update ToDos

readme.md

@@ -634,7 +634,10 @@ a means of exploring some very beautiful math.
 
 ## TODO
 
-* Update CLI option text in this readme to include Zipf distribution options
+* Consider whether a graph database might be a better option than keeping things in memory.
+* Look at fastUtil for more performant maps and arrays. Note that there is an optional jGraphT library to work with fastUtil (see FastutilMapIntVertexGraph, for example).
+* Consider implementing an option to use the jGrapht sparse graph representation for a lower memory cost with very large graphs (tens or hundreds of thousands of distinct sequences).
+* ~~Update CLI option text in this readme to include Zipf distribution options~~
 * ~~Try invoking GC at end of workloads to reduce paging to disk~~ DONE
 * ~~Hold graph data in memory until another graph is read-in? ABANDONED UNABANDONED~~ DONE
     * ~~*No, this won't work, because BiGpairSEQ simulations alter the underlying graph based on filtering constraints. Changes would cascade with multiple experiments.*~~

src/main/java/Simulator.java

@@ -1,3 +1,4 @@
+import org.jgrapht.Graphs;
 import org.jgrapht.alg.interfaces.MatchingAlgorithm;
 import org.jgrapht.alg.matching.MaximumWeightBipartiteMatching;
 import org.jgrapht.graph.DefaultWeightedEdge;
@@ -147,39 +148,25 @@ public class Simulator implements GraphModificationFunctions {
             vertexLabelValue++;
         }
         betaVertices.forEach(graph::addVertex);
-        //add edges
+        //add edges (best so far)
+        int edgesAddedCount = 0;
         for(Vertex a: alphaVertices) {
+            Set<Integer> a_wells = a.getRecord().getWells();
             for(Vertex b: betaVertices) {
-                double weight = 0.0;
+                Set<Integer> sharedWells = new HashSet<>(a_wells);
-                for (Integer well: a.getRecord().getWells()) {
+                sharedWells.retainAll(b.getRecord().getWells());
-                    if (b.getRecord().isInWell(well)) {
+                if (!sharedWells.isEmpty()) {
-                        weight += 1.0;
+                    Graphs.addEdge(graph, a, b, (double) sharedWells.size());
                 }
-                }
+                edgesAddedCount++;
-                if (weight != 0.0) {
+                if (edgesAddedCount % 10000000 == 0) { //collect garbage every 10,000,000 edges
-                    DefaultWeightedEdge edge = graph.addEdge(a, b);
+                    System.out.println(edgesAddedCount + " edges added");
-                    graph.setEdgeWeight(edge, weight);
+                    //request garbage collection
+                    System.gc();
+                    System.out.println("Garbage collection requested");
                 }
             }
         }
-//        //add edges
-//        for(Vertex a: alphaVertices) {
-//            for(Integer well: a.getRecord().getWells()) {
-//                for (Vertex b: betaVertices) {
-//                    if (b.getRecord().isInWell(well)) {
-//                        DefaultWeightedEdge edge = graph.getEdge(a, b);
-//                        if (edge == null) {
-//                            edge = graph.addEdge(a, b);
-//                            graph.setEdgeWeight(edge, 1.0);
-//                        }
-//                        else {
-//                            double weight = graph.getEdgeWeight(edge);
-//                            graph.setEdgeWeight(edge, weight + 1.0);
-//                        }
-//                    }
-//                }
-//            }
-//        }
         if(verbose){System.out.println("Graph created");}
         //stop timing
         Instant stop = Instant.now();