refactor to construct the bipartite graph directly, rather than by using an adjacency matrix and a graph generator.

.idea/artifacts/BiGpairSEQ_Sim_jar.xml

@@ -1,16 +1,27 @@
 <component name="ArtifactManager">
-  <artifact type="jar" build-on-make="true" name="BiGpairSEQ_Sim:jar">
+  <artifact type="jar" name="BiGpairSEQ_Sim:jar">
     <output-path>$PROJECT_DIR$/out/artifacts/BiGpairSEQ_Sim_jar</output-path>
     <root id="archive" name="BiGpairSEQ_Sim.jar">
       <element id="directory" name="META-INF">
-        <element id="file-copy" path="$PROJECT_DIR$/src/main/java/META-INF/MANIFEST.MF" />
+        <element id="file-copy" path="$PROJECT_DIR$/META-INF/MANIFEST.MF" />
       </element>
-      <element id="module-output" name="BigPairSEQ" />
+      <element id="module-output" name="BiGpairSEQ_Sim" />
-      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/jgrapht/jgrapht-core/1.5.1/jgrapht-core-1.5.1.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/jgrapht/jgrapht-core/1.5.2/jgrapht-core-1.5.2.jar" path-in-jar="/" />
-      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/jheaps/jheaps/0.13/jheaps-0.13.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-rng-sampling/1.6/commons-rng-sampling-1.6.jar" path-in-jar="/" />
-      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/commons-cli/commons-cli/1.5.0/commons-cli-1.5.0.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-csv/1.14.0/commons-csv-1.14.0.jar" path-in-jar="/" />
-      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-csv/1.9.0/commons-csv-1.9.0.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/jetbrains/annotations/26.0.2/annotations-26.0.2.jar" path-in-jar="/" />
-      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/jetbrains/annotations/23.0.0/annotations-23.0.0.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/jgrapht/jgrapht-io/1.5.2/jgrapht-io-1.5.2.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-rng-simple/1.6/commons-rng-simple-1.6.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/commons-io/commons-io/2.18.0/commons-io-2.18.0.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-rng-core/1.6/commons-rng-core-1.6.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/commons-codec/commons-codec/1.18.0/commons-codec-1.18.0.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-rng-client-api/1.6/commons-rng-client-api-1.6.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/commons-cli/commons-cli/1.9.0/commons-cli-1.9.0.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/antlr/antlr4-runtime/4.12.0/antlr4-runtime-4.12.0.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apfloat/apfloat/1.10.1/apfloat-1.10.1.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/apache/commons/commons-text/1.10.0/commons-text-1.10.0.jar" path-in-jar="/" />
+      <element id="extracted-dir" path="$MAVEN_REPOSITORY$/org/jheaps/jheaps/0.14/jheaps-0.14.jar" path-in-jar="/" />
     </root>
   </artifact>
 </component>

.idea/compiler.xml

@@ -7,6 +7,7 @@
         <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
         <outputRelativeToContentRoot value="true" />
         <module name="BigPairSEQ" />
+        <module name="BiGpairSEQ_Sim" />
       </profile>
     </annotationProcessing>
   </component>

.idea/jarRepositories.xml

@@ -1,20 +1,35 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="RemoteRepositoriesConfiguration">
+    <remote-repository>
+      <option name="id" value="my-internal-site" />
+      <option name="name" value="my-internal-site" />
+      <option name="url" value="https://myserver/repo" />
+    </remote-repository>
+    <remote-repository>
+      <option name="id" value="central" />
+      <option name="name" value="Central Repository" />
+      <option name="url" value="https://repo1.maven.org/maven2" />
+    </remote-repository>
+    <remote-repository>
+      <option name="id" value="central repo" />
+      <option name="name" value="central repo" />
+      <option name="url" value="https://repo1.maven.org/maven2/" />
+    </remote-repository>
     <remote-repository>
       <option name="id" value="central" />
       <option name="name" value="Central Repository" />
       <option name="url" value="https://repo.maven.apache.org/maven2" />
     </remote-repository>
-    <remote-repository>
-      <option name="id" value="central" />
-      <option name="name" value="Maven Central repository" />
-      <option name="url" value="https://repo1.maven.org/maven2" />
-    </remote-repository>
     <remote-repository>
       <option name="id" value="jboss.community" />
       <option name="name" value="JBoss Community repository" />
       <option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
     </remote-repository>
+    <remote-repository>
+      <option name="id" value="34d16bdc-85f0-48ee-8e8b-144091765be1" />
+      <option name="name" value="34d16bdc-85f0-48ee-8e8b-144091765be1" />
+      <option name="url" value="https://repository.mulesoft.org/nexus/content/repositories/public/" />
+    </remote-repository>
   </component>
 </project>

.idea/libraries/apache_commons_csv.xml

@@ -1,8 +1,10 @@
 <component name="libraryTable">
   <library name="apache.commons.csv" type="repository">
-    <properties maven-id="org.apache.commons:commons-csv:1.9.0" />
+    <properties maven-id="org.apache.commons:commons-csv:1.14.0" />
     <CLASSES>
-      <root url="jar://$MAVEN_REPOSITORY$/org/apache/commons/commons-csv/1.9.0/commons-csv-1.9.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/apache/commons/commons-csv/1.14.0/commons-csv-1.14.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/commons-io/commons-io/2.18.0/commons-io-2.18.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/commons-codec/commons-codec/1.18.0/commons-codec-1.18.0.jar!/" />
     </CLASSES>
     <JAVADOC />
     <SOURCES />

.idea/libraries/commons_cli.xml

@@ -1,8 +1,8 @@
 <component name="libraryTable">
   <library name="commons.cli" type="repository">
-    <properties maven-id="commons-cli:commons-cli:1.5.0" />
+    <properties maven-id="commons-cli:commons-cli:1.9.0" />
     <CLASSES>
-      <root url="jar://$MAVEN_REPOSITORY$/commons-cli/commons-cli/1.5.0/commons-cli-1.5.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/commons-cli/commons-cli/1.9.0/commons-cli-1.9.0.jar!/" />
     </CLASSES>
     <JAVADOC />
     <SOURCES />

.idea/libraries/commons_rng_1.xml

@@ -1,13 +0,0 @@
-<component name="libraryTable">
-  <library name="commons-rng-1">
-    <CLASSES>
-      <root url="file://$USER_HOME$/Downloads/commons-rng-1.6" />
-    </CLASSES>
-    <JAVADOC />
-    <SOURCES>
-      <root url="file://$USER_HOME$/Downloads/commons-rng-1.6" />
-    </SOURCES>
-    <jarDirectory url="file://$USER_HOME$/Downloads/commons-rng-1.6" recursive="false" />
-    <jarDirectory url="file://$USER_HOME$/Downloads/commons-rng-1.6" recursive="false" type="SOURCES" />
-  </library>
-</component>

.idea/libraries/jgrapht_core.xml

@@ -1,9 +1,10 @@
 <component name="libraryTable">
   <library name="jgrapht.core" type="repository">
-    <properties maven-id="org.jgrapht:jgrapht-core:1.5.1" />
+    <properties maven-id="org.jgrapht:jgrapht-core:1.5.2" />
     <CLASSES>
-      <root url="jar://$MAVEN_REPOSITORY$/org/jgrapht/jgrapht-core/1.5.1/jgrapht-core-1.5.1.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jgrapht/jgrapht-core/1.5.2/jgrapht-core-1.5.2.jar!/" />
-      <root url="jar://$MAVEN_REPOSITORY$/org/jheaps/jheaps/0.13/jheaps-0.13.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jheaps/jheaps/0.14/jheaps-0.14.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/apfloat/apfloat/1.10.1/apfloat-1.10.1.jar!/" />
     </CLASSES>
     <JAVADOC />
     <SOURCES />

.idea/libraries/jgrapht_io.xml

@@ -1,13 +1,14 @@
 <component name="libraryTable">
   <library name="jgrapht.io" type="repository">
-    <properties maven-id="org.jgrapht:jgrapht-io:1.5.1" />
+    <properties maven-id="org.jgrapht:jgrapht-io:1.5.2" />
     <CLASSES>
-      <root url="jar://$MAVEN_REPOSITORY$/org/jgrapht/jgrapht-io/1.5.1/jgrapht-io-1.5.1.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jgrapht/jgrapht-io/1.5.2/jgrapht-io-1.5.2.jar!/" />
-      <root url="jar://$MAVEN_REPOSITORY$/org/jgrapht/jgrapht-core/1.5.1/jgrapht-core-1.5.1.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jgrapht/jgrapht-core/1.5.2/jgrapht-core-1.5.2.jar!/" />
-      <root url="jar://$MAVEN_REPOSITORY$/org/jheaps/jheaps/0.13/jheaps-0.13.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jheaps/jheaps/0.14/jheaps-0.14.jar!/" />
-      <root url="jar://$MAVEN_REPOSITORY$/org/antlr/antlr4-runtime/4.8-1/antlr4-runtime-4.8-1.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/apfloat/apfloat/1.10.1/apfloat-1.10.1.jar!/" />
-      <root url="jar://$MAVEN_REPOSITORY$/org/apache/commons/commons-text/1.8/commons-text-1.8.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/antlr/antlr4-runtime/4.12.0/antlr4-runtime-4.12.0.jar!/" />
-      <root url="jar://$MAVEN_REPOSITORY$/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/apache/commons/commons-text/1.10.0/commons-text-1.10.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/apache/commons/commons-lang3/3.12.0/commons-lang3-3.12.0.jar!/" />
     </CLASSES>
     <JAVADOC />
     <SOURCES />

pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>org.example</groupId>
-    <artifactId>TCellSim</artifactId>
+    <artifactId>BiGpairSEQ_Sim</artifactId>
     <version>1.0-SNAPSHOT</version>
     <build>
         <plugins>
@@ -26,8 +26,48 @@
             <version>RELEASE</version>
             <scope>compile</scope>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-rng-simple -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-rng-simple</artifactId>
+            <version>1.6</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-rng-sampling</artifactId>
+            <version>1.6</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-csv -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+            <version>1.14.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.jgrapht/jgrapht-core -->
+        <dependency>
+            <groupId>org.jgrapht</groupId>
+            <artifactId>jgrapht-core</artifactId>
+            <version>1.5.2</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.jgrapht/jgrapht-io -->
+        <dependency>
+            <groupId>org.jgrapht</groupId>
+            <artifactId>jgrapht-io</artifactId>
+            <version>1.5.2</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.jheaps/jheaps -->
+        <dependency>
+            <groupId>org.jheaps</groupId>
+            <artifactId>jheaps</artifactId>
+            <version>0.14</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/commons-cli/commons-cli -->
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <version>1.9.0</version>
+        </dependency>
     </dependencies>
-
     <properties>
         <maven.compiler.source>11</maven.compiler.source>
         <maven.compiler.target>11</maven.compiler.target>

src/main/java/Plate.java

@@ -13,8 +13,7 @@ TODO: Implement discrete frequency distributions using Vose's Alias Method
  */
 
 
-import org.apache.commons.rng.UniformRandomProvider;
+
-import org.apache.commons.rng.core.BaseProvider;
 import org.apache.commons.rng.sampling.distribution.RejectionInversionZipfSampler;
 import org.apache.commons.rng.simple.JDKRandomWrapper;
 
@@ -242,7 +241,7 @@ public class Plate {
                                     sequencesAndMisreads.put(currentSequence, new ArrayList<>());
                                 }
                                 //The specific misread hasn't happened before
-                                if (rand.nextDouble() >= errorCollisionRate || sequencesAndMisreads.get(currentSequence).size() == 0) {
+                                if (rand.nextDouble() >= errorCollisionRate || sequencesAndMisreads.get(currentSequence).isEmpty()) {
                                     //The misread doesn't collide with a real sequence already on the plate and some sequences have already been read
                                     if(rand.nextDouble() >= realSequenceCollisionRate || !sequenceMap.isEmpty()){
                                         StringBuilder spurious = new StringBuilder(currentSequence);

src/main/java/Simulator.java

@@ -70,58 +70,97 @@ public class Simulator implements GraphModificationFunctions {
             if(verbose){System.out.println("Total beta sequence wells removed: " + betaWellsRemoved);}
         }
 
-        //construct the graph. For simplicity, going to make
+        /*
-        if(verbose){System.out.println("Making vertex maps");}
+         * The commented out code below works beautifully for small enough graphs. However, after implementing a
-        //For the SimpleWeightedBipartiteGraphMatrixGenerator, all vertices must have
+         * Zipf distribution and attempting to simulate Experiment 3 from the paper again, I discovered that
-        //distinct numbers associated with them. Since I'm using a 2D array, that means
+         * this method uses too much memory. Even a 120GB heap is not enough to build this adjacency matrix.
-        //distinct indices between the rows and columns. vertexStartValue lets me track where I switch
+         * So I'm going to attempt to build this graph directly and see if that is less memory intensive
-        //from numbering rows to columns, so I can assign unique numbers to every vertex, and then
+         */
-        //subtract the vertexStartValue from betas to use their vertex labels as array indices
+//        //construct the graph. For simplicity, going to make
-        int vertexStartValue = 0;
+//        if(verbose){System.out.println("Making vertex maps");}
-        //keys are sequential integer vertices, values are alphas
+//        //For the SimpleWeightedBipartiteGraphMatrixGenerator, all vertices must have
-        Map<String, Integer> plateAtoVMap = makeSequenceToVertexMap(alphaSequences, vertexStartValue);
+//        //distinct numbers associated with them. Since I'm using a 2D array, that means
-        //new start value for vertex to beta map should be one more than final vertex value in alpha map
+//        //distinct indices between the rows and columns. vertexStartValue lets me track where I switch
-        vertexStartValue += plateAtoVMap.size();
+//        //from numbering rows to columns, so I can assign unique numbers to every vertex, and then
-        //keys are betas, values are sequential integers
+//        //subtract the vertexStartValue from betas to use their vertex labels as array indices
-        Map<String, Integer> plateBtoVMap = makeSequenceToVertexMap(betaSequences, vertexStartValue);
+//        int vertexStartValue = 0;
-        if(verbose){System.out.println("Vertex maps made");}
+//        //keys are sequential integer vertices, values are alphas
-        //make adjacency matrix for bipartite graph generator
+//        Map<String, Integer> plateAtoVMap = makeSequenceToVertexMap(alphaSequences, vertexStartValue);
-        //(technically this is only 1/4 of an adjacency matrix, but that's all you need
+//        //new start value for vertex to beta map should be one more than final vertex value in alpha map
-        //for a bipartite graph, and all the SimpleWeightedBipartiteGraphMatrixGenerator class expects.)
+//        vertexStartValue += plateAtoVMap.size();
-        if(verbose){System.out.println("Making adjacency matrix");}
+//        //keys are betas, values are sequential integers
-        double[][] weights = new double[plateAtoVMap.size()][plateBtoVMap.size()];
+//        Map<String, Integer> plateBtoVMap = makeSequenceToVertexMap(betaSequences, vertexStartValue);
-        fillAdjacencyMatrix(weights, vertexStartValue, alphaSequences, betaSequences, plateAtoVMap, plateBtoVMap);
+//        if(verbose){System.out.println("Vertex maps made");}
-        if(verbose){System.out.println("Adjacency matrix made");}
+//        //make adjacency matrix for bipartite graph generator
+//        //(technically this is only 1/4 of an adjacency matrix, but that's all you need
+//        //for a bipartite graph, and all the SimpleWeightedBipartiteGraphMatrixGenerator class expects.)
+//        if(verbose){System.out.println("Making adjacency matrix");}
+//        double[][] weights = new double[plateAtoVMap.size()][plateBtoVMap.size()];
+//        fillAdjacencyMatrix(weights, vertexStartValue, alphaSequences, betaSequences, plateAtoVMap, plateBtoVMap);
+//        if(verbose){System.out.println("Adjacency matrix made");}
+//        //make bipartite graph
+//        if(verbose){System.out.println("Making bipartite weighted graph");}
+//        //the graph object
+//        SimpleWeightedGraph<Vertex, DefaultWeightedEdge> graph =
+//                new SimpleWeightedGraph<>(DefaultWeightedEdge.class);
+//        //the graph generator
+//        SimpleWeightedBipartiteGraphMatrixGenerator graphGenerator = new SimpleWeightedBipartiteGraphMatrixGenerator();
+//        //the list of alpha vertices
+//        List<Vertex> alphaVertices = new ArrayList<>();
+//        for (String seq : plateAtoVMap.keySet()) {
+//            Vertex alphaVertex = new Vertex(alphaSequences.get(seq), plateAtoVMap.get(seq));
+//            alphaVertices.add(alphaVertex);
+//        }
+//        //Sort to make sure the order of vertices in list matches the order of the adjacency matrix
+//        Collections.sort(alphaVertices);
+//        //Add ordered list of vertices to the graph
+//        graphGenerator.first(alphaVertices);
+//        //the list of beta vertices
+//        List<Vertex> betaVertices = new ArrayList<>();
+//        for (String seq : plateBtoVMap.keySet()) {
+//            Vertex betaVertex = new Vertex(betaSequences.get(seq), plateBtoVMap.get(seq));
+//            betaVertices.add(betaVertex);
+//        }
+//        //Sort to make sure the order of vertices in list matches the order of the adjacency matrix
+//        Collections.sort(betaVertices);
+//        //Add ordered list of vertices to the graph
+//        graphGenerator.second(betaVertices);
+//        //use adjacency matrix of weight created previously
+//        graphGenerator.weights(weights);
+//        graphGenerator.generateGraph(graph);
+
         //make bipartite graph
         if(verbose){System.out.println("Making bipartite weighted graph");}
         //the graph object
         SimpleWeightedGraph<Vertex, DefaultWeightedEdge> graph =
                 new SimpleWeightedGraph<>(DefaultWeightedEdge.class);
-        //the graph generator
+        int vertexLabelValue = 0;
-        SimpleWeightedBipartiteGraphMatrixGenerator graphGenerator = new SimpleWeightedBipartiteGraphMatrixGenerator();
+        //create and add alpha sequence vertices
-        //the list of alpha vertices
         List<Vertex> alphaVertices = new ArrayList<>();
-        for (String seq : plateAtoVMap.keySet()) {
+        for (Map.Entry<String, SequenceRecord> entry: alphaSequences.entrySet()) {
-            Vertex alphaVertex = new Vertex(alphaSequences.get(seq), plateAtoVMap.get(seq));
+            alphaVertices.add(new Vertex(entry.getValue(), vertexLabelValue));
-            alphaVertices.add(alphaVertex);
+            vertexLabelValue++;
         }
-        //Sort to make sure the order of vertices in list matches the order of the adjacency matrix
+        alphaVertices.forEach(graph::addVertex);
-        Collections.sort(alphaVertices);
+        //add beta sequence vertices
-        //Add ordered list of vertices to the graph
-        graphGenerator.first(alphaVertices);
-        //the list of beta vertices
         List<Vertex> betaVertices = new ArrayList<>();
-        for (String seq : plateBtoVMap.keySet()) {
+        for (Map.Entry<String, SequenceRecord> entry: betaSequences.entrySet()) {
-            Vertex betaVertex = new Vertex(betaSequences.get(seq), plateBtoVMap.get(seq));
+            betaVertices.add(new Vertex(entry.getValue(), vertexLabelValue));
-            betaVertices.add(betaVertex);
+            vertexLabelValue++;
         }
-        //Sort to make sure the order of vertices in list matches the order of the adjacency matrix
+        betaVertices.forEach(graph::addVertex);
-        Collections.sort(betaVertices);
+        //add edges
-        //Add ordered list of vertices to the graph
+        for(Vertex alpha: alphaVertices){
-        graphGenerator.second(betaVertices);
+            for(Vertex beta: betaVertices) {
-        //use adjacency matrix of weight created previously
+                Set<Integer> sharedWells = alpha.getRecord().getWells();
-        graphGenerator.weights(weights);
+                sharedWells.retainAll(beta.getRecord().getWells());
-        graphGenerator.generateGraph(graph);
+                if(!sharedWells.isEmpty()) {
+                    DefaultWeightedEdge edge = graph.addEdge(alpha, beta);
+                    graph.setEdgeWeight(edge, sharedWells.size());
+                }
+            }
+        }
+
         if(verbose){System.out.println("Graph created");}
         //stop timing
         Instant stop = Instant.now();