- Update about page

- Update sign in page
- Update meta
- Add json state server to FW links
- up # instances to 8
- experimental test

scripts/deploy.sh

@@ -24,7 +24,7 @@ gcloud run deploy --source . \
         --set-env-vars "BUILD_TIMESTAMP=${TIMESTAMP}" \
         --region us-east1 \
         --allow-unauthenticated \
-        --min-instances 6 \
+        --min-instances 8 \
         --cpu 2 \
         --memory 8G \
         codex && \

src/data/downloadable_files_metadata.json

@@ -7,7 +7,7 @@
       "morphology_clusters.csv.gz": 736744,
       "connectivity_clusters.csv.gz": 522912,
       "connections.csv.gz": 26124953,
-      "labels.csv.gz": 3164654,
+      "labels.csv.gz": 3248647,
       "coordinates.csv.gz": 4584155,
       "neuropil_synapse_table.csv.gz": 4490002
     },
@@ -82,17 +82,17 @@
         "col 5 - nt_type": "6 unique values"
       },
       "labels.csv.gz": {
-        "# rows": "112,928 (+ header)",
+        "# rows": "115,756 (+ header)",
         "# columns": "9",
-        "col 1 - root_id": "80,328 unique values",
-        "col 2 - label": "6,468 unique values",
+        "col 1 - root_id": "82,946 unique values",
+        "col 2 - label": "6,479 unique values",
         "col 3 - user_id": "132 unique values",
-        "col 4 - position": "91,917 unique values",
-        "col 5 - supervoxel_id": "91,197 unique values",
+        "col 4 - position": "94,743 unique values",
+        "col 5 - supervoxel_id": "94,019 unique values",
         "col 6 - label_id": "all rows contain unique values",
-        "col 7 - date_created": "34,481 unique values",
-        "col 8 - user_name": "129 unique values in 112,278 rows, empty in 650 rows",
-        "col 9 - user_affiliation": "52 unique values in 112,215 rows, empty in 713 rows"
+        "col 7 - date_created": "34,870 unique values",
+        "col 8 - user_name": "129 unique values in 115,106 rows, empty in 650 rows",
+        "col 9 - user_affiliation": "52 unique values in 115,043 rows, empty in 713 rows"
       },
       "coordinates.csv.gz": {
         "# rows": "207,700 (+ header)",

src/templates/about_codex.html

@@ -17,9 +17,9 @@ <h6>Status</h6>
         </ul>
         <h6>Credits, Terms and Privacy notice</h6>
         <ul>
-            <li>Codex & FlyWire <a href="https://flywire.ai/tos">Terms of Service and Privacy</a> notice</li>
+            <li>Codex serves data from the <a href="https://flywire.ai">FlyWire</a> connectome - see <a href="{{url_for('base.about_flywire')}}">data credits page</a> for details</li>
             <li>Supported by BRAIN Initiative grant MH117815, MH129268 and U24 NS126935 to Murthy and Seung</li>
-            <li>Codex serves data from the FlyWire connectome - see <a href="{{url_for('base.about_flywire')}}">data credits page</a> for details</li>
+            <li>Codex & FlyWire <a href="https://flywire.ai/tos">Terms of Service and Privacy</a> notice</li>
         </ul>
         <h6>Creators @ Princeton Neuroscience Institute</h6>
         <ul>
@@ -37,8 +37,8 @@ <h6>Creators @ Princeton Neuroscience Institute</h6>
         </ul>
         <h6>Links and Resources</h6>
         <ul>
-            <li>Codex <a href="https://github.com/murthylab/codex" target="_blank">source code</a> (type in your GitHub
-                handle below to request access)
+            <li>Codex <a href="https://github.com/murthylab/codex" target="_blank">source code</a> (open to public soon, type in your GitHub
+                handle below to request early access)
             </li>
             <li>Codex <a href="{{url_for('base.demo_clip')}}" target="_blank">demos <i class="fa-brands fa-youtube"></i></a> and <a href="{{url_for('base.todo_list')}}" target="_blank">TODO List</a> open for comments</li>
         </ul>

src/templates/auth.html

@@ -22,7 +22,9 @@ <h1 class="display-4">Codex</h1>
             </div>
             <div class="g_id_signin" data-type="standard"></div>
             <br>
-            <em style="color:grey;">Need help? {{support_email}}</em>
+            <em style="color:grey;">No G account / need help getting access? {{support_email}}</em>
+            <br>
+            <small style="color:grey;">Additional sign-in methods coming soon. For general info visit the <a href="https://flywire.ai" target="_blank">FlyWire homepage</a>.</small>
         </div>
         <hr class="my-4">
     </div>

src/templates/base.html

@@ -25,18 +25,19 @@ <h6 style="background: orange; color: white; text-align: center; font-size: 14px
 
     <div class="collapse navbar-collapse" id="navbarSupportedContent">
         <ul class="navbar-nav mr-auto">
-          {% set tabs = [['explore', 'Explore'],
-                        ['search', 'Search'],
-                        ['stats', 'Stats'],
-                        ['cell_details', 'Cell Info'],
-                        ['connectivity', 'Network'],
-                        ['path_length', 'Pathways'],
-                        ['neuropils', 'Neuropils'],
-                        ['heatmaps', 'Heatmaps'],
+          {% set tabs = [['explore', 'Explore', 'app'],
+                        ['search', 'Search', 'app'],
+                        ['stats', 'Stats', 'app'],
+                        ['cell_details', 'Cell Info', 'app'],
+                        ['connectivity', 'Network', 'app'],
+                        ['path_length', 'Pathways', 'app'],
+                        ['neuropils', 'Neuropils', 'app'],
+                        ['heatmaps', 'Heatmaps', 'app'],
+                        ['about_codex', 'About', 'base'],
                         ]%}
         {% for page in tabs %}
           <li class="nav-item text-nowrap">
-            <a onclick="loading(event);" class="nav-link" id="nav_{{ page[0] }}" href="{{url_for('app.' + page[0])}}">{{ page[1] }}</a>
+            <a onclick="loading(event);" class="nav-link" id="nav_{{ page[0] }}" href="{{url_for(page[2] + '.' + page[0])}}">{{ page[1] }}</a>
           </li>
         {% endfor %}
 
@@ -45,7 +46,6 @@ <h6 style="background: orange; color: white; text-align: center; font-size: 14px
                   More
                 </a>
                 <div class="dropdown-menu" aria-labelledby="nav_more_active">
-                  <a class="dropdown-item" id="nav_about_codex" href="{{url_for('base.about_codex')}}">About Codex</a>
                   <a class="dropdown-item" id="nav_about_flywire" href="{{url_for('base.about_flywire')}}">FlyWire Info & Credits</a>
                   <a class="dropdown-item" id="nav_faq" href="{{url_for('base.faq')}}">FAQ</a>
                   <a class="dropdown-item" id="nav_optic_lobe_tagging" href="{{url_for('app.optic_lobe_tagging')}}">Optic Lobe Tagging</a>

src/utils/graph_algos.py

@@ -16,11 +16,13 @@ def neighbors(sources, neighbor_sets):
 
 
 # given set of sources, calculates the distance to all other reachable nodes into a dict (rid -> distance)
-def reachable_nodes(sources, neighbor_sets, stop_target=None):
+def reachable_nodes(sources, neighbor_sets, stop_target=None, max_depth=None):
     depth = 0
     reached = {s: 0 for s in sources}
     frontier = set(sources)
     while frontier:
+        if max_depth is not None and depth == max_depth:
+            break
         if stop_target is not None and stop_target in frontier:
             break
         depth += 1

src/utils/nglui.py

@@ -64,6 +64,7 @@ def url_for_root_ids(root_ids, version, point_to="ngl", position=None):
             "layer": seg_layer_name,
             "visible": True,
         }
+        config["jsonStateServer"] = "https://globalv1.flywire-daf.com/nglstate/post"
 
         return f"https://ngl.flywire.ai/#!{urllib.parse.quote(json.dumps(config))}"
     else:

tests/experimental/test_ol_tagging.py

@@ -4,6 +4,8 @@
 from unittest import TestCase
 
 from src.data.brain_regions import REGIONS
+from src.data.local_data_loader import read_csv, write_csv
+from src.utils.graph_algos import reachable_nodes
 from src.utils.stats import jaccard_binary
 
 from tests import get_testing_neuron_db
@@ -1859,3 +1861,47 @@ def test_jaccard_in_class_vs_out_of_class(self):
         print(
             f"Diff type avg Emb score: {sum(diff_type_sscores) / len(diff_type_sscores)}"
         )
+
+    def test_dsx_fru_tables(self):
+        dsxfru_table = read_csv(
+            "../../static/experimental_data/dsxfru/Dsxfru_630_V6.csv"
+        )
+        print(dsxfru_table[0])
+        rid_col = dsxfru_table[0].index("RootID_630")
+        dsxfru_rids = [int(r[rid_col]) for r in dsxfru_table[1:]]
+        self.assertEqual(len(dsxfru_rids), len(set(dsxfru_rids)))
+        not_in_ds = []
+        for rid in dsxfru_rids:
+            if not self.neuron_db.is_in_dataset(rid):
+                not_in_ds.append(rid)
+        print(len(not_in_ds))
+        dsxfru_rids = set(dsxfru_rids) - set(not_in_ds)
+        print(len(dsxfru_rids))
+
+        con_rows = self.neuron_db.connections_.rows_between_sets(
+            dsxfru_rids, dsxfru_rids
+        )
+        syn_counts = defaultdict(int)
+        for r in con_rows:
+            syn_counts[(r[0], r[1])] += r[3]
+        syn_counts_table = [["from", "to", "syn_count"]] + [
+            [k[0], k[1], v] for k, v in syn_counts.items()
+        ]
+        write_csv(
+            "../../static/experimental_data/dsxfru/dsxfru_syn_table_v6.csv",
+            rows=syn_counts_table,
+        )
+
+        ins, outs = self.neuron_db.input_output_partner_sets()
+        path_lengths = []
+        for r in dsxfru_rids:
+            reachable = reachable_nodes([r], neighbor_sets=outs, max_depth=3)
+            for k, v in reachable.items():
+                if v > 0:
+                    path_lengths.append([r, k, v])
+
+        path_table = [["from", "to", "shortest_path_length"]] + path_lengths
+        write_csv(
+            "../../static/experimental_data/dsxfru/dsxfru_path_length_table_v6.csv",
+            rows=path_table,
+        )