update

tests/experimental/test_catalog_assets.py

@@ -287,5 +287,5 @@ def test_make_catalog_figures(self):
                 colorizer=type_color,
             )
 
-            plt.savefig(f"../../static/experimental_data/catalog_assets/fig_{k}.png")
+            plt.savefig(f"../../static/experimental_data/ol_catalog_assets/fig_{k}.png")
             plt.close(fig)

tests/experimental/test_dsx_fru.py

@@ -1,15 +1,22 @@
+import json
 import math
 import random
 from collections import defaultdict
 from unittest import TestCase
+import matplotlib.pyplot as plt
+import networkx as nx
 
 from src.data.local_data_loader import read_csv, write_csv
-from src.utils.formatting import format_dict_by_largest_value
+from src.utils.formatting import format_dict_by_largest_value, display
 from tests import get_testing_neuron_db
 
 DSX_FRU_TABLE_FNAME = "../../src/data/dsxfru/Dsxfru_630_V6.csv"
 
 
+def to_percent(frac):
+    return f"{round(frac * 100)}%"
+
+
 class DsxFruTest(TestCase):
     @classmethod
     def setUpClass(cls):
@@ -21,6 +28,11 @@ def setUpClass(cls):
         cls.dsx_fru_root_id_to_type = {
             int(r[rid_idx]): r[type_idx] for r in dsx_fru_table[1:]
         }
+        cls.dsxfru_type_to_list = defaultdict(list)
+        for rid, tp in cls.dsx_fru_root_id_to_type.items():
+            cls.dsxfru_type_to_list[tp].append(rid)
+
+        cls.dsx_fry_types = sorted(cls.dsxfru_type_to_list.keys())
 
         def extract_pos_xyz(nd):
             pos = nd["position"][0].replace("[", "").replace("]", "").split(" ")
@@ -40,6 +52,15 @@ def allowed(nd):
             if allowed(nd):
                 cls.super_class_to_rids_without_kc[nd["super_class"]].append(rid)
 
+    def dftclass(self, root_id):
+        if root_id in self.dsx_fru_root_id_to_type:
+            return self.dsx_fru_root_id_to_type[root_id]
+        nd = self.neuron_db.neuron_data[root_id]
+        return nd["class"] or "other"
+
+    def type_color(self, tp):
+        return f"C{self.dsx_fry_types.index(tp)}"
+
     def test_summarize_data(self):
         print(
             f"Cells: {len(self.dsx_fru_root_id_to_type)}, "
@@ -114,88 +135,206 @@ def matching_set(rid_list):
             table.append([f"matched network #{i}", dist] + list(mrids))
             write_csv(filename="matched_networks_dsxfru_v6.csv", rows=table)
 
-    def test_similar_and_mirror_cells(self):
-        # data_rows = read_csv("../../static/experimental_data/updated_dsxfru_with_status_filtered_571_fixed.csv")
-        data_rows = read_csv("../../static/experimental_data/Dsxfru_630_V1.csv")
-        rid_col_idx = data_rows[0].index("RootID_630")
-        self.assertGreater(rid_col_idx, 0)
-        type_col_idx = data_rows[0].index("SubTypeName")
-        self.assertGreater(type_col_idx, 0)
-        data_rid_to_type = {
-            int(r[rid_col_idx]): r[type_col_idx] for r in data_rows[1:] if r
-        }
-        data_rids_filtered = [
-            r for r in data_rid_to_type.keys() if r in self.neuron_db.neuron_data
-        ]
-        print(
-            f"dataset root ids: {len(data_rid_to_type)}, exist in 630: {len(data_rids_filtered)}"
+    def collect_data(self):
+
+        type_to_dftorclass_inputs = defaultdict(lambda: defaultdict(int))
+        type_to_dftorclass_outputs = defaultdict(lambda: defaultdict(int))
+
+        # todo
+        type_to_sensory_ranks = {tp: {"olfactory": 5, "visual": 14, "mechano": 8} for tp in self.dsx_fry_types}
+
+        type_to_dsxfru_input_synapses = defaultdict(int)
+        type_to_dsxfru_output_synapses = defaultdict(int)
+        type_to_non_dsxfru_input_synapses = defaultdict(int)
+        type_to_non_dsxfru_output_synapses = defaultdict(int)
+
+        result = {}
+
+        ins, outs = self.neuron_db.input_output_partners_with_synapse_counts()
+        for tp, rids in self.dsxfru_type_to_list.items():
+            avg_in_degree = round(sum([len(ins[rid]) for rid in rids]) / len(rids))
+            avg_out_degree = round(sum([len(outs[rid]) for rid in rids]) / len(rids))
+            avg_in_synapses = round(
+                sum([sum(ins[rid].values()) for rid in rids]) / len(rids)
+            )
+            avg_out_synapses = round(
+                sum([sum(outs[rid].values()) for rid in rids]) / len(rids)
+            )
+
+            result[tp] = {
+                "num_cells": len(rids),
+                "avg_length_nm": round(
+                    sum([self.neuron_db.neuron_data[rid]["length_nm"] for rid in rids])
+                    / len(rids)
+                ),
+                "avg_area_nm": round(
+                    sum([self.neuron_db.neuron_data[rid]["area_nm"] for rid in rids])
+                    / len(rids)
+                ),
+                "avg_volume_nm": round(
+                    sum([self.neuron_db.neuron_data[rid]["size_nm"] for rid in rids])
+                    / len(rids)
+                ),
+                "avg_in_degree": avg_in_degree,
+                "avg_out_degree": avg_out_degree,
+                "avg_in_synapses": avg_in_synapses,
+                "avg_out_synapses": avg_out_synapses,
+                "avg_in_synapses_from_dsx_fru": round(
+                    type_to_dsxfru_input_synapses[tp] / len(rids)
+                ),
+                "avg_in_synapses_from_non_dsx_fru": round(
+                    type_to_non_dsxfru_input_synapses[tp] / len(rids)
+                ),
+                "avg_out_synapses_to_dsx_fru": round(
+                    type_to_dsxfru_output_synapses[tp] / len(rids)
+                ),
+                "avg_out_synapses_to_non_dsx_fru": round(
+                    type_to_non_dsxfru_output_synapses[tp] / len(rids)
+                ),
+            }
+
+        print(json.dumps(result, indent=2))
+        return result
+
+    def add_network(self, subplot, top_nodes, mid_nodes, bottom_nodes):
+        graph = nx.DiGraph()
+        node_names = []
+        for n in top_nodes + mid_nodes + bottom_nodes:
+            graph.add_node(n)
+            node_names.append(n)
+        for n1 in top_nodes:
+            for n2 in mid_nodes:
+                graph.add_edge(n1, n2)
+        for n1 in mid_nodes:
+            for n2 in bottom_nodes:
+                graph.add_edge(n1, n2)
+
+        positions = {}
+
+        def calc_node_positions(nodes, layer):
+            space = round(100 / max(len(nodes), 1))
+            for i, n in enumerate(nodes):
+                positions[n] = (round((i + 1 / 2) * space), layer)
+
+        calc_node_positions(top_nodes, 2)
+        calc_node_positions(mid_nodes, 1)
+        calc_node_positions(bottom_nodes, 0)
+
+        nx.draw(
+            graph,
+            with_labels=True,
+            node_color=[self.type_color(nn.replace(" ", "")) for nn in node_names],
+            pos=positions,
+            node_shape="o",
+            node_size=[len(nn) ** 2 * 60 for nn in node_names],
+            ax=subplot,
         )
-        print(
-            f"Types:{set(data_rid_to_type.values())}, num cells without type: {len([v for v in data_rid_to_type.values() if not v])}"
+        subplot.set_title(
+            f"Average connectivity by type/class", y=-0.1
         )
 
-        sides = defaultdict(int)
-        type_to_cells = defaultdict(list)
-        for r in data_rids_filtered:
-            type_to_cells[data_rid_to_type[r] or "NO_TYPE"].append(r)
-            sides[self.neuron_db.neuron_data[r]["side"]] += 1
-        print(dict(sides))
-
-        non_dsx_fru_matches = set()
-
-        for r in data_rids_filtered:
-            side = self.neuron_db.neuron_data[r]["side"]
-            sims = self.neuron_db.neuron_data[r]["similar_cell_scores"]
-            sims = dict(sorted(sims.items(), key=lambda item: -item[1]))
-            section_started = False
-            cell_matches = [r]
-            for c, s in sims.items():
-                if c in data_rids_filtered:
-                    continue
-                else:
-                    non_dsx_fru_matches.add(c)
-                    cell_matches.append(c)
-                if not section_started:
-                    section_started = True
-                    print(
-                        f"### Similar cells for {r} ({side}) that are not marked as DSX/FRU"
-                    )
-
-                msg = "is DSX/FRU" if c in data_rids_filtered else "not DSC/FRU"
-                c_side = self.neuron_db.neuron_data[c]["side"]
-                # log_dev_url_for_root_ids(f"match={s} {c} | {c_side} | {msg}", [r, c], point_to_prod=True)
-                print(f"match={s} {c} | {c_side} | {msg} {c}")
-            if section_started:
-                # log_dev_url_for_root_ids(f"all matches", cell_matches, point_to_prod=True)
-                print(f"all matches {cell_matches}")
-
-        print(f"# Total num. matches outside dsx/fru: {len(non_dsx_fru_matches)}")
-
-        output_lines = []
-
-        for tp, cell_ids in type_to_cells.items():
-            output_lines.append("---------------")
-            output_lines.append(f"Type {tp}")
-            output_lines.append("current:")
-            output_lines.append(str(cell_ids))
-            similar_cell_ids = set()
-            for r in cell_ids:
-                sims = self.neuron_db.neuron_data[r]["similar_cell_scores"]
-                sims = dict(sorted(sims.items(), key=lambda item: -item[1]))
-                for c, s in sims.items():
-                    if s < 5 or c in data_rids_filtered:
-                        continue
-                    else:
-                        non_dsx_fru_matches.add(c)
-                        similar_cell_ids.add(c)
-            output_lines.append("candidates:")
-            output_lines.append(
-                str(list(similar_cell_ids)) if similar_cell_ids else "not found"
-            )
+    @staticmethod
+    def add_barchart(subplot, title, ylabel, legend, data, colorizer):
+        # first trim to top 10, then sort by key
+        data = sorted(data, key=lambda x: -x[1])[:10]
+        data = sorted(data, key=lambda x: x[0])
+        bars = [d[0] for d in data]
+        counts = [d[1] for d in data]
+        bar_labels = [d[2] for d in data]
+        bar_colors = [colorizer(bar) for bar in bars]
+
+        subplot.bar(bars, counts, label=bar_labels, color=bar_colors)
+
+        subplot.set_ylabel(ylabel)
+        subplot.set_title(title)
+        if legend:
+            subplot.legend(title=legend)
 
-        output_lines.append("================")
-        output_lines.append(
-            f"# Total num. candidates identified: {len(non_dsx_fru_matches)}"
+    @staticmethod
+    def add_table(subplot, title, subtitle, data_dict):
+        # Data for the table
+        data = [[k, display(v)] for k, v in data_dict.items()]
+
+        # Set titles for the figure and the subplot respectively
+        subplot.set_title(
+            f"{title}\n\n{subtitle}", fontsize=18, fontweight="bold", color="purple"
         )
 
-        print("\n".join(output_lines))
+        # Hide axes
+        subplot.axis("off")
+
+        # Plot the table
+        table = subplot.table(
+            cellText=data,
+            colLabels=None,
+            cellLoc="left",
+            loc="center",
+            edges="open",
+        )
+
+        # adjust 2nd col to the right
+        for key, cell in table._cells.items():
+            cell.PAD = 0.03  # Adjust padding to move text to the right for the second column (optional)
+            if key[1] == 1:  # Align the text in the second column to the right
+                cell._text.set_horizontalalignment("right")
+
+        # Customize the table
+        table.auto_set_font_size(False)
+        table.set_fontsize(16)
+
+        # Adjust table layout
+        table.scale(1, 2.5)
+
+    def test_make_catalog_figures(self):
+        meta_data = self.collect_data()
+        for k in self.dsx_fry_types:
+            v = meta_data[k]
+            fig, ax = plt.subplots(nrows=2, ncols=3, figsize=(32, 20))
+
+            self.add_network(
+                subplot=ax[1][2],
+                top_nodes=[f"  {nn}   " for nn in v["predicate_input_types"]],
+                mid_nodes=[f"   {k}   "],
+                bottom_nodes=[f"   {nn}  " for nn in v["predicate_output_types"]],
+            )
+
+            table_data_dict = {
+                "Avg. in degree": meta_data[k]["avg_in_degree"],
+                "Avg. out degree": meta_data[k]["avg_out_degree"],
+                "Avg. in synapses": meta_data[k]["avg_in_synapses"],
+                "Avg. out synapses": meta_data[k]["avg_out_synapses"],
+                "Avg. cable length (nm)": meta_data[k]["avg_length_nm"],
+                "Avg. surface area (nm^2)": meta_data[k]["avg_area_nm"],
+                "Avg. volume (nm^3)": meta_data[k]["avg_volume_nm"],
+            }
+            self.add_table(
+                subplot=ax[0][0],
+                title=f"{k}",
+                subtitle=f'{display(meta_data[k]["num_cells"])} cells',
+                data_dict=table_data_dict,
+            )
+            self.add_barchart(
+                subplot=ax[1][0],
+                title="Upstream Partner Types",
+                ylabel="avg. num cells",
+                legend=None,
+                data=[
+                    (k, v, None)
+                    for k, v in meta_data[k]["avg_in_partners_by_type"].items()
+                ],
+                colorizer=self.type_color,
+            )
+            self.add_barchart(
+                subplot=ax[1][1],
+                title="Downstream Partner Types",
+                ylabel="avg. num cells",
+                legend=None,
+                data=[
+                    (k, v, None)
+                    for k, v in meta_data[k]["avg_out_partners_by_type"].items()
+                ],
+                colorizer=self.type_color,
+            )
+
+            plt.savefig(f"../../static/experimental_data/dsx_fru_catalog_assets/fig_{k}.png")
+            plt.close(fig)