Merge branch 'main' into feature/optimization

.gitignore

@@ -1,4 +1,6 @@
 tests/**/compiled
+tests/_machinable/remotes
+tests/_machinable/storage
 examples/datasets
 
 # Byte-compiled / optimized / DLL files

src/miv_simulator/interface/connections.py

@@ -62,3 +62,16 @@ def __call__(self):
 
     def on_write_meta_data(self):
         return MPI.COMM_WORLD.Get_rank() == 0
+
+    def compute_context(self):
+        context = super().compute_context()
+        del context["config"]["filepath"]
+        del context["config"]["forest_filepath"]
+        del context["config"]["io_size"]
+        del context["config"]["chunk_size"]
+        del context["config"]["value_chunk_size"]
+        del context["config"]["cache_size"]
+        del context["config"]["write_size"]
+        del context["config"]["ranks"]
+        context["predicate"]["uses"] = sorted([u.hash for u in self.uses])
+        return context

src/miv_simulator/interface/distances.py

@@ -1,10 +1,9 @@
 from typing import Optional, Tuple
 
 import logging
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field
 
 from machinable import Component
-from machinable.config import Field
 from miv_simulator import config, simulator
 from mpi4py import MPI
 
@@ -53,3 +52,14 @@ def __call__(self):
 
     def on_write_meta_data(self):
         return MPI.COMM_WORLD.Get_rank() == 0
+
+    def compute_context(self):
+        context = super().compute_context()
+        del context["config"]["filepath"]
+        del context["config"]["io_size"]
+        del context["config"]["chunk_size"]
+        del context["config"]["value_chunk_size"]
+        del context["config"]["cache_size"]
+        del context["config"]["ranks"]
+        context["predicate"]["uses"] = sorted([u.hash for u in self.uses])
+        return context

src/miv_simulator/interface/network.py

@@ -68,7 +68,7 @@ def launch(self):
                     "io_size": 1,
                     "write_size": 0,
                 },
-                uses=list(self.synapse_forest.values()),
+                uses=self.synapse_forest[population],
             ).launch()
             for population in config.synapses
         }
@@ -85,7 +85,7 @@ def launch(self):
                     "cache_size": 20,
                     "write_size": 100,
                 },
-                uses=list(self.synapses.values()),
+                uses=self.synapses[population],
             ).launch()
             for population in config.synapses
         }
@@ -102,3 +102,8 @@ def launch(self):
         ).launch()
 
         return self
+
+    def compute_context(self):
+        context = super().compute_context()
+        del context["config"]
+        return context

src/miv_simulator/interface/network_architecture.py

@@ -128,3 +128,13 @@ def generate_connections(self, version=None, uses=None):
             + normversion(version),
             uses=_join(self, uses),
         )
+
+    def compute_context(self):
+        context = super().compute_context()
+        del context["config"]["filepath"]
+        del context["config"]["io_size"]
+        del context["config"]["chunk_size"]
+        del context["config"]["value_chunk_size"]
+        del context["config"]["ranks"]
+        context["predicate"]["uses"] = sorted([u.hash for u in self.uses])
+        return context

src/miv_simulator/interface/neuroh5_graph.py

@@ -34,9 +34,9 @@ def __call__(self) -> None:
                     populations.append(p)
                     if p in self.connections:
                         raise ValueError(
-                            f"Redundant distance connection specification for population {p}"
+                            f"Redundant distance connection specification for population {p}. "
                             f"Found duplicate in {u.config.forest_filepath}, while already "
-                            f"defined in {self.connections[p].config.forest_filepath}"
+                            f"defined in {self.connections[p].config.forest_filepath} ({populations})"
                         )
                     self.connections[p] = u
             elif name == "synapse_forest":
@@ -90,23 +90,13 @@ def __call__(self) -> None:
             },
         )
 
-    def on_compute_predicate(self):
-        def generate_uid(use):
-            if getattr(use, "refreshed_at", None) is not None:
-                return f"{use.uuid}-{use.refreshed_at}"
-            return use.uuid
-
-        return {
-            "uses": sorted(
-                map(
-                    generate_uid,
-                    self.uses,
-                )
-            )
-        }
-
     def files(self) -> Dict[str, str]:
         return {
             "cells": self.graph.cells_filepath,
             "connections": self.graph.connections_filepath,
         }
+
+    def compute_context(self):
+        context = super().compute_context()
+        context["predicate"]["uses"] = sorted([u.hash for u in self.uses])
+        return context

src/miv_simulator/interface/synapse_forest.py

@@ -1,6 +1,7 @@
 from machinable import Component
 from miv_simulator import config, simulator
 from pydantic import BaseModel, Field, ConfigDict
+from machinable.utils import file_hash
 
 
 class GenerateSynapseForest(Component):
@@ -28,3 +29,13 @@ def __call__(self) -> None:
             morphology=self.config.morphology,
         )
         print("generate_synapse_forest() completed")
+
+    def compute_context(self):
+        # remove filepath in favor of uses
+        context = super().compute_context()
+        del context["config"]["filepath"]
+        context["config"]["morphology"] = file_hash(
+            context["config"]["morphology"]
+        )
+        context["predicate"]["uses"] = sorted([u.hash for u in self.uses])
+        return context

src/miv_simulator/interface/synapses.py

@@ -64,3 +64,16 @@ def __call__(self):
 
     def on_write_meta_data(self):
         return MPI.COMM_WORLD.Get_rank() == 0
+
+    def compute_context(self):
+        context = super().compute_context()
+        del context["config"]["forest_filepath"]
+        del context["config"]["template_path"]
+        del context["config"]["mechanisms_path"]
+        del context["config"]["io_size"]
+        del context["config"]["write_size"]
+        del context["config"]["chunk_size"]
+        del context["config"]["value_chunk_size"]
+        del context["config"]["ranks"]
+        context["predicate"]["uses"] = sorted([u.hash for u in self.uses])
+        return context

src/miv_simulator/mechanisms.py

@@ -49,6 +49,8 @@ def compile(
     file_data = {}
     if recursive:
         mod_files = glob(os.path.join(src, "**/*.mod"), recursive=True)
+        # ignore output_path if part of this directory
+        mod_files = [f for f in mod_files if output_path not in f]
     else:
         mod_files = glob(os.path.join(src, "*.mod"))
     for m in sorted(mod_files, key=lambda x: x.replace(src, "")):

tests/_machinable/project.py

@@ -0,0 +1,8 @@
+from machinable import Project
+
+
+class TestEnv(Project):
+    def on_resolve_remotes(self):
+        return {
+            "mpi": "url+https://raw.githubusercontent.com/machinable-org/machinable/2670e9626eb548f6ce2301923be1f49642086d8c/docs/examples/mpi-execution/mpi.py",
+        }

tests/mechanisms/test_mechanisms.py

@@ -9,6 +9,10 @@ def test_mechanisms_compile(tmp_path):
     d = str(tmp_path / "mechanisms")
     os.makedirs(d)
 
+    with open("tests/mechanisms/Gfluct3.mod", "r") as f:
+        with open(os.path.join(d, "Gfluct3.mod"), "w") as g:
+            g.write(f.read())
+
     # invalid directory
     with pytest.raises(FileNotFoundError):
         compile()
@@ -22,6 +26,10 @@ def test_mechanisms_compile(tmp_path):
     compiled_path = compile(d, force=True)
     assert len(os.path.basename(compiled_path)) == 64
 
+    h1 = compile(d, force=True, return_hash=True)
+    h2 = compile(d, force=True, return_hash=True)
+    assert h1 == h2
+
 
 def test_mechanisms_compile_and_load(tmp_path):
     d = str(tmp_path / "mechanisms")

tests/test_interface.py

@@ -0,0 +1,37 @@
+import pytest
+import os
+
+
+@pytest.mark.skipif(
+    not os.environ.get("MIV_SIMULATOR_TEST_INTERFACE_SRC"),
+    reason="Environment variables not available.",
+)
+def test_interface(tmp_path):
+    from machinable import get
+    from miv_simulator.mechanisms import compile
+
+    source = os.environ["MIV_SIMULATOR_TEST_INTERFACE_SRC"]
+    wd = os.path.dirname(__file__)
+
+    debug = True
+    storage_directory = str(tmp_path / "storage")
+    if debug:
+        storage_directory = f"{wd}/_machinable/storage"
+
+    with get("machinable.index", storage_directory), get(
+        "machinable.project", wd
+    ):
+        with get("mpi", {"ranks": -1}) as run:
+            get(
+                "miv_simulator.interface.network",
+                {
+                    "config_filepath": f"{source}/config/Microcircuit.yaml",
+                    "mechanisms_path": compile(f"{source}/mechanisms"),
+                    "template_path": f"{source}/templates",
+                    "morphology_path": f"{source}/morphology",
+                },
+            ).launch()
+
+        for component in run.executables:
+            print(component)
+            assert component.cached()