impl index spec

python/tests/test_base_install/test_index_spec.py

@@ -0,0 +1,144 @@
+from raphtory import Graph, IndexSpecBuilder
+from raphtory import filter
+
+def init_graph(graph):
+    nodes = [
+        (1, "pometry", {"p1": 5, "p2": 50}, "fire_nation", {"x": True}),
+        (1, "raphtory", {"p1": 10, "p2": 100}, "water_tribe", {"y": False}),
+    ]
+    for t, name, props, group, const_props in nodes:
+        n = graph.add_node(t, name, props, group)
+        n.add_constant_properties(const_props)
+
+    edges = [
+        (1, "pometry", "raphtory", {"e_p1": 3.2, "e_p2": 10.0}, "Football", {"e_x": True}),
+        (1, "raphtory", "pometry", {"e_p1": 4.0, "e_p2": 20.0}, "Baseball", {"e_y": False}),
+    ]
+    for t, src, dst, props, label, const_props in edges:
+        e = graph.add_edge(t, src, dst, props, label)
+        e.add_constant_properties(const_props, label)
+
+    return graph
+
+
+def search_nodes(graph, filter_expr):
+    return sorted(n.name for n in graph.search_nodes(filter_expr, 10, 0))
+
+
+def search_edges(graph, filter_expr):
+    return sorted(f"{e.src.name}->{e.dst.name}" for e in graph.search_edges(filter_expr, 10, 0))
+
+
+def test_with_all_props_index_spec():
+    graph = init_graph(Graph())
+    spec = (
+        IndexSpecBuilder(graph)
+        .with_all_node_props()
+        .with_all_edge_props()
+        .build()
+    )
+
+    graph.create_index_in_ram_with_spec(spec)
+
+    f1 = filter.Property("p1") == 5
+    f2 = filter.Property("x") == True
+    assert search_nodes(graph, f1 & f2) == ["pometry"]
+
+    f1 = filter.Property("e_p1") < 5.0
+    f2 = filter.Property("e_y") == False
+    assert sorted(search_edges(graph, f1 & f2)) == sorted(["raphtory->pometry"])
+
+
+def test_with_selected_props_index_spec():
+    graph = init_graph(Graph())
+    spec = (
+        IndexSpecBuilder(graph)
+        .with_const_node_props(["y"])
+        .with_temp_node_props(["p1"])
+        .with_const_edge_props(["e_y"])
+        .with_temp_edge_props(["e_p1"])
+        .build()
+    )
+
+    graph.create_index_in_ram_with_spec(spec)
+
+    f1 = filter.Property("p1") == 5
+    f2 = filter.Property("y") == False
+    assert sorted(search_nodes(graph, f1 | f2)) == sorted(["pometry", "raphtory"])
+
+    f = filter.Property("y") == False
+    assert search_nodes(graph, f) == ["raphtory"]
+
+    f1 = filter.Property("e_p1") < 5.0
+    f2 = filter.Property("e_y") == False
+    assert sorted(search_edges(graph, f1 | f2)) == sorted(["pometry->raphtory", "raphtory->pometry"])
+
+
+def test_with_invalid_property_returns_error():
+    graph = init_graph(Graph())
+    try:
+        IndexSpecBuilder(graph).with_const_node_props(["xyz"])
+        assert False, "Expected error for unknown property"
+    except Exception as e:
+        assert "xyz" in str(e)
+
+
+def test_build_empty_spec_by_default():
+    graph = init_graph(Graph())
+    spec = IndexSpecBuilder(graph).build()
+
+    graph.create_index_in_ram_with_spec(spec)
+
+    f1 = filter.Property("p1") == 5
+    f2 = filter.Property("x") == True
+    assert sorted(search_nodes(graph, f1 & f2)) == ["pometry"]
+
+    f1 = filter.Property("e_p1") < 5.0
+    f2 = filter.Property("e_y") == False
+    assert sorted(search_edges(graph, f1 | f2)) == sorted(["pometry->raphtory", "raphtory->pometry"])
+
+
+def test_mixed_node_and_edge_props_index_spec():
+    graph = init_graph(Graph())
+    spec = (
+        IndexSpecBuilder(graph)
+        .with_const_node_props(["x"])
+        .with_all_temp_node_props()
+        .with_all_edge_props()
+        .build()
+    )
+
+    graph.create_index_in_ram_with_spec(spec)
+
+    f1 = filter.Property("p1") == 5
+    f2 = filter.Property("y") == False
+    assert sorted(search_nodes(graph, f1 | f2)) == sorted(["pometry", "raphtory"])
+
+    f1 = filter.Property("e_p1") < 5.0
+    f2 = filter.Property("e_y") == False
+    assert sorted(search_edges(graph, f1 | f2)) == sorted(["pometry->raphtory", "raphtory->pometry"])
+
+
+def test_get_index_spec():
+    graph = init_graph(Graph())
+    spec = (
+        IndexSpecBuilder(graph)
+        .with_const_node_props(["x"])
+        .with_all_temp_node_props()
+        .with_all_edge_props()
+        .build()
+    )
+
+    graph.create_index_in_ram_with_spec(spec)
+
+    returned_spec = graph.get_index_spec()
+
+    node_const_names = {name for (name, _, _) in returned_spec.node_const_props}
+    node_temp_names = {name for (name, _, _) in returned_spec.node_temp_props}
+    edge_const_names = {name for (name, _, _) in returned_spec.edge_const_props}
+    edge_temp_names = {name for (name, _, _) in returned_spec.edge_temp_props}
+
+    assert "x" in node_const_names
+    assert "p1" in node_temp_names or "p2" in node_temp_names
+    assert "e_x" in edge_const_names or "e_y" in edge_const_names
+    assert "e_p1" in edge_temp_names or "e_p2" in edge_temp_names

raphtory/src/core/utils/errors.rs

@@ -386,6 +386,9 @@ pub enum GraphError {
     #[error("Failed to create index in ram")]
     FailedToCreateIndexInRam,
 
+    #[error("Unknown property key: {0}")]
+    UnknownProperty(String),
+
     #[error("Your window and step must be of the same type: duration (string) or epoch (int)")]
     MismatchedIntervalTypes,
 }

raphtory/src/db/api/storage/storage.rs

@@ -21,9 +21,9 @@ use crate::{
     },
 };
 
-use crate::db::api::{
-    storage::graph::edges::edge_storage_ops::EdgeStorageOps,
-    view::internal::{InheritEdgeHistoryFilter, InheritNodeHistoryFilter, InternalStorageOps},
+use crate::db::api::view::{
+    internal::{InheritEdgeHistoryFilter, InheritNodeHistoryFilter, InternalStorageOps},
+    IndexSpec,
 };
 #[cfg(feature = "search")]
 use crate::search::graph_index::GraphIndex;
@@ -140,27 +140,38 @@ impl Storage {
 
 #[cfg(feature = "search")]
 impl Storage {
+    pub(crate) fn get_index_spec(&self) -> Result<IndexSpec, GraphError> {
+        let index = self.index.get().ok_or(GraphError::GraphIndexIsMissing)?;
+        Ok(index.index_spec.read().clone())
+    }
+
+    pub(crate) fn get_or_load_index(&self, path: PathBuf) -> Result<&GraphIndex, GraphError> {
+        self.index.get_or_try_init(|| {
+            let index = GraphIndex::load_from_path(&self.graph, &path)?;
+            Ok(index)
+        })
+    }
+
     pub(crate) fn get_or_create_index(
         &self,
-        path: Option<PathBuf>,
+        index_spec: IndexSpec,
     ) -> Result<&GraphIndex, GraphError> {
+        if let Some(index) = self.index.get() {
+            index.update(&self.graph, index_spec.clone())?;
+        };
         self.index.get_or_try_init(|| {
-            if let Some(path) = path {
-                Ok::<_, GraphError>(GraphIndex::load_from_path(&path)?)
-            } else {
-                let cache_path = self.get_cache().map(|cache| cache.folder.get_base_path());
-                Ok::<_, GraphError>(GraphIndex::create_from_graph(
-                    &self.graph,
-                    false,
-                    cache_path,
-                )?)
-            }
+            let cached_graph_path = self.get_cache().map(|cache| cache.folder.get_base_path());
+            let index = GraphIndex::create(&self.graph, false, cached_graph_path, index_spec)?;
+            Ok(index)
         })
     }
 
-    pub(crate) fn get_or_create_index_in_ram(&self) -> Result<&GraphIndex, GraphError> {
+    pub(crate) fn get_or_create_index_in_ram(
+        &self,
+        index_spec: IndexSpec,
+    ) -> Result<&GraphIndex, GraphError> {
         let index = self.index.get_or_try_init(|| {
-            Ok::<_, GraphError>(GraphIndex::create_from_graph(&self.graph, true, None)?)
+            Ok::<_, GraphError>(GraphIndex::create(&self.graph, true, None, index_spec)?)
         })?;
         if index.path.is_some() {
             Err(GraphError::FailedToCreateIndexInRam)
@@ -316,9 +327,6 @@ impl InternalAdditionOps for Storage {
         #[cfg(feature = "proto")]
         self.if_cache(|cache| cache.resolve_node_property(prop, id, &dtype, is_static));
 
-        #[cfg(feature = "search")]
-        self.if_index(|index| index.create_node_property_index(id, prop, &dtype, is_static))?;
-
         Ok(id)
     }
 
@@ -335,9 +343,6 @@ impl InternalAdditionOps for Storage {
         #[cfg(feature = "proto")]
         self.if_cache(|cache| cache.resolve_edge_property(prop, id, &dtype, is_static));
 
-        #[cfg(feature = "search")]
-        self.if_index(|index| index.create_edge_property_index(id, prop, &dtype, is_static))?;
-
         Ok(id)
     }
 
@@ -375,7 +380,7 @@ impl InternalAdditionOps for Storage {
         });
 
         #[cfg(feature = "search")]
-        self.if_index(|index| index.add_edge_update(&self.graph, id, t, src, dst, layer, props))?;
+        self.if_index(|index| index.add_edge_update(&self.graph, id, t, layer, props))?;
 
         Ok(id)
     }
@@ -393,12 +398,7 @@ impl InternalAdditionOps for Storage {
         self.if_cache(|cache| cache.add_edge_update(t, edge, props, layer));
 
         #[cfg(feature = "search")]
-        self.if_index(|index| {
-            let ee = self.graph.edge_entry(edge);
-            let src = ee.src();
-            let dst = ee.dst();
-            index.add_edge_update(&self.graph, Existing(edge), t, src, dst, layer, props)
-        })?;
+        self.if_index(|index| index.add_edge_update(&self.graph, Existing(edge), t, layer, props))?;
 
         Ok(())
     }
@@ -451,7 +451,7 @@ impl InternalPropertyAdditionOps for Storage {
         self.if_cache(|cache| cache.add_node_cprops(vid, props));
 
         #[cfg(feature = "search")]
-        self.if_index(|index| index.add_node_constant_properties(&self.graph, vid, props))?;
+        self.if_index(|index| index.add_node_constant_properties(vid, props))?;
 
         Ok(())
     }
@@ -468,7 +468,7 @@ impl InternalPropertyAdditionOps for Storage {
         self.if_cache(|cache| cache.add_node_cprops(vid, props));
 
         #[cfg(feature = "search")]
-        self.if_index(|index| index.update_node_constant_properties(&self.graph, vid, props))?;
+        self.if_index(|index| index.update_node_constant_properties(vid, props))?;
 
         Ok(())
     }
@@ -486,7 +486,7 @@ impl InternalPropertyAdditionOps for Storage {
         self.if_cache(|cache| cache.add_edge_cprops(eid, layer, props));
 
         #[cfg(feature = "search")]
-        self.if_index(|index| index.add_edge_constant_properties(&self.graph, eid, layer, props))?;
+        self.if_index(|index| index.add_edge_constant_properties(eid, layer, props))?;
 
         Ok(())
     }
@@ -504,9 +504,7 @@ impl InternalPropertyAdditionOps for Storage {
         self.if_cache(|cache| cache.add_edge_cprops(eid, layer, props));
 
         #[cfg(feature = "search")]
-        self.if_index(|index| {
-            index.update_edge_constant_properties(&self.graph, eid, layer, props)
-        })?;
+        self.if_index(|index| index.update_edge_constant_properties(eid, layer, props))?;
 
         Ok(())
     }