Merge pull request #1019 from WolframResearch/feature/build-with-abso…

…lute-paths Added option to build vector databases in target directory with absolute paths
WolframResearch · Jan 9, 2025 · 94dd11e · 94dd11e
2 parents 3990981 + 77cc3b3
commit 94dd11e
Showing 1 changed file with 42 additions and 9 deletions.
diff --git a/Developer/VectorDatabases/VectorDatabaseBuilder.wl b/Developer/VectorDatabases/VectorDatabaseBuilder.wl
@@ -56,6 +56,7 @@ $incrementalBuildBatchSize = 512;
 $dbConnectivity            = 16;
 $dbExpansionAdd            = 256;
 $dbExpansionSearch         = 2048;
+$relativePaths             = Automatic;
 
 (* ::**************************************************************************************************************:: *)
 (* ::Subsection::Closed:: *)
@@ -143,15 +144,17 @@ BuildVectorDatabase // ClearAll;
 BuildVectorDatabase // Options = {
     "Connectivity"    :> $dbConnectivity,
     "ExpansionAdd"    :> $dbExpansionAdd,
-    "ExpansionSearch" :> $dbExpansionSearch
+    "ExpansionSearch" :> $dbExpansionSearch,
+    "RelativePaths"   :> $relativePaths
 };
 
 BuildVectorDatabase[ All, opts: OptionsPattern[ ] ] :=
     Block[
         {
             $dbConnectivity    = OptionValue[ "Connectivity"    ],
             $dbExpansionAdd    = OptionValue[ "ExpansionAdd"    ],
-            $dbExpansionSearch = OptionValue[ "ExpansionSearch" ]
+            $dbExpansionSearch = OptionValue[ "ExpansionSearch" ],
+            $relativePaths     = checkRelativePaths[ OptionValue[ "RelativePaths" ], True ]
         },
         AssociationMap[ BuildVectorDatabase, FileBaseName /@ getVectorDBSourceFile @ All ]
     ];
@@ -161,16 +164,30 @@ BuildVectorDatabase[ name_String, opts: OptionsPattern[ ] ] := Enclose[
         {
             $dbConnectivity    = OptionValue[ "Connectivity"    ],
             $dbExpansionAdd    = OptionValue[ "ExpansionAdd"    ],
-            $dbExpansionSearch = OptionValue[ "ExpansionSearch" ]
+            $dbExpansionSearch = OptionValue[ "ExpansionSearch" ],
+            $relativePaths     = checkRelativePaths[ OptionValue[ "RelativePaths" ], True ]
         },
-        WithCleanup[
-            SetDirectory @ ensureDirectory @ $vectorDBTargetDirectory,
-            ConfirmMatch[ buildVectorDatabase @ name, $$vectorDatabase, "Build" ],
-            ResetDirectory[ ]
+        If[ TrueQ @ $relativePaths,
+            WithCleanup[
+                SetDirectory @ ensureDirectory @ $vectorDBTargetDirectory,
+                ConfirmMatch[ buildVectorDatabase @ name, $$vectorDatabase, "Build" ],
+                ResetDirectory[ ]
+            ],
+            ConfirmMatch[ buildVectorDatabase @ name, $$vectorDatabase, "Build" ]
         ]
     ]
 ];
 
+BuildVectorDatabase[ id_, dir_, opts: OptionsPattern[ ] ] := Enclose[
+    Block[
+        {
+            $vectorDBTargetDirectory = ConfirmBy[ GeneralUtilities`EnsureDirectory @ dir, DirectoryQ, "Directory" ],
+            $relativePaths           = checkRelativePaths[ OptionValue[ "RelativePaths" ], False ]
+        },
+        BuildVectorDatabase[ id, opts ]
+    ]
+];
+
 
 buildVectorDatabase // ClearAll;
 
@@ -180,7 +197,12 @@ buildVectorDatabase[ name_String ] :=
         loadEmbeddingCache[ ];
 
         dir = ConfirmBy[ ensureDirectory @ { $vectorDBTargetDirectory, name }, DirectoryQ, "Directory" ];
-        rel = ConfirmBy[ ResourceFunction[ "RelativePath" ][ dir ], DirectoryQ, "Relative" ];
+
+        rel = If[ TrueQ @ $relativePaths,
+                  ConfirmBy[ ResourceFunction[ "RelativePath" ][ dir ], DirectoryQ, "Relative" ],
+                  dir
+              ];
+
         src = ConfirmBy[ getVectorDBSourceFile @ name, FileExistsQ, "File" ];
 
         DeleteFile /@ FileNames[ { "*.wxf", "*.usearch" }, dir ];
@@ -231,7 +253,10 @@ buildVectorDatabase[ name_String ] :=
             ConfirmBy[ rewriteDBData[ rel, name ], FileExistsQ, "Rewrite" ];
 
             built = ConfirmMatch[
-                VectorDatabaseObject @ File @ FileNameJoin @ { rel, name <> ".wxf" },
+                If[ TrueQ @ $relativePaths,
+                    VectorDatabaseObject @ File @ FileNameJoin @ { rel, name <> ".wxf" },
+                    VectorDatabaseObject[ File @ FileNameJoin @ { dir, name <> ".wxf" }, OverwriteTarget -> True ]
+                ],
                 $$vectorDatabase,
                 "Result"
             ];
@@ -265,6 +290,14 @@ buildVectorDatabase[ name_String ] :=
         ConfirmMatch[ built, $$vectorDatabase, "Result" ]
     ];
 
+(* ::**************************************************************************************************************:: *)
+(* ::Subsubsection::Closed:: *)
+(*checkRelativePaths*)
+checkRelativePaths // ClearAll;
+checkRelativePaths[ relative: True|False, default_ ] := relative;
+checkRelativePaths[ relative_, default: True|False ] := default;
+checkRelativePaths[ relative_, default_ ] := True;
+
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsection::Closed:: *)
 (*setDBDefaults*)