feat(cli): Allow vectorising hidden files (#70)

* Allow vectorising hidden files

* Do not forcibly ignore `.git`

Author: jpmelos

src/vectorcode/cli_utils.py

@@ -52,6 +52,7 @@ class CliAction(Enum):
 class Config:
     no_stderr: bool = False
     recursive: bool = False
+    include_hidden: bool = False
     to_be_deleted: list[str] = field(default_factory=list)
     pipe: bool = False
     action: Optional[CliAction] = None
@@ -202,6 +203,12 @@ def get_cli_parser():
         default=False,
         help="Recursive indexing for directories.",
     )
+    vectorise_parser.add_argument(
+        "--include-hidden",
+        action="store_true",
+        default=False,
+        help="Include hidden files.",
+    )
     vectorise_parser.add_argument(
         "--force",
         "-f",
@@ -306,6 +313,7 @@ async def parse_cli_args(args: Optional[Sequence[str]] = None):
     files = []
     query = None
     recursive = False
+    include_hidden = False
     number_of_result = 1
     force = False
     chunk_size = -1
@@ -319,6 +327,7 @@ async def parse_cli_args(args: Optional[Sequence[str]] = None):
         case "vectorise":
             files = main_args.file_paths
             recursive = main_args.recursive
+            include_hidden = main_args.include_hidden
             force = main_args.force
             chunk_size = main_args.chunk_size
             overlap_ratio = main_args.overlap
@@ -344,6 +353,7 @@ async def parse_cli_args(args: Optional[Sequence[str]] = None):
         project_root=main_args.project_root,
         query=query,
         recursive=recursive,
+        include_hidden=include_hidden,
         n_result=number_of_result,
         pipe=main_args.pipe,
         force=force,
@@ -436,7 +446,7 @@ def expand_path(path: PathLike, absolute: bool = False) -> PathLike:
 
 
 async def expand_globs(
-    paths: list[PathLike], recursive: bool = False
+    paths: list[PathLike], recursive: bool = False, include_hidden: bool = False
 ) -> list[PathLike]:
     result = set()
     stack = paths
@@ -445,9 +455,19 @@ async def expand_globs(
         if os.path.isfile(curr):
             result.add(expand_path(curr))
         elif "**" in str(curr):
-            stack.extend(glob.glob(str(curr), recursive=True))
+            stack.extend(
+                glob.glob(str(curr), recursive=True, include_hidden=include_hidden)
+            )
         elif "*" in str(curr):
-            stack.extend(glob.glob(str(curr), recursive=recursive))
+            stack.extend(
+                glob.glob(str(curr), recursive=recursive, include_hidden=include_hidden)
+            )
         elif os.path.isdir(curr) and recursive:
-            stack.extend(glob.glob(os.path.join(curr, "**", "*"), recursive=recursive))
+            stack.extend(
+                glob.glob(
+                    os.path.join(curr, "**", "*"),
+                    recursive=recursive,
+                    include_hidden=include_hidden,
+                )
+            )
     return list(result)

src/vectorcode/subcommands/vectorise.py

@@ -122,8 +122,7 @@ def load_files_from_include(project_root: str) -> list[str]:
     include_file_path = os.path.join(project_root, ".vectorcode", "vectorcode.include")
     if os.path.isfile(include_file_path):
         with open(include_file_path) as fin:
-            specs = pathspec.PathSpec.from_lines(
-                pattern_factory="gitwildmatch",
+            specs = pathspec.GitIgnoreSpec.from_lines(
                 lines=(os.path.expanduser(i) for i in fin.readlines()),
             )
         return [
@@ -144,12 +143,15 @@ async def vectorise(configs: Config) -> int:
         return 1
     if not verify_ef(collection, configs):
         return 1
-    gitignore_path = os.path.join(str(configs.project_root), ".gitignore")
+
     files = await expand_globs(
         configs.files or load_files_from_include(str(configs.project_root)),
         recursive=configs.recursive,
+        include_hidden=configs.include_hidden,
     )
+
     if not configs.force:
+        gitignore_path = os.path.join(str(configs.project_root), ".gitignore")
         specs = [
             gitignore_path,
         ]

tests/subcommands/test_vectorise.py

@@ -85,19 +85,15 @@ def test_show_stats_pipe_true(capsys):
 
 def test_exclude_paths_by_spec():
     paths = ["file1.py", "file2.py", "exclude.py"]
-    specs = pathspec.PathSpec.from_lines(
-        pattern_factory="gitwildmatch", lines=["exclude.py"]
-    )
+    specs = pathspec.GitIgnoreSpec.from_lines(lines=["exclude.py"])
     excluded_paths = exclude_paths_by_spec(paths, specs)
     assert "exclude.py" not in excluded_paths
     assert len(excluded_paths) == 2
 
 
 def test_include_paths_by_spec():
     paths = ["file1.py", "file2.py", "include.py"]
-    specs = pathspec.PathSpec.from_lines(
-        pattern_factory="gitwildmatch", lines=["include.py", "file1.py"]
-    )
+    specs = pathspec.GitIgnoreSpec.from_lines(lines=["include.py", "file1.py"])
     included_paths = include_paths_by_spec(paths, specs)
     assert "file2.py" not in included_paths
     assert len(included_paths) == 2

tests/test_cli_utils.py

@@ -342,6 +342,26 @@ async def test_parse_cli_args_vectorise():
         assert config.files == ["file1.txt"]
 
 
+@pytest.mark.asyncio
+async def test_parse_cli_args_vectorise_recursive_dir():
+    with patch("sys.argv", ["vectorcode", "vectorise", "-r", "."]):
+        config = await parse_cli_args()
+        assert config.action == CliAction.vectorise
+        assert config.files == ["."]
+        assert config.recursive is True
+        assert config.include_hidden is False
+
+
+@pytest.mark.asyncio
+async def test_parse_cli_args_vectorise_recursive_dir_include_hidden():
+    with patch("sys.argv", ["vectorcode", "vectorise", "-r", "."]):
+        config = await parse_cli_args()
+        assert config.action == CliAction.vectorise
+        assert config.files == ["."]
+        assert config.recursive is True
+        assert config.include_hidden is False
+
+
 @pytest.mark.asyncio
 async def test_parse_cli_args_vectorise_no_files():
     with patch("sys.argv", ["vectorcode", "vectorise"]):

tests/test_main.py

@@ -215,13 +215,16 @@ async def test_async_main_cli_action_query(monkeypatch):
 @pytest.mark.asyncio
 async def test_async_main_cli_action_vectorise(monkeypatch):
     mock_cli_args = MagicMock(
-        no_stderr=False, project_root=".", action=CliAction.vectorise
+        no_stderr=False,
+        project_root=".",
+        action=CliAction.vectorise,
+        include_hidden=True,
     )
     monkeypatch.setattr(
         "vectorcode.main.parse_cli_args", AsyncMock(return_value=mock_cli_args)
     )
     mock_final_configs = MagicMock(
-        host="test_host", port=1234, action=CliAction.vectorise
+        host="test_host", port=1234, action=CliAction.vectorise, include_hidden=True
     )
     monkeypatch.setattr(
         "vectorcode.main.get_project_config",