Expose incremental indexing support in the indexer CLI

Indexer-PiperOrigin-RevId: 807878969

Author: OSS-Fuzz Team

infra/indexer/frontend/frontend.cc

@@ -118,12 +118,13 @@ clang::tooling::CommandLineArguments ExtraArgumentsAdjuster(
 // perform indexing on a compilation database.
 std::vector<std::pair<std::unique_ptr<clang::tooling::FrontendActionFactory>,
                       clang::tooling::ArgumentsAdjuster>>
-GetIndexActions(FileCopier& file_copier, MergeQueue& merge_queue) {
+GetIndexActions(FileCopier& file_copier, MergeQueue& merge_queue,
+                bool support_incremental_indexing) {
   std::vector<std::pair<std::unique_ptr<clang::tooling::FrontendActionFactory>,
                         clang::tooling::ArgumentsAdjuster>>
       actions;
-  auto index_action =
-      std::make_unique<IndexActionFactory>(file_copier, merge_queue);
+  auto index_action = std::make_unique<IndexActionFactory>(
+      file_copier, merge_queue, support_incremental_indexing);
   auto adjuster = clang::tooling::combineAdjusters(RemoveClangArgumentsAdjuster,
                                                    ExtraArgumentsAdjuster);
   actions.push_back(

infra/indexer/frontend/frontend.h

@@ -39,7 +39,8 @@ std::vector<std::string> ParseCommandLine(absl::string_view commandLine);
 // perform indexing on a compilation database.
 std::vector<std::pair<std::unique_ptr<clang::tooling::FrontendActionFactory>,
                       clang::tooling::ArgumentsAdjuster>>
-GetIndexActions(FileCopier& file_copier, MergeQueue& merge_queue);
+GetIndexActions(FileCopier& file_copier, MergeQueue& merge_queue,
+                bool support_incremental_indexing = false);
 }  // namespace indexer
 }  // namespace oss_fuzz
 

infra/indexer/frontend/index_action.cc

@@ -15,6 +15,7 @@
 #include "indexer/frontend/index_action.h"
 
 #include <memory>
+#include <optional>
 #include <string>
 #include <utility>
 #include <vector>
@@ -29,6 +30,8 @@
 #include "absl/strings/match.h"
 #include "absl/strings/string_view.h"
 #include "clang/AST/ASTConsumer.h"
+#include "clang/Basic/FileEntry.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Lex/Pragma.h"
 #include "clang/Lex/Preprocessor.h"
@@ -41,23 +44,43 @@ namespace oss_fuzz {
 namespace indexer {
 class AstConsumer : public clang::ASTConsumer {
  public:
-  explicit AstConsumer(InMemoryIndex& index, clang::CompilerInstance& compiler)
-      : index_(index), compiler_(compiler) {}
+  AstConsumer(InMemoryIndex& index, clang::CompilerInstance& compiler,
+              bool support_incremental_indexing = false)
+      : index_(index),
+        compiler_(compiler),
+        support_incremental_indexing_(support_incremental_indexing) {}
   ~AstConsumer() override = default;
 
   void HandleTranslationUnit(clang::ASTContext& context) override {
+    if (support_incremental_indexing_) {
+      const clang::SourceManager& source_manager = context.getSourceManager();
+      const clang::FileID main_file_id = source_manager.getMainFileID();
+      const clang::OptionalFileEntryRef main_file =
+          source_manager.getFileEntryRefForID(main_file_id);
+      CHECK(main_file.has_value()) << "Couldn't retrieve the main file entry";
+
+      const clang::FileManager& file_manager = source_manager.getFileManager();
+      llvm::SmallString<256> absolute_path(main_file->getName());
+      file_manager.makeAbsolutePath(absolute_path);
+
+      index_.SetTranslationUnit({absolute_path.data(), absolute_path.size()});
+    }
+
     AstVisitor visitor(index_, context, compiler_);
     visitor.TraverseDecl(context.getTranslationUnitDecl());
   }
 
  private:
   InMemoryIndex& index_;
   clang::CompilerInstance& compiler_;
+  const bool support_incremental_indexing_;
 };
 
-IndexAction::IndexAction(FileCopier& file_copier, MergeQueue& merge_queue)
+IndexAction::IndexAction(FileCopier& file_copier, MergeQueue& merge_queue,
+                         bool support_incremental_indexing)
     : index_(std::make_unique<InMemoryIndex>(file_copier)),
-      merge_queue_(merge_queue) {}
+      merge_queue_(merge_queue),
+      support_incremental_indexing_(support_incremental_indexing) {}
 
 bool IndexAction::BeginSourceFileAction(clang::CompilerInstance& compiler) {
   CHECK(index_);
@@ -79,15 +102,20 @@ void IndexAction::EndSourceFileAction() { merge_queue_.Add(std::move(index_)); }
 
 std::unique_ptr<clang::ASTConsumer> IndexAction::CreateASTConsumer(
     clang::CompilerInstance& compiler, llvm::StringRef path) {
-  return std::make_unique<AstConsumer>(*index_, compiler);
+  return std::make_unique<AstConsumer>(*index_, compiler,
+                                       support_incremental_indexing_);
 }
 
 IndexActionFactory::IndexActionFactory(FileCopier& file_copier,
-                                       MergeQueue& merge_queue)
-    : file_copier_(file_copier), merge_queue_(merge_queue) {}
+                                       MergeQueue& merge_queue,
+                                       bool support_incremental_indexing)
+    : file_copier_(file_copier),
+      merge_queue_(merge_queue),
+      support_incremental_indexing_(support_incremental_indexing) {}
 
 std::unique_ptr<clang::FrontendAction> IndexActionFactory::create() {
-  return std::make_unique<IndexAction>(file_copier_, merge_queue_);
+  return std::make_unique<IndexAction>(file_copier_, merge_queue_,
+                                       support_incremental_indexing_);
 }
 }  // namespace indexer
 }  // namespace oss_fuzz

infra/indexer/frontend/index_action.h

@@ -16,6 +16,7 @@
 #define OSS_FUZZ_INFRA_INDEXER_FRONTEND_INDEX_ACTION_H_
 
 #include <memory>
+#include <string>
 
 #include "indexer/index/file_copier.h"
 #include "indexer/index/in_memory_index.h"
@@ -32,28 +33,32 @@ namespace indexer {
 // indexer/frontend.h should be used instead.
 class IndexAction : public clang::ASTFrontendAction {
  public:
-  explicit IndexAction(FileCopier& file_copier, MergeQueue& merge_queue);
+  explicit IndexAction(FileCopier& file_copier, MergeQueue& merge_queue,
+                       bool support_incremental_indexing = false);
 
   bool BeginSourceFileAction(clang::CompilerInstance& compiler) override;
   void EndSourceFileAction() override;
 
   std::unique_ptr<clang::ASTConsumer> CreateASTConsumer(
-      clang::CompilerInstance& compiler, llvm::StringRef) override;
+      clang::CompilerInstance& compiler, llvm::StringRef path) override;
 
  private:
   std::unique_ptr<InMemoryIndex> index_;
   MergeQueue& merge_queue_;
+  bool support_incremental_indexing_;
 };
 
 class IndexActionFactory : public clang::tooling::FrontendActionFactory {
  public:
-  explicit IndexActionFactory(FileCopier& file_copier, MergeQueue& merge_queue);
+  IndexActionFactory(FileCopier& file_copier, MergeQueue& merge_queue,
+                     bool support_incremental_indexing = false);
 
   std::unique_ptr<clang::FrontendAction> create() override;
 
  private:
   FileCopier& file_copier_;
   MergeQueue& merge_queue_;
+  const bool support_incremental_indexing_;
 };
 }  // namespace indexer
 }  // namespace oss_fuzz

infra/indexer/index/file_copier.cc

@@ -41,18 +41,20 @@ void PreparePath(std::string& path) {
 
 FileCopier::FileCopier(absl::string_view base_path,
                        absl::string_view index_path,
-                       const std::vector<std::string>& extra_paths)
+                       const std::vector<std::string>& extra_paths,
+                       bool exist_ok)
     : base_path_(base_path),
       extra_paths_(extra_paths),
-      index_path_(index_path) {
+      index_path_(index_path),
+      exist_ok_(exist_ok) {
   PreparePath(base_path_);
   for (std::string& extra_path : extra_paths_) {
     PreparePath(extra_path);
   }
 }
 
 std::string FileCopier::AbsoluteToIndexPath(absl::string_view path) const {
-  CHECK(path.starts_with("/")) << "Absolute path expected";
+  CHECK(path.starts_with("/")) << "Absolute path expected: " << path;
 
   std::string result = std::string(path);
   if (!base_path_.empty() && absl::StartsWith(path, base_path_)) {
@@ -70,12 +72,12 @@ std::string FileCopier::AbsoluteToIndexPath(absl::string_view path) const {
 }
 
 void FileCopier::RegisterIndexedFile(absl::string_view index_path) {
-  absl::MutexLock lock(&mutex_);
+  absl::MutexLock lock(mutex_);
   indexed_files_.emplace(index_path);
 }
 
 void FileCopier::CopyIndexedFiles() {
-  absl::MutexLock lock(&mutex_);
+  absl::MutexLock lock(mutex_);
 
   for (const std::string& indexed_path : indexed_files_) {
     std::filesystem::path src_path = indexed_path;
@@ -101,9 +103,13 @@ void FileCopier::CopyIndexedFiles() {
                         << dst_path.parent_path()
                         << " (error: " << error_code.message() << ")";
 
-    QCHECK(std::filesystem::copy_file(src_path, dst_path, error_code))
-        << "Failed to copy file " << src_path << " to " << dst_path
-        << " (error: " << error_code.message() << ")";
+    using std::filesystem::copy_options;
+    const copy_options options =
+        exist_ok_ ? copy_options::skip_existing : copy_options::none;
+    std::filesystem::copy_file(src_path, dst_path, options, error_code);
+    QCHECK(!error_code) << "Failed to copy file " << src_path << " to "
+                        << dst_path << " (error: " << error_code.message()
+                        << ")";
   }
 }
 }  // namespace indexer

infra/indexer/index/file_copier.h

@@ -33,7 +33,8 @@ namespace indexer {
 class FileCopier {
  public:
   FileCopier(absl::string_view base_path, absl::string_view index_path,
-             const std::vector<std::string>& extra_paths);
+             const std::vector<std::string>& extra_paths,
+             bool exist_ok = false);
   FileCopier(const FileCopier&) = delete;
 
   // Takes an absolute path. Rewrites this path into the representation it will
@@ -51,6 +52,7 @@ class FileCopier {
   std::string base_path_;
   std::vector<std::string> extra_paths_;
   const std::filesystem::path index_path_;
+  const bool exist_ok_;
 
   absl::Mutex mutex_;
   absl::flat_hash_set<std::string> indexed_files_ ABSL_GUARDED_BY(mutex_);