C#: Add functionality to detect overlay mode and integrate in extraction context.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -41,16 +41,20 @@ public class Analyser : IDisposable
 
         public IPathCache PathCache { get; }
 
+        public IOverlayInfo OverlayInfo { get; }
+
         protected Analyser(
             IProgressMonitor pm,
             ILogger logger,
             PathTransformer pathTransformer,
             IPathCache pathCache,
+            IOverlayInfo overlayInfo,
             bool addAssemblyTrapPrefix)
         {
             Logger = logger;
             PathTransformer = pathTransformer;
             PathCache = pathCache;
+            OverlayInfo = overlayInfo;
             this.addAssemblyTrapPrefix = addAssemblyTrapPrefix;
             this.progressMonitor = pm;
 
@@ -158,7 +162,7 @@ private void DoAnalyseReferenceAssembly(PortableExecutableReference r)
 
                     if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
                     {
-                        var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
+                        var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), OverlayInfo, addAssemblyTrapPrefix);
 
                         foreach (var module in assembly.Modules)
                         {
@@ -195,7 +199,7 @@ private void DoExtractTree(SyntaxTree tree)
                 var currentTaskId = IncrementTaskCount();
                 ReportProgressTaskStarted(currentTaskId, sourcePath);
 
-                var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
+                var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), OverlayInfo, addAssemblyTrapPrefix);
                 // Ensure that the file itself is populated in case the source file is totally empty
                 var root = tree.GetRoot();
                 Entities.File.Create(cx, root.SyntaxTree.FilePath);
@@ -234,7 +238,7 @@ private void DoAnalyseCompilation()
                 var assembly = compilation.Assembly;
                 var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
                 compilationTrapFile = trapWriter;  // Dispose later
-                var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
+                var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), OverlayInfo, addAssemblyTrapPrefix);
 
                 compilationEntity = Entities.Compilation.Create(cx);
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/BinaryLogAnalyser.cs

@@ -8,7 +8,7 @@ namespace Semmle.Extraction.CSharp
     public class BinaryLogAnalyser : Analyser
     {
         public BinaryLogAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
-            : base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
+            : base(pm, logger, pathTransformer, pathCache, new TrivialOverlayInfo(), addAssemblyTrapPrefix)
         {
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs

@@ -29,6 +29,12 @@ public class Context
         /// </summary>
         public bool ShouldAddAssemblyTrapPrefix { get; }
 
+        /// <summary>
+        /// Holds if trap only should be created for types and member signatures (and not for expressions and statements).
+        /// This is the case for all unchanged files, when running in overlay mode.
+        /// </summary>
+        public bool OnlyScaffold { get; }
+
         public IList<object> TrapStackSuffix { get; } = new List<object>();
 
         private int GetNewId() => TrapWriter.IdCounter++;
@@ -523,13 +529,14 @@ internal void CacheLambdaParameterSymbol(IParameterSymbol param, SyntaxNode synt
 
         internal CommentProcessor CommentGenerator { get; } = new CommentProcessor();
 
-        public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, bool shouldAddAssemblyTrapPrefix = false)
+        public Context(ExtractionContext extractionContext, Compilation c, TrapWriter trapWriter, IExtractionScope scope, IOverlayInfo overlayInfo, bool shouldAddAssemblyTrapPrefix = false)
         {
             ExtractionContext = extractionContext;
             TrapWriter = trapWriter;
             ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix;
             Compilation = c;
             this.scope = scope;
+            OnlyScaffold = overlayInfo.IsOverlayMode && scope is SourceScope ss && overlayInfo.OnlyMakeScaffold(ss.SourceTree.FilePath);
         }
 
         public bool FromSource => scope is SourceScope;

csharp/extractor/Semmle.Extraction.CSharp/Extractor/OverlayInfo.cs

@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp
+{
+    public interface IOverlayInfo
+    {
+        /// <summary>
+        /// True, if the extractor is running in overlay mode.
+        /// </summary>
+        bool IsOverlayMode { get; }
+
+        /// <summary>
+        /// Returns true, if the given file is not in the set of changed files.
+        /// </summary>
+        /// <param name="filePath">A source file path</param>
+        bool OnlyMakeScaffold(string filePath);
+    }
+
+
+    /// <summary>
+    /// An instance of this class is used when overlay is not enabled.
+    /// </summary>
+    public class TrivialOverlayInfo : IOverlayInfo
+    {
+        public TrivialOverlayInfo() { }
+
+        public bool IsOverlayMode { get; } = false;
+
+        public bool OnlyMakeScaffold(string filePath) => false;
+    }
+
+    /// <summary>
+    /// An instance of this class is used for detecting
+    /// (1) Whether overlay is enabled.
+    /// (2) Fetch the changed files that should be fully extracted as a part
+    /// of the overlay extraction.
+    /// </summary>
+    public class OverlayInfo : IOverlayInfo
+    {
+        private readonly ILogger logger;
+        private readonly HashSet<string> changedFiles;
+
+        public OverlayInfo(ILogger logger, string json)
+        {
+            this.logger = logger;
+            changedFiles = ParseJson(json);
+        }
+
+        public bool IsOverlayMode { get; } = true;
+
+        public bool OnlyMakeScaffold(string filePath) => !changedFiles.Contains(filePath);
+
+        /// <summary>
+        /// Private type only used to parse overlay changes JSON files.
+        /// 
+        /// The content of such a file has the format
+        /// {
+        ///  "changes": [
+        ///    "app/controllers/about_controller.xyz",
+        ///    "app/models/about.xyz"
+        ///  ]
+        /// }
+        /// </summary>
+        public record ChangedFiles
+        {
+            public string[]? Changes { get; set; }
+        }
+
+        private HashSet<string> ParseJson(string json)
+        {
+            try
+            {
+                var options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
+                var changedFiles = JsonSerializer.Deserialize<ChangedFiles>(json, options);
+                return changedFiles?.Changes is string[] changes
+                    ? new HashSet<string>(changes)
+                    : [];
+            }
+            catch (JsonException)
+            {
+                logger.LogError("Overlay: Unable to parse the JSON content from the overlay changes file");
+                return [];
+            }
+        }
+    }
+
+    public static class OverlayInfoFactory
+    {
+        /// <summary>
+        /// The returned object is used to decide, whether
+        /// (1) The extractor is running in overlay mode.
+        /// (2) Which files to only extract scaffolds for (unchanged files)
+        /// </summary>
+        /// <param name="mode">The extraction mode</param>
+        /// <param name="logger">A logger</param>
+        /// <returns>An overlay information object.</returns>
+        public static IOverlayInfo Make(ILogger logger)
+        {
+            if (EnvironmentVariables.GetOverlayChangesFilePath() is string path)
+            {
+                try
+                {
+                    var json = File.ReadAllText(path);
+                    return new OverlayInfo(logger, json);
+                }
+                catch
+                {
+                    logger.LogError("Overlay: Unexpected error while reading the overlay changes file.");
+                }
+
+            }
+
+            return new TrivialOverlayInfo();
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

@@ -9,7 +9,7 @@ namespace Semmle.Extraction.CSharp
     public class StandaloneAnalyser : Analyser
     {
         public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
-            : base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
+            : base(pm, logger, pathTransformer, pathCache, OverlayInfoFactory.Make(logger), addAssemblyTrapPrefix)
         {
         }
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

@@ -14,7 +14,7 @@ public class TracingAnalyser : Analyser
         private bool init;
 
         public TracingAnalyser(IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix)
-            : base(pm, logger, pathTransformer, pathCache, addAssemblyTrapPrefix)
+            : base(pm, logger, pathTransformer, pathCache, new TrivialOverlayInfo(), addAssemblyTrapPrefix)
         {
         }
 

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -53,5 +53,19 @@ public static IEnumerable<string> GetURLs(string name)
         {
             return Environment.GetEnvironmentVariable(name)?.Split(" ", StringSplitOptions.RemoveEmptyEntries) ?? [];
         }
+
+        /// <summary>
+        /// Used to
+        /// (1) Detect whether the extractor should run in overlay mode.
+        /// (2) Returns the path to the file containing a list of changed files
+        /// in JSON format.
+        ///
+        /// The environment variable is only set in case the extraction is supposed to be
+        /// performed in overlay mode. Furthermore, this only applies to buildless extraction.
+        /// </summary>
+        public static string? GetOverlayChangesFilePath()
+        {
+            return Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OVERLAY_CHANGES");
+        }
     }
 }