WIP: Reenable modified filepaths entry

This should greatly speed up hash calculation time

Author: tinder-maxwellelliott

cli/BUILD

@@ -1,7 +1,5 @@
-load("@rules_java//java:defs.bzl", "java_binary", "java_library", "java_proto_library")
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_binary", "kt_jvm_library", "kt_jvm_test")
-load("@rules_jvm_external//:defs.bzl", "artifact")
+load("@rules_java//java:defs.bzl", "java_binary")
+load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library", "kt_jvm_test")
 
 config_setting(
     name = "enable_debug",
@@ -10,6 +8,13 @@ config_setting(
     },
 )
 
+config_setting(
+    name = "macos",
+    constraint_values = [
+        "@platforms//os:macos",
+    ],
+)
+
 java_binary(
     name = "bazel-diff",
     jvm_flags = select({
@@ -39,6 +44,10 @@ kt_jvm_library(
 
 kt_jvm_test(
     name = "BuildGraphHasherTest",
+    jvm_flags = select({
+        ":macos": ["-Djava.security.manager=allow"],
+        "//conditions:default": [],
+    }),
     test_class = "com.bazel_diff.hash.BuildGraphHasherTest",
     runtime_deps = [":cli-test-lib"],
 )
@@ -48,42 +57,70 @@ kt_jvm_test(
     data = [
         ":src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts",
     ],
+    jvm_flags = select({
+        ":macos": ["-Djava.security.manager=allow"],
+        "//conditions:default": [],
+    }),
     test_class = "com.bazel_diff.hash.SourceFileHasherTest",
     runtime_deps = [":cli-test-lib"],
 )
 
 kt_jvm_test(
     name = "CalculateImpactedTargetsInteractorTest",
+    jvm_flags = select({
+        ":macos": ["-Djava.security.manager=allow"],
+        "//conditions:default": [],
+    }),
     test_class = "com.bazel_diff.interactor.CalculateImpactedTargetsInteractorTest",
     runtime_deps = [":cli-test-lib"],
 )
 
 kt_jvm_test(
     name = "NormalisingPathConverterTest",
+    jvm_flags = select({
+        ":macos": ["-Djava.security.manager=allow"],
+        "//conditions:default": [],
+    }),
     test_class = "com.bazel_diff.cli.converter.NormalisingPathConverterTest",
     runtime_deps = [":cli-test-lib"],
 )
 
 kt_jvm_test(
     name = "OptionsConverterTest",
+    jvm_flags = select({
+        ":macos": ["-Djava.security.manager=allow"],
+        "//conditions:default": [],
+    }),
     test_class = "com.bazel_diff.cli.converter.OptionsConverterTest",
     runtime_deps = [":cli-test-lib"],
 )
 
 kt_jvm_test(
     name = "DeserialiseHashesInteractorTest",
+    jvm_flags = select({
+        ":macos": ["-Djava.security.manager=allow"],
+        "//conditions:default": [],
+    }),
     test_class = "com.bazel_diff.interactor.DeserialiseHashesInteractorTest",
     runtime_deps = [":cli-test-lib"],
 )
 
 kt_jvm_test(
     name = "BazelRuleTest",
+    jvm_flags = select({
+        ":macos": ["-Djava.security.manager=allow"],
+        "//conditions:default": [],
+    }),
     test_class = "com.bazel_diff.bazel.BazelRuleTest",
     runtime_deps = [":cli-test-lib"],
 )
 
 kt_jvm_test(
     name = "E2ETest",
+    jvm_flags = select({
+        ":macos": ["-Djava.security.manager=allow"],
+        "//conditions:default": [],
+    }),
     test_class = "com.bazel_diff.e2e.E2ETest",
     runtime_deps = [":cli-test-lib"],
 )
@@ -94,6 +131,10 @@ kt_jvm_test(
         ":src/test/kotlin/com/bazel_diff/io/fixture/correct.json",
         ":src/test/kotlin/com/bazel_diff/io/fixture/wrong.json",
     ],
+    jvm_flags = select({
+        ":macos": ["-Djava.security.manager=allow"],
+        "//conditions:default": [],
+    }),
     test_class = "com.bazel_diff.io.ContentHashProviderTest",
     runtime_deps = [
         ":cli-test-lib",

cli/src/main/kotlin/com/bazel_diff/bazel/BazelClient.kt

@@ -2,9 +2,10 @@ package com.bazel_diff.bazel
 
 import com.bazel_diff.log.Logger
 import com.google.devtools.build.lib.query2.proto.proto2api.Build
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
+import java.nio.file.Path
 import java.util.*
+import org.koin.core.component.inject
+import org.koin.core.component.KoinComponent
 
 class BazelClient(private val useCquery: Boolean, private val fineGrainedHashExternalRepos: Set<String>) : KoinComponent {
     private val logger: Logger by inject()
@@ -63,5 +64,18 @@ class BazelClient(private val useCquery: Boolean, private val fineGrainedHashExt
 
         return targets
     }
+
+    suspend fun queryModifiedSourcefileTargets(
+        modifiedFilepaths: Set<Path>
+    ): List<Build.Target> {
+        val queryEpoch = Calendar.getInstance().getTimeInMillis()
+        val allReposToQuery = fineGrainedHashExternalRepos.map { "@$it" }
+        val fineGrainedHashExternalReposSourceTargets = queryService.query("kind('source file', ${allReposToQuery.joinToString(" + ") { "'$it//...:all-targets'" }})")
+        val modifiedSourceFileTargets = queryService.query("kind('source file', ${modifiedFilepaths.joinToString(" + ") { "'$it'" }})")
+        val queryDuration = Calendar.getInstance().getTimeInMillis() - queryEpoch
+        logger.i { "Modified source files queried in $queryDuration" }
+
+        return fineGrainedHashExternalReposSourceTargets + modifiedSourceFileTargets
+    }
 }
 

cli/src/main/kotlin/com/bazel_diff/cli/GenerateHashesCommand.kt

@@ -118,6 +118,12 @@ class GenerateHashesCommand : Callable<Int> {
     )
     var ignoredRuleHashingAttributes: Set<String> = emptySet()
 
+     @CommandLine.Option(
+        names = ["-m", "--modified-filepaths"],
+        description = ["Experimental: A text file containing a newline separated list of filepaths, these filepaths should represent the modified files between the specified revisions and will be used to scope what files are hashed during hash generation."]
+    )
+    var modifiedFilepaths: File? = null
+
     @CommandLine.Spec
     lateinit var spec: CommandLine.Model.CommandSpec
 
@@ -142,7 +148,12 @@ class GenerateHashesCommand : Callable<Int> {
             )
         }
 
-        return when (GenerateHashesInteractor().execute(seedFilepaths, outputPath, ignoredRuleHashingAttributes)) {
+        return when (GenerateHashesInteractor().execute(
+            seedFilepaths,
+            outputPath,
+            ignoredRuleHashingAttributes,
+            modifiedFilepaths
+        )) {
             true -> CommandLine.ExitCode.OK
             false -> CommandLine.ExitCode.SOFTWARE
         }.also { stopKoin() }

cli/src/main/kotlin/com/bazel_diff/cli/VersionProvider.kt

@@ -4,6 +4,6 @@ import picocli.CommandLine.IVersionProvider
 
 class VersionProvider : IVersionProvider {
     override fun getVersion(): Array<String> {
-        return arrayOf("4.8.2")
+        return arrayOf("5.0.0")
     }
 }

cli/src/main/kotlin/com/bazel_diff/hash/BuildGraphHasher.kt

@@ -8,18 +8,19 @@ import com.bazel_diff.extensions.toHexString
 import com.bazel_diff.log.Logger
 import com.google.common.collect.Sets
 import com.google.devtools.build.lib.query2.proto.proto2api.Build
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.async
-import kotlinx.coroutines.runBlocking
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
+import java.io.File
 import java.nio.file.Path
+import java.util.Calendar
+import java.util.concurrent.atomic.AtomicReference
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.ConcurrentMap
-import java.util.concurrent.atomic.AtomicReference
 import java.util.stream.Collectors
 import kotlin.io.path.readBytes
-import java.util.Calendar
+import kotlinx.coroutines.async
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import org.koin.core.component.inject
+import org.koin.core.component.KoinComponent
 
 class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
     private val targetHasher: TargetHasher by inject()
@@ -28,7 +29,8 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
 
     fun hashAllBazelTargetsAndSourcefiles(
         seedFilepaths: Set<Path> = emptySet(),
-        ignoredAttrs: Set<String> = emptySet()
+        ignoredAttrs: Set<String> = emptySet(),
+        modifiedFilepaths: Set<Path> = emptySet()
     ): Map<String, String> {
         /**
          * Bazel will lock parallel queries but this is still allowing us to hash source files while executing a parallel query
@@ -38,6 +40,9 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
              * Source query is usually faster than targets query, so we prioritise it first
              */
             val sourceTargetsFuture = async(Dispatchers.IO) {
+                if (!modifiedFilepaths.isEmpty()) {
+                    return@async bazelClient.queryModifiedSourcefileTargets(modifiedFilepaths)
+                }
                 bazelClient.queryAllSourcefileTargets()
             }
             val sourceTargets = sourceTargetsFuture.await()
@@ -67,7 +72,9 @@ class BuildGraphHasher(private val bazelClient: BazelClient) : KoinComponent {
         )
     }
 
-    private fun hashSourcefiles(targets: List<Build.Target>): ConcurrentMap<String, ByteArray> {
+    private fun hashSourcefiles(
+        targets: List<Build.Target>
+    ): ConcurrentMap<String, ByteArray> {
         val exception = AtomicReference<Exception?>(null)
         val result: ConcurrentMap<String, ByteArray> = targets.parallelStream()
             .map { target: Build.Target ->

cli/src/main/kotlin/com/bazel_diff/interactor/GenerateHashesInteractor.kt

@@ -18,7 +18,12 @@ class GenerateHashesInteractor : KoinComponent {
     private val logger: Logger by inject()
     private val gson: Gson by inject()
 
-    fun execute(seedFilepaths: File?, outputPath: File?, ignoredRuleHashingAttributes: Set<String>): Boolean {
+    fun execute(
+        seedFilepaths: File?,
+        outputPath: File?,
+        ignoredRuleHashingAttributes: Set<String>,
+        modifiedFilepaths: File?
+    ): Boolean {
         return try {
             val epoch = Calendar.getInstance().getTimeInMillis()
             var seedFilepathsSet: Set<Path> = when {
@@ -31,9 +36,20 @@ class GenerateHashesInteractor : KoinComponent {
                 }
                 else -> emptySet()
             }
+            var modifiedFilepathsSet: Set<Path> = when {
+                modifiedFilepaths != null -> {
+                    BufferedReader(FileReader(modifiedFilepaths)).use {
+                        it.readLines()
+                            .map { line: String -> File(line).toPath() }
+                            .toSet()
+                    }
+                }
+                else -> emptySet()
+            }
             val hashes = buildGraphHasher.hashAllBazelTargetsAndSourcefiles(
                 seedFilepathsSet,
-                ignoredRuleHashingAttributes
+                ignoredRuleHashingAttributes,
+                modifiedFilepathsSet
             )
             when (outputPath) {
                 null -> FileWriter(FileDescriptor.out)