Skip to content

Commit 0dcb2b6

Browse files
tgengtgeng
authored
Make fineGrainedHashExternalRepos work with BzlMod (#207)
With Bzlmod, the location of external repository is no longer `<exec_root>/external/<repo_name>`. Hence, a devoted repo resolver is added and it resolves bzlmod repo location by doing a `bazel query`. The result is cached to mitigate performance impact of such a call.
1 parent 27adb8d commit 0dcb2b6

File tree

9 files changed

+180
-55
lines changed

9 files changed

+180
-55
lines changed

‎cli/src/main/kotlin/com/bazel_diff/di/Modules.kt‎

Lines changed: 29 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,7 @@ package com.bazel_diff.di
22

33
import com.bazel_diff.bazel.BazelClient
44
import com.bazel_diff.bazel.BazelQueryService
5-
import com.bazel_diff.hash.BuildGraphHasher
6-
import com.bazel_diff.hash.RuleHasher
7-
import com.bazel_diff.hash.SourceFileHasher
8-
import com.bazel_diff.hash.TargetHasher
5+
import com.bazel_diff.hash.*
96
import com.bazel_diff.io.ContentHashProvider
107
import com.bazel_diff.log.Logger
118
import com.bazel_diff.log.StderrLogger
@@ -23,46 +20,46 @@ import java.nio.file.Paths
2320

2421
@OptIn(ExperimentalCoroutinesApi::class)
2522
fun hasherModule(
26-
workingDirectory: Path,
27-
bazelPath: Path,
28-
contentHashPath: File?,
29-
startupOptions: List<String>,
30-
commandOptions: List<String>,
31-
cqueryOptions: List<String>,
32-
useCquery: Boolean,
33-
keepGoing: Boolean,
34-
fineGrainedHashExternalRepos: Set<String>,
23+
workingDirectory: Path,
24+
bazelPath: Path,
25+
contentHashPath: File?,
26+
startupOptions: List<String>,
27+
commandOptions: List<String>,
28+
cqueryOptions: List<String>,
29+
useCquery: Boolean,
30+
keepGoing: Boolean,
31+
fineGrainedHashExternalRepos: Set<String>,
3532
): Module = module {
33+
val result = runBlocking {
34+
process(
35+
bazelPath.toString(), "info", "output_base",
36+
stdout = Redirect.CAPTURE,
37+
workingDirectory = workingDirectory.toFile(),
38+
stderr = Redirect.PRINT,
39+
destroyForcibly = true,
40+
)
41+
}
42+
val outputPath = Paths.get(result.output.single())
3643
val debug = System.getProperty("DEBUG", "false").equals("true")
3744
single {
3845
BazelQueryService(
39-
workingDirectory,
40-
bazelPath,
41-
startupOptions,
42-
commandOptions,
43-
cqueryOptions,
44-
keepGoing,
45-
debug
46+
workingDirectory,
47+
bazelPath,
48+
startupOptions,
49+
commandOptions,
50+
cqueryOptions,
51+
keepGoing,
52+
debug
4653
)
4754
}
4855
single { BazelClient(useCquery, fineGrainedHashExternalRepos) }
4956
single { BuildGraphHasher(get()) }
5057
single { TargetHasher() }
5158
single { RuleHasher(useCquery, fineGrainedHashExternalRepos) }
5259
single { SourceFileHasher(fineGrainedHashExternalRepos) }
60+
single { ExternalRepoResolver(workingDirectory, bazelPath, outputPath) }
5361
single(named("working-directory")) { workingDirectory }
54-
single(named("output-base")) {
55-
val result = runBlocking {
56-
process(
57-
bazelPath.toString(), "info", "output_base",
58-
stdout = Redirect.CAPTURE,
59-
workingDirectory = workingDirectory.toFile(),
60-
stderr = Redirect.PRINT,
61-
destroyForcibly = true,
62-
)
63-
}
64-
Paths.get(result.output.single())
65-
}
62+
single(named("output-base")) { outputPath }
6663
single { ContentHashProvider(contentHashPath) }
6764
}
6865

‎cli/src/main/kotlin/com/bazel_diff/hash/ExternalRepoResolver.kt‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package com.bazel_diff.hash
2+
3+
import com.google.common.cache.CacheBuilder
4+
import com.google.common.cache.CacheLoader
5+
import org.koin.core.component.KoinComponent
6+
import java.nio.file.Files
7+
import java.nio.file.Path
8+
import java.nio.file.Paths
9+
10+
class ExternalRepoResolver(
11+
private val workingDirectory: Path,
12+
private val bazelPath: Path,
13+
private val outputBase: Path,
14+
) : KoinComponent {
15+
private val externalRoot: Path by lazy {
16+
outputBase.resolve("external")
17+
}
18+
19+
private val cache = CacheBuilder.newBuilder().build(CacheLoader.from { repoName: String ->
20+
val externalRepoRoot = externalRoot.resolve(repoName)
21+
if (Files.exists(externalRepoRoot)) {
22+
return@from externalRepoRoot
23+
}
24+
resolveBzlModPath(repoName)
25+
})
26+
27+
fun resolveExternalRepoRoot(repoName: String): Path {
28+
return cache.get(repoName)
29+
}
30+
31+
private fun resolveBzlModPath(repoName: String): Path {
32+
// Query result line should look something like "<exec root>/external/<canonical repo name>/some/bazel/target: <kind> <label>"
33+
val queryResultLine = runProcessAndCaptureFirstLine(bazelPath.toString(), "query", "@$repoName//...", "--output", "location")
34+
val path = Paths.get(queryResultLine.split(": ", limit = 2)[0])
35+
val bzlModRelativePath = path.relativize(externalRoot).first()
36+
return externalRoot.resolve(bzlModRelativePath)
37+
}
38+
39+
private fun runProcessAndCaptureFirstLine(vararg command: String): String {
40+
val process = ProcessBuilder(*command).directory(workingDirectory.toFile()).start()
41+
process.inputStream.bufferedReader().use {
42+
// read the first line and close the stream so that Bazel doesn't need to continue
43+
// output all the query result.
44+
return it.readLine()
45+
}
46+
}
47+
}

‎cli/src/main/kotlin/com/bazel_diff/hash/SourceFileHasher.kt‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ class SourceFileHasher : KoinComponent {
1313
private val workingDirectory: Path
1414
private val logger: Logger
1515
private val relativeFilenameToContentHash: Map<String, String>?
16-
private val outputBase: Path
1716
private val fineGrainedHashExternalRepos: Set<String>
17+
private val externalRepoResolver: ExternalRepoResolver
1818

1919
init {
2020
val logger: Logger by inject()
@@ -26,16 +26,16 @@ class SourceFileHasher : KoinComponent {
2626
this.workingDirectory = workingDirectory
2727
val contentHashProvider: ContentHashProvider by inject()
2828
relativeFilenameToContentHash = contentHashProvider.filenameToHash
29-
val outputBase: Path by inject(qualifier = named("output-base"))
30-
this.outputBase = outputBase
3129
this.fineGrainedHashExternalRepos = fineGrainedHashExternalRepos
30+
val externalRepoResolver: ExternalRepoResolver by inject()
31+
this.externalRepoResolver = externalRepoResolver
3232
}
3333

34-
constructor(workingDirectory: Path, outputBase: Path, relativeFilenameToContentHash: Map<String, String>?, fineGrainedHashExternalRepos: Set<String> = emptySet()) {
34+
constructor(workingDirectory: Path, relativeFilenameToContentHash: Map<String, String>?, externalRepoResolver: ExternalRepoResolver, fineGrainedHashExternalRepos: Set<String> = emptySet()) {
3535
this.workingDirectory = workingDirectory
36-
this.outputBase = outputBase
3736
this.relativeFilenameToContentHash = relativeFilenameToContentHash
3837
this.fineGrainedHashExternalRepos = fineGrainedHashExternalRepos
38+
this.externalRepoResolver = externalRepoResolver
3939
}
4040

4141
fun digest(sourceFileTarget: BazelSourceFileTarget): ByteArray {
@@ -55,7 +55,8 @@ class SourceFileHasher : KoinComponent {
5555
return@sha256
5656
}
5757
val relativePath = Paths.get(parts[1].removePrefix(":").replace(':', '/'))
58-
outputBase.resolve("external/$repoName").resolve(relativePath)
58+
val externalRepoRoot = externalRepoResolver.resolveExternalRepoRoot(repoName)
59+
externalRepoRoot.resolve(relativePath)
5960
} else {
6061
return@sha256
6162
}

‎cli/src/test/kotlin/com/bazel_diff/Modules.kt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
package com.bazel_diff
22

33
import com.bazel_diff.bazel.BazelClient
4-
import com.bazel_diff.hash.BuildGraphHasher
5-
import com.bazel_diff.hash.RuleHasher
6-
import com.bazel_diff.hash.SourceFileHasher
7-
import com.bazel_diff.hash.TargetHasher
4+
import com.bazel_diff.hash.*
85
import com.bazel_diff.io.ContentHashProvider
96
import com.bazel_diff.log.Logger
107
import com.google.gson.GsonBuilder
@@ -14,15 +11,18 @@ import org.koin.dsl.module
1411
import java.nio.file.Paths
1512

1613
fun testModule(): Module = module {
14+
val outputBase = Paths.get("output-base")
15+
val workingDirectory = Paths.get("working-directory")
1716
single<Logger> { SilentLogger }
1817
single { BazelClient(false, emptySet()) }
1918
single { BuildGraphHasher(get()) }
2019
single { TargetHasher() }
2120
single { RuleHasher(false, emptySet()) }
21+
single { ExternalRepoResolver(workingDirectory, Paths.get("bazel"), outputBase) }
2222
single { SourceFileHasher() }
2323
single { GsonBuilder().disableHtmlEscaping().setPrettyPrinting().create() }
24-
single(named("working-directory")) { Paths.get("working-directory") }
25-
single(named("output-base")) { Paths.get("output-base") }
24+
single(named("working-directory")) { workingDirectory }
25+
single(named("output-base")) { outputBase }
2626
single { ContentHashProvider(null) }
2727
}
2828

‎cli/src/test/kotlin/com/bazel_diff/e2e/E2ETest.kt‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,62 @@ class E2ETest {
123123
assertThat(actual).isEqualTo(expected)
124124
}
125125

126+
@Test
127+
fun testFineGrainedHashBzlMod() {
128+
// The difference between these two snapshots is simply upgrading the Guava version.
129+
// Following is the diff. (The diff on maven_install.json is omitted)
130+
//
131+
// diff --git a/MODULE.bazel b/MODULE.bazel
132+
// index 9a58823..3ffded3 100644
133+
// --- a/MODULE.bazel
134+
// +++ b/MODULE.bazel
135+
// @@ -4,7 +4,7 @@ maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")
136+
// maven.install(
137+
// artifacts = [
138+
// "junit:junit:4.12",
139+
// - "com.google.guava:guava:31.1-jre",
140+
// + "com.google.guava:guava:32.0.0-jre",
141+
// ],
142+
// lock_file = "//:maven_install.json",
143+
// repositories = [
144+
//
145+
// The project contains a single target that depends on Guava:
146+
// //src/main/java/com/integration:guava-user
147+
//
148+
// So this target, its derived targets, and all other changed external targets should be
149+
// the only impacted targets.
150+
val projectA = extractFixtureProject("/fixture/fine-grained-hash-bzlmod-test-1.zip")
151+
val projectB = extractFixtureProject("/fixture/fine-grained-hash-bzlmod-test-2.zip")
152+
153+
val workingDirectoryA = projectA
154+
val workingDirectoryB = projectB
155+
val bazelPath = "bazel"
156+
val outputDir = temp.newFolder()
157+
val from = File(outputDir, "starting_hashes.json")
158+
val to = File(outputDir, "final_hashes.json")
159+
val impactedTargetsOutput = File(outputDir, "impacted_targets.txt")
160+
161+
val cli = CommandLine(BazelDiff())
162+
//From
163+
cli.execute(
164+
"generate-hashes", "-w", workingDirectoryA.absolutePath, "-b", bazelPath, "--fineGrainedHashExternalRepos", "bazel_diff_maven", from.absolutePath
165+
)
166+
//To
167+
cli.execute(
168+
"generate-hashes", "-w", workingDirectoryB.absolutePath, "-b", bazelPath, "--fineGrainedHashExternalRepos", "bazel_diff_maven", to.absolutePath
169+
)
170+
//Impacted targets
171+
cli.execute(
172+
"get-impacted-targets", "-sh", from.absolutePath, "-fh", to.absolutePath, "-o", impactedTargetsOutput.absolutePath
173+
)
174+
175+
val actual: Set<String> = impactedTargetsOutput.readLines().filter { it.isNotBlank() }.toSet()
176+
val expected: Set<String> =
177+
javaClass.getResourceAsStream("/fixture/fine-grained-hash-bzlmod-test-impacted-targets.txt").use { it.bufferedReader().readLines().filter { it.isNotBlank() }.toSet() }
178+
179+
assertThat(actual).isEqualTo(expected)
180+
}
181+
126182
// TODO: re-enable the test after https://github.com/bazelbuild/bazel/issues/21010 is fixed
127183
@Ignore("cquery mode is broken with Bazel 7 because --transition=lite is crashes due to https://github.com/bazelbuild/bazel/issues/21010")
128184
@Test

‎cli/src/test/kotlin/com/bazel_diff/hash/SourceFileHasherTest.kt‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,13 @@ import java.nio.file.Files
1515
import java.nio.file.Paths
1616

1717

18-
internal class SourceFileHasherTest: KoinTest {
18+
internal class SourceFileHasherTest : KoinTest {
1919
private val repoAbsolutePath = Paths.get("").toAbsolutePath()
2020
private val outputBasePath = Files.createTempDirectory("SourceFileHasherTest")
2121
private val fixtureFileTarget = "//cli/src/test/kotlin/com/bazel_diff/hash/fixture:foo.ts"
2222
private val fixtureFileContent: ByteArray
2323
private val seed = "seed".toByteArray()
24+
private val externalRepoResolver = ExternalRepoResolver(repoAbsolutePath, Paths.get("bazel"), outputBasePath)
2425

2526
init {
2627
val path = Paths.get("cli/src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts")
@@ -35,7 +36,7 @@ internal class SourceFileHasherTest: KoinTest {
3536

3637
@Test
3738
fun testHashConcreteFile() = runBlocking {
38-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
39+
val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
3940
val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
4041
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
4142
val expected = sha256 {
@@ -48,7 +49,7 @@ internal class SourceFileHasherTest: KoinTest {
4849

4950
@Test
5051
fun testHashConcreteFileInExternalRepo() = runBlocking {
51-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null, setOf("external_repo"))
52+
val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver, setOf("external_repo"))
5253
val externalRepoFilePath = outputBasePath.resolve("external/external_repo/path/to/my_file.txt")
5354
Files.createDirectories(externalRepoFilePath.parent)
5455
val externalRepoFileTarget = "@external_repo//path/to:my_file.txt"
@@ -66,7 +67,7 @@ internal class SourceFileHasherTest: KoinTest {
6667

6768
@Test
6869
fun testSoftHashConcreteFile() = runBlocking {
69-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
70+
val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
7071
val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
7172
val actual = hasher.softDigest(bazelSourceFileTarget)?.toHexString()
7273
val expected = sha256 {
@@ -79,7 +80,7 @@ internal class SourceFileHasherTest: KoinTest {
7980

8081
@Test
8182
fun testSoftHashNonExistedFile() = runBlocking {
82-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
83+
val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
8384
val bazelSourceFileTarget = BazelSourceFileTarget("//i/do/not/exist", seed)
8485
val actual = hasher.softDigest(bazelSourceFileTarget)
8586
assertThat(actual).isNull()
@@ -88,7 +89,7 @@ internal class SourceFileHasherTest: KoinTest {
8889
@Test
8990
fun testSoftHashExternalTarget() = runBlocking {
9091
val target = "@bazel-diff//some:file"
91-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
92+
val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
9293
val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
9394
val actual = hasher.softDigest(bazelSourceFileTarget)
9495
assertThat(actual).isNull()
@@ -97,7 +98,7 @@ internal class SourceFileHasherTest: KoinTest {
9798
@Test
9899
fun testHashNonExistedFile() = runBlocking {
99100
val target = "//i/do/not/exist"
100-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
101+
val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
101102
val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
102103
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
103104
val expected = sha256 {
@@ -110,7 +111,7 @@ internal class SourceFileHasherTest: KoinTest {
110111
@Test
111112
fun testHashExternalTarget() = runBlocking {
112113
val target = "@bazel-diff//some:file"
113-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, null)
114+
val hasher = SourceFileHasher(repoAbsolutePath, null, externalRepoResolver)
114115
val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
115116
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
116117
val expected = sha256 {}.toHexString()
@@ -120,7 +121,7 @@ internal class SourceFileHasherTest: KoinTest {
120121
@Test
121122
fun testHashWithProvidedContentHash() = runBlocking {
122123
val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts" to "foo-content-hash")
123-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, filenameToContentHash)
124+
val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash, externalRepoResolver)
124125
val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
125126
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
126127
val expected = sha256 {
@@ -134,7 +135,7 @@ internal class SourceFileHasherTest: KoinTest {
134135
@Test
135136
fun testHashWithProvidedContentHashButNotInKey() = runBlocking {
136137
val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts" to "foo-content-hash")
137-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, filenameToContentHash)
138+
val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash, externalRepoResolver)
138139
val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
139140
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
140141
val expected = sha256 {
@@ -149,7 +150,7 @@ internal class SourceFileHasherTest: KoinTest {
149150
fun testHashWithProvidedContentHashWithLeadingColon() = runBlocking {
150151
val targetName = "//:cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts"
151152
val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts" to "foo-content-hash")
152-
val hasher = SourceFileHasher(repoAbsolutePath, outputBasePath, filenameToContentHash)
153+
val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash, externalRepoResolver)
153154
val bazelSourceFileTarget = BazelSourceFileTarget(targetName, seed)
154155
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
155156
val expected = sha256 {

‎cli/src/test/resources/fixture/fine-grained-hash-bzlmod-test-1.zip‎

7.98 KB
Binary file not shown.

‎cli/src/test/resources/fixture/fine-grained-hash-bzlmod-test-2.zip‎

8.02 KB
Binary file not shown.

0 commit comments

Comments
 (0)