Skip to content

Commit ed0555b

Browse files
fa93hwstinder-maxwellelliott
fa93hws
and
tinder-maxwellelliott
authored
feature: Allow user to provide their own content hash to minimise IO operation (#129)
* content hash class to parse json file, di and args * use provided content hash whenever possible * update docs * use existing deserialiser Co-authored-by: Maxwell Elliott <[email protected]>
1 parent 06c903b commit ed0555b

File tree

13 files changed

+310
-13
lines changed

13 files changed

+310
-13
lines changed

‎README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,11 @@ workspace.
9898
-co, --bazelCommandOptions=<bazelCommandOptions>
9999
Additional space separated Bazel command options used
100100
when invoking Bazel
101+
--contentHashPath=<contentHashPath>
102+
Path to content hash json file. It's a map which maps
103+
relative file path from workspace path to its
104+
content hash. Files in this map will skip content
105+
hashing and use provided value
101106
-h, --help Show this help message and exit.
102107
-k, --[no-]keep_going This flag controls if `bazel query` will be executed
103108
with the `--keep_going` flag or not. Disabling this

‎cli/BUILD‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,15 @@ kt_jvm_test(
5959
runtime_deps = [":cli-test-lib"],
6060
)
6161

62+
kt_jvm_test(
63+
name = "SourceFileHasherTest",
64+
data = [
65+
":src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts",
66+
],
67+
test_class = "com.bazel_diff.hash.SourceFileHasherTest",
68+
runtime_deps = [":cli-test-lib"],
69+
)
70+
6271
kt_jvm_test(
6372
name = "CalculateImpactedTargetsInteractorTest",
6473
test_class = "com.bazel_diff.interactor.CalculateImpactedTargetsInteractorTest",
@@ -89,6 +98,18 @@ kt_jvm_test(
8998
runtime_deps = [":cli-test-lib"],
9099
)
91100

101+
kt_jvm_test(
102+
name = "ContentHashProviderTest",
103+
data = [
104+
":src/test/kotlin/com/bazel_diff/io/fixture/correct.json",
105+
":src/test/kotlin/com/bazel_diff/io/fixture/wrong.json",
106+
],
107+
test_class = "com.bazel_diff.io.ContentHashProviderTest",
108+
runtime_deps = [
109+
":cli-test-lib",
110+
],
111+
)
112+
92113
kt_jvm_library(
93114
name = "cli-test-lib",
94115
testonly = True,

‎cli/src/main/kotlin/com/bazel_diff/cli/GenerateHashesCommand.kt‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ class GenerateHashesCommand : Callable<Int> {
4040
)
4141
lateinit var bazelPath: Path
4242

43+
@CommandLine.Option(
44+
names = ["--contentHashPath"],
45+
description = ["Path to content hash json file. It's a map which maps relative file path from workspace path to its content hash. Files in this map will skip content hashing and use provided value"],
46+
scope = CommandLine.ScopeType.INHERIT,
47+
required = false
48+
)
49+
var contentHashPath: File? = null
50+
4351
@CommandLine.Option(
4452
names = ["-so", "--bazelStartupOptions"],
4553
description = ["Additional space separated Bazel client startup options used when invoking Bazel"],
@@ -81,12 +89,14 @@ class GenerateHashesCommand : Callable<Int> {
8189

8290
override fun call(): Int {
8391
val output = validateOutput(outputPath)
92+
validate(contentHashPath=contentHashPath)
8493

8594
startKoin {
8695
modules(
8796
hasherModule(
8897
workspacePath,
8998
bazelPath,
99+
contentHashPath,
90100
bazelStartupOptions,
91101
bazelCommandOptions,
92102
keepGoing,
@@ -108,4 +118,15 @@ class GenerateHashesCommand : Callable<Int> {
108118
"No output path specified."
109119
)
110120
}
121+
122+
private fun validate(contentHashPath: File?) {
123+
contentHashPath?.let {
124+
if (!it.canRead()) {
125+
throw CommandLine.ParameterException(
126+
spec.commandLine(),
127+
"Incorrect contentHashFilePath: file doesn't exist or can't be read."
128+
)
129+
}
130+
}
131+
}
111132
}

‎cli/src/main/kotlin/com/bazel_diff/di/Modules.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,17 +6,20 @@ import com.bazel_diff.hash.BuildGraphHasher
66
import com.bazel_diff.hash.RuleHasher
77
import com.bazel_diff.hash.SourceFileHasher
88
import com.bazel_diff.hash.TargetHasher
9+
import com.bazel_diff.io.ContentHashProvider
910
import com.bazel_diff.log.Logger
1011
import com.bazel_diff.log.StdoutLogger
1112
import com.google.gson.GsonBuilder
1213
import org.koin.core.module.Module
1314
import org.koin.core.qualifier.named
1415
import org.koin.dsl.module
16+
import java.io.File
1517
import java.nio.file.Path
1618

1719
fun hasherModule(
1820
workingDirectory: Path,
1921
bazelPath: Path,
22+
contentHashPath: File?,
2023
startupOptions: List<String>,
2124
commandOptions: List<String>,
2225
keepGoing: Boolean?,
@@ -38,6 +41,7 @@ fun hasherModule(
3841
single { RuleHasher() }
3942
single { SourceFileHasher() }
4043
single(named("working-directory")) { workingDirectory }
44+
single { ContentHashProvider(contentHashPath) }
4145
}
4246

4347
fun loggingModule(verbose: Boolean) = module {

‎cli/src/main/kotlin/com/bazel_diff/hash/SourceFileHasher.kt‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.bazel_diff.hash
22

33
import com.bazel_diff.bazel.BazelSourceFileTarget
4+
import com.bazel_diff.io.ContentHashProvider
45
import com.bazel_diff.log.Logger
56
import org.koin.core.component.KoinComponent
67
import org.koin.core.component.inject
@@ -9,21 +10,46 @@ import java.nio.file.Path
910
import java.nio.file.Paths
1011

1112
class SourceFileHasher : KoinComponent {
12-
private val workingDirectory: Path by inject(qualifier = named("working-directory"))
13-
private val logger: Logger by inject()
13+
private val workingDirectory: Path
14+
private val logger: Logger
15+
private val relativeFilenameToContentHash: Map<String, String>?
16+
init {
17+
val logger: Logger by inject()
18+
this.logger = logger
19+
}
20+
21+
constructor() {
22+
val workingDirectory: Path by inject(qualifier = named("working-directory"))
23+
this.workingDirectory = workingDirectory
24+
val contentHashProvider: ContentHashProvider by inject()
25+
relativeFilenameToContentHash = contentHashProvider.filenameToHash
26+
}
27+
28+
constructor(workingDirectory: Path, relativeFilenameToContentHash: Map<String, String>?) {
29+
this.workingDirectory = workingDirectory
30+
this.relativeFilenameToContentHash = relativeFilenameToContentHash
31+
}
1432

1533
fun digest(sourceFileTarget: BazelSourceFileTarget): ByteArray {
1634
return sha256 {
1735
val name = sourceFileTarget.name
1836
if (name.startsWith("//")) {
1937
val filenameSubstring = name.substring(2)
20-
val filenamePath = filenameSubstring.replaceFirst(":".toRegex(), "/")
21-
val absoluteFilePath = Paths.get(workingDirectory.toString(), filenamePath)
22-
val file = absoluteFilePath.toFile()
23-
if (file.exists() && file.isFile) {
24-
putFile(file)
38+
val filenamePath = filenameSubstring.replaceFirst(
39+
":".toRegex(),
40+
if (filenameSubstring.startsWith(":")) "" else "/"
41+
)
42+
if (relativeFilenameToContentHash?.contains(filenamePath) == true) {
43+
val contentHash = relativeFilenameToContentHash.getValue(filenamePath)
44+
safePutBytes(contentHash.toByteArray())
2545
} else {
26-
logger.w { "File $absoluteFilePath not found" }
46+
val absoluteFilePath = Paths.get(workingDirectory.toString(), filenamePath)
47+
val file = absoluteFilePath.toFile()
48+
if (file.exists() && file.isFile) {
49+
putFile(file)
50+
} else {
51+
logger.w { "File $absoluteFilePath not found" }
52+
}
2753
}
2854
safePutBytes(sourceFileTarget.seed)
2955
safePutBytes(name.toByteArray())

‎cli/src/main/kotlin/com/bazel_diff/interactor/DeserialiseHashesInteractor.kt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.bazel_diff.interactor
22

33
import com.google.gson.Gson
4+
import com.google.gson.reflect.TypeToken
45
import org.koin.core.component.KoinComponent
56
import org.koin.core.component.inject
67
import java.io.File
@@ -13,7 +14,7 @@ class DeserialiseHashesInteractor : KoinComponent {
1314
* @param file path to file that has been pre-validated
1415
*/
1516
fun execute(file: File): Map<String, String> {
16-
val gsonHash: Map<String, String> = HashMap()
17-
return gson.fromJson(FileReader(file), gsonHash.javaClass)
17+
val shape = object : TypeToken<Map<String, String>>() {}.type
18+
return gson.fromJson(FileReader(file), shape)
1819
}
1920
}

‎cli/src/main/kotlin/com/bazel_diff/io/ContentHashProvider.kt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package com.bazel_diff.io
2+
3+
import com.bazel_diff.interactor.DeserialiseHashesInteractor
4+
import java.io.File
5+
6+
class ContentHashProvider(file: File?) {
7+
// filename relative to workspace -> content hash of the file
8+
val filenameToHash: Map<String, String>? = if (file == null) null else readJson(file)
9+
10+
private fun readJson(file: File): Map<String, String> {
11+
val deserialiser = DeserialiseHashesInteractor()
12+
return deserialiser.execute(file)
13+
}
14+
}

‎cli/src/test/kotlin/com/bazel_diff/Modules.kt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,17 @@
11
package com.bazel_diff
22

33
import com.bazel_diff.bazel.BazelClient
4-
import com.bazel_diff.bazel.BazelQueryService
54
import com.bazel_diff.hash.BuildGraphHasher
65
import com.bazel_diff.hash.RuleHasher
76
import com.bazel_diff.hash.SourceFileHasher
87
import com.bazel_diff.hash.TargetHasher
8+
import com.bazel_diff.io.ContentHashProvider
99
import com.bazel_diff.log.Logger
10-
import com.bazel_diff.log.StdoutLogger
1110
import com.google.gson.GsonBuilder
1211
import org.koin.core.module.Module
1312
import org.koin.core.qualifier.named
1413
import org.koin.dsl.module
15-
import java.nio.file.Path
14+
import java.nio.file.Paths
1615

1716
fun testModule(): Module = module {
1817
single<Logger> { SilentLogger }
@@ -22,6 +21,8 @@ fun testModule(): Module = module {
2221
single { RuleHasher() }
2322
single { SourceFileHasher() }
2423
single { GsonBuilder().setPrettyPrinting().create() }
24+
single(named("working-directory")) { Paths.get("working-directory") }
25+
single { ContentHashProvider(null) }
2526
}
2627

2728
object SilentLogger : Logger {

‎cli/src/test/kotlin/com/bazel_diff/hash/SourceFileHasherTest.kt‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
package com.bazel_diff.hash
2+
3+
import assertk.assertThat
4+
import assertk.assertions.isEqualTo
5+
import assertk.assertions.isNull
6+
import com.bazel_diff.bazel.BazelSourceFileTarget
7+
import com.bazel_diff.extensions.toHexString
8+
import com.bazel_diff.testModule
9+
import kotlinx.coroutines.runBlocking
10+
import org.junit.Rule
11+
import org.junit.Test
12+
import org.koin.test.KoinTest
13+
import org.koin.test.KoinTestRule
14+
import java.nio.file.Files
15+
import java.nio.file.Paths
16+
17+
18+
internal class SourceFileHasherTest: KoinTest {
19+
private val repoAbsolutePath = Paths.get("").toAbsolutePath()
20+
private val fixtureFileTarget = "//cli/src/test/kotlin/com/bazel_diff/hash/fixture:foo.ts"
21+
private val fixtureFileContent: ByteArray
22+
private val seed = "seed".toByteArray()
23+
24+
init {
25+
val path = Paths.get("cli/src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts")
26+
fixtureFileContent = Files.readAllBytes(path)
27+
}
28+
29+
30+
@get:Rule
31+
val koinTestRule = KoinTestRule.create {
32+
modules(testModule())
33+
}
34+
35+
@Test
36+
fun testHashConcreteFile() = runBlocking {
37+
val hasher = SourceFileHasher(repoAbsolutePath, null)
38+
val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
39+
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
40+
val expected = sha256 {
41+
safePutBytes(fixtureFileContent)
42+
safePutBytes(seed)
43+
safePutBytes(fixtureFileTarget.toByteArray())
44+
}.toHexString()
45+
assertThat(actual).isEqualTo(expected)
46+
}
47+
48+
@Test
49+
fun testSoftHashConcreteFile() = runBlocking {
50+
val hasher = SourceFileHasher(repoAbsolutePath, null)
51+
val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
52+
val actual = hasher.softDigest(bazelSourceFileTarget)?.toHexString()
53+
val expected = sha256 {
54+
safePutBytes(fixtureFileContent)
55+
safePutBytes(seed)
56+
safePutBytes(fixtureFileTarget.toByteArray())
57+
}.toHexString()
58+
assertThat(actual).isEqualTo(expected)
59+
}
60+
61+
@Test
62+
fun testSoftHashNonExistedFile() = runBlocking {
63+
val hasher = SourceFileHasher(repoAbsolutePath, null)
64+
val bazelSourceFileTarget = BazelSourceFileTarget("//i/do/not/exist", seed)
65+
val actual = hasher.softDigest(bazelSourceFileTarget)
66+
assertThat(actual).isNull()
67+
}
68+
69+
@Test
70+
fun testSoftHashExternalTarget() = runBlocking {
71+
val target = "@bazel-diff//some:file"
72+
val hasher = SourceFileHasher(repoAbsolutePath, null)
73+
val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
74+
val actual = hasher.softDigest(bazelSourceFileTarget)
75+
assertThat(actual).isNull()
76+
}
77+
78+
@Test
79+
fun testHashNonExistedFile() = runBlocking {
80+
val target = "//i/do/not/exist"
81+
val hasher = SourceFileHasher(repoAbsolutePath, null)
82+
val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
83+
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
84+
val expected = sha256 {
85+
safePutBytes(seed)
86+
safePutBytes(target.toByteArray())
87+
}.toHexString()
88+
assertThat(actual).isEqualTo(expected)
89+
}
90+
91+
@Test
92+
fun testHashExternalTarget() = runBlocking {
93+
val target = "@bazel-diff//some:file"
94+
val hasher = SourceFileHasher(repoAbsolutePath, null)
95+
val bazelSourceFileTarget = BazelSourceFileTarget(target, seed)
96+
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
97+
val expected = sha256 {}.toHexString()
98+
assertThat(actual).isEqualTo(expected)
99+
}
100+
101+
@Test
102+
fun testHashWithProvidedContentHash() = runBlocking {
103+
val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts" to "foo-content-hash")
104+
val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash)
105+
val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
106+
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
107+
val expected = sha256 {
108+
safePutBytes("foo-content-hash".toByteArray())
109+
safePutBytes(seed)
110+
safePutBytes(fixtureFileTarget.toByteArray())
111+
}.toHexString()
112+
assertThat(actual).isEqualTo(expected)
113+
}
114+
115+
@Test
116+
fun testHashWithProvidedContentHashButNotInKey() = runBlocking {
117+
val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts" to "foo-content-hash")
118+
val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash)
119+
val bazelSourceFileTarget = BazelSourceFileTarget(fixtureFileTarget, seed)
120+
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
121+
val expected = sha256 {
122+
safePutBytes(fixtureFileContent)
123+
safePutBytes(seed)
124+
safePutBytes(fixtureFileTarget.toByteArray())
125+
}.toHexString()
126+
assertThat(actual).isEqualTo(expected)
127+
}
128+
129+
@Test
130+
fun testHashWithProvidedContentHashWithLeadingColon() = runBlocking {
131+
val targetName = "//:cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts"
132+
val filenameToContentHash = hashMapOf("cli/src/test/kotlin/com/bazel_diff/hash/fixture/bar.ts" to "foo-content-hash")
133+
val hasher = SourceFileHasher(repoAbsolutePath, filenameToContentHash)
134+
val bazelSourceFileTarget = BazelSourceFileTarget(targetName, seed)
135+
val actual = hasher.digest(bazelSourceFileTarget).toHexString()
136+
val expected = sha256 {
137+
safePutBytes("foo-content-hash".toByteArray())
138+
safePutBytes(seed)
139+
safePutBytes(targetName.toByteArray())
140+
}.toHexString()
141+
assertThat(actual).isEqualTo(expected)
142+
}
143+
}

‎cli/src/test/kotlin/com/bazel_diff/hash/fixture/foo.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
console.log('123')

0 commit comments

Comments
 (0)