Skip to content

Commit a7f32bd

Browse files
benbbenb
authored
File Hashing for Incremental Builds (#1923)
* Fix test build * Add incremental hash option to swift-driver * source file hashing for incremental * external dependency hashing for incremental * Bump serialized version
1 parent 6089687 commit a7f32bd

23 files changed

+351
-74
lines changed

‎Sources/SwiftDriver/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ add_library(SwiftDriver
113113
Utilities/DateAdditions.swift
114114
Utilities/Diagnostics.swift
115115
Utilities/FileList.swift
116+
Utilities/FileMetadata.swift
116117
Utilities/FileType.swift
117118
Utilities/PredictableRandomNumberGenerator.swift
118119
Utilities/RelativePathAdditions.swift

‎Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import struct TSCBasic.ByteString
2424
import struct TSCBasic.Diagnostic
2525
import struct TSCBasic.FileInfo
2626
import struct TSCBasic.RelativePath
27+
import struct TSCBasic.SHA256
2728
import var TSCBasic.localFileSystem
2829
import var TSCBasic.stderrStream
2930
import var TSCBasic.stdoutStream
@@ -44,7 +45,6 @@ extension Driver.ErrorDiagnostics: CustomStringConvertible {
4445
}
4546
}
4647

47-
4848
/// The Swift driver.
4949
public struct Driver {
5050
public enum Error: Swift.Error, Equatable, DiagnosticData {
@@ -212,8 +212,8 @@ public struct Driver {
212212
/// The set of input files
213213
@_spi(Testing) public let inputFiles: [TypedVirtualPath]
214214

215-
/// The last time each input file was modified, recorded at the start of the build.
216-
@_spi(Testing) public let recordedInputModificationDates: [TypedVirtualPath: TimePoint]
215+
/// The last time each input file was modified, and the file's SHA256 hash, recorded at the start of the build.
216+
@_spi(Testing) public let recordedInputMetadata: [TypedVirtualPath: FileMetadata]
217217

218218
/// The mapping from input files to output files for each kind.
219219
let outputFileMap: OutputFileMap?
@@ -950,11 +950,20 @@ public struct Driver {
950950
// Classify and collect all of the input files.
951951
let inputFiles = try Self.collectInputFiles(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine, fileSystem: self.fileSystem)
952952
self.inputFiles = inputFiles
953-
self.recordedInputModificationDates = .init(uniqueKeysWithValues:
954-
Set(inputFiles).compactMap {
955-
guard let modTime = try? fileSystem
956-
.lastModificationTime(for: $0.file) else { return nil }
957-
return ($0, modTime)
953+
954+
let incrementalFileHashes = parsedOptions.hasFlag(positive: .enableIncrementalFileHashing,
955+
negative: .disableIncrementalFileHashing,
956+
default: false)
957+
self.recordedInputMetadata = .init(uniqueKeysWithValues:
958+
Set(inputFiles).compactMap { inputFile -> (TypedVirtualPath, FileMetadata)? in
959+
guard let modTime = try? fileSystem.lastModificationTime(for: inputFile.file) else { return nil }
960+
if incrementalFileHashes {
961+
guard let data = try? fileSystem.readFileContents(inputFile.file) else { return nil }
962+
let hash = SHA256().hash(data).hexadecimalRepresentation
963+
return (inputFile, FileMetadata(mTime: modTime, hash: hash))
964+
} else {
965+
return (inputFile, FileMetadata(mTime: modTime))
966+
}
958967
})
959968

960969
do {
@@ -1073,7 +1082,7 @@ public struct Driver {
10731082
outputFileMap: outputFileMap,
10741083
incremental: self.shouldAttemptIncrementalCompilation,
10751084
parsedOptions: parsedOptions,
1076-
recordedInputModificationDates: recordedInputModificationDates)
1085+
recordedInputMetadata: recordedInputMetadata)
10771086

10781087
self.supportedFrontendFlags =
10791088
try Self.computeSupportedCompilerArgs(of: self.toolchain,
@@ -1912,7 +1921,7 @@ extension Driver {
19121921
}
19131922
try executor.execute(job: inPlaceJob,
19141923
forceResponseFiles: forceResponseFiles,
1915-
recordedInputModificationDates: recordedInputModificationDates)
1924+
recordedInputMetadata: recordedInputMetadata)
19161925
}
19171926

19181927
// If requested, warn for options that weren't used by the driver after the build is finished.
@@ -1957,7 +1966,7 @@ extension Driver {
19571966
delegate: jobExecutionDelegate,
19581967
numParallelJobs: numParallelJobs ?? 1,
19591968
forceResponseFiles: forceResponseFiles,
1960-
recordedInputModificationDates: recordedInputModificationDates)
1969+
recordedInputMetadata: recordedInputMetadata)
19611970
}
19621971

19631972
public func writeIncrementalBuildInformation(_ jobs: [Job]) {

‎Sources/SwiftDriver/Execution/DriverExecutor.swift

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,25 @@ public protocol DriverExecutor {
2323

2424
/// Execute a single job and capture the output.
2525
@discardableResult
26+
func execute(job: Job,
27+
forceResponseFiles: Bool,
28+
recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult
29+
2630
func execute(job: Job,
2731
forceResponseFiles: Bool,
2832
recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> ProcessResult
2933

34+
3035
/// Execute multiple jobs, tracking job status using the provided execution delegate.
3136
/// Pass in the `IncrementalCompilationState` to allow for incremental compilation.
3237
/// Pass in the `InterModuleDependencyGraph` to allow for module dependency tracking.
38+
func execute(workload: DriverExecutorWorkload,
39+
delegate: JobExecutionDelegate,
40+
numParallelJobs: Int,
41+
forceResponseFiles: Bool,
42+
recordedInputMetadata: [TypedVirtualPath: FileMetadata]
43+
) throws
44+
3345
func execute(workload: DriverExecutorWorkload,
3446
delegate: JobExecutionDelegate,
3547
numParallelJobs: Int,
@@ -38,6 +50,13 @@ public protocol DriverExecutor {
3850
) throws
3951

4052
/// Execute multiple jobs, tracking job status using the provided execution delegate.
53+
func execute(jobs: [Job],
54+
delegate: JobExecutionDelegate,
55+
numParallelJobs: Int,
56+
forceResponseFiles: Bool,
57+
recordedInputMetadata: [TypedVirtualPath: FileMetadata]
58+
) throws
59+
4160
func execute(jobs: [Job],
4261
delegate: JobExecutionDelegate,
4362
numParallelJobs: Int,
@@ -53,6 +72,34 @@ public protocol DriverExecutor {
5372
func description(of job: Job, forceResponseFiles: Bool) throws -> String
5473
}
5574

75+
extension DriverExecutor {
76+
public func execute(job: Job,
77+
forceResponseFiles: Bool,
78+
recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> ProcessResult
79+
{
80+
return try execute(job: job, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime })
81+
}
82+
83+
84+
public func execute(workload: DriverExecutorWorkload,
85+
delegate: JobExecutionDelegate,
86+
numParallelJobs: Int,
87+
forceResponseFiles: Bool,
88+
recordedInputMetadata: [TypedVirtualPath: FileMetadata]
89+
) throws {
90+
try execute(workload: workload, delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime })
91+
}
92+
93+
public func execute(jobs: [Job],
94+
delegate: JobExecutionDelegate,
95+
numParallelJobs: Int,
96+
forceResponseFiles: Bool,
97+
recordedInputMetadata: [TypedVirtualPath: FileMetadata]
98+
) throws {
99+
try execute(jobs: jobs, delegate: delegate, numParallelJobs: numParallelJobs, forceResponseFiles: forceResponseFiles, recordedInputModificationDates: recordedInputMetadata.mapValues { $0.mTime })
100+
}
101+
}
102+
56103
public struct DriverExecutorWorkload {
57104
public let continueBuildingAfterErrors: Bool
58105
public enum Kind {
@@ -96,10 +143,10 @@ extension DriverExecutor {
96143
func execute<T: Decodable>(job: Job,
97144
capturingJSONOutputAs outputType: T.Type,
98145
forceResponseFiles: Bool,
99-
recordedInputModificationDates: [TypedVirtualPath: TimePoint]) throws -> T {
146+
recordedInputMetadata: [TypedVirtualPath: FileMetadata]) throws -> T {
100147
let result = try execute(job: job,
101148
forceResponseFiles: forceResponseFiles,
102-
recordedInputModificationDates: recordedInputModificationDates)
149+
recordedInputMetadata: recordedInputMetadata)
103150

104151
if (result.exitStatus != .terminated(code: EXIT_SUCCESS)) {
105152
let returnCode = Self.computeReturnCode(exitStatus: result.exitStatus)

‎Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,8 @@ internal extension InterModuleDependencyGraph {
195195
reporter?.report("Unable to 'stat' \(inputPath.description)")
196196
return false
197197
}
198+
// SHA256 hashes for these files from the previous build are not
199+
// currently stored, so we can only check timestamps
198200
if inputModTime > outputModTime {
199201
reporter?.reportExplicitDependencyOutOfDate(moduleName,
200202
inputPath: inputPath.description)

‎Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ public extension Driver {
235235
try self.executor.execute(job: preScanJob,
236236
capturingJSONOutputAs: InterModuleDependencyImports.self,
237237
forceResponseFiles: forceResponseFiles,
238-
recordedInputModificationDates: recordedInputModificationDates)
238+
recordedInputMetadata: recordedInputMetadata)
239239
}
240240
return imports
241241
}
@@ -312,7 +312,7 @@ public extension Driver {
312312
try self.executor.execute(job: scannerJob,
313313
capturingJSONOutputAs: InterModuleDependencyGraph.self,
314314
forceResponseFiles: forceResponseFiles,
315-
recordedInputModificationDates: recordedInputModificationDates)
315+
recordedInputMetadata: recordedInputMetadata)
316316
}
317317
return dependencyGraph
318318
}

‎Sources/SwiftDriver/IncrementalCompilation/BuildRecord.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ extension BuildRecord {
4646
init(jobs: [Job],
4747
finishedJobResults: [BuildRecordInfo.JobResult],
4848
skippedInputs: Set<TypedVirtualPath>?,
49-
compilationInputModificationDates: [TypedVirtualPath: TimePoint],
49+
compilationInputModificationDates: [TypedVirtualPath: FileMetadata],
5050
actualSwiftVersion: String,
5151
argsHash: String,
5252
timeBeforeFirstJob: TimePoint,
@@ -57,10 +57,10 @@ extension BuildRecord {
5757
entry.job.inputsGeneratingCode.map { ($0, entry.result) }
5858
})
5959
let inputInfosArray = compilationInputModificationDates
60-
.map { input, modDate -> (VirtualPath, InputInfo) in
60+
.map { input, metadata -> (VirtualPath, InputInfo) in
6161
let status = InputInfo.Status( wasSkipped: skippedInputs?.contains(input),
6262
jobResult: jobResultsByInput[input])
63-
return (input.file, InputInfo(status: status, previousModTime: modDate))
63+
return (input.file, InputInfo(status: status, previousModTime: metadata.mTime, hash: metadata.hash))
6464
}
6565

6666
self.init(

‎Sources/SwiftDriver/IncrementalCompilation/BuildRecordInfo.swift

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ import class Dispatch.DispatchQueue
4848
@_spi(Testing) public let actualSwiftVersion: String
4949
@_spi(Testing) public let timeBeforeFirstJob: TimePoint
5050
let diagnosticEngine: DiagnosticsEngine
51-
let compilationInputModificationDates: [TypedVirtualPath: TimePoint]
51+
let compilationInputModificationDates: [TypedVirtualPath: FileMetadata]
5252
private var explicitModuleDependencyGraph: InterModuleDependencyGraph? = nil
5353

5454
private var finishedJobResults = [JobResult]()
@@ -64,7 +64,7 @@ import class Dispatch.DispatchQueue
6464
actualSwiftVersion: String,
6565
timeBeforeFirstJob: TimePoint,
6666
diagnosticEngine: DiagnosticsEngine,
67-
compilationInputModificationDates: [TypedVirtualPath: TimePoint])
67+
compilationInputModificationDates: [TypedVirtualPath: FileMetadata])
6868
{
6969
self.buildRecordPath = buildRecordPath
7070
self.fileSystem = fileSystem
@@ -85,7 +85,7 @@ import class Dispatch.DispatchQueue
8585
outputFileMap: OutputFileMap?,
8686
incremental: Bool,
8787
parsedOptions: ParsedOptions,
88-
recordedInputModificationDates: [TypedVirtualPath: TimePoint]
88+
recordedInputMetadata: [TypedVirtualPath: FileMetadata]
8989
) {
9090
// Cannot write a buildRecord without a path.
9191
guard let buildRecordPath = try? Self.computeBuildRecordPath(
@@ -99,7 +99,7 @@ import class Dispatch.DispatchQueue
9999
}
100100
let currentArgsHash = BuildRecordArguments.computeHash(parsedOptions)
101101
let compilationInputModificationDates =
102-
recordedInputModificationDates.filter { input, _ in
102+
recordedInputMetadata.filter { input, _ in
103103
input.type.isPartOfSwiftCompilation
104104
}
105105

‎Sources/SwiftDriver/IncrementalCompilation/FirstWaveComputer.swift

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ extension IncrementalCompilationState {
2626
let showJobLifecycle: Bool
2727
let alwaysRebuildDependents: Bool
2828
let explicitModulePlanner: ExplicitDependencyBuildPlanner?
29+
let useHashes: Bool
2930
/// If non-null outputs information for `-driver-show-incremental` for input path
3031
private let reporter: Reporter?
3132

@@ -46,6 +47,7 @@ extension IncrementalCompilationState {
4647
self.alwaysRebuildDependents = initialState.incrementalOptions.contains(
4748
.alwaysRebuildDependents)
4849
self.explicitModulePlanner = explicitModulePlanner
50+
self.useHashes = initialState.incrementalOptions.contains(.useFileHashesInModuleDependencyGraph)
4951
self.reporter = reporter
5052
}
5153

@@ -303,8 +305,9 @@ extension IncrementalCompilationState.FirstWaveComputer {
303305
/// The status of the input file.
304306
let status: InputInfo.Status
305307
/// If `true`, the modification time of this input matches the modification
306-
/// time recorded from the prior build in the build record.
307-
let datesMatch: Bool
308+
/// time recorded from the prior build in the build record, or the hash of
309+
/// its contents match.
310+
let metadataMatch: Bool
308311
}
309312

310313
// Find the inputs that have changed since last compilation, or were marked as needed a build
@@ -313,16 +316,21 @@ extension IncrementalCompilationState.FirstWaveComputer {
313316
) -> [ChangedInput] {
314317
jobsInPhases.compileJobs.compactMap { job in
315318
let input = job.primaryInputs[0]
316-
let modDate = buildRecordInfo.compilationInputModificationDates[input] ?? .distantFuture
319+
let metadata = buildRecordInfo.compilationInputModificationDates[input] ?? FileMetadata(mTime: .distantFuture)
317320
let inputInfo = outOfDateBuildRecord.inputInfos[input.file]
318321
let previousCompilationStatus = inputInfo?.status ?? .newlyAdded
319322
let previousModTime = inputInfo?.previousModTime
323+
let previousHash = inputInfo?.hash
324+
325+
assert(metadata.hash != nil || !useHashes)
320326

321327
switch previousCompilationStatus {
322-
case .upToDate where modDate == previousModTime:
328+
case .upToDate where metadata.mTime == previousModTime:
323329
reporter?.report("May skip current input:", input)
324330
return nil
325-
331+
case .upToDate where useHashes && (metadata.hash == previousHash):
332+
reporter?.report("May skip current input (identical hash):", input)
333+
return nil
326334
case .upToDate:
327335
reporter?.report("Scheduling changed input", input)
328336
case .newlyAdded:
@@ -332,9 +340,10 @@ extension IncrementalCompilationState.FirstWaveComputer {
332340
case .needsNonCascadingBuild:
333341
reporter?.report("Scheduling noncascading build", input)
334342
}
343+
let metadataMatch = metadata.mTime == previousModTime || (useHashes && metadata.hash == previousHash)
335344
return ChangedInput(typedFile: input,
336345
status: previousCompilationStatus,
337-
datesMatch: modDate == previousModTime)
346+
metadataMatch: metadataMatch )
338347
}
339348
}
340349

@@ -383,7 +392,7 @@ extension IncrementalCompilationState.FirstWaveComputer {
383392
) -> [TypedVirtualPath] {
384393
changedInputs.compactMap { changedInput in
385394
let inputIsUpToDate =
386-
changedInput.datesMatch && !inputsMissingOutputs.contains(changedInput.typedFile)
395+
changedInput.metadataMatch && !inputsMissingOutputs.contains(changedInput.typedFile)
387396
let basename = changedInput.typedFile.file.basename
388397

389398
// If we're asked to always rebuild dependents, all we need to do is

‎Sources/SwiftDriver/IncrementalCompilation/IncrementalCompilationState+Extensions.swift

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,10 @@ extension IncrementalCompilationState {
373373
/// Enables additional handling of explicit module build artifacts:
374374
/// Additional reading and writing of the inter-module dependency graph.
375375
public static let explicitModuleBuild = Options(rawValue: 1 << 6)
376+
377+
/// Enables use of file hashes as a fallback in the case that a timestamp
378+
/// change might invalidate a node
379+
public static let useFileHashesInModuleDependencyGraph = Options(rawValue: 1 << 7)
376380
}
377381
}
378382

‎Sources/SwiftDriver/IncrementalCompilation/IncrementalDependencyAndInputSetup.swift

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,12 @@ extension IncrementalCompilationState {
8989
if driver.parsedOptions.contains(.driverExplicitModuleBuild) {
9090
options.formUnion(.explicitModuleBuild)
9191
}
92+
if driver.parsedOptions.hasFlag(positive: .enableIncrementalFileHashing,
93+
negative: .disableIncrementalFileHashing,
94+
default: false) {
95+
options.formUnion(.useFileHashesInModuleDependencyGraph)
96+
}
97+
9298
return options
9399
}
94100
}
@@ -128,6 +134,9 @@ extension IncrementalCompilationState {
128134
@_spi(Testing) public var emitDependencyDotFileAfterEveryImport: Bool {
129135
options.contains(.emitDependencyDotFileAfterEveryImport)
130136
}
137+
@_spi(Testing) public var useFileHashesInModuleDependencyGraph: Bool {
138+
options.contains(.useFileHashesInModuleDependencyGraph)
139+
}
131140

132141
@_spi(Testing) public init(
133142
_ options: Options,

‎Sources/SwiftDriver/IncrementalCompilation/InputInfo.swift

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
//===----------------------------------------------------------------------===//
1212

1313
import struct TSCBasic.ProcessResult
14+
import struct TSCBasic.SHA256
1415

1516
/// Contains information about the current status of an input to the incremental
1617
/// build.
@@ -25,9 +26,12 @@ import struct TSCBasic.ProcessResult
2526
/// The last known modification time of this input.
2627
/*@_spi(Testing)*/ public let previousModTime: TimePoint
2728

28-
/*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint) {
29+
/*@_spi(Testing)*/ public let hash: String?
30+
31+
/*@_spi(Testing)*/ public init(status: Status, previousModTime: TimePoint, hash: String?) {
2932
self.status = status
3033
self.previousModTime = previousModTime
34+
self.hash = hash
3135
}
3236
}
3337

0 commit comments

Comments
 (0)