Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Maven support disk cache fixes #9824

Merged
merged 8 commits into from
Jan 24, 2025
13 changes: 12 additions & 1 deletion analyzer/src/main/kotlin/Analyzer.kt
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,13 @@ class Analyzer(private val config: AnalyzerConfiguration, private val labels: Ma
}

private fun analyzeInParallel(managedFiles: Map<PackageManager, List<File>>): AnalyzerResult {
val state = AnalyzerState()
logger.info { "Calling before resolution hooks for ${managedFiles.size} manager(s). " }

managedFiles.forEach { (manager, definitionFiles) ->
manager.beforeResolution(definitionFiles)
}

val state = AnalyzerState()
val packageManagerDependencies = determinePackageManagerDependencies(managedFiles)

runBlocking {
Expand All @@ -187,6 +192,12 @@ class Analyzer(private val config: AnalyzerConfiguration, private val labels: Ma
}
}

logger.info { "Calling after resolution hooks for ${managedFiles.size} manager(s). " }

managedFiles.forEach { (manager, definitionFiles) ->
manager.afterResolution(definitionFiles)
}

val excludes = managedFiles.keys.firstOrNull()?.excludes ?: Excludes.EMPTY
return state.buildResult(excludes)
}
Expand Down
16 changes: 8 additions & 8 deletions analyzer/src/main/kotlin/PackageManager.kt
Original file line number Diff line number Diff line change
Expand Up @@ -250,14 +250,18 @@ abstract class PackageManager(
PackageManagerDependencyResult(mustRunBefore = emptySet(), mustRunAfter = emptySet())

/**
* Optional step to run before dependency resolution, like checking for prerequisites.
* Optional step to run before dependency resolution, like checking for prerequisites. This function is called
* before [resolveDependencies] is called for any enabled package manager. It does not respect any "mustRunAfter"
* configuration.
*/
protected open fun beforeResolution(definitionFiles: List<File>) {}
open fun beforeResolution(definitionFiles: List<File>) {}

/**
* Optional step to run after dependency resolution, like cleaning up temporary files.
* Optional step to run after dependency resolution, like cleaning up temporary files. This function is called after
* [resolveDependencies] has finished for all enabled package managers. It does not respect any "mustRunAfter"
* configuration.
*/
protected open fun afterResolution(definitionFiles: List<File>) {}
open fun afterResolution(definitionFiles: List<File>) {}

/**
* Generate the final result to be returned by this package manager. This function is called at the very end of the
Expand All @@ -283,8 +287,6 @@ abstract class PackageManager(

val result = mutableMapOf<File, List<ProjectAnalyzerResult>>()

beforeResolution(definitionFiles)

definitionFiles.forEach { definitionFile ->
val relativePath = definitionFile.relativeTo(analysisRoot).invariantSeparatorsPath.ifEmpty { "." }

Expand Down Expand Up @@ -321,8 +323,6 @@ abstract class PackageManager(
logger.info { "$managerName resolved dependencies for path '$relativePath' in $duration." }
}

afterResolution(definitionFiles)

return createPackageManagerResult(result).addDependencyGraphIfMissing()
}

Expand Down
76 changes: 76 additions & 0 deletions analyzer/src/test/kotlin/AnalyzerTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Copyright (C) 2025 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
* License-Filename: LICENSE
*/

package org.ossreviewtoolkit.analyzer

import io.kotest.core.spec.style.WordSpec
import io.kotest.matchers.collections.containExactly
import io.kotest.matchers.should

import java.io.File
import java.io.IOException

import org.ossreviewtoolkit.analyzer.Analyzer.ManagedFileInfo
import org.ossreviewtoolkit.model.ProjectAnalyzerResult
import org.ossreviewtoolkit.model.config.AnalyzerConfiguration
import org.ossreviewtoolkit.model.config.RepositoryConfiguration

class AnalyzerTest : WordSpec({

Check warning on line 34 in analyzer/src/test/kotlin/AnalyzerTest.kt

View workflow job for this annotation

GitHub Actions / qodana-scan

Unused symbol

Class "AnalyzerTest" is never used
"analyze()" should {
"call afterResolution() even if resolveDependencies() throws" {
val analyzerConfig = AnalyzerConfiguration()
val repoConfig = RepositoryConfiguration()
val analyzer = Analyzer(analyzerConfig)
val analysisRoot = File(".").absoluteFile

val manager = DummyPackageManager(analysisRoot, analyzerConfig, repoConfig)

val info = ManagedFileInfo(
absoluteProjectPath = analysisRoot,
managedFiles = mapOf(manager to listOf(analysisRoot.resolve("Dummy"))),
repositoryConfiguration = repoConfig
)

analyzer.analyze(info)

manager.calls should containExactly("beforeResolution", "resolveDependencies", "afterResolution")
}
}
})

private class DummyPackageManager(
analysisRoot: File,
analyzerConfig: AnalyzerConfiguration,
repoConfig: RepositoryConfiguration
) : PackageManager("Dummy", "Project", analysisRoot, analyzerConfig, repoConfig) {
val calls = mutableListOf<String>()

override fun beforeResolution(definitionFiles: List<File>) {
calls += "beforeResolution"
}

override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> {
calls += "resolveDependencies"
throw IOException()
}

override fun afterResolution(definitionFiles: List<File>) {
calls += "afterResolution"
}
}
22 changes: 14 additions & 8 deletions analyzer/src/testFixtures/kotlin/TestUtils.kt
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,9 @@
package org.ossreviewtoolkit.analyzer

import io.kotest.inspectors.forAll
import io.kotest.matchers.collections.haveSize
import io.kotest.matchers.collections.shouldBeSingleton
import io.kotest.matchers.collections.shouldHaveAtLeastSize
import io.kotest.matchers.nulls.shouldNotBeNull
import io.kotest.matchers.should

import java.io.File
import java.time.Instant
Expand All @@ -45,15 +44,22 @@ import org.ossreviewtoolkit.model.config.ScopeExcludeReason
import org.ossreviewtoolkit.utils.test.USER_DIR

fun PackageManager.resolveSingleProject(definitionFile: File, resolveScopes: Boolean = false): ProjectAnalyzerResult {
val managerResult = resolveDependencies(listOf(definitionFile), emptyMap())
val definitionFiles = listOf(definitionFile)

return managerResult.projectResults[definitionFile].let { resultList ->
resultList.shouldNotBeNull()
resultList should haveSize(1)
val result = resultList.single()
beforeResolution(definitionFiles)
val managerResult = resolveDependencies(definitionFiles, emptyMap())

if (resolveScopes) managerResult.resolveScopes(result) else result
val resultList = managerResult.projectResults[definitionFile]
resultList.shouldNotBeNull()
resultList.shouldBeSingleton()

val result = resultList.single().let {
if (resolveScopes) managerResult.resolveScopes(it) else it
}

afterResolution(definitionFiles)

return result
}

/**
Expand Down
10 changes: 7 additions & 3 deletions plugins/package-managers/gradle/src/main/kotlin/Gradle.kt
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ class Gradle(
* A workspace reader that is backed by the local Gradle artifact cache.
*/
private class GradleCacheReader : WorkspaceReader {
private val workspaceRepository = WorkspaceRepository("gradleCache")
private val workspaceRepository = WorkspaceRepository("gradle/remote-artifacts")
private val gradleCacheRoot = GRADLE_USER_HOME.resolve("caches/modules-2/files-2.1")

override fun findArtifact(artifact: Artifact): File? {
Expand Down Expand Up @@ -162,8 +162,8 @@ class Gradle(
override fun getRepository() = workspaceRepository
}

private val maven = MavenSupport(GradleCacheReader())
private val dependencyHandler = GradleDependencyHandler(managerName, projectType, maven)
private val mavenSupport = MavenSupport(GradleCacheReader())
private val dependencyHandler = GradleDependencyHandler(managerName, projectType, mavenSupport)
private val graphBuilder = DependencyGraphBuilder(dependencyHandler)

// The path to the root project. In a single-project, just points to the project path.
Expand Down Expand Up @@ -328,4 +328,8 @@ class Gradle(
}
}
}

override fun afterResolution(definitionFiles: List<File>) {
mavenSupport.close()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,14 @@ class MavenFunTest : StringSpec({
val definitionFileLib = getAssetFile("projects/synthetic/maven/lib/pom.xml")
val expectedResultFile = getAssetFile("projects/synthetic/maven-expected-output-app.yml")

// app depends on lib, so we also have to pass the pom.xml of lib to resolveDependencies so that it is
// available in the Maven.projectsByIdentifier cache. Otherwise, resolution of transitive dependencies would
// not work.
val managerResult = create("Maven").resolveDependencies(
listOf(definitionFileApp, definitionFileLib),
emptyMap()
)
// The "app" project depends on the "lib" project, so the "pom.xml" of the "lib" project also has to be passed
// to [resolveDependencies], so that it is available in the [Maven.projectsByIdentifier] cache. Otherwise,
// resolution of transitive dependencies would not work.
val managerResult = with(create("Maven")) {
val definitionFiles = listOf(definitionFileApp, definitionFileLib)
beforeResolution(definitionFiles)
resolveDependencies(definitionFiles, emptyMap()).also { afterResolution(definitionFiles) }
}

val result = managerResult.projectResults[definitionFileApp]

Expand Down
14 changes: 9 additions & 5 deletions plugins/package-managers/maven/src/main/kotlin/Maven.kt
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ class Maven(
}

private inner class LocalProjectWorkspaceReader : WorkspaceReader {
private val workspaceRepository = WorkspaceRepository()
private val workspaceRepository = WorkspaceRepository("maven/remote-artifacts")

override fun findArtifact(artifact: Artifact) =
artifact.takeIf { it.extension == "pom" }?.let {
Expand All @@ -92,7 +92,7 @@ class Maven(
override fun getRepository() = workspaceRepository
}

private val mvn = MavenSupport(LocalProjectWorkspaceReader())
private val mavenSupport = MavenSupport(LocalProjectWorkspaceReader())

private val localProjectBuildingResults = mutableMapOf<String, ProjectBuildingResult>()

Expand All @@ -102,10 +102,10 @@ class Maven(
private val sbtMode = options["sbtMode"].toBoolean()

override fun beforeResolution(definitionFiles: List<File>) {
localProjectBuildingResults += mvn.prepareMavenProjects(definitionFiles)
localProjectBuildingResults += mavenSupport.prepareMavenProjects(definitionFiles)

val localProjects = localProjectBuildingResults.mapValues { it.value.project }
val dependencyHandler = MavenDependencyHandler(managerName, projectType, mvn, localProjects, sbtMode)
val dependencyHandler = MavenDependencyHandler(managerName, projectType, mavenSupport, localProjects, sbtMode)
graphBuilder = DependencyGraphBuilder(dependencyHandler)
}

Expand All @@ -114,7 +114,7 @@ class Maven(

override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> {
val workingDir = definitionFile.parentFile
val projectBuildingResult = mvn.buildMavenProject(definitionFile)
val projectBuildingResult = mavenSupport.buildMavenProject(definitionFile)
val mavenProject = projectBuildingResult.project
val projectId = Identifier(
type = projectType,
Expand Down Expand Up @@ -172,6 +172,10 @@ class Maven(

return listOf(ProjectAnalyzerResult(project, emptySet(), issues))
}

override fun afterResolution(definitionFiles: List<File>) {
mavenSupport.close()
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.ossreviewtoolkit.plugins.packagemanagers.maven.utils

import java.io.Closeable
import java.io.File
import java.net.URI

Expand Down Expand Up @@ -100,10 +101,16 @@
private val File?.safePath: String
get() = this?.invariantSeparatorsPath ?: "<unknown file>"

class MavenSupport(private val workspaceReader: WorkspaceReader) {
class MavenSupport(private val workspaceReader: WorkspaceReader) : Closeable {

Check warning on line 104 in plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt

View check run for this annotation

Codecov / codecov/patch

plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt#L104

Added line #L104 was not covered by tests
private val container = createContainer()
private val repositorySystemSession = createRepositorySystemSession(workspaceReader)

private val remoteArtifactCache = DiskCache(
directory = ortDataDirectory.resolve("cache/analyzer/${workspaceReader.repository.contentType}"),
maxCacheSizeInBytes = 1.gibibytes,
maxCacheEntryAgeInSeconds = 6.hours.inWholeSeconds

Check warning on line 111 in plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt

View check run for this annotation

Codecov / codecov/patch

plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt#L108-L111

Added lines #L108 - L111 were not covered by tests
)

// The MavenSettingsBuilder class is deprecated, but internally it uses its successor SettingsBuilder. Calling
// MavenSettingsBuilder requires less code than calling SettingsBuilder, so use it until it is removed.
@Suppress("DEPRECATION")
Expand Down Expand Up @@ -587,6 +594,10 @@
legacySupport.session = null
}
}

override fun close() {
remoteArtifactCache.close()
}

Check warning on line 600 in plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt

View check run for this annotation

Codecov / codecov/patch

plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt#L599-L600

Added lines #L599 - L600 were not covered by tests
}

private val PACKAGING_TYPES = setOf(
Expand All @@ -597,12 +608,6 @@
"jenkins-module", "orbit", "so", "zip"
)

private val remoteArtifactCache = DiskCache(
directory = ortDataDirectory.resolve("cache/analyzer/maven/remote-artifacts"),
maxCacheSizeInBytes = 1.gibibytes,
maxCacheEntryAgeInSeconds = 6.hours.inWholeSeconds
)

private fun createContainer(): PlexusContainer {
val configuration = DefaultContainerConfiguration().apply {
autoWiring = true
Expand Down
6 changes: 1 addition & 5 deletions plugins/package-managers/node/src/main/kotlin/npm/Npm.kt
Original file line number Diff line number Diff line change
Expand Up @@ -129,11 +129,7 @@
).let { listOf(it) }
}

override fun beforeResolution(definitionFiles: List<File>) {
// We do not actually depend on any features specific to an NPM version, but we still want to stick to a
// fixed minor version to be sure to get consistent results.
NpmCommand.checkVersion()
}
override fun beforeResolution(definitionFiles: List<File>) = NpmCommand.checkVersion()

Check warning on line 132 in plugins/package-managers/node/src/main/kotlin/npm/Npm.kt

View check run for this annotation

Codecov / codecov/patch

plugins/package-managers/node/src/main/kotlin/npm/Npm.kt#L132

Added line #L132 was not covered by tests

private fun listModules(workingDir: File, issues: MutableList<Issue>): ModuleInfo {
val listProcess = NpmCommand.run(workingDir, "list", "--depth", "Infinity", "--json", "--long")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,7 @@
workingDir = workingDir
).requireSuccess()

override fun beforeResolution(definitionFiles: List<File>) =
// We do not actually depend on any features specific to a PNPM version, but we still want to stick to a
// fixed major version to be sure to get consistent results.
PnpmCommand.checkVersion()
override fun beforeResolution(definitionFiles: List<File>) = PnpmCommand.checkVersion()

Check warning on line 136 in plugins/package-managers/node/src/main/kotlin/pnpm/Pnpm.kt

View check run for this annotation

Codecov / codecov/patch

plugins/package-managers/node/src/main/kotlin/pnpm/Pnpm.kt#L136

Added line #L136 was not covered by tests

internal fun getRemotePackageDetails(packageName: String): PackageJson? {
packageDetailsCache[packageName]?.let { return it }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,7 @@
}
}

override fun beforeResolution(definitionFiles: List<File>) =
// We do not actually depend on any features specific to a Yarn version, but we still want to stick to a
// fixed minor version to be sure to get consistent results.
YarnCommand.checkVersion()
override fun beforeResolution(definitionFiles: List<File>) = YarnCommand.checkVersion()

Check warning on line 134 in plugins/package-managers/node/src/main/kotlin/yarn/Yarn.kt

View check run for this annotation

Codecov / codecov/patch

plugins/package-managers/node/src/main/kotlin/yarn/Yarn.kt#L134

Added line #L134 was not covered by tests

override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> {
val workingDir = definitionFile.parentFile
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -177,9 +177,7 @@
isCorepackEnabledInManifest(workingDir)
}

override fun beforeResolution(definitionFiles: List<File>) =
// We depend on a version >= 2, so we check the version for safety.
definitionFiles.forEach { checkVersion(it.parentFile) }
override fun beforeResolution(definitionFiles: List<File>) = definitionFiles.forEach { checkVersion(it.parentFile) }

Check warning on line 180 in plugins/package-managers/node/src/main/kotlin/yarn2/Yarn2.kt

View check run for this annotation

Codecov / codecov/patch

plugins/package-managers/node/src/main/kotlin/yarn2/Yarn2.kt#L180

Added line #L180 was not covered by tests

override fun resolveDependencies(definitionFile: File, labels: Map<String, String>): List<ProjectAnalyzerResult> {
val workingDir = definitionFile.parentFile
Expand Down
Loading
Loading