From 04c00b5f0cfc0b7ecb4a451d7d90697b2bb2fdcd Mon Sep 17 00:00:00 2001 From: Wolfgang Klenk Date: Mon, 11 Nov 2024 10:13:44 +0100 Subject: [PATCH] refactor(vcs): Enable to provide VCS-specific configuration options While VCS implementations are already plugins, they are not yet configurable. VCS implementations require common configurations (e.g., `revision`, `recursive`) and should support also VCS-specific configurations if they are consumed via their API. This allows to add functionality to individual VCS implementations without the need to implement them for all of them. Fixes #8556. Signed-off-by: Wolfgang Klenk --- cli/src/funTest/kotlin/AnalyzerFunTest.kt | 6 +- downloader/src/main/kotlin/Downloader.kt | 4 +- .../src/main/kotlin/VersionControlSystem.kt | 99 +++++++++++-------- .../kotlin/VersionControlSystemFactory.kt | 26 +++++ .../test/kotlin/VersionControlSystemTest.kt | 7 +- .../schemas/ort-configuration-schema.json | 22 +++++ .../kotlin/config/DownloaderConfiguration.kt | 28 +++++- .../VersionControlSystemConfiguration.kt | 35 +++++++ model/src/main/resources/reference.yml | 11 +++ .../config/DownloaderConfigurationTest.kt | 54 ++++++++++ .../git/src/main/kotlin/Git.kt | 97 ++++++++++++++---- .../git/src/main/kotlin/GitRepo.kt | 21 +++- ...iewtoolkit.downloader.VersionControlSystem | 2 - ...kit.downloader.VersionControlSystemFactory | 2 + .../mercurial/src/main/kotlin/Mercurial.kt | 18 +++- ...it.downloader.VersionControlSystemFactory} | 2 +- .../subversion/src/main/kotlin/Subversion.kt | 18 +++- ...it.downloader.VersionControlSystemFactory} | 2 +- .../kotlin/SafeDeleteRecursivelyFunTest.kt | 4 +- 19 files changed, 380 insertions(+), 78 deletions(-) create mode 100644 downloader/src/main/kotlin/VersionControlSystemFactory.kt create mode 100644 model/src/main/kotlin/config/VersionControlSystemConfiguration.kt create mode 100644 model/src/test/kotlin/config/DownloaderConfigurationTest.kt delete mode 100644 plugins/version-control-systems/git/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem create mode 100644 plugins/version-control-systems/git/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory rename plugins/version-control-systems/mercurial/src/main/resources/META-INF/services/{org.ossreviewtoolkit.downloader.VersionControlSystem => org.ossreviewtoolkit.downloader.VersionControlSystemFactory} (81%) rename plugins/version-control-systems/subversion/src/main/resources/META-INF/services/{org.ossreviewtoolkit.downloader.VersionControlSystem => org.ossreviewtoolkit.downloader.VersionControlSystemFactory} (79%) diff --git a/cli/src/funTest/kotlin/AnalyzerFunTest.kt b/cli/src/funTest/kotlin/AnalyzerFunTest.kt index aac10978c5348..7ca955eabccbf 100644 --- a/cli/src/funTest/kotlin/AnalyzerFunTest.kt +++ b/cli/src/funTest/kotlin/AnalyzerFunTest.kt @@ -35,6 +35,7 @@ import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.AnalyzerConfiguration import org.ossreviewtoolkit.model.config.PackageManagerConfiguration import org.ossreviewtoolkit.model.config.RepositoryConfiguration +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.model.toYaml import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.GitRepo import org.ossreviewtoolkit.utils.test.getAssetFile @@ -52,7 +53,10 @@ class AnalyzerFunTest : WordSpec({ revision = "31588aa8f8555474e1c3c66a359ec99e4cd4b1fa" ) ) - val outputDir = tempdir().also { GitRepo().download(pkg, it) } + val outputDir = tempdir().also { + GitRepo.Factory().create(VersionControlSystemConfiguration()) + .download(pkg, it) + } val result = analyze(outputDir, packageManagers = emptySet()).toYaml() diff --git a/downloader/src/main/kotlin/Downloader.kt b/downloader/src/main/kotlin/Downloader.kt index 6e5b49d87988f..bcfe638ea6cde 100644 --- a/downloader/src/main/kotlin/Downloader.kt +++ b/downloader/src/main/kotlin/Downloader.kt @@ -232,7 +232,7 @@ class Downloader(private val config: DownloaderConfiguration) { var applicableVcs: VersionControlSystem? = null if (pkg.vcsProcessed.type != VcsType.UNKNOWN) { - applicableVcs = VersionControlSystem.forType(pkg.vcsProcessed.type) + applicableVcs = VersionControlSystem.forType(pkg.vcsProcessed.type, config.versionControlSystems) logger.info { applicableVcs?.let { "Detected VCS type '${it.type}' from type name '${pkg.vcsProcessed.type}'." @@ -241,7 +241,7 @@ class Downloader(private val config: DownloaderConfiguration) { } if (applicableVcs == null) { - applicableVcs = VersionControlSystem.forUrl(pkg.vcsProcessed.url) + applicableVcs = VersionControlSystem.forUrl(pkg.vcsProcessed.url, config.versionControlSystems) logger.info { applicableVcs?.let { "Detected VCS type '${it.type}' from URL ${pkg.vcsProcessed.url}." diff --git a/downloader/src/main/kotlin/VersionControlSystem.kt b/downloader/src/main/kotlin/VersionControlSystem.kt index 4d16e31b4861d..0de3a0ff1ad35 100644 --- a/downloader/src/main/kotlin/VersionControlSystem.kt +++ b/downloader/src/main/kotlin/VersionControlSystem.kt @@ -24,13 +24,14 @@ import java.io.IOException import org.apache.logging.log4j.kotlin.logger +import org.ossreviewtoolkit.downloader.VersionControlSystemFactory.Companion.ALL import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType import org.ossreviewtoolkit.model.config.LicenseFilePatterns +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.model.orEmpty import org.ossreviewtoolkit.utils.common.CommandLineTool -import org.ossreviewtoolkit.utils.common.Plugin import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.uppercaseFirstChar import org.ossreviewtoolkit.utils.ort.ORT_REPO_CONFIG_FILENAME @@ -44,22 +45,26 @@ abstract class VersionControlSystem( * the version control system is available. */ private val commandLineTool: CommandLineTool? = null -) : Plugin { +) { companion object { - /** - * All [version control systems][VersionControlSystem] available in the classpath, sorted by their priority. - */ - val ALL by lazy { - Plugin.getAll().toList().sortedByDescending { (_, vcs) -> vcs.priority }.toMap() - } - /** * Return the applicable VCS for the given [vcsType], or null if none is applicable. */ - fun forType(vcsType: VcsType) = - ALL.values.find { - it.isAvailable() && it.isApplicableType(vcsType) + fun forType( + vcsType: VcsType, + versionControlSystems: Map = emptyMap() + ) = ALL.values.filter { vcsFactory -> vcsFactory.type == vcsType.toString() } + .map { vcsFactory -> + // If there is a configuration for the VCS type, use it, otherwise create + // the VCS with an empty configuration. + versionControlSystems[vcsFactory.type]?.let { vcsConfig -> + vcsFactory.parseConfig( + options = vcsConfig.options, + secrets = emptyMap() + ).let { parsedVcsConfig -> vcsFactory.create(parsedVcsConfig) } + } ?: vcsFactory.create(VersionControlSystemConfiguration()) } + .firstOrNull { vcs -> vcs.isAvailable() } /** * A map to cache the [VersionControlSystem], if any, for previously queried URLs. This helps to speed up @@ -72,7 +77,7 @@ abstract class VersionControlSystem( * Return the applicable VCS for the given [vcsUrl], or null if none is applicable. */ @Synchronized - fun forUrl(vcsUrl: String) = + fun forUrl(vcsUrl: String, versionControlSystems: Map = emptyMap()) = // Do not use getOrPut() here as it cannot handle null values, also see // https://youtrack.jetbrains.com/issue/KT-21392. if (vcsUrl in urlToVcsMap) { @@ -82,12 +87,25 @@ abstract class VersionControlSystem( when (val type = VcsHost.parseUrl(vcsUrl).type) { VcsType.UNKNOWN -> { // ...then eventually try to determine the type also dynamically. - ALL.values.find { - it.isAvailable() && it.isApplicableUrl(vcsUrl) - } + ALL.values + .map { vcsFactory -> + // If there is a configuration for the VCS type, use it, otherwise create + // the VCS with an empty configuration. + versionControlSystems[vcsFactory.type]?.let { vcsConfig -> + vcsFactory.parseConfig( + options = vcsConfig.options, + secrets = emptyMap() + ) + .let { parsedVcsConfig -> + vcsFactory.create(parsedVcsConfig) + } + } + + ?: vcsFactory.create(VersionControlSystemConfiguration()) + }.firstOrNull { vcs -> vcs.isAvailable() && vcs.isApplicableUrl(vcsUrl) } } - else -> forType(type) + else -> forType(type, versionControlSystems) }.also { urlToVcsMap[vcsUrl] = it } @@ -109,28 +127,31 @@ abstract class VersionControlSystem( return if (absoluteVcsDirectory in dirToVcsMap) { dirToVcsMap[absoluteVcsDirectory] } else { - ALL.values.asSequence().mapNotNull { - if (it is CommandLineTool && !it.isInPath()) { - null - } else { - it.getWorkingTree(absoluteVcsDirectory) - } - }.find { - try { - it.isValid() - } catch (e: IOException) { - e.showStackTrace() - - logger.debug { - "Exception while validating ${it.vcsType} working tree, treating it as non-applicable: " + - e.collectMessages() + ALL.values.asSequence() + .map { vcsFactory -> vcsFactory.create(VersionControlSystemConfiguration()) } + .mapNotNull { + if (it is CommandLineTool && !it.isInPath()) { + null + } else { + it.getWorkingTree(absoluteVcsDirectory) } - - false + }.find { + try { + it.isValid() + } catch (e: IOException) { + e.showStackTrace() + + logger.debug { + "Exception while validating ${it.vcsType} working tree, " + + "treating it as non-applicable: " + + e.collectMessages() + } + + false + } + }.also { + dirToVcsMap[absoluteVcsDirectory] = it } - }.also { - dirToVcsMap[absoluteVcsDirectory] = it - } } } @@ -165,9 +186,9 @@ abstract class VersionControlSystem( } /** - * The priority in which this VCS should be probed. A higher value means a higher priority. + * The type of CVS that is supported by this VCS plugin. */ - protected open val priority: Int = 0 + abstract val type: String /** * A list of symbolic names that point to the latest revision. diff --git a/downloader/src/main/kotlin/VersionControlSystemFactory.kt b/downloader/src/main/kotlin/VersionControlSystemFactory.kt new file mode 100644 index 0000000000000..bab59ca7b9884 --- /dev/null +++ b/downloader/src/main/kotlin/VersionControlSystemFactory.kt @@ -0,0 +1,26 @@ +package org.ossreviewtoolkit.downloader + +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration +import org.ossreviewtoolkit.utils.common.Plugin +import org.ossreviewtoolkit.utils.common.TypedConfigurablePluginFactory + +/** + * An abstract class to be implemented by factories for [version contral systems][VersionControlSystem]. + * The constructor parameter [type] denotes which VCS type is supported by this plugin. + * The constructor parameter [priority] is used to determine the order in which the VCS plugins are used. + */ +abstract class VersionControlSystemFactory(override val type: String, val priority: Int) : + TypedConfigurablePluginFactory { + companion object { + /** + * All [version control system factories][VersionControlSystemFactory] available in the classpath, + * associated by their names, sorted by priority. + */ + val ALL by lazy { + Plugin.getAll() + .toList() + .sortedByDescending { (_, vcsFactory) -> vcsFactory.priority } + .toMap() + } + } +} diff --git a/downloader/src/test/kotlin/VersionControlSystemTest.kt b/downloader/src/test/kotlin/VersionControlSystemTest.kt index 9349902a37a18..e1801e170bdaf 100644 --- a/downloader/src/test/kotlin/VersionControlSystemTest.kt +++ b/downloader/src/test/kotlin/VersionControlSystemTest.kt @@ -33,6 +33,7 @@ import java.lang.UnsupportedOperationException import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git import org.ossreviewtoolkit.utils.common.CommandLineTool @@ -87,7 +88,8 @@ class VersionControlSystemTest : WordSpec({ every { workingTree.guessRevisionName(any(), any()) } returns "v1.6.0" - Git().getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf( + Git.Factory().create(VersionControlSystemConfiguration()) + .getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf( "v1.6.0" ) } @@ -110,7 +112,8 @@ class VersionControlSystemTest : WordSpec({ every { workingTree.listRemoteBranches() } returns listOf("main") every { workingTree.listRemoteTags() } returns emptyList() - Git().getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf( + Git.Factory().create(VersionControlSystemConfiguration()) + .getRevisionCandidates(workingTree, pkg, allowMovingRevisions = true) shouldBeSuccess listOf( "master", "main" ) diff --git a/integrations/schemas/ort-configuration-schema.json b/integrations/schemas/ort-configuration-schema.json index 44042cdebaaf2..4e96bf00b7573 100644 --- a/integrations/schemas/ort-configuration-schema.json +++ b/integrations/schemas/ort-configuration-schema.json @@ -89,6 +89,28 @@ "items": { "$ref": "#/definitions/SourceCodeOrigins" } + }, + "versionControlSystems": { + "type": "object", + "properties": { + "Git": { + "type": "object", + "properties": { + "options": { + "type": "object", + "properties": { + "submoduleHistoryDepth": { + "type": "integer", + "minimum": 1 + }, + "updateNestedSubmodules": { + "type": "boolean" + } + } + } + } + } + } } } }, diff --git a/model/src/main/kotlin/config/DownloaderConfiguration.kt b/model/src/main/kotlin/config/DownloaderConfiguration.kt index 86245caeabeab..217a77f0217fe 100644 --- a/model/src/main/kotlin/config/DownloaderConfiguration.kt +++ b/model/src/main/kotlin/config/DownloaderConfiguration.kt @@ -44,9 +44,35 @@ data class DownloaderConfiguration( * Configuration of the considered source code origins and their priority order. This must not be empty and not * contain any duplicates. */ - val sourceCodeOrigins: List = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT) + val sourceCodeOrigins: List = listOf(SourceCodeOrigin.VCS, SourceCodeOrigin.ARTIFACT), + + /** + * Version control system specific configurations. The key needs to match VCS type, + * e.g. "Git" for the Git version control system. + */ + val versionControlSystems: Map = emptyMap() ) { + /** + * A copy of [versionControlSystems] with case-insensitive keys. + */ + private val versionControlSystemsCaseInsensitive: Map = + versionControlSystems.toSortedMap(String.CASE_INSENSITIVE_ORDER) + init { sourceCodeOrigins.requireNotEmptyNoDuplicates() + + val duplicateVersionControlSystems = + versionControlSystems.keys - versionControlSystemsCaseInsensitive.keys.toSet() + + require(duplicateVersionControlSystems.isEmpty()) { + "The following version control systems have duplicate configuration: " + + "${duplicateVersionControlSystems.joinToString()}." + } } + + /** + * Get a [VersionControlSystemConfiguration] from [versionControlSystems]. + * The difference to accessing the map directly is that [vcsType] can be case-insensitive. + */ + fun getVersionControlSystemConfiguration(vcsType: String) = versionControlSystemsCaseInsensitive[vcsType] } diff --git a/model/src/main/kotlin/config/VersionControlSystemConfiguration.kt b/model/src/main/kotlin/config/VersionControlSystemConfiguration.kt new file mode 100644 index 0000000000000..a6da508f7f657 --- /dev/null +++ b/model/src/main/kotlin/config/VersionControlSystemConfiguration.kt @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.config + +import com.fasterxml.jackson.annotation.JsonInclude + +import org.ossreviewtoolkit.utils.common.Options + +/** + * The configuration for a Version Control System (VCS). + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +data class VersionControlSystemConfiguration( + /** + * Custom configuration options. See the documentation of the respective class for available options. + */ + val options: Options = emptyMap() +) diff --git a/model/src/main/resources/reference.yml b/model/src/main/resources/reference.yml index 8f76fe92fb783..32aead0119e78 100644 --- a/model/src/main/resources/reference.yml +++ b/model/src/main/resources/reference.yml @@ -167,6 +167,17 @@ ort: sourceCodeOrigins: [VCS, ARTIFACT] + # Optional VCS-specific configuration options. + versionControlSystems: + Git: + options: + # Depth of the commit history to fetch when updating submodules + submoduleHistoryDepth: 10 + + # A flag to control whether nested submodules should be updated (true), or if only the submodules + # on the first layer should be considered (false). + updateNestedSubmodules: true + scanner: skipConcluded: true skipExcluded: true diff --git a/model/src/test/kotlin/config/DownloaderConfigurationTest.kt b/model/src/test/kotlin/config/DownloaderConfigurationTest.kt new file mode 100644 index 0000000000000..cc8fbc9cba07c --- /dev/null +++ b/model/src/test/kotlin/config/DownloaderConfigurationTest.kt @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 The ORT Project Authors (see ) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +package org.ossreviewtoolkit.model.config + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.WordSpec +import io.kotest.matchers.shouldBe + +class DownloaderConfigurationTest : WordSpec({ + "DownloaderConfiguration()" should { + "throw an exception on duplicate VCS configurations" { + shouldThrow { + DownloaderConfiguration( + versionControlSystems = mapOf( + "Git" to VersionControlSystemConfiguration(), + "git" to VersionControlSystemConfiguration() + ) + ) + } + } + } + + "getVersionControlSystemConfiguration" should { + "return configuration case-insensitively" { + val vcsConfiguration = VersionControlSystemConfiguration() + val downloaderConfiguration = DownloaderConfiguration( + versionControlSystems = mapOf( + "MyVCS" to vcsConfiguration + ) + ) + + downloaderConfiguration.getVersionControlSystemConfiguration("MyVCS") shouldBe vcsConfiguration + downloaderConfiguration.getVersionControlSystemConfiguration("myvcs") shouldBe vcsConfiguration + downloaderConfiguration.getVersionControlSystemConfiguration("MYVCS") shouldBe vcsConfiguration + } + } +}) diff --git a/plugins/version-control-systems/git/src/main/kotlin/Git.kt b/plugins/version-control-systems/git/src/main/kotlin/Git.kt index 1f77eec9d153d..48740ffb68d8c 100644 --- a/plugins/version-control-systems/git/src/main/kotlin/Git.kt +++ b/plugins/version-control-systems/git/src/main/kotlin/Git.kt @@ -27,7 +27,7 @@ import java.security.PublicKey import org.apache.logging.log4j.kotlin.logger -import org.eclipse.jgit.api.Git +import org.eclipse.jgit.api.Git as JGit import org.eclipse.jgit.api.LsRemoteCommand import org.eclipse.jgit.api.errors.GitAPIException import org.eclipse.jgit.errors.UnsupportedCredentialItem @@ -45,10 +45,13 @@ import org.eclipse.jgit.transport.sshd.ServerKeyDatabase import org.eclipse.jgit.transport.sshd.SshdSessionFactory import org.ossreviewtoolkit.downloader.VersionControlSystem +import org.ossreviewtoolkit.downloader.VersionControlSystemFactory import org.ossreviewtoolkit.downloader.WorkingTree import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.utils.common.CommandLineTool +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.common.safeMkdirs @@ -59,8 +62,18 @@ import org.ossreviewtoolkit.utils.ort.showStackTrace import org.semver4j.RangesList import org.semver4j.RangesListFactory -// TODO: Make this configurable. -const val GIT_HISTORY_DEPTH = 50 +const val DEFAULT_GIT_HISTORY_DEPTH = 50 + +/** + * The name of the option to specify the depth of the commit history to fetch when updating submodules. + */ +const val OPTION_SUBMODULE_HISTORY_DEPTH = "submoduleHistoryDepth" + +/** + * The name of the option to specify if nested submodules should be updated, or if only the submodules + * on the first layer should be considered. + */ +const val OPTION_UPDATE_NESTED_SUBMODULES = "updateNestedSubmodules" // Replace prefixes of Git submodule repository URLs. private val REPOSITORY_URL_PREFIX_REPLACEMENTS = listOf( @@ -84,8 +97,10 @@ object GitCommand : CommandLineTool { override fun displayName(): String = "Git" } -class Git : VersionControlSystem(GitCommand) { +class Git internal constructor(val vcsConfig: VersionControlSystemConfiguration = VersionControlSystemConfiguration()) : + VersionControlSystem(GitCommand) { companion object { + init { // Make sure that JGit uses the exact same authentication information as ORT itself. This addresses // discrepancies in the way .netrc files are interpreted between JGit's and ORT's implementation. @@ -117,13 +132,22 @@ class Git : VersionControlSystem(GitCommand) { } override val type = VcsType.GIT.toString() - override val priority = 100 override val latestRevisionNames = listOf("HEAD", "@") + class Factory : VersionControlSystemFactory(VcsType.GIT.toString(), 100) { + override fun create(config: VersionControlSystemConfiguration): VersionControlSystem { + return Git(config) + } + + override fun parseConfig(options: Options, secrets: Options): VersionControlSystemConfiguration { + TODO("Not yet implemented") + } + } + override fun getVersion() = GitCommand.getVersion(null) override fun getDefaultBranchName(url: String): String { - val refs = Git.lsRemoteRepository().setRemote(url).callAsMap() + val refs = JGit.lsRemoteRepository().setRemote(url).callAsMap() return (refs["HEAD"] as? SymbolicRef)?.target?.name?.removePrefix("refs/heads/") ?: "master" } @@ -138,7 +162,7 @@ class Git : VersionControlSystem(GitCommand) { override fun initWorkingTree(targetDir: File, vcs: VcsInfo): WorkingTree { try { - Git.init().setDirectory(targetDir).call().use { git -> + JGit.init().setDirectory(targetDir).call().use { git -> git.remoteAdd().setName("origin").setUri(URIish(vcs.url)).call() if (Os.isWindows) { @@ -173,12 +197,22 @@ class Git : VersionControlSystem(GitCommand) { recursive: Boolean ): Result = (workingTree as GitWorkingTree).useRepo { - Git(this).use { git -> + JGit(this).use { git -> logger.info { "Updating working tree from ${workingTree.getRemoteUrl()}." } - updateWorkingTreeWithoutSubmodules(workingTree, git, revision).mapCatching { + val historyDepth = + vcsConfig.options[OPTION_SUBMODULE_HISTORY_DEPTH]?.toInt() ?: DEFAULT_GIT_HISTORY_DEPTH + updateWorkingTreeWithoutSubmodules(workingTree, git, revision, historyDepth).mapCatching { // In case this throws the exception gets encapsulated as a failure. - if (recursive) updateSubmodules(workingTree) + if (recursive) { + val updateNestedSubmodules = + vcsConfig.options[OPTION_UPDATE_NESTED_SUBMODULES]?.toBoolean() ?: true + updateSubmodules( + workingTree, + recursive = updateNestedSubmodules, + historyDepth = historyDepth + ) + } revision } @@ -187,13 +221,14 @@ class Git : VersionControlSystem(GitCommand) { private fun updateWorkingTreeWithoutSubmodules( workingTree: WorkingTree, - git: Git, - revision: String + git: JGit, + revision: String, + historyDepth: Int ): Result = runCatching { - logger.info { "Trying to fetch only revision '$revision' with depth limited to $GIT_HISTORY_DEPTH." } + logger.info { "Trying to fetch only revision '$revision' with depth limited to $historyDepth." } - val fetch = git.fetch().setDepth(GIT_HISTORY_DEPTH) + val fetch = git.fetch().setDepth(historyDepth) // See https://git-scm.com/docs/gitrevisions#_specifying_revisions for how Git resolves ambiguous // names. In particular, tag names have higher precedence than branch names. @@ -211,13 +246,13 @@ class Git : VersionControlSystem(GitCommand) { it.showStackTrace() logger.info { "Could not fetch only revision '$revision': ${it.collectMessages()}" } - logger.info { "Falling back to fetching all refs with depth limited to $GIT_HISTORY_DEPTH." } + logger.info { "Falling back to fetching all refs with depth limited to $historyDepth." } - git.fetch().setDepth(GIT_HISTORY_DEPTH).setTagOpt(TagOpt.FETCH_TAGS).call() + git.fetch().setDepth(historyDepth).setTagOpt(TagOpt.FETCH_TAGS).call() }.recoverCatching { it.showStackTrace() - logger.info { "Could not fetch with only a depth of $GIT_HISTORY_DEPTH: ${it.collectMessages()}" } + logger.info { "Could not fetch with only a depth of $historyDepth: ${it.collectMessages()}" } logger.info { "Falling back to fetch everything including tags." } git.fetch().setUnshallow(true).setTagOpt(TagOpt.FETCH_TAGS).call() @@ -272,7 +307,14 @@ class Git : VersionControlSystem(GitCommand) { revision } - private fun updateSubmodules(workingTree: WorkingTree) { + /** + * Initialize, update, and clone all the submodules in a working tree. + * + * If [recursive] is set to true, then the operations are not only performed on the + * submodules in the top-level of the working tree, but also on the submodules of the submodules, and so on. + * If [recursive] is set to false, only the submodules on the top-level are initialized, updated, and cloned. + */ + private fun updateSubmodules(workingTree: WorkingTree, recursive: Boolean, historyDepth: Int) { if (!workingTree.getRootPath().resolve(".gitmodules").isFile) return val insteadOf = REPOSITORY_URL_PREFIX_REPLACEMENTS.map { (prefix, replacement) -> @@ -281,14 +323,27 @@ class Git : VersionControlSystem(GitCommand) { runCatching { // TODO: Migrate this to JGit once https://bugs.eclipse.org/bugs/show_bug.cgi?id=580731 is implemented. - workingTree.runGit("submodule", "update", "--init", "--recursive", "--depth", "$GIT_HISTORY_DEPTH") + val updateArgs = mutableListOf("submodule", "update", "--init", "--depth", "$historyDepth").apply { + if (recursive) { add("--recursive") } + } + + workingTree.runGit(*updateArgs.toTypedArray()) insteadOf.forEach { - workingTree.runGit("submodule", "foreach", "--recursive", "git config $it") + val foreachArgs = mutableListOf("submodule", "foreach").apply { + if (recursive) { add("--recursive") } + add("git config $it") + } + + workingTree.runGit(*foreachArgs.toTypedArray()) } }.recover { // As Git's dumb HTTP transport does not support shallow capabilities, also try to not limit the depth. - workingTree.runGit("submodule", "update", "--recursive") + val fallbackArgs = mutableListOf("submodule", "update").apply { + if (recursive) { add("--recursive") } + } + + workingTree.runGit(*fallbackArgs.toTypedArray()) } } diff --git a/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt b/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt index 26cdcb7e28498..99bc1b121860e 100644 --- a/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt +++ b/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt @@ -33,11 +33,14 @@ import org.apache.logging.log4j.kotlin.logger import org.eclipse.jgit.lib.SymbolicRef import org.ossreviewtoolkit.downloader.VersionControlSystem +import org.ossreviewtoolkit.downloader.VersionControlSystemFactory import org.ossreviewtoolkit.downloader.WorkingTree import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.model.utils.parseRepoManifestPath import org.ossreviewtoolkit.utils.common.CommandLineTool +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.ProcessCapture import org.ossreviewtoolkit.utils.common.collectMessages @@ -86,11 +89,22 @@ object GitRepoCommand : CommandLineTool { override fun displayName(): String = "GitRepo" } -class GitRepo : VersionControlSystem(GitRepoCommand) { +@Suppress("UnusedPrivateProperty") +class GitRepo internal constructor() : VersionControlSystem(GitRepoCommand) { + override val type = VcsType.GIT_REPO.toString() - override val priority = 50 override val latestRevisionNames = listOf("HEAD", "@") + class Factory : VersionControlSystemFactory(VcsType.GIT_REPO.toString(), 50) { + override fun create(config: VersionControlSystemConfiguration): VersionControlSystem { + return GitRepo() + } + + override fun parseConfig(options: Options, secrets: Options): VersionControlSystemConfiguration { + TODO("Not yet implemented") + } + } + override fun getVersion() = GitRepoCommand.getVersion(null) override fun getDefaultBranchName(url: String): String { @@ -136,7 +150,8 @@ class GitRepo : VersionControlSystem(GitRepoCommand) { paths.forEach { path -> // Add the nested Repo project. - val workingTree = Git().getWorkingTree(getRootPath().resolve(path)) + val workingTree = Git.Factory().create(VersionControlSystemConfiguration()) + .getWorkingTree(getRootPath().resolve(path)) nested[path] = workingTree.getInfo() // Add the Git submodules of the nested Repo project. diff --git a/plugins/version-control-systems/git/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem b/plugins/version-control-systems/git/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem deleted file mode 100644 index afa98de81107d..0000000000000 --- a/plugins/version-control-systems/git/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem +++ /dev/null @@ -1,2 +0,0 @@ -org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git -org.ossreviewtoolkit.plugins.versioncontrolsystems.git.GitRepo diff --git a/plugins/version-control-systems/git/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory b/plugins/version-control-systems/git/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory new file mode 100644 index 0000000000000..cbf1e1ad58dec --- /dev/null +++ b/plugins/version-control-systems/git/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory @@ -0,0 +1,2 @@ +org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git$Factory +org.ossreviewtoolkit.plugins.versioncontrolsystems.git.GitRepo$Factory diff --git a/plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt b/plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt index 613ab0e117b1b..711ddf3227118 100644 --- a/plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt +++ b/plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt @@ -24,10 +24,13 @@ import java.io.File import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.downloader.VersionControlSystem +import org.ossreviewtoolkit.downloader.VersionControlSystemFactory import org.ossreviewtoolkit.downloader.WorkingTree import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.utils.common.CommandLineTool +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.ProcessCapture const val MERCURIAL_LARGE_FILES_EXTENSION = "largefiles = " @@ -47,11 +50,22 @@ object MercurialCommand : CommandLineTool { override fun displayName(): String = "Mercurial" } -class Mercurial : VersionControlSystem(MercurialCommand) { +@Suppress("UnusedPrivateProperty") +class Mercurial internal constructor() : VersionControlSystem(MercurialCommand) { + override val type = VcsType.MERCURIAL.toString() - override val priority = 20 override val latestRevisionNames = listOf("tip") + class Factory : VersionControlSystemFactory(VcsType.MERCURIAL.toString(), 20) { + override fun create(config: VersionControlSystemConfiguration): VersionControlSystem { + return Mercurial() + } + + override fun parseConfig(options: Options, secrets: Options): VersionControlSystemConfiguration { + TODO("Not yet implemented") + } + } + override fun getVersion() = MercurialCommand.getVersion(null) override fun getDefaultBranchName(url: String) = "default" diff --git a/plugins/version-control-systems/mercurial/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem b/plugins/version-control-systems/mercurial/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory similarity index 81% rename from plugins/version-control-systems/mercurial/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem rename to plugins/version-control-systems/mercurial/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory index 844294e265439..df59e625125df 100644 --- a/plugins/version-control-systems/mercurial/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem +++ b/plugins/version-control-systems/mercurial/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory @@ -1 +1 @@ -org.ossreviewtoolkit.plugins.versioncontrolsystems.mercurial.Mercurial +org.ossreviewtoolkit.plugins.versioncontrolsystems.mercurial.Mercurial$Factory diff --git a/plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt b/plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt index 5a4a217fb32f8..46119dd1d7a0c 100644 --- a/plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt +++ b/plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt @@ -28,9 +28,12 @@ import java.nio.file.Paths import org.apache.logging.log4j.kotlin.logger import org.ossreviewtoolkit.downloader.VersionControlSystem +import org.ossreviewtoolkit.downloader.VersionControlSystemFactory import org.ossreviewtoolkit.downloader.WorkingTree import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration +import org.ossreviewtoolkit.utils.common.Options import org.ossreviewtoolkit.utils.common.collectMessages import org.ossreviewtoolkit.utils.ort.OrtAuthenticator import org.ossreviewtoolkit.utils.ort.OrtProxySelector @@ -52,16 +55,27 @@ import org.tmatesoft.svn.core.wc.SVNClientManager import org.tmatesoft.svn.core.wc.SVNRevision import org.tmatesoft.svn.util.Version -class Subversion : VersionControlSystem() { +@Suppress("UnusedPrivateProperty") +class Subversion internal constructor() : VersionControlSystem() { + private val ortAuthManager = OrtSVNAuthenticationManager() private val clientManager = SVNClientManager.newInstance().apply { setAuthenticationManager(ortAuthManager) } override val type = VcsType.SUBVERSION.toString() - override val priority = 10 override val latestRevisionNames = listOf("HEAD") + class Factory : VersionControlSystemFactory(VcsType.SUBVERSION.toString(), 10) { + override fun create(config: VersionControlSystemConfiguration): VersionControlSystem { + return Subversion() + } + + override fun parseConfig(options: Options, secrets: Options): VersionControlSystemConfiguration { + TODO("Not yet implemented") + } + } + override fun getVersion(): String = Version.getVersionString() override fun getDefaultBranchName(url: String) = "trunk" diff --git a/plugins/version-control-systems/subversion/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem b/plugins/version-control-systems/subversion/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory similarity index 79% rename from plugins/version-control-systems/subversion/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem rename to plugins/version-control-systems/subversion/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory index 4eee8dd95ed90..0e8aaafe45dce 100644 --- a/plugins/version-control-systems/subversion/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystem +++ b/plugins/version-control-systems/subversion/src/main/resources/META-INF/services/org.ossreviewtoolkit.downloader.VersionControlSystemFactory @@ -1 +1 @@ -org.ossreviewtoolkit.plugins.versioncontrolsystems.subversion.Subversion +org.ossreviewtoolkit.plugins.versioncontrolsystems.subversion.Subversion$Factory diff --git a/utils/common/src/funTest/kotlin/SafeDeleteRecursivelyFunTest.kt b/utils/common/src/funTest/kotlin/SafeDeleteRecursivelyFunTest.kt index 4c3a6e14e223b..52317eee0d504 100644 --- a/utils/common/src/funTest/kotlin/SafeDeleteRecursivelyFunTest.kt +++ b/utils/common/src/funTest/kotlin/SafeDeleteRecursivelyFunTest.kt @@ -29,6 +29,7 @@ import java.io.IOException import org.ossreviewtoolkit.model.Package import org.ossreviewtoolkit.model.VcsInfo import org.ossreviewtoolkit.model.VcsType +import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git class SafeDeleteRecursivelyFunTest : WordSpec({ @@ -58,7 +59,8 @@ class SafeDeleteRecursivelyFunTest : WordSpec({ ) val nodeDir = tempdir().resolve("node-dir") - Git().download(pkg, nodeDir) + Git.Factory().create(VersionControlSystemConfiguration()) + .download(pkg, nodeDir) shouldNotThrow { nodeDir.safeDeleteRecursively()