feat(vcs): Add Git-specific configuration options for submodule handling

For large repositories with many layers of nested Git submodules, the download process can be very time-consuming and often results in duplicate projects in the tree of nested submodules. This feature introduces configuration options to limit the recursive checkout of nested Git submodules to the first layer, optimizing performance and reducing redundancy. Additionally, it also allows to limit the depth of commit history to fetch when downloading the projects. Signed-off-by: Wolfgang Klenk <[email protected]>
oss-review-toolkit · Nov 11, 2024 · c66ccf6 · c66ccf6
1 parent 9370898
commit c66ccf6
Show file tree

Hide file tree

Showing 8 changed files with 49 additions and 21 deletions.
diff --git a/integrations/schemas/ort-configuration-schema.json b/integrations/schemas/ort-configuration-schema.json
@@ -89,6 +89,28 @@
                     "items": {
                         "$ref": "#/definitions/SourceCodeOrigins"
                     }
+                },
+                "versionControlSystems": {
+                    "type": "object",
+                    "properties": {
+                        "Git": {
+                            "type": "object",
+                            "properties": {
+                                "options": {
+                                    "type": "object",
+                                    "properties": {
+                                        "submoduleHistoryDepth": {
+                                            "type": "integer",
+                                            "minimum": 1
+                                        },
+                                        "updateNestedSubmodules": {
+                                            "type": "boolean"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
                 }
             }
         },

diff --git a/plugins/version-control-systems/git/src/funTest/kotlin/GitDownloadFunTest.kt b/plugins/version-control-systems/git/src/funTest/kotlin/GitDownloadFunTest.kt
@@ -43,7 +43,7 @@ private const val REPO_REV_FOR_VERSION = "371b23f37da064687518bace268d607a92ecbe
 private const val REPO_PATH_FOR_VERSION = "specs"
 
 class GitDownloadFunTest : StringSpec() {
-    private val git = Git()
+    private val git = Git.create()
     private lateinit var outputDir: File
 
     override suspend fun beforeTest(testCase: TestCase) {

diff --git a/plugins/version-control-systems/git/src/funTest/kotlin/GitFunTest.kt b/plugins/version-control-systems/git/src/funTest/kotlin/GitFunTest.kt
@@ -44,7 +44,7 @@ private val tags = mapOf(
 )
 
 class GitFunTest : WordSpec({
-    val git = Git()
+    val git = Git.create()
     val vcsInfo = VcsInfo(
         type = VcsType.GIT,
         url = "https://github.com/oss-review-toolkit/ort-test-data-git.git",

diff --git a/plugins/version-control-systems/git/src/funTest/kotlin/GitWorkingTreeFunTest.kt b/plugins/version-control-systems/git/src/funTest/kotlin/GitWorkingTreeFunTest.kt
@@ -34,7 +34,7 @@ import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
 
 class GitWorkingTreeFunTest : StringSpec({
-    val git = Git()
+    val git = Git.create()
     val repoDir = tempdir()
     val vcsInfo = VcsInfo(
         type = VcsType.GIT,

diff --git a/plugins/version-control-systems/git/src/main/kotlin/Git.kt b/plugins/version-control-systems/git/src/main/kotlin/Git.kt
@@ -45,17 +45,11 @@ import org.eclipse.jgit.transport.sshd.ServerKeyDatabase
 import org.eclipse.jgit.transport.sshd.SshdSessionFactory
 
 import org.ossreviewtoolkit.downloader.VersionControlSystem
-import org.ossreviewtoolkit.downloader.VersionControlSystemFactory
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
 import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
-import org.ossreviewtoolkit.utils.common.CommandLineTool
-import org.ossreviewtoolkit.utils.common.Options
-import org.ossreviewtoolkit.utils.common.Os
-import org.ossreviewtoolkit.utils.common.collectMessages
-import org.ossreviewtoolkit.utils.common.safeMkdirs
-import org.ossreviewtoolkit.utils.common.withoutPrefix
+import org.ossreviewtoolkit.utils.common.*
 import org.ossreviewtoolkit.utils.ort.requestPasswordAuthentication
 import org.ossreviewtoolkit.utils.ort.showStackTrace
 
@@ -97,10 +91,9 @@ object GitCommand : CommandLineTool {
     override fun displayName(): String = "Git"
 }
 
-class Git internal constructor(
-    private val vcsConfig: VersionControlSystemConfiguration = VersionControlSystemConfiguration()
-) : VersionControlSystem(GitCommand) {
+class Git private constructor(val vcsConfig: VersionControlSystemConfiguration): VersionControlSystem(GitCommand) {
     companion object {
+        const val VCS_NAME = "Git"
 
         init {
             // Make sure that JGit uses the exact same authentication information as ORT itself. This addresses
@@ -130,19 +123,26 @@ class Git internal constructor(
 
             SshSessionFactory.setInstance(sessionFactory)
         }
+
+        // Use the factory to create a new instance of Git (convenience function)
+        fun create() : Git {
+            return Factory().create(VersionControlSystemConfiguration()) as Git
+        }
     }
 
     override val type = VcsType.GIT.toString()
+    override val priority = 100
     override val latestRevisionNames = listOf("HEAD", "@")
 
-    class Factory : VersionControlSystemFactory(VcsType.GIT.toString(), 100) {
+    class Factory : VersionControlSystemFactory<VersionControlSystemConfiguration>(VCS_NAME) {
         override fun create(config: VersionControlSystemConfiguration): VersionControlSystem {
             return Git(config)
         }
 
         override fun parseConfig(options: Options, secrets: Options): VersionControlSystemConfiguration {
             TODO("Not yet implemented")
         }
+
     }
 
     override fun getVersion() = GitCommand.getVersion(null)

diff --git a/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt b/plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt
@@ -90,14 +90,22 @@ object GitRepoCommand : CommandLineTool {
 }
 
 @Suppress("UnusedPrivateProperty")
-class GitRepo internal constructor() : VersionControlSystem(GitRepoCommand) {
+class GitRepo private constructor(private val vcsConfig: VersionControlSystemConfiguration) :
+    VersionControlSystem(GitRepoCommand) {
+
+    companion object {
+        // Use the factory to create a new instance of GitRepo (convenience function)
+        fun create(): GitRepo {
+            return Factory().create(VersionControlSystemConfiguration()) as GitRepo
+        }
+    }
 
     override val type = VcsType.GIT_REPO.toString()
     override val latestRevisionNames = listOf("HEAD", "@")
 
     class Factory : VersionControlSystemFactory(VcsType.GIT_REPO.toString(), 50) {
         override fun create(config: VersionControlSystemConfiguration): VersionControlSystem {
-            return GitRepo()
+            return GitRepo(config)
         }
 
         override fun parseConfig(options: Options, secrets: Options): VersionControlSystemConfiguration {
@@ -150,8 +158,7 @@ class GitRepo internal constructor() : VersionControlSystem(GitRepoCommand) {
 
                     paths.forEach { path ->
                         // Add the nested Repo project.
-                        val workingTree = Git.Factory().create(VersionControlSystemConfiguration())
-                            .getWorkingTree(getRootPath().resolve(path))
+                        val workingTree = Git.create().getWorkingTree(getRootPath().resolve(path))
                         nested[path] = workingTree.getInfo()
 
                         // Add the Git submodules of the nested Repo project.

diff --git a/plugins/version-control-systems/git/src/test/kotlin/GitTest.kt b/plugins/version-control-systems/git/src/test/kotlin/GitTest.kt
@@ -46,7 +46,7 @@ import org.ossreviewtoolkit.utils.ort.requestPasswordAuthentication
 
 class GitTest : WordSpec({
     // Make sure that the initialization logic runs.
-    val git = Git()
+    val git = Git.create()
 
     var originalCredentialsProvider: CredentialsProvider? = null
     var originalAuthenticator: Authenticator? = null

diff --git a/utils/common/src/funTest/kotlin/SafeDeleteRecursivelyFunTest.kt b/utils/common/src/funTest/kotlin/SafeDeleteRecursivelyFunTest.kt
@@ -29,7 +29,6 @@ import java.io.IOException
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
-import org.ossreviewtoolkit.model.config.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.plugins.versioncontrolsystems.git.Git
 
 class SafeDeleteRecursivelyFunTest : WordSpec({
@@ -59,7 +58,7 @@ class SafeDeleteRecursivelyFunTest : WordSpec({
             )
 
             val nodeDir = tempdir().resolve("node-dir")
-            Git.Factory().create(VersionControlSystemConfiguration()).download(pkg, nodeDir)
+            Git.create().download(pkg, nodeDir)
 
             shouldNotThrow<IOException> {
                 nodeDir.safeDeleteRecursively()