New SPDX expression parser

gradle/libs.versions.toml

@@ -11,7 +11,6 @@ kotlinPlugin = "1.9.22"
 mavenPublishPlugin = "0.27.0"
 versionsPlugin = "0.51.0"
 
-antlr = "4.13.1"
 asciidoctorj = "2.5.11"
 asciidoctorjPdf = "2.3.12"
 clikt = "4.2.2"
@@ -80,7 +79,6 @@ plugin-graalVmNativeImage = { module = "org.graalvm.buildtools:native-gradle-plu
 plugin-kotlin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlinPlugin" }
 plugin-mavenPublish = { module = "com.vanniktech:gradle-maven-publish-plugin", version.ref = "mavenPublishPlugin" }
 
-antlr = { module = "org.antlr:antlr4", version.ref = "antlr" }
 asciidoctorj = { module = "org.asciidoctor:asciidoctorj", version.ref = "asciidoctorj" }
 asciidoctorj-pdf = { module = "org.asciidoctor:asciidoctorj-pdf", version.ref = "asciidoctorjPdf" }
 awsS3 = { module = "software.amazon.awssdk:s3", version.ref = "s3" }
@@ -124,6 +122,7 @@ kotest-assertions-core = { module = "io.kotest:kotest-assertions-core", version.
 kotest-assertions-json = { module = "io.kotest:kotest-assertions-json", version.ref = "kotest" }
 kotest-extensions-junitXml = { module = "io.kotest:kotest-extensions-junitxml", version.ref = "kotest" }
 kotest-framework-api = { module = "io.kotest:kotest-framework-api", version.ref = "kotest" }
+kotest-framework-datatest = { module = "io.kotest:kotest-framework-datatest", version.ref = "kotest" }
 kotest-framework-engine = { module = "io.kotest:kotest-framework-engine", version.ref = "kotest" }
 kotest-runner-junit5 = { module = "io.kotest:kotest-runner-junit5", version.ref = "kotest" }
 kotlinx-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version.ref = "kotlinxCoroutines" }

utils/spdx/build.gradle.kts

@@ -28,50 +28,14 @@ import java.net.URI
 val spdxLicenseListVersion: String by project
 
 plugins {
-    // Apply core plugins.
-    antlr
-
     // Apply precompiled plugins.
     id("ort-library-conventions")
 
     // Apply third-party plugins.
     alias(libs.plugins.download)
 }
 
-tasks.withType<AntlrTask>().configureEach {
-    arguments = arguments + listOf("-visitor")
-
-    doLast {
-        // Work around https://github.com/antlr/antlr4/issues/4128.
-        outputDirectory.walk()
-            .filter { it.isFile && it.extension == "java" }
-            .forEach { javaFile ->
-                val lines = javaFile.readLines()
-
-                val text = buildString {
-                    lines.mapIndexed { index, line ->
-                        val patchedLine = when {
-                            index == 0 && line.startsWith("// Generated from ") -> line.replace('\\', '/')
-                            else -> line
-                        }
-
-                        appendLine(patchedLine)
-                    }
-                }
-
-                javaFile.writeText(text)
-            }
-    }
-}
-
-sourceSets.configureEach {
-    val generateGrammarSource = tasks.named(getTaskName("generate", "GrammarSource"))
-    java.srcDir(generateGrammarSource.map { files() })
-}
-
 dependencies {
-    antlr(libs.antlr)
-
     api(libs.jackson.databind)
 
     implementation(projects.utils.commonUtils)
@@ -81,6 +45,7 @@ dependencies {
     implementation(libs.jackson.module.kotlin)
 
     testImplementation(libs.kotest.assertions.json)
+    testImplementation(libs.kotest.framework.datatest)
 }
 
 if (Authenticator.getDefault() == null) {

utils/spdx/src/main/kotlin/SpdxExpression.kt

@@ -23,11 +23,9 @@ import com.fasterxml.jackson.annotation.JsonCreator
 import com.fasterxml.jackson.databind.annotation.JsonSerialize
 import com.fasterxml.jackson.databind.ser.std.ToStringSerializer
 
-import org.antlr.v4.runtime.CharStreams
-import org.antlr.v4.runtime.CommonTokenStream
-
 import org.ossreviewtoolkit.utils.spdx.SpdxConstants.DOCUMENT_REF_PREFIX
 import org.ossreviewtoolkit.utils.spdx.SpdxConstants.LICENSE_REF_PREFIX
+import org.ossreviewtoolkit.utils.spdx.parser.SpdxExpressionParser
 
 /**
  * An SPDX expression as defined by version 2.1 of the [SPDX specification, appendix IV][1].
@@ -86,21 +84,8 @@ sealed class SpdxExpression {
          * allowed ([ALLOW_DEPRECATED][Strictness.ALLOW_DEPRECATED]) or only current license identifiers are allowed
          * ([ALLOW_CURRENT][Strictness.ALLOW_CURRENT]). Throws an [SpdxException] if the string cannot be parsed.
          */
-        fun parse(expression: String, strictness: Strictness): SpdxExpression {
-            val charStream = CharStreams.fromString(expression)
-            val lexer = SpdxExpressionLexer(charStream).apply {
-                removeErrorListeners()
-                addErrorListener(SpdxErrorListener())
-            }
-
-            val tokenStream = CommonTokenStream(lexer)
-            val parser = SpdxExpressionParser(tokenStream).apply {
-                removeErrorListeners()
-                addErrorListener(SpdxErrorListener())
-            }
-
-            return SpdxExpressionDefaultVisitor(strictness).visit(parser.licenseExpression())
-        }
+        fun parse(expression: String, strictness: Strictness): SpdxExpression =
+            SpdxExpressionParser(expression, strictness).parse()
     }
 
     /**

utils/spdx/src/main/kotlin/parser/Exceptions.kt

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+package org.ossreviewtoolkit.utils.spdx.parser
+
+import kotlin.reflect.KClass
+
+import org.ossreviewtoolkit.utils.spdx.SpdxException
+
+/**
+ * An exception to indicate that an [SpdxExpressionLexer] error occurred.
+ */
+class SpdxExpressionLexerException(val char: Char, val position: Int) :
+    SpdxException("Unexpected character '$char' at position $position.")
+
+/**
+ * An exception to indicate that an [SpdxExpressionParser] error occurred. [token] is the unexpected token that caused
+ * the exception, if it is `null` that means the end of the input was reached unexpectedly. [expectedTokenTypes] are the
+ * expected token types, if available.
+ */
+class SpdxExpressionParserException(
+    val token: Token?,
+    vararg val expectedTokenTypes: KClass<out Token> = emptyArray()
+) : SpdxException(
+    buildString {
+        append("Unexpected token '$token'")
+
+        if (expectedTokenTypes.size == 1) {
+            append(", expected ${expectedTokenTypes.first().simpleName}")
+        } else if (expectedTokenTypes.size > 1) {
+            append(", expected one of ${expectedTokenTypes.joinToString { it.simpleName.orEmpty() }}")
+        }
+
+        append(".")
+    }
+)