Use published conformance tests (#126)

Use the published JSpecify artifacts instead of requiring a local clone
of https://github.com/jspecify/jspecify. Allow the user to include a
local clone with `--included-build path/to/jspecify`, and use those
artifacts instead in that case.

Part of #107.

---------

Co-authored-by: Werner Dietl <[email protected]>

Author: netdpb

.github/workflows/build.yml

@@ -10,10 +10,15 @@ jobs:
       run:
         working-directory: jspecify-reference-checker
     steps:
-      - name: Check out the code
+      - name: Check out jspecify-reference checker
         uses: actions/checkout@v4
         with:
           path: jspecify-reference-checker
+      - name: Check out jspecify
+        uses: actions/checkout@v4
+        with:
+          repository: jspecify/jspecify
+          path: jspecify
       - name: Set up Java
         uses: actions/setup-java@v4
         with:
@@ -22,13 +27,15 @@ jobs:
       - name: Set up Gradle
         uses: gradle/gradle-build-action@v2
       - name: Build and Test
-        run: ./gradlew build conformanceTests demoTest
+        run: ./gradlew build conformanceTests demoTest --include-build ../jspecify
         env:
           SHALLOW: 1
       - name: Check out jspecify/samples-google-prototype
         if: always()
-        run: git checkout samples-google-prototype
+        run: |
+          git fetch --depth=1 origin samples-google-prototype
+          git checkout samples-google-prototype
         working-directory: jspecify
       - name: Run Samples Tests
         if: always()
-        run: ./gradlew jspecifySamplesTest
+        run: ./gradlew jspecifySamplesTest --include-build ../jspecify

README.md

@@ -58,20 +58,26 @@ git clone https://github.com/jspecify/jspecify-reference-checker
 cd jspecify-reference-checker
 ```
 
-Now build it, which will retrieve a lot of other code too, and will take 10-15 minutes:
+Now build it, which will retrieve a lot of other code too, and will take 10-15
+minutes:
 
 ```sh
 ./gradlew assemble
 ```
 
-Now try the sample file:
+### Demonstration
+
+Run the checker on the sample file:
 
 ```sh
 cd $root_dir/jspecify-reference-checker
 ./demo SimpleSample.java
 ```
 
-After ~10 seconds delay, you should see
+(If you haven't [built the reference checker](#building) yet, this will build it
+the first time you run it.)
+
+After that, you should see:
 
 ```
 SimpleSample.java:7: error: [nullness] incompatible types in return.

build.gradle

@@ -10,17 +10,32 @@ plugins {
 }
 
 repositories {
+    mavenLocal()
+    maven {
+        // Nexus snapshot repository
+        url = "https://s01.oss.sonatype.org/content/repositories/snapshots/"
+    }
     mavenCentral()
 }
 
-configurations {
-    errorproneJavac
-    conformanceTestSuite
-}
-
 ext {
     checkerFramework = gradle.includedBuild("checker-framework")
-    jspecify = gradle.includedBuild("jspecify")
+
+    // null if not included with `--include-build path/to/jspecify`
+    jspecify = gradle.includedBuilds.find { it.name == 'jspecify' }
+}
+
+configurations {
+    errorproneJavac
+    conformanceTestSuite {
+        // When including a local JSpecify build, depend on the local test suite.
+        // See https://docs.gradle.org/current/userguide/cross_project_publications.html#sec:variant-aware-sharing.
+        if (jspecify != null) {
+            attributes {
+                attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, 'conformance-tests'))
+            }
+        }
+    }
 }
 
 java {
@@ -41,16 +56,20 @@ dependencies {
     testImplementation libs.jspecify.conformanceTestFramework
     testRuntimeOnly libs.jsr305 // jsr305 annotations are in some of the samples
 
-    // TODO: Depend on a group:artifact:version rather than a file.
-    conformanceTestSuite files("${jspecify.projectDir}/conformance-tests/build/distributions/conformance-tests-0.0.0-SNAPSHOT.zip")
+    conformanceTestSuite libs.jspecify.conformanceTests
 
     errorproneJavac libs.errorProne.javac
     errorprone libs.errorProne.core
 }
 
 // Assemble checker-framework when assembling the reference checker.
 assemble.dependsOn(checkerFramework.task(":assemble"))
-assemble.dependsOn(jspecify.task(":assemble"))
+
+// If built with `--include-build path/to/jspecify` then
+// assemble jspecify when assembling the reference checker.
+if (jspecify != null) {
+    assemble.dependsOn(jspecify.task(':assemble'))
+}
 
 tasks.withType(JavaCompile).configureEach {
     options.compilerArgs.add("-Xlint:all")
@@ -110,15 +129,14 @@ test {
 tasks.register('jspecifySamplesTest', Test) {
     description = 'Run the checker against the JSpecify samples.'
     group = 'verification'
+    shouldRunAfter test
     include '**/NullSpecTest$Lenient.class'
     include '**/NullSpecTest$Strict.class'
 
     inputs.files(unzipConformanceTestSuite)
 }
 
 tasks.register('unzipConformanceTestSuite', Copy) {
-    // TODO: Don't explicitly depend on an included build's task.
-    dependsOn jspecify.task(':conformance-tests:build')
     dependsOn configurations.conformanceTestSuite
     from zipTree(configurations.conformanceTestSuite.singleFile)
     into layout.buildDirectory.dir("conformanceTests")
@@ -127,6 +145,7 @@ tasks.register('unzipConformanceTestSuite', Copy) {
 tasks.register('conformanceTests', Test) {
     group = 'verification'
     include '**/ConformanceTest.class'
+    shouldRunAfter test
 
     // Conformance tests
     inputs.files(unzipConformanceTestSuite)
@@ -140,7 +159,6 @@ tasks.register('conformanceTests', Test) {
     }
 
     // Conformance tests run on the samples directory
-    inputs.dir("${unzipConformanceTestSuite.destinationDir}/samples")
     inputs.files("tests/ConformanceTestOnSamples-report.txt")
     doFirst {
         systemProperties([
@@ -168,7 +186,7 @@ tasks.register('demoTest', Exec)  {
     ignoreExitValue = true
     errorOutput = new ByteArrayOutputStream()
     doLast {
-        if (!errorOutput.toString().startsWith("SimpleSample.java:7: error:")) {
+        if (!errorOutput.toString().contains("SimpleSample.java:7: error:")) {
             throw new AssertionError("`./demo SimpleSample.java` did not run correctly. Error output:\n$errorOutput")
         }
     }

demo

@@ -1,15 +1,32 @@
 #!/bin/sh
 # A quick and easy way to run the reference checker on some standalone code.
-# If you set the CLASSPATH environment variable it will ues it, adding its own entries to that list.
+# If you set the CLASSPATH environment variable it will use it, adding its own entries to that list.
 # To integrate the checker into a more complex build, reading the below should give you what you need to know.
 
 dir=$(dirname $0)
-ourclasspath="$dir/../jspecify/build/libs/jspecify-0.0.0-SNAPSHOT.jar:$dir/build/libs/jspecify-reference-checker.jar"
+jspecify="${dir}/../jspecify/build/libs/jspecify-0.0.0-SNAPSHOT.jar"
+if [ ! -e "${jspecify}" ]; then
+  version=0.3.0
+  jspecify="${dir}/build/jspecify-${version}.jar"
+  if [ ! -e "${jspecify}" ]; then
+    echo "Downloading $(basename "${jspecify}") from Maven central"
+    mvn -q org.apache.maven.plugins:maven-dependency-plugin:3.6.1:copy \
+      -Dartifact="org.jspecify:jspecify:${version}" \
+      -DoutputDirectory="$(dirname "${jspecify}")"
+  fi
+fi
+jspecify_reference_checker="${dir}/build/libs/jspecify-reference-checker.jar"
+if [ ! -e "${jspecify_reference_checker}" ]; then
+  echo "Assembling jspecify-reference-checker"
+  ./gradlew assemble
+fi
 
-export CLASSPATH="$ourclasspath:$CLASSPATH"
+ourclasspath="${jspecify}:${jspecify_reference_checker}"
+
+export CLASSPATH="${ourclasspath}:$CLASSPATH"
 
 $dir/../checker-framework/checker/bin/javac \
-  -processorpath "$ourclasspath" \
+  -processorpath "${ourclasspath}" \
   -processor com.google.jspecify.nullness.NullSpecChecker \
   -AcheckImpl \
   -AassumePure \

docs/development.md

@@ -0,0 +1,46 @@
+# Development
+
+## Codevelopment with Checker Framework fork
+
+This project depends on
+an [unreleased fork of the Checker Framework][jspecify-checker-framework].
+(The [main-eisop branch] represents ongoing work to depend on a released version
+of the [EISOP] fork instead.)
+
+Because of that dependency, this build clones that unreleased fork into the
+sibling directory `../checker-framwork`.
+_That_ build then clones some other projects into other sibling directories. It
+expects `../jdk` to contain an annotated JDK, so our build
+clones [JSpecify's][jspecify-jdk] there.
+
+## Codevelopment with JSpecify
+
+This project depends on two artifacts built from the main [JSpecify repo]
+[jspecify-jspecify]:
+
+1. the annotations: `org.jspecify:jspecify`
+2. the conformance test suite: `org.jspecify.conformance:conformance-tests`
+
+For each of those dependencies, developers can depend on either a published
+version (fixed or snapshot) or a local build.
+
+In order to depend on a specific commit or uncommitted state of those artifacts,
+clone the repo (or your fork) somewhere, and pass
+`--include-build path/to/jspecify` to Gradle when building. The local clone will
+be used for both the annotations and the conformance test suite.
+
+By default the reference checker depends on version `0.3.0` of the annotations,
+and version `0.0.0-SNAPSHOT` of the conformance test suite.
+
+In order to depend on a different published version of either artifact, set
+Gradle properties on the command line.
+
+* `-Porg.jspecify:jspecify=0.2.0` would change the version of the annotations.
+* `-Porg.jspecify.conformance:conformance-tests=0.3.0` would change the version
+  of the conformance test suite.
+
+[EISOP]: https://github.com/eisop/checker-framework
+[jspecify-checker-framework]: https://github.com/jspecify/checker-framework
+[jspecify-jdk]: https://github.com/jspecify/jdk
+[jspecify-jspecify]: https://github.com/jspecify/jspecify
+[main-eisop branch]: https://github.com/jspecify/jspecify-reference-checker/tree/main-eisop

gradle.properties

@@ -0,0 +1,3 @@
+org.jspecify\:jspecify=0.3.0
+org.jspecify.conformance\:conformance-test-framework=0.0.0-SNAPSHOT
+org.jspecify.conformance\:conformance-tests=0.0.0-SNAPSHOT

initialize-project

@@ -63,10 +63,6 @@ git_clone() {
   run "${git[@]}" "https://github.com/jspecify/${repo}.git" "../${repo}"
 }
 
-# Fetch all branches even when $SHALLOW is set so we can check out
-# samples-google-prototype to run samples tests.
-git_clone jspecify --no-single-branch
-
 git_clone jdk --depth 1 --single-branch
 
 git_clone checker-framework

settings.gradle

@@ -6,19 +6,11 @@ include 'conformance-test-framework'
 // See https://docs.gradle.org/current/userguide/composite_builds.html#included_build_declaring_substitutions
 includeBuild(".")
 
-def initializeProject() {
-    exec {
-        executable './initialize-project'
-    }
-}
-
-includeBuild("../jspecify") {
-    initializeProject()
+exec {
+    executable './initialize-project'
 }
 
-includeBuild("../checker-framework") {
-    initializeProject()
-}
+includeBuild("../checker-framework")
 
 dependencyResolutionManagement {
     versionCatalogs {
@@ -33,11 +25,19 @@ dependencyResolutionManagement {
             library("errorProne-core", "com.google.errorprone:error_prone_core:2.18.0")
             library("errorProne-javac", "com.google.errorprone:javac:9+181-r4173-1")
             library("guava", "com.google.guava:guava:31.1-jre")
-            library("jspecify", "org.jspecify:jspecify:0.0.0-SNAPSHOT")
-            library("jspecify-conformanceTestFramework", "org.jspecify.conformance:conformance-test-framework:0.0.0-SNAPSHOT")
+
+            // Related JSpecify project versions are specified in gradle.properties, and can be overridden on the Gradle command line.
+            library("jspecify", libraryVersion('org.jspecify:jspecify'))
+            library("jspecify-conformanceTestFramework", libraryVersion('org.jspecify.conformance:conformance-test-framework'))
+            library("jspecify-conformanceTests", libraryVersion('org.jspecify.conformance:conformance-tests'))
+
             library("jsr305", "com.google.code.findbugs:jsr305:3.0.2")
             library("junit", "junit:junit:4.12")
             library("truth", "com.google.truth:truth:1.1.3")
         }
     }
 }
+
+String libraryVersion(String groupArtifact) {
+    return "$groupArtifact:${settings[groupArtifact]}"
+}