Add pre-commit

.github/workflows/ci.yml

@@ -10,8 +10,60 @@ env:
   graalvm_version: '26-ea'
 
 jobs:
-  build:
+  # This is done as one single separate step to work around rate limits for the API
+  maven_bundle_url:
+    runs-on: ubuntu-latest
+    outputs:
+      maven_bundle_url: ${{ steps.set_url.outputs.maven_bundle_url }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Java 21
+        id: java21
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 21
+
+      - name: Set Maven Bundle URL
+        id: set_url
+        run: echo "maven_bundle_url=$(./scripts/maven-bundle-url.sh | tail -n 1)" >> $GITHUB_OUTPUT
+        shell: bash
+
+  pre_commit:
     runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Java 21
+        id: java21
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 21
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install pre-commit
+        run: python -m pip install --upgrade pip pre-commit
+        shell: bash
+
+      - name: run pre-commit
+        run: pre-commit run --all-files
+        shell: bash
+
+  build:
+    runs-on: ${{ matrix.os }}
+    # pre_commit not needed, but is in 'needs' to avoid wasting CI cycles in case it doesn't pass
+    needs: [maven_bundle_url, pre_commit]
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -25,6 +77,7 @@ jobs:
 
       - name: Set GRADLE_JAVA_HOME
         run: echo "GRADLE_JAVA_HOME=${{ steps.java21.outputs.path }}" >> $GITHUB_ENV
+        shell: bash
 
       - name: Set up GraalVM
         uses: graalvm/setup-graalvm@v1
@@ -33,31 +86,33 @@ jobs:
           java-version: ${{ env.graalvm_version }}
 
       - name: Setup Polyglot Maven bundle
-        run: ./scripts/setup-maven-bundle.sh "${{ github.workspace }}/maven-resource-bundle"
+        run: ./scripts/maven-bundle-setup.sh "${{ github.workspace }}/maven-resource-bundle" "${{ needs.maven_bundle_url.outputs.maven_bundle_url }}"
+        shell: bash
 
       - name: Build project (skip tests)
         run: mvn --batch-mode -DskipTests -s settings.xml -Dgradle.java.home=$GRADLE_JAVA_HOME package verify
-
-      - name: Archive Maven build outputs (all targets)
-        # We need all "target" directories from submodules
-        run: |
-          mkdir build-artifacts
-          find . -type d -name target | tar -czf build-artifacts/all-targets.tgz --files-from=-
         shell: bash
 
       - name: Upload build artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: build-artifacts
-          path: build-artifacts/all-targets.tgz
+          name: build-artifacts-${{ runner.os }}
+          path: ./**/target/**
 
-  integration-tests:
-    needs: build
-    runs-on: ubuntu-latest
+  integration:
+    needs: [build, maven_bundle_url]
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: [ubuntu-latest, macos-latest] # TODO: windows-latest
         test_name: [ test_jbang_integration.py, test_maven_plugin.py, test_gradle_plugin.py ]
-
+        include:
+          # We need to skip native-image tests on MacOS runners, because they do not have enough
+          # RAM to reliably build a native image of GraalPy from Gradle (somehow Maven native-image
+          # tests seems to work, but we may need to disable them in the future too)
+          - test_name: test_gradle_plugin.py
+            os: macos-latest
+            extra_test_args: "--skip-native-image"
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -74,13 +129,9 @@ jobs:
       - name: Download build artifacts
         uses: actions/download-artifact@v4
         with:
-          name: build-artifacts
+          name: build-artifacts-${{ runner.os }}
           path: .
 
-      - name: Extract build artifacts
-        run: |
-          tar -xzf all-targets.tgz
-
       - name: Setup Java 21
         id: java21
         uses: actions/setup-java@v4
@@ -90,6 +141,7 @@ jobs:
 
       - name: Set GRADLE_JAVA_HOME
         run: echo "GRADLE_JAVA_HOME=${{ steps.java21.outputs.path }}" >> $GITHUB_ENV
+        shell: bash
 
       - name: Set up GraalVM
         uses: graalvm/setup-graalvm@v1
@@ -98,59 +150,63 @@ jobs:
           java-version: ${{ env.graalvm_version }}
 
       - name: Setup Polyglot Maven bundle
-        run: ./scripts/setup-maven-bundle.sh "${{ github.workspace }}/maven-resource-bundle"
+        run: ./scripts/maven-bundle-setup.sh "${{ github.workspace }}/maven-resource-bundle" "${{ needs.maven_bundle_url.outputs.maven_bundle_url }}"
+        shell: bash
 
       - name: Run Maven install
         run: mvn --batch-mode -s settings.xml -Dgradle.java.home=$GRADLE_JAVA_HOME -DskipTests install
+        shell: bash
 
-      - name: Install JBang
-        if: matrix.test_name == 'test_jbang_integration.py'
+      - name: Install JBang (Linux/macOS)
+        if: runner.os != 'Windows' && matrix.test_name == 'test_jbang_integration.py'
         run: |
           curl -Ls https://sh.jbang.dev | bash
           echo "$HOME/.jbang/bin" >> $GITHUB_PATH
+        shell: bash
+
+      - name: Install JBang (Windows)
+        if: runner.os == 'Windows' && matrix.test_name == 'test_jbang_integration.py'
+        run: choco install jbang
+        shell: pwsh
 
       - name: Run integration tests
         run: |
-          mvn --batch-mode -s settings.xml -Dgradle.java.home=$GRADLE_JAVA_HOME exec:java@integration-tests \
-              -Dintegration.tests.args="${{ matrix.test_name }} $TEST_FLAGS"
+          mvn --batch-mode -s settings.xml -Dgradle.java.home=$GRADLE_JAVA_HOME exec:java@integration-tests -Dintegration.tests.args="${{ matrix.test_name }} ${{ matrix.extra_test_args }} $TEST_FLAGS"
+        shell: bash
 
   unit-tests:
-    needs: build
-    runs-on: ubuntu-latest
+    needs: [build, maven_bundle_url]
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: [ubuntu-latest, macos-latest] # TODO: windows-latest
+        tests: [unit-cp, unit-mp, unit-it-mp, isolate-it]
+        exclude: # TODO: find out why the resources are not in the bundle (the same for Windows)
+          - os: macos-latest
+            tests: isolate-it
         include:
-          - job_name: "unit"
+          - tests: unit-cp
             maven_args: "test"
-            display: "All Unit Tests"
-          - job_name: "unit-modulepath"
+          - tests: unit-mp
             maven_args: >-
               -pl org.graalvm.python.embedding -Ptest.modules -am test
-            display: "Embedding Unit Tests Using Modulepath"
-          - job_name: "integration-modulepath"
+          - tests: unit-it-mp
             maven_args: >-
               -pl org.graalvm.python.embedding -Ptest.modules.integration -Dtest=org.graalvm.python.embedding.test.integration.** -am test
-            display: "Embedding Integration Unit Tests Using Modulepath"
-          - job_name: "isolate-integration"
+          - tests: isolate-it
             maven_args: >-
               -pl org.graalvm.python.embedding -Pisolate -Dpolyglot.engine.AllowExperimentalOptions=true -Dpolyglot.engine.SpawnIsolate=true -Dpolyglot.engine.IsolateMode=external -Dtest=org.graalvm.python.embedding.test.integration.** -am test
-            display: "Isolated Embedding Integration Unit Tests"
-
-    name: ${{ matrix.display }}
+    name: "${{ matrix.tests }} (${{ matrix.os }})"
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
 
       - name: Download build artifacts
         uses: actions/download-artifact@v4
         with:
-          name: build-artifacts
+          name: build-artifacts-${{ runner.os }}
           path: .
 
-      - name: Extract build artifacts
-        run: |
-          tar -xzf all-targets.tgz
-
       - name: Setup Java 21
         id: java21
         uses: actions/setup-java@v4
@@ -166,12 +222,16 @@ jobs:
 
       - name: Set GRADLE_JAVA_HOME
         run: echo "GRADLE_JAVA_HOME=${{ steps.java21.outputs.path }}" >> $GITHUB_ENV
+        shell: bash
 
       - name: Setup Polyglot Maven bundle
-        run: ./scripts/setup-maven-bundle.sh "${{ github.workspace }}/maven-resource-bundle"
+        run: ./scripts/maven-bundle-setup.sh "${{ github.workspace }}/maven-resource-bundle" "${{ needs.maven_bundle_url.outputs.maven_bundle_url }}"
+        shell: bash
 
       - name: Run Maven install
         run: mvn --batch-mode -s settings.xml -Dgradle.java.home=$GRADLE_JAVA_HOME -DskipTests install
+        shell: bash
 
       - name: Run Maven tests
-        run: mvn --batch-mode -s settings.xml -Dgradle.java.home=$GRADLE_JAVA_HOME ${{ matrix.maven_args }}
+        run: mvn --batch-mode -s settings.xml -Dgradle.java.home=$GRADLE_JAVA_HOME ${{ matrix.maven_args }}
+        shell: bash

.github/workflows/javainterfacegen.yml

@@ -38,4 +38,4 @@ jobs:
         with:
           name: javainterfacegen-${{ matrix.os_name }}-${{ env.ARCH }}
           path: |
-            javainterfacegen/target/javainterfacegen
+            javainterfacegen/target/javainterfacegen

.github/workflows/tests-release-jbang.yml

@@ -25,4 +25,4 @@ jobs:
           mvn --batch-mode exec:java@integration-tests \
               -Dproject.polyglot.version=24.1.2 \
               -Dintegration.tests.args="test_jbang_integration.py \
-              --jbang-graalpy-version=24.2.1"
+              --jbang-graalpy-version=24.2.1"

.gitignore

@@ -14,4 +14,4 @@ __pycache__
 *.swp
 .flattened-pom.xml
 settings.xml
-maven-bundle-*
+/maven-bundle-*

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+wrapperVersion=3.3.2
+distributionType=only-script
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.10/apache-maven-3.9.10-bin.zip

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: check-xml
+    -   id: check-executables-have-shebangs
+    -   id: check-shebang-scripts-are-executable
+    -   id: check-symlinks
+    -   id: mixed-line-ending
+        args: ['--fix=no']
+-   repo: local
+    hooks:
+    -   id: maven-validate-spotless
+        name: Maven validate & spotless:apply
+        entry: ./mvnw
+        args: ['validate', 'spotless:apply']
+        language: system
+        pass_filenames: false

DEVELOPMENT.md

@@ -1,3 +1,14 @@
+## Building for a Release
+
+* Verify that the following properties in `pom.xml` match the expected versions for the release:
+  * `revision`: the version for graalpy-extensions artifacts, e.g., `org.graalvm.python.embedding`
+  * `project.polyglot.version`: version of the polyglot artifacts to use, e.g., the version of `org.graalvm.polyglot:polyglot` dependency
+* The polyglot artifacts of given version must be available in Maven central or some additional
+  Maven repository that can be configured, for example, using Maven's setting.xml mechanism
+* Run Maven with `-P release` to sign the artifacts
+* Note: the files that are expected to be deployed include usual: `*.jar`, `*.asc`, `*.pom`,
+  and in case of `org.graalvm.python.embedding` also `*.sigfile`, which is used for API stability checking
+
 ## How to develop against the latest GraalPy, GraalVM SDK, and Truffle:
 
 ### Option 1: Use pre-built Maven bundle
@@ -67,9 +78,9 @@ mvn install exec:java@integration-tests -Dintegration.test.args="test_maven_plug
 ```
 
 The integration tests are driven by Python and implemented using unittest framework, which is
-part of Python standard library. The tests expect that the relevant Maven artifacts are available, 
+part of Python standard library. The tests expect that the relevant Maven artifacts are available,
 which can be achieved, for example, by `mvn install`. But they can also run on released artifacts
-published in Mavencentral or some snapshot repository configured in Maven settings. 
+published in Mavencentral or some snapshot repository configured in Maven settings.
 
 The whole execution of the tests is wrapped in Maven goal `exec:java@integration-tests`, which passes
 some necessary arguments to the test driver Python script. One can pass additional arguments for the
@@ -80,4 +91,4 @@ verbosity level.
 ## Changing version
 
 - property `revision` in top level `pom.xml`
-- property `graalpy.version` in `graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml` (TODO: propagate from revision)
+- property `graalpy.version` in `graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/pom.xml` (TODO: propagate from revision)

graalpy-archetype-polyglot-app/src/main/resources/META-INF/maven/archetype-metadata.xml

@@ -24,4 +24,4 @@
       </includes>
     </fileSet>
   </fileSets>
-</archetype-descriptor>
+</archetype-descriptor>

graalpy-archetype-polyglot-app/src/main/resources/archetype-resources/src/main/java/GraalPy.java

@@ -61,7 +61,7 @@ public static void main(String[] args) {
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
-            
+
             context.eval(source);
 
             // retrieve the python PyHello class
@@ -70,7 +70,7 @@ public static void main(String[] args) {
             // and cast it to the Hello interface which matches PyHello
             Hello hello = pyHello.as(Hello.class);
             hello.hello("java");
-            
+
         } catch (PolyglotException e) {
             if (e.isExit()) {
                 System.exit(e.getExitStatus());

graalpy-maven-plugin/pom.xml

@@ -76,12 +76,7 @@ SOFTWARE.
       <groupId>${project.groupId}</groupId>
       <artifactId>python-embedding-tools</artifactId>
       <version>${project.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.graalvm.python</groupId>
-      <artifactId>python-launcher</artifactId>
-      <version>${project.polyglot.version}</version>
-      <scope>runtime</scope>
+      <scope>compile</scope>
     </dependency>
   </dependencies>
 </project>

integration-tests/gradle/gradle-test-project/gradle.properties

@@ -1,4 +1,4 @@
 # the jdk used to run the gradle demon has to be <= 22
 # this will be replaced by the integration tests:
 # pass -Dgradle.java.home to Maven or --gradle-java-home to the Python test driver
-org.gradle.java.home={GRADLE_JAVA_HOME}
+org.gradle.java.home={GRADLE_JAVA_HOME}

integration-tests/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties

@@ -4,4 +4,4 @@ distributionUrl=https://services.gradle.org/distributions/gradle-8.9-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
+zipStorePath=wrapper/dists