Add GPULlama3.java as model provider to run on GPUs

.github/workflows/build-pull-request.yml

@@ -66,14 +66,15 @@ jobs:
                 | jq -R -s -c 'split("\n")[:-1]')
 
           # Integration tests (without the in-process embedding models)
-          # Remove JLama and Llama3 from the list
+          # Remove JLama, Llama3 and GPU Llama3 from the list
           cd integration-tests
           IT_MODULES=$( \
               find . -mindepth 2 -maxdepth 2 -type f -name 'pom.xml' -exec dirname {} \; \
                 | sed 's|^\./||' \
                 | sort -u \
                 | grep -v jlama \
                 | grep -v llama3-java \
+                | grep -v gpu-llama3 \
                 | grep -v in-process-embedding-models \
                 | jq -R -s -c 'split("\n")[:-1]')
 
@@ -143,6 +144,13 @@ jobs:
         run: |
           ./mvnw -B clean install -DskipTests -Dno-format -ntp -f model-providers/jlama/pom.xml
 
+      # Build Jlama if JDK >= 21
+      # It's not build by default as it requires Java 21+
+      - name: Build GPU Llama3 extension
+        if: ${{ matrix.java >= 21 }}
+        run: |
+          ./mvnw -B clean install -DskipTests -Dno-format -ntp -f model-providers/gpu-llama3/pom.xml
+
       # Build Llama3.java if JDK >= 22. See https://x.com/tjake/status/1849141171475399083?t=EpgVJCPLC17fCXio0FvnhA&s=19 for the reason
       - name: Build Llama3-java extension
         if: ${{ matrix.java >= 22 }}

docs/modules/ROOT/pages/includes/attributes.adoc

@@ -1,4 +1,4 @@
 :project-version: 1.3.1
-:langchain4j-version: 1.6.0
-:langchain4j-embeddings-version: 1.6.0-beta12
+:langchain4j-version: 1.8.0
+:langchain4j-embeddings-version: 1.8.0-beta15
 :examples-dir: ./../examples/

integration-tests/gpu-llama3/README.md

@@ -0,0 +1,51 @@
+### How to run the integrated tests:
+
+#### 1) Install TornadoVM:
+
+```bash
+cd ~
+git clone [email protected]:beehive-lab/TornadoVM.git
+cd ~/TornadoVM
+./bin/tornadovm-installer --jdk jdk21 --backend opencl
+source setvars.sh
+```
+
+Note that the above steps:
+- Set `TORNADOVM_SDK` environment variable to the path of the TornadoVM SDK.
+- Create the `tornado-argfile` under `~/TornadoVM` which contains all the required JVM arguments to enable TornadoVM.
+- The argfile is automatically used in Quarkus dev mode; however, in production mode, you need to manually pass the argfile to the JVM (see step 3).
+
+#### 2) Build Quarkus-langchain4j:
+
+```bash
+cd ~
+git clone [email protected]:mikepapadim/quarkus-langchain4j.git
+cd ~/quarkus-langchain4j
+git checkout gpu-llama3-integration
+mvn clean install -DskipTests
+```
+
+#### 3) Run the integrated tests:
+
+##### 3.1 Deploy the Quarkus app:
+
+```bash
+cd ~/quarkus-langchain4j/integration-tests/gpullama3
+```
+- For *dev* mode, run:
+```
+mvn quarkus:dev
+```
+
+- For *production* mode, run:
+```bash
+java @~/TornadoVM/tornado-argfile -jar target/quarkus-app/quarkus-run.jar
+```
+##### 3.2 Send requests to the Quarkus app:
+
+when quarkus is running, open a new terminal and run:
+
+```bash
+curl http://localhost:8080/chat/blocking
+```
+

integration-tests/gpu-llama3/pom.xml

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>io.quarkiverse.langchain4j</groupId>
+        <artifactId>quarkus-langchain4j-integration-tests-parent</artifactId>
+        <version>999-SNAPSHOT</version>
+    </parent>
+    <artifactId>quarkus-langchain4j-integration-test-gpu-llama3</artifactId>
+    <name>Quarkus LangChain4j - Integration Tests - GPULlama3</name>
+    <properties>
+        <skipITs>true</skipITs>
+        <maven.compiler.release>21</maven.compiler.release>
+        <quarkus.version>3.18.0</quarkus.version>
+        <!-- TornadoVM argfile path -->
+        <tornado.argfile>${env.TORNADO_SDK}/../../../tornado-argfile</tornado.argfile>
+    </properties>
+    <dependencies>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-rest-jackson</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkiverse.langchain4j</groupId>
+            <artifactId>quarkus-langchain4j-gpu-llama3</artifactId>
+            <version>999-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-junit5</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.rest-assured</groupId>
+            <artifactId>rest-assured</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-devtools-testing</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- Make sure the deployment artifact is built before executing this module -->
+        <dependency>
+            <groupId>io.quarkiverse.langchain4j</groupId>
+            <artifactId>quarkus-langchain4j-gpu-llama3-deployment</artifactId>
+            <version>999-SNAPSHOT</version>
+            <type>pom</type>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>io.quarkus</groupId>
+                <artifactId>quarkus-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>build</goal>
+                        </goals>
+                    </execution>
+                </executions>
+                <configuration>
+                    <!-- Pass tornado-argfile to dev mode -->
+                    <jvmArgs>@${tornado.argfile}</jvmArgs>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>integration-test</goal>
+                            <goal>verify</goal>
+                        </goals>
+                        <configuration>
+                            <argLine>@${tornado.argfile}</argLine>
+                            <systemPropertyVariables>
+                                <native.image.path>${project.build.directory}/${project.build.finalName}-runner</native.image.path>
+                                <java.util.logging.manager>org.jboss.logmanager.LogManager</java.util.logging.manager>
+                                <maven.home>${maven.home}</maven.home>
+                            </systemPropertyVariables>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

integration-tests/gpu-llama3/src/main/java/org/acme/example/gpullama3/chat/ChatLanguageModelResource.java

@@ -0,0 +1,22 @@
+package org.acme.example.gpullama3.chat;
+
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+
+import dev.langchain4j.model.chat.ChatModel;
+
+@Path("chat")
+public class ChatLanguageModelResource {
+
+    private final ChatModel chatModel;
+
+    public ChatLanguageModelResource(ChatModel chatModel) {
+        this.chatModel = chatModel;
+    }
+
+    @GET
+    @Path("blocking")
+    public String blocking() {
+        return chatModel.chat("When was the nobel prize for economics first awarded?");
+    }
+}

integration-tests/gpu-llama3/src/main/resources/application.properties

@@ -0,0 +1,12 @@
+quarkus.langchain4j.gpu-llama3.include-models-in-artifact=false
+
+# Configure GPULlama3
+quarkus.langchain4j.gpu-llama3.enable-integration=true
+quarkus.langchain4j.gpu-llama3.chat-model.model-name=beehive-lab/Llama-3.2-1B-Instruct-GGUF
+quarkus.langchain4j.gpu-llama3.chat-model.quantization=FP16
+quarkus.langchain4j.gpu-llama3.chat-model.temperature=0.7
+quarkus.langchain4j.gpu-llama3.chat-model.max-tokens=513
+
+# other supported models:
+#model-name=ggml-org/Qwen3-0.6B-GGUF
+#quantization=f16

integration-tests/pom.xml

@@ -56,7 +56,18 @@
         <module>llama3-java</module>
       </modules>
     </profile>
-        <profile>
+    <profile>
+      <id>TornadoVM</id>
+      <activation>
+        <property>
+          <name>tornado</name>
+        </property>
+      </activation>
+      <modules>
+        <module>gpu-llama3</module>
+      </modules>
+    </profile>
+    <profile>
             <id>default-project-deps</id>
             <activation>
                 <property>
@@ -107,6 +118,11 @@
                         <artifactId>quarkus-langchain4j-easy-rag</artifactId>
                         <version>999-SNAPSHOT</version>
                     </dependency>
+                    <dependency>
+                        <groupId>io.quarkiverse.langchain4j</groupId>
+                        <artifactId>quarkus-langchain4j-gpullama3</artifactId>
+                        <version>${quarkus-langchain4j.version}</version>
+                    </dependency>
                     <dependency>
                         <groupId>io.quarkiverse.langchain4j</groupId>
                         <artifactId>quarkus-langchain4j-hugging-face</artifactId>
@@ -122,6 +138,11 @@
                         <artifactId>quarkus-langchain4j-llama3-java</artifactId>
                         <version>999-SNAPSHOT</version>
                     </dependency>
+                  <dependency>
+                    <groupId>io.quarkiverse.langchain4j</groupId>
+                    <artifactId>quarkus-langchain4j-gpu-llama3</artifactId>
+                    <version>999-SNAPSHOT</version>
+                  </dependency>
                     <dependency>
                         <groupId>io.quarkiverse.langchain4j</groupId>
                         <artifactId>quarkus-langchain4j-mcp</artifactId>

model-providers/gpu-llama3/deployment/pom.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>io.quarkiverse.langchain4j</groupId>
+        <artifactId>quarkus-langchain4j-gpu-llama3-parent</artifactId>
+        <version>999-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>quarkus-langchain4j-gpu-llama3-deployment</artifactId>
+    <name>Quarkus LangChain4j - GPULlama3 - Deployment</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.quarkiverse.langchain4j</groupId>
+            <artifactId>quarkus-langchain4j-gpu-llama3</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <!-- what is this -->
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-arc-deployment</artifactId>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.quarkiverse.langchain4j</groupId>
+            <artifactId>quarkus-langchain4j-core-deployment</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-junit5-internal</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <annotationProcessorPaths>
+                        <path>
+                            <groupId>io.quarkus</groupId>
+                            <artifactId>quarkus-extension-processor</artifactId>
+                        </path>
+                    </annotationProcessorPaths>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

model-providers/gpu-llama3/deployment/src/main/java/io/quarkiverse/langchain4j/gpullama3/deployment/ChatModelBuildConfig.java

@@ -0,0 +1,16 @@
+package io.quarkiverse.langchain4j.gpullama3.deployment;
+
+import java.util.Optional;
+
+import io.quarkus.runtime.annotations.ConfigDocDefault;
+import io.quarkus.runtime.annotations.ConfigGroup;
+
+@ConfigGroup
+public interface ChatModelBuildConfig {
+
+    /**
+     * Whether the model should be enabled
+     */
+    @ConfigDocDefault("true")
+    Optional<Boolean> enabled();
+}