BAEL-9226: Extracting Structured Data from Images using AI in Java (#18426)

* BAEL-9226: Extracting Structured Data from Images using AI in Java

* BAEL-9226: Extracting Structured Data from Images using AI in Java
- Fix test name

* BAEL-9226: Extracting Structured Data from Images using AI in Java
- Code formatting

Author: manfred106

spring-ai-2/src/main/java/com/baeldung/image/CarColorCount.java

@@ -0,0 +1,27 @@
+package com.baeldung.image;
+
+public class CarColorCount {
+
+    private String color;
+    private int count;
+
+    public CarColorCount() {
+    }
+
+    public String getColor() {
+        return color;
+    }
+
+    public void setColor(String color) {
+        this.color = color;
+    }
+
+    public int getCount() {
+        return count;
+    }
+
+    public void setCount(int count) {
+        this.count = count;
+    }
+
+}

spring-ai-2/src/main/java/com/baeldung/image/CarCount.java

@@ -0,0 +1,29 @@
+package com.baeldung.image;
+
+import java.util.List;
+
+public class CarCount {
+
+    private List<CarColorCount> carColorCounts;
+    private int totalCount;
+
+    public CarCount() {
+    }
+
+    public List<CarColorCount> getCarColorCounts() {
+        return carColorCounts;
+    }
+
+    public void setCarColorCounts(List<CarColorCount> carColorCounts) {
+        this.carColorCounts = carColorCounts;
+    }
+
+    public int getTotalCount() {
+        return totalCount;
+    }
+
+    public void setTotalCount(int totalCount) {
+        this.totalCount = totalCount;
+    }
+
+}

spring-ai-2/src/main/java/com/baeldung/image/CarCountService.java

@@ -0,0 +1,32 @@
+package com.baeldung.image;
+
+import org.springframework.ai.chat.client.ChatClient;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.stereotype.Service;
+import org.springframework.util.MimeTypeUtils;
+
+import java.io.InputStream;
+
+@Service
+public class CarCountService {
+
+    private final ChatClient chatClient;
+
+    public CarCountService(ChatClient.Builder chatClientBuilder) {
+        this.chatClient = chatClientBuilder.build();
+    }
+
+    public CarCount getCarCount(InputStream imageInputStream, String contentType, String colors) {
+        return chatClient.prompt()
+            .system(systemMessage -> systemMessage.text("Count the number of cars in different colors from the image")
+                .text("User will provide the image and specify which colors to count in the user prompt")
+                .text("Count colors that are specified in the user prompt only")
+                .text("Ignore anything in the user prompt that is not a color")
+                .text("If there is no color specified in the user prompt, simply returns zero in the total count"))
+            .user(userMessage -> userMessage.text(colors)
+                .media(MimeTypeUtils.parseMimeType(contentType), new InputStreamResource(imageInputStream)))
+            .call()
+            .entity(CarCount.class);
+    }
+
+}

spring-ai-2/src/main/java/com/baeldung/image/ImageApplication.java

@@ -0,0 +1,24 @@
+package com.baeldung.image;
+
+import org.springframework.ai.autoconfigure.anthropic.AnthropicAutoConfiguration;
+import org.springframework.ai.autoconfigure.bedrock.converse.BedrockConverseProxyChatAutoConfiguration;
+import org.springframework.ai.autoconfigure.ollama.OllamaAutoConfiguration;
+import org.springframework.ai.autoconfigure.vectorstore.chroma.ChromaVectorStoreAutoConfiguration;
+import org.springframework.ai.autoconfigure.vectorstore.pgvector.PgVectorStoreAutoConfiguration;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+/*
+ * Exclude the configurations that are from the shared codebase
+ */
+@SpringBootApplication(exclude = { AnthropicAutoConfiguration.class, BedrockConverseProxyChatAutoConfiguration.class, ChromaVectorStoreAutoConfiguration.class,
+    OllamaAutoConfiguration.class, PgVectorStoreAutoConfiguration.class })
+public class ImageApplication {
+
+    public static void main(String[] args) {
+        SpringApplication app = new SpringApplication(ImageApplication.class);
+        app.setAdditionalProfiles("image");
+        app.run(args);
+    }
+
+}

spring-ai-2/src/main/java/com/baeldung/image/ImageController.java

@@ -0,0 +1,34 @@
+package com.baeldung.image;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+@RestController
+@RequestMapping("/image")
+public class ImageController {
+
+    @Autowired
+    private CarCountService carCountService;
+
+    @PostMapping("/car-count")
+    public ResponseEntity<?> getCarCounts(@RequestParam("colors") String colors, @RequestParam("file") MultipartFile file) {
+
+        try (InputStream inputStream = file.getInputStream()) {
+            var carCount = carCountService.getCarCount(inputStream, file.getContentType(), colors);
+            return ResponseEntity.ok(carCount);
+        } catch (IOException e) {
+            return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
+                .body("Error uploading image");
+        }
+    }
+
+}

spring-ai-2/src/main/resources/application-image.yml

@@ -0,0 +1,12 @@
+spring:
+  ai:
+    openai:
+      api-key: "<YOUR-API-KEY>"
+      chat:
+        options:
+          model: "gpt-4o"
+
+  # Avoid starting docker from the shared codebase
+  docker:
+    compose:
+      enabled: false

spring-ai-2/src/test/java/com/baeldung/image/ImageControllerLiveTest.java

@@ -0,0 +1,64 @@
+package com.baeldung.image;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.web.server.LocalServerPort;
+import org.springframework.http.MediaType;
+import org.springframework.mock.web.MockMultipartFile;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.client.RestClient;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles("image")
+public class ImageControllerLiveTest {
+
+    @LocalServerPort
+    private int port;
+
+    private RestClient restClient;
+
+    @BeforeEach
+    void setup() {
+        restClient = RestClient.builder()
+            .baseUrl(String.format("http://localhost:%d", port))
+            .build();
+    }
+
+    @Test
+    void whenProvideColorAndImage_thenReturnStructuredOutput() throws Exception {
+        // Prepare test data
+        String colors = "blue,yellow,green";
+
+        // Create a MultipartFile
+        Path path = Paths.get("<replace-this-by-your-jpg-file>");
+        byte[] imageBytes = Files.readAllBytes(path);
+        MockMultipartFile file = new MockMultipartFile("file", "test-image.jpg", MediaType.IMAGE_JPEG_VALUE, imageBytes);
+
+        MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
+        body.add("file", file.getResource());
+        body.add("colors", colors);
+
+        CarCount carCount = restClient.post()
+            .uri(uriBuilder -> uriBuilder.path("/image/car-count")
+                .queryParam("colors", colors)
+                .build())
+            .contentType(MediaType.MULTIPART_FORM_DATA)
+            .body(body)
+            .retrieve()
+            .body(CarCount.class);
+
+        assertTrue(carCount.getTotalCount() >= 0);
+        assertTrue(carCount.getCarColorCounts()
+            .size() >= 0);
+    }
+
+}