SED-4816 Introduce AttachmentStorage

step-controller/step-controller-base-plugins/src/main/java/step/plugins/streaming/StepStreamingResourceManager.java

@@ -1,26 +1,41 @@
 package step.plugins.streaming;
 
+import org.bson.types.ObjectId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import step.attachments.AttachmentMeta;
+import step.attachments.SkippedAttachmentMeta;
+import step.attachments.StreamingAttachmentMeta;
 import step.constants.LiveReportingConstants;
 import step.core.GlobalContext;
 import step.core.access.User;
 import step.core.deployment.AuthorizationException;
+import step.core.execution.ExecutionContext;
+import step.core.objectenricher.ObjectEnricher;
 import step.core.objectenricher.ObjectHookRegistry;
 import step.framework.server.Session;
 import step.framework.server.access.AuthorizationManager;
-import step.resources.StreamingResourceContentProvider;
-import step.streaming.common.*;
+import step.plugins.streaming.util.MimeTypeUtil;
+import step.resources.AttachmentStorage;
+import step.streaming.common.QuotaExceededException;
+import step.streaming.common.StreamingResourceMetadata;
+import step.streaming.common.StreamingResourceReference;
+import step.streaming.common.StreamingResourceUploadContext;
+import step.streaming.common.StreamingResourceUploadContexts;
 import step.streaming.server.DefaultStreamingResourceManager;
 import step.streaming.server.StreamingResourcesStorageBackend;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Objects;
 import java.util.function.Function;
+import java.util.regex.Pattern;
 
-public class StepStreamingResourceManager extends DefaultStreamingResourceManager implements StreamingResourceContentProvider {
+public class StepStreamingResourceManager extends DefaultStreamingResourceManager implements AttachmentStorage {
     private static final Logger logger = LoggerFactory.getLogger(StepStreamingResourceManager.class);
     static final String ATTRIBUTE_STEP_SESSION = "stepSession";
+    private static final Pattern MIME_TYPE_PATTERN = Pattern.compile("^[a-zA-Z0-9!#$&^_.+-]+/[a-zA-Z0-9!#$&^_.+-]+$");
 
 
     private final AuthorizationManager<User, Session<User>> authorizationManager;
@@ -139,8 +154,84 @@ public void checkDownloadPermission(StreamingResource resource, Session<User> st
         }
     }
 
+    /* The following methods implement the AttachmentStorage interface */
+
     @Override
-    public InputStream getResourceContentStream(String resourceId) throws Exception {
+    public InputStream getAttachmentStream(String resourceId) throws Exception {
         return openStream(resourceId, 0, getStatus(resourceId).getCurrentSize());
     }
+
+    @Override
+    public AttachmentMeta saveAttachment(Object executionContext, byte[] content, String filename, String mimeType) {
+        if (!(executionContext instanceof ExecutionContext context)) {
+            String className = executionContext == null ? "null" : executionContext.getClass().getName();
+            return new SkippedAttachmentMeta(filename, mimeType, "UNEXPECTED: Invalid execution context type of class " + className);
+        }
+        // Java 16+ Voodoo: the (negative) instanceof check above actually defines the "context" variable and keeps it in scope here. :-)
+        // This is intentional; the reasoning is to keep the "happy path" straightforward, by allowing for that kind of guard clauses with early returns.
+        StreamingResourceUploadContexts uploadContexts = context.get(StreamingResourceUploadContexts.class);
+
+        if (uploadContexts == null) {
+            return new SkippedAttachmentMeta(filename, mimeType, "UNEXPECTED: no StreamingResourceUploadContexts found in execution context");
+        }
+
+        // We don't need to react to any events, but we need an UploadContext to convey relevant contextual information about the attachment.
+        StreamingResourceUploadContext uploadContext = new StreamingResourceUploadContext();
+        ObjectEnricher enricher = context.getObjectEnricher();
+        if (enricher != null) {
+            uploadContext.getAttributes().put(LiveReportingConstants.ACCESSCONTROL_ENRICHER, enricher);
+        }
+        uploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_EXECUTION_ID, context.getExecutionId());
+        uploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_VARIABLES_MANAGER, context.getVariablesManager());
+        uploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_REPORT_NODE, context.getCurrentReportNode());
+        uploadContexts.registerContext(uploadContext);
+
+        try {
+            return createAttachmentFromContent(content, filename, mimeType, uploadContext.contextId);
+        } catch (Exception e) {
+            return new SkippedAttachmentMeta(filename, mimeType, e.getMessage());
+        } finally {
+            uploadContexts.unregisterContext(uploadContext);
+        }
+    }
+
+    public AttachmentMeta createAttachmentFromContent(byte[] content, String filename, String mimeType, String uploadContextId) throws QuotaExceededException, IOException {
+        Objects.requireNonNull(filename, "filename must not be null");
+        if (mimeType == null) {
+            // Try to determine from filename; remains null for unknown extensions
+            mimeType = MimeTypeUtil.getMimeTypeForFilename(filename);
+        }
+        mimeType = sanitizeMimeType(mimeType);
+        boolean supportsLineAccess = isLineAccessSupported(mimeType);
+        StreamingResourceMetadata metadata = new StreamingResourceMetadata(filename, mimeType, supportsLineAccess);
+        String resourceId = registerNewResource(metadata, uploadContextId);
+        StreamingAttachmentMeta meta = new StreamingAttachmentMeta(new ObjectId(resourceId), filename, mimeType);
+        // Saving will be completely finished, in a single call, because all data is already present from the beginning.
+        writeChunk(resourceId, new ByteArrayInputStream(content), true);
+        var status = markCompleted(resourceId);
+        meta.setCurrentNumberOfLines(status.getNumberOfLines());
+        meta.setCurrentSize(status.getCurrentSize());
+        meta.setStatus(StreamingAttachmentMeta.Status.COMPLETED);
+        return meta;
+    }
+
+    private static String sanitizeMimeType(String mimeType) {
+        if (mimeType == null) {
+            // fallback if no mimeType was given
+            return "application/octet-stream";
+        }
+        mimeType = mimeType.trim(); // just in case
+        if (!MIME_TYPE_PATTERN.matcher(mimeType).matches()) {
+            logger.warn("Invalid mime type \"{}\", replacing with generic application/octet-stream", mimeType);
+            return "application/octet-stream";
+        }
+        return mimeType;
+    }
+
+    private static boolean isLineAccessSupported(String mimeType) {
+        // Very simple heuristic, but should catch almost all useful cases
+        return mimeType.startsWith("text/")
+            || mimeType.equals("application/xml")
+            || mimeType.equals("application/json");
+    }
 }

step-controller/step-controller-base-plugins/src/main/java/step/plugins/streaming/StreamingResourcesControllerPlugin.java

@@ -1,6 +1,5 @@
 package step.plugins.streaming;
 
-import ch.exense.commons.app.Configuration;
 import jakarta.servlet.http.HttpSession;
 import jakarta.websocket.Endpoint;
 import jakarta.websocket.HandshakeResponse;
@@ -10,17 +9,14 @@
 import org.slf4j.LoggerFactory;
 import step.constants.LiveReportingConstants;
 import step.core.GlobalContext;
-import step.core.controller.StepControllerPlugin;
 import step.core.deployment.ObjectHookControllerPlugin;
-import step.core.execution.AbstractExecutionEngineContext;
 import step.core.execution.ExecutionContext;
 import step.core.execution.ExecutionEngineContext;
 import step.core.plugins.AbstractControllerPlugin;
 import step.core.plugins.Plugin;
 import step.engine.plugins.AbstractExecutionEnginePlugin;
 import step.engine.plugins.ExecutionEnginePlugin;
 import step.resources.ResourceManagerControllerPlugin;
-import step.resources.StreamingResourceContentProvider;
 import step.streaming.common.StreamingResourceUploadContexts;
 import step.streaming.server.FilesystemStreamingResourcesStorageBackend;
 import step.streaming.server.StreamingResourceManager;
@@ -66,6 +62,7 @@ public void serverStart(GlobalContext context) throws Exception {
         manager = new StepStreamingResourceManager(context, catalog, storage, referenceProducer, uploadContexts);
 
         context.put(StepStreamingResourceManager.class, manager);
+        context.setAttachmentStorage(manager);
         context.getServiceRegistrationCallback().registerService(StreamingResourceServices.class);
 
         context.getServiceRegistrationCallback().registerWebsocketEndpoint(makeUploadConfig(manager));
@@ -113,12 +110,6 @@ private ServerEndpointConfig makeDownloadConfig(StepStreamingResourceManager man
     @Override
     public ExecutionEnginePlugin getExecutionEnginePlugin() {
         return new AbstractExecutionEnginePlugin() {
-            @Override
-            public void initializeExecutionEngineContext(AbstractExecutionEngineContext parentContext, ExecutionEngineContext executionEngineContext) {
-                // required by AP reporting for fetching attachments
-                executionEngineContext.put(StreamingResourceContentProvider.class, manager);
-            }
-
             @Override
             public void initializeExecutionContext(ExecutionEngineContext executionEngineContext, ExecutionContext executionContext) {
                 // Makes streaming available to the execution

step-controller/step-controller-base-plugins/src/main/java/step/plugins/streaming/util/MimeTypeUtil.java

@@ -0,0 +1,70 @@
+package step.plugins.streaming.util;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+public class MimeTypeUtil {
+    private MimeTypeUtil() {
+    }
+
+    private static final Map<String, String> mimeTypeForExtension = makeMap();
+
+    private static Map<String, String> makeMap() {
+        Map<String, String> map = new HashMap<>();
+
+        // Text & Code
+        List.of("txt", "log", "ini", "conf", "java", "c", "cpp", "h", "hpp", "sh", "bat", "ps1", "sql", "php", "rb", "pl", "go", "rs")
+            .forEach(m -> map.put(m, "text/plain"));
+        List.of("md", "markdown").forEach(m -> map.put(m, "text/markdown"));
+        map.put("csv", "text/csv");
+        List.of("htm", "html").forEach(m -> map.put(m, "text/html"));
+
+        // Application Data
+        map.put("json", "application/json");
+        map.put("xml", "application/xml");
+        List.of("yaml", "yml").forEach(m -> map.put(m, "application/yaml"));
+        map.put("pdf", "application/pdf");
+
+        // Images
+        List.of("jpg", "jpeg").forEach(m -> map.put(m, "image/jpeg"));
+        map.put("png", "image/png");
+        map.put("gif", "image/gif");
+        map.put("bmp", "image/bmp");
+        map.put("webp", "image/webp");
+        List.of("tiff", "tif").forEach(m -> map.put(m, "image/tiff"));
+        map.put("svg", "image/svg+xml");
+        map.put("heic", "image/heic");
+        map.put("avif", "image/avif");
+
+        // Video
+        List.of("mp4", "m4v").forEach(m -> map.put(m, "video/mp4"));
+        map.put("mkv", "video/x-matroska");
+        map.put("mov", "video/quicktime");
+        map.put("avi", "video/x-msvideo");
+        map.put("wmv", "video/x-ms-vwm");
+        map.put("flw", "video/x-flv");
+        map.put("webm", "video/webm");
+        map.put("mpeg", "video/mpeg");
+        map.put("mpg", "video/mpg");
+        map.put("3gp", "video/3gpp");
+        map.put("3g2", "video/3gpp2");
+        map.put("ogv", "video/ogg");
+        return map;
+    }
+
+    public static String getMimeTypeForFilename(String filename) {
+        if (filename == null) {
+            // unexpected, but let's be prudent
+            return null;
+        }
+        int dot = filename.lastIndexOf('.');
+        if (dot == -1) {
+            // no extension
+            return null;
+        }
+        String extension = filename.substring(dot + 1).toLowerCase(Locale.ROOT);
+        return mimeTypeForExtension.get(extension);
+    }
+}

step-controller/step-controller-server/src/test/java/step/core/GlobalContextBuilder.java

@@ -60,7 +60,14 @@
 import step.functions.accessor.FunctionAccessor;
 import step.functions.accessor.FunctionEntity;
 import step.functions.accessor.InMemoryFunctionAccessorImpl;
-import step.resources.*;
+import step.resources.InMemoryResourceAccessor;
+import step.resources.InMemoryResourceRevisionAccessor;
+import step.resources.LocalResourceManagerImpl;
+import step.resources.ResourceAccessor;
+import step.resources.ResourceEntity;
+import step.resources.ResourceManager;
+import step.resources.ResourceRevision;
+import step.resources.ResourceRevisionAccessor;
 
 import java.io.File;
 import java.io.IOException;
@@ -102,8 +109,10 @@ public static GlobalContext createGlobalContext() throws CircularDependencyExcep
         ResourceAccessor resourceAccessor = new InMemoryResourceAccessor();
         InMemoryResourceRevisionAccessor resourceRevisionAccessor = new InMemoryResourceRevisionAccessor();
         try {
-            File rootFolder = FileHelper.createTempFolder();
-            ResourceManager resourceManager = new ResourceManagerImpl(rootFolder, resourceAccessor, resourceRevisionAccessor);
+            // WARNING: The following folder is leaked unless it's cleaned up
+            // in the referencing unit tests (e.g. using ResourceManager#cleanup) - tracked in SED-4849
+            File resourceRootFolder = FileHelper.createTempFolder("step-resourceManager-");
+            ResourceManager resourceManager = new LocalResourceManagerImpl(resourceRootFolder, resourceAccessor, resourceRevisionAccessor);
             context.setResourceManager(resourceManager);
         } catch (IOException e) {
             logger.error("Unable to create temp folder for the resource manager", e);

step-core/src/main/java/step/attachments/FileResolver.java

@@ -18,18 +18,18 @@
  ******************************************************************************/
 package step.attachments;
 
-import java.io.Closeable;
-import java.io.File;
-import java.io.IOException;
-
 import org.bson.types.ObjectId;
 import step.resources.Resource;
 import step.resources.ResourceManager;
 import step.resources.ResourceRevisionFileHandle;
 
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+
 public class FileResolver {
 
-    public static final String ATTACHMENT_PREFIX = "attachment:";
+    public static final String ATTACHMENT_PREFIX_OBSOLETE = "attachment:";
     public static final String RESOURCE_PREFIX = "resource:";
     public static final String RESOURCE_PATH_SEPARATOR = ":";
 
@@ -46,9 +46,9 @@ public ResourceManager getResourceManager() {
 
     public File resolve(String path) {
         File file;
-        if (path.startsWith(ATTACHMENT_PREFIX)) {
-            throw new RuntimeException("Attachments have been migrated to the ResourceManager. The reference " + path +
-                " isn't valid anymore. Your attachment should be migrated to the ResourceManager.");
+        if (path.startsWith(ATTACHMENT_PREFIX_OBSOLETE)) {
+            throw new RuntimeException("Attachments have been migrated to use the AttachmentStorage. The reference " + path +
+                " isn't valid anymore. Please update your code to use the AttachmentStorage instead.");
         } else if (path.startsWith(RESOURCE_PREFIX)) {
             file = getResourceRevisionFileHandleForPath(path).getResourceFile();
         } else {
@@ -114,7 +114,7 @@ protected static String extractResourceSubPath(String path) {
     public FileHandle resolveFileHandle(String path) {
         File file;
         ResourceRevisionFileHandle resourceRevisionFileHandle;
-        if (path.startsWith(ATTACHMENT_PREFIX)) {
+        if (path.startsWith(ATTACHMENT_PREFIX_OBSOLETE)) {
             throw new RuntimeException("Attachments have been migrated to the ResourceManager. The reference " + path +
                 " isn't valid anymore. Your attachment should be migrated to the ResourceManager.");
         } else if (path.startsWith(RESOURCE_PREFIX)) {