Add Azure Blob Storage as a storage engine for attachments

.gitignore

@@ -118,3 +118,4 @@ dist
 !**/.yarn/versions
 
 dist-types/
+app-config.local.yaml

docs/config.md

@@ -43,7 +43,7 @@ The allowed configuration values are:
 
 ## Storage
 
-- storage.type, string, what kind of storage is used to upload images used in questions. Default is `database`. Available values are 'filesystem', 'database' and 's3'.
+- storage.type, string, what kind of storage is used to upload images used in questions. Default is `database`. Available values are 'filesystem', 'database', 's3' and 'azure'.
 - storage.maxSizeImage, number, the maximum allowed size of upload files in bytes. Default is `2500000`
 - storage.folder, string, what folder is used to storage temporarily images to convert and send to frontend. Default is `/tmp/backstage-qeta-images`
 - storage.allowedMimeTypes, string[], A list of allowed upload formats. Default: `png,jpg,jpeg,gif`
@@ -53,5 +53,10 @@ The allowed configuration values are:
 - storage.secretAccessKey, string, secret access key for S3 storage, optional
 - storage.region, string, region for S3 storage, optional
 - storage.sessionToken, string, AWS session token, optional
+- storage.blobStorageAccountName, string, Azure Blob Storage account name, optional
+- storage.blobStorageConnectionString, string, Connection String to Azure Blob Storage, optional
+- storage.blobStorageContainer, string, Azure Blob Storage container name, optional. Default `backstage-qeta-images`
+
+> Note: For Azure Blob Storage you can either use passwordless authentication by configuring `blobStorageAccountName`. This requires your Backstage backend to run as an Azure Managed Identity. Alternatively, you can use `blobStorageConnectionString` to authenticate with a connection string.
 
 Additionally, there are more config values for the [OpenAI module](ai.md).

plugins/qeta-backend/configSchema.d.ts

@@ -79,7 +79,7 @@ export interface Config {
      * @visibility backend
      */
     storage?: {
-      type?: 'database' | 'filesystem' | 's3';
+      type?: 'database' | 'filesystem' | 's3' | 'azure';
       folder?: string;
       maxSizeImage?: number;
       allowedMimeTypes?: string[];
@@ -88,6 +88,9 @@ export interface Config {
       secretAccessKey?: string;
       region?: string;
       sessionToken?: string;
+      blobStorageAccountName?: string;
+      blobStorageConnectionString?: string;
+      blobStorageContainer?: string;
     };
     /**
      * Stats config

plugins/qeta-backend/package.json

@@ -48,6 +48,8 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.540.0",
+    "@azure/identity": "^4.5.0",
+    "@azure/storage-blob": "^12.26.0",
     "@backstage/backend-defaults": "backstage:^",
     "@backstage/backend-plugin-api": "backstage:^",
     "@backstage/catalog-client": "backstage:^",

plugins/qeta-backend/src/service/routes/attachments.ts

@@ -4,16 +4,15 @@ import multiparty from 'multiparty';
 import FilesystemStoreEngine from '../upload/filesystem';
 import DatabaseStoreEngine from '../upload/database';
 import S3StoreEngine from '../upload/s3';
+import AzureBlobStorageEngine from '../upload/azureBlobStorage';
 import fs from 'fs';
 import FileType from 'file-type';
 import { File, RouteOptions } from '../types';
 import {
-  DeleteObjectCommand,
-  DeleteObjectCommandOutput,
-  GetObjectCommand,
-  GetObjectCommandOutput,
-} from '@aws-sdk/client-s3';
-import { getS3Client } from '../util';
+  AttachmentStorageEngine,
+  AttachmentStorageEngineOptions,
+} from '../upload/attachmentStorageEngine';
+import { getUsername } from '../util';
 
 const DEFAULT_IMAGE_SIZE_LIMIT = 2500000;
 const DEFAULT_MIME_TYPES = [
@@ -23,13 +22,32 @@ const DEFAULT_MIME_TYPES = [
   'image/gif',
 ];
 
+const getStorageEngine = (
+  storageType: string,
+  options: AttachmentStorageEngineOptions,
+): AttachmentStorageEngine => {
+  switch (storageType) {
+    case 'azure':
+      return AzureBlobStorageEngine(options);
+    case 's3':
+      return S3StoreEngine(options);
+    case 'filesystem':
+      return FilesystemStoreEngine(options);
+    case 'database':
+    default:
+      return DatabaseStoreEngine(options);
+  }
+};
+
 export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
   const { database, config } = options;
 
   // POST /attachments
   router.post('/attachments', async (request, response) => {
     let attachment: Attachment;
 
+    const username = await getUsername(request, options);
+
     const storageType =
       config?.getOptionalString('qeta.storage.type') || 'database';
     const maxSizeImage =
@@ -40,9 +58,8 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
       DEFAULT_MIME_TYPES;
 
     const form = new multiparty.Form();
-    const fileSystemEngine = FilesystemStoreEngine(options);
-    const databaseEngine = DatabaseStoreEngine(options);
-    const s3Engine = S3StoreEngine(options);
+
+    const engine = getStorageEngine(storageType, options);
 
     form.parse(request, async (err, _fields, files) => {
       if (err) {
@@ -84,6 +101,7 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
       };
 
       const opts = {
+        creator: username,
         postId: request.query.postId ? Number(request.query.postId) : undefined,
         answerId: request.query.answerId
           ? Number(request.query.answerId)
@@ -93,18 +111,7 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
           : undefined,
       };
 
-      switch (storageType) {
-        case 's3':
-          attachment = await s3Engine.handleFile(file, opts);
-          break;
-        case 'filesystem':
-          attachment = await fileSystemEngine.handleFile(file, opts);
-          break;
-        case 'database':
-        default:
-          attachment = await databaseEngine.handleFile(file, opts);
-          break;
-      }
+      attachment = await engine.handleFile(file, opts);
       response.json(attachment);
     });
   });
@@ -119,42 +126,12 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
       return;
     }
 
-    const getS3ImageBuffer = async () => {
-      const bucket = config.getOptionalString('qeta.storage.bucket');
-      if (!bucket) {
-        throw new Error('Bucket name is required for S3 storage');
-      }
-      const s3 = getS3Client(config);
-      const object: GetObjectCommandOutput = await s3.send(
-        new GetObjectCommand({
-          Bucket: bucket,
-          Key: attachment.path,
-        }),
-      );
-
-      if (!object.Body) {
-        return undefined;
-      }
-      const bytes = await object.Body.transformToByteArray();
-      return Buffer.from(bytes);
-    };
-
-    let imageBuffer: Buffer | undefined;
-    switch (attachment.locationType) {
-      case 's3':
-        imageBuffer = await getS3ImageBuffer();
-        break;
-      case 'filesystem':
-        imageBuffer = await fs.promises.readFile(attachment.path);
-        break;
-      default:
-      case 'database':
-        imageBuffer = attachment.binaryImage;
-        break;
-    }
+    const engine = getStorageEngine(attachment.locationType, options);
+    const imageBuffer = await engine.getAttachmentBuffer(attachment);
 
     if (!imageBuffer) {
-      response.status(500).send('Attachment buffer is undefined');
+      response.status(404).end();
+      return;
     }
 
     response.writeHead(200, {
@@ -186,34 +163,8 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
       return;
     }
 
-    const deleteS3Image = async () => {
-      const bucket = config.getOptionalString('qeta.storage.bucket');
-      if (!bucket) {
-        throw new Error('Bucket name is required for S3 storage');
-      }
-      const s3 = getS3Client(config);
-      const output: DeleteObjectCommandOutput = await s3.send(
-        new DeleteObjectCommand({
-          Bucket: bucket,
-          Key: attachment.path,
-        }),
-      );
-      if (output.$metadata.httpStatusCode !== 204) {
-        throw new Error('Failed to delete object');
-      }
-    };
-
-    switch (attachment.locationType) {
-      case 's3':
-        await deleteS3Image();
-        break;
-      case 'filesystem':
-        await fs.promises.rm(attachment.path);
-        break;
-      default:
-      case 'database':
-        break;
-    }
+    const engine = getStorageEngine(attachment.locationType, options);
+    await engine.deleteAttachment(attachment);
 
     const result = await database.deleteAttachment(uuid);
     if (!result) {

plugins/qeta-backend/src/service/upload/attachmentStorageEngine.ts

@@ -0,0 +1,23 @@
+import { Attachment } from '@drodil/backstage-plugin-qeta-common';
+import { File } from '../types';
+import { Config } from '@backstage/config/index';
+import { QetaStore } from '../../database/QetaStore';
+
+export type AttachmentStorageEngineOptions = {
+  config: Config;
+  database: QetaStore;
+};
+
+export interface AttachmentStorageEngine {
+  handleFile: (
+    file: File,
+    options?: {
+      creator: string;
+      postId?: number;
+      answerId?: number;
+      collectionId?: number;
+    },
+  ) => Promise<Attachment>;
+  getAttachmentBuffer: (attachment: Attachment) => Promise<Buffer | undefined>;
+  deleteAttachment(attachment: Attachment): Promise<void>;
+}

plugins/qeta-backend/src/service/upload/azureBlobStorage.ts

@@ -0,0 +1,94 @@
+import * as fs from 'fs';
+import { v4 as uuidv4 } from 'uuid';
+import { Config } from '@backstage/config';
+import { QetaStore } from '../../database/QetaStore';
+import { Attachment } from '@drodil/backstage-plugin-qeta-common';
+import { File } from '../types';
+import { getAzureBlobServiceClient } from '../util';
+import {
+  AttachmentStorageEngine,
+  AttachmentStorageEngineOptions,
+} from './attachmentStorageEngine';
+
+class AzureBlobStorageEngine implements AttachmentStorageEngine {
+  config: Config;
+  database: QetaStore;
+  backendBaseUrl: string;
+  qetaUrl: string;
+  container: string;
+
+  constructor(opts: AttachmentStorageEngineOptions) {
+    this.config = opts.config;
+    this.database = opts.database;
+    this.backendBaseUrl = this.config.getString('backend.baseUrl');
+    this.qetaUrl = `${this.backendBaseUrl}/api/qeta/attachments`;
+    this.container =
+      this.config.getOptionalString('qeta.storage.blobStorageContainer') ||
+      'backstage-qeta-images';
+  }
+
+  handleFile = async (
+    file: File,
+    options?: { postId?: number; answerId?: number; collectionId?: number },
+  ): Promise<Attachment> => {
+    const imageUuid = uuidv4();
+    const filename = `image-${imageUuid}-${Date.now()}.${file.ext}`;
+
+    const imageURI = `${this.qetaUrl}/${imageUuid}`;
+    const client = getAzureBlobServiceClient(this.config);
+    const container = client.getContainerClient(this.container);
+    if (!(await container.exists())) {
+      await container.create();
+    }
+
+    await container.uploadBlockBlob(
+      filename,
+      fs.createReadStream(file.path),
+      file.size,
+    );
+
+    return await this.database.postAttachment({
+      uuid: imageUuid,
+      locationType: 'azure',
+      locationUri: imageURI,
+      extension: file.ext,
+      mimeType: file.mimeType,
+      path: filename,
+      binaryImage: undefined,
+      ...options,
+    });
+  };
+
+  getAttachmentBuffer = async (attachment: Attachment) => {
+    const client = getAzureBlobServiceClient(this.config);
+    const container = client.getContainerClient(this.container);
+
+    if (!(await container.exists())) {
+      return undefined;
+    }
+
+    const blob = container.getBlockBlobClient(attachment.path);
+    if (await blob.exists()) {
+      return blob.downloadToBuffer();
+    }
+
+    return undefined;
+  };
+
+  deleteAttachment = async (attachment: Attachment) => {
+    const client = getAzureBlobServiceClient(this.config);
+    const container = client.getContainerClient(this.container);
+    if (!(await container.exists())) {
+      return;
+    }
+
+    const blob = container.getBlockBlobClient(attachment.path);
+    if (await blob.exists()) {
+      await blob.delete();
+    }
+  };
+}
+
+export default (opts: AttachmentStorageEngineOptions) => {
+  return new AzureBlobStorageEngine(opts);
+};

plugins/qeta-backend/src/service/upload/database.ts

@@ -2,19 +2,19 @@ import { Config } from '@backstage/config';
 import { QetaStore } from '../../database/QetaStore';
 import { File } from '../types';
 import { v4 as uuidv4 } from 'uuid';
+import {
+  AttachmentStorageEngine,
+  AttachmentStorageEngineOptions,
+} from './attachmentStorageEngine';
+import { Attachment } from '@drodil/backstage-plugin-qeta-common';
 
-type Options = {
-  config: Config;
-  database: QetaStore;
-};
-
-class DatabaseStoreEngine {
+class DatabaseStoreEngine implements AttachmentStorageEngine {
   config: Config;
   database: QetaStore;
   backendBaseUrl: string;
   qetaUrl: string;
 
-  constructor(opts: Options) {
+  constructor(opts: AttachmentStorageEngineOptions) {
     this.config = opts.config;
     this.database = opts.database;
     this.backendBaseUrl = this.config.getString('backend.baseUrl');
@@ -38,8 +38,16 @@ class DatabaseStoreEngine {
       ...options,
     });
   };
+
+  getAttachmentBuffer = async (attachment: Attachment) => {
+    return attachment.binaryImage;
+  };
+
+  deleteAttachment = async (_attachment: Attachment) => {
+    // Nothing to do here, since the attachment is stored in the database
+  };
 }
 
-export default (opts: Options) => {
+export default (opts: AttachmentStorageEngineOptions) => {
   return new DatabaseStoreEngine(opts);
 };