Feat: S3 Compatible Protocol (#444)

* feat: s3 protocol

Author: fenos

.env.sample

@@ -65,10 +65,18 @@ UPLOAD_FILE_SIZE_LIMIT=524288000
 UPLOAD_FILE_SIZE_LIMIT_STANDARD=52428800
 UPLOAD_SIGNED_URL_EXPIRATION_TIME=60
 
+#######################################
+# TUS Protocol
+#######################################
 TUS_URL_PATH=/upload/resumable
 TUS_URL_EXPIRY_MS=3600000
 TUS_PART_SIZE=50
 
+#######################################
+# S3 Protocol
+#######################################
+S3_PROTOCOL_ACCESS_KEY_ID=b585f311d839730f8a980a3457be2787
+S3_PROTOCOL_ACCESS_KEY_SECRET=67d161a7a8a46a24a17a75b26e7724f11d56b8d49a119227c66b13b6595601fb
 
 #######################################
 # Storage Backend Driver

.env.test.sample

@@ -2,8 +2,12 @@ AUTHENTICATED_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJhdXRoZW50aWNhd
 ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiYW5vbiIsImlhdCI6MTYxMzUzMTk4NSwiZXhwIjoxOTI5MTA3OTg1fQ.mqfi__KnQB4v6PkIjkhzfwWrYyF94MEbSC6LnuvVniE
 SERVICE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoic2VydmljZV9yb2xlIiwiaWF0IjoxNjEzNTMxOTg1LCJleHAiOjE5MjkxMDc5ODV9.th84OKK0Iz8QchDyXZRrojmKSEZ-OuitQm_5DvLiSIc
 
+S3_PROTOCOL_ACCESS_KEY_ID=b585f311d839730f8a980a3457be2787
+S3_PROTOCOL_ACCESS_KEY_SECRET=67d161a7a8a46a24a17a75b26e7724f11d56b8d49a119227c66b13b6595601fb
+S3_PROTOCOL_ALLOWS_SERVICE_KEY_AS_SECRET=false
+
 TENANT_ID=bjhaohmqunupljrqypxz
-ENABLE_DEFAULT_METRICS=false
+DEFAULT_METRICS_ENABLED=false
 PG_QUEUE_ENABLE=false
 MULTI_TENANT=false
 ADMIN_API_KEYS=apikey
@@ -18,3 +22,4 @@ AWS_DEFAULT_REGION=ap-southeast-1
 STORAGE_S3_ENDPOINT=http://127.0.0.1:9000
 STORAGE_S3_PROTOCOL=http
 STORAGE_S3_FORCE_PATH_STYLE=true
+REQUEST_X_FORWARDED_HOST_REGEXP=

.github/workflows/ci.yml

@@ -56,16 +56,13 @@ jobs:
           SERVICE_KEY: ${{ secrets.SERVICE_KEY }}
           TENANT_ID: ${{ secrets.TENANT_ID }}
           REGION: ${{ secrets.REGION }}
-          POSTGREST_URL: ${{ secrets.POSTGREST_URL }}
           GLOBAL_S3_BUCKET: ${{ secrets.GLOBAL_S3_BUCKET }}
           PGRST_JWT_SECRET: ${{ secrets.PGRST_JWT_SECRET }}
           AUTHENTICATED_KEY: ${{ secrets.AUTHENTICATED_KEY }}
           DATABASE_URL: postgresql://postgres:[email protected]/postgres
-          PGOPTIONS: -c search_path=storage,public
           FILE_SIZE_LIMIT: '52428800'
           STORAGE_BACKEND: s3
           MULTITENANT_DATABASE_URL: postgresql://postgres:[email protected]:5433/postgres
-          POSTGREST_URL_SUFFIX: /rest/v1
           ADMIN_API_KEYS: apikey
           ENABLE_IMAGE_TRANSFORMATION: true
           IMGPROXY_URL: http://127.0.0.1:50020
@@ -79,6 +76,9 @@ jobs:
           ENABLE_DEFAULT_METRICS: false
           PG_QUEUE_ENABLE: false
           MULTI_TENANT: false
+          S3_PROTOCOL_ACCESS_KEY_ID: ${{ secrets.TENANT_ID }}
+          S3_PROTOCOL_ACCESS_KEY_SECRET: ${{ secrets.SERVICE_KEY }}
+
 
       - name: Upload coverage results to Coveralls
         uses: coverallsapp/github-action@master

README.md

@@ -6,11 +6,19 @@ A scalable, light-weight object storage service.
 
 > Read [this post](https://supabase.io/blog/2021/03/30/supabase-storage) on why we decided to build a new object storage service.
 
+- Multi-protocol support (HTTP, TUS, S3)
 - Uses Postgres as its datastore for storing metadata
 - Authorization rules are written as Postgres Row Level Security policies
-- Integrates with S3 as the storage backend (with more in the pipeline!)
+- Integrates with S3 Compatible Storages
 - Extremely lightweight and performant
 
+
+**Supported Protocols**
+
+- [x] HTTP/REST
+- [x] TUS Resumable Upload
+- [x] S3 Compatible API
+
 ![Architecture](./static/architecture.png?raw=true 'Architecture')
 
 ## Documentation

docker-compose.yml

@@ -6,7 +6,6 @@ services:
     image: supabase/storage-api:latest
     ports:
       - '5000:5000'
-      - '5001:5001'
     depends_on:
       tenant_db:
         condition: service_healthy
@@ -39,7 +38,7 @@ services:
       UPLOAD_SIGNED_URL_EXPIRATION_TIME: 120
       TUS_URL_PATH: /upload/resumable
       TUS_URL_EXPIRY_MS: 3600000
-      # Image Tranformation
+      # Image Transformation
       IMAGE_TRANSFORMATION_ENABLED: "true"
       IMGPROXY_URL: http://imgproxy:8080
       IMGPROXY_REQUEST_TIMEOUT: 15

jest-setup.ts

@@ -1,3 +1,7 @@
 import { getConfig, setEnvPaths } from './src/config'
 
 setEnvPaths(['.env.test', '.env'])
+
+beforeEach(() => {
+  getConfig({ reload: true })
+})

jest.config.js

@@ -4,7 +4,7 @@ module.exports = {
   transform: {
     '^.+\\.(t|j)sx?$': 'ts-jest',
   },
-  setupFiles: ['<rootDir>/jest-setup.ts'],
+  setupFilesAfterEnv: ['<rootDir>/jest-setup.ts'],
   testEnvironment: 'node',
   testPathIgnorePatterns: ['node_modules', 'dist'],
   coverageProvider: 'v8',

jest.sequencer.js

@@ -11,15 +11,21 @@ const isTusTest = (test) => {
   return test.path.includes('tus')
 }
 
+const isS3Test = (test) => {
+  return test.path.includes('s3')
+}
+
 class CustomSequencer extends Sequencer {
   sort(tests) {
     const copyTests = Array.from(tests)
-    const normalTests = copyTests.filter((t) => !isRLSTest(t) && !isTusTest(t))
+    const normalTests = copyTests.filter((t) => !isRLSTest(t) && !isTusTest(t) && !isS3Test(t))
     const tusTests = copyTests.filter((t) => isTusTest(t))
+    const s3Tests = copyTests.filter((t) => isS3Test(t))
     const rlsTests = copyTests.filter((t) => isRLSTest(t))
     return super
       .sort(normalTests)
       .concat(tusTests)
+      .concat(s3Tests)
       .concat(rlsTests.sort((a, b) => (a.path > b.path ? 1 : -1)))
   }
 }

migrations/multitenant/0008-tenants-s3-credentials.sql

@@ -0,0 +1,46 @@
+
+
+CREATE TABLE IF NOT EXISTS tenants_s3_credentials (
+    id UUID PRIMARY KEY default gen_random_uuid(),
+    description text NOT NULL,
+    tenant_id text REFERENCES tenants(id) ON DELETE CASCADE,
+    access_key text NOT NULL,
+    secret_key text NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS tenants_s3_credentials_tenant_id_idx ON tenants_s3_credentials(tenant_id);
+CREATE UNIQUE INDEX IF NOT EXISTS tenants_s3_credentials_access_key_idx ON tenants_s3_credentials(tenant_id, access_key);
+
+
+CREATE OR REPLACE FUNCTION tenants_s3_credentials_update_notify_trigger ()
+    RETURNS TRIGGER
+AS $$
+BEGIN
+    PERFORM
+        pg_notify('tenants_s3_credentials_update', '"' || NEW.id || ':' || NEW.access_key || '"');
+    RETURN NULL;
+END;
+$$
+    LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION tenants_s3_credentials_delete_notify_trigger ()
+    RETURNS TRIGGER
+AS $$
+BEGIN
+    PERFORM
+        pg_notify('tenants_s3_credentials_update', '"' || OLD.id || ':' || OLD.access_key || '"');
+    RETURN NULL;
+END;
+$$
+    LANGUAGE plpgsql;
+
+CREATE TRIGGER tenants_s3_credentials_update_notify_trigger
+    AFTER UPDATE ON tenants_s3_credentials
+    FOR EACH ROW
+EXECUTE PROCEDURE tenants_s3_credentials_update_notify_trigger ();
+
+CREATE TRIGGER tenants_s3_credentials_delete_notify_trigger
+    AFTER DELETE ON tenants_s3_credentials
+    FOR EACH ROW
+EXECUTE PROCEDURE tenants_s3_credentials_delete_notify_trigger ();

migrations/multitenant/0009-add-scope-token-column-to-tenants-s3.sql

@@ -0,0 +1,3 @@
+
+
+ALTER TABLE tenants_s3_credentials ADD COLUMN claims json NOT NULL DEFAULT '{}';

migrations/tenant/0020-list-objects-with-delimiter.sql

@@ -0,0 +1,43 @@
+
+
+CREATE OR REPLACE FUNCTION storage.list_objects_with_delimiter(bucket_id text, prefix_param text, delimiter_param text, max_keys integer default 100, start_after text DEFAULT '', next_token text DEFAULT '')
+    RETURNS TABLE (name text, id uuid, metadata jsonb, updated_at timestamptz) AS
+$$
+BEGIN
+    RETURN QUERY EXECUTE
+        'SELECT DISTINCT ON(name COLLATE "C") * from (
+            SELECT
+                CASE
+                    WHEN position($2 IN substring(name from length($1) + 1)) > 0 THEN
+                        substring(name from 1 for length($1) + position($2 IN substring(name from length($1) + 1)))
+                    ELSE
+                        name
+                END AS name, id, metadata, updated_at
+            FROM
+                storage.objects
+            WHERE
+                bucket_id = $5 AND
+                name ILIKE $1 || ''%'' AND
+                CASE
+                    WHEN $6 != '''' THEN
+                    name COLLATE "C" > $6
+                ELSE true END
+                AND CASE
+                    WHEN $4 != '''' THEN
+                        CASE
+                            WHEN position($2 IN substring(name from length($1) + 1)) > 0 THEN
+                                substring(name from 1 for length($1) + position($2 IN substring(name from length($1) + 1))) COLLATE "C" > $4
+                            ELSE
+                                name COLLATE "C" > $4
+                            END
+                    ELSE
+                        true
+                END
+            ORDER BY
+                name COLLATE "C" ASC) as e order by name COLLATE "C" LIMIT $3'
+        USING prefix_param, delimiter_param, max_keys, next_token, bucket_id, start_after;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE INDEX IF NOT EXISTS idx_objects_bucket_id_name
+    ON storage.objects (bucket_id, (name COLLATE "C"));

migrations/tenant/0021-s3-multipart-uploads.sql

@@ -0,0 +1,84 @@
+CREATE TABLE IF NOT EXISTS storage.s3_multipart_uploads (
+    id text PRIMARY KEY,
+    in_progress_size int NOT NULL default 0,
+    upload_signature text NOT NULL,
+    bucket_id text NOT NULL references storage.buckets(id),
+    key text COLLATE "C" NOT NULL ,
+    version text NOT NULL,
+    owner_id text NULL,
+    created_at timestamptz NOT NULL default now()
+);
+
+CREATE TABLE IF NOT EXISTS storage.s3_multipart_uploads_parts (
+     id uuid PRIMARY KEY default gen_random_uuid(),
+     upload_id text NOT NULL references storage.s3_multipart_uploads(id) ON DELETE CASCADE,
+     size int NOT NULL default 0,
+     part_number int NOT NULL,
+     bucket_id text NOT NULL references storage.buckets(id),
+     key text COLLATE "C" NOT NULL,
+     etag text NOT NULL,
+     owner_id text NULL,
+     version text NOT NULL,
+     created_at timestamptz NOT NULL default now()
+);
+
+CREATE INDEX IF NOT EXISTS idx_multipart_uploads_list
+    ON storage.s3_multipart_uploads (bucket_id, (key COLLATE "C"), created_at ASC);
+
+CREATE OR REPLACE FUNCTION storage.list_multipart_uploads_with_delimiter(bucket_id text, prefix_param text, delimiter_param text, max_keys integer default 100, next_key_token text DEFAULT '', next_upload_token text default '')
+    RETURNS TABLE (key text, id text, created_at timestamptz) AS
+$$
+BEGIN
+    RETURN QUERY EXECUTE
+        'SELECT DISTINCT ON(key COLLATE "C") * from (
+            SELECT
+                CASE
+                    WHEN position($2 IN substring(key from length($1) + 1)) > 0 THEN
+                        substring(key from 1 for length($1) + position($2 IN substring(key from length($1) + 1)))
+                    ELSE
+                        key
+                END AS key, id, created_at
+            FROM
+                storage.s3_multipart_uploads
+            WHERE
+                bucket_id = $5 AND
+                key ILIKE $1 || ''%'' AND
+                CASE
+                    WHEN $4 != '''' AND $6 = '''' THEN
+                        CASE
+                            WHEN position($2 IN substring(key from length($1) + 1)) > 0 THEN
+                                substring(key from 1 for length($1) + position($2 IN substring(key from length($1) + 1))) COLLATE "C" > $4
+                            ELSE
+                                key COLLATE "C" > $4
+                            END
+                    ELSE
+                        true
+                END AND
+                CASE
+                    WHEN $6 != '''' THEN
+                        id COLLATE "C" > $6
+                    ELSE
+                        true
+                    END
+            ORDER BY
+                key COLLATE "C" ASC, created_at ASC) as e order by key COLLATE "C" LIMIT $3'
+        USING prefix_param, delimiter_param, max_keys, next_key_token, bucket_id, next_upload_token;
+END;
+$$ LANGUAGE plpgsql;
+
+ALTER TABLE storage.s3_multipart_uploads ENABLE ROW LEVEL SECURITY;
+ALTER TABLE storage.s3_multipart_uploads_parts ENABLE ROW LEVEL SECURITY;
+
+DO $$
+DECLARE
+    anon_role text = COALESCE(current_setting('storage.anon_role', true), 'anon');
+    authenticated_role text = COALESCE(current_setting('storage.authenticated_role', true), 'authenticated');
+    service_role text = COALESCE(current_setting('storage.service_role', true), 'service_role');
+BEGIN
+    EXECUTE 'revoke all on storage.s3_multipart_uploads from ' || anon_role || ', ' || authenticated_role;
+    EXECUTE 'revoke all on storage.s3_multipart_uploads_parts from ' || anon_role || ', ' || authenticated_role;
+    EXECUTE 'GRANT ALL ON TABLE storage.s3_multipart_uploads TO ' || service_role;
+    EXECUTE 'GRANT ALL ON TABLE storage.s3_multipart_uploads_parts TO ' || service_role;
+    EXECUTE 'GRANT SELECT ON TABLE storage.s3_multipart_uploads TO ' || authenticated_role || ', ' || anon_role;
+    EXECUTE 'GRANT SELECT ON TABLE storage.s3_multipart_uploads_parts TO ' || authenticated_role || ', ' || anon_role;
+END$$;

migrations/tenant/0022-s3-multipart-uploads-big-ints.sql

@@ -0,0 +1,2 @@
+ALTER TABLE storage.s3_multipart_uploads ALTER COLUMN in_progress_size TYPE bigint;
+ALTER TABLE storage.s3_multipart_uploads_parts ALTER COLUMN size TYPE bigint;