Implement Git LFS using Azure Functions and Storage Blob services

git-lfs-azure/requirements.txt

@@ -1 +1,2 @@
 azure-functions
+azure-storage-blob==1.5.0

git-lfs-azure/src/main/function.json → git-lfs-azure/src/main/batch/function.json

@@ -5,7 +5,11 @@
       "authLevel": "anonymous",
       "type": "httpTrigger",
       "direction": "in",
-      "name": "req"
+      "name": "req",
+      "methods": [
+        "post"
+      ],
+      "route": "objects/batch"
     },
     {
       "type": "http",

git-lfs-azure/src/main/batch/function.py

@@ -0,0 +1,138 @@
+import json
+import logging
+import os
+import re
+from datetime import datetime, timedelta, timezone
+
+import azure.functions as func
+from azure.storage.blob import blockblobservice, BlobPermissions
+
+try:
+    from ..core import *
+except:
+    from core import *
+
+logger = logging.getLogger()
+logger.setLevel(os.getenv('LOG_LEVEL', 'INFO'))
+
+__all__ = (
+    'BlobLargeFileStorage',
+    'Factory',
+
+    'process',
+    'main'
+)
+
+
+class BlobLargeFileStorage(LargeFileStorage):
+
+    def __init__(self, env=os.environ):
+        self.storage_account_name = env['STORAGE_ACCOUNT']
+        self.storage_account_primary_key = env['STORAGE_ACCOUNT_PRIMARY_KEY']
+        self.storage_container_name = env['STORAGE_CONTAINER']
+        self.client = blockblobservice.BlockBlobService(
+            account_name=self.storage_account_name,
+            account_key=self.storage_account_primary_key
+        )
+
+    def exists(self, oid: str) -> bool:
+        return self.client.exists(self.storage_container_name, oid)
+
+    def prepare_download(self, oid: str, size: int) -> BatchResponse.ObjectLfs.Action:
+        expiry = datetime.utcnow().replace(microsecond=0, tzinfo=timezone.utc) + timedelta(hours=1)
+
+        return BatchResponse.ObjectLfs.Action(
+            href=self.generate_blob_shared_access_signature_url(
+                oid, BlobPermissions.READ, expiry
+            ),
+            expires_at=expiry.isoformat()
+        )
+
+    def prepare_upload(self, oid: str, size: int) -> BatchResponse.ObjectLfs.Action:
+        expiry = datetime.utcnow().replace(microsecond=0, tzinfo=timezone.utc) + timedelta(hours=1)
+
+        return BatchResponse.ObjectLfs.Action(
+            href=self.generate_blob_shared_access_signature_url(
+                oid, BlobPermissions.WRITE, expiry
+            ),
+            header={
+                'x-ms-blob-type': 'BlockBlob'
+            },
+            expires_at=expiry.isoformat()
+        )
+
+    def generate_blob_shared_access_signature_url(self, oid, permission, expiry):
+        sas = self.client.generate_blob_shared_access_signature(
+            self.storage_container_name, oid, permission, expiry
+        )
+
+        return f'https://{self.storage_account_name}.blob.core.windows.net/{self.storage_container_name}/{oid}?{sas}'
+
+
+class KeyInsensitiveDict(dict):
+    def __getitem__(self, key):
+        return dict.__getitem__(self, key.lower())
+
+    def get(self, key, *args, **kwargs):
+        return dict.get(self, key.lower(), *args, **kwargs)
+
+
+class Factory:
+    @staticmethod
+    def create_batch_facade():
+        return BatchFacade(BlobLargeFileStorage())
+
+
+def process(request: func.HttpRequest, context: func.Context) -> func.HttpResponse:
+    try:
+        facade = Factory.create_batch_facade()
+
+        response = facade.process(
+            HttpRequest(
+                path=re.split('(api)', request.url)[2],
+                method=request.method,
+                headers=KeyInsensitiveDict(request.headers),
+                parameters=KeyInsensitiveDict(request.params),
+                body=request.get_body().decode()
+            )
+        )
+
+        return func.HttpResponse(
+            status_code=response.status_code,
+            headers=response.headers,
+            body=json.dumps(
+                dataclass_as_dict(response.body)
+            )
+        )
+
+    except HttpError as e:
+        return func.HttpResponse(
+            status_code=e.code,
+            mimetype='application/json',
+            body=json.dumps(
+                {
+                    'message': e.message,
+                    'request_id': context.invocation_id
+                }
+            )
+        )
+
+    except Exception as e:
+        logger.error(e, exc_info=True)
+        return func.HttpResponse(
+            status_code=500,
+            mimetype='application/json',
+            body=json.dumps(
+                {
+                    'message': 'Internal Server Error',
+                    'request_id': context.invocation_id
+                }
+            )
+        )
+
+
+def main(req: func.HttpRequest, context: func.Context) -> func.HttpResponse:
+    logger.info('azure.HttpRequest: %s' % vars(req))
+    res = process(req, context)
+    logger.info('azure.HttpResponse: %s' % vars(res))
+    return res

git-lfs-azure/src/main/local.settings.json

@@ -0,0 +1,6 @@
+{
+  "IsEncrypted": false,
+  "Values": {
+    "FUNCTIONS_WORKER_RUNTIME": "python"
+  }
+}

git-lfs-azure/src/test/batch/function_test.py

@@ -0,0 +1,155 @@
+import unittest
+from datetime import datetime
+from unittest import mock
+
+from azure.storage.blob import BlobPermissions
+
+from batch import function
+
+RFC3339_REGEX = r'^([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))$'
+
+
+class BlobLargeFileStorageTestCase(unittest.TestCase):
+
+    @mock.patch('azure.storage.blob.blockblobservice.BlockBlobService')
+    def setUp(self, mock_blockblobservice):
+        self.mock_client = mock.MagicMock()
+        mock_blockblobservice.return_value = self.mock_client
+        self.lfs = function.BlobLargeFileStorage(
+            {
+                'STORAGE_ACCOUNT': 'StorageAccountName',
+                'STORAGE_ACCOUNT_PRIMARY_KEY': 'StorageAccountPrimaryKeyId',
+                'STORAGE_CONTAINER': 'StorageContainerName'
+            }
+        )
+
+    def test_exists(self):
+        self.mock_client.exists.return_value = True
+
+        actual = self.lfs.exists('happyface.jpg')
+
+        self.assertTrue(actual)
+        self.mock_client.exists.assert_called_once_with('StorageContainerName', 'happyface.jpg')
+
+    def test_not_exists(self):
+        self.mock_client.exists.return_value = False
+
+        actual = self.lfs.exists('happyface.jpg')
+
+        self.assertFalse(actual)
+        self.mock_client.exists.assert_called_once_with('StorageContainerName', 'happyface.jpg')
+
+    def test_generate_blob_shared_access_signature_url(self):
+        self.mock_client.generate_blob_shared_access_signature.return_value = 'ZA1XSW2EDC'
+        expiry = datetime.fromtimestamp(1590558507)
+
+        actual = self.lfs.generate_blob_shared_access_signature_url(
+            'happyface.jpg', BlobPermissions.WRITE, expiry
+        )
+
+        self.assertEqual(
+            actual, 'https://StorageAccountName.blob.core.windows.net/StorageContainerName/happyface.jpg?ZA1XSW2EDC')
+        self.mock_client.generate_blob_shared_access_signature.assert_called_once_with(
+            'StorageContainerName', 'happyface.jpg', BlobPermissions.WRITE, expiry)
+
+    def test_prepare_download(self):
+        with mock.patch('batch.function.BlobLargeFileStorage.generate_blob_shared_access_signature_url') as mock_method:
+            mock_method.return_value = 'https://example.com'
+
+            actual = self.lfs.prepare_download('happyface.jpg', 123)
+
+            self.assertIsNotNone(actual)
+            self.assertEqual(actual.href, 'https://example.com')
+            self.assertRegex(actual.expires_at, RFC3339_REGEX)
+
+    def test_prepare_upload(self):
+        with mock.patch('batch.function.BlobLargeFileStorage.generate_blob_shared_access_signature_url') as mock_method:
+            mock_method.return_value = 'https://example.com'
+
+            actual = self.lfs.prepare_upload('happyface.jpg', 123)
+
+            self.assertIsNotNone(actual)
+            self.assertEqual(actual.href, 'https://example.com')
+            self.assertRegex(actual.expires_at, RFC3339_REGEX)
+            self.assertEqual(actual.header.get('x-ms-blob-type'), 'BlockBlob')
+
+
+class KeyInsensitiveDictTestCase(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.dictionary = function.KeyInsensitiveDict(
+            {
+                'foo': 'bar'
+            }
+        )
+
+    def test_getitem(self):
+        actual = self.dictionary['Foo']
+
+        self.assertEqual(actual, 'bar')
+
+    def test_get(self):
+        actual = self.dictionary.get('Foo')
+
+        self.assertEqual(actual, 'bar')
+
+    def test_get_missing_value(self):
+        actual = self.dictionary.get('baz')
+
+        self.assertIsNone(actual)
+
+
+class FunctionTestCase(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.mock_request = mock.MagicMock()
+        self.mock_request.url = 'https://foo.com/api/objects/batch'
+        self.mock_context = mock.MagicMock()
+        self.mock_context.invocation_id = 'uuid'
+        self.mock_response = mock.MagicMock()
+        self.mock_facade = mock.MagicMock()
+
+    def test_main(self):
+        with mock.patch('batch.function.process') as mock_process_method:
+            mock_process_method.return_value = self.mock_response
+
+            actual = function.main(self.mock_request, self.mock_context)
+
+            self.assertEqual(actual, self.mock_response)
+            mock_process_method.assert_called_once_with(self.mock_request, self.mock_context)
+
+    def test_process_http_error(self):
+        from core import HttpError
+        with mock.patch('batch.function.Factory.create_batch_facade') as mock_factory:
+            mock_factory.return_value = self.mock_facade
+            self.mock_facade.process.side_effect = HttpError(123, 'foo')
+
+            actual = function.process(self.mock_request, self.mock_context)
+
+            self.assertIsNotNone(actual)
+            self.assertEqual(actual.status_code, 123)
+            self.assertEqual(actual.get_body(), b'{"message": "foo", "request_id": "uuid"}')
+
+    def test_process_internal_server_error(self):
+        with mock.patch('batch.function.Factory.create_batch_facade') as mock_factory:
+            mock_factory.return_value = self.mock_facade
+            self.mock_facade.process.side_effect = KeyError('foo')
+
+            actual = function.process(self.mock_request, self.mock_context)
+
+            self.assertIsNotNone(actual)
+            self.assertEqual(actual.status_code, 500)
+            self.assertEqual(actual.get_body(), b'{"message": "Internal Server Error", "request_id": "uuid"}')
+
+    def test_process(self):
+        from core import HttpResponse, BatchResponse
+
+        with mock.patch('batch.function.Factory.create_batch_facade') as mock_factory:
+            mock_factory.return_value = self.mock_facade
+            self.mock_facade.process.return_value = HttpResponse(BatchResponse('foo', []))
+
+            actual = function.process(self.mock_request, self.mock_context)
+
+            self.assertIsNotNone(actual)
+            self.assertEqual(actual.status_code, 200)
+            self.assertEqual(actual.get_body(), b'{"transfer": "foo", "objects": []}')

infrastructure/azure/Makefile

@@ -42,10 +42,19 @@ init:
 package:
 	rm -rf $(CURDIR)/function_source.zip
 	for module in 'core' 'azure' ; do \
-		cd $(CURDIR)/../../git-lfs-$$module/src/ ; \
-		zip -r $(CURDIR)/function_source.zip ./main/ ; \
+  		module_dir=$(CURDIR)/../../git-lfs-$$module ; \
+  		\
+		cd $$module_dir/src/main/ ; \
+		zip -r $(CURDIR)/function_source.zip ./ ; \
+		\
+		cd $$module_dir ; \
+		rm -rf .python_packages ; \
+		docker run -v `pwd`:/mnt/ --entrypoint '/bin/bash' \
+			mcr.microsoft.com/azure-functions/python:2.0-python3.7-appservice -c \
+			"pip install -r /mnt/requirements.txt -t /mnt/.python_packages/lib/site-packages" ; \
+		zip -r $(CURDIR)/function_source.zip .python_packages/lib/site-packages ; \
+		rm -rf .python_packages ; \
 	done
-	cd $(CURDIR)/../../git-lfs-azure/ && zip -r $(CURDIR)/function_source.zip host.json requirements.txt
 
 .PHONY: plan
 plan: configure init package

infrastructure/azure/outputs.tf

@@ -30,5 +30,5 @@ output "function_app_name" {
 
 output "function_app_endpoint" {
   description = "The endpoint of the Function App"
-  value       = "https://${azurerm_function_app.function_app.default_hostname}/api/main/"
+  value       = "https://${azurerm_function_app.function_app.default_hostname}/api/"
 }