bytebase · adela-bytebase · Jan 16, 2025 · Jan 16, 2025
diff --git a/.github/workflows/bb-masking-column.yml b/.github/workflows/bb-masking-column.yml
@@ -0,0 +1,189 @@
+name: Bytebase Masking Policy Update
+on:
+  pull_request:
+    types: [closed]
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  bytebase-masking-1:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      - name: Login Bytebase
+        id: bytebase-login
+        uses: bytebase/[email protected]
+        with:
+          bytebase-url: ${{ secrets.BYTEBASE_URL }}
+          service-key: ${{ secrets.BYTEBASE_SERVICE_KEY }}
+          service-secret: ${{ secrets.BYTEBASE_SERVICE_SECRET }}
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v42
+        with:
+          files: |
+            masking/databases/**/**/database-catalog.json
+            masking/projects/**/masking-exception.json
+          since_last_remote_commit: true
+          fetch_depth: 0
+          include_all_old_new_renamed_files: true
+
+      - name: Debug changed files in detail
+        run: |
+          echo "All changed files:"
+          echo "${{ steps.changed-files.outputs.all_changed_files }}"
+          echo "Added files:"
+          echo "${{ steps.changed-files.outputs.added_files }}"
+          echo "Modified files:"
+          echo "${{ steps.changed-files.outputs.modified_files }}"
+          echo "Contains database-catalog.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'database-catalog.json') }}"
+          echo "Contains masking-exception.json: ${{ contains(steps.changed-files.outputs.all_changed_files, 'masking-exception.json') }}"
+          echo "Raw output:"
+          echo "${{ toJSON(steps.changed-files.outputs) }}"
+
+      - name: Apply column masking policy
+        id: apply-column-masking
+        if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, '/database-catalog.json') }}
+        run: |
+          # Process all database-catalog.json files
+          echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "database-catalog.json" | while read -r CHANGED_FILE; do
+            echo "Processing: $CHANGED_FILE"
+            INSTANCE_NAME=$(echo "$CHANGED_FILE" | sed -n 's/masking\/databases\/\([^/]*\)\/\([^/]*\).*/\1/p')
+            DATABASE_NAME=$(echo "$CHANGED_FILE" | sed -n 's/masking\/databases\/\([^/]*\)\/\([^/]*\).*/\2/p')
+            echo "INSTANCE_NAME=$INSTANCE_NAME"
+            echo "DATABASE_NAME=$DATABASE_NAME"
+
+            response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/instances/${INSTANCE_NAME}/databases/${DATABASE_NAME}/catalog" \
+              --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \
+              --header "Content-Type: application/json" \
+              --data @"$CHANGED_FILE")
+
+            # Extract status code and response body
+            status_code=$(echo "$response" | tail -n1)
+            body=$(echo "$response" | sed '$d')
+
+            echo "Status code: $status_code"
+            echo "Response body: $body"
+
+            # Append to outputs (with unique identifiers)
+            echo "status_code_${DATABASE_NAME}=${status_code}" >> $GITHUB_OUTPUT
+            echo "response_${DATABASE_NAME}<<EOF" >> $GITHUB_OUTPUT
+            echo "${body}" >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+
+            if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then
+              echo "Failed with status code: $status_code for database: $DATABASE_NAME"
+              exit 1
+            fi
+          done
+
+      - name: Apply masking exception policy
+        id: apply-masking-exception
+        if: ${{ steps.changed-files.outputs.any_changed == 'true' && contains(steps.changed-files.outputs.all_changed_files, '/masking-exception.json') }}
+        run: |
+          # Process all masking-exception.json files
+          echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep "masking-exception.json" | while read -r CHANGED_FILE; do
+            echo "Processing: $CHANGED_FILE"
+            PROJECT_NAME=$(echo "$CHANGED_FILE" | sed -n 's/masking\/projects\/\([^/]*\).*/\1/p')
+            echo "PROJECT_NAME=$PROJECT_NAME"
+
+            response=$(curl -s -w "\n%{http_code}" --request PATCH "${{ steps.bytebase-login.outputs.api_url }}/projects/${PROJECT_NAME}/policies/masking_exception?allow_missing=true&update_mask=payload" \
+              --header "Authorization: Bearer ${{ steps.bytebase-login.outputs.token }}" \
+              --header "Content-Type: application/json" \
+              --data @"$CHANGED_FILE")
+
+            # Extract status code and response body
+            status_code=$(echo "$response" | tail -n1)
+            body=$(echo "$response" | sed '$d')
+
+            echo "Status code: $status_code"
+            echo "Response body: $body"
+
+            # Append to outputs (with unique identifiers)
+            echo "status_code_${PROJECT_NAME}=${status_code}" >> $GITHUB_OUTPUT
+            echo "response_${PROJECT_NAME}<<EOF" >> $GITHUB_OUTPUT
+            echo "${body}" >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+
+            if [[ $status_code -lt 200 || $status_code -ge 300 ]]; then
+              echo "Failed with status code: $status_code for project: $PROJECT_NAME"
+              exit 1
+            fi
+          done
+
+      - name: Comment on PR
+        uses: actions/github-script@v7
+        env:
+          CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        with:
+          script: |
+            const changedFiles = process.env.CHANGED_FILES || '';
+            let commentBody = `### Masking Policy Update Summary\n\n`;
+
+            // Add status of merge
+            commentBody += `✅ **PR Status:** Merged\n\n`;
+
+            // Add changed files section
+            commentBody += `📝 **Changed Files:**\n\n`;
+            if (changedFiles.trim()) {
+              commentBody += changedFiles.split(' ').map(f => `- ${f}`).join('\n');
+            } else {
+              commentBody += `None`;
+            }
+            commentBody += '\n\n';
+
+            // Add API calls summary
+            commentBody += `🔄 **API Calls:**\n\n`;
+            let apiCallsFound = false;
+
+            if (changedFiles.includes('database-catalog.json')) {
+              const maskingStatuses = Object.keys(${{ toJSON(steps.apply-column-masking.outputs) }} || {})
+                .filter(key => key.startsWith('status_code_'))
+                .map(key => ({
+                  name: key.replace('status_code_', ''),
+                  status: ${{ toJSON(steps.apply-column-masking.outputs) }}[key]
+                }));
+
+              maskingStatuses.forEach(({name, status}) => {
+                apiCallsFound = true;
+                const success = status >= 200 && status < 300;
+                commentBody += `- Column Masking (${name}): ${success ? '✅' : '❌'} ${status}\n`;
+              });
+            }
+
+            if (changedFiles.includes('masking-exception.json')) {
+              const exceptionStatuses = Object.keys(${{ toJSON(steps.apply-masking-exception.outputs) }} || {})
+                .filter(key => key.startsWith('status_code_'))
+                .map(key => ({
+                  name: key.replace('status_code_', ''),
+                  status: ${{ toJSON(steps.apply-masking-exception.outputs) }}[key]
+                }));
+
+              exceptionStatuses.forEach(({name, status}) => {
+                apiCallsFound = true;
+                const success = status >= 200 && status < 300;
+                commentBody += `- Masking Exception (${name}): ${success ? '✅' : '❌'} ${status}\n`;
+              });
+            }
+
+            if (!apiCallsFound) {
+              commentBody += `None`;
+            }
+
+            await github.rest.issues.createComment({
+              ...context.repo,
+              issue_number: context.issue.number,
+              body: commentBody
+            });
diff --git a/masking/databases/test-sample-instance/hr_test/database-catalog.json b/masking/databases/test-sample-instance/hr_test/database-catalog.json
@@ -10,7 +10,7 @@
             "columns": [
               {
                 "name": "amount",
-                "semanticType": "bb.default",
+                "semanticType": "bb.default-partial",
                 "labels": {},
                 "classification": ""
               }

diff --git a/masking/projects/project-sample/masking-exception.json b/masking/projects/project-sample/masking-exception.json
@@ -5,7 +5,7 @@
     "maskingExceptions": [
       {
         "action": "EXPORT",
-        "member": "user:dev@x.com",
+        "member": "user:dev@example.com",
         "condition": {
           "expression": "resource.instance_id == \"prod-sample-instance\" && resource.database_name == \"hr_prod\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"",
           "title": "",
@@ -14,7 +14,7 @@
       },
       {
         "action": "QUERY",
-        "member": "user:dev2@x.com",
+        "member": "user:dev2@example.com",
         "condition": {
           "expression": "resource.instance_id == \"prod-sample-instance\" && resource.database_name == \"hr_prod\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"",
           "title": "",
@@ -23,7 +23,7 @@
       },
       {
         "action": "QUERY",
-        "member": "group:contractor@x.com",
+        "member": "group:contractor@example.com",
         "condition": {
           "expression": "resource.instance_id == \"prod-sample-instance\" && resource.database_name == \"hr_prod\" && resource.schema_name == \"public\" && resource.table_name == \"salary\" && resource.column_name == \"amount\"",
           "title": "",