From b1b5bfbaef16dba726d4ccfe2811ebf50836e05b Mon Sep 17 00:00:00 2001 From: nalbam Date: Thu, 22 Aug 2024 11:01:08 +0900 Subject: [PATCH 1/3] chore: Update sync-notion and start-ingestion workflows --- .github/workflows/4-sync-notion.yml.stop | 48 -------------- .github/workflows/5-sync-notion.yml | 68 ++++++++++++++++++++ .github/workflows/6-start-ingestion.yml.stop | 11 ++-- bin/.env.example | 6 +- bin/notion_exporter.py | 6 +- serverless.yml | 12 ++-- 6 files changed, 88 insertions(+), 63 deletions(-) delete mode 100644 .github/workflows/4-sync-notion.yml.stop create mode 100644 .github/workflows/5-sync-notion.yml diff --git a/.github/workflows/4-sync-notion.yml.stop b/.github/workflows/4-sync-notion.yml.stop deleted file mode 100644 index 4d0c74c..0000000 --- a/.github/workflows/4-sync-notion.yml.stop +++ /dev/null @@ -1,48 +0,0 @@ -name: sync-notion - -on: - # push: - # branches: - # - main - - schedule: - - cron: "0 20 * * 0-4" - -env: - NOTION_PAGE_NAME: "notion" - NOTION_PAGE_ID: "0c7c08203a9b4435a4ca07b6454151d7" - - AWS_DEST_PATH: ${{ vars.AWS_DEST_PATH }} - -jobs: - sync: - runs-on: ubuntu-latest - - steps: - - name: Checkout 🛎️ - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup Python 3.9 🐍 - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - - name: Setup Dependencies - run: pip install python-notion-exporter - - - name: Run Notion Exporter - env: - NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }} - NOTION_FILE_TOKEN: ${{ secrets.NOTION_FILE_TOKEN }} - run: | - python bin/notion_exporter.py - - - name: Sync to AWS S3 Data Source - run: | - aws s3 sync build/ ${{ env.AWS_DEST_PATH }} --delete - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" diff --git a/.github/workflows/5-sync-notion.yml b/.github/workflows/5-sync-notion.yml new file mode 100644 index 0000000..a49ea3e --- /dev/null +++ b/.github/workflows/5-sync-notion.yml @@ -0,0 +1,68 @@ +name: sync-notion + +on: + push: + branches: + - main + + schedule: + - cron: "0 20 * * 0-4" + +env: + ENABLE_NOTION_SYNC: ${{ vars.ENABLE_NOTION_SYNC }} + + NOTION_PAGE_NAME: "nalbam" + NOTION_PAGE_ID: "7aace0412a82431996f61a29225a95ec" + + AWS_DEST_PATH: ${{ vars.AWS_DEST_PATH }} + + KNOWLEDGE_BASE_ID: ${{ vars.KNOWLEDGE_BASE_ID }} + DATA_SOURCE_ID: ${{ vars.DATA_SOURCE_ID }} + +jobs: + sync: + runs-on: ubuntu-latest + + steps: + - name: Checkout 🛎️ + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python 3.9 🐍 + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Setup Dependencies + run: pip install python-notion-exporter + + - name: Run Notion Exporter + if: env.ENABLE_NOTION_SYNC == 'Yes' + env: + NOTION_TOKEN_V2: ${{ secrets.NOTION_TOKEN_V2 }} + NOTION_FILE_TOKEN: ${{ secrets.NOTION_FILE_TOKEN }} + run: | + python bin/notion_exporter.py + + - name: Sync to AWS S3 Data Source + if: env.ENABLE_NOTION_SYNC == 'Yes' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + run: | + aws s3 sync --delete \ + build/${{ env.NOTION_PAGE_NAME }}/ \ + ${{ env.AWS_DEST_PATH }}/${{ env.NOTION_PAGE_NAME }}/ + + - name: Sync to AWS Bedrock Knowledge Base + if: env.ENABLE_NOTION_SYNC == 'Yes' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: "us-east-1" + run: | + aws bedrock-agent start-ingestion-job \ + --knowledge-base-id ${{ env.KNOWLEDGE_BASE_ID }} \ + --data-source-id ${{ env.DATA_SOURCE_ID }} diff --git a/.github/workflows/6-start-ingestion.yml.stop b/.github/workflows/6-start-ingestion.yml.stop index 762c3c5..e2a3045 100644 --- a/.github/workflows/6-start-ingestion.yml.stop +++ b/.github/workflows/6-start-ingestion.yml.stop @@ -9,6 +9,8 @@ on: - cron: "0 21 * * 0-4" env: + ENABLE_NOTION_SYNC: ${{ vars.ENABLE_NOTION_SYNC }} + KNOWLEDGE_BASE_ID: ${{ vars.KNOWLEDGE_BASE_ID }} DATA_SOURCE_ID: ${{ vars.DATA_SOURCE_ID }} @@ -28,11 +30,12 @@ jobs: python-version: 3.9 - name: Sync to AWS Bedrock Knowledge Base - run: | - aws bedrock-agent start-ingestion-job \ - --knowledge-base-id ${{ env.KNOWLEDGE_BASE_ID }} \ - --data-source-id ${{ env.DATA_SOURCE_ID }} + if: env.ENABLE_NOTION_SYNC == 'Yes' env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: "us-east-1" + run: | + aws bedrock-agent start-ingestion-job \ + --knowledge-base-id ${{ env.KNOWLEDGE_BASE_ID }} \ + --data-source-id ${{ env.DATA_SOURCE_ID }} diff --git a/bin/.env.example b/bin/.env.example index 9600e30..d7b3a8f 100644 --- a/bin/.env.example +++ b/bin/.env.example @@ -1,5 +1,5 @@ -NOTION_TOKEN = "v02%3Auser_token_or_cookies" +NOTION_TOKEN_V2 = "v02%3Auser_token_or_cookies" NOTION_FILE_TOKEN = "v02%3Afile_token" -NOTION_PAGE_ID = "0c7c08203a9b4435a4ca07b6454151d7" -NOTION_PAGE_NAME = "notion" +NOTION_PAGE_NAME = "nalbam" +NOTION_PAGE_ID = "7aace0412a82431996f61a29225a95ec" diff --git a/bin/notion_exporter.py b/bin/notion_exporter.py index e1216e4..63f736d 100644 --- a/bin/notion_exporter.py +++ b/bin/notion_exporter.py @@ -3,16 +3,16 @@ from python_notion_exporter import NotionExporter, ExportType, ViewExportType -NOTION_TOKEN = os.getenv("NOTION_TOKEN") +NOTION_TOKEN_V2 = os.getenv("NOTION_TOKEN_V2") NOTION_FILE_TOKEN = os.getenv("NOTION_FILE_TOKEN") -NOTION_PAGE_ID = os.getenv("NOTION_PAGE_ID", "0c7c08203a9b4435a4ca07b6454151d7") NOTION_PAGE_NAME = os.getenv("NOTION_PAGE_NAME", "demo") +NOTION_PAGE_ID = os.getenv("NOTION_PAGE_ID", "7aace0412a82431996f61a29225a95ec") if __name__ == "__main__": exporter = NotionExporter( - token_v2=NOTION_TOKEN, + token_v2=NOTION_TOKEN_V2, file_token=NOTION_FILE_TOKEN, pages={NOTION_PAGE_NAME: NOTION_PAGE_ID}, export_directory="build", diff --git a/serverless.yml b/serverless.yml index b3b4ad8..8ca4120 100644 --- a/serverless.yml +++ b/serverless.yml @@ -53,11 +53,13 @@ resources: TimeToLiveSpecification: AttributeName: expire_at Enabled: true - # S3Bucket: - # Type: AWS::S3::Bucket - # Properties: - # BucketName: - # Fn::Sub: ${self:provider.environment.BASE_NAME}-${AWS::AccountId} + + S3Bucket: + Type: AWS::S3::Bucket + Properties: + BucketName: + Fn::Sub: ${self:provider.environment.BASE_NAME}-${AWS::AccountId} + # KnowledgeBase: # Type: AWS::Bedrock::KnowledgeBase # Properties: From 5ef2192ac5b1f14fba0672189546d4649e27e7ea Mon Sep 17 00:00:00 2001 From: nalbam Date: Thu, 22 Aug 2024 11:26:13 +0900 Subject: [PATCH 2/3] chore: Add tags to S3Bucket resource in serverless.yml --- serverless.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/serverless.yml b/serverless.yml index 8ca4120..6e8984e 100644 --- a/serverless.yml +++ b/serverless.yml @@ -53,12 +53,18 @@ resources: TimeToLiveSpecification: AttributeName: expire_at Enabled: true + Tags: + - Key: Name + Value: ${self:provider.environment.BASE_NAME}-context S3Bucket: Type: AWS::S3::Bucket Properties: BucketName: Fn::Sub: ${self:provider.environment.BASE_NAME}-${AWS::AccountId} + Tags: + - Key: Name + Value: ${self:provider.environment.BASE_NAME}-${AWS::AccountId} # KnowledgeBase: # Type: AWS::Bedrock::KnowledgeBase From 9ebea33d83aaca532680d10f475a176f998c02cf Mon Sep 17 00:00:00 2001 From: nalbam Date: Thu, 22 Aug 2024 11:29:00 +0900 Subject: [PATCH 3/3] chore: Update S3Bucket resource tags in serverless.yml --- serverless.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/serverless.yml b/serverless.yml index 6e8984e..1fb3136 100644 --- a/serverless.yml +++ b/serverless.yml @@ -54,8 +54,8 @@ resources: AttributeName: expire_at Enabled: true Tags: - - Key: Name - Value: ${self:provider.environment.BASE_NAME}-context + - Key: Project + Value: ${self:provider.environment.BASE_NAME} S3Bucket: Type: AWS::S3::Bucket @@ -63,8 +63,8 @@ resources: BucketName: Fn::Sub: ${self:provider.environment.BASE_NAME}-${AWS::AccountId} Tags: - - Key: Name - Value: ${self:provider.environment.BASE_NAME}-${AWS::AccountId} + - Key: Project + Value: ${self:provider.environment.BASE_NAME} # KnowledgeBase: # Type: AWS::Bedrock::KnowledgeBase