Merge branch 'staging' into fix-asset-uploader-error-logging

Author: alex-chew

README.md

@@ -30,23 +30,37 @@ If you have previously set up a v1 developer portal (non-SAM deployed), you will
 #### Deploy
 
 Run:
+
 >In the command below, replace the `your-lambda-artifacts-bucket-name` with the name of a bucket that you manage and that already exists. Then, run:
+
 ```bash
-sam package --template-file ./cloudformation/template.yaml --output-template-file ./cloudformation/packaged.yaml --s3-bucket your-lambda-artifacts-bucket-name
+sam package --template-file ./cloudformation/template.yaml \
+    --output-template-file ./cloudformation/packaged.yaml \
+    --s3-bucket your-lambda-artifacts-bucket-name
 ```
 
-Then run: 
+Then run:
+
 >In the command below, replace the `your-lambda-artifacts-bucket-name` with the name of a bucket that you manage and that already exists, and replace `custom-prefix` with some prefix that is globally unique, like your org name or username. Then, run:
+
 ```bash
-sam deploy --template-file ./cloudformation/packaged.yaml --stack-name "dev-portal" --s3-bucket your-lambda-artifacts-bucket-name --capabilities CAPABILITY_NAMED_IAM --parameter-overrides DevPortalSiteS3BucketName="custom-prefix-dev-portal-static-assets" ArtifactsS3BucketName="custom-prefix-dev-portal-artifacts" CognitoDomainNameOrPrefix="custom-prefix"
+sam deploy --template-file ./cloudformation/packaged.yaml \
+    --stack-name "dev-portal" \
+    --s3-bucket your-lambda-artifacts-bucket-name \
+    --capabilities CAPABILITY_NAMED_IAM \
+    --parameter-overrides \
+    DevPortalSiteS3BucketName="custom-prefix-dev-portal-static-assets" \
+    ArtifactsS3BucketName="custom-prefix-dev-portal-artifacts" \
+    CognitoDomainNameOrPrefix="custom-prefix"
 ```
 
 The command will exit when the stack creation is successful. If you'd like to watch it create in real-time, you can log into the cloudformation console.
 
 To get the URL for the newly created developer portal instance, find the websiteURL field in the cloudformation console's outputs or run this command:
 
 ```bash
-aws cloudformation describe-stacks --query "Stacks[?StackName=='dev-portal'][Outputs[?OutputKey=='WebsiteURL']][][].OutputValue"
+aws cloudformation describe-stacks --query \
+    "Stacks[?StackName=='dev-portal'][Outputs[?OutputKey=='WebsiteURL']][][].OutputValue"
 ```
 
 You can override any of the parameters in the template using the `--parameter-overrides key="value"` format. This will be necessary if you intend to deploy several instances of the developer portal or customize some of the features. You can see a full list of overridable parameters in `cloudformation/template.yaml` under the `Parameters` section.

cloudformation/template.yaml

@@ -74,6 +74,10 @@ Parameters:
     Type: String
     Description: By default, a static asset rebuild doesn't overwrite custom-content. Provide the value `overwrite-content` to replace the custom-content with your local version. Don't do this unless you know what you're doing -- all custom changes in your s3 bucket will be lost. 
     Default: ''
+    AllowedValues:
+      - 'overwrite-content'
+      - ''
+    ConstraintDescription: Malformed input - Parameter StaticAssetRebuildMode value must be either 'overwrite-content' or left blank.
 
   MarketplaceSubscriptionTopicProductCode:
     Type: String
@@ -108,11 +112,19 @@ Parameters:
     Type: String
     Description: Only applicable if creating a custom domain name for your dev portal. Defaults to false, and you'll need to provide your own nameserver hosting. If set to true, a Route53 HostedZone and RecordSet are created for you.
     Default: 'false'
+    AllowedValues: 
+      - 'false'
+      - 'true'
+    ConstraintDescription: Malformed input - Parameter UseRoute53Nameservers value must be either 'true' or 'false'
 
   DevelopmentMode:
     Type: String
     Description: Enabling this weakens security features (OAI, SSL, site S3 bucket with public read ACLs, Cognito callback verification, CORS, etc.) for easier development. It also breaks frontend routing (except to /index.html), including deep linking and page refresh. Do not enable this in production! Additionally, do not update a stack that was previously in development mode to be a production stack; instead, make a new stack that has never been in development mode.
     Default: 'false'
+    AllowedValues: 
+      - 'false'
+      - 'true'
+    ConstraintDescription: Malformed input - Parameter DevelopmentMode value must be either 'true' or 'false'
 
 Conditions:
   UseCustomDomainName: !And [!And [!Not [!Equals [!Ref CustomDomainName, '']], !Not [!Equals [!Ref CustomDomainNameAcmCertArn, '']]], !Condition NotDevelopmentMode]

lambdas/backend/_common/customers-controller.js

@@ -3,6 +3,7 @@
 
 'use strict'
 const AWS = require('aws-sdk')
+const { getAllUsagePlans } = require('../shared/get-all-usage-plans')
 
 const dynamoDb = new AWS.DynamoDB.DocumentClient()
 const apigateway = new AWS.APIGateway()
@@ -195,10 +196,9 @@ function getUsagePlansForCustomer(cognitoIdentityId, error, callback) {
                 keyId,
                 limit: 1000
             }
-            apigateway.getUsagePlans(params, (err, usagePlansData) => {
-                if (err) error(err)
-                else callback(usagePlansData)
-            })
+            getAllUsagePlans(apigateway, params)
+                .then(usagePlansData => callback({ items: usagePlansData }))
+                .catch(err => error(err))
         }
     })
 }
@@ -209,26 +209,22 @@ function getUsagePlanForProductCode(productCode, error, callback) {
     // do a linear scan of usage plans for name matching productCode
     var params = {
         limit: 1000
-    };
-    apigateway.getUsagePlans(params, function(err, data) {
-        if (err) {
-            error(err)
-        } else {
-            console.log(`Got usage plans ${JSON.stringify(data.items)}`)
+    }
+    getAllUsagePlans(apigateway, params).then(usagePlans => {
+        console.log(`Got usage plans ${JSON.stringify(usagePlans)}`)
 
-            // note: ensure that only one usage plan maps to a given marketplace product code
-            const usageplan = data.items.find(function (item) {
-                return item.productCode !== undefined && item.productCode === productCode
-            })
-            if (usageplan !== undefined) {
-                console.log(`Found usage plan matching ${productCode}`)
-                callback(usageplan)
-            } else {
-                console.log(`Couldn't find usageplan matching product code ${productCode}`)
-                error(`Couldn't find usageplan matching product code ${productCode}`)
-            }
+        // note: ensure that only one usage plan maps to a given marketplace product code
+        const usageplan = usagePlans.find(function (item) {
+            return item.productCode !== undefined && item.productCode === productCode
+        })
+        if (usageplan !== undefined) {
+            console.log(`Found usage plan matching ${productCode}`)
+            callback(usageplan)
+        } else {
+            console.log(`Couldn't find usageplan matching product code ${productCode}`)
+            error(`Couldn't find usageplan matching product code ${productCode}`)
         }
-    });
+    }).catch(err => error(err))
 }
 
 function updateCustomerMarketplaceId(cognitoIdentityId, marketplaceCustomerId, error, success) {
@@ -323,16 +319,6 @@ function updateCustomerApiKeyId(cognitoIdentityId, apiKeyId, error, success) {
     })
 }
 
-// function getUsagePlans(error, callback) {
-//     const params = {
-//         limit: 1000
-//     }
-//     apigateway.getUsagePlans(params, (err, data) => {
-//         if (err) error(err)
-//         else callback(data)
-//     })
-// }
-
 module.exports = {
     ensureCustomerItem,
     subscribe,

lambdas/backend/express-route-handlers.js

@@ -3,6 +3,7 @@ const feedbackController = require('./_common/feedback-controller.js')
 const AWS = require('aws-sdk')
 const catalog = require('./catalog/index')
 const hash = require('object-hash')
+const { getAllUsagePlans } = require('./shared/get-all-usage-plans')
 
 const Datauri = require('datauri')
 
@@ -421,7 +422,7 @@ async function getAdminCatalogVisibility(req, res) {
             })
         })
 
-        let usagePlans = await exports.apigateway.getUsagePlans().promise()
+        let usagePlans = await getAllUsagePlans(exports.apigateway)
 
         // In the case of apiGateway APIs, the client doesn't know if there are usage plan associated or not
         // so we need to provide that information. This can't be merged with the above loop:
@@ -431,7 +432,7 @@ async function getAdminCatalogVisibility(req, res) {
         visibility.apiGateway.map((apiEntry) => {
             apiEntry.subscribable = false
 
-            usagePlans.items.forEach((usagePlan) => {
+            usagePlans.forEach((usagePlan) => {
                 usagePlan.apiStages.forEach((apiStage) => {
                     if(apiEntry.id === apiStage.apiId && apiEntry.stage === apiStage.stage) {
                         apiEntry.subscribable = true

lambdas/backend/shared/get-all-usage-plans.js

@@ -0,0 +1,33 @@
+/**
+ * Fetches all usage plans, combining all pages into a single array.
+ *
+ * @param apiGateway
+ *    an instance of `AWS.APIGateway` to use for API calls
+ *
+ * @param paramOverrides
+ *    a parameter object passed in calls to `APIGateway.getUsagePlans`
+ *
+ * @returns
+ *    a Promise resolving with an array of items returned from
+ *    `APIGateway.getUsagePlans` calls
+ */
+async function getAllUsagePlans(apiGateway, paramOverrides = {}) {
+  // The maximum allowed value of `limit` is 500 according to
+  // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/APIGateway.html#getUsagePlans-property
+  const defaultParams = {limit: 500, ...paramOverrides}
+
+  console.log('Fetching first page of usage plans')
+  let response = await apiGateway.getUsagePlans(defaultParams).promise()
+  const usagePlans = response.items
+
+  while (response.position) {
+    console.log(`Fetching next page of usage plans, at position=[${response.position}]`)
+    const nextParams = {...defaultParams, position: response.position}
+    response = await apiGateway.getUsagePlans(nextParams).promise()
+    usagePlans.push(...response.items)
+  }
+
+  return usagePlans
+}
+
+module.exports = { getAllUsagePlans }

lambdas/catalog-updater/index.js

@@ -12,6 +12,8 @@ let AWS = require('aws-sdk'),
   bucketName = '',
   hash = require('object-hash')
 
+const { getAllUsagePlans } = require('./shared/get-all-usage-plans')
+
 /**
  * Takes in an s3 listObjectsV2 object and returns whether it's a "swagger file" (one ending in .JSON, .YAML, or .YML),
  * and whether it's in the catalog folder (S3 Key starts with "catalog/").
@@ -185,10 +187,9 @@ function buildCatalog(swaggerFiles, sdkGeneration) {
     generic: []
   }
 
-  return exports.gateway.getUsagePlans({}).promise()
-    .then((result) => {
-      console.log(`usagePlans: ${JSON.stringify(result.items, null, 4)}`)
-      let usagePlans = result.items
+  return getAllUsagePlans(exports.gateway)
+    .then(usagePlans => {
+      console.log(`usagePlans: ${JSON.stringify(usagePlans, null, 4)}`)
       for (let i = 0; i < usagePlans.length; i++) {
           catalog.apiGateway[i] = usagePlanToCatalogObject(usagePlans[i], swaggerFiles, sdkGeneration)
       }
@@ -252,4 +253,4 @@ exports = module.exports = {
   gateway: new AWS.APIGateway(),
   handler,
   hash
-}
+}

lambdas/catalog-updater/shared/get-all-usage-plans.js

@@ -0,0 +1,33 @@
+/**
+ * Fetches all usage plans, combining all pages into a single array.
+ *
+ * @param apiGateway
+ *    an instance of `AWS.APIGateway` to use for API calls
+ *
+ * @param paramOverrides
+ *    a parameter object passed in calls to `APIGateway.getUsagePlans`
+ *
+ * @returns
+ *    a Promise resolving with an array of items returned from
+ *    `APIGateway.getUsagePlans` calls
+ */
+async function getAllUsagePlans(apiGateway, paramOverrides = {}) {
+  // The maximum allowed value of `limit` is 500 according to
+  // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/APIGateway.html#getUsagePlans-property
+  const defaultParams = {limit: 500, ...paramOverrides}
+
+  console.log('Fetching first page of usage plans')
+  let response = await apiGateway.getUsagePlans(defaultParams).promise()
+  const usagePlans = response.items
+
+  while (response.position) {
+    console.log(`Fetching next page of usage plans, at position=[${response.position}]`)
+    const nextParams = {...defaultParams, position: response.position}
+    response = await apiGateway.getUsagePlans(nextParams).promise()
+    usagePlans.push(...response.items)
+  }
+
+  return usagePlans
+}
+
+module.exports = { getAllUsagePlans }

lambdas/shared/__tests__/get-all-usage-plans-test.js

@@ -0,0 +1,106 @@
+const { getAllUsagePlans } = require('../get-all-usage-plans')
+
+const promiser = require('../../setup-jest').promiser
+
+const mockUsagePlanItem = () => ({
+  id: '1a2b3c',
+  name: '1a2b3c',
+  apiStages: [
+    {
+      apiId: 'anmlcrckrs',
+      stage: 'prod',
+    },
+    {
+      apiId: 'jlpnochips',
+      stage: 'beta',
+    },
+  ],
+  throttle: {
+    burstLimit: 10,
+    rateLimit: 10,
+  },
+  quota: {
+    limit: 10000,
+    offset: 0,
+    period: 'DAY',
+  }
+})
+
+describe('getAllUsagePlans', () => {
+  test('returns all usage plans, when none exist', async () => {
+    const mockApiGateway = {
+      getUsagePlans: jest.fn().mockReturnValueOnce(promiser({
+        items: []
+      }))
+    }
+
+    const result = await getAllUsagePlans(mockApiGateway)
+    const mocked = mockApiGateway.getUsagePlans.mock
+    expect(mocked.calls.length).toBe(1)
+    expect(mocked.calls[0][0]).not.toHaveProperty('position')
+    expect(result).toHaveLength(0)
+  })
+
+  test('returns all usage plans, when only one page of usage plans exists', async () => {
+    const mockApiGateway = {
+      getUsagePlans: jest.fn().mockReturnValueOnce(promiser({
+        items: [
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+        ]
+      }))
+    }
+
+    const result = await getAllUsagePlans(mockApiGateway)
+    const mocked = mockApiGateway.getUsagePlans.mock
+    expect(mocked.calls.length).toBe(1)
+    expect(mocked.calls[0][0]).not.toHaveProperty('position')
+    expect(result).toHaveLength(4)
+  })
+
+  test('returns all usage plans, when multiple pages of usage plans exist', async () => {
+    const mockApiGateway = {
+      getUsagePlans: jest.fn().mockReturnValueOnce(promiser({
+        items: [
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+        ],
+        position: 'qwertyuiopasdf%3D%3D',
+      })).mockReturnValueOnce(promiser({
+        items: [
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+        ],
+        position: 'zxcvbnm1234567%3D%3D',
+      })).mockReturnValueOnce(promiser({
+        items: [
+          mockUsagePlanItem(),
+          mockUsagePlanItem(),
+        ],
+      }))
+    }
+
+    const result = await getAllUsagePlans(mockApiGateway)
+    const mocked = mockApiGateway.getUsagePlans.mock
+    expect(mocked.calls.length).toBe(3)
+    expect(mocked.calls[0][0]).not.toHaveProperty('position')
+    expect(mocked.calls[1][0]).toHaveProperty('position', 'qwertyuiopasdf%3D%3D')
+    expect(mocked.calls[2][0]).toHaveProperty('position', 'zxcvbnm1234567%3D%3D')
+    expect(result).toHaveLength(10)
+  })
+
+  test('passes through an API Gateway request error', async () => {
+    const expectedError = {}
+    const mockApiGateway = {
+      getUsagePlans: jest.fn().mockReturnValueOnce(promiser(null, expectedError))
+    }
+
+    await expect(getAllUsagePlans(mockApiGateway)).rejects.toStrictEqual(expectedError)
+  })
+})