feat: remote hosted screenshot testing (material-components#12)

Author: patrickrodee

.eslintrc.yml

@@ -9,3 +9,5 @@ rules:
   require-jsdoc: off
   valid-jsdoc: off
   switch-colon-spacing: 0
+  max-len: [error, 120]
+  indent:  [error, 2, {"SwitchCase":1}]

.travis.yml

@@ -5,6 +5,10 @@ git:
 dist: trusty
 sudo: required
 
+branches:
+  only:
+    - master
+
 matrix:
   include:
     - node_js: 8
@@ -24,5 +28,3 @@ matrix:
         - ./test/screenshot/start.sh
         - sleep 10s
         - npm run test:image-diff
-      after_script:
-        - COMMIT_HASH=$(git rev-parse --short HEAD) npm run upload:screenshots

package-lock.json

@@ -5,7 +5,7 @@
   "scripts": {
     "start": "webpack-dev-server --config test/screenshot/webpack.config.js --content-base test/screenshot",
     "stop": "./test/screenshot/stop.sh",
-    "capture": "./node_modules/.bin/mocha --compilers js:babel-core/register --ui tdd test/screenshot/capture-suite.js",
+    "capture": "cross-env MDC_COMMIT_HASH=$(git rev-parse --short HEAD) MDC_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) mocha --compilers js:babel-core/register --ui tdd --timeout 30000 test/screenshot/capture-suite.js",
     "commitmsg": "validate-commit-msg",
     "fix": "eslint --fix packages test",
     "lint": "eslint packages test",
@@ -15,7 +15,7 @@
     "test:watch": "karma start karma.local.js --auto-watch",
     "test:unit": "karma start karma.local.js --single-run",
     "test:unit-ci": "karma start karma.ci.js --single-run",
-    "test:image-diff": "./node_modules/.bin/mocha --compilers js:babel-core/register --ui tdd --timeout 30000 test/screenshot/diff-suite.js",
+    "test:image-diff": "cross-env MDC_COMMIT_HASH=$(git rev-parse --short HEAD) MDC_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) mocha --compilers js:babel-core/register --ui tdd --timeout 30000 test/screenshot/diff-suite.js",
     "upload:screenshots": "node ./test/screenshot/upload-screenshots.js"
   },
   "config": {
@@ -36,13 +36,12 @@
     "babel-preset-react": "^6.24.1",
     "capture-chrome": "^2.0.0",
     "chai": "^4.1.2",
+    "cross-env": "^5.1.4",
     "css-loader": "^0.28.10",
     "eslint": "^3.19.0",
     "eslint-config-google": "^0.9.1",
     "eslint-plugin-react": "^7.7.0",
     "extract-text-webpack-plugin": "^3.0.2",
-    "fs-readfile-promise": "^3.0.1",
-    "glob": "^7.1.2",
     "husky": "^0.14.3",
     "istanbul": "^0.4.5",
     "istanbul-instrumenter-loader": "^3.0.0",

package.json

@@ -0,0 +1,4 @@
+{
+  "temporary-package/index.html": "57de91e42bd22a845074bae80f71eca3902ab7c6719f129960bc9018c79db95a",
+  "temporary-package/foo.html": "91f0795700eaba345092dd7507e046daa0d33e1136bad7992011ab454ddb1faf"
+}

test/screenshot/golden.json

@@ -1,60 +1,209 @@
+import {Readable} from 'stream';
+import {createHash} from 'crypto';
+import {readFile, writeFile} from 'fs';
+import {promisify} from 'util';
 import puppeteer from 'puppeteer';
 import compareImages from 'resemblejs/compareImages';
-import {promisify} from 'util';
-import {readFile, writeFile} from 'fs';
 import {assert} from 'chai';
+import Storage from '@google-cloud/storage';
 
 import comparisonOptions from './screenshot-comparison-options';
 
 const readFilePromise = promisify(readFile);
 const writeFilePromise = promisify(writeFile);
 
+const serviceAccountKey = process.env.MDC_GCLOUD_SERVICE_ACCOUNT_KEY;
+const branchName = process.env.MDC_BRANCH_NAME;
+const commitHash = process.env.MDC_COMMIT_HASH;
+const goldenFilePath = './test/screenshot/golden.json';
+const bucketName = 'screenshot-uploads';
+const defaultMetadata = {
+  commit: commitHash,
+  branch: branchName,
+};
+
+const storage = new Storage({
+  credentials: JSON.parse(serviceAccountKey),
+});
+
+const bucket = storage.bucket(bucketName);
+
 export default class Screenshot {
+  /**
+   * @param {string} urlPath The URL path to test
+   */
   constructor(urlPath) {
+    /** @private {string} */
     this.urlPath_ = urlPath;
-    this.imagePath_ = `${urlPath}.golden.png`;
-    this.snapshotImagePath_ = `${urlPath}.snapshot.png`;
-    this.diffPath_ = `${urlPath}.diff.png`;
     // TODO allow clients to specify capture-chrome options, like viewport size
   }
 
+  /**
+   * Captures a screenshot of the test URL and marks it as the new golden image
+   */
   capture() {
     test(this.urlPath_, async () => {
-      const url = `http://localhost:8080/${this.urlPath_}`;
-      const imagePath = `./test/screenshot/${this.imagePath_}`;
-      await this.createScreenshotTask_(url, imagePath);
+      const golden = await this.takeScreenshot_();
+      const goldenHash = this.generateImageHash_(golden);
+      const goldenPath = this.getImagePath_(goldenHash, 'golden');
+      await Promise.all([
+        this.saveImage_(goldenPath, golden),
+        this.saveGoldenHash_(goldenHash),
+      ]);
       return;
     });
   }
 
+  /**
+   * Diffs a screenshot of the test URL with the existing golden screenshot
+   */
   diff() {
     test(this.urlPath_, async () => {
-      const url = `http://localhost:8080/${this.urlPath_}`;
-      const imagePath = `./test/screenshot/${this.imagePath_}`;
-      const snapshotImagePath = `./test/screenshot/${this.snapshotImagePath_}`;
-      const diffPath = `./test/screenshot/${this.diffPath_}`;
-
-      const [newScreenshot, oldScreenshot] = await Promise.all([
-        this.createScreenshotTask_(url, snapshotImagePath),
-        readFilePromise(imagePath),
+      // Get the golden file path from the golden hash
+      const goldenHash = await this.getGoldenHash_();
+      const goldenPath = this.getImagePath_(goldenHash, 'golden');
+
+      // Take a snapshot and download the golden iamge
+      const [snapshot, golden] = await Promise.all([
+        this.takeScreenshot_(),
+        this.readImage_(goldenPath),
       ]);
 
-      const data = await compareImages(newScreenshot, oldScreenshot,
-        comparisonOptions);
+      // Compare the images
+      const data = await compareImages(snapshot, golden, comparisonOptions);
+      const diff = data.getBuffer();
+
+      // Use the same hash for the snapshot path and diff path so it's easy can associate the two
+      const snapshotHash = this.generateImageHash_(snapshot);
+      const snapshotPath = this.getImagePath_(snapshotHash, 'snapshot');
+      const diffPath = this.getImagePath_(snapshotHash, 'diff');
+      const metadata = {golden: goldenHash};
+
+      // Save the snapshot and the diff
+      await Promise.all([
+        this.saveImage_(snapshotPath, snapshot, metadata),
+        this.saveImage_(diffPath, diff, metadata),
+      ]);
+
+      return assert.isBelow(Number(data.misMatchPercentage), 0.01);
+    });
+  }
+
+  /**
+   * Generates a unique hash from an image's contents
+   * @param {!Buffer} imageBuffer The image buffer to hash
+   * @return {string}
+   * @private
+   */
+  generateImageHash_(imageBuffer) {
+    return createHash('sha256').update(imageBuffer).digest('hex');
+  }
+
+  /**
+   * Returns the golden hash
+   * @return {string|undefined}
+   * @private
+   */
+  async getGoldenHash_() {
+    const goldenFile = await readFilePromise(goldenFilePath);
+    const goldenJSON = JSON.parse(goldenFile);
+    return goldenJSON[this.urlPath_];
+  }
+
+  /**
+   * Returns the correct image path
+   * @param {string} imageHash The image hash
+   * @param {string} imageType The image type
+   * @return {string|undefined}
+   * @private
+   */
+  getImagePath_(imageHash, imageType) {
+    if (imageType === 'golden') {
+      return `${this.urlPath_}/${imageHash}.golden.png`;
+    }
+
+    if (['snapshot', 'diff'].includes(imageType)) {
+      return `${this.urlPath_}/${commitHash}/${imageHash}.${imageType}.png`;
+    }
+  }
+
+  /**
+   * Downloads an image from Google Cloud Storage
+   * @param {string} gcsFilePath The file path on Google Cloud Storage
+   * @return {Buffer}
+   * @private
+   */
+  async readImage_(gcsFilePath) {
+    const data = await bucket.file(gcsFilePath).download();
+    return data[0];
+  }
 
-      await writeFilePromise(diffPath, data.getBuffer());
+  /**
+   * Saves the golden hash
+   * @param {string} goldenHash The hash of the golden image
+   * @private
+   */
+  async saveGoldenHash_(goldenHash) {
+    const goldenFile = await readFilePromise(goldenFilePath);
+    const goldenJSON = JSON.parse(goldenFile);
+    goldenJSON[this.urlPath_] = goldenHash;
+    const goldenContent = JSON.stringify(goldenJSON, null, '  ');
+    await writeFilePromise(goldenFilePath, `${goldenContent}\r\n`);
+  }
 
-      assert.isBelow(Number(data.misMatchPercentage), 0.01);
+  /**
+   * Saves the given image to Google Cloud Storage with optional metadata
+   * @param {string} imagePath The path to the image
+   * @param {!Buffer} imageBuffer The image buffer
+   * @param {!Object=} customMetadata Optional custom metadata
+   * @private
+   */
+  async saveImage_(imagePath, imageBuffer, customMetadata={}) {
+    const metadata = Object.assign({}, defaultMetadata, customMetadata);
+    const file = bucket.file(imagePath);
+
+    // Check if file exists and exit if it does
+    const [exists] = await file.exists();
+    if (exists) {
+      console.log('✔︎ No changes to', imagePath);
       return;
+    }
+
+    // Create a new stream from the image buffer
+    let stream = new Readable();
+    stream.push(imageBuffer);
+    stream.push(null);
+
+    // The promise is resolved or rejected inside the stream event callbacks
+    return new Promise((resolve, reject) => {
+      stream.pipe(file.createWriteStream())
+        .on('error', (err) => {
+          reject(err);
+        }).on('finish', async () => {
+          try {
+            // Make the image public and set it's metadata
+            await file.makePublic();
+            await file.setMetadata({metadata});
+            console.log('✔︎ Uploaded', imagePath);
+            resolve();
+          } catch (err) {
+            reject(err);
+          }
+        });
     });
   }
 
-  async createScreenshotTask_(url, path) {
+  /**
+   * Takes a screenshot of the URL
+   * @return {!Buffer}
+   * @private
+   */
+  async takeScreenshot_() {
     const browser = await puppeteer.launch();
     const page = await browser.newPage();
-    await page.goto(url);
-    const image = await page.screenshot({path});
+    await page.goto(`http://localhost:8080/${this.urlPath_}`);
+    const imageBuffer = await page.screenshot();
     await browser.close();
-    return image;
+    return imageBuffer;
   }
 }