Upload to s3 (#50)

Currently the docker build uses a shared volume (`/out`) to place the built files on the system. This change allows us to PUT the tarballs to any url via the environment.

This allows us to leverage hyper.sh or AWS ECS Fargate (or any other docker service) to build and upload the files.

This upgrades npm for docker and vagrant. It's much faster and uses less memory.

For vagrant you should rebuild your vm to get the new script and npm.

Author: reconbot

.travis.yml

@@ -8,5 +8,5 @@ before_install:
 - docker build ./ -t t2-compiler:dev
 
 script:
-- docker run --rm -v `pwd`/out:/out t2-compiler:dev [email protected] /out 6.5.0 release
+- docker run --rm -v `pwd`/out:/out t2-compiler:dev [email protected] 6.5.0 release /out
 - test -f out/serialport-4.0.7-Release-node-v48-linux-mipsel.tgz

Dockerfile

@@ -18,10 +18,11 @@ RUN ["/bin/bash", "-c", "curl -s -o - https://raw.githubusercontent.com/creation
 # Install node 6.5.0
 RUN ["/bin/bash", "-c", ". /root/.nvm/nvm.sh \
   && nvm install 6.5.0 \
+  && npm install -g npm \
   && npm install -g pre-gypify node-pre-gyp node-gyp \
   && node-gyp install 6.5.0 \
   "]
 
-COPY ./compile.sh /root/
+COPY compile.sh upload-files.js /root/
 ENTRYPOINT ["/root/compile.sh"]
 CMD ["--help"]

README.md

@@ -19,7 +19,7 @@ Install vagrant
 
 ```
 vagrant up
-./compile-vagrant.sh serialport@2.0.5
+./compile-vagrant.sh serialport@6
 ```
 
 Look in the 'out' directory
@@ -29,8 +29,8 @@ Look in the 'out' directory
 If you want to use docker you can run;
 
 ```bash
-# puts the output in the `./out` directory (wont overwrite existing files)
-./compile-docker.sh serialport@4.0.0
+# puts the output in the `./out` directory (adds new files)
+./compile-docker.sh serialport@6
 ```
 
 To update to the latest t2-compiler from docker hub.
@@ -39,6 +39,13 @@ To update to the latest t2-compiler from docker hub.
 docker pull tessel/t2-compiler
 ```
 
+To output the build on the last line of docker output in JSON containing BASE64 encoded strings
+```bash
+docker run --rm tessel/t2-compiler $1 6.5.0 release JSON
+# build output followed by file contents
+# {"serialport-6.0.3-Release-node-v46-linux-mipsel.tgz":"H4s....."}
+```
+
 #### Developing the compiler
 
 To build your local Dockerfile
@@ -52,7 +59,9 @@ docker images
 # t2-compiler:dev      latest              75f126974601        About a minute ago   1.281 GB
 
 # Run the local image you've built
-docker run --rm -v `pwd`/out:/out t2-compiler:dev [email protected] /out
+docker run --rm -v `pwd`/out:/out t2-compiler:dev serialport 6.5.0 release /out
+# or
+docker run --rm t2-compiler:dev serialport 6.5.0 release JSON
 ```
 
 To get an interactive shell run

Vagrantfile

@@ -38,6 +38,7 @@ Vagrant.configure(2) do |config|
     # Install node 6.5.0
     . /root/.nvm/nvm.sh \
       && nvm install 6.5.0 \
+      && npm install -g npm \
       && npm install -g pre-gypify node-pre-gyp node-gyp \
       && node-gyp install 6.5.0
 

compile-docker.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -ex
-docker run --rm -v `pwd`/out:/out tessel/t2-compiler $1 /out 6.5.0 release
-docker run --rm -v `pwd`/out:/out tessel/t2-compiler $1 /out 6.5.0 debug
+docker run --rm -v `pwd`/out:/out tessel/t2-compiler $1 6.5.0 release /out
+docker run --rm -v `pwd`/out:/out tessel/t2-compiler $1 6.5.0 debug /out

compile-vagrant.sh

@@ -3,12 +3,12 @@ set -e
 
 PACKAGE_NAME=$1
 
-if [[ "$PACKAGE_NAME" == '' ]]; then
+if [[ $PACKAGE_NAME == '' ]]; then
   echo 'Usage: compile.sh package-name'
   exit 1
 fi
 
 cd $(dirname $0)
 mkdir -p ./out
-vagrant ssh -c "sudo su root /root/compile.sh $PACKAGE_NAME /root/out/ 6.5.0 release"
-vagrant ssh -c "sudo su root /root/compile.sh $PACKAGE_NAME /root/out/ 6.5.0 debug"
+vagrant ssh -c "sudo su root /root/compile.sh $PACKAGE_NAME 6.5.0 release /root/out/"
+vagrant ssh -c "sudo su root /root/compile.sh $PACKAGE_NAME 6.5.0 debug /root/out/"

compile.sh

@@ -1,25 +1,40 @@
 #!/bin/bash
 set -e
-
+echo "# compile.sh $1 $2 $3 $4"
 PACKAGE_NAME=$1
-OUTPUT_DIR=$2
+
+echo "loading nvm"
 . /root/.nvm/nvm.sh
-[ -n "$3" ] && nvm use $3
-RELEASE_TYPE=$4
+
+RELEASE_TYPE=$3
+OUTPUT_DIR=$4
+
+if [ -n $2 ]; then
+  echo "# installing node $2"
+  nvm install $2
+fi
+
 NODE_VERSION=`node -p process.versions.node`
 
 ARCH=mipsel
 export STAGING_DIR=/root
 
-if [[ "$PACKAGE_NAME" == '--help' || "$PACKAGE_NAME" == '' ]]; then
+if [[ $PACKAGE_NAME == '--help' || $PACKAGE_NAME == '' ]]; then
   echo "Usage: "
-	echo "    $0 [package name]<@version> [output_dir/]"
+	echo "    $0 [package name]<@version> [nodeversion releaseType outputdir]"
 	exit 1
 fi
 
-if [ ! -d "$OUTPUT_DIR" ]; then
-  (>&2 echo "ERROR: The output directory ${OUTPUT_DIR} doesn't exist")
-  exit 1
+if [[ $OUTPUT_DIR != 'S3' ]]; then
+  if [ ! -d "$OUTPUT_DIR" ]; then
+    (>&2 echo "ERROR: The output directory ${OUTPUT_DIR} doesn't exist, must be a valid directory or JSON")
+    exit 1
+  fi
+else
+  if [ ! -n $S3_RELEASE_URL ]; then
+    (>&2 echo "ERROR: The S3_RELEASE_URL variable should be set for upload to S3")
+    exit 1
+  fi
 fi
 
 cd $(dirname $0)
@@ -43,7 +58,7 @@ export PATH=$TOOLCHAIN_DIR/bin:$PATH
 export CPPPATH=$TARGET_DIR/usr/include
 export LIBPATH=$TARGET_DIR/usr/lib
 
-#TODO: anything better than this hack?
+# TODO: anything better than this hack?
 OPTS="-I $SYSROOT/usr/include -L $TOOLCHAIN_DIR/lib -L $SYSROOT/usr/lib -L $SYSROOT/lib"
 
 export CC="${TARGET_CROSS}gcc $OPTS"
@@ -70,10 +85,15 @@ if [[ $RELEASE_TYPE == "debug" ]]; then
   echo "Debug build"
   node-pre-gyp rebuild --target_platform=linux --target_arch=$ARCH --target=$NODE_VERSION --debug
   node-pre-gyp package --target_platform=linux --target_arch=$ARCH --target=$NODE_VERSION --debug
-  mv -vn build/stage/*.tgz $OUTPUT_DIR
 else
   echo "Release build"
   node-pre-gyp rebuild --target_platform=linux --target_arch=$ARCH --target=$NODE_VERSION
   node-pre-gyp package --target_platform=linux --target_arch=$ARCH --target=$NODE_VERSION
-  mv -vn build/stage/*.tgz $OUTPUT_DIR
+fi
+
+cd $(dirname $0)
+if [[ $OUTPUT_DIR == 'S3' ]]; then
+  node ./upload-files.js build/package/build/stage
+else
+  mv -vn build/package/build/stage/*.tgz $OUTPUT_DIR
 fi

test-output-url.js

@@ -0,0 +1,12 @@
+const AWS = require('aws-sdk')
+const s3 = new AWS.S3()
+
+const s3Bucket = 'packages.tessel.io'
+
+const url = s3.getSignedUrl('putObject', {
+  Key: 'test/' + process.argv[2],
+  Bucket: s3Bucket,
+  ACL: 'public-read'
+})
+
+console.log(url)

upload-files.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+const fs = require('fs')
+const https = require('https')
+const path = require('path')
+const url = require('url')
+const outputDir = process.argv[2]
+
+const releaseURL = url.parse(process.env.S3_RELEASE_URL)
+
+if (!outputDir) {
+  console.log('Error no output dir')
+  console.log(`Usage: ${__filename} OUTPUT_DIR`)
+  process.exit(1)
+}
+
+const files = fs.readdirSync(outputDir).filter(file => file.match(/\.tgz$/))
+
+files.forEach(filename => {
+  console.log(`Starting request to S3 for ${filename}: ${S3_RELEASE_URL}`)
+
+  const stream = fs.createReadStream(path.join(outputDir, filename))
+  const request = https.request(
+    Object.assign(
+      releaseURL,
+      {
+        method: 'PUT',
+        headers: {
+          'Content-Length': fs.statSync(path.join(outputDir, filename)).size,
+        },
+      }
+    ),
+    (response) => response.pipe(process.stdout),
+  )
+
+  stream.pipe(request)
+})