Merge pull request #41 from gw2efficiency/refactor-background-job-queue

Refactor jobs into background queues & use configuration files

.gitignore

@@ -3,4 +3,4 @@ node_modules/
 .idea/
 build/
 .DS_Store
-newrelic.js
+config/environment.production.js

.travis.yml

@@ -12,3 +12,4 @@ before_script:
   - tar -xvf /tmp/mongodb.tgz
   - mkdir /tmp/data
   - ${PWD}/mongodb-linux-x86_64-3.2.0/bin/mongod --dbpath /tmp/data --bind_ip 127.0.0.1 --port 27017 &> /dev/null &
+  - redis-server /etc/redis/redis.conf --port 6379 &

README.md

@@ -13,45 +13,78 @@
 
 - **Requirements:**
   - [MongoDB](http://mongodb.org/) as the database layer
+  - [Redis](http://redis.io/) for the priority job queue
   - A process manager to keep the processes running, in this example [pm2](https://github.com/Unitech/pm2)
   - (Optional) Some sort of caching like [Varnish](https://www.varnish-cache.org/)
 
 ```sh
-# Clone the repository and build the worker and server files
+# Clone the repository
 git clone https://github.com/gw2efficiency/gw2-api.com
 cd gw2-api.com/
+
+# Install the dependencies and build all files
 npm install
 npm run build
 
-# Build the initial database (this may take a few minutes)
-node build/bin/rebuild.js
+# Build the initial database using the cli tool
+node build/bin/cli.js full-rebuild
+
+# Start the job processing cluster
+pm2 start build/bin/worker.js --name="gw2api-worker" -i 3
+
+# Start the job scheduling
+pm2 start build/bin/scheduler.js --name="gw2api-scheduler"
 
-# Start a server cluster with 5 processes
-# Note: to start the server with logging to keymetrics.io
-# set the env variable "ENVIRONMENT=production"
+# Start the job monitoring interface (port 3000)
+KUE_USER='user' KUE_PASSWORD='password' pm2 start build/bin/kue.js --name="gw2api-kue"
+
+# Start a server cluster for API routes (port 8080)
 pm2 start build/bin/server.js --name="gw2api-server" -i 5
+```
 
-# Start the background worker
-pm2 start build/bin/worker.js --name="gw2api-worker"
+Logs for all processes will be written in `~/.pm2/logs`. 
 
-# Note: Logs will be written in "~/.pm2/logs"
+## CLI
+
+You can fire off any jobs using the included cli tool, either directly in the cli process
+or as usual as a queued job, which gets processed by the worker processes:
+
+```bash
+node build/bin/cli.js <job-name> # Execute job in this process
+node build/bin/cli.js <job-name> -q # Push the job in the queue
 ```
 
-## Rebuild specific parts
+The following jobs exist:
+
+- `full-rebuild` (clear the database and execute all jobs)
+- `item-list` (get all items)
+- `item-prices` (update the item prices)
+- `item-values` (calculate the item values)
+- `recipe-list` (get all recipes)
+- `crafting-prices` (calculate the crafting prices)
+- `skin-list` (get all skins)
+- `skin-prices` (calculate all skin prices)
+- `gem-price-history` (get the gem price history)
+- `item-last-known-prices` (get the last known sell prices, if they are missing)
+
+## Running in production
 
-You can also rebuild specific parts of the database using the following commands:
+When running the servers in production, use the `NODE_ENV` environment variable
+to set the environment to `production`. Note that this will load the 
+`config/environment.production.js` configuration, so you will have to copy the default 
+`config/environment.js` and make sure the values match your setup.
 
 ```bash
-node build/bin/rebuild.js # Full rebuild
-node build/bin/rebuild.js items # Rebuild all item specific data
-node build/bin/rebuild.js recipes # Rebuild all recipe specific data
-node build/bin/rebuild.js skins # Rebuild all skin specific data
-node build/bin/rebuild.js gems # Rebuild all gem specific data
+NODE_ENV=production pm2 start build/bin/worker.js --name="gw2api-worker" -i 3
+NODE_ENV=production pm2 start build/bin/scheduler.js --name="gw2api-scheduler"
+NODE_ENV=production pm2 start build/bin/kue.js --name="gw2api-kue"
+NODE_ENV=production pm2 start build/bin/server.js --name="gw2api-server" -i 5
 ```
 
 ## Tests
 
-**Note:** Requires a running instance of mongodb and will execute on a test database
+**Note:** Requires a running instance of mongodb and will execute on a test database. Also
+requires a running instance of redis and will **flush all existing jobs**.
 
 ```
 npm test

package.json

@@ -11,17 +11,21 @@
   "license": "MIT",
   "dependencies": {
     "babel-polyfill": "^6.7.4",
-    "chalk": "^1.1.1",
+    "basic-auth-connect": "^1.0.0",
+    "chalk": "^1.1.3",
+    "commander": "^2.9.0",
     "cron-parser": "^2.0.2",
     "escape-string-regexp": "^1.0.5",
     "exit": "^0.1.2",
+    "express": "^4.13.4",
     "gw2e-account-value": "^1.0.0",
     "gw2e-async-promises": "^1.0.2",
     "gw2e-gw2api-client": "^1.0.0",
     "gw2e-gw2api-scraping": "^1.0.1",
     "gw2e-recipe-calculation": "^1.0.1",
     "gw2e-recipe-nesting": "^2.0.0",
     "gw2e-requester": "^1.0.0",
+    "kue": "^0.11.0",
     "mongodb": "^2.1.7",
     "node-schedule": "^1.1.0",
     "pmx": "^0.6.1",

src/bin/cli.js

@@ -0,0 +1,64 @@
+require('babel-polyfill')
+const commander = require('commander')
+const mongo = require('../helpers/mongo.js')
+const createJob = require('../helpers/createJob.js')
+const chalk = require('chalk')
+const jobList = require('../config/jobs.js')
+
+// Fire off the cli
+let jobName = false
+commander
+  .arguments('<job>')
+  .option('-q, --queue', 'Queue the job (by default jobs get executed directly)')
+  .action((job) => jobName = job)
+  .parse(process.argv)
+
+// Check if the job name is valid
+let jobNames = jobList.map(j => j.name)
+if (!jobName || jobNames.indexOf(jobName) === -1) {
+  jobNames = jobNames.map(j => `  - ${j}`).join('\n')
+  console.log(chalk.bold.red(`You have to specify a valid job name:\n${jobNames}`))
+  process.exit()
+}
+
+let job = jobList.find(j => j.name === jobName)
+let verb = commander.queue ? 'Queueing' : 'Executing'
+console.log(chalk.green(`${verb} job "${job.data.title}" [${job.name}]`))
+
+// Mock the "done" functionality that jobs expect and
+// log all results or errors out to the user
+function doneMock (err, result) {
+  if (err) {
+    console.log(chalk.bold.red(`An error occurred in the job:\n${err}`))
+    process.exit()
+  }
+
+  let output = result
+    ? `Job finished successfully:\n${result}`
+    : 'Job finished successfully'
+  console.log(chalk.green(output))
+  process.exit()
+}
+
+// Queue the job and wait for it to be processed by the workers
+if (commander.queue) {
+  createJob({
+    name: job.name,
+    data: {title: `[CLI] ${job.data.title}`},
+    priority: 'high',
+    callback: doneMock
+  })
+
+  console.log(chalk.gray('Waiting till the queued job gets processed...'))
+}
+
+// Execute the job in this process
+if (!commander.queue) {
+  const jobFunction = require(job.path)
+  let jobMock = {log: console.log}
+
+  mongo.connect().then(() => {
+    jobFunction(jobMock, doneMock)
+      .catch(err => console.log(chalk.bold.red(`An error occurred in the job:\n${err}`)))
+  })
+}

src/bin/kue.js

@@ -0,0 +1,13 @@
+const kue = require('../helpers/kue.js')
+const express = require('express')
+const auth = require('basic-auth-connect')
+const logger = require('../helpers/logger.js')
+const config = require('../config/application.js')
+
+const app = express()
+kue.createQueue()
+app.use(auth(config.kue.username, config.kue.password))
+app.use('/', kue.app)
+app.listen(config.kue.port, () => logger.info(`Server listening on port ${config.kue.port}`))
+
+module.exports = app

src/bin/scheduler.js

@@ -0,0 +1,11 @@
+// This process just handles the scheduling of background jobs.
+// It pushes the tasks into a priority queue to be picked up by the worker processes.
+
+const createJob = require('../helpers/createJob.js')
+const jobList = require('../config/jobs.js')
+
+jobList
+  .filter(job => job.schedule)
+  .map(job => createJob(job))
+
+module.exports = jobList

src/bin/server.js

@@ -1,7 +1,8 @@
 require('babel-polyfill')
+const config = require('../config/application.js')
 
 // If we are on the production environment, enable logging
-if (process.env.ENVIRONMENT === 'production') {
+if (config.keymetrics.logging) {
   require('pmx').init({http: true})
 }
 
@@ -20,7 +21,7 @@ server.use(restify.queryParser())
 server.use(restify.bodyParser())
 setupRoutes(server, routes)
 setupErrorHandling(server)
-server.listen(8080, () => logger.info('Server listening on port 8080'))
+server.listen(config.server.port, () => logger.info(`Server listening on port ${config.server.port}`))
 
 // Export the server for testing purposes
 module.exports = server

src/bin/worker.js

@@ -1,9 +1,15 @@
 require('babel-polyfill')
+const kue = require('../helpers/kue.js')
+const queue = kue.createQueue()
 const mongo = require('../helpers/mongo.js')
-const schedule = require('../helpers/worker.js')
-const scheduledTasks = require('../config/schedule.js')
+const wrapJob = require('../helpers/wrapJob.js')
+const jobList = require('../config/jobs.js')
 
-// Connect to the database and start the scheduling
 mongo.connect().then(() => {
-  scheduledTasks.map(task => schedule(task[0], task[1]))
+  jobList.map(job => {
+    let jobFunction = require(job.path)
+    queue.process(job.name, wrapJob(jobFunction))
+  })
+
+  console.log(`${jobList.length} jobs loaded, waiting for queued entries`)
 })

src/config/application.js

@@ -0,0 +1,10 @@
+// This is the file that the application loads, which then in turn
+// loads the environment configuration for the specific NODE_ENV
+
+const path = process.env.NODE_ENV === 'production'
+  ? '../config/environment.production.js'
+  : '../config/environment.js'
+
+const configuration = require(path)
+
+module.exports = configuration

src/config/environment.js

@@ -0,0 +1,37 @@
+const environment = {
+  server: {
+    // port of the api server
+    port: 8080,
+    // languages to support and the default language (needs to be in the languages)
+    languages: ['de', 'en', 'fr', 'es'],
+    defaultLanguage: 'en'
+  },
+  kue: {
+    // port, username and password of the job monitoring server
+    port: 4000,
+    username: 'username',
+    password: 'password',
+    // redis prefix for queued jobs
+    prefix: 'kue-'
+  },
+  mongo: {
+    // mongodb connection url
+    url: 'mongodb://127.0.0.1:27017/gw2api'
+  },
+  redis: {
+    // redis connection settings, you can see all options here:
+    // https://github.com/NodeRedis/node_redis#options-object-properties
+    port: 6379,
+    host: '127.0.0.1'
+  },
+  keymetrics: {
+    // enable/disable logging http requests to http://keymetrics.io
+    logging: false
+  },
+  gw2api: {
+    // retries for failed api requests (to the official api)
+    retries: 5
+  }
+}
+
+module.exports = environment