diff --git a/server/package.json b/server/package.json index 331ef42..bd56014 100644 --- a/server/package.json +++ b/server/package.json @@ -4,7 +4,7 @@ "description": "Deighton AR Server", "main": "src/server.js", "scripts": { - "start": "babel-node src/index.js", + "start": "babel-node src/dar-server.js", "start:prod": "NODE_ENV=production npm start", "build": "babel src -d dist -s", "serve": "NODE_ENV=production node dist/server.js", diff --git a/server/src/DARServer.js b/server/src/DARServer.js new file mode 100644 index 0000000..84d316e --- /dev/null +++ b/server/src/DARServer.js @@ -0,0 +1,98 @@ +import childProcess from 'child_process' +import path from 'path' +import chalk from 'chalk' +import timers from 'timers' +import autoBind from 'auto-bind2' + +export class DARServer { + constructor(toolName, log) { + autoBind(this) + this.toolName = toolName + this.log = log + this.actors = [ + { name: 'api' }, + { name: 'email' }, + { name: 'image' }, + ] + } + + restart(actor) { + const timeSinceStart = Date.now() - actor.startTime + + if (timeSinceStart < actor.timeToInit) { + actor.backOff += 1 + } else { + actor.backOff = 0 + } + + const backOffTime = (Math.pow(2, Math.min(actor.backOff, 7)) - 1) * 1000 + + if (backOffTime > 0) { + this.log.warn(`Actor '${actor.name}' died quickly, waiting ${Math.floor(backOffTime / 1000)} seconds to restart`) + } else { + this.log.warn(`Actor ${actor.name} died, restarting`) + } + + timers.setTimeout(() => { + actor.starts += 1 + actor.startTime = Date.now() + actor.proc = childProcess.fork(actor.modulePath) + actor.proc.on('exit', (code, signal) => { + // Don't restart if the exit was clean or Control+C + if (code === 0 || signal === 'SIGINT') { + this.log.info(`Actor ${actor.name} terminated normally`) + return + } + + this.restart(actor) + }) + this.log.info(`Restarted actor '${actor.name}', pid ${actor.proc.pid}`) + }, backOffTime) + } + + async run() { + let promises = [] + + this.actors.forEach((actor) => { + promises.push(new Promise((resolve, reject) => { + actor.modulePath = path.join(__dirname, actor.name) + actor.startTime = Date.now() + actor.backOff = 0 + actor.timeToInit = actor.timeToInit || 3000 + actor.proc = childProcess.fork(actor.modulePath) + actor.proc.on('exit', (code, signal) => { + const timeSinceStart = Date.now() - actor.startTime + + if (timeSinceStart < actor.timeToInit) { + this.log.error(`Actor '${actor.name}' exited during initialization`) + + if (this.actors) { + this.actors.forEach((otherActor) => { + if (otherActor !== actor) { + this.log.info(`Terminating actor ${actor.name}, pid ${otherActor.proc.pid}`) + otherActor.proc.kill('SIGINT') + otherActor.proc = null + } + }) + + actor.proc = null + this.actors = null + } + + return reject(new Error('All actors must initialize on first start')) + } + + this.restart(actor) + }) + timers.setTimeout(() => { + if (actor.proc) { + resolve() + } + }, actor.timeToInit) + this.log.info(`Started actor '${actor.name}', pid ${actor.proc.pid}`) + })) // new Promise() + }) + + return Promise.all(promises) + } +} diff --git a/server/src/dar-server.js b/server/src/dar-server.js new file mode 100644 index 0000000..7936570 --- /dev/null +++ b/server/src/dar-server.js @@ -0,0 +1,27 @@ +#!/usr/bin/env node +import { DARServer } from './DARServer' +import pino from 'pino' +import * as pinoExpress from 'pino-pretty-express' +import path from 'path' + +const serviceName = 'dar-server' +const isProduction = (process.env.NODE_ENV == 'production') +let log = null + +if (isProduction) { + log = pino( { name: serviceName }, + fs.createWriteStream(path.join(config.get('logDir'), serviceName + '.log')) + ) +} else { + const pretty = pinoExpress.pretty({}) + pretty.pipe(process.stdout) + log = pino({ name: serviceName }, pretty) +} + +const tool = new DARServer(path.basename(process.argv[1], '.js'), log) + +tool.run(process.argv.slice(2)).then((exitCode) => { + process.exitCode = exitCode +}).catch((err) => { + console.error(err) +}) diff --git a/server/src/server.js b/server/src/server.js deleted file mode 100644 index 3e13fbb..0000000 --- a/server/src/server.js +++ /dev/null @@ -1,9 +0,0 @@ -import childProcess from 'child_process' - -const actors = [ 'api', 'email', 'image' ] - -// TODO: spawn index.js in each of these sub-directories - -// TODO: If any child exits, wait for a back-off period and then restart - -// TODO: Gradually increase back-off period if process fails again too quickly