Merge pull request #19 from abhishek97/runModes

add support for mode parameter for POST /runs

Author: championswimmer

config.js

@@ -23,5 +23,14 @@ exports = module.exports = {
     PASS: process.env.AMQP_PASS || 'codingblocks',
     HOST: process.env.AMQP_HOST || 'localhost',
     PORT: process.env.AMQP_PORT || 5672
+  },
+
+  S3: {
+    endpoint: process.env.S3_ENDPOINT || 'localhost',
+    port: process.env.S3_PORT || 9000,
+    ssl: process.env.S3_SSL || false,
+    accessKey: process.env.S3_ACCESS_KEY || '',
+    secretKey: process.env.S3_SECRET_KEY || '',
+    bucket: process.env.S3_BUCKET || 'judge-submissions'
   }
 }

migrations/001-addOutputsToSubmission.sql

@@ -0,0 +1,2 @@
+alter table submissions 
+  add column outputs varchar[];

package.json

@@ -1,6 +1,6 @@
 {
   "name": "judge-api",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Judge API",
   "main": "dist/server.js",
   "repository": "https://github.com/coding-blocks/judge-api",
@@ -11,17 +11,21 @@
     "@types/amqplib": "^0.5.4",
     "amqplib": "^0.5.2",
     "apidoc": "^0.17.6",
+    "axios": "^0.18.0",
     "base-64": "^0.1.0",
     "debug": "^4.0.0",
     "express": "^4.16.2",
+    "minio": "^7.0.3",
     "pg": "^7.4.3",
     "pg-hstore": "^2.3.2",
-    "sequelize": "^4.22.6"
+    "sequelize": "^4.22.6",
+    "uuid": "^3.3.2"
   },
   "devDependencies": {
     "@types/chai": "^4.0.4",
     "@types/debug": "^0.0.30",
     "@types/express": "^4.0.39",
+    "@types/minio": "^7.0.1",
     "@types/mocha": "^5.0.0",
     "@types/request": "^2.0.8",
     "@types/sequelize": "^4.0.79",

src/db/models.ts

@@ -27,7 +27,7 @@ const Langs = db.define('langs', {
 })
 export type LangsAttributes = { lang_slug: string, lang_name:string, lang_version: string }
 
-const Submissions = db.define('submissions', {
+const Submissions = db.define<SubmissionInstance ,SubmissionAttributes>('submissions', {
   id: {
     type: Sequelize.INTEGER,
     autoIncrement: true,
@@ -42,7 +42,8 @@ const Submissions = db.define('submissions', {
   },
   start_time: Sequelize.DATE,
   end_time: Sequelize.DATE,
-  results: Sequelize.ARRAY(Sequelize.INTEGER)
+  results: Sequelize.ARRAY(Sequelize.INTEGER),
+  outputs: Sequelize.ARRAY(Sequelize.STRING),
 }, {
   paranoid: true, // We do not want to lose any submission data
   timestamps: false // Start and end times are already logged
@@ -53,8 +54,11 @@ export type SubmissionAttributes = {
   start_time: Date
   end_time?: Date
   results?: Array<number>
+  outputs?: Array<string>
 }
 
+export type SubmissionInstance = Sequelize.Instance<SubmissionAttributes> & SubmissionAttributes
+
 const ApiKeys = db.define('apikeys', {
   id: {
     type: Sequelize.INTEGER,

src/routes/api/run.ts

@@ -1,15 +1,20 @@
 import {Response, Router, Request} from 'express'
-import {SubmissionAttributes, Submissions} from '../../db/models'
+import axios from 'axios'
+
+import {SubmissionAttributes, Submissions, db} from '../../db/models'
 import {RunJob, queueJob, successListener} from '../../rabbitmq/jobqueue'
 import {isInvalidRunRequest} from '../../validators/SubmissionValidators'
+import {upload} from '../../utils/s3'
 import config = require('../../../config')
 
 const route: Router = Router()
 
 export type RunRequestBody = {
   source: string, //Base64 encoded
   lang: string,
-  stdin: string
+  stdin: string,
+  mode: string,
+  callback?: string
 }
 export interface RunRequest extends Request {
   body: RunRequestBody
@@ -21,7 +26,77 @@ export interface RunResponse {
   stderr: string
 }
 
-const runPool: {[x: number]: Response} = {}
+export type RunPoolElement = {
+  mode: string,
+  res: Response,
+  callback?: string
+}
+
+const runPool: {[x: number]: RunPoolElement} = {}
+
+const handleTimeoutForSubmission = function (submissionId: number) {
+  const job = runPool[submissionId]
+  const errorResponse = {
+    id: submissionId,
+    code: 408,
+    message: "Compile/Run timed out",
+  }
+
+  switch (job.mode) {
+    case 'sync': 
+      job.res.status(408).json(errorResponse)
+      break;
+    case 'callback':
+      axios.post(job.callback, errorResponse)
+  }
+}
+
+const handleSuccessForSubmission = function (result: RunResponse) {
+  const job = runPool[result.id]
+  switch (job.mode) {
+    case 'sync':
+      job.res.status(200).json(result)
+      break;
+    case 'callback':
+      // send a post request to callback 
+      (async () => {
+        // 1. upload the result to s3 and get the url
+        const {url} = await upload(result)
+
+        // 2. save the url in db
+        await Submissions.update(<any>{
+          outputs: [url]
+        }, {
+          where: {
+            id: result.id
+          }
+        })
+
+        // make the callback request
+        await axios.post(job.callback, {id: result.id, outputs: [url]})
+      })()
+      break;
+  }
+}
+
+/**
+ * Returns a runPoolElement for request
+ */
+const getRunPoolElement = function (body: RunRequestBody, res: Response): RunPoolElement {
+  switch (body.mode) {
+    case 'sync':
+      return ({
+        mode: 'sync',
+        res
+      })
+    case 'callback':
+      return ({
+        mode: 'callback',
+        res,
+        callback: body.callback
+      })
+  }
+}
 
 /**
  * @api {post} /runs POST /runs
@@ -33,6 +108,8 @@ const runPool: {[x: number]: Response} = {}
  * @apiParam {String(Base64)} source source code to run (encoded in base64)
  * @apiParam {Enum} lang Language of code to execute
  * @apiParam {String(Base64)} input [Optional] stdin input for the program (encoded in base64)
+ * @apiParam {Enum} mode [Optional] mode for request. Default = `sync`, see: https://github.com/coding-blocks/judge-api/issues/16
+ * @apiParam {String)} callback [Optional] callback url for request. Required for `mode = callback`
  *
  * @apiUse AvailableLangs
  *
@@ -41,14 +118,26 @@ const runPool: {[x: number]: Response} = {}
  * @apiSuccess {String(Base64)} stderr Output of stderr of execution (encoded in base64)
  * @apiSuccess {Number} statuscode Result of operation
  *
- * @apiSuccessExample {JSON} Success-Response:
+ * @apiSuccessExample {JSON} Success-Response(mode=sync):
  *  HTTP/1.1 200 OK
  *  {
  *    "id": 10,
  *    "statuscode": 0,
  *    "stdout": "NA0KMg0KMw=="
  *    "stderr": "VHlwZUVycm9y"
  *  }
+ *  @apiSuccessExample {JSON} Success-Response(mode=callback):
+ *  HTTP/1.1 200 OK
+ *  {
+ *    "id": 10
+ *  }
+ * 
+ *  @apiSuccessExample {JSON} Body for Callback(mode=callback):
+ *  HTTP/1.1 200 OK
+ *  {
+ *    "id": 10,
+ *    "outputs": ["http://localhost/judge-submissions/file.json"]
+ *  }
  */
 route.post('/', (req, res, next) => {
   const invalidRequest = isInvalidRunRequest(req)
@@ -70,19 +159,24 @@ route.post('/', (req, res, next) => {
       lang: req.body.lang,
       stdin: req.body.stdin
     })
+
     // Put into pool and wait for judge-worker to respond
-    runPool[submission.id] = res
+    runPool[submission.id] = getRunPoolElement(req.body, res)
+
     setTimeout(() => {
       if (runPool[submission.id]) {
-        runPool[submission.id].status(408).json({
-          id: submission.id,
-          code: 408,
-          message: "Compile/Run timed out",
-        })
+        handleTimeoutForSubmission(submission.id)
         delete runPool[submission.id]
       }
     }, config.RUN.TIMEOUT)
 
+    switch (req.body.mode) {
+      case 'callback':
+        res.json({
+          id: submission.id
+        })
+    }
+
   }).catch(err => {
     res.status(501).json({
       code: 501,
@@ -97,10 +191,10 @@ route.post('/', (req, res, next) => {
  */
 successListener.on('success', (result: RunResponse) => {
   if (runPool[result.id]) {
-    runPool[result.id].status(200).json(result)
+    handleSuccessForSubmission(result)
     delete runPool[result.id]
   }
-  Submissions.update({
+  Submissions.update(<any>{
     end_time: new Date()
   }, {
     where: {

src/utils/s3.ts

@@ -0,0 +1,27 @@
+import Minio = require('minio')
+import v4 =  require('uuid/v4')
+import config = require('../../config')
+
+const client = new Minio.Client({
+  endPoint: config.S3.endpoint,
+  port: config.S3.port,
+  useSSL: config.S3.ssl,
+  accessKey: config.S3.accessKey,
+  secretKey: config.S3.secretKey,
+})
+
+export type savedFile = {
+  etag: string,
+  url: string
+}
+
+export const urlForFilename = (bucket: string, filename: string) : string => `http${config.S3.ssl ? 's': ''}://${config.S3.endpoint}/${bucket}/${filename}`
+
+export const upload = function (object:object, filename:string = v4() + '.json' ,bucket:string = config.S3.bucket) : Promise<savedFile> {
+  return new Promise((resolve, reject) => {
+    client.putObject(bucket, filename, JSON.stringify(object), function(err, etag) {
+      if (err) return reject(err)
+      resolve({etag, url: urlForFilename(bucket, filename) })
+    })
+  })
+}

src/validators/SubmissionValidators.ts

@@ -12,6 +12,15 @@ export function isInvalidRunRequest(req: Request): Error | boolean {
   if (!req.body.stdin) {
     req.body.stdin = ''
   }
+  if (!req.body.mode) {
+    req.body.mode = 'sync'
+  }
+  if (!['sync', 'callback'].includes(req.body.mode)) {
+    return new Error('Mode must be one of sync, callback')
+  }
+  if (req.body.mode === 'callback' && !req.body.callback) {
+    return new Error('Must specify a callback for callback mode')
+  }
 
   return false
 }

tsconfig.json

@@ -1,7 +1,7 @@
 {
   "compilerOptions": {
     "module": "commonjs",
-    "target": "es2015",
+    "target": "es2016",
     "moduleResolution": "node",
     "allowSyntheticDefaultImports": true,
     "sourceMap": true,