Implement crawling controller to fetch directory URLs containing OpenAPI definitions. - Implementing Queue-Based Architecture of Downloading Index Files from Common Crawl Server Using RabbitMQ NOTE: All code is contained within the downloadAndProcessIndexFilesInBackground() function

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+  "prettier.jsxSingleQuote": true,
+  "prettier.singleQuote": true
+}

src/server/.editorconfig

@@ -0,0 +1,31 @@
+################################################
+#   ╔═╗╔╦╗╦╔╦╗╔═╗╦═╗┌─┐┌─┐┌┐┌┌─┐┬┌─┐
+#   ║╣  ║║║ ║ ║ ║╠╦╝│  │ ││││├┤ ││ ┬
+#  o╚═╝═╩╝╩ ╩ ╚═╝╩╚═└─┘└─┘┘└┘└  ┴└─┘
+#
+# > Formatting conventions for your Sails app.
+#
+# This file (`.editorconfig`) exists to help
+# maintain consistent formatting throughout the
+# files in your Sails app.
+#
+# For the sake of convention, the Sails team's
+# preferred settings are included here out of the
+# box.  You can also change this file to fit your
+# team's preferences (for example, if all of the
+# developers on your team have a strong preference
+# for tabs over spaces),
+#
+# To review what each of these options mean, see:
+# http://editorconfig.org/
+#
+################################################
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true

src/server/.eslintignore

@@ -0,0 +1 @@
+

src/server/.eslintrc

@@ -0,0 +1,85 @@
+{
+  //   ╔═╗╔═╗╦  ╦╔╗╔╔╦╗┬─┐┌─┐
+  //   ║╣ ╚═╗║  ║║║║ ║ ├┬┘│
+  //  o╚═╝╚═╝╩═╝╩╝╚╝ ╩ ┴└─└─┘
+  // A set of basic code conventions designed to encourage quality and consistency
+  // across your Sails app's code base.  These rules are checked against
+  // automatically any time you run `npm test`.
+  // 
+  // > Note: If you're using mocha, you'll want to add an extra override file to your
+  // > `test/` folder so that eslint will tolerate mocha-specific globals like `before`
+  // > and `describe`.
+  // Designed for ESLint v4.
+  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+  // For more information about any of the rules below, check out the relevant
+  // reference page on eslint.org.  For example, to get details on "no-sequences",
+  // you would visit `http://eslint.org/docs/rules/no-sequences`.  If you're unsure
+  // or could use some advice, come by https://sailsjs.com/support.
+  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+  "env": {
+    "node": true
+  },
+
+  "parserOptions": {
+    "ecmaVersion": 2018
+  },
+
+  "globals": {
+    // If "no-undef" is enabled below, be sure to list all global variables that
+    // are used in this app's backend code (including the globalIds of models):
+    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    "Promise": true,
+    "sails": true,
+    "_": true
+    // …and any others (e.g. `"Organization": true`)
+    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+  },
+
+  "rules": {
+    "block-scoped-var":             ["error"],
+    "callback-return":              ["error", ["done", "proceed", "next", "onwards", "callback", "cb"]],
+    "camelcase":                    ["warn", {"properties":"always"}],
+    "comma-style":                  ["warn", "last"],
+    "curly":                        ["warn"],
+    "eqeqeq":                       ["error", "always"],
+    "eol-last":                     ["warn"],
+    "handle-callback-err":          ["error"],
+    "indent":                       ["warn", 2, {
+      "SwitchCase": 1,
+      "MemberExpression": "off",
+      "FunctionDeclaration": {"body":1, "parameters":"off"},
+      "FunctionExpression": {"body":1, "parameters":"off"},
+      "CallExpression": {"arguments":"off"},
+      "ArrayExpression": 1,
+      "ObjectExpression": 1,
+      "ignoredNodes": ["ConditionalExpression"]
+    }],
+    "linebreak-style":              ["error", "unix"],
+    "no-dupe-keys":                 ["error"],
+    "no-duplicate-case":            ["error"],
+    "no-extra-semi":                ["warn"],
+    "no-labels":                    ["error"],
+    "no-mixed-spaces-and-tabs":     [2, "smart-tabs"],
+    "no-redeclare":                 ["warn"],
+    "no-return-assign":             ["error", "always"],
+    "no-sequences":                 ["error"],
+    "no-trailing-spaces":           ["warn"],
+    "no-undef":                     ["off"],
+    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    // ^^Note: If this "no-undef" rule is enabled (set to `["error"]`), then all model globals
+    // (e.g. `"Organization": true`) should be included above under "globals".
+    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    "no-unexpected-multiline":      ["warn"],
+    "no-unreachable":               ["warn"],
+    "no-unused-vars":               ["warn", {"caughtErrors":"all", "caughtErrorsIgnorePattern": "^unused($|[A-Z].*$)", "argsIgnorePattern": "^unused($|[A-Z].*$)", "varsIgnorePattern": "^unused($|[A-Z].*$)" }],
+    "no-use-before-define":         ["error", {"functions":false}],
+    "one-var":                      ["warn", "never"],
+    "prefer-arrow-callback":        ["warn", {"allowNamedFunctions":true}],
+    "quotes":                       ["warn", "single", {"avoidEscape":false, "allowTemplateLiterals":true}],
+    "semi":                         ["warn", "always"],
+    "semi-spacing":                 ["warn", {"before":false, "after":true}],
+    "semi-style":                   ["warn", "last"]
+  }
+
+}

src/server/.gitignore

@@ -0,0 +1,134 @@
+################################################
+#   ┌─┐┬┌┬┐╦╔═╗╔╗╔╔═╗╦═╗╔═╗
+#   │ ┬│ │ ║║ ╦║║║║ ║╠╦╝║╣
+#  o└─┘┴ ┴ ╩╚═╝╝╚╝╚═╝╩╚═╚═╝
+#
+# > Files to exclude from your app's repo.
+#
+# This file (`.gitignore`) is only relevant if
+# you are using git.
+#
+# It exists to signify to git that certain files
+# and/or directories should be ignored for the
+# purposes of version control.
+#
+# This keeps tmp files and sensitive credentials
+# from being uploaded to  your repository.  And
+# it allows you to configure your app for your
+# machine without accidentally committing settings
+# which will smash the local settings of other
+# developers on your team.
+#
+# Some reasonable defaults are included below,
+# but, of course, you should modify/extend/prune
+# to fit your needs!
+#
+################################################
+
+
+################################################
+# Local Configuration
+#
+# Explicitly ignore files which contain:
+#
+# 1. Sensitive information you'd rather not push to
+#    your git repository.
+#    e.g., your personal API keys or passwords.
+#
+# 2. Developer-specific configuration
+#    Basically, anything that would be annoying
+#    to have to change every time you do a
+#    `git pull` on your laptop.
+#    e.g. your local development database, or
+#    the S3 bucket you're using for file uploads
+#    during development.
+#
+################################################
+
+config/local.js
+
+
+################################################
+# Dependencies
+#
+#
+# When releasing a production app, you _could_
+# hypothetically include your node_modules folder
+# in your git repo, but during development, it
+# is always best to exclude it, since different
+# developers may be working on different kernels,
+# where dependencies would need to be recompiled
+# anyway.
+#
+# Most of the time, the node_modules folder can
+# be excluded from your code repository, even
+# in production, thanks to features like the
+# package-lock.json file / NPM shrinkwrap.
+#
+# But no matter what, since this is a Sails app,
+# you should always push up the package-lock.json
+# or shrinkwrap file to your repository, to avoid
+# accidentally pulling in upgraded dependencies
+# and breaking your code.
+#
+# That said, if you are having trouble with
+# dependencies, (particularly when using
+# `npm link`) this can be pretty discouraging.
+# But rather than just adding the lockfile to
+# your .gitignore, try this first:
+# ```
+#     rm -rf node_modules
+#     rm package-lock.json
+#     npm install
+# ```
+#
+# [?] For more tips/advice, come by and say hi
+#     over at https://sailsjs.com/support
+#
+################################################
+
+node_modules
+
+
+################################################
+#
+# > Do you use bower?
+# > re: the bower_components dir, see this:
+# > http://addyosmani.com/blog/checking-in-front-end-dependencies/
+# > (credit Addy Osmani, @addyosmani)
+#
+################################################
+
+
+################################################
+# Temporary files generated by Sails/Waterline.
+################################################
+
+.tmp
+
+
+################################################
+# Miscellaneous
+#
+# Common files generated by text editors,
+# operating systems, file systems, dbs, etc.
+################################################
+
+*~
+*#
+.DS_STORE
+.netbeans
+nbproject
+.idea
+*.iml
+.vscode
+.node_history
+dump.rdb
+
+npm-debug.log
+lib-cov
+*.seed
+*.log
+*.out
+*.pid
+

src/server/.npmrc

@@ -0,0 +1,11 @@
+######################
+#   ╔╗╔╔═╗╔╦╗┬─┐┌─┐  #
+#   ║║║╠═╝║║║├┬┘│    #
+#  o╝╚╝╩  ╩ ╩┴└─└─┘  #
+######################
+
+# Hide NPM log output unless it is related to an error of some kind:
+loglevel=error
+
+# Make "npm audit" an opt-in thing for subsequent installs within this app:
+audit=false

src/server/.sailsrc

@@ -0,0 +1,12 @@
+{
+  "generators": {
+    "modules": {}
+  },
+  "_generatedWith": {
+    "sails": "1.5.4",
+    "sails-generate": "2.0.8"
+  },
+  "hooks": {
+    "grunt": false
+  }
+}

src/server/README.md

@@ -0,0 +1,26 @@
+# server
+
+a [Sails v1](https://sailsjs.com) application
+
+
+### Links
+
++ [Sails framework documentation](https://sailsjs.com/get-started)
++ [Version notes / upgrading](https://sailsjs.com/documentation/upgrading)
++ [Deployment tips](https://sailsjs.com/documentation/concepts/deployment)
++ [Community support options](https://sailsjs.com/support)
++ [Professional / enterprise options](https://sailsjs.com/enterprise)
+
+
+### Version info
+
+This app was originally generated on Sat Jun 10 2023 12:22:10 GMT+0530 (India Standard Time) using Sails v1.5.4.
+
+<!-- Internally, Sails used [`sails-generate@2.0.8`](https://github.com/balderdashy/sails-generate/tree/v2.0.8/lib/core-generators/new). -->
+
+
+
+<!--
+Note:  Generators are usually run using the globally-installed `sails` CLI (command-line interface).  This CLI version is _environment-specific_ rather than app-specific, thus over time, as a project's dependencies are upgraded or the project is worked on by different developers on different computers using different versions of Node.js, the Sails dependency in its package.json file may differ from the globally-installed Sails CLI release it was originally generated with.  (Be sure to always check out the relevant [upgrading guides](https://sailsjs.com/upgrading) before upgrading the version of Sails used by your app.  If you're stuck, [get help here](https://sailsjs.com/support).)
+-->
+

src/server/api/constants/Constants.js

@@ -0,0 +1,7 @@
+const CC_SERVER_URL = 'https://index.commoncrawl.org/';
+const SCHEDULE = '0 0 * * 0';
+const RABBITMQ_URL = 'amqp://localhost:5672';
+const QUEUE_NAME = 'background-jobs';
+const FILE_PATH = `${__dirname}/../../data/output.txt`;
+
+module.exports = { CC_SERVER_URL, SCHEDULE, RABBITMQ_URL, QUEUE_NAME, FILE_PATH };

src/server/api/controllers/CrawlingController.js

@@ -0,0 +1,62 @@
+const { selectDataSources } = require('../utils/SelectDataSources');
+
+module.exports = {
+  /**
+   * Endpoint for starting the crawling process.
+   * Controller for the 'start-crawling' route. Initiates the crawling process based on the provided data source.
+   * Retrieves API definitions from the selected data source and sends the response accordingly.
+   *
+   * @route POST /api/v1/run/crawler
+   * @param {Object} req - The request object.
+   * @param {Object} res - The response object.
+   * @param {Object} req.body - The payload of the request.
+   * @param {string} req.body.dataSource - The data source for crawling.
+   * @param {boolean} req.query.latest - Indicates whether to use the latest data.
+   * @returns {Array<string>} An array of API definition URLs, or an error response.
+   * @throws {Error} If an error occurs during the crawling process.
+   *
+   * @example
+   * Request body example:
+   * {
+   *   'dataSource': 'commonCrawl'
+   * }
+   *
+   * @example
+   * Query parameter example:
+   * /api/v1/run/crawler?latest=true
+   * or
+   * /api/v1/run/crawler?latest=false
+   *
+   * @example
+   * Response example:
+   * [
+   *   'https://api.example.com/definition1',
+   *   'https://api.example.com/definition2',
+   *   ...
+   * ]
+   */
+  startCrawling: async function (req, res) {
+    try {
+      const { dataSource } = req.body;
+      /**
+       * If req.query.latest is truthy and equals the string 'true', then latest is set to true; else latest is set to false;
+       * otherwise, it is set to the default value true.
+       */
+      const latest = req.query.latest ? req.query.latest === 'true' : true;
+
+      if (!dataSource) {
+        return res.badRequest('Data source not provided');
+      }
+
+      try {
+        await selectDataSources(dataSource, latest);
+      } catch (error) {
+        throw error;
+      }
+
+      return res.status(202).json({message: 'downloading and processing has begun.'});
+    } catch (error) {
+      return res.serverError(error);
+    }
+  },
+};