completed openapi web search project server.

src/server/api/constants/Constants.js

@@ -1,6 +1,11 @@
 const CC_SERVER_URL = 'https://index.commoncrawl.org/';
 const RABBIT_MQ_URL = 'amqp://localhost:5672';
+const ELASTIC_LOCAL_URL = 'https://localhost:9200';
+const ELASTIC_HTTP_CRED = '/home/priyanshu/Downloads/ELK/primary-node/config/certs/http_ca.crt';
+const INDEX_NAME = 'openapi_definition';
 const QUEUE_NAME = 'index-files-jobs';
+const ELASTIC_USERNAME = 'elastic';
+const ELASTIC_PASSWORD = 'pxnS=+fLwN-0j2z=iPgX';
 const FILE_PATH = `${__dirname}/../../dist/data/output.txt`;
 const RESULTS_FILE_PATH = `${__dirname}/../../dist/results/results.txt`;
 const DIST_PATH = `${__dirname}/../../dist`;
@@ -12,4 +17,5 @@ const SUB_DOMAIN_REGEX = /^(?:https?:\/\/)?api\.[^\/\s]+\.[^\/\s]+(?:\/[^\/\s]*)
 const KEYWORD_REGEX = /^(?:https?:\/\/)?[^\/\s]+(?:\/[^\/\s]+)*(?:\/(openapi|swagger))(?:\/[^\/\s]*)*$/gi;
 const BASE_URL = 'http://localhost:1337';
 
-module.exports = { CC_SERVER_URL, RABBIT_MQ_URL, QUEUE_NAME, FILE_PATH, CHUNK_SIZE, PROGRESS_BAR_WIDTH, URL_REGEX, API_DEFINITION_REGEX, SUB_DOMAIN_REGEX, KEYWORD_REGEX, RESULTS_FILE_PATH, DIST_PATH, BASE_URL };
+module.exports = { CC_SERVER_URL, RABBIT_MQ_URL, QUEUE_NAME, FILE_PATH, CHUNK_SIZE, PROGRESS_BAR_WIDTH, URL_REGEX, API_DEFINITION_REGEX, SUB_DOMAIN_REGEX, KEYWORD_REGEX, RESULTS_FILE_PATH, DIST_PATH, BASE_URL, ELASTIC_LOCAL_URL,
+     ELASTIC_HTTP_CRED, INDEX_NAME, ELASTIC_USERNAME, ELASTIC_PASSWORD };

src/server/api/controllers/IndexingController.js

@@ -0,0 +1,66 @@
+const SwaggerParser = require('swagger-parser');
+const { fetchDefinitionsFromDBUtils } = require('../utils/DBUtils');
+const {
+  transformingDefinitionForIndexing,
+  checkForIndex,
+  removeIndex,
+} = require('../utils/ElasticsearchUtils');
+const { INDEX_NAME } = require('../constants/Constants');
+
+module.exports = {
+  /**
+   * Starts the process of indexing validated OpenAPI definitions.
+   *
+   * @function startIndexing
+   * @async
+   * @param {Object} req - The Express request object.
+   * @param {Object} res - The Express response object.
+   * @returns {Object} The response indicating the indexing status.
+   * @throws {400} If there are no OpenAPI definitions to fetch.
+   * @throws {500} If an internal server error occurs during indexing.
+   *
+   * @example
+   * // Request:
+   * // POST /start-indexing
+   * //
+   * // Response:
+   * // 200 OK
+   * // 'OpenAPI definitions have been indexed!'
+   */
+  startIndexing: async function (req, res) {
+    try {
+      const client = sails.client;
+      const transformedResults = [];
+      const validatedAPIDefinitions = await fetchDefinitionsFromDBUtils();
+
+      if (validatedAPIDefinitions.length === 0) {
+        return res.badRequest('There are no OpenAPI definitions to fetch!');
+      }
+
+      for (const { url } of validatedAPIDefinitions) {
+        const document = await SwaggerParser.parse(url);
+        transformedResults.push(
+          transformingDefinitionForIndexing(document, url)
+        );
+      }
+
+      const isExists = await checkForIndex(client);
+      if (isExists) {
+        await removeIndex(client);
+      }
+
+      await client.helpers.bulk({
+        datasource: transformedResults,
+        onDocument() {
+          return {
+            index: { _index: INDEX_NAME },
+          };
+        },
+      });
+
+      return res.send('OpenAPI definitions has been indexed!');
+    } catch (error) {
+      res.serverError(error);
+    }
+  },
+};

src/server/api/controllers/SearchController.js

@@ -0,0 +1,51 @@
+const { searchForQuery } = require('../utils/ElasticsearchUtils');
+
+module.exports = {
+  /**
+   * Starts a search operation based on provided keywords.
+   *
+   * @function startSearching
+   * @async
+   * @param {Object} req - The Express request object.
+   * @param {Object} res - The Express response object.
+   * @returns {Object} The JSON response containing search results.
+   * @throws {400} If the query parameter is missing.
+   * @throws {500} If an internal server error occurs.
+   *
+   * @example
+   * // Request:
+   * // GET /search?q=authentication
+   * //
+   * // Response:
+   * // 200 OK
+   * // [
+   * //   {
+   * //     '_id': '123',
+   * //     '_score': 1.0,
+   * //     '_source': {
+   * //       'url': 'https://api.example.com',
+   * //       'content': {
+   * //         /* ... * /
+   * //       }
+   * //     }
+   * //   },
+   * //   /* ... * /
+   * // ]
+   */
+  startSearching: async function (req, res) {
+    try {
+      const client = sails.client;
+      const keywords = req.query.q;
+
+      if (keywords === undefined) {
+        return res.badRequest('Query not found!');
+      }
+
+      const searchResults = await searchForQuery(client, keywords);
+
+      return res.json(searchResults);
+    } catch (error) {
+      return res.serverError(error.message);
+    }
+  },
+};

src/server/api/utils/ConnectElasticSearchUtils.js

@@ -0,0 +1,22 @@
+const fs = require('fs');
+const { Client } = require('@elastic/elasticsearch');
+const { ELASTIC_LOCAL_URL, ELASTIC_HTTP_CRED, ELASTIC_USERNAME, ELASTIC_PASSWORD } = require('../constants/Constants');
+
+const client = new Client({
+  node: ELASTIC_LOCAL_URL,
+  log: 'error',
+  auth: {
+    username: ELASTIC_USERNAME,
+    password: ELASTIC_PASSWORD
+  },
+  tls: {
+    ca: fs.readFileSync(ELASTIC_HTTP_CRED),
+    rejectUnauthorized: false
+  }
+});
+
+module.exports = {
+  getClient: function() {
+    return client;
+  },
+};

src/server/api/utils/DBUtils.js

@@ -7,7 +7,7 @@ module.exports = {
    * @function cleanDB
    * @param {Object} model - The database model to clean (e.g., Waterline model).
    * @throws {Error} If there is an error during the database cleaning process, it will be thrown.
- */
+   */
   cleanDB: async function (model) {
     try {
       await model.destroy({});
@@ -28,23 +28,23 @@ module.exports = {
    *                                        Use 'aes' for ascending and 'des' for descending.
    * @returns {Promise<Array>} A Promise that resolves with an array of index files fetched from the database.
    * @throws {Error} If there is an error during the database query, it will be thrown.
- */
+   */
   fetchIndexFilesFromDBUtils: async function (filter) {
     try {
       const orderMap = {
-        'aes': 1,
-        'des': -1
+        aes: 1,
+        des: -1,
       };
 
       const skp = filter.skip === undefined ? 0 : parseInt(filter.skip);
-      const lmt = filter.limit === undefined ? Infinity : parseInt(filter.limit);
+      const lmt =
+        filter.limit === undefined ? Infinity : parseInt(filter.limit);
       const srt = filter.sort === undefined ? 1 : orderMap[filter.sort];
 
       return await IndexFilesModel.find({})
         .sort({ URL: srt })
         .skip(skp)
         .limit(lmt);
-
     } catch (error) {
       throw error;
     }
@@ -57,11 +57,11 @@ module.exports = {
    * @param {Array<string>} indexFiles - An array of index file URLs to be saved to the database.
    * @returns {Promise<void>} A Promise that resolves when all index file URLs have been saved to the database.
    * @throws {Error} If there is an error during the database insertion process, it will be thrown.
- */
+   */
   saveIndexFiles: async function (indexFiles) {
     try {
-      indexFiles.forEach(async url => {
-        await IndexFilesModel.create({URL: url});
+      indexFiles.forEach(async (url) => {
+        await IndexFilesModel.create({ URL: url });
       });
     } catch (error) {
       throw error;
@@ -75,18 +75,36 @@ module.exports = {
    * @param {string[]} definitions - An array of API definitions (URLs) to be saved.
    * @throws {Error} If there's an error while saving the definitions to the file or the database.
    * @returns {Promise<void>} A Promise that resolves once all definitions have been saved.
- */
-  saveAPIsDefinitions: async function(definitions) {
+   */
+  saveAPIsDefinitions: async function (definitions) {
     try {
-      definitions.forEach(async url => {
-        validatingService(url).then(async () => {
-          await APIsDefinitionsModel.create({url});
-        }).catch(error => {
-          console.log('Invalid definition!',error.message);
-        });
+      definitions.forEach(async (url) => {
+        validatingService(url)
+          .then(async () => {
+            await APIsDefinitionsModel.create({ url });
+          })
+          .catch((error) => {
+            console.log(`\nInvalid definition! ${error.message}\n`);
+          });
       });
-    } catch(error) {
+    } catch (error) {
       throw error;
     }
-  }
-}
+  },
+
+  /**
+   * Fetches API definitions from the database using utility functions.
+   *
+   * @async
+   * @function fetchDefinitionsFromDBUtils
+   * @returns {Promise<Array>} A promise that resolves to an array of API definitions.
+   * @throws {Error} If an error occurs while fetching the API definitions from the database.
+   */
+  fetchDefinitionsFromDBUtils: async function () {
+    try {
+      return await APIsDefinitionsModel.find({});
+    } catch (error) {
+      throw error;
+    }
+  },
+};

src/server/api/utils/ElasticsearchUtils.js

@@ -0,0 +1,78 @@
+const { INDEX_NAME } = require('../constants/Constants');
+
+module.exports = {
+  /**
+   * Transforms an API definition document into an indexable format for search purposes.
+   *
+   * @function transformingDefinitionForIndexing
+   * @param {Object} document - The API definition document to be transformed.
+   * @param {string} url - The URL associated with the API definition.
+   * @returns {Object} An indexable representation of the API definition.
+   * @throws {Error} If an error occurs during the transformation process.
+   */
+  transformingDefinitionForIndexing: function (document, url) {
+    try {
+      const indexableInfo = {
+        url,
+        info: document.info,
+        tags: document.tags !== undefined ? document.tags : null,
+        servers: document.servers !== undefined ? document.servers : null,
+      };
+
+      return {
+        url: url,
+        content: indexableInfo,
+      };
+    } catch (error) {
+      throw error;
+    }
+  },
+
+  checkForIndex: async function (client) {
+    try {
+      return await client.indices.exists({ index: INDEX_NAME });
+    } catch (error) {
+      throw error;
+    }
+  },
+
+  removeIndex: async function (client) {
+    try {
+      await client.indices.delete({ index: INDEX_NAME });
+    } catch (error) {
+      throw error;
+    }
+  },
+
+  /**
+   * Searches for API definitions in an Elasticsearch index based on specified keywords.
+   *
+   * @async
+   * @function searchForQuery
+   * @param {Object} client - The Elasticsearch client used for performing the search.
+   * @param {string} keywords - The keywords to search for in the API definitions.
+   * @returns {Promise<Array<Object>>} A promise that resolves to an array of matched API definition hits.
+   * @throws {Error} If an error occurs during the search process.
+   */
+  searchForQuery: async function (client, keywords) {
+    try {
+      const response = await client.search({
+        index: INDEX_NAME,
+        query: {
+          multi_match: {
+            query: keywords,
+            fields: [
+              'content.info.title',
+              'content.info.description',
+              'content.tags.name',
+            ],
+          },
+        },
+      });
+
+      return response.hits.hits;
+    } catch (error) {
+      throw error;
+    }
+  },
+};