Merge pull request #6135 from Countly/feat/version-spread

Cookiezaurs · web-flow · commit 95f12778bf7e · 2025-04-16T15:25:28.000+03:00
Spread version saving to db
diff --git a/api/parts/data/usage.js b/api/parts/data/usage.js
@@ -398,7 +398,7 @@ usage.returnAllProcessedMetrics = function(params) {
                 continue;
             }
 
-            if (recvMetricValue) {
+            if (recvMetricValue !== undefined && recvMetricValue !== null) {
                 var escapedMetricVal = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":");
                 processedMetrics[tmpMetric.short_code] = escapedMetricVal;
             }
@@ -705,7 +705,7 @@ function processMetrics(user, uniqueLevelsZero, uniqueLevelsMonth, params, done)
                     continue;
                 }
 
-                if (recvMetricValue) {
+                if (recvMetricValue !== undefined && recvMetricValue !== null) {
                     recvMetricValue = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":");
                     postfix = common.crypto.createHash("md5").update(recvMetricValue).digest('base64')[0];
                     metaToFetch[predefinedMetrics[i].db + params.app_id + "_" + dateIds.zero + "_" + postfix] = {
@@ -767,7 +767,7 @@ function processMetrics(user, uniqueLevelsZero, uniqueLevelsMonth, params, done)
                     continue;
                 }
 
-                if (recvMetricValue) {
+                if (recvMetricValue !== undefined && recvMetricValue !== null) {
                     escapedMetricVal = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":");
                     postfix = common.crypto.createHash("md5").update(escapedMetricVal).digest('base64')[0];
 
@@ -1110,6 +1110,15 @@ plugins.register("/sdk/user_properties", async function(ob) {
                 userProps[key] = up[key];
             }
         }
+
+        if (params.qstring.metrics._app_version) {
+            const versionComponents = common.parseAppVersion(params.qstring.metrics._app_version);
+            if (versionComponents.success) {
+                userProps.av_major = versionComponents.major;
+                userProps.av_minor = versionComponents.minor;
+                userProps.av_patch = versionComponents.patch;
+            }
+        }
     }
 
     if (params.qstring.session_duration) {
diff --git a/api/utils/common.js b/api/utils/common.js
@@ -1,8 +1,7 @@
 /**
-* Module for some common utility functions and references
-* @module api/utils/common
-*/
-
+ * Module for some common utility functions and references
+ * @module api/utils/common
+ */
 /**
  * @typedef {import('../../types/requestProcessor').Params} Params
  * @typedef {import('../../types/common').TimeObject} TimeObject
@@ -12,19 +11,20 @@
 
 /** @lends module:api/utils/common **/
 /** @type(import('../../types/common').Common) */
-var common = {};
+const common = {};
 
 /** @type(import('moment-timezone')) */
-var moment = require('moment-timezone');
-var crypto = require('crypto'),
-    logger = require('./log.js'),
-    mcc_mnc_list = require('mcc-mnc-list'),
-    plugins = require('../../plugins/pluginManager.js'),
-    countlyConfig = require('./../config', 'dont-enclose'),
-    argon2 = require('argon2'),
-    mongodb = require('mongodb'),
-    getRandomValues = require('get-random-values'),
-    _ = require('lodash');
+const moment = require('moment-timezone');
+const crypto = require('crypto');
+const logger = require('./log.js');
+const mcc_mnc_list = require('mcc-mnc-list');
+const plugins = require('../../plugins/pluginManager.js');
+const countlyConfig = require('./../config', 'dont-enclose');
+const argon2 = require('argon2');
+const mongodb = require('mongodb');
+const getRandomValues = require('get-random-values');
+const semver = require('semver');
+const _ = require('lodash');
 
 var matchHtmlRegExp = /"|'|&(?!amp;|quot;|#39;|lt;|gt;|#46;|#36;)|<|>/;
 var matchLessHtmlRegExp = /[<>]/;
@@ -210,6 +210,9 @@ common.dbUserMap = {
     'platform': 'p',
     'platform_version': 'pv',
     'app_version': 'av',
+    'app_version_major': 'av_major',
+    'app_version_minor': 'av_minor',
+    'app_version_patch': 'av_patch',
     'last_begin_session_timestamp': 'lbst',
     'last_end_session_timestamp': 'lest',
     'has_ongoing_session': 'hos',
@@ -2055,6 +2058,134 @@ common.versionCompare = function(v1, v2, options) {
     return compareParts;
 };
 
+/**
+ * Parse app_version into major, minor, patch components
+ * @param {string|number} version - The version to parse
+ * @returns {object} Object containing major, minor, patch, original version, and success flag
+ */
+common.parseAppVersion = function(version) {
+    try {
+        if (typeof version !== 'string') {
+            version = String(version);
+        }
+
+        // Ensure version has at least one decimal point
+        if (version.indexOf('.') === -1) {
+            version += '.0';
+        }
+
+        const parsedVersion = semver.valid(semver.coerce(version));
+        if (parsedVersion) {
+            const versionObj = semver.parse(parsedVersion);
+            if (versionObj) {
+                return {
+                    major: versionObj.major,
+                    minor: versionObj.minor,
+                    patch: versionObj.patch,
+                    original: version,
+                    success: true
+                };
+            }
+        }
+    }
+    catch (error) {
+        // Silently catch any errors from semver library
+        // console.error('Error parsing app version:', error);
+    }
+
+    // Return only original version with success=false if parsing fails or throws an exception
+    return {
+        original: version,
+        success: false
+    };
+};
+
+/**
+ *  Check if a version string follows some kind of scheme (there is only semantic versioning (semver) for now)
+ *  @param {string} inpVersion - an app version string
+ *  @return {array} [regex.exec result, version scheme name]
+ */
+common.checkAppVersion = function(inpVersion) {
+    // Regex is from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+    const semverRgx = /(^v?)(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/;
+    // Half semver is similar to semver but with only one dot
+    const halfSemverRgx = /(^v?)(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/;
+
+    let execResult = semverRgx.exec(inpVersion);
+
+    if (execResult) {
+        return [execResult, 'semver'];
+    }
+
+    execResult = halfSemverRgx.exec(inpVersion);
+
+    if (execResult) {
+        return [execResult, 'halfSemver'];
+    }
+
+    return [null, null];
+};
+
+/**
+ *  Transform a version string so it will be numerically correct when sorted
+ *  For example '1.10.2' will be transformed to '100001.100010.100002'
+ *  So when sorted ascending it will come after '1.2.0' ('100001.100002.100000')
+ *  @param {string} inpVersion - an app version string
+ *  @return {string} the transformed app version
+ *  @note Imported and moved from @module plugins/crashes/api/parts/version (which has now been deprecated)
+ */
+common.transformAppVersion = function(inpVersion) {
+    const [execResult, versionScheme] = common.checkAppVersion(inpVersion);
+
+    if (execResult === null) {
+        // Version string does not follow any scheme, just return it
+        return inpVersion;
+    }
+
+    // Mark version parts based on semver scheme
+    let prefixIdx = 1;
+    let majorIdx = 2;
+    let minorIdx = 3;
+    let patchIdx = 4;
+    let preReleaseIdx = 5;
+    let buildIdx = 6;
+
+    if (versionScheme === 'halfSemver') {
+        patchIdx -= 1;
+        preReleaseIdx -= 1;
+        buildIdx -= 1;
+    }
+
+    let transformed = '';
+    // Rejoin version parts to a new string
+    for (let idx = prefixIdx; idx < buildIdx; idx += 1) {
+        let part = execResult[idx];
+
+        if (part) {
+            if (idx >= majorIdx && idx <= patchIdx) {
+                part = 100000 + parseInt(part, 10);
+            }
+
+            if (idx >= minorIdx && idx <= patchIdx) {
+                part = '.' + part;
+            }
+
+            if (idx === preReleaseIdx) {
+                part = '-' + part;
+            }
+
+            if (idx === buildIdx) {
+                part = '+' + part;
+            }
+
+            transformed += part;
+        }
+    }
+
+    return transformed;
+};
+
+
 common.adjustTimestampByTimezone = function(ts, tz) {
     var d = moment();
     if (tz) {
diff --git a/package.json b/package.json
@@ -85,6 +85,7 @@
     "properties-parser": "0.6.0",
     "puppeteer": "^23.8.0",
     "sass": "1.86.0",
+    "semver": "^7.7.1",
     "sqlite3": "^5.1.7",
     "tslib": "^2.6.3",
     "uglify-js": "3.19.3",
diff --git a/plugins/crashes/api/api.js b/plugins/crashes/api/api.js
@@ -8,7 +8,6 @@ var plugin = {},
     Duplex = require('stream').Duplex,
     Promise = require("bluebird"),
     trace = require("./parts/stacktrace.js"),
-    versionUtils = require('./parts/version.js'),
     { DEFAULT_MAX_CUSTOM_FIELD_KEYS } = require('./parts/custom_field.js'),
     plugins = require('../../pluginManager.js'),
     { validateCreate, validateRead, validateUpdate, validateDelete } = require('../../../api/utils/rights.js');
@@ -468,6 +467,19 @@ plugins.setConfigs("crashes", {
                     if ('app_version' in report && typeof report.app_version !== 'string') {
                         report.app_version += '';
                     }
+
+                    // Parse app_version into separate major, minor, patch version fields
+                    if ('app_version' in report) {
+                        const versionComponents = common.parseAppVersion(report.app_version);
+
+                        // Only store the components as separate fields if parsing was successful
+                        if (versionComponents.success) {
+                            report.app_version_major = versionComponents.major;
+                            report.app_version_minor = versionComponents.minor;
+                            report.app_version_patch = versionComponents.patch;
+                        }
+                    }
+
                     let updateData = {$inc: {}};
 
                     updateData.$inc["data.crashes"] = 1;
@@ -511,6 +523,9 @@ plugins.setConfigs("crashes", {
                                     { name: "muted", type: "l" },
                                     { name: "background", type: "l" },
                                     { name: "app_version", type: "l" },
+                                    { name: "app_version_major", type: "n" },
+                                    { name: "app_version_minor", type: "n" },
+                                    { name: "app_version_patch", type: "n" },
                                     { name: "ram_current", type: "n" },
                                     { name: "ram_total", type: "n" },
                                     { name: "disk_current", type: "n" },
@@ -600,7 +615,7 @@ plugins.setConfigs("crashes", {
                                     groupInsert.is_resolved = false;
                                     groupInsert.startTs = report.ts;
                                     groupInsert.latest_version = report.app_version;
-                                    groupInsert.latest_version_for_sort = versionUtils.transformAppVersion(report.app_version);
+                                    groupInsert.latest_version_for_sort = common.transformAppVersion(report.app_version);
                                     groupInsert.error = report.error;
                                     groupInsert.lrid = report._id + "";
 
@@ -722,7 +737,7 @@ plugins.setConfigs("crashes", {
                                         if (!isNew && crashGroup) {
                                             if (crashGroup.latest_version && common.versionCompare(report.app_version.replace(/\./g, ":"), crashGroup.latest_version.replace(/\./g, ":")) > 0) {
                                                 group.latest_version = report.app_version;
-                                                group.latest_version_for_sort = versionUtils.transformAppVersion(report.app_version);
+                                                group.latest_version_for_sort = common.transformAppVersion(report.app_version);
                                             }
                                             if (plugins.getConfig('crashes').same_app_version_crash_update) {
                                                 if (crashGroup.latest_version && common.versionCompare(report.app_version.replace(/\./g, ":"), crashGroup.latest_version.replace(/\./g, ":")) >= 0) {
diff --git a/plugins/crashes/api/parts/version.js b/plugins/crashes/api/parts/version.js
@@ -31,11 +31,13 @@ function checkAppVersion(inpVersion) {
 
 module.exports = {
     /**
+     *  @deprecated
      *  Transform a version string so it will be numerically correct when sorted
      *  For example '1.10.2' will be transformed to '100001.100010.100002'
      *  So when sorted ascending it will come after '1.2.0' ('100001.100002.100000')
      *  @param {string} inpVersion - an app version string
      *  @return {string} the transformed app version
+     *  @note These method are now moved to @module api/utils/common
      */
     transformAppVersion: function(inpVersion) {
         const [execResult, versionScheme] = checkAppVersion(inpVersion);
diff --git a/types/common.d.ts b/types/common.d.ts

Original file line number	Diff line number	Diff line change
`@@ -398,7 +398,7 @@ usage.returnAllProcessedMetrics = function(params) {`
`398`	`398`	`continue;`
`399`	`399`	`}`
`400`	`400`
`401`		`- if (recvMetricValue) {`
	`401`	`+ if (recvMetricValue !== undefined && recvMetricValue !== null) {`
`402`	`402`	`var escapedMetricVal = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":");`
`403`	`403`	`processedMetrics[tmpMetric.short_code] = escapedMetricVal;`
`404`	`404`	`}`
`@@ -705,7 +705,7 @@ function processMetrics(user, uniqueLevelsZero, uniqueLevelsMonth, params, done)`
`705`	`705`	`continue;`
`706`	`706`	`}`
`707`	`707`
`708`		`- if (recvMetricValue) {`
	`708`	`+ if (recvMetricValue !== undefined && recvMetricValue !== null) {`
`709`	`709`	`recvMetricValue = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":");`
`710`	`710`	`postfix = common.crypto.createHash("md5").update(recvMetricValue).digest('base64')[0];`
`711`	`711`	`metaToFetch[predefinedMetrics[i].db + params.app_id + "_" + dateIds.zero + "_" + postfix] = {`
`@@ -767,7 +767,7 @@ function processMetrics(user, uniqueLevelsZero, uniqueLevelsMonth, params, done)`
`767`	`767`	`continue;`
`768`	`768`	`}`
`769`	`769`
`770`		`- if (recvMetricValue) {`
	`770`	`+ if (recvMetricValue !== undefined && recvMetricValue !== null) {`
`771`	`771`	`escapedMetricVal = (recvMetricValue + "").replace(/^\$/, "").replace(/\./g, ":");`
`772`	`772`	`postfix = common.crypto.createHash("md5").update(escapedMetricVal).digest('base64')[0];`
`773`	`773`
`@@ -1110,6 +1110,15 @@ plugins.register("/sdk/user_properties", async function(ob) {`
`1110`	`1110`	`userProps[key] = up[key];`
`1111`	`1111`	`}`
`1112`	`1112`	`}`
	`1113`	`+`
	`1114`	`+ if (params.qstring.metrics._app_version) {`
	`1115`	`+ const versionComponents = common.parseAppVersion(params.qstring.metrics._app_version);`
	`1116`	`+ if (versionComponents.success) {`
	`1117`	`+ userProps.av_major = versionComponents.major;`
	`1118`	`+ userProps.av_minor = versionComponents.minor;`
	`1119`	`+ userProps.av_patch = versionComponents.patch;`
	`1120`	`+ }`
	`1121`	`+ }`
`1113`	`1122`	`}`
`1114`	`1123`
`1115`	`1124`	`if (params.qstring.session_duration) {`