feat(scorecard): implement cleanup expired metrics mechanism

Signed-off-by: Ihor Mykhno <imykhno@redhat.com>

Author: imykhno

workspaces/scorecard/plugins/scorecard-backend/config.d.ts

@@ -19,6 +19,8 @@ import { SchedulerServiceTaskScheduleDefinitionConfig } from '@backstage/backend
 export interface Config {
   /** Configuration for scorecard plugin */
   scorecard?: {
+    /** Number of days to retain metric data in the database. Older data will be automatically cleaned up. Default: 365 days */
+    dataRetentionDays?: number;
     /** Configuration for scorecard metric providers */
     plugins?: {
       /** Configuration for datasource */

workspaces/scorecard/plugins/scorecard-backend/src/constants/metricSchedules.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const DEFAULT_DATA_RETENTION_DAYS = 365;

workspaces/scorecard/plugins/scorecard-backend/src/database/DatabaseMetricValuesStore.ts

@@ -49,4 +49,13 @@ export class DatabaseMetricValuesStore implements MetricValuesStore {
           .groupBy('metric_id'),
       );
   }
+
+  /**
+   * Delete metric values that are older than the given date
+   */
+  async cleanupExpiredMetrics(olderThan: Date): Promise<number> {
+    return await this.knex(this.tableName)
+      .where('timestamp', '<', olderThan)
+      .del();
+  }
 }

workspaces/scorecard/plugins/scorecard-backend/src/database/MetricValuesStore.ts

@@ -41,4 +41,9 @@ export interface MetricValuesStore {
     catalog_entity_ref: string,
     metric_ids: string[],
   ): Promise<DbMetricValue[]>;
+
+  /**
+   * Delete metric values that are older than the given date
+   */
+  cleanupExpiredMetrics(olderThan: Date): Promise<number>;
 }

workspaces/scorecard/plugins/scorecard-backend/src/providers/MetricProvidersScheduler.ts

@@ -31,6 +31,8 @@ import {
   MetricValuesStore,
 } from '../database/MetricValuesStore';
 import { MetricProvidersRegistry } from './MetricProvidersRegistry';
+import { DEFAULT_DATA_RETENTION_DAYS } from '../constants/metricSchedules';
+import { daysToMilliseconds } from '../utils/daysToMilliseconds';
 
 export class MetricProvidersScheduler {
   private static readonly CATALOG_BATCH_SIZE = 50;
@@ -46,14 +48,19 @@ export class MetricProvidersScheduler {
   ) {}
 
   async schedule() {
+    this.scheduleCleanupExpiredMetrics();
+    this.scheduleLoadProviderMetrics();
+  }
+
+  private async scheduleLoadProviderMetrics() {
+    let countScheduledProviders = 0;
     const providers = this.metricProvidersRegistry.listProviders();
-    const scheduledProviders: string[] = [];
 
     await Promise.all(
       providers.map(async provider => {
         try {
           await this.scheduleProvider(provider);
-          scheduledProviders.push(provider.getProviderId());
+          countScheduledProviders++;
         } catch (e) {
           this.logger.warn(
             `Failed to schedule provider ${provider.getProviderId()}, ${e}`,
@@ -63,7 +70,7 @@ export class MetricProvidersScheduler {
     );
 
     this.logger.info(
-      `Scheduled ${scheduledProviders.length}/${providers.length} metric providers`,
+      `Scheduled ${countScheduledProviders}/${providers.length} metric providers`,
     );
   }
 
@@ -173,9 +180,40 @@ export class MetricProvidersScheduler {
           this.config.getConfig(schedulePath),
         )
       : ({
-          frequency: { hours: 2 },
+          frequency: { hours: 1 },
           timeout: { minutes: 15 },
           initialDelay: { seconds: 3 },
         } as SchedulerServiceTaskScheduleDefinition);
   }
+
+  private async cleanupExpiredMetric(logger: LoggerService): Promise<void> {
+    const dataRetentionDays =
+      this.config.getOptionalNumber('scorecard.dataRetentionDays') ??
+      DEFAULT_DATA_RETENTION_DAYS;
+    const olderThan = new Date(
+      Date.now() - daysToMilliseconds(dataRetentionDays),
+    );
+
+    const deletedMetrics = await this.metricValuesStore.cleanupExpiredMetrics(
+      olderThan,
+    );
+    logger.info(
+      `Deleted ${deletedMetrics} abandoned metrics older than ${dataRetentionDays} days`,
+    );
+  }
+
+  /**
+   * Schedule a task to delete abandoned metrics
+   */
+  private async scheduleCleanupExpiredMetrics() {
+    await this.scheduler.scheduleTask({
+      id: 'cleanup-expired-metrics',
+      frequency: { days: 1 },
+      timeout: { minutes: 2 },
+      initialDelay: { seconds: 3 },
+      fn: async () => {
+        await this.cleanupExpiredMetric(this.logger);
+      },
+    });
+  }
 }

workspaces/scorecard/plugins/scorecard-backend/src/utils/daysToMilliseconds.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const daysToMilliseconds = (days: number) => {
+  return days * 24 * 60 * 60 * 1000;
+};