tracking rules_version and minor changes

Author: Akshit Maheshwary

api_app/analyzers_manager/file_analyzers/capa_info.py

@@ -4,15 +4,18 @@
 import json
 import logging
 import os
+import shutil
 import subprocess
 from shlex import quote
 from zipfile import ZipFile
 
 import requests
 from django.conf import settings
+from django.utils import timezone
 
 from api_app.analyzers_manager.classes import FileAnalyzer
 from api_app.analyzers_manager.exceptions import AnalyzerRunException
+from api_app.analyzers_manager.models import AnalyzerRulesFileVersion, PythonModule
 from tests.mock_utils import if_mock_connections, patch
 
 logger = logging.getLogger(__name__)
@@ -28,9 +31,44 @@ class CapaInfo(FileAnalyzer):
     shellcode: bool
     arch: str
     timeout: float = 15
+    force_pull_signatures: bool = False
+
+    def _check_if_latest_version(self, latest_version: str) -> bool:
+
+        analyzer_rules_file_version = AnalyzerRulesFileVersion.objects.filter(
+            python_module=self.python_module
+        ).first()
+
+        if analyzer_rules_file_version is None:
+            return False
+
+        return latest_version == analyzer_rules_file_version.last_downloaded_version
 
     @classmethod
-    def _unzip(cls):
+    def _update_rules_file_version(cls, latest_version: str, file_url: str):
+        capa_module = PythonModule.objects.get(
+            module="capa_info.CapaInfo",
+            base_path="api_app.analyzers_manager.file_analyzers",
+        )
+
+        analyzer_rules_file_version, created = (
+            AnalyzerRulesFileVersion.objects.update_or_create(
+                python_module=capa_module,
+                defaults={
+                    "last_downloaded_version": latest_version,
+                    "download_url": file_url,
+                    "downloaded_at": timezone.now(),
+                },
+            )
+        )
+
+        if created:
+            logger.info(f"Created new entry for {capa_module} rules file version")
+        else:
+            logger.info(f"Updated existing entry for {capa_module} rules file version")
+
+    @classmethod
+    def _unzip_rules(cls):
         logger.info(f"Extracting rules at {RULES_LOCATION}")
         with ZipFile(RULES_FILE, mode="r") as archive:
             archive.extractall(
@@ -41,31 +79,46 @@ def _unzip(cls):
     @classmethod
     def _download_rules(cls, latest_version: str):
 
-        if not os.path.exists(RULES_LOCATION):
-            os.makedirs(RULES_LOCATION)
+        if os.path.exists(RULES_LOCATION):
+            logger.info(f"Removing existing rules at {RULES_LOCATION}")
+            shutil.rmtree(RULES_LOCATION)
+
+        os.makedirs(RULES_LOCATION)
+        logger.info(f"Created fresh rules directory at {RULES_LOCATION}")
 
         file_to_download = latest_version + ".zip"
         file_url = RULES_URL + file_to_download
         try:
 
             response = requests.get(file_url, stream=True)
-            logger.info(f"Started downloading rules from {file_url}")
+            logger.info(
+                f"Started downloading rules with version: {latest_version} from {file_url}"
+            )
             with open(RULES_FILE, mode="wb+") as file:
                 for chunk in response.iter_content(chunk_size=10 * 1024):
                     file.write(chunk)
 
+            cls._update_rules_file_version(latest_version, file_url)
+            logger.info(f"Bumped up version number in db to {latest_version}")
+
         except Exception as e:
             logger.error(f"Failed to download rules with error: {e}")
             raise AnalyzerRunException("Failed to download rules")
 
-        logger.info(f"Rules have been successfully downloaded at {RULES_LOCATION}")
+        logger.info(
+            f"Rules with version: {latest_version} have been successfully downloaded at {RULES_LOCATION}"
+        )
 
     @classmethod
     def _download_signatures(cls) -> None:
         logger.info(f"Downloading signatures at {SIGNATURE_LOCATION} now")
 
-        if not os.path.exists(SIGNATURE_LOCATION):
-            os.makedirs(SIGNATURE_LOCATION)
+        if os.path.exists(SIGNATURE_LOCATION):
+            logger.info(f"Removing existing signatures at {SIGNATURE_LOCATION}")
+            shutil.rmtree(SIGNATURE_LOCATION)
+
+        os.makedirs(SIGNATURE_LOCATION)
+        logger.info(f"Created fresh signatures directory at {SIGNATURE_LOCATION}")
 
         signatures_url = "https://api.github.com/repos/mandiant/capa/contents/sigs"
         try:
@@ -77,28 +130,29 @@ def _download_signatures(cls) -> None:
                 filename = signature["name"]
                 download_url = signature["download_url"]
 
+                signature_file_path = os.path.join(SIGNATURE_LOCATION, filename)
+
                 sig_content = requests.get(download_url, stream=True)
-                with open(filename, mode="wb") as file:
+                with open(signature_file_path, mode="wb") as file:
                     for chunk in sig_content.iter_content(chunk_size=10 * 1024):
                         file.write(chunk)
 
         except Exception as e:
             logger.error(f"Failed to download signature: {e}")
             raise AnalyzerRunException("Failed to update signatures")
-        logger.info("Successfully updated singatures")
+        logger.info("Successfully updated signatures")
 
     @classmethod
     def update(cls) -> bool:
         try:
-            logger.info("Updating capa rules and signatures")
+            logger.info("Updating capa rules")
             response = requests.get(
                 "https://api.github.com/repos/mandiant/capa-rules/releases/latest"
             )
             latest_version = response.json()["tag_name"]
             cls._download_rules(latest_version)
-            cls._unzip()
-            cls._download_signatures()
-            logger.info("Successfully updated capa rules and signatures")
+            cls._unzip_rules()
+            logger.info("Successfully updated capa rules")
 
             return True
 
@@ -109,16 +163,22 @@ def update(cls) -> bool:
 
     def run(self):
         try:
-            if (
-                not (
-                    os.path.isdir(RULES_LOCATION) and os.path.isdir(SIGNATURE_LOCATION)
-                )
-                and not self.update()
-            ):
 
-                raise AnalyzerRunException(
-                    "Couldn't update capa rules or signatures successfully"
-                )
+            response = requests.get(
+                "https://api.github.com/repos/mandiant/capa-rules/releases/latest"
+            )
+            latest_version = response.json()["tag_name"]
+
+            update_status = (
+                True if self._check_if_latest_version(latest_version) else self.update()
+            )
+
+            if self.force_pull_signatures or not os.path.isdir(SIGNATURE_LOCATION):
+                self._download_signatures()
+
+            if not (os.path.isdir(RULES_LOCATION)) and not update_status:
+
+                raise AnalyzerRunException("Couldn't update capa rules")
 
             command: list[str] = ["/usr/local/bin/capa", "--quiet", "--json"]
             shell_code_arch = "sc64" if self.arch == "64" else "sc32"
@@ -136,7 +196,9 @@ def run(self):
 
             command.append(quote(self.filepath))
 
-            logger.info(f"Starting CAPA analysis for {self.filename}")
+            logger.info(
+                f"Starting CAPA analysis for {self.filename} with hash: {self.md5} and command: {command}"
+            )
 
             process: subprocess.CompletedProcess = subprocess.run(
                 command,
@@ -147,13 +209,20 @@ def run(self):
             )
 
             result = json.loads(process.stdout)
-            logger.info("CAPA analysis successfully completed")
+            result["command_executed"] = command
+            result["rules_version"] = latest_version
+
+            logger.info(
+                f"CAPA analysis successfully completed for file: {self.filename} with hash {self.md5}"
+            )
 
         except subprocess.CalledProcessError as e:
             stderr = e.stderr
-            logger.info(f"Capa Info failed to run for {self.filename} with command {e}")
+            logger.info(
+                f"Capa Info failed to run for {self.filename} with hash: {self.md5} with command {e}"
+            )
             raise AnalyzerRunException(
-                f" Analyzer for {self.filename} failed with error: {stderr}"
+                f" Analyzer for {self.filename} with hash: {self.md5} failed with error: {stderr}"
             )
 
         return result

api_app/analyzers_manager/migrations/0164_update_capa.py

@@ -6,14 +6,29 @@
 def migrate(apps, schema_editor):
     PythonModule = apps.get_model("api_app", "PythonModule")
     Parameter = apps.get_model("api_app", "Parameter")
+    CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule")
     AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
 
     pm = PythonModule.objects.get(
         module="capa_info.CapaInfo",
         base_path="api_app.analyzers_manager.file_analyzers",
     )
 
+    new_crontab, created = CrontabSchedule.objects.get_or_create(
+        minute="0",
+        hour="0",
+        day_of_week="*",
+        day_of_month="*",
+        month_of_year="*",
+        timezone="UTC",
+    )
+    if created:
+        pm.update_schedule = new_crontab
+        pm.full_clean()
+        pm.save()
+
     AnalyzerConfig.objects.filter(python_module=pm).update(soft_time_limit=1800)
+    AnalyzerConfig.objects.filter(python_module=pm).update(docker_based=False)
 
     p1 = Parameter(
         name="timeout",
@@ -24,9 +39,21 @@ def migrate(apps, schema_editor):
         python_module=pm,
     )
 
+    p2 = Parameter(
+        name="force_pull_signatures",
+        type="bool",
+        description="Force download signatures from flare-capa github repository",
+        is_secret=False,
+        required=False,
+        python_module=pm,
+    )
+
     p1.full_clean()
     p1.save()
 
+    p2.full_clean()
+    p2.save()
+
 
 class Migration(migrations.Migration):
 

api_app/analyzers_manager/migrations/0165_analyzerrulesfileversion.py

@@ -0,0 +1,43 @@
+# Generated by Django 4.2.17 on 2025-09-05 19:42
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api_app", "0071_delete_last_elastic_report"),
+        ("analyzers_manager", "0164_update_capa"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="AnalyzerRulesFileVersion",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "last_downloaded_version",
+                    models.CharField(max_length=50, null=True, blank=True),
+                ),
+                ("download_url", models.URLField(null=True, blank=True)),
+                ("downloaded_at", models.DateTimeField(auto_now_add=True)),
+                (
+                    "python_module",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="rules_version",
+                        to="api_app.pythonmodule",
+                    ),
+                ),
+            ],
+        ),
+    ]

api_app/analyzers_manager/models.py

@@ -350,3 +350,13 @@ def plugin_type(cls) -> str:
     @property
     def config_exception(cls):
         return AnalyzerConfigurationException
+
+
+class AnalyzerRulesFileVersion(models.Model):
+    last_downloaded_version = models.CharField(max_length=50, blank=True, null=True)
+    download_url = models.URLField(max_length=200, blank=True, null=True)
+    downloaded_at = models.DateTimeField(auto_now_add=True)
+
+    python_module = models.ForeignKey(
+        PythonModule, on_delete=models.PROTECT, related_name="rules_version"
+    )

requirements/project-requirements.txt

@@ -91,7 +91,7 @@ wad==0.4.6
 debloat==1.6.4
 phonenumbers==9.0.3
 die-python==0.4.0
-# guarddog==2.1.0 # version greater than 2.1.0 raises dependency conflicts. Commenting this out due to dependency conflict.
+# guarddog==2.1.0 # version greater than 2.1.0 raises dependency conflicts. Commenting this out due to dependency conflicts.
 jbxapi==3.23.0
 flare-floss==3.1.1
 flare-capa==9.2.1