Merge pull request #2543 from blacklanternsecurity/upgrade-gowitness

Upgrade Gowitness

Author: TheTechromancer

bbot/modules/gowitness.py

@@ -16,7 +16,7 @@ class gowitness(BaseModule):
     flags = ["active", "safe", "web-screenshots"]
     meta = {"description": "Take screenshots of webpages", "created_date": "2022-07-08", "author": "@TheTechromancer"}
     options = {
-        "version": "2.4.2",
+        "version": "3.0.5",
         "threads": 0,
         "timeout": 10,
         "resolution_x": 1440,
@@ -161,6 +161,7 @@ async def handle_batch(self, *events):
                 key = e.data["url"]
             event_dict[key] = e
         stdin = "\n".join(list(event_dict))
+        self.hugeinfo(f"Gowitness input: {stdin}")
 
         try:
             async for line in self.run_process_live(self.command, input=stdin, idle_timeout=self.idle_timeout):
@@ -174,12 +175,14 @@ async def handle_batch(self, *events):
         new_screenshots = await self.get_new_screenshots()
         for filename, screenshot in new_screenshots.items():
             url = screenshot["url"]
+            url = self.helpers.clean_url(url).geturl()
             final_url = screenshot["final_url"]
             filename = self.screenshot_path / screenshot["filename"]
             filename = filename.relative_to(self.scan.home)
             # NOTE: this prevents long filenames from causing problems in BBOT, but gowitness will still fail to save it.
             filename = self.helpers.truncate_filename(filename)
             webscreenshot_data = {"path": str(filename), "url": final_url}
+            self.hugewarning(event_dict)
             parent_event = event_dict[url]
             await self.emit_event(
                 webscreenshot_data,
@@ -191,11 +194,11 @@ async def handle_batch(self, *events):
         # emit URLs
         new_network_logs = await self.get_new_network_logs()
         for url, row in new_network_logs.items():
-            ip = row["ip"]
+            ip = row["remote_ip"]
             status_code = row["status_code"]
             tags = [f"status-{status_code}", f"ip-{ip}", "spider-danger"]
 
-            _id = row["url_id"]
+            _id = row["result_id"]
             parent_url = self.screenshots_taken[_id]
             parent_event = event_dict[parent_url]
             if url and url.startswith("http"):
@@ -210,7 +213,7 @@ async def handle_batch(self, *events):
         # emit technologies
         new_technologies = await self.get_new_technologies()
         for row in new_technologies.values():
-            parent_id = row["url_id"]
+            parent_id = row["result_id"]
             parent_url = self.screenshots_taken[parent_id]
             parent_event = event_dict[parent_url]
             technology = row["value"]
@@ -224,28 +227,29 @@ async def handle_batch(self, *events):
 
     def construct_command(self):
         # base executable
-        command = ["gowitness"]
+        command = ["gowitness", "scan"]
         # chrome path
         if self.chrome_path is not None:
             command += ["--chrome-path", str(self.chrome_path)]
         # db path
-        command += ["--db-path", str(self.db_path)]
+        command += ["--write-db"]
+        command += ["--write-db-uri", f"sqlite://{self.db_path}"]
         # screenshot path
         command += ["--screenshot-path", str(self.screenshot_path)]
         # user agent
-        command += ["--user-agent", f"{self.scan.useragent}"]
+        command += ["--chrome-user-agent", f"{self.scan.useragent}"]
         # proxy
         if self.proxy:
-            command += ["--proxy", str(self.proxy)]
+            command += ["--chrome-proxy", str(self.proxy)]
         # resolution
-        command += ["--resolution-x", str(self.resolution_x)]
-        command += ["--resolution-y", str(self.resolution_y)]
-        # input
-        command += ["file", "-f", "-"]
+        command += ["--chrome-window-x", str(self.resolution_x)]
+        command += ["--chrome-window-y", str(self.resolution_y)]
         # threads
         command += ["--threads", str(self.threads)]
         # timeout
         command += ["--timeout", str(self.timeout)]
+        # input
+        command += ["file", "-f", "-"]
         return command
 
     async def get_new_screenshots(self):
@@ -254,8 +258,10 @@ async def get_new_screenshots(self):
             async with aiosqlite.connect(str(self.db_path)) as con:
                 con.row_factory = aiosqlite.Row
                 con.text_factory = self.helpers.smart_decode
-                async with con.execute("SELECT * FROM urls") as cur:
+                async with con.execute("SELECT * FROM results") as cur:
+                    self.critical(f"CUR: {cur}")
                     async for row in cur:
+                        self.critical(f"SCREENSHOT: {row}")
                         row = dict(row)
                         _id = row["id"]
                         if _id not in self.screenshots_taken:
@@ -270,8 +276,9 @@ async def get_new_network_logs(self):
                 con.row_factory = aiosqlite.Row
                 async with con.execute("SELECT * FROM network_logs") as cur:
                     async for row in cur:
+                        self.critical(f"NETWORK LOG: {row}")
                         row = dict(row)
-                        url = row["final_url"]
+                        url = row["url"]
                         if url not in self.connections_logged:
                             self.connections_logged.add(url)
                             network_logs[url] = row
@@ -284,6 +291,7 @@ async def get_new_technologies(self):
                 con.row_factory = aiosqlite.Row
                 async with con.execute("SELECT * FROM technologies") as cur:
                     async for row in cur:
+                        self.critical(f"TECHNOLOGY: {row}")
                         _id = row["id"]
                         if _id not in self.technologies_found:
                             self.technologies_found.add(_id)

bbot/test/conftest.py

@@ -1,8 +1,8 @@
 import os
 import ssl
 import time
-import shutil
 import pytest
+import shutil
 import asyncio
 import logging
 from pathlib import Path

bbot/test/test_step_2/module_tests/test_module_gowitness.py

@@ -51,9 +51,9 @@ def check(self, module_test, events):
         )
 
         screenshots_path = self.home_dir / "scans" / module_test.scan.name / "gowitness" / "screenshots"
-        screenshots = list(screenshots_path.glob("*.png"))
+        screenshots = list(screenshots_path.glob("*.jpeg"))
         assert len(screenshots) == 1, (
-            f"{len(screenshots):,} .png files found at {screenshots_path}, should have been 1"
+            f"{len(screenshots):,} .jpeg files found at {screenshots_path}, should have been 1"
         )
         assert 1 == len([e for e in events if e.type == "URL" and e.data == "http://127.0.0.1:8888/"])
         assert 1 == len(
@@ -74,9 +74,9 @@ class TestGowitness_Social(TestGowitness):
 
     def check(self, module_test, events):
         screenshots_path = self.home_dir / "scans" / module_test.scan.name / "gowitness" / "screenshots"
-        screenshots = list(screenshots_path.glob("*.png"))
+        screenshots = list(screenshots_path.glob("*.jpeg"))
         assert len(screenshots) == 2, (
-            f"{len(screenshots):,} .png files found at {screenshots_path}, should have been 2"
+            f"{len(screenshots):,} .jpeg files found at {screenshots_path}, should have been 2"
         )
         assert 2 == len([e for e in events if e.type == "WEBSCREENSHOT"])
         assert 1 == len(
@@ -86,7 +86,7 @@ def check(self, module_test, events):
                 if e.type == "WEBSCREENSHOT" and e.data["url"] == "http://127.0.0.1:8888/blacklanternsecurity"
             ]
         )
-        assert 1 == len(
+        assert len(
             [
                 e
                 for e in events