Major vhost (virtualhost) rewrite

bbot/core/event/base.py

@@ -1604,16 +1604,17 @@ def _pretty_string(self):
         return self.data["technology"]
 
 
-class VHOST(DictHostEvent):
+class VIRTUAL_HOST(DictHostEvent):
     class _data_validator(BaseModel):
         host: str
-        vhost: str
+        virtual_host: str
+        description: str
         url: Optional[str] = None
         _validate_url = field_validator("url")(validators.validate_url)
         _validate_host = field_validator("host")(validators.validate_host)
 
     def _pretty_string(self):
-        return self.data["vhost"]
+        return self.data["virtual_host"]
 
 
 class PROTOCOL(DictHostEvent):

bbot/core/helpers/command.py

@@ -195,7 +195,7 @@ async def _spawn_proc(self, *command, **kwargs):
             raise ValueError("stdin and input arguments may not both be used.")
         kwargs["stdin"] = asyncio.subprocess.PIPE
 
-    log.hugeverbose(f"run: {' '.join(command)}")
+    log.debug(f"run: {' '.join(command)}")
     try:
         proc = await asyncio.create_subprocess_exec(*command, **kwargs)
         return proc, _input, command

bbot/core/helpers/misc.py

@@ -11,9 +11,11 @@
 import regex as re
 import subprocess as sp
 
+
 from pathlib import Path
 from contextlib import suppress
 from unidecode import unidecode  # noqa F401
+from typing import Iterable, Awaitable, Optional
 from asyncio import create_task, gather, sleep, wait_for  # noqa
 from urllib.parse import urlparse, quote, unquote, urlunparse, urljoin  # noqa F401
 
@@ -2590,30 +2592,101 @@ def parse_port_string(port_string):
     return ports
 
 
-async def as_completed(coros):
+async def as_completed(
+    coroutines: Iterable[Awaitable],
+    max_concurrent: Optional[int] = None,
+):
     """
-    Async generator that yields completed Tasks as they are completed.
+    Yield completed coroutines as they finish with optional concurrency limiting.
+    All coroutines are scheduled as tasks internally for execution.
 
-    Args:
-        coros (iterable): An iterable of coroutine objects or asyncio Tasks.
+    Guarantees cleanup:
+    - If the consumer breaks early or an internal cancellation is detected, all remaining
+      tasks are cancelled and awaited (with return_exceptions=True) to avoid
+      "Task exception was never retrieved" warnings.
+    """
+    it = iter(coroutines)
 
-    Yields:
-        asyncio.Task: A Task object that has completed its execution.
+    running: set[asyncio.Task] = set()
+    limit = max_concurrent or float("inf")
 
-    Examples:
-        >>> async def main():
-        ...     async for task in as_completed([coro1(), coro2(), coro3()]):
-        ...         result = task.result()
-        ...         print(f'Task completed with result: {result}')
+    async def _cancel_and_drain_remaining():
+        if not running:
+            return
+        for t in running:
+            t.cancel()
+        try:
+            await asyncio.gather(*running, return_exceptions=True)
+        finally:
+            running.clear()
 
-        >>> asyncio.run(main())
+    # Prime the running set up to the concurrency limit (or all, if unlimited)
+    try:
+        while len(running) < limit:
+            coro = next(it)
+            running.add(asyncio.create_task(coro))
+    except StopIteration:
+        pass
+
+    # Dedup state for repeated error messages
+    _last_err = {"msg": None, "count": 0}
+
+    try:
+        # Drain: yield completed tasks, backfill from the iterator as slots free up
+        while running:
+            done, running = await asyncio.wait(running, return_when=asyncio.FIRST_COMPLETED)
+            for task in done:
+                # Immediately backfill one slot per completed task, if more work remains
+                try:
+                    coro = next(it)
+                    running.add(asyncio.create_task(coro))
+                except StopIteration:
+                    pass
+
+                # If task raised, handle cancellation gracefully and dedupe noisy repeats
+                if task.exception() is not None:
+                    e = task.exception()
+                    if in_exception_chain(e, (KeyboardInterrupt, asyncio.CancelledError)):
+                        # Quietly stop if we're being cancelled
+                        log.info("as_completed: cancellation detected; exiting early")
+                        await _cancel_and_drain_remaining()
+                        return
+                    # Build a concise message
+                    msg = f"as_completed yielded exception: {e}"
+                    if msg == _last_err["msg"]:
+                        _last_err["count"] += 1
+                        if _last_err["count"] <= 3:
+                            log.warning(msg)
+                        elif _last_err["count"] % 10 == 0:
+                            log.warning(f"{msg} (repeated {_last_err['count']}x)")
+                        else:
+                            log.debug(msg)
+                    else:
+                        _last_err["msg"] = msg
+                        _last_err["count"] = 1
+                        log.warning(msg)
+                yield task
+    finally:
+        # If the consumer breaks early or an error bubbles, ensure we don't leak tasks
+        await _cancel_and_drain_remaining()
+
+
+def get_waf_strings():
+    """
+    Returns a list of common WAF (Web Application Firewall) detection strings.
+
+    Returns:
+        list: List of WAF detection strings
+
+    Examples:
+        >>> waf_strings = get_waf_strings()
+        >>> "The requested URL was rejected" in waf_strings
+        True
     """
-    tasks = {coro if isinstance(coro, asyncio.Task) else asyncio.create_task(coro): coro for coro in coros}
-    while tasks:
-        done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED)
-        for task in done:
-            tasks.pop(task)
-            yield task
+    return [
+        "The requested URL was rejected",
+        "This content has been blocked",
+    ]
 
 
 def clean_dns_record(record):

bbot/core/helpers/names_generator.py

@@ -42,6 +42,7 @@
     "crumbly",
     "cryptic",
     "cuddly",
+    "cursed",
     "cute",
     "dark",
     "dastardly",
@@ -158,6 +159,7 @@
     "mysterious",
     "nascent",
     "naughty",
+    "nautical",
     "nefarious",
     "negligent",
     "neurotic",
@@ -169,6 +171,7 @@
     "overmedicated",
     "overwhelming",
     "overzealous",
+    "pacific",
     "paranoid",
     "pasty",
     "peckish",
@@ -347,7 +350,9 @@
     "brittany",
     "bruce",
     "bryan",
+    "buhner",
     "caitlyn",
+    "cal",
     "caleb",
     "cameron",
     "carl",
@@ -432,6 +437,7 @@
     "evan",
     "evelyn",
     "faramir",
+    "felix",
     "florence",
     "fox",
     "frances",
@@ -458,6 +464,7 @@
     "gollum",
     "grace",
     "gregory",
+    "griffey",
     "gus",
     "hagrid",
     "hank",
@@ -472,6 +479,7 @@
     "homer",
     "howard",
     "hunter",
+    "ichiro",
     "irene",
     "isaac",
     "isabella",
@@ -515,6 +523,7 @@
     "judy",
     "julia",
     "julie",
+    "julio",
     "justin",
     "karen",
     "katherine",
@@ -547,6 +556,7 @@
     "logan",
     "lois",
     "lori",
+    "lou",
     "louis",
     "louise",
     "lucius",
@@ -578,6 +588,7 @@
     "mildred",
     "milhouse",
     "monica",
+    "moose",
     "nancy",
     "natalie",
     "nathan",
@@ -694,6 +705,8 @@
     "wendy",
     "william",
     "willie",
+    "wilson",
+    "woo",
     "worf",
     "wormtongue",
     "xavier",