Require an API key for github_workflows

.gitattributes

@@ -5,4 +5,4 @@
 *.txt text eol=lf
 *.json text eol=lf
 *.md text eol=lf
-*.sh text eol=lf
+*.sh text eol=lf

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: Bug Report
+about: Create a report to help us improve
+title: ""
+labels: bug
+assignees: ""
+---
+
+**Describe the bug**
+What happened vs what was expected?
+
+**BBOT Command**
+Example: `bbot -m httpx -t evilcorp.com`
+
+**OS, BBOT Installation Method + Version**
+Example: `OS: Arch Linux, Installation method: pip, BBOT version: 1.0.3.545`
+Note: You can get the BBOT version with `bbot --version`
+Note: BBOT is designed from the ground up to run on Linux. Windows and MacOS are not officially supported. If you are using one of these platforms, it's recommended to use Docker.
+
+**BBOT Config**
+Attach your full BBOT preset (to show it, add `--current-preset` to your BBOT command).
+
+**Logs/Screenshots**
+If possible, produce the bug while `--debug` is enabled, and attach the relevant parts of the output.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,10 @@
+---
+name: Feature Request
+about: Request a new feature
+title: ""
+labels: enhancement
+assignees: ""
+---
+
+**Description**
+Which feature would you like to see added to BBOT? What are its use cases?

.github/dependabot.yml

@@ -0,0 +1,17 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    target-branch: "dev"
+    open-pull-requests-limit: 10
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly
+    target-branch: "dev" 

.github/workflows/distro_tests.yml

@@ -0,0 +1,72 @@
+name: Tests (Linux Distros)
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-distros:
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "kalilinux/kali-rolling", "parrotsec/security"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Python and Poetry
+        run: |
+          if [ -f /etc/os-release ]; then
+            . /etc/os-release
+            if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ] || [ "$ID" = "kali" ] || [ "$ID" = "parrotsec" ]; then
+              export DEBIAN_FRONTEND=noninteractive
+              apt-get update
+              apt-get -y install curl git bash build-essential docker.io libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev
+            elif [ "$ID" = "alpine" ]; then
+              apk add --no-cache bash gcc g++ musl-dev libffi-dev docker curl git make openssl-dev bzip2-dev zlib-dev xz-dev sqlite-dev
+            elif [ "$ID" = "arch" ]; then
+              pacman -Syu --noconfirm curl docker git bash base-devel
+            elif [ "$ID" = "fedora" ]; then
+              dnf install -y curl docker git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel python3-libdnf5
+            elif [ "$ID" = "gentoo" ]; then
+              echo "media-libs/libglvnd X" >> /etc/portage/package.use/libglvnd
+              emerge-webrsync
+              emerge --update --newuse dev-vcs/git media-libs/mesa curl docker bash
+            fi
+          fi
+
+          # Re-run the script with bash
+          exec bash -c "
+            curl https://pyenv.run | bash
+            export PATH=\"$HOME/.pyenv/bin:\$PATH\"
+            export PATH=\"$HOME/.local/bin:\$PATH\"
+            eval \"\$(pyenv init --path)\"
+            eval \"\$(pyenv init -)\"
+            eval \"\$(pyenv virtualenv-init -)\"
+            pyenv install 3.11
+            pyenv global 3.11
+            pyenv rehash
+            python3.11 -m pip install --user pipx
+            python3.11 -m pipx ensurepath
+            pipx install poetry
+          "
+      - name: Set OS Environment Variable
+        run: echo "OS_NAME=${{ matrix.os }}" | sed 's|[:/]|_|g' >> $GITHUB_ENV
+      - name: Run tests
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+          export PATH="$HOME/.pyenv/bin:$PATH"
+          export PATH="$HOME/.pyenv/shims:$PATH"
+          export BBOT_DISTRO_TESTS=true
+          poetry env use python3.11
+          poetry install
+          poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
+      - name: Upload Debug Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pytest-debug-logs-${{ env.OS_NAME }}
+          path: pytest_debug.log

.github/workflows/docs_updater.yml

@@ -0,0 +1,34 @@
+name: Daily Docs Update
+
+on:
+  schedule:
+    - cron: '30 2 * * *'  # Runs daily at 2:30 AM UTC, a less congested time
+  workflow_dispatch:      # Allows manual triggering of the workflow
+
+jobs:
+  update_docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
+          ref: dev  # Checkout the dev branch
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install
+      - name: Generate docs
+        run: |
+          poetry run bbot/scripts/docs.py
+      - name: Create or Update Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
+          branch: update-docs
+          base: dev
+          title: "Automated Docs Update"
+          body: "This is an automated pull request to update the documentation."

.github/workflows/tests.yml

@@ -1,47 +1,171 @@
-name: tests
-on: [push, pull_request]
+name: Tests
+on:
+  push:
+    branches:
+      - stable
+      - dev
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      # if one python version fails, let the others finish
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: ${{ matrix.python-version }}
+      - name: Set Python Version Environment Variable
+        run: echo "PYTHON_VERSION=${{ matrix.python-version }}" | sed 's|[:/]|_|g' >> $GITHUB_ENV
       - name: Install dependencies
         run: |
           pip install poetry
           poetry install
-      - name: Run tests with pytest
+      - name: Lint
         run: |
-          poetry run pytest --disable-warnings --cov-report xml:cov.xml --cov=bbot .
-      - uses: codecov/codecov-action@v2
+          poetry run ruff check
+          poetry run ruff format --check
+      - name: Run tests
+        run: |
+          poetry run pytest -vv --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+      - name: Upload Debug Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pytest-debug-logs-${{ env.PYTHON_VERSION }}
+          path: pytest_debug.log
+      - name: Upload Code Coverage
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./cov.xml
-          fail_ci_if_error: true
           verbose: true
-  pypi:
-    runs-on: ubuntu-latest
+  publish_code:
     needs: test
-    if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/stable')
     continue-on-error: true
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: "3.x"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install build
-      - name: Build package
+          pip install poetry build
+          poetry self add "poetry-dynamic-versioning[plugin]"
+      - name: Build Pypi package
+        if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'
         run: python -m build
-      - name: Publish package
-        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Publish Pypi package
+        if: github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev'
+        uses: pypa/gh-action-pypi-publish@release/v1.12
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
+      - name: Get BBOT version
+        id: version
+        run: echo "BBOT_VERSION=$(poetry version | cut -d' ' -f2)" >> $GITHUB_OUTPUT
+      - name: Publish to Docker Hub (dev)
+        if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        with:
+          name: blacklanternsecurity/bbot
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          tags: "latest,dev,${{ steps.version.outputs.BBOT_VERSION }}"
+      - name: Publish to Docker Hub (stable)
+        if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
+        uses: elgohr/Publish-Docker-Github-Action@v5
+        with:
+          name: blacklanternsecurity/bbot
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          tags: "stable,${{ steps.version.outputs.BBOT_VERSION }}"
+      - name: Docker Hub Description
+        if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          repository: blacklanternsecurity/bbot
+    outputs:
+      BBOT_VERSION: ${{ steps.version.outputs.BBOT_VERSION }}
+  publish_docs:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/stable' || github.ref == 'refs/heads/dev')
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.BBOT_DOCS_UPDATER_PAT }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
+      - uses: actions/cache@v4
+        with:
+          key: mkdocs-material-${{ env.cache_id }}
+          path: .cache
+          restore-keys: |
+            mkdocs-material-
+      - name: Install dependencies
+        run: |
+          pip install poetry
+          poetry install --only=docs
+      - name: Configure Git
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git fetch origin gh-pages:refs/remotes/origin/gh-pages
+          if git show-ref --verify --quiet refs/heads/gh-pages; then
+            git branch -f gh-pages origin/gh-pages
+          else
+            git branch --track gh-pages origin/gh-pages
+          fi
+      - name: Generate docs (stable branch)
+        if: github.ref == 'refs/heads/stable'
+        run: |
+          poetry run mike deploy Stable
+      - name: Generate docs (dev branch)
+        if: github.ref == 'refs/heads/dev'
+        run: |
+          poetry run mike deploy Dev
+      - name: Publish docs
+        run: |
+          git switch gh-pages
+          git push
+  # tag_commit:
+  #   needs: publish_code
+  #   runs-on: ubuntu-latest
+  #   if: github.event_name == 'push' && github.ref == 'refs/heads/stable'
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{ github.head_ref }}
+  #         fetch-depth: 0 # Fetch all history for all tags and branches
+  #     - name: Configure git
+  #       run: |
+  #         git config --local user.email "info@blacklanternsecurity.com"
+  #         git config --local user.name "GitHub Actions"
+  #     - name: Tag commit
+  #       run: |
+  #         VERSION="${{ needs.publish_code.outputs.BBOT_VERSION }}"
+  #         if [[ "${{ github.ref }}" == "refs/heads/dev" ]]; then
+  #           TAG_MESSAGE="Dev Release $VERSION"
+  #         elif [[ "${{ github.ref }}" == "refs/heads/stable" ]]; then
+  #           TAG_MESSAGE="Stable Release $VERSION"
+  #         fi
+  #         git tag -a $VERSION -m "$TAG_MESSAGE"
+  #         git push origin --tags