From 13764b2c0d05c2e5281881487b0e5932e969139f Mon Sep 17 00:00:00 2001 From: Dave O'Connor Date: Mon, 13 Jan 2025 14:25:17 -0800 Subject: [PATCH 1/8] Added nix for developer setup (#1379) --- .envrc | 8 ++ .gitignore | 2 +- docs/nix_based_setup.md | 140 ++++++++++++++++++++++++++++++++ flake.lock | 61 ++++++++++++++ flake.nix | 96 ++++++++++++++++++++++ justfile | 46 +++++++++-- scripts/load_production_data.sh | 5 +- 7 files changed, 348 insertions(+), 10 deletions(-) create mode 100644 .envrc create mode 100644 docs/nix_based_setup.md create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..37ef80f76 --- /dev/null +++ b/.envrc @@ -0,0 +1,8 @@ +# we have checks to account for first time setup +if [ -f ".env" ]; then + dotenv +fi +if [ -f "$HOME/.config/nix/nix.conf" ]; then + echo "Entering nix shell environment." + use flake +fi diff --git a/.gitignore b/.gitignore index ff1a33c01..7940db741 100644 --- a/.gitignore +++ b/.gitignore @@ -147,7 +147,7 @@ media python.log # direnv -.envrc +.direnv .local_env .DS_Store diff --git a/docs/nix_based_setup.md b/docs/nix_based_setup.md new file mode 100644 index 000000000..c141a8d63 --- /dev/null +++ b/docs/nix_based_setup.md @@ -0,0 +1,140 @@ +# Boost.org Website + +## Overview + +A Django based website that will power a new Boost website. See the [documentation](./docs/README.md) for more information about maintaining this project. + +Links: + +- https://www.stage.boost.cppalliance.org/ - staging +- https://www.boost.org/ - production + +--- + +## Local Development Setup + +This project uses Python 3.11, Docker, and Docker Compose. + +This document describes how to set up a development environment using Nix, which is a package manager that allows for reproducible builds and development environments, like a better encapsulated declarative cross-platform Homebrew. + +For a basic rundown on Nix, this video could be useful https://www.youtube.com/watch?v=yQwW8dkuHqw + +1. Install the following according to the instructions for your platform if not already installed: + 1. Direnv - https://direnv.net/docs/installation.html (don't install OS packaged version, must be >= 2.35.0) and then configure your shell to add the hook as per the direnv docs. + 2. Docker Engine + * Linux - https://docs.docker.com/engine/install/ + * MacOS - https://orbstack.dev/ or https://github.com/abiosoft/colima ? + * Windows - ? + 3. Just - https://just.systems/man/en/packages.html + 4. Nix - https://nixos.org/download/ (multi-user, y to all options) +2. Clone this https://github.com/boostorg/website-v2.git repository to your machine. +3. cd into the repository directory. +4. In a terminal run `just bootstrap` in the root of the checked out repository to install the necessary development dependencies and generate the .env file. + * This will ask you to log in to your google account to allow permissions for the production database load later. Ask Sam for permissions on the database backup drive with an email address associated with Google. +5. Update the generated .env file with the necessary environment variables. Where you can't retrieve these yourself, you can ask someone for some in #boost-website on the slack server at https://ccplang.slack.com. The minimum that must be set is: + * GITHUB_TOKEN - a personal access token for the GitHub API, from your profile + * STATIC_CONTENT_AWS_ACCESS_KEY_ID - ask for this + * STATIC_CONTENT_AWS_SECRET_ACCESS_KEY - ask for this +5. Run `just setup` to build services, and build the JS and CSS assets. If docker fails with permissions errors, reboot your machine. +6. Run `just load_production_data` to download live data from the backup server. +7. Run `docker compose up` to start the server. + +### Social Login with django-allauth + +Follow these instructions to use the social logins through django-allauth on your local machine. + +See https://testdriven.io/blog/django-social-auth/ for more information. + +#### Github +- Go to https://github.com/settings/applications/new and add a new OAuth application +- Set `http://localhost:8000` as the Homepage URL +- Set `http://localhost:8000/accounts/github/login/callback/` as the Callback URL +- Click whether you want to enable the device flow + - The GitHub screen that registers a new OAuth app +- On completion copy the Client ID and Client Secret to the `.env` file as values of `GITHUB_OAUTH_CLIENT_ID` and `GITHUB_OAUTH_CLIENT_SECRET`. +- Run `direnv allow` and restart your docker containers. + +Setup should be complete and you should be able to see an option to "Use Github" on the sign up page. + +To test the flow including authorizing Github for the Boost account, log into your GitHub account settings and click **Applications** in the left menu. Find the "Boost" authorization and delete it. The next time you log into Boost with this GitHub account, you will have to re-authorize it. + +The 'Authorized OAuth Apps' tab in your GitHub Applications + +This setup process is not something that can currently be automated through terraform because of a lack of relevant Github API endpoints to create Oauth credentials. + +#### Google + +More detailed instructions at: + +https://docs.allauth.org/en/latest/socialaccount/providers/google.html + +1. Update the `.env` file with values for: + 1. `TF_VAR_google_cloud_email` (the email address of your Google Cloud account) + 2. `TF_VAR_google_organization_domain` (usually the domain of your Google Cloud account, e.g. "boost.org" if you will be using a @boost.org email address) + 3. `TF_VAR_google_cloud_project_name` (optional, default: localboostdev) - needs to change if destroyed and a setup is needed within 30 days +2. Run `just development-tofu-init` to initialize tofu. +3. Run `just development-tofu-plan` to confirm the planned changes. +4. Run `just development-tofu-apply` to apply the changes. +5. Go to https://console.developers.google.com/ + 1. Search for the newly created project, named "Boost Development" (ID: localboostdev by default). + 2. Type "credentials" in the search input at the top of the page. + 3. Select "Credentials" under "APIs & Services". + 1. Click "+ CREATE CREDENTIALS" + 2. Select "OAuth Client ID" + 3. Select Application Type: "Web application" + 4. Name: "Boost Development" (arbitrary) + 5. For "Authorized Javascript Origins" use:`http://localhost:8000` + 6. For "Authorized Redirect URIs" use: + * `http://localhost:8000/accounts/google/login/callback/` + * `http://localhost:8000/accounts/google/login/callback/?flowName=GeneralOAuthFlow` + 7. Save +6. From the page that's displayed, update the `.env` file with values for the following: + - `GOOGLE_OAUTH_CLIENT_ID` should be similar to "k235bn2b1l1(...)asdsk.apps.googleusercontent.com" + - `GOOGLE_OAUTH_CLIENT_SECRET` should be similar to "LAJACO(...)KLAI612ANAD" +7. Run `docker compose down && docker compose up` and restart your docker containers. + +Point 5 above can not be automated through terraform because of a lack of relevant Google Cloud API endpoints to create Oauth credentials. + +Setup should be complete and you should be able to see an option to "Use Google" on the sign up page. + +#### Additional Notes on allauth login flows: +**Working locally**: If you need to run through the login flows multiple times, create a superuser so you can log into the admin. Then, log into the admin and delete your "Social Account" from the admin. This will test a fresh connection to GitHub for your logged-in GitHub user. + +## Syncing EmailData Locally (optional) + +To work with mailinglist data locally, the django application expects to be +able to query a copy of the hyperkitty database from HYPERKITTY_DATABASE_NAME. +Then, `just manage sync_mailinglist_stats` management command can be run. + +## Debugging +For local development there is Django Debug Toolbar, and the option to set a debugger. + +In your env: +- Django Debug Toolbar, enabled by default, can be disabled by setting DEBUG_TOOLBAR=False +- IDE Debugging, disabled by default, can be enabled by uncommenting `PYTHONBREAKPOINT` in your .env file. + +### Set Up Pycharm +You can set up your IDE with a new "Python Debug Server" configuration as: + +PyCharm Debugger Settings + +### Debugger Usage +To use the debugger add `breakpoint()` on a line in the code before you want to start debugging and then add breakpoints by clicking on the gutter. The debugger will stop at these point, you can then step/inspect the variables. + + +## Troubleshooting + +### Docker +Keep in mind if there are issues with docker that the host docker daemon on your machine and the docker daemon in the nix setup may not match. It's a good idea to keep both up to date. + +### Direnv +when you switch to the directory you may see direnv exporting a bunch of environment variables as below. + +The installer configures direnv to suppress those but it's a recent configuration option, so may be worth checking for an update if you see them. + +## Disk space +Should you find you're running short on disk space, to delete previous versioned store data you can run `nix-collect-garbage -d`. Reentering the directory will then reinstall all the current dependencies again. It's probably a good idea to run that periodically. + +```shell +direnv: export +ALLOWED_HOSTS +AR +AS... +``` diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..e13c6442b --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1736755442, + "narHash": "sha256-a3MMEY7i/wdF0gb7WFNTn6onzaiMOvwj7OerRVenA8o=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ef56e777fedaa4da8c66a150081523c5de1e0171", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..0c7cbdd85 --- /dev/null +++ b/flake.nix @@ -0,0 +1,96 @@ +{ + description = "Boost.org development environment."; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils, ... }@inputs: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + }; + # https://nixos.wiki/wiki/Google_Cloud_SDK + gdk = pkgs.google-cloud-sdk.withExtraComponents( with pkgs.google-cloud-sdk.components; [ + gke-gcloud-auth-plugin + ]); + # Install a Ruby gem from rubygems.org + asciidoctorBoostGem = pkgs.stdenv.mkDerivation rec { + pname = "asciidoctor-boost"; + version = "0.1.7"; + sha = "ce139448812a9848219ce4cdb521c83c16009406a9d16efbc90bb24e94a46c24"; + + src = pkgs.fetchurl { + url = "https://rubygems.org/downloads/${pname}-${version}.gem"; + sha256 = "${sha}"; + }; + dontUnpack = true; + nativeBuildInputs = [ pkgs.ruby ]; + buildPhase = "true"; # Nothing to compile. + installPhase = '' + # Create a temporary gem directory + mkdir -p $out + # Set GEM_HOME to install gems locally under $out. + export GEM_HOME=$out + # Install the gem into GEM_HOME. + ${pkgs.ruby}/bin/gem install ${src} --no-document --ignore-dependencies + ''; + meta = { + description = "Asciidoctor Boost Ruby Gem installed from rubygems.org"; + homepage = "https://rubygems.org/gems/asciidoctor-boost"; + license = "BSL-1.0"; + }; + }; + + in { + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + # general system + # e.g. this could contain docker client if we wanted that to be consistent, + # though we need the daemon on the host anyway so it's redundant + # general project + awscli + gdk + just + opentofu + # frontend + nodejs_22 # matches Dockerfile, due for upgrade? + yarn + # backend + asciidoctor + asciidoctorBoostGem + pre-commit + python311 # matches Dockerfile, due for upgrade? + python311.pkgs.black + python311.pkgs.isort + python311.pkgs.pip-tools + ]; + # Host system installation workflow goes into the bootstrap justfile target. + # Project specific installation and execution workflow should go here. + shellHook = '' + if [ ! -f .git/hooks/pre-commit ]; then + pre-commit install + fi + if [ ! -d .venv ]; then + python3.11 -m venv .venv + . .venv/bin/activate + pip install -r requirements.txt -r requirements-dev.txt + else + . .venv/bin/activate + fi + if [ ! -f .env ]; then + cp env.template .env + echo ".env created, you should update its contents" + fi + # google cloud login + gcloud auth list --format="value(account)" | grep -q . || { + echo "Not logged in. Running gcloud auth login..." + gcloud auth login + } + ''; + }; + } + ); +} diff --git a/justfile b/justfile index 7b4156ffa..7cc033d22 100644 --- a/justfile +++ b/justfile @@ -12,13 +12,43 @@ ENV_FILE := ".env" # ---- @bootstrap: ## installs/updates all dependencies - #!/usr/bin/env bash - set -euo pipefail - if [ ! -f "{{ENV_FILE}}" ]; then - echo "{{ENV_FILE}} created" - cp env.template {{ENV_FILE}} + command -v direnv >/dev/null 2>&1 || { echo >&2 "Direnv is required but not installed. see: https://direnv.net/docs/installation.html - Aborting."; exit 1; } + command -v nix >/dev/null 2>&1 || { echo >&2 "Nix is required but not installed. see: https://nixos.org/download.html - Aborting."; exit 1; } + command -v just >/dev/null 2>&1 || { echo >&2 "Just is required but not installed. see: https://just.systems/man/en/packages.html - Aborting."; exit 1; } + command -v docker >/dev/null 2>&1 || { echo >&2 "Docker is required but not installed. see: docs for links - Aborting."; exit 1; } + + shell_name=$(basename "$SHELL") && \ + echo $shell_name && \ + if [ "$shell_name" = "zsh" ] && command -v zsh >/dev/null; then \ + zsh -i -c 'echo ${precmd_functions} | grep -q _direnv_hook' || { echo "❌ direnv hook is NOT installed in Zsh"; exit 1; }; \ + elif ([ "$shell_name" = "pwsh" ] || [ "$shell_name" = "powershell" ]) && command -v "$shell_name" >/dev/null; then \ + "$shell_name" -NoProfile -Command '$function:prompt.ToString() | grep -q direnv' || { echo "❌ direnv hook is NOT installed in PowerShell"; exit 1; }; \ + else \ + echo "ℹ️ Unsupported shell for checking direnv hook: $shell_name. Ensure you have the direnv shell hook eval set up correctly if there are problems."; \ fi - docker compose --file {{COMPOSE_FILE}} build --force-rm + + if [ ! -d $HOME/.config/direnv/direnv.toml ]; then \ + mkdir -p $HOME/.config/direnv; \ + printf "[global]\nhide_env_diff = true\nload_dotenv = true\n" > $HOME/.config/direnv/direnv.toml; \ + fi + if [ ! -d $HOME/.config/nix ]; then \ + mkdir -p $HOME/.config/nix; \ + printf "experimental-features = nix-command flakes\n" > $HOME/.config/nix/nix.conf; \ + fi + # check if the docker group exists, create if not + if [ ! $(getent group docker) ]; then \ + echo "ℹ️ Adding docker group..."; \ + sudo groupadd docker; \ + fi + + # check if user is in docker group, add if not + if [ $(id -Gn | grep -c docker) -eq 0 ]; then \ + echo "ℹ️ Adding docker group"; \ + sudo usermod -aG docker $USER; \ + echo "ℹ️ Added docker user. Please close the shell and open a new one."; \ + fi + echo "Bootstrapping complete, update your .env and run 'just setup'" + echo "If you have issues with docker permissions running just setup try restarting your machine." @rebuild: ## rebuilds containers docker compose kill @@ -31,7 +61,7 @@ ENV_FILE := ".env" @build: ## builds containers docker compose pull - DOCKER_BUILDKIT=1 docker compose build + docker compose build @cibuild: ## invoked by continuous integration servers to run tests python -m pytest @@ -49,6 +79,8 @@ alias shell := console @setup: ## sets up a project to be used for the first time docker compose --file {{COMPOSE_FILE}} build --force-rm docker compose --file docker-compose.yml run --rm web python manage.py migrate --noinput + npm install + npm run build @test_pytest *args: ## runs pytest (optional: test file/pattern, -v for verbose, -vv for very verbose) -docker compose run --rm -e DEBUG_TOOLBAR="False" web pytest -s --create-db {{ args }} diff --git a/scripts/load_production_data.sh b/scripts/load_production_data.sh index d9a4d7250..f0a506466 100755 --- a/scripts/load_production_data.sh +++ b/scripts/load_production_data.sh @@ -49,11 +49,12 @@ source .env download_media_file() { # download all files from the PROD_MEDIA_CONTENT bucket and copy to Docker container - [ -z "$PROD_MEDIA_CONTENT_AWS_ACCESS_KEY_ID" ] && { + # todo: remove the changeme check and remove 'changeme' as the default, use nothing instead + [[ -z "$PROD_MEDIA_CONTENT_AWS_ACCESS_KEY_ID" || "$PROD_MEDIA_CONTENT_AWS_ACCESS_KEY_ID" == "changeme" ]] && { echo "Error: PROD_MEDIA_CONTENT_AWS_ACCESS_KEY_ID not set in .env"; return 1; } - [ -z "$PROD_MEDIA_CONTENT_AWS_SECRET_ACCESS_KEY" ] && { + [[ -z "$PROD_MEDIA_CONTENT_AWS_SECRET_ACCESS_KEY" || "$PROD_MEDIA_CONTENT_AWS_SECRET_ACCESS_KEY" = "changeme" ]] && { echo "Error: PROD_MEDIA_CONTENT_AWS_SECRET_ACCESS_KEY not set in .env"; return 1; } From 567ab85430445d92fbaabe1cf8190e48ca37ea58 Mon Sep 17 00:00:00 2001 From: Dave O'Connor Date: Thu, 25 Sep 2025 14:45:13 -0700 Subject: [PATCH 2/8] Docs updates, separates nix and non-nix install processes, edits to simplify for both. Update to add index for coherency for devs and reduce some redundancy. (#1379) --- .envrc | 2 +- README.md | 114 ++++++-------- docs/README.md | 2 +- docs/allauth_setup.md | 59 ++++++++ ...s.md => development_setup_notes_native.md} | 76 ++-------- docs/development_setup_notes_nix.md | 73 +++++++++ docs/first_time_data_import.md | 7 + docs/nix_based_setup.md | 140 ------------------ docs/non-dev-server-allauth-setup.md | 2 +- justfile | 2 +- 10 files changed, 206 insertions(+), 271 deletions(-) create mode 100644 docs/allauth_setup.md rename docs/{development_setup_notes.md => development_setup_notes_native.md} (69%) create mode 100644 docs/development_setup_notes_nix.md delete mode 100644 docs/nix_based_setup.md diff --git a/.envrc b/.envrc index 37ef80f76..0523577b3 100644 --- a/.envrc +++ b/.envrc @@ -3,6 +3,6 @@ if [ -f ".env" ]; then dotenv fi if [ -f "$HOME/.config/nix/nix.conf" ]; then - echo "Entering nix shell environment." + echo "Loading nix shell environment." use flake fi diff --git a/README.md b/README.md index 4fe6feb79..11870d7c3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Django based website that will power a new Boost website. See the [documentati Links: -- https://www.stage.boost.cppalliance.org/ - staging +- https://stage.boost.org/ - staging - https://www.boost.org/ - production --- @@ -15,54 +15,65 @@ Links: This project will use Python 3.11, Docker, and Docker Compose. -Instructions to install those packages are included in [development_setup_notes.md](docs/development_setup_notes.md). - -**NOTE**: All of these various `docker compose` commands, along with other helpful -developer utility commands, are codified in our `justfile` and can be ran with -less typing. - -You will need to install `just`, by [following the documentation](https://just.systems/man/en/) - -**Environment Variables**: Copy file `env.template` to `.env` and adjust values to match your local environment. See [Environment Variables](docs/env_vars.md) for more information. - -```shell -$ cp env.template .env -``` - -**NOTE**: Double check that the exposed port assigned to the PostgreSQL -container does not clash with a database or other server you have running -locally. - -Then run: +There are two options for development setups, nix and native. The native setup doesn't require nix to be installed. The nix-based setup works in a similar way to a python venv, with everything encapsulated for the project. Note: it is not a vm. It handles installation of all of the application and development environment's dependencies automatically: + +- The python version relevant to the project +- awscli +- gdk +- just +- opentofu +- nodejs +- yarn +- asciidoctor +- asciidoctor-boost gem +- pre-commit +- black +- isort +- pip-tools + + +1. Development System setup + 1. Basic Setup Options: + 1. [Nix-based](docs/development_setup_notes_nix.md) + 1. [Native](docs/development_setup_notes_native.md) + 1. [Initial Data Setup](docs/first_time_data_import.md) + 1. [Allauth Social Login Setup](docs/allauth_setup.md) +1. System Concepts + 1. [GitHub Data Syncing](docs/syncing_data_with_github.md) + 1. [Rendered Content Caching](docs/caching_rendered_content.md) + 1. [Static Content from S3](docs/static_content.md) + 1. ['manage' Commands](docs/commands.md) + 1. [API](docs/api.md) + 1. [RSS Feeds](docs/rss_feeds.md) + 1. [Env Vars](docs/env_vars.md) + 1. [Calendar](docs/calendar.md) + 1. [News](docs/news.md) +1. Instructional + 1. [Dependency Updates](docs/dependencies.md) + 1. [Release Reports](docs/release_reports.md) + 1. [User Management](docs/user_management.md) + 1. [Non-Dev Server Allauth Setup](docs/non-dev-server-allauth-setup.md) + 1. [Admin Features](docs/admin.md) + 1. [Mailing List Setup](docs/mailing_list.md) + +After going through the "Development System setup" steps above to create the Docker image, install dependencies, and start the services in `docker-compose.yml`, run: ```shell # start our services (and build them if necessary) $ docker compose up # to create database migrations -$ docker compose run --rm web python manage.py makemigrations +$ just makemigrations # to run database migrations -$ docker compose run --rm web python manage.py migrate +$ just migrate # to create a superuser -$ docker compose run --rm web python manage.py createsuperuser +$ just manage createsuperuser ``` -This will create the Docker image, install dependencies, start the services -defined in `docker-compose.yml`, and start the webserver. - styles.css is still missing in a local docker-compose environment. Steps to add it: -``` -# One-time setup -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash -. ~/.bashrc -nvm install 20 -nvm use 20 -npm install -g yarn -``` - ``` # Each time - rebuild styles.css yarn @@ -90,13 +101,13 @@ $ docker compose down To run the tests, execute: ```shell -$ docker compose run --rm web pytest +$ just test ``` or run: ```shell -$ just test +$ docker compose run --rm web pytest ``` ## Yarn and Tailwind @@ -125,22 +136,6 @@ currently generated by `docker compose build` and is included in the Docker imag --- -## Generating Local Data - -To **add real Boost libraries and sync all the data from GitHub and S3**, set appropriate values in your new .env file according to [Environment Variables](docs/env_vars.md) for `GITHUB_TOKEN`, `STATIC_CONTENT_AWS_ACCESS_KEY_ID`, `STATIC_CONTENT_AWS_SECRET_ACCESS_KEY`, `STATIC_CONTENT_BUCKET_NAME`, `STATIC_CONTENT_REGION`, `STATIC_CONTENT_AWS_S3_ENDPOINT_URL` then run: - -```bash -docker compose run --rm web python manage.py update_libraries --local -``` - -Those values can be gotten from another developer in the `#boost-website` Slack channel. - -The `--local` flag speeds up the command a lot by excluding the retrieval of data you generally don't need. For more information, see `update_libraries` in [Management Commands](docs/commands.md). - -Then as a superuser log into the admin interface, go to "Versions" and click on the "import new releases" button in the top right. - ---- - ## Setting up Mailman/Hyperkitty locally ``shell sudo apt-get install sassc @@ -168,7 +163,7 @@ psql -U postgres -W hyperkitty_db < /lists_stage_web.staging-db1-2.2025-02-06-08 To work with mailinglist data locally, the django application expects to be able to query a copy of the hyperkitty database from HYPERKITTY_DATABASE_NAME. -Then, the `sync_mailinglist_stats` management command can be run. +Then, `just manage sync_mailinglist_stats` management command can be run. ## Deploying @@ -190,17 +185,6 @@ We use [pre-commit hooks](https://pre-commit.com/) to check code for style, synt | [Ruff](https://github.com/charliermarsh/ruff) | Wrapper around `flake8` and `isort`, among other linters | | [Djhtml](https://github.com/rtts/djhtml) | Auto-formats Django templates | -### Setup and Usage - -| Description | Command | -| ---- | ------- | -| 1. Install the `pre-commit` package using `pip` | `pip install pre-commit` | -| 2. Install our list of pre-commit hooks locally | `pre-commit install` | -| 3. Run all hooks for changed files before commit | `pre-commit run` | -| 4. Run specific hook before commit | `pre-commit run {hook}` | -| 5. Run hooks for all files, even unchanged ones | `pre-commit run --all-files` | -| 6. Commit without running pre-commit hooks | `git commit -m "Your commit message" --no-verify` | - Example commands for running specific hooks: | Hook | Example | diff --git a/docs/README.md b/docs/README.md index 600ca72db..48485ef11 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,7 +4,7 @@ - [Boost Mailing List](./mailing_list.md) -Includes OAuth instructions - [Caching and the `RenderedContent` model](./caching_rendered_content.md) - [Dependency Management](./dependencies.md) -- [Development Setup Notes](./development_setup_notes.md) +- [Development Setup Notes](development_setup_notes_native.md) - [Environment Variables](./env_vars.md) - [Events Calendar](./calendar.md) - [Example Files](./examples/README.md) - Contains samples of `libraries.json`. `.gitmodules`, and other files that Boost data depends on diff --git a/docs/allauth_setup.md b/docs/allauth_setup.md new file mode 100644 index 000000000..ca4e847de --- /dev/null +++ b/docs/allauth_setup.md @@ -0,0 +1,59 @@ +### Social Login with django-allauth + +Follow these instructions to use the social logins through django-allauth on your local machine. + +See https://testdriven.io/blog/django-social-auth/ for more information. + +#### Github +- Go to https://github.com/settings/applications/new and add a new OAuth application +- Set `http://localhost:8000` as the Homepage URL +- Set `http://localhost:8000/accounts/github/login/callback/` as the Callback URL +- Click whether you want to enable the device flow + - The GitHub screen that registers a new OAuth app +- On completion copy the Client ID and Client Secret to the `.env` file as values of `GITHUB_OAUTH_CLIENT_ID` and `GITHUB_OAUTH_CLIENT_SECRET`. +- Run `direnv allow` and restart your docker containers. + +Setup should be complete and you should be able to see an option to "Use Github" on the sign up page. + +To test the flow including authorizing Github for the Boost account, log into your GitHub account settings and click **Applications** in the left menu. Find the "Boost" authorization and delete it. The next time you log into Boost with this GitHub account, you will have to re-authorize it. + +The 'Authorized OAuth Apps' tab in your GitHub Applications + +This setup process is not something that can currently be automated through terraform because of a lack of relevant Github API endpoints to create Oauth credentials. + +#### Google + +More detailed instructions at: + +https://docs.allauth.org/en/latest/socialaccount/providers/google.html + +1. Update the `.env` file with values for: + 1. `TF_VAR_google_cloud_email` (the email address of your Google Cloud account) + 2. `TF_VAR_google_organization_domain` (usually the domain of your Google Cloud account, e.g. "boost.org" if you will be using a @boost.org email address) + 3. `TF_VAR_google_cloud_project_name` (optional, default: localboostdev) - needs to change if destroyed and a setup is needed within 30 days +2. Run `just development-tofu-init` to initialize tofu. +3. Run `just development-tofu-plan` to confirm the planned changes. +4. Run `just development-tofu-apply` to apply the changes. +5. Go to https://console.developers.google.com/ + 1. Search for the newly created project, named "Boost Development" (ID: localboostdev by default). + 2. Type "credentials" in the search input at the top of the page. + 3. Select "Credentials" under "APIs & Services". + 1. Click "+ CREATE CREDENTIALS" + 2. Select "OAuth Client ID" + 3. Select Application Type: "Web application" + 4. Name: "Boost Development" (arbitrary) + 5. For "Authorized Javascript Origins" use:`http://localhost:8000` + 6. For "Authorized Redirect URIs" use: + * `http://localhost:8000/accounts/google/login/callback/` + * `http://localhost:8000/accounts/google/login/callback/?flowName=GeneralOAuthFlow` + 7. Save +6. From the page that's displayed, update the `.env` file with values for the following: + - `GOOGLE_OAUTH_CLIENT_ID` should be similar to "k235bn2b1l1(...)asdsk.apps.googleusercontent.com" + - `GOOGLE_OAUTH_CLIENT_SECRET` should be similar to "LAJACO(...)KLAI612ANAD" + +Point 5 above can not be automated through terraform because of a lack of relevant Google Cloud API endpoints to create Oauth credentials. + +Setup should be complete and you should be able to see an option to "Use Google" on the sign up page. + +#### Additional Notes on allauth login flows: +**Working locally**: If you need to run through the login flows multiple times, create a superuser so you can log into the admin. Then, log into the admin and delete your "Social Account" from the admin. This will test a fresh connection to GitHub for your logged-in GitHub user. diff --git a/docs/development_setup_notes.md b/docs/development_setup_notes_native.md similarity index 69% rename from docs/development_setup_notes.md rename to docs/development_setup_notes_native.md index ecc9884c0..e0fd4ad32 100644 --- a/docs/development_setup_notes.md +++ b/docs/development_setup_notes_native.md @@ -271,72 +271,24 @@ cd website-v2 cp env.template .env ``` -Edit the .env, adding AWS keys. - -Continue to the instructions in the top-level README.md file. - -## Local Development - -### Social Login with django-allauth - -Follow these instructions to use the social logins through django-allauth on your local machine. - -See https://testdriven.io/blog/django-social-auth/ for more information. - -#### Github -- Go to https://github.com/settings/applications/new and add a new OAuth application -- Set `http://localhost:8000` as the Homepage URL -- Set `http://localhost:8000/accounts/github/login/callback/` as the Callback URL -- Click whether you want to enable the device flow - - The GitHub screen that registers a new OAuth app -- On completion copy the Client ID and Client Secret to the `.env` file as values of `GITHUB_OAUTH_CLIENT_ID` and `GITHUB_OAUTH_CLIENT_SECRET`. -- Run `direnv allow` and restart your docker containers. +Edit the .env, adding AWS keys, and adjust values to match your local environment. See [Environment Variables](docs/env_vars.md) for more information. -Setup should be complete and you should be able to see an option to "Use Github" on the sign up page. +**NOTE**: Double check that the exposed port assigned to the PostgreSQL +container does not clash with a database or other server you have running +locally. -To test the flow including authorizing Github for the Boost account, log into your GitHub account settings and click **Applications** in the left menu. Find the "Boost" authorization and delete it. The next time you log into Boost with this GitHub account, you will have to re-authorize it. +### Pre-commit Hooks Setup -The 'Authorized OAuth Apps' tab in your GitHub Applications +| Description | Command | +| ---- | ------- | +| 1. Install the `pre-commit` package using `pip` | `pip install pre-commit` | +| 2. Install our list of pre-commit hooks locally | `pre-commit install` | +| 3. Run all hooks for changed files before commit | `pre-commit run` | +| 4. Run specific hook before commit | `pre-commit run {hook}` | +| 5. Run hooks for all files, even unchanged ones | `pre-commit run --all-files` | +| 6. Commit without running pre-commit hooks | `git commit -m "Your commit message" --no-verify` | -This setup process is not something that can currently be automated through terraform because of a lack of relevant Github API endpoints to create Oauth credentials. - -#### Google - -More detailed instructions at: - -https://docs.allauth.org/en/latest/socialaccount/providers/google.html - -1. Update the `.env` file with values for: - 1. `TF_VAR_google_cloud_email` (the email address of your Google Cloud account) - 2. `TF_VAR_google_organization_domain` (usually the domain of your Google Cloud account, e.g. "boost.org" if you will be using an @boost.org email address) - 3. `TF_VAR_google_cloud_project_name` (optional, default: localboostdev) - needs to change if destroyed and a setup is needed within 30 days -2. Run `just development-tofu-init` to initialize tofu. -3. Run `just development-tofu-plan` to confirm the planned changes. -4. Run `just development-tofu-apply` to apply the changes. -5. Go to https://console.developers.google.com/ - 1. Search for the newly created project, named "Boost Development" (ID: localboostdev by default). - 2. Type "credentials" in the search input at the top of the page. - 3. Select "Credentials" under "APIs & Services". - 1. Click "+ CREATE CREDENTIALS" - 2. Select "OAuth Client ID" - 3. Select Application Type: "Web application" - 4. Name: "Boost Development" (arbitrary) - 5. For "Authorized Javascript Origins" use:`http://localhost:8000` - 6. For "Authorized Redirect URIs" use: - * `http://localhost:8000/accounts/google/login/callback/` - * `http://localhost:8000/accounts/google/login/callback/?flowName=GeneralOAuthFlow` - 7. Save -6. From the page that's displayed, update the `.env` file with values for the following: - - `GOOGLE_OAUTH_CLIENT_ID` should be similar to "k235bn2b1l1(...)asdsk.apps.googleusercontent.com" - - `GOOGLE_OAUTH_CLIENT_SECRET` should be similar to "LAJACO(...)KLAI612ANAD" -7. Run `direnv allow` and restart your docker containers. - -Point 5 above can not be automated through terraform because of a lack of relevant Google Cloud API endpoints to create Oauth credentials. - -Setup should be complete and you should be able to see an option to "Use Google" on the sign up page. - -#### Additional Notes: -**Working locally**: If you need to run through the login flows multiple times, create a superuser so you can log into the admin. Then, log into the admin and delete your "Social Account" from the admin. This will test a fresh connection to GitHub for your logged-in GitHub user. +Continue to the instructions in the top-level README.md file. ### Debugging For local development there is Django Debug Toolbar, and the option to set a debugger. diff --git a/docs/development_setup_notes_nix.md b/docs/development_setup_notes_nix.md new file mode 100644 index 000000000..8b733fbcd --- /dev/null +++ b/docs/development_setup_notes_nix.md @@ -0,0 +1,73 @@ +# Boost.org Website + +## Overview + +A Django based website that will power a new Boost website. See the [documentation](./docs/README.md) for more information about maintaining this project. + +Links: + +- https://www.stage.boost.cppalliance.org/ - staging +- https://www.boost.org/ - production + +--- + +## Local Development Setup + +This project uses Python 3.11, Docker, and Docker Compose. + +This document describes how to set up a development environment using Nix, which is a package manager that allows for reproducible builds and development environments, like a better encapsulated declarative cross-platform Homebrew. + +For a basic rundown on Nix, this video could be useful https://www.youtube.com/watch?v=yQwW8dkuHqw + +1. Install the following according to the instructions for your platform if not already installed: + 1. Direnv - https://direnv.net/docs/installation.html (don't install OS packaged version, must be >= 2.35.0) and then configure your shell to add the hook as per the direnv docs. + 2. Docker Engine + * Linux - https://docs.docker.com/engine/install/ + * MacOS - https://orbstack.dev/ or https://github.com/abiosoft/colima ? + * Windows - ? + 3. Just - https://just.systems/man/en/packages.html + 4. Nix - https://nixos.org/download/ (multi-user, y to all options) +2. Clone this https://github.com/boostorg/website-v2.git repository to your machine. +3. cd into the repository directory. +4. In a terminal run `just bootstrap-nix` in the root of the checked out repository to install the necessary development dependencies and generate the .env file. + * This will ask you to log in to your google account to allow permissions for the production database load later. Ask Sam for permissions on the database backup drive with an email address associated with Google. +5. Update the generated .env file with the necessary environment variables. Where you can't retrieve these yourself, you can ask someone for some in #boost-website on the slack server at https://ccplang.slack.com. The minimum that must be set is: + * GITHUB_TOKEN - a personal access token for the GitHub API, from your profile + * STATIC_CONTENT_AWS_ACCESS_KEY_ID - ask for this + * STATIC_CONTENT_AWS_SECRET_ACCESS_KEY - ask for this +5. Run `just setup` to build services, and build the JS and CSS assets. If docker fails with permissions errors, reboot your machine. +6. Run `just load_production_data` to download live data from the backup server. +7. Run `docker compose up` to start the server. + + +## Debugging +For local development there is Django Debug Toolbar, and the option to set a debugger. + +In your env: +- Django Debug Toolbar, enabled by default, can be disabled by setting DEBUG_TOOLBAR=False +- IDE Debugging, disabled by default, can be enabled by uncommenting `PYTHONBREAKPOINT` in your .env file. + +### Set Up Pycharm +You can set up your IDE with a new "Python Debug Server" configuration as: + +PyCharm Debugger Settings + +### Debugger Usage +To use the debugger add `breakpoint()` on a line in the code before you want to start debugging and then add breakpoints by clicking on the gutter. The debugger will stop at these point, you can then step/inspect the variables. + +## Troubleshooting + +### Docker +Keep in mind if there are issues with docker that the host docker daemon on your machine and the docker daemon in the nix setup may not match. It's a good idea to keep both up to date. + +### Direnv +when you switch to the directory you may see direnv exporting a bunch of environment variables as below. + +The installer configures direnv to suppress those but it's a recent configuration option, so may be worth checking for an update if you see them. + +## Disk space +Should you find you're running short on disk space, to delete previous versioned store data you can run `nix-collect-garbage -d`. Reentering the directory will then reinstall all the current dependencies again. It's probably a good idea to run that periodically. + +```shell +direnv: export +ALLOWED_HOSTS +AR +AS... +``` diff --git a/docs/first_time_data_import.md b/docs/first_time_data_import.md index f5cc8125d..d801b6b8e 100644 --- a/docs/first_time_data_import.md +++ b/docs/first_time_data_import.md @@ -1,10 +1,17 @@ # Populating the Database for the First Time +- [Development Environments](#development-environments) - [Deployed Environments](#deployed-environments) - [Further Reading](#further-reading) This document contains information about importing Boost Versions (also called Releases), Libraries, and the data associated with those objects. It is concerned with importing data in **deployed environments**, but at the bottom of the page there is a section on importing data for **local development**. +## Development Environments + +```bash +just load_production_data +```` + ## Deployed Environments There are several steps to populating the database with historical Boost data, because we retrieve Boost data from multiple sources. diff --git a/docs/nix_based_setup.md b/docs/nix_based_setup.md deleted file mode 100644 index c141a8d63..000000000 --- a/docs/nix_based_setup.md +++ /dev/null @@ -1,140 +0,0 @@ -# Boost.org Website - -## Overview - -A Django based website that will power a new Boost website. See the [documentation](./docs/README.md) for more information about maintaining this project. - -Links: - -- https://www.stage.boost.cppalliance.org/ - staging -- https://www.boost.org/ - production - ---- - -## Local Development Setup - -This project uses Python 3.11, Docker, and Docker Compose. - -This document describes how to set up a development environment using Nix, which is a package manager that allows for reproducible builds and development environments, like a better encapsulated declarative cross-platform Homebrew. - -For a basic rundown on Nix, this video could be useful https://www.youtube.com/watch?v=yQwW8dkuHqw - -1. Install the following according to the instructions for your platform if not already installed: - 1. Direnv - https://direnv.net/docs/installation.html (don't install OS packaged version, must be >= 2.35.0) and then configure your shell to add the hook as per the direnv docs. - 2. Docker Engine - * Linux - https://docs.docker.com/engine/install/ - * MacOS - https://orbstack.dev/ or https://github.com/abiosoft/colima ? - * Windows - ? - 3. Just - https://just.systems/man/en/packages.html - 4. Nix - https://nixos.org/download/ (multi-user, y to all options) -2. Clone this https://github.com/boostorg/website-v2.git repository to your machine. -3. cd into the repository directory. -4. In a terminal run `just bootstrap` in the root of the checked out repository to install the necessary development dependencies and generate the .env file. - * This will ask you to log in to your google account to allow permissions for the production database load later. Ask Sam for permissions on the database backup drive with an email address associated with Google. -5. Update the generated .env file with the necessary environment variables. Where you can't retrieve these yourself, you can ask someone for some in #boost-website on the slack server at https://ccplang.slack.com. The minimum that must be set is: - * GITHUB_TOKEN - a personal access token for the GitHub API, from your profile - * STATIC_CONTENT_AWS_ACCESS_KEY_ID - ask for this - * STATIC_CONTENT_AWS_SECRET_ACCESS_KEY - ask for this -5. Run `just setup` to build services, and build the JS and CSS assets. If docker fails with permissions errors, reboot your machine. -6. Run `just load_production_data` to download live data from the backup server. -7. Run `docker compose up` to start the server. - -### Social Login with django-allauth - -Follow these instructions to use the social logins through django-allauth on your local machine. - -See https://testdriven.io/blog/django-social-auth/ for more information. - -#### Github -- Go to https://github.com/settings/applications/new and add a new OAuth application -- Set `http://localhost:8000` as the Homepage URL -- Set `http://localhost:8000/accounts/github/login/callback/` as the Callback URL -- Click whether you want to enable the device flow - - The GitHub screen that registers a new OAuth app -- On completion copy the Client ID and Client Secret to the `.env` file as values of `GITHUB_OAUTH_CLIENT_ID` and `GITHUB_OAUTH_CLIENT_SECRET`. -- Run `direnv allow` and restart your docker containers. - -Setup should be complete and you should be able to see an option to "Use Github" on the sign up page. - -To test the flow including authorizing Github for the Boost account, log into your GitHub account settings and click **Applications** in the left menu. Find the "Boost" authorization and delete it. The next time you log into Boost with this GitHub account, you will have to re-authorize it. - -The 'Authorized OAuth Apps' tab in your GitHub Applications - -This setup process is not something that can currently be automated through terraform because of a lack of relevant Github API endpoints to create Oauth credentials. - -#### Google - -More detailed instructions at: - -https://docs.allauth.org/en/latest/socialaccount/providers/google.html - -1. Update the `.env` file with values for: - 1. `TF_VAR_google_cloud_email` (the email address of your Google Cloud account) - 2. `TF_VAR_google_organization_domain` (usually the domain of your Google Cloud account, e.g. "boost.org" if you will be using a @boost.org email address) - 3. `TF_VAR_google_cloud_project_name` (optional, default: localboostdev) - needs to change if destroyed and a setup is needed within 30 days -2. Run `just development-tofu-init` to initialize tofu. -3. Run `just development-tofu-plan` to confirm the planned changes. -4. Run `just development-tofu-apply` to apply the changes. -5. Go to https://console.developers.google.com/ - 1. Search for the newly created project, named "Boost Development" (ID: localboostdev by default). - 2. Type "credentials" in the search input at the top of the page. - 3. Select "Credentials" under "APIs & Services". - 1. Click "+ CREATE CREDENTIALS" - 2. Select "OAuth Client ID" - 3. Select Application Type: "Web application" - 4. Name: "Boost Development" (arbitrary) - 5. For "Authorized Javascript Origins" use:`http://localhost:8000` - 6. For "Authorized Redirect URIs" use: - * `http://localhost:8000/accounts/google/login/callback/` - * `http://localhost:8000/accounts/google/login/callback/?flowName=GeneralOAuthFlow` - 7. Save -6. From the page that's displayed, update the `.env` file with values for the following: - - `GOOGLE_OAUTH_CLIENT_ID` should be similar to "k235bn2b1l1(...)asdsk.apps.googleusercontent.com" - - `GOOGLE_OAUTH_CLIENT_SECRET` should be similar to "LAJACO(...)KLAI612ANAD" -7. Run `docker compose down && docker compose up` and restart your docker containers. - -Point 5 above can not be automated through terraform because of a lack of relevant Google Cloud API endpoints to create Oauth credentials. - -Setup should be complete and you should be able to see an option to "Use Google" on the sign up page. - -#### Additional Notes on allauth login flows: -**Working locally**: If you need to run through the login flows multiple times, create a superuser so you can log into the admin. Then, log into the admin and delete your "Social Account" from the admin. This will test a fresh connection to GitHub for your logged-in GitHub user. - -## Syncing EmailData Locally (optional) - -To work with mailinglist data locally, the django application expects to be -able to query a copy of the hyperkitty database from HYPERKITTY_DATABASE_NAME. -Then, `just manage sync_mailinglist_stats` management command can be run. - -## Debugging -For local development there is Django Debug Toolbar, and the option to set a debugger. - -In your env: -- Django Debug Toolbar, enabled by default, can be disabled by setting DEBUG_TOOLBAR=False -- IDE Debugging, disabled by default, can be enabled by uncommenting `PYTHONBREAKPOINT` in your .env file. - -### Set Up Pycharm -You can set up your IDE with a new "Python Debug Server" configuration as: - -PyCharm Debugger Settings - -### Debugger Usage -To use the debugger add `breakpoint()` on a line in the code before you want to start debugging and then add breakpoints by clicking on the gutter. The debugger will stop at these point, you can then step/inspect the variables. - - -## Troubleshooting - -### Docker -Keep in mind if there are issues with docker that the host docker daemon on your machine and the docker daemon in the nix setup may not match. It's a good idea to keep both up to date. - -### Direnv -when you switch to the directory you may see direnv exporting a bunch of environment variables as below. - -The installer configures direnv to suppress those but it's a recent configuration option, so may be worth checking for an update if you see them. - -## Disk space -Should you find you're running short on disk space, to delete previous versioned store data you can run `nix-collect-garbage -d`. Reentering the directory will then reinstall all the current dependencies again. It's probably a good idea to run that periodically. - -```shell -direnv: export +ALLOWED_HOSTS +AR +AS... -``` diff --git a/docs/non-dev-server-allauth-setup.md b/docs/non-dev-server-allauth-setup.md index fbc6f1b65..0bc0abe22 100644 --- a/docs/non-dev-server-allauth-setup.md +++ b/docs/non-dev-server-allauth-setup.md @@ -1,6 +1,6 @@ # Production/Staging Server setup for allauth -For development see [development_setup_notes.md](development_setup_notes.md). +For development see [development_setup_notes_native.md](development_setup_notes_native.md). For this setup adjustments will need to be made to the values as applicable for each server and service. diff --git a/justfile b/justfile index 7cc033d22..8d1968d60 100644 --- a/justfile +++ b/justfile @@ -11,7 +11,7 @@ ENV_FILE := ".env" # - https://github.blog/2015-06-30-scripts-to-rule-them-all/ # ---- -@bootstrap: ## installs/updates all dependencies +@bootstrap-nix: ## installs/updates all dependencies command -v direnv >/dev/null 2>&1 || { echo >&2 "Direnv is required but not installed. see: https://direnv.net/docs/installation.html - Aborting."; exit 1; } command -v nix >/dev/null 2>&1 || { echo >&2 "Nix is required but not installed. see: https://nixos.org/download.html - Aborting."; exit 1; } command -v just >/dev/null 2>&1 || { echo >&2 "Just is required but not installed. see: https://just.systems/man/en/packages.html - Aborting."; exit 1; } From 7f2303603e5482b625249df019732bd30ccfebda Mon Sep 17 00:00:00 2001 From: daveoconnor Date: Wed, 22 Oct 2025 13:24:26 -0700 Subject: [PATCH 3/8] Upgrade django to 5.2, python to 3.13 (#1915) --- .github/workflows/actions-gcp.yaml | 10 +- .github/workflows/actions.yml | 10 +- .pre-commit-config.yaml | 7 +- README.md | 2 +- core/calendar.py | 6 +- core/migrations/0003_sitesettings_and_more.py | 2 +- core/models.py | 2 +- core/views.py | 58 +++++++ docker/Dockerfile | 4 +- docs/dependencies.md | 11 ++ justfile | 5 + marketing/views.py | 8 +- news/feeds.py | 6 +- news/notifications.py | 2 +- news/tests/test_feeds.py | 10 +- pyproject.toml | 2 +- requirements-dev.in | 2 +- requirements-dev.txt | 6 +- requirements.in | 4 +- requirements.txt | 150 +++++++++--------- slack/migrations/0001_initial.py | 4 +- slack/models.py | 4 +- users/admin.py | 4 +- users/views.py | 2 +- versions/feeds.py | 6 +- versions/tests/test_feeds.py | 10 +- 26 files changed, 207 insertions(+), 130 deletions(-) diff --git a/.github/workflows/actions-gcp.yaml b/.github/workflows/actions-gcp.yaml index e66ef9324..57fba5edb 100644 --- a/.github/workflows/actions-gcp.yaml +++ b/.github/workflows/actions-gcp.yaml @@ -25,7 +25,7 @@ jobs: services: postgres: - image: postgres:12 + image: postgres:16 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -44,10 +44,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.11 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.13 - uses: actions/cache@v4 with: @@ -133,10 +133,10 @@ jobs: run: | git fetch --depth=1 origin +refs/tags/*:refs/tags/* || true - - name: Set up Python 3.11 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.13 - name: Install Python dependencies run: | diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 3997f5815..607b8cb63 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -15,7 +15,7 @@ jobs: services: postgres: - image: postgres:12 + image: postgres:16 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -34,10 +34,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.11 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.13 - uses: actions/cache@v4 with: @@ -85,10 +85,10 @@ jobs: run: | git fetch --depth=1 origin +refs/tags/*:refs/tags/* || true - - name: Set up Python 3.11 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.13 - name: Install Python dependencies run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e0c0f116b..8ac7766dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,14 @@ default_language_version: - python: python3.11 + python: python3.13 exclude: .*migrations\/.*|static\/img\/.*|static\/animations\/.*|static\/js\/boost-gecko\/.*|kube\/boost\/templates\/.*\.yaml repos: + - repo: https://github.com/adamchainz/django-upgrade + rev: "1.27.0" + hooks: + - id: django-upgrade + args: [--target-version, "5.2"] # Replace with Django version - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: diff --git a/README.md b/README.md index 11870d7c3..728b7bb73 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Links: ## Local Development Setup -This project will use Python 3.11, Docker, and Docker Compose. +This project will use Python 3.13, Docker, and Docker Compose. There are two options for development setups, nix and native. The native setup doesn't require nix to be installed. The nix-based setup works in a similar way to a python venv, with everything encapsulated for the project. Note: it is not a vm. It handles installation of all of the application and development environment's dependencies automatically: diff --git a/core/calendar.py b/core/calendar.py index 80831e9eb..d51d64092 100644 --- a/core/calendar.py +++ b/core/calendar.py @@ -20,10 +20,8 @@ def get_calendar(min_time=None, single_events=True, order_by="startTime"): https://developers.google.com/calendar/api/v3/reference/events/list """ if not min_time: - min_time = ( - datetime.datetime.utcnow().isoformat() + "Z" - ) # 'Z' indicates UTC time - + # 'Z' indicates UTC time + min_time = datetime.datetime.now(datetime.timezone.utc).isoformat() + "Z" url = f"https://www.googleapis.com/calendar/v3/calendars/{settings.BOOST_CALENDAR}/events?key={settings.CALENDAR_API_KEY}&timeMin={min_time}&singleEvents={single_events}&orderBy={order_by}" headers = {"Accept": "application/json"} diff --git a/core/migrations/0003_sitesettings_and_more.py b/core/migrations/0003_sitesettings_and_more.py index ce31eb678..7d38c73a1 100644 --- a/core/migrations/0003_sitesettings_and_more.py +++ b/core/migrations/0003_sitesettings_and_more.py @@ -28,7 +28,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="sitesettings", constraint=models.CheckConstraint( - check=models.Q(("id", 1)), name="core_sitesettings_single_instance" + condition=models.Q(("id", 1)), name="core_sitesettings_single_instance" ), ), migrations.AlterModelOptions( diff --git a/core/models.py b/core/models.py index 63e0d47e5..e74c75951 100644 --- a/core/models.py +++ b/core/models.py @@ -72,7 +72,7 @@ class Meta: # check constraint to only allow id=1 to exist models.CheckConstraint( name="%(app_label)s_%(class)s_single_instance", - check=models.Q(id=1), + condition=models.Q(id=1), ), ] verbose_name_plural = "Site Settings" diff --git a/core/views.py b/core/views.py index 4631a7b7e..2d408c20f 100644 --- a/core/views.py +++ b/core/views.py @@ -1,5 +1,6 @@ import os +import requests from django.utils import timezone from urllib.parse import urljoin @@ -16,11 +17,14 @@ HttpResponse, HttpResponseNotFound, HttpResponseRedirect, + HttpRequest, ) from django.shortcuts import redirect from django.template.loader import render_to_string from django.urls import reverse +from django.utils.decorators import method_decorator from django.views import View +from django.views.decorators.cache import never_cache from django.views.generic import TemplateView from config.settings import ENABLE_DB_CACHE @@ -942,3 +946,57 @@ def get(self, request, requested_version): if requested_version == "release": new_path = "/libraries/" return HttpResponseRedirect(new_path) + + +@method_decorator(never_cache, name="dispatch") +class QRCodeView(View): + """Handles QR code urls, sending them to Plausible, then redirecting to the desired url. + + QR code urls are formatted /qrc//desired/path/to/content/, and will + result in a redirect to /desired/path/to/content/. + + E.g. https://www.boost.org/qrc/pv-01/library/latest/beast/ will send this full url to Plausible, + then redirect to https://www.boost.org/library/latest/beast/ + """ + + def get(self, request: HttpRequest, campaign_identifier: str, main_path: str = ""): + absolute_url = request.build_absolute_uri(request.path) + referrer = request.headers.get("referer", "") + user_agent = request.headers.get("user-agent", "") + + plausible_payload = { + "name": "pageview", + "domain": "qrc.boost.org", + "url": absolute_url, + "referrer": referrer, + } + + headers = {"Content-Type": "application/json", "User-Agent": user_agent} + + client_ip = request.headers.get("x-forwarded-for", "").split(",")[0].strip() + client_ip = client_ip or request.META.get("REMOTE_ADDR") + + if client_ip: + headers["X-Forwarded-For"] = client_ip + + try: + requests.post( + "https://plausible.io/api/event", + json=plausible_payload, + headers=headers, + timeout=2.0, + ) + except Exception as e: + # Don’t interrupt the redirect - just log it + logger.error(f"Plausible event post failed: {e}") + + # Now that we've sent the request url to plausible, we can redirect to the main_path + # Preserve the original querystring, if any. + # Example: /qrc/3/library/latest/algorithm/?x=1 -> /library/latest/algorithm/?x=1 + # `main_path` is everything after qrc// thanks to . + redirect_path = "/" + main_path if main_path else "/" + qs = request.META.get("QUERY_STRING") + if qs: + redirect_path = f"{redirect_path}?{qs}" + + return HttpResponseRedirect(redirect_path) diff --git a/docker/Dockerfile b/docker/Dockerfile index f1c6c4ced..d200df23d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # syntax = docker/dockerfile:experimental -FROM python:3.11-slim AS builder-py +FROM python:3.13-slim AS builder-py ARG LOCAL_DEVELOPMENT @@ -42,7 +42,7 @@ RUN yarn build # Final image. -FROM python:3.11-slim AS release +FROM python:3.13-slim AS release RUN apt update && apt install -y git libpq-dev ruby ruby-dev && rm -rf /var/lib/apt/lists/* diff --git a/docs/dependencies.md b/docs/dependencies.md index 20f77514e..d48c9bb23 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -7,3 +7,14 @@ 1. Run `just pip-compile`, which will add the dependency to `requirements.txt` 1. Run `just rebuild` to rebuild your Docker image to include the new dependencies 2. Run `just up` and continue with development + +## Upgrading dependencies + +To upgrade all dependencies to their latest versions, run: + +1. `just pip-compile-upgrade`. +2. Get the django version from requirements.txt and set the `DJANGO_VERSION` value in /justfile +3. Update the `--target-version` args value for django-upgrade in .pre-commit-config.yaml to match +3. In a venv with installed packages run `just run-django-upgrade` to upgrade python code. +4. `just build` to create new docker images. +5. Tear down docker containers and restart with the newly built images, then test. diff --git a/justfile b/justfile index 8d1968d60..a351e0750 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,7 @@ set dotenv-load := false COMPOSE_FILE := "docker-compose.yml" ENV_FILE := ".env" +DJANGO_VERSION := "5.2" @_default: just --list @@ -154,6 +155,10 @@ alias shell := console fi @cd development-tofu; direnv allow && tofu destroy +@run-django-upgrade: + [ -n "${VIRTUAL_ENV-}" ] || { echo "❌ Activate your venv first."; exit 1; } + -git ls-files -z -- '*.py' | xargs -0r django-upgrade --target {{DJANGO_VERSION}} + # Dependency management @pip-compile ARGS='': ## rebuilds our pip requirements docker compose run --rm web uv pip compile {{ ARGS }} ./requirements.in --no-strip-extras --output-file ./requirements.txt diff --git a/marketing/views.py b/marketing/views.py index f59aabb57..473184f1f 100644 --- a/marketing/views.py +++ b/marketing/views.py @@ -28,8 +28,8 @@ class PlausibleRedirectView(View): def get(self, request: HttpRequest, campaign_identifier: str, main_path: str = ""): absolute_url = request.build_absolute_uri(request.path) - referrer = request.META.get("HTTP_REFERER", "") - user_agent = request.META.get("HTTP_USER_AGENT", "") + referrer = request.headers.get("referer", "") + user_agent = request.headers.get("user-agent", "") plausible_payload = { "name": "pageview", @@ -40,7 +40,7 @@ def get(self, request: HttpRequest, campaign_identifier: str, main_path: str = " headers = {"Content-Type": "application/json", "User-Agent": user_agent} - client_ip = request.META.get("HTTP_X_FORWARDED_FOR", "").split(",")[0].strip() + client_ip = request.headers.get("x-forwarded-for", "").split(",")[0].strip() client_ip = client_ip or request.META.get("REMOTE_ADDR") if client_ip: @@ -85,7 +85,7 @@ def dispatch(self, request, *args, **kwargs): if original_referrer := self.request.session.get("original_referrer", ""): self.referrer = original_referrer else: - self.referrer = self.request.META.get("HTTP_REFERER", "") + self.referrer = self.request.headers.get("referer", "") return super().dispatch(request, *args, **kwargs) def get_template_names(self): diff --git a/news/feeds.py b/news/feeds.py index a234d128b..1dddf0c16 100644 --- a/news/feeds.py +++ b/news/feeds.py @@ -1,7 +1,7 @@ -from datetime import datetime +from datetime import datetime, timezone from django.contrib.syndication.views import Feed from django.utils.feedgenerator import Atom1Feed -from django.utils.timezone import make_aware, utc +from django.utils.timezone import make_aware from django.utils.html import urlize, linebreaks from .models import Entry @@ -22,7 +22,7 @@ def item_pubdate(self, item): publish_date = item.publish_at if publish_date: datetime_obj = datetime.combine(publish_date, datetime.min.time()) - aware_datetime_obj = make_aware(datetime_obj, timezone=utc) + aware_datetime_obj = make_aware(datetime_obj, timezone=timezone.utc) return aware_datetime_obj def item_description(self, item): diff --git a/news/notifications.py b/news/notifications.py index 90fad54a7..4b945d7b3 100644 --- a/news/notifications.py +++ b/news/notifications.py @@ -60,7 +60,7 @@ def generate_magic_approval_link(entry_slug: str, moderator_id: int): def send_email_news_needs_moderation(request, entry): recipient_list = [ u - for u in moderators().select_related("preferences").only("email") + for u in moderators().select_related("preferences").only("email", "preferences") if entry.tag in u.preferences.allow_notification_others_news_needs_moderation ] if not recipient_list: diff --git a/news/tests/test_feeds.py b/news/tests/test_feeds.py index 66c48f6c5..e1714d1e5 100644 --- a/news/tests/test_feeds.py +++ b/news/tests/test_feeds.py @@ -1,5 +1,5 @@ -from datetime import datetime, timedelta -from django.utils.timezone import make_aware, now, utc +from datetime import datetime, timedelta, timezone +from django.utils.timezone import make_aware, now from model_bakery import baker from ..feeds import RSSNewsFeed, AtomNewsFeed @@ -22,7 +22,8 @@ def test_item_pubdate(make_entry): feed = RSSNewsFeed() published_entry = make_entry(moderator=baker.make("users.User"), approved_at=now()) expected_datetime = make_aware( - datetime.combine(published_entry.publish_at, datetime.min.time()), timezone=utc + datetime.combine(published_entry.publish_at, datetime.min.time()), + timezone=timezone.utc, ) assert feed.item_pubdate(published_entry) == expected_datetime @@ -51,6 +52,7 @@ def test_item_pubdate_atom(make_entry): feed = AtomNewsFeed() published_entry = make_entry(moderator=baker.make("users.User"), approved_at=now()) expected_datetime = make_aware( - datetime.combine(published_entry.publish_at, datetime.min.time()), timezone=utc + datetime.combine(published_entry.publish_at, datetime.min.time()), + timezone=timezone.utc, ) assert feed.item_pubdate(published_entry) == expected_datetime diff --git a/pyproject.toml b/pyproject.toml index 136dc41ae..41fdea450 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ whitelist-regex = ["test_.*"] [tool.ruff] line-length = 88 -target-version = "py311" +target-version = "py313" [tool.black] line-length = 88 diff --git a/requirements-dev.in b/requirements-dev.in index 0082442ad..31ef89211 100644 --- a/requirements-dev.in +++ b/requirements-dev.in @@ -1,3 +1,3 @@ -c requirements.txt django-debug-toolbar -pydevd-pycharm==243.26053.29 # pinned to appropriate version for current pycharm +pydevd-pycharm==252.26830.99 # pinned to appropriate version for current pycharm diff --git a/requirements-dev.txt b/requirements-dev.txt index f1a54f7b0..6a027f896 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,16 +1,16 @@ # This file was autogenerated by uv via the following command: # uv pip compile ./requirements-dev.in --no-strip-extras --output-file ./requirements-dev.txt -asgiref==3.9.1 +asgiref==3.10.0 # via # -c ./requirements.txt # django -django==4.2.24 +django==5.2.7 # via # -c ./requirements.txt # django-debug-toolbar django-debug-toolbar==6.0.0 # via -r ./requirements-dev.in -pydevd-pycharm==243.26053.29 +pydevd-pycharm==252.26830.99 # via -r ./requirements-dev.in sqlparse==0.5.3 # via diff --git a/requirements.in b/requirements.in index eadb27685..7653c1ec2 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -Django>=4.0, <5.0 +Django>=5.0, <6.0 bumpversion django-admin-env-notice django-allauth @@ -12,7 +12,7 @@ django-health-check django-imagekit django-oauth-toolkit django-redis -django-rest-auth +django-upgrade django-widget-tweaks djangorestframework environs[django] diff --git a/requirements.txt b/requirements.txt index 2a7e14a0e..86770c138 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,17 +2,17 @@ # uv pip compile ./requirements.in --no-strip-extras --output-file ./requirements.txt aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.12.15 +aiohttp==3.13.1 # via algoliasearch aiosignal==1.4.0 # via aiohttp -algoliasearch==4.27.0 +algoliasearch==4.30.0 # via -r ./requirements.in amqp==5.3.1 # via kombu annotated-types==0.7.0 # via pydantic -anyio==4.10.0 +anyio==4.11.0 # via # httpx # openai @@ -22,7 +22,7 @@ argon2-cffi==25.1.0 # via minio argon2-cffi-bindings==25.1.0 # via argon2-cffi -asgiref==3.9.1 +asgiref==3.10.0 # via # django # django-allauth @@ -32,21 +32,21 @@ asttokens==3.0.0 # via stack-data async-timeout==5.0.1 # via algoliasearch -attrs==25.3.0 +attrs==25.4.0 # via # aiohttp # interrogate -beautifulsoup4==4.13.5 +beautifulsoup4==4.14.2 # via -r ./requirements.in -billiard==4.2.1 +billiard==4.2.2 # via celery -black==25.1.0 +black==25.9.0 # via -r ./requirements.in -boto3==1.40.24 +boto3==1.40.56 # via # -r ./requirements.in # django-bakery -botocore==1.40.24 +botocore==1.40.56 # via # boto3 # s3transfer @@ -56,14 +56,14 @@ bumpversion==0.6.0 # via -r ./requirements.in celery==5.5.3 # via -r ./requirements.in -certifi==2025.8.3 +certifi==2025.10.5 # via # elasticsearch # httpcore # httpx # minio # requests -cffi==1.17.1 +cffi==2.0.0 # via # argon2-cffi-bindings # cryptography @@ -71,9 +71,9 @@ cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via -r ./requirements.in -charset-normalizer==3.4.3 +charset-normalizer==3.4.4 # via requests -click==8.2.1 +click==8.3.0 # via # black # celery @@ -92,9 +92,9 @@ colorama==0.4.6 # via interrogate contourpy==1.3.3 # via matplotlib -coverage[toml]==7.10.6 +coverage[toml]==7.11.0 # via pytest-cov -cryptography==45.0.7 +cryptography==46.0.3 # via # -r ./requirements.in # jwcrypto @@ -107,11 +107,11 @@ distlib==0.4.0 # via virtualenv distro==1.9.0 # via openai -dj-database-url==2.2.0 +dj-database-url==3.0.1 # via environs dj-email-url==1.0.6 # via environs -django==4.2.24 +django==5.2.7 # via # -r ./requirements.in # dj-database-url @@ -126,13 +126,12 @@ django==4.2.24 # django-js-asset # django-oauth-toolkit # django-redis - # django-rest-auth # django-storages # djangorestframework # model-bakery django-admin-env-notice==1.0.1 # via -r ./requirements.in -django-allauth[socialaccount]==65.11.1 +django-allauth[socialaccount]==65.12.1 # via -r ./requirements.in django-anymail[mailgun]==13.1 # via -r ./requirements.in @@ -144,7 +143,7 @@ django-cache-url==3.4.5 # via environs django-click==2.4.1 # via -r ./requirements.in -django-cors-headers==4.7.0 +django-cors-headers==4.9.0 # via -r ./requirements.in django-countries==7.6.1 # via -r ./requirements.in @@ -156,53 +155,51 @@ django-haystack==3.3.0 # via -r ./requirements.in django-health-check==3.20.0 # via -r ./requirements.in -django-imagekit==5.0.0 +django-imagekit==6.0.0 # via -r ./requirements.in django-js-asset==3.1.2 # via django-mptt django-mptt==0.14.0 # via -r ./requirements.in -django-oauth-toolkit==3.0.1 +django-oauth-toolkit==3.1.0 # via -r ./requirements.in django-redis==6.0.0 # via -r ./requirements.in -django-rest-auth==0.9.5 - # via -r ./requirements.in django-storages==1.14.6 # via -r ./requirements.in django-test-plus==2.3.0 # via -r ./requirements.in django-tracer==0.9.3 # via -r ./requirements.in +django-upgrade==1.29.0 + # via -r ./requirements.in django-widget-tweaks==1.5.0 # via -r ./requirements.in djangorestframework==3.16.1 - # via - # -r ./requirements.in - # django-rest-auth -elasticsearch==7.17.12 + # via -r ./requirements.in +elasticsearch==7.9.1 # via -r ./requirements.in environs[django]==14.3.0 # via -r ./requirements.in executing==2.2.1 # via stack-data -faker==37.6.0 +faker==37.11.0 # via -r ./requirements.in -fastcore==1.8.8 +fastcore==1.8.13 # via ghapi -filelock==3.19.1 +filelock==3.20.0 # via virtualenv -fonttools==4.59.2 +fonttools==4.60.1 # via matplotlib -frozenlist==1.7.0 +frozenlist==1.8.0 # via # aiohttp # aiosignal fs==2.4.16 # via django-bakery -gevent==25.8.2 +gevent==25.9.1 # via -r ./requirements.in -ghapi==1.0.6 +ghapi==1.0.8 # via -r ./requirements.in greenlet==3.2.4 # via @@ -216,19 +213,19 @@ httpcore==1.0.9 # via httpx httpx==0.28.1 # via openai -identify==2.6.1 +identify==2.6.15 # via pre-commit -idna==3.10 +idna==3.11 # via # anyio # httpx # requests # yarl -iniconfig==2.1.0 +iniconfig==2.3.0 # via pytest interrogate==1.7.0 # via -r ./requirements.in -ipython==9.5.0 +ipython==9.6.0 # via -r ./requirements.in ipython-pygments-lexers==1.1.1 # via ipython @@ -236,7 +233,7 @@ itsdangerous==2.2.0 # via -r ./requirements.in jedi==0.19.2 # via ipython -jiter==0.10.0 +jiter==0.11.1 # via openai jmespath==1.0.1 # via @@ -252,21 +249,21 @@ kiwisolver==1.4.9 # via matplotlib kombu==5.5.4 # via celery -lxml==6.0.1 +lxml==6.0.2 # via -r ./requirements.in marshmallow==4.0.1 # via environs -matplotlib==3.10.6 +matplotlib==3.10.7 # via wordcloud matplotlib-inline==0.1.7 # via ipython -minio==7.2.16 +minio==7.2.18 # via -r ./requirements.in -mistletoe==1.4.0 +mistletoe==1.5.0 # via -r ./requirements.in model-bakery==1.20.5 # via -r ./requirements.in -multidict==6.6.4 +multidict==6.7.0 # via # aiohttp # yarl @@ -274,7 +271,7 @@ mypy-extensions==1.1.0 # via black nodeenv==1.9.1 # via pre-commit -numpy==2.3.2 +numpy==2.3.4 # via # contourpy # matplotlib @@ -283,9 +280,9 @@ oauthlib==3.3.1 # via # django-allauth # django-oauth-toolkit -openai==1.102.0 +openai==2.6.0 # via -r ./requirements.in -packaging==24.1 +packaging==25.0 # via # black # django-haystack @@ -303,13 +300,13 @@ pexpect==4.9.0 # via ipython pilkit==3.0 # via django-imagekit -pillow==11.3.0 +pillow==12.0.0 # via # -r ./requirements.in # matplotlib # pilkit # wordcloud -platformdirs==4.4.0 +platformdirs==4.5.0 # via # black # virtualenv @@ -323,13 +320,13 @@ prompt-toolkit==3.0.52 # via # click-repl # ipython -propcache==0.3.2 +propcache==0.4.1 # via # aiohttp # yarl psycogreen==1.0.2 # via -r ./requirements.in -psycopg2-binary==2.9.10 +psycopg2-binary==2.9.11 # via -r ./requirements.in ptyprocess==0.7.0 # via pexpect @@ -337,33 +334,33 @@ pure-eval==0.2.3 # via stack-data py==1.11.0 # via interrogate -pycparser==2.22 +pycparser==2.23 # via cffi pycryptodome==3.23.0 # via minio -pydantic==2.11.9 +pydantic==2.12.3 # via # algoliasearch # openai -pydantic-core==2.33.2 +pydantic-core==2.41.4 # via pydantic pygments==2.19.2 # via # ipython # ipython-pygments-lexers # pytest -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.1 # via # django-allauth # redis -pyparsing==3.2.0 +pyparsing==3.2.5 # via matplotlib pytest==8.4.2 # via # -r ./requirements.in # pytest-cov # pytest-django -pytest-cov==6.2.1 +pytest-cov==7.0.0 # via -r ./requirements.in pytest-django==4.11.1 # via -r ./requirements.in @@ -378,9 +375,11 @@ python-dotenv==1.1.1 # via environs python-frontmatter==1.1.0 # via -r ./requirements.in -python-json-logger==3.3.0 +python-json-logger==4.0.0 # via -r ./requirements.in -pyyaml==6.0.2 +pytokens==0.2.0 + # via black +pyyaml==6.0.3 # via # pre-commit # python-frontmatter @@ -399,26 +398,24 @@ requests==2.32.5 # responses responses==0.25.8 # via -r ./requirements.in -s3transfer==0.13.1 +s3transfer==0.14.0 # via boto3 setuptools==80.9.0 # via # fs # zope-event - # zope-interface six==1.17.0 # via # django-bakery - # django-rest-auth # fs # python-dateutil -slack-sdk==3.36.0 +slack-sdk==3.37.0 # via -r ./requirements.in sniffio==1.3.1 # via # anyio # openai -soupsieve==2.6 +soupsieve==2.8 # via beautifulsoup4 sqlparse==0.5.3 # via django @@ -428,6 +425,8 @@ structlog==25.4.0 # via -r ./requirements.in tabulate==0.9.0 # via interrogate +tokenize-rt==6.2.0 + # via django-upgrade tqdm==4.67.1 # via openai traitlets==5.14.3 @@ -439,7 +438,6 @@ typing-extensions==4.15.0 # aiosignal # anyio # beautifulsoup4 - # dj-database-url # django-countries # ipython # jwcrypto @@ -448,7 +446,7 @@ typing-extensions==4.15.0 # pydantic # pydantic-core # typing-inspection -typing-inspection==0.4.1 +typing-inspection==0.4.2 # via pydantic tzdata==2025.2 # via @@ -456,7 +454,7 @@ tzdata==2025.2 # kombu unidecode==1.4.0 # via -r ./requirements.in -urllib3==1.26.20 +urllib3==2.5.0 # via # algoliasearch # botocore @@ -465,26 +463,26 @@ urllib3==1.26.20 # minio # requests # responses -uv==0.8.15 +uv==0.9.5 # via -r ./requirements.in vine==5.1.0 # via # amqp # celery # kombu -virtualenv==20.34.0 +virtualenv==20.35.3 # via pre-commit -wcwidth==0.2.13 +wcwidth==0.2.14 # via prompt-toolkit wheel==0.45.1 # via -r ./requirements.in -whitenoise==6.9.0 +whitenoise==6.11.0 # via -r ./requirements.in wordcloud==1.9.4 # via -r ./requirements.in -yarl==1.20.1 +yarl==1.22.0 # via aiohttp -zope-event==5.1.1 +zope-event==6.0 # via gevent -zope-interface==7.2 +zope-interface==8.0.1 # via gevent diff --git a/slack/migrations/0001_initial.py b/slack/migrations/0001_initial.py index 87d664842..4da1ebad6 100644 --- a/slack/migrations/0001_initial.py +++ b/slack/migrations/0001_initial.py @@ -146,7 +146,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="thread", constraint=models.CheckConstraint( - check=django.db.models.lookups.GreaterThanOrEqual( + condition=django.db.models.lookups.GreaterThanOrEqual( django.db.models.functions.comparison.Cast( "last_update_ts", output_field=models.FloatField() ), @@ -172,7 +172,7 @@ class Migration(migrations.Migration): migrations.AddConstraint( model_name="channelupdategap", constraint=models.CheckConstraint( - check=django.db.models.lookups.GreaterThan( + condition=django.db.models.lookups.GreaterThan( django.db.models.functions.comparison.Cast( "newest_message_ts", output_field=models.FloatField() ), diff --git a/slack/models.py b/slack/models.py index 180762f1a..28e148485 100644 --- a/slack/models.py +++ b/slack/models.py @@ -61,7 +61,7 @@ class ChannelUpdateGap(models.Model): class Meta: constraints = [ models.CheckConstraint( - check=models.lookups.GreaterThan( + condition=models.lookups.GreaterThan( models.functions.Cast( "newest_message_ts", output_field=models.FloatField() ), @@ -110,7 +110,7 @@ class Meta: unique_together = [("channel", "thread_ts")] constraints = [ models.CheckConstraint( - check=models.lookups.GreaterThanOrEqual( + condition=models.lookups.GreaterThanOrEqual( models.functions.Cast( "last_update_ts", output_field=models.FloatField() ), diff --git a/users/admin.py b/users/admin.py index 9890cf62e..0e622244b 100644 --- a/users/admin.py +++ b/users/admin.py @@ -5,6 +5,7 @@ from .models import User +@admin.register(User) class EmailUserAdmin(UserAdmin): fieldsets = ( (None, {"fields": ("email", "password")}), @@ -59,6 +60,3 @@ class EmailUserAdmin(UserAdmin): "claimed", ) search_fields = ("email", "display_name__unaccent") - - -admin.site.register(User, EmailUserAdmin) diff --git a/users/views.py b/users/views.py index 777befe71..5f276332d 100644 --- a/users/views.py +++ b/users/views.py @@ -330,7 +330,7 @@ def render_to_response(self, context, **response_kwargs): ): # check if user is on pages that require CSRF but don't require login # (auth pages where anonymous users submit forms) - referer = self.request.META.get("HTTP_REFERER", "") + referer = self.request.headers.get("referer", "") current_path = self.request.path # paths that anonymous users can access and have forms diff --git a/versions/feeds.py b/versions/feeds.py index 7cc4f70da..dfd27d451 100644 --- a/versions/feeds.py +++ b/versions/feeds.py @@ -1,7 +1,7 @@ -from datetime import datetime +from datetime import datetime, timezone from django.contrib.syndication.views import Feed from django.utils.feedgenerator import Atom1Feed -from django.utils.timezone import make_aware, utc +from django.utils.timezone import make_aware from core.models import RenderedContent from .models import Version @@ -24,7 +24,7 @@ def item_pubdate(self, item): release_date = item.release_date if release_date: datetime_obj = datetime.combine(release_date, datetime.min.time()) - aware_datetime_obj = make_aware(datetime_obj, timezone=utc) + aware_datetime_obj = make_aware(datetime_obj, timezone=timezone.utc) return aware_datetime_obj def item_description(self, item): diff --git a/versions/tests/test_feeds.py b/versions/tests/test_feeds.py index 14fbcc32b..25f8d0147 100644 --- a/versions/tests/test_feeds.py +++ b/versions/tests/test_feeds.py @@ -1,5 +1,5 @@ -from datetime import datetime -from django.utils.timezone import make_aware, utc +from datetime import datetime, timezone +from django.utils.timezone import make_aware from ..feeds import RSSVersionFeed, AtomVersionFeed @@ -16,7 +16,8 @@ def test_items(version, old_version): def test_item_pubdate(version): feed = RSSVersionFeed() expected_datetime = make_aware( - datetime.combine(version.release_date, datetime.min.time()), timezone=utc + datetime.combine(version.release_date, datetime.min.time()), + timezone=timezone.utc, ) assert feed.item_pubdate(version) == expected_datetime @@ -51,6 +52,7 @@ def test_items_atom(version, old_version): def test_item_pubdate_atom(version): feed = AtomVersionFeed() expected_datetime = make_aware( - datetime.combine(version.release_date, datetime.min.time()), timezone=utc + datetime.combine(version.release_date, datetime.min.time()), + timezone=timezone.utc, ) assert feed.item_pubdate(version) == expected_datetime From 1d205fc198fc1e5751ef33a93a6f2910639a9e14 Mon Sep 17 00:00:00 2001 From: Julio Estrada Date: Mon, 27 Oct 2025 14:51:26 -0400 Subject: [PATCH 4/8] Remove "toc-hidden" class from original_docs.html --- templates/original_docs.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/original_docs.html b/templates/original_docs.html index 1ba930e5c..74c39feda 100644 --- a/templates/original_docs.html +++ b/templates/original_docs.html @@ -2,7 +2,7 @@ {% load i18n %} {% get_current_language as LANGUAGE_CODE %} - + From aa64fcd4378df8e9587398d10a66458cc5bd31bd Mon Sep 17 00:00:00 2001 From: daveoconnor Date: Tue, 28 Oct 2025 14:57:26 -0700 Subject: [PATCH 5/8] Account for develop/master branch paths in docs version alert regex (#1984) (#1985) --- libraries/mixins.py | 2 +- versions/models.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libraries/mixins.py b/libraries/mixins.py index c37cae04b..f565e03eb 100644 --- a/libraries/mixins.py +++ b/libraries/mixins.py @@ -42,7 +42,7 @@ def get_context_data(self, **kwargs): current_version_kwargs.update( { "content_path": re.sub( - r"([_0-9]+)/(\S+)", + r"([_0-9]+|master|develop)/(\S+)", rf"{LATEST_RELEASE_URL_PATH_STR}/\2", current_version_kwargs.get("content_path"), ) diff --git a/versions/models.py b/versions/models.py index 290573f01..bd7468618 100755 --- a/versions/models.py +++ b/versions/models.py @@ -326,5 +326,8 @@ def docs_path_to_boost_name(content_path): Convert a documentation content path to the Boost version name. e.g. "1_79_0/doc/html/accumulators.html" to "boost-1.79.0" """ - result = re.sub(r"^([_0-9]+)(/\S+)", r"boost-\1", content_path) + if content_path.startswith("develop") or content_path.startswith("master"): + result = content_path.split("/")[0] + else: + result = re.sub(r"^([_0-9]+)(/\S+)", r"boost-\1", content_path) return result.replace("_", ".") From c6add302f7a97bbba1f46822cba0c47c1e4b6293 Mon Sep 17 00:00:00 2001 From: daveoconnor Date: Wed, 29 Oct 2025 11:30:34 -0700 Subject: [PATCH 6/8] Fix issue with events no longer showing on homepage (#1990) (#1991) --- core/calendar.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/calendar.py b/core/calendar.py index d51d64092..ffb16e51e 100644 --- a/core/calendar.py +++ b/core/calendar.py @@ -1,4 +1,4 @@ -import datetime +from datetime import datetime, timezone from django.conf import settings import requests from collections import defaultdict @@ -21,7 +21,9 @@ def get_calendar(min_time=None, single_events=True, order_by="startTime"): """ if not min_time: # 'Z' indicates UTC time - min_time = datetime.datetime.now(datetime.timezone.utc).isoformat() + "Z" + # we replace +00:00 with Z because google no longer seems to be fully rfc3339 + # compliant for this parameter, even with a valid format ending with +00:00 + min_time = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") url = f"https://www.googleapis.com/calendar/v3/calendars/{settings.BOOST_CALENDAR}/events?key={settings.CALENDAR_API_KEY}&timeMin={min_time}&singleEvents={single_events}&orderBy={order_by}" headers = {"Accept": "application/json"} From ff272826bc9a73e4f597366348a493598cfc766e Mon Sep 17 00:00:00 2001 From: Dave O'Connor Date: Wed, 22 Oct 2025 15:01:58 -0700 Subject: [PATCH 7/8] Nix Python 3.13 updates (#1379) --- flake.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 0c7cbdd85..9ffc6744c 100644 --- a/flake.nix +++ b/flake.nix @@ -62,10 +62,10 @@ asciidoctor asciidoctorBoostGem pre-commit - python311 # matches Dockerfile, due for upgrade? - python311.pkgs.black - python311.pkgs.isort - python311.pkgs.pip-tools + python313 # matches Dockerfile, due for upgrade? + python313.pkgs.black + python313.pkgs.isort + python313.pkgs.pip-tools ]; # Host system installation workflow goes into the bootstrap justfile target. # Project specific installation and execution workflow should go here. @@ -74,7 +74,7 @@ pre-commit install fi if [ ! -d .venv ]; then - python3.11 -m venv .venv + python3.13 -m venv .venv . .venv/bin/activate pip install -r requirements.txt -r requirements-dev.txt else From 4830c06da833ff60c8aa6a741b1a8770417767ad Mon Sep 17 00:00:00 2001 From: Dave O'Connor Date: Mon, 3 Nov 2025 14:19:50 -0800 Subject: [PATCH 8/8] Updated load_production_data script to default download hyperkitty data, related docs and minor tweaks. --- README.md | 14 +++++--------- docs/development_setup_notes_nix.md | 2 +- docs/first_time_data_import.md | 3 +++ env.template | 2 +- scripts/load_production_data.sh | 20 ++++++++++---------- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 728b7bb73..5b4ab48a1 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ There are two options for development setups, nix and native. The native setup d 1. Development System setup + 1. Give your ssh key to Sam so he can add it to the boost.cpp.al server. 1. Basic Setup Options: 1. [Nix-based](docs/development_setup_notes_nix.md) 1. [Native](docs/development_setup_notes_native.md) @@ -137,6 +138,9 @@ currently generated by `docker compose build` and is included in the Docker imag --- ## Setting up Mailman/Hyperkitty locally + +Note: This is optional - not required for the website to function, here for reference in case a need arises. + ``shell sudo apt-get install sassc git clone git@gitlab.com:mailman/hyperkitty.git @@ -147,18 +151,10 @@ pip install psycopg2-binary `` change settings.py to use postgres database: 'ENGINE': 'django.db.backends.postgresql_psycopg2', -Update database values in settings to use the same host, user, password, and the database name as in the .env file value for `HYPERKITTY_DATABASE_NAME`. +Update database values in settings to use the same host, user, password, and the database name as in the .env file value for `HYPERKITTY_DATABASE_NAME` (`lists_production_web` by default). run `django-admin migrate --pythonpath example_project --settings settings` -Give your ssh key to Sam so he can add it to the boost.cpp.al server, and then download the mailman db archive and cp the sql to the docker container -```shell -scp {user}@staging-db1.boost.cpp.al:/tmp/lists_stage_web.staging-db1-2.2025-02-06-08-00-01.sql.gz . -docker cp lists_stage_web.staging-db1-2.2025-02-06-08-00-01.sql website-v2-web-1:/lists_stage_web.staging-db1-2.2025-02-06-08-00-01.sql -docker exec -it website-v2-web-1 /bin/bash -psql -U postgres -W hyperkitty_db < /lists_stage_web.staging-db1-2.2025-02-06-08-00-01.sql -``` - ## Syncing EmailData Locally To work with mailinglist data locally, the django application expects to be diff --git a/docs/development_setup_notes_nix.md b/docs/development_setup_notes_nix.md index 8b733fbcd..819b04689 100644 --- a/docs/development_setup_notes_nix.md +++ b/docs/development_setup_notes_nix.md @@ -36,7 +36,7 @@ For a basic rundown on Nix, this video could be useful https://www.youtube.com/w * STATIC_CONTENT_AWS_ACCESS_KEY_ID - ask for this * STATIC_CONTENT_AWS_SECRET_ACCESS_KEY - ask for this 5. Run `just setup` to build services, and build the JS and CSS assets. If docker fails with permissions errors, reboot your machine. -6. Run `just load_production_data` to download live data from the backup server. +6. Run `just load_production_data` to download live data from the backup server for the website and mailman/hyperkitty. That script can be run regularly to update data from the live servers. You will lose all local data. 7. Run `docker compose up` to start the server. diff --git a/docs/first_time_data_import.md b/docs/first_time_data_import.md index d801b6b8e..dff78883f 100644 --- a/docs/first_time_data_import.md +++ b/docs/first_time_data_import.md @@ -8,10 +8,13 @@ This document contains information about importing Boost Versions (also called R ## Development Environments +Download the production data for the website and mailman/hyperkitty. ```bash just load_production_data ```` +That script can be run regularly to update data from the live servers. You will lose all local data. + ## Deployed Environments There are several steps to populating the database with historical Boost data, because we retrieve Boost data from multiple sources. diff --git a/env.template b/env.template index 85699bcd4..bb4238b90 100644 --- a/env.template +++ b/env.template @@ -33,7 +33,7 @@ PROD_MEDIA_CONTENT_REGION=$STATIC_CONTENT_REGION PROD_MEDIA_CONTENT_AWS_S3_ENDPOINT_URL=$STATIC_CONTENT_AWS_S3_ENDPOINT_URL # Mailman database settings -HYPERKITTY_DATABASE_NAME="hyperkitty_db" +HYPERKITTY_DATABASE_NAME="lists_production_web" DATABASE_URL="postgresql://postgres@db:5432/postgres" DATABASE_TYPE="postgres" DATABASE_CLASS="mailman.database.postgresql.PostgreSQLDatabase" diff --git a/scripts/load_production_data.sh b/scripts/load_production_data.sh index f0a506466..f4d7379fb 100755 --- a/scripts/load_production_data.sh +++ b/scripts/load_production_data.sh @@ -11,7 +11,7 @@ set -eu # # READ IN COMMAND-LINE OPTIONS -TEMP=$(getopt -o h:: --long help::,lists::,only-lists:: -- "$@") +TEMP=$(getopt -o h:: --long help::,no-web::,no-lists:: -- "$@") eval set -- "$TEMP" # extract options and their arguments into variables. @@ -19,14 +19,14 @@ while true ; do case "$1" in -h|--help) helpmessage=""" -usage: load_production_data.sh [-h] [--lists] [--only-lists] +usage: load_production_data.sh [-h] [--no-web] [--no-lists] -Load production data. By default this will import the main website database. +Load production data. By default this will import both the main website database and mailing list databases. optional arguments: -h, --help Show this help message and exit - --lists Import mailing list dbs also. - --only-lists Import mailing list database and not the main web database. + --no-web Skip importing the main website database. + --no-lists Skip importing mailing list databases. """ echo "" @@ -34,10 +34,10 @@ optional arguments: echo "" exit 0 ;; - --lists) - lists_option="yes" ; shift 2 ;; - --only-lists) - lists_option="yes" ; skip_web_option="yes" ; shift 2 ;; + --no-web) + skip_web_option="yes" ; shift 2 ;; + --no-lists) + skip_lists_option="yes" ; shift 2 ;; --) shift ; break ;; *) echo "Internal error!" ; exit 1 ;; esac @@ -201,7 +201,7 @@ if [ "${skip_web_option:-}" != "yes" ]; then } fi -if [ "${lists_option:-}" = "yes" ]; then +if [ "${skip_lists_option:-}" != "yes" ]; then download_latest_db_dump lists_web_db || { echo "Failed to download and restore latest lists_web_db dump"; exit 1;