diff --git a/config/settings/base.py b/config/settings/base.py index 7bb600ef..832f4873 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -290,6 +290,9 @@ ADMINS = [("Mitchell Kotler", "mitch@muckrock.com")] # https://docs.djangoproject.com/en/dev/ref/settings/#managers MANAGERS = ADMINS +# Chunk size for CSV exports using .iterator() to process large querysets +# without loading all records into memory at once +CSV_EXPORT_CHUNK_SIZE = env.int("CSV_EXPORT_CHUNK_SIZE", default=2000) # LOGGING # ------------------------------------------------------------------------------ diff --git a/documentcloud/addons/admin.py b/documentcloud/addons/admin.py index f8662bd7..7b8db70a 100644 --- a/documentcloud/addons/admin.py +++ b/documentcloud/addons/admin.py @@ -1,4 +1,5 @@ # Django +from django.conf import settings from django.contrib import admin, messages from django.db.models import JSONField from django.forms import widgets @@ -128,7 +129,10 @@ def export_runs_as_csv(self, request, queryset): """Export selected Add-On Runs to CSV.""" field_names = [ "addon_id", + "addon_name", "user_id", + "user_name", + "user_email", "run_id", "status", "rating", @@ -143,11 +147,28 @@ def export_runs_as_csv(self, request, queryset): writer = csv.writer(response) writer.writerow(field_names) - for run in queryset: + limited_queryset = queryset.select_related("addon", "user").only( + "addon_id", + "user_id", + "run_id", + "status", + "rating", + "credits_spent", + "created_at", + "updated_at", + "addon__name", + "user__name", + "user__email", + ) + + for run in limited_queryset.iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE): writer.writerow( [ run.addon_id, + run.addon.name, run.user_id, + run.user.name, + run.user.email, run.run_id, run.status, run.rating, diff --git a/documentcloud/organizations/admin.py b/documentcloud/organizations/admin.py index 2e7d0c79..f347c5d1 100644 --- a/documentcloud/organizations/admin.py +++ b/documentcloud/organizations/admin.py @@ -1,4 +1,5 @@ # Django +from django.conf import settings from django.contrib import admin from django.http import HttpResponse @@ -41,7 +42,7 @@ def export_ai_credit_logs(self, request, queryset): writer = csv.writer(response) writer.writerow(field_names) - for log in queryset: + for log in queryset.iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE): writer.writerow( [ str(log.organization), diff --git a/documentcloud/statistics/admin.py b/documentcloud/statistics/admin.py index 66a3249e..81d9babe 100644 --- a/documentcloud/statistics/admin.py +++ b/documentcloud/statistics/admin.py @@ -1,4 +1,5 @@ # Django +from django.conf import settings from django.contrib import admin from django.http import HttpResponse @@ -29,7 +30,7 @@ def export_statistics_as_csv(self, request, queryset): writer = csv.writer(response) writer.writerow(field_names) - for obj in queryset: + for obj in queryset.iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE): row = [] for field_name in field_names: value = getattr(obj, field_name) diff --git a/documentcloud/users/admin.py b/documentcloud/users/admin.py index 61cc6e3a..a92fb0cf 100644 --- a/documentcloud/users/admin.py +++ b/documentcloud/users/admin.py @@ -1,4 +1,5 @@ # Django +from django.conf import settings from django.contrib import admin from django.http.response import HttpResponse from django.urls.conf import re_path @@ -47,7 +48,7 @@ def format_date(date): writer.writerow(["username", "name", "email", "last_login", "date_joined"]) for user in User.objects.only( "username", "name", "email", "last_login", "created_at" - ).iterator(chunk_size=2000): + ).iterator(chunk_size=settings.CSV_EXPORT_CHUNK_SIZE): writer.writerow( [ user.username,