Skip to content

Commit 0ed2d03

Browse files
tdrueztdruez
authored
Add ability to download all output results formats #1880 (#1910)
Signed-off-by: tdruez <tdruez@nexb.com>
1 parent 2da4d37 commit 0ed2d03

File tree

12 files changed

+138
-30
lines changed

12 files changed

+138
-30
lines changed

‎CHANGELOG.rst‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
Changelog
22
=========
33

4+
v35.4.1 (unreleased)
5+
--------------------
6+
7+
- Add ability to download all output results formats as a zipfile for a given project.
8+
https://github.com/aboutcode-org/scancode.io/issues/1880
9+
410
v35.4.0 (2025-09-30)
511
--------------------
612

‎docs/command-line-interface.rst‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -419,10 +419,11 @@ Displays status information about the ``PROJECT`` project.
419419

420420
.. _cli_output:
421421

422-
`$ scanpipe output --project PROJECT --format {json,csv,xlsx,spdx,cyclonedx,attribution}`
423-
-----------------------------------------------------------------------------------------
422+
`$ scanpipe output --project PROJECT --format {json,csv,xlsx,spdx,cyclonedx,attribution,...}`
423+
---------------------------------------------------------------------------------------------
424424

425-
Outputs the ``PROJECT`` results as JSON, XLSX, CSV, SPDX, CycloneDX, and Attribution.
425+
Outputs the ``PROJECT`` results as JSON, XLSX, CSV, SPDX, CycloneDX,
426+
ORT package-list.yml, and Attribution.
426427
The output files are created in the ``PROJECT`` :guilabel:`output/` directory.
427428

428429
Multiple formats can be provided at once::

‎docs/output-files.rst‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,6 @@ Additional sheets are included **only when relevant** (i.e., when data is availa
285285

286286
SPDX
287287
^^^^
288-
289288
ScanCode.io can generate Software Bill of Materials (SBOM) in the **SPDX** format,
290289
which is an open standard for communicating software component information.
291290
SPDX is widely used for license compliance, security analysis, and software supply
@@ -309,7 +308,6 @@ The SPDX output includes:
309308

310309
CycloneDX
311310
^^^^^^^^^
312-
313311
ScanCode.io can generate **CycloneDX** SBOMs, a lightweight standard designed for
314312
security and dependency management. CycloneDX is optimized for vulnerability analysis
315313
and software supply chain risk assessment.

‎docs/rest-api.rst‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -694,10 +694,16 @@ Finally, use this action to download the project results in the provided
694694
``output_format`` as an attachment file.
695695

696696
Data:
697-
- ``output_format``: ``json``, ``xlsx``, ``spdx``, ``cyclonedx``, ``attribution``
697+
- ``output_format``: ``json``, ``xlsx``, ``spdx``, ``cyclonedx``, ``attribution``,
698+
``all_formats``, ``all_outputs``
698699

699700
``GET /api/projects/d4ed9405-5568-45ad-99f6-782a9b82d1d2/results_download/?output_format=cyclonedx``
700701

702+
.. note::
703+
Use ``all_formats`` to generate a zip file containing all output formats for a
704+
project, while ``all_outputs`` can be used to obtain a zip file of all existing
705+
output files for that project.
706+
701707
.. tip::
702708
Refer to :ref:`output_files` to learn more about the available output formats.
703709

‎scanpipe/api/views.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,10 @@ def results_download(self, request, *args, **kwargs):
171171
output_file = output.to_attribution(project)
172172
elif format == "ort-package-list":
173173
output_file = output.to_ort_package_list_yml(project)
174+
elif format == "all_formats":
175+
output_file = output.to_all_formats(project)
176+
elif format == "all_outputs":
177+
output_file = output.to_all_outputs(project)
174178
else:
175179
message = {"status": f"Format {format} not supported."}
176180
return Response(message, status=status.HTTP_400_BAD_REQUEST)

‎scanpipe/pipes/output.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,12 @@
2626
import json
2727
import re
2828
import uuid
29+
import zipfile
2930
from operator import attrgetter
3031
from pathlib import Path
3132

3233
from django.apps import apps
34+
from django.core.files.base import ContentFile
3335
from django.core.serializers.json import DjangoJSONEncoder
3436
from django.forms.models import model_to_dict
3537
from django.template import Context
@@ -1138,3 +1140,43 @@ def to_ort_package_list_yml(project):
11381140
"attribution": to_attribution,
11391141
"ort-package-list": to_ort_package_list_yml,
11401142
}
1143+
1144+
1145+
def make_zip_from_files(files):
1146+
"""Return an in-memory zipfile given a list of (filename, file_path) pairs."""
1147+
zip_buffer = io.BytesIO()
1148+
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
1149+
for filename, file_path in files:
1150+
with open(file_path, "rb") as f:
1151+
zip_file.writestr(filename, f.read())
1152+
zip_buffer.seek(0)
1153+
return zip_buffer
1154+
1155+
1156+
def to_all_formats(project):
1157+
"""Generate all output formats for a project and return a Django File-like zip."""
1158+
files = []
1159+
for output_function in FORMAT_TO_FUNCTION_MAPPING.values():
1160+
output_file = output_function(project)
1161+
filename = safe_filename(f"{project.name}_{output_file.name}")
1162+
files.append((filename, output_file))
1163+
1164+
zip_buffer = make_zip_from_files(files)
1165+
1166+
# Wrap it into a Django File-like object
1167+
zip_file = ContentFile(zip_buffer.getvalue())
1168+
zip_file.name = safe_filename(f"{project.name}_outputs.zip")
1169+
1170+
return zip_file
1171+
1172+
1173+
def to_all_outputs(project):
1174+
"""Return a Django File-like zip containing all existing project's output/ files."""
1175+
files = [(path.name, path) for path in project.output_path.glob("*")]
1176+
zip_buffer = make_zip_from_files(files)
1177+
1178+
# Wrap it into a Django File-like object
1179+
zip_file = ContentFile(zip_buffer.getvalue())
1180+
zip_file.name = safe_filename(f"{project.name}_outputs.zip")
1181+
1182+
return zip_file

‎scanpipe/templates/scanpipe/dropdowns/project_download_dropdown.html‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@
4040
<a href="{% url 'project_results' project.slug 'ort-package-list' %}" class="dropdown-item">
4141
<strong>ORT (package-list)</strong>
4242
</a>
43+
<hr class="dropdown-divider" />
44+
<a href="{% url 'project_results' project.slug 'all_formats' %}" class="dropdown-item">
45+
<strong>All formats</strong>
46+
</a>
4347
</div>
4448
</div>
4549
</div>

‎scanpipe/templates/scanpipe/includes/project_downloads.html‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,16 @@
11
<article class="message is-success">
2-
<div class="message-body">
3-
Download results as:
2+
<div class="message-body p-3">
3+
<span class="icon"><i class="fa-solid fa-download"></i></span>
4+
Download results:
45
<a class="tag is-success is-medium ml-2" href="{% url 'project_results' project.slug 'json' %}">
5-
<span class="icon mr-1"><i class="fa-solid fa-download"></i></span>JSON
6+
JSON
67
</a>
78
<a class="tag is-success is-medium" href="{% url 'project_results' project.slug 'xlsx' %}">
8-
<span class="icon mr-1"><i class="fa-solid fa-download"></i></span>XLSX
9+
XLSX
910
</a>
1011
<div class="dropdown is-hoverable">
1112
<div class="dropdown-trigger">
1213
<button class="button tag is-success is-medium has-text-weight-normal" aria-haspopup="true" aria-controls="dropdown-menu-spdx">
13-
<span class="icon">
14-
<i class="fa-solid fa-download" aria-hidden="true"></i>
15-
</span>
1614
<span>SPDX</span>
1715
<span class="icon is-small">
1816
<i class="fa-solid fa-angle-down" aria-hidden="true"></i>
@@ -33,9 +31,6 @@
3331
<div class="dropdown is-hoverable">
3432
<div class="dropdown-trigger">
3533
<button class="button tag is-success is-medium has-text-weight-normal" aria-haspopup="true" aria-controls="dropdown-menu-cyclonedx">
36-
<span class="icon">
37-
<i class="fa-solid fa-download" aria-hidden="true"></i>
38-
</span>
3934
<span>CycloneDX</span>
4035
<span class="icon is-small">
4136
<i class="fa-solid fa-angle-down" aria-hidden="true"></i>
@@ -57,14 +52,11 @@
5752
</div>
5853
</div>
5954
<a class="tag is-success is-medium" href="{% url 'project_results' project.slug 'attribution' %}">
60-
<span class="icon mr-1"><i class="fa-solid fa-download"></i></span>Attribution
55+
Attribution
6156
</a>
6257
<div class="dropdown is-hoverable">
6358
<div class="dropdown-trigger">
6459
<button class="button tag is-success is-medium has-text-weight-normal" aria-haspopup="true" aria-controls="dropdown-menu-ort">
65-
<span class="icon">
66-
<i class="fa-solid fa-download" aria-hidden="true"></i>
67-
</span>
6860
<span>Tools formats</span>
6961
<span class="icon is-small">
7062
<i class="fa-solid fa-angle-down" aria-hidden="true"></i>
@@ -79,5 +71,9 @@
7971
</div>
8072
</div>
8173
</div>
74+
<span class="p-1">|</span>
75+
<a class="tag is-success is-medium" href="{% url 'project_results' project.slug 'all_formats' %}">
76+
All formats
77+
</a>
8278
</div>
8379
</article>

‎scanpipe/templates/scanpipe/panels/project_outputs.html‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,10 @@
1818
</div>
1919
</div>
2020
{% endfor %}
21+
<div class="panel-block">
22+
<a class="button is-link is-outlined is-fullwidth" href="{% url 'project_results' project.slug 'all_outputs' %}">
23+
<span class="icon mr-1"><i class="fa-solid fa-download"></i></span>
24+
Download all outputs
25+
</a>
26+
</div>
2127
</article>

‎scanpipe/tests/pipes/test_output.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import shutil
2727
import tempfile
2828
import uuid
29+
import zipfile
2930
from dataclasses import dataclass
3031
from pathlib import Path
3132
from unittest import mock
@@ -633,6 +634,39 @@ def test_scanpipe_pipes_outputs_to_to_ort_package_list_yml(self):
633634
expected_file = self.data / "asgiref" / "asgiref-3.3.0.package-list.yml"
634635
self.assertResultsEqual(expected_file, output_file.read_text())
635636

637+
def test_scanpipe_pipes_outputs_to_all_formats(self):
638+
fixtures = self.data / "asgiref" / "asgiref-3.3.0_fixtures.json"
639+
call_command("loaddata", fixtures, **{"verbosity": 0})
640+
project = Project.objects.get(name="asgiref")
641+
642+
with self.assertNumQueries(35):
643+
output_file = output.to_all_formats(project=project)
644+
645+
self.assertEqual("asgiref_outputs.zip", output_file.name)
646+
647+
with zipfile.ZipFile(output_file, "r") as zip_ref:
648+
zip_contents = zip_ref.namelist()
649+
file_count = len(zip_contents)
650+
651+
expected_file_count = len(output.FORMAT_TO_FUNCTION_MAPPING)
652+
self.assertEqual(expected_file_count, file_count)
653+
654+
def test_scanpipe_pipes_outputs_to_all_outputs(self):
655+
fixtures = self.data / "asgiref" / "asgiref-3.3.0_fixtures.json"
656+
call_command("loaddata", fixtures, **{"verbosity": 0})
657+
project = Project.objects.get(name="asgiref")
658+
659+
with self.assertNumQueries(0):
660+
output_file = output.to_all_outputs(project=project)
661+
662+
self.assertEqual("asgiref_outputs.zip", output_file.name)
663+
664+
with zipfile.ZipFile(output_file, "r") as zip_ref:
665+
zip_contents = zip_ref.namelist()
666+
file_count = len(zip_contents)
667+
668+
self.assertEqual(len(project.output_root), file_count)
669+
636670
def test_scanpipe_pipes_outputs_make_unknown_license_object(self):
637671
licensing = get_licensing()
638672
parsed_expression = licensing.parse("some-unknown-license")

0 commit comments

Comments
 (0)