diff --git a/bin/pipelines.json.js b/bin/pipelines.json.js index 0def81283b..5a041630c8 100644 --- a/bin/pipelines.json.js +++ b/bin/pipelines.json.js @@ -1,5 +1,6 @@ #! /usr/bin/env node import { octokit, getDocFiles, getGitHubFile, githubFolderExists } from "../sites/main-site/src/components/octokit.js"; +import { fetchCO2FootprintFiles } from "./s3Utils.js"; import { promises as fs, writeFileSync, existsSync } from "fs"; import yaml from "js-yaml"; @@ -491,6 +492,17 @@ export const writePipelinesJson = async () => { nf_core_version = extractNfCoreVersion(nfCoreYml); } + // Check if this release has plugins already stored either as dev or as "main"/"master" + let plugins = data[`main_nextflow_config_plugins`] || data[`master_nextflow_config_plugins`] || []; + + // Fetch CO2 footprint files if nf-co2footprint plugin is used and we are not in the dev tree + console.log(plugins); + let co2footprint_files = null; + if (plugins.join(" ").includes("nf-co2footprint") && tag_name !== "dev") { + console.log(`Fetching CO2 footprint files for ${name} ${tag_name}`); + co2footprint_files = await fetchCO2FootprintFiles(name, tag_sha); + } + let components = await octokit .request("GET /repos/{owner}/{repo}/contents/{path}?ref={ref}", { owner: "nf-core", @@ -556,6 +568,8 @@ export const writePipelinesJson = async () => { components, nextflow_version, nf_core_version, + plugins, + co2footprint_files, }; }), ); diff --git a/bin/s3Utils.js b/bin/s3Utils.js new file mode 100644 index 0000000000..d78a0d353a --- /dev/null +++ b/bin/s3Utils.js @@ -0,0 +1,122 @@ +import { S3Client, ListObjectsV2Command, GetObjectCommand } from "@aws-sdk/client-s3"; + +/** + * Creates an anonymous S3 client for public bucket access + */ +export function createS3Client() { + return new S3Client({ + region: "eu-west-1", + signer: { sign: async (request) => request }, + credentials: { + accessKeyId: "", + secretAccessKey: "", + }, + }); +} + +/** + * Fetches all objects with a given prefix from S3 + * @param {string} bucketName - S3 bucket name + * @param {string} prefix - Object key prefix + * @param {string} [delimiter] - Optional delimiter for folder-like structure + * @returns {Promise<{bucketContents: Array, commonPrefixes: Array}>} Object containing Contents and CommonPrefixes arrays + */ +export async function fetchS3Objects(bucketName, prefix, delimiter) { + const client = createS3Client(); + + let bucketContents = []; + let commonPrefixes = []; + let isTruncated = true; + let continuationToken; + + try { + while (isTruncated) { + const response = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + Prefix: prefix, + Delimiter: delimiter, + ContinuationToken: continuationToken, + }), + ); + + if (response.KeyCount === 0) { + break; + } + + if (response.Contents) { + bucketContents.push(...response.Contents); + } + + if (response.CommonPrefixes) { + commonPrefixes.push(...response.CommonPrefixes); + } + + isTruncated = response.IsTruncated ?? false; + continuationToken = response.NextContinuationToken; + } + + return { bucketContents, commonPrefixes }; + } catch (error) { + console.warn(`Failed to fetch S3 objects for prefix ${prefix}:`, error); + return { bucketContents: [], commonPrefixes: [] }; + } +} + +/** + * Fetches and parses CO2 footprint summary file from S3 for a given pipeline and commit SHA + * @param {string} pipelineName - Name of the pipeline (e.g., 'rnaseq') + * @param {string} commitSha - Git commit SHA for the release + * @returns {Promise<{filename: string, co2e_emissions: number|null, energy_consumption: number|null}|null>} CO2 footprint data object with filename, co2e_emissions, and energy_consumption, or null if not found + */ +export async function fetchCO2FootprintFiles(pipelineName, commitSha) { + console.log(`Fetching CO2 footprint files for ${pipelineName} ${commitSha}`); + const bucketName = "nf-core-awsmegatests"; + const prefix = `${pipelineName}/results-${commitSha}/`; + + const { bucketContents } = await fetchS3Objects(bucketName, prefix); + + // Filter for CO2 footprint summary file matching the pattern + const co2FootprintFile = bucketContents.find((file) => { + const fileName = file.Key?.split("/").pop() || ""; + return /co2footprint_summary_.*\.txt$/.test(fileName); + }); + console.log(`CO2 footprint file found: ${co2FootprintFile?.Key}`); + + if (!co2FootprintFile) { + return null; + } + + const client = createS3Client(); + + try { + const response = await client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: co2FootprintFile.Key, + }), + ); + + // Read the stream and convert to string + const bodyContents = await response.Body?.transformToString(); + + if (!bodyContents) { + return null; + } + + // Parse CO2e emissions and energy consumption + const co2Match = bodyContents.match(/CO2e emissions:\s*([\d.]+)\s*g/); + const energyMatch = bodyContents.match(/Energy consumption:\s*([\d.]+)\s*Wh/); + + const fileName = co2FootprintFile.Key?.split("/").pop() || ""; + + return { + filename: fileName, + co2e_emissions: co2Match ? parseFloat(co2Match[1]) : null, + energy_consumption: energyMatch ? parseFloat(energyMatch[1]) : null, + }; + } catch (error) { + console.warn(`Failed to fetch CO2 footprint file ${co2FootprintFile.Key}:`, error); + return null; + } +} diff --git a/public/pipelines.json b/public/pipelines.json index 2f5669d73b..13ab2ea9cf 100644 --- a/public/pipelines.json +++ b/public/pipelines.json @@ -26081,7 +26081,7 @@ "fork": false, "url": "https://api.github.com/repos/nf-core/multiplesequencealign", "created_at": "2023-07-24T14:30:12Z", - "updated_at": "2025-10-02T01:27:27Z", + "updated_at": "2025-10-02T11:57:09Z", "pushed_at": "2025-07-09T08:24:51Z", "homepage": "https://nf-co.re/multiplesequencealign", "size": 20538, @@ -26290,7 +26290,17 @@ ] }, "nextflow_version": "!>=25.04.2", - "nf_core_version": "3.2.1" + "nf_core_version": "3.2.1", + "plugins": [ + "nf-schema@2.2.0", + "nf-co2footprint@1.0.0-beta", + "nf-prov@1.2.2" + ], + "co2footprint_files": { + "filename": "co2footprint_summary_2025-05-28_13-38-21.txt", + "co2e_emissions": 169.72, + "energy_consumption": 357.36 + } }, { "tag_name": "1.1.0", @@ -26446,7 +26456,13 @@ ] }, "nextflow_version": "!>=25.04.2", - "nf_core_version": "3.2.1" + "nf_core_version": "3.2.1", + "plugins": [ + "nf-schema@2.2.0", + "nf-co2footprint@1.0.0-beta", + "nf-prov@1.2.2" + ], + "co2footprint_files": null } ] }, diff --git a/sites/main-site/src/components/pipeline/PipelineSidebar.astro b/sites/main-site/src/components/pipeline/PipelineSidebar.astro index 85ad17ff01..39849a4b04 100644 --- a/sites/main-site/src/components/pipeline/PipelineSidebar.astro +++ b/sites/main-site/src/components/pipeline/PipelineSidebar.astro @@ -23,6 +23,10 @@ export interface Props { modules?: string[]; subworkflows?: string[]; }; + co2footprint_files: { + co2e_emissions: number; + energy_consumption: number; + }; }[]; stargazers_count: number; subscribers_count: number; @@ -178,6 +182,15 @@ const subworkflows: string[] = meta.releases[0].components?.subworkflows || []; }, ]} /> + 2e emissions ", + value: meta.releases[0].co2footprint_files.co2e_emissions + " g", + }, + { title: "energy consumption", value: meta.releases[0].co2footprint_files.energy_consumption + " Wh" }, + ]} + /> { modules.length > 0 && ( diff --git a/sites/main-site/src/components/sidebar/SidebarStatsRow.astro b/sites/main-site/src/components/sidebar/SidebarStatsRow.astro index 122b3170f9..1091f48aca 100644 --- a/sites/main-site/src/components/sidebar/SidebarStatsRow.astro +++ b/sites/main-site/src/components/sidebar/SidebarStatsRow.astro @@ -17,9 +17,8 @@ export interface Props { if (entry.title && entry.value) { return (
-
+
{entry.icon && } - {entry.title}
diff --git a/sites/main-site/src/layouts/ResultsLayout.astro b/sites/main-site/src/layouts/ResultsLayout.astro index b5d9b1ba3a..1e01b75d1c 100644 --- a/sites/main-site/src/layouts/ResultsLayout.astro +++ b/sites/main-site/src/layouts/ResultsLayout.astro @@ -1,7 +1,7 @@ --- import PipelinePageLayout from "@layouts/PipelinePageLayout.astro"; import pipelines_json from "@public/pipelines.json"; -import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3"; +import { fetchS3Objects } from "../../../../bin/s3Utils"; import AWSFile from "@components/pipeline/AWSFile.astro"; import AWSFilePreview from "@components/pipeline/AWSFilePreview.astro"; @@ -34,41 +34,13 @@ const pathParts = path.split("/"); let filename = Astro.url.searchParams.get("file"); const description = meta?.description; const depth = pathParts.length; -let awsResponse; let isTruncated = commonPrefixes.length === 0; if (bucketContents.length === 0 && commonPrefixes.length === 0) { - let client = new S3Client({ - region: "eu-west-1", - signer: { sign: async (request) => request }, - credentials: { - accessKeyId: "", - secretAccessKey: "", - }, - }); - - while (isTruncated) { - awsResponse = await client.send( - new ListObjectsV2Command({ - Bucket: bucketName, - ContinuationToken: awsResponse?.NextContinuationToken, - Prefix: path + "/", - Delimiter: "/", - }), - ); - - if (awsResponse.KeyCount === 0) { - break; - } - if (awsResponse.Contents) { - bucketContents.push(...awsResponse.Contents); - } - if (awsResponse.CommonPrefixes) { - commonPrefixes.push(...awsResponse.CommonPrefixes); - } - - isTruncated = awsResponse.IsTruncated; - } + const result = await fetchS3Objects(bucketName, path + "/", "/"); + bucketContents = result.bucketContents; + commonPrefixes = result.commonPrefixes; + isTruncated = false; } let directories: { name: string | undefined; path: string; depth: number; size: number; lastModified: Date }[] = []; let files: { name: string | undefined; path: string; depth: number; size: number; lastModified: Date }[] = [];