diff --git a/docker/rucio-quotas/Dockerfile b/docker/rucio-quotas/Dockerfile new file mode 100644 index 00000000..3b572fe8 --- /dev/null +++ b/docker/rucio-quotas/Dockerfile @@ -0,0 +1,4 @@ +FROM registry.cern.ch/cmsmonitoring/cmsmon-py:test + +# Copy only the specific files you need instead of cloning entire repos +COPY src/ ./ diff --git a/docker/rucio-quotas/README.md b/docker/rucio-quotas/README.md new file mode 100644 index 00000000..4a2bf03b --- /dev/null +++ b/docker/rucio-quotas/README.md @@ -0,0 +1,3 @@ +# Rucio Quotas Docker Image + +Calculates the quotas of Rucio storage elements (RSE), i.e., sites, in TB. It runs daily and fills the [CMS Rucio Quotas and usage monitoring](https://cmsdatapop.web.cern.ch/cmsdatapop/rucio/quotas.html) website of the CMS Data Popularity service. diff --git a/docker/rucio-quotas/src/cron4rucio_quotas.sh b/docker/rucio-quotas/src/cron4rucio_quotas.sh new file mode 100755 index 00000000..91b837d8 --- /dev/null +++ b/docker/rucio-quotas/src/cron4rucio_quotas.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -e +TZ=UTC +myname=$(basename "$0") +script_dir="$(cd "$(dirname "$0")" && pwd)" +# Get nice util functions +. "${script_dir}"/utils.sh + +# ---------------------------------------------------------------------------------------------------------- Run in K8S +if [ -n "$K8S_ENV" ]; then + # $1: output, shift is mandatory because rucio/setup-py3.sh also waits for $1 + output_=$1; shift + + util4logi "${myname} is starting.." + util_cron_send_start "$myname" "1h" + + # Rucio API setup + export X509_USER_PROXY=/etc/proxy/proxy + export RUCIO_HOME=/cvmfs/cms.cern.ch/rucio/x86_64/rhel9/py3/current + util_kerberos_auth_with_keytab /etc/secrets/keytab + python3 "${script_dir}"/rucio_quotas.py \ + --output "$output_" \ + --template_dir "${script_dir}/rucio_quotas_html" 2>&1 + + util_cron_send_end "$myname" "1h" "$?" + util4logi "${myname} successfully finished." + exit 0 + # break +fi +# Run in LxPlus for test ---------------------------------------------------------------------------------------------- + +source /cvmfs/cms.cern.ch/cmsset_default.sh >/dev/null +source /cvmfs/cms.cern.ch/rucio/setup-py3.sh >/dev/null +output=$(voms-proxy-init -voms cms -rfc -valid 192:00 2>&1) +ec=$? +if [ $ec -ne 0 ]; then + echo "$output" - exit code: $ec + exit $ec +fi + +py_input_args=( + --output "/eos/user/c/cmsmonit/www/rucio/quotas.html" + --template_dir "${script_dir}/../src/html/rucio_quotas" +) + +# Catch output to not print successful jobs stdout to email, print when failed +output=$( + "${script_dir}"/../venv/bin/python3 \ + "${script_dir}"/../src/python/CMSMonitoring/rucio_quotas.py "${py_input_args[@]}" 2>&1 +) +ec=$? +if [ $ec -ne 0 ]; then + echo "$output" - exit code: $ec + exit $ec +fi diff --git a/docker/rucio-quotas/src/rucio_quotas.py b/docker/rucio-quotas/src/rucio_quotas.py new file mode 100644 index 00000000..48c46ca5 --- /dev/null +++ b/docker/rucio-quotas/src/rucio_quotas.py @@ -0,0 +1,79 @@ +# !/usr/bin/env python +# -*- coding: utf-8 -*- +# Author: Benedikt Maier +# Create html table for Rucio RSE quotas +import click +import os +from datetime import datetime + +import pandas as pd +from rucio.client import Client + + +@click.command() +@click.option("--output", required=True, help="For example: /eos/.../www/test/test.html") +@click.option("--template_dir", required=True, + help="Html directory for main html template. For example: ~/CMSMonitoring/src/html/rucio_quotas") +def main(output=None, template_dir=None): + client = Client() + + # DISK, no tape + # rse_type=DISK or TAPE + # cms_type=real or test + # tier<3, exclude T3 + # tier>0, exclude T0, + # \(T2_US_Caltech_Ceph|T2_PL_Warsaw) (exclude) + + RSE_EXPRESSION = r"rse_type=DISK&cms_type=real&tier<3&tier>0\(T2_US_Caltech_Ceph|T2_PL_Warsaw)" + + rses = list(client.list_rses(rse_expression=RSE_EXPRESSION)) + rse_names = [name["rse"] for name in rses] + rse_names.sort() + + static_space = [] + used_space = [] + fraction_space = [] + free_space = [] + rse_names_final = [] + + for rse in rse_names: + site_usage = list(client.get_rse_usage(rse)) + sources = {el["source"]: el["used"] for el in site_usage if el["source"] in ["static", "rucio"]} + if len(sources) == 2: # checking if both rucio and static info exists + static_space.append(sources["static"] * 1e-12) + used_space.append(sources["rucio"] * 1e-12) + fraction_space.append(100. * used_space[-1] / static_space[-1]) + free_space.append(static_space[-1] - used_space[-1]) + rse_names_final.append(rse) + + # print(f"rse_names: {rse_names_final},\nlen: {len(rse_names)}\n") + # print(f"fraction_space: {fraction_space},\nlen: {len(fraction_space)}\n") + # print(f"free_space: {free_space},\nlen: {len(free_space)}\n") + # print(f"static_space: {static_space},\nlen: {len(static_space)}\n") + # print(f"used_space: {used_space},\nlen: {len(used_space)}\n") + + data = {'RSE': rse_names_final, ' Used space ': used_space, ' Total space ': static_space, + ' Fraction used (%) ': fraction_space, ' Free space ': free_space} + + df = pd.DataFrame.from_dict(data=data).round(1) + # df.style.set_properties(subset=[' Used space '], **{'width': '300px'}) + html = df.to_html(escape=False, index=False, col_space='150px') + html = html.replace( + 'table border="1" class="dataframe"', + 'table id="dataframe" class="display compact" style="width:100%;"', + ) + html = html.replace('style="text-align: right;"', "") + + with open(os.path.join(template_dir, "htmltemplate.html")) as f: + htm_template = f.read() + + current_date = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + main_html = htm_template.replace("XXX", current_date) + main_html = main_html.replace("____MAIN_BLOCK____", html) + + with open(output, "w+") as f: + f.write(main_html) + + +if __name__ == "__main__": + main() diff --git a/docker/rucio-quotas/src/rucio_quotas_html/htmltemplate.html b/docker/rucio-quotas/src/rucio_quotas_html/htmltemplate.html new file mode 100644 index 00000000..81ccba1a --- /dev/null +++ b/docker/rucio-quotas/src/rucio_quotas_html/htmltemplate.html @@ -0,0 +1,133 @@ + + + + + + + + + + + +
+
+ CMS +

+ CMS Rucio quotas and usage monitoring +

+ Last Update: XXX +
+
+ +
+ + + ____MAIN_BLOCK____ + +
+ + + + + + + + + + + + + + + + +