Create rucio-quotas Docker image

docker/rucio-quotas/Dockerfile

@@ -0,0 +1,4 @@
+FROM registry.cern.ch/cmsmonitoring/cmsmon-py:test
+
+# Copy only the specific files you need instead of cloning entire repos
+COPY src/ ./

docker/rucio-quotas/README.md

@@ -0,0 +1,3 @@
+# Rucio Quotas Docker Image
+
+Calculates the quotas of Rucio storage elements (RSE), i.e., sites, in TB. It runs daily and fills the [CMS Rucio Quotas and usage monitoring](https://cmsdatapop.web.cern.ch/cmsdatapop/rucio/quotas.html) website of the CMS Data Popularity service.

docker/rucio-quotas/src/cron4rucio_quotas.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -e
+TZ=UTC
+myname=$(basename "$0")
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+# Get nice util functions
+. "${script_dir}"/utils.sh
+
+# ---------------------------------------------------------------------------------------------------------- Run in K8S
+if [ -n "$K8S_ENV" ]; then
+    # $1: output, shift is mandatory because rucio/setup-py3.sh also waits for $1
+    output_=$1; shift
+
+    util4logi "${myname} is starting.."
+    util_cron_send_start "$myname" "1h"
+
+    # Rucio API setup
+    export X509_USER_PROXY=/etc/proxy/proxy
+    export RUCIO_HOME=/cvmfs/cms.cern.ch/rucio/x86_64/rhel9/py3/current
+    util_kerberos_auth_with_keytab /etc/secrets/keytab
+    python3 "${script_dir}"/rucio_quotas.py \
+        --output "$output_" \
+        --template_dir "${script_dir}/rucio_quotas_html" 2>&1
+
+    util_cron_send_end "$myname" "1h" "$?"
+    util4logi "${myname} successfully finished."
+    exit 0
+    # break
+fi
+# Run in LxPlus for test ----------------------------------------------------------------------------------------------
+
+source /cvmfs/cms.cern.ch/cmsset_default.sh >/dev/null
+source /cvmfs/cms.cern.ch/rucio/setup-py3.sh >/dev/null
+output=$(voms-proxy-init -voms cms -rfc -valid 192:00 2>&1)
+ec=$?
+if [ $ec -ne 0 ]; then
+    echo "$output" - exit code: $ec
+    exit $ec
+fi
+
+py_input_args=(
+    --output "/eos/user/c/cmsmonit/www/rucio/quotas.html"
+    --template_dir "${script_dir}/../src/html/rucio_quotas"
+)
+
+# Catch	output to not print successful jobs stdout to email, print when failed
+output=$(
+    "${script_dir}"/../venv/bin/python3 \
+        "${script_dir}"/../src/python/CMSMonitoring/rucio_quotas.py "${py_input_args[@]}" 2>&1
+)
+ec=$?
+if [ $ec -ne 0 ]; then
+    echo "$output" - exit code: $ec
+    exit $ec
+fi

docker/rucio-quotas/src/rucio_quotas.py

@@ -0,0 +1,79 @@
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Author: Benedikt Maier <benedikt [DOT] maier AT cern [DOT] ch>
+# Create html table for Rucio RSE quotas
+import click
+import os
+from datetime import datetime
+
+import pandas as pd
+from rucio.client import Client
+
+
+@click.command()
+@click.option("--output", required=True, help="For example: /eos/.../www/test/test.html")
+@click.option("--template_dir", required=True,
+              help="Html directory for main html template. For example: ~/CMSMonitoring/src/html/rucio_quotas")
+def main(output=None, template_dir=None):
+    client = Client()
+
+    # DISK, no tape
+    # rse_type=DISK or TAPE
+    # cms_type=real or test
+    # tier<3, exclude T3
+    # tier>0, exclude T0,
+    # \(T2_US_Caltech_Ceph|T2_PL_Warsaw) (exclude)
+
+    RSE_EXPRESSION = r"rse_type=DISK&cms_type=real&tier<3&tier>0\(T2_US_Caltech_Ceph|T2_PL_Warsaw)"
+
+    rses = list(client.list_rses(rse_expression=RSE_EXPRESSION))
+    rse_names = [name["rse"] for name in rses]
+    rse_names.sort()
+
+    static_space = []
+    used_space = []
+    fraction_space = []
+    free_space = []
+    rse_names_final = []
+
+    for rse in rse_names:
+        site_usage = list(client.get_rse_usage(rse))
+        sources = {el["source"]: el["used"] for el in site_usage if el["source"] in ["static", "rucio"]}
+        if len(sources) == 2:  # checking if both rucio and static info exists
+            static_space.append(sources["static"] * 1e-12)
+            used_space.append(sources["rucio"] * 1e-12)
+            fraction_space.append(100. * used_space[-1] / static_space[-1])
+            free_space.append(static_space[-1] - used_space[-1])
+            rse_names_final.append(rse)
+
+    # print(f"rse_names: {rse_names_final},\nlen: {len(rse_names)}\n")
+    # print(f"fraction_space: {fraction_space},\nlen: {len(fraction_space)}\n")
+    # print(f"free_space: {free_space},\nlen: {len(free_space)}\n")
+    # print(f"static_space: {static_space},\nlen: {len(static_space)}\n")
+    # print(f"used_space: {used_space},\nlen: {len(used_space)}\n")
+
+    data = {'RSE': rse_names_final, '   Used space   ': used_space, '   Total space   ': static_space,
+            '   Fraction used (%)   ': fraction_space, '   Free space   ': free_space}
+
+    df = pd.DataFrame.from_dict(data=data).round(1)
+    # df.style.set_properties(subset=['   Used space   '], **{'width': '300px'})
+    html = df.to_html(escape=False, index=False, col_space='150px')
+    html = html.replace(
+        'table border="1" class="dataframe"',
+        'table id="dataframe" class="display compact" style="width:100%;"',
+    )
+    html = html.replace('style="text-align: right;"', "")
+
+    with open(os.path.join(template_dir, "htmltemplate.html")) as f:
+        htm_template = f.read()
+
+    current_date = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+    main_html = htm_template.replace("XXX", current_date)
+    main_html = main_html.replace("____MAIN_BLOCK____", html)
+
+    with open(output, "w+") as f:
+        f.write(main_html)
+
+
+if __name__ == "__main__":
+    main()

docker/rucio-quotas/src/rucio_quotas_html/htmltemplate.html

@@ -0,0 +1,133 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="https://cdn.datatables.net/1.10.20/css/jquery.dataTables.min.css">
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
+    <link rel="stylesheet" href="https://cdn.datatables.net/1.11.4/css/dataTables.bootstrap.min.css">
+    <link rel="stylesheet" href="https://cdn.datatables.net/buttons/2.2.2/css/buttons.bootstrap.min.css">
+    <style>
+        body {
+	        font-family: 'Trebuchet MS', sans-serif;
+        }
+        /* Search bar */
+        .dataTables_filter input {
+          border: 7px solid Tomato;
+          width: 400px;
+          font-size: 16px;
+          font-weight: bold;
+        }
+        table td {
+            word-break: break-all;
+        }
+        /* From 2nd column, align to right */
+        table td:nth-child(n+2) {
+            text-align: right;
+        }
+        /* First row bold */
+        table td:nth-child(1) {
+            font-weight: bold;
+        }
+        /* Different background color for even and odd columns  */
+        #dataframe tr:nth-child(even) {
+          /* background-color: #dddfff; */
+        }
+        /* No carriage return for values, no break lines */
+        #dataframe tr td {
+          width: 1%;
+          white-space: nowrap;
+        }
+        /* button */
+        div.dt-buttons {
+          float: right;
+        }
+    </style>
+</head>
+<body>
+  <div class="container" style="display:block; width:70%">
+    <div class="cms">
+        <img src="https://cds.cern.ch/record/1306150/files/cmsLogo_image.jpg"
+            alt="CMS" style="width: 5%; float:left">
+        <h3 style="width: 100%;">
+            CMS Rucio quotas and usage monitoring
+        </h3>
+        <small>Last Update: XXX</small>
+    </div>
+    <div class="w3-container" style="margin-left: 3%;">
+      <button style="font-size: 18px; background-color: white; color: black; border: 5px solid #f9ccac;" onclick="explainFunction()">
+        &darr; How to interpret this table &darr;
+      </button>
+    </div>
+    <div id="explanations" style="display: none; margin-top: 2%;">
+        <pre>
+    Source code: <a href="https://github.com/dmwm/CMSMonitoring/blob/master/src/python/CMSMonitoring/rucio_quotas.py">CMSMonitoring/rucio_quotas.py</a>
+    Source code: <a href="https://github.com/dmwm/CMSMonitoring/blob/master/src/html/rucio_quotas/htmltemplate.html">htmltemplate.html</a>
+    This page shows the quotas of Rucio storage elements (RSE), i.e., sites, in TB.
+        </pre>
+	</div>
+
+  ____MAIN_BLOCK____
+
+  </div>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
+    <script type="text/javascript" src="https://code.jquery.com/jquery-3.5.1.js"></script>
+    <script type="text/javascript" src="https://cdn.datatables.net/1.11.4/js/jquery.dataTables.min.js"></script>
+    <script type="text/javascript" src="https://cdn.datatables.net/1.11.4/js/dataTables.bootstrap.min.js"></script>
+    <script type="text/javascript" src="https://cdn.datatables.net/buttons/2.2.2/js/dataTables.buttons.min.js"></script>
+    <script type="text/javascript" src="https://cdn.datatables.net/buttons/2.2.2/js/buttons.bootstrap.min.js"></script>
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.1.3/jszip.min.js"></script>
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.1.53/pdfmake.min.js"></script>
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.1.53/vfs_fonts.js"></script>
+    <script type="text/javascript" src="https://cdn.datatables.net/buttons/2.2.2/js/buttons.html5.min.js"></script>
+    <script type="text/javascript" src="https://cdn.datatables.net/buttons/2.2.2/js/buttons.print.min.js"></script>
+    <script type="text/javascript" src="https://cdn.datatables.net/buttons/2.2.2/js/buttons.colVis.min.js"></script>
+    <script>
+        function explainFunction() {
+          var x = document.getElementById("explanations");
+          if (x.style.display === "none") {
+            x.style.display = "block";
+          } else {
+            x.style.display = "none";
+          }
+        }
+    </script>
+    <script>
+        var url = new URL(window.location.href);
+        var searchString = url.searchParams.get("search");
+        if (searchString == null){
+          searchString = ''
+        }
+        $(document).ready(function () {
+            var dt = $('#dataframe').DataTable( {
+                "orderCellsTop": true,
+                "dom": "fBrtpli",
+                "order": [[ 0, "asc" ]],
+                "pageLength" : 300,
+                "scrollX": false,
+                "oSearch": { "sSearch": searchString },
+                language: {
+                    search: "_INPUT_",
+                    searchPlaceholder: "--- Search RSE name ---",
+                },
+                lengthChange: false,
+                  buttons: [
+                  'copy',
+                  'excel',
+                  'pdf',
+                  'colvis',
+                  {
+                        text: 'Copy search link to clipboard',
+                        action: function ( e, dt, node, config ) {
+                          url.searchParams.set('search', dt.search());
+                          //window.location.replace(url.href);
+                          navigator.clipboard.writeText(url.href);
+                        }
+                    }
+                ]
+            });
+            dt.buttons().container().appendTo( '#example_wrapper .col-sm-4:eq(0)' );
+        });
+    </script>
+</body>
+<!-- final -->
+</html>