From 105706e4230970d105c9b1b584b70b4e1de11217 Mon Sep 17 00:00:00 2001 From: Felix Exner Date: Fri, 9 May 2025 10:43:28 +0200 Subject: [PATCH 1/5] Check links using lychee --- .github/workflows/check_links.yml | 42 +++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 20 +++++++-------- 2 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/check_links.yml diff --git a/.github/workflows/check_links.yml b/.github/workflows/check_links.yml new file mode 100644 index 00000000..bc0fcee4 --- /dev/null +++ b/.github/workflows/check_links.yml @@ -0,0 +1,42 @@ +name: Check Links +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + +jobs: + check_links: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Restore lychee cache + id: restore-cache + uses: actions/cache/restore@v4 + with: + path: .lycheecache + key: cache-lychee-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: cache-lychee- + - name: Link Checker + id: lychee + uses: lycheeverse/lychee-action@v2 + with: + fail: true + args: > + --verbose + --no-progress + --cache + --cache-exclude-status 429 + --max-cache-age 2d + './**/*.md' './**/*.html' './**/*.rst' './**/*.cpp' './**/*.h' './**/*.py' + --exclude '^http://192\.168\.56\.101' + --exclude 'kernel\.org\/pub\/linux\/kernel' + --exclude 'releases/download/v\$%7BURCAP_VERSION%7D/externalcontrol-\$%7BURCAP_VERSION%7D\.jar' + --threads 1 + - name: Save lychee cache + uses: actions/cache/save@v4 + if: always() + with: + path: .lycheecache + key: ${{ steps.restore-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78a00955..e9296a7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,13 +127,13 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} flags: start_ursim - check_links: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Check URLs - run: | - .github/helpers/check_urls.sh \ - -d ".git build CMakeModules debian" \ - -f "package.xml urcl_architecture.svg trajectory_interface.svg dataflow.graphml start_ursim.sh ursim_docker.rst" \ - -p "vnc\.html opensource\.org\/licenses\/BSD-3-Clause kernel\.org\/pub\/linux\/kernel" + #check_links: + #runs-on: ubuntu-latest + #steps: + #- uses: actions/checkout@v4 + #- name: Check URLs + #run: | + #.github/helpers/check_urls.sh \ + #-d ".git build CMakeModules debian" \ + #-f "package.xml urcl_architecture.svg trajectory_interface.svg dataflow.graphml start_ursim.sh ursim_docker.rst" \ + #-p "vnc\.html opensource\.org\/licenses\/BSD-3-Clause kernel\.org\/pub\/linux\/kernel" From b3e0db5912234cf996051274a1c34d8560d8b09e Mon Sep 17 00:00:00 2001 From: Felix Exner Date: Fri, 9 May 2025 11:01:41 +0200 Subject: [PATCH 2/5] Do not cache most common errors --- .github/workflows/check_links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_links.yml b/.github/workflows/check_links.yml index bc0fcee4..c4f20756 100644 --- a/.github/workflows/check_links.yml +++ b/.github/workflows/check_links.yml @@ -27,7 +27,7 @@ jobs: --verbose --no-progress --cache - --cache-exclude-status 429 + --cache-exclude-status '429, 403, 404, 502' --max-cache-age 2d './**/*.md' './**/*.html' './**/*.rst' './**/*.cpp' './**/*.h' './**/*.py' --exclude '^http://192\.168\.56\.101' From 507ff218348fbbc6245703848147b7809ec6f27b Mon Sep 17 00:00:00 2001 From: Felix Exner Date: Mon, 12 May 2025 10:03:34 +0200 Subject: [PATCH 3/5] Attempt to reduce 429s on GitHub --- .github/workflows/check_links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_links.yml b/.github/workflows/check_links.yml index c4f20756..eecf026b 100644 --- a/.github/workflows/check_links.yml +++ b/.github/workflows/check_links.yml @@ -33,7 +33,7 @@ jobs: --exclude '^http://192\.168\.56\.101' --exclude 'kernel\.org\/pub\/linux\/kernel' --exclude 'releases/download/v\$%7BURCAP_VERSION%7D/externalcontrol-\$%7BURCAP_VERSION%7D\.jar' - --threads 1 + --max-concurrency 1 - name: Save lychee cache uses: actions/cache/save@v4 if: always() From f25f994285464832894f75a87d7ccc2b62bcf394 Mon Sep 17 00:00:00 2001 From: Felix Exner Date: Wed, 14 May 2025 11:41:20 +0200 Subject: [PATCH 4/5] Remove old link checker --- .github/helpers/check_urls.sh | 71 ----------------------------------- .github/workflows/ci.yml | 11 ------ 2 files changed, 82 deletions(-) delete mode 100755 .github/helpers/check_urls.sh diff --git a/.github/helpers/check_urls.sh b/.github/helpers/check_urls.sh deleted file mode 100755 index d3aa1861..00000000 --- a/.github/helpers/check_urls.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -set -e - -IGNORE_FILES="" -IGNORE_PATTERNS="" - -while getopts ":f:d:p:" opt; do - case "${opt}" in - f) - IGNORE_FILES="${OPTARG}";; - d) - IGNORE_DIRS="${OPTARG}";; - p) - IGNORE_PATTERNS="${OPTARG}";; - \?) - echo "Invalid option -$OPTARG" - exit;; - esac -done - -read -r -a ignore_files <<<"$IGNORE_FILES" -read -r -a ignore_dirs <<<"$IGNORE_DIRS" -read -r -a ignore_patterns <<<"$IGNORE_PATTERNS" - -IGNORE_FILES_ARG="" -for item in "${ignore_files[@]}"; do - IGNORE_FILES_ARG="$IGNORE_FILES_ARG --exclude=$item" -done -IGNORE_DIRS_ARG="" -for item in "${ignore_dirs[@]}"; do - IGNORE_DIRS_ARG="$IGNORE_DIRS_ARG --exclude-dir=$item" -done - -#Find URLs in code: -urls=$(grep -oP '(http|ftp|https):\/\/([a-zA-Z0-9_-]+(?:(?:\.[a-zA-Z0-9_-]+)+))([a-zA-Z0-9_.,@?^=%&:\/~+#-]*[a-zA-Z0-9_@?^=%&\/~+#-])?' -rI $IGNORE_FILES_ARG $IGNORE_DIRS_ARG) - -fail_counter=0 - -FAILED_LINKS=() -CHECKED_LINKS=() -for item in $urls; do -# echo $item - skip=0 - for pattern in "${ignore_patterns[@]}"; do - [[ "$item" =~ $pattern ]] && skip=1 - done - - if [[ $skip == 1 ]]; then - echo "SKIPPING $item" - continue - fi - - filename=$(echo "$item" | cut -d':' -f1) - url=$(echo "$item" | cut -d':' -f2-) - echo -n "Checking $url from file $filename" - if [[ $(echo ${CHECKED_LINKS[@]} | fgrep -w $url) ]]; then - echo -e " \033[36malready checked\033[0m" - elif ! curl --head --silent --fail "$url" 2>&1 > /dev/null; then - echo -e " \033[0;31mNOT FOUND\033[0m" - FAILED_LINKS+=("$url from file $filename") - ((fail_counter=fail_counter+1)) - else - echo -e " \033[32mok\033[0m" - fi - CHECKED_LINKS+=("$url") -done - -echo "Failed files:" -printf '%s\n' "${FAILED_LINKS[@]}" -exit $fail_counter diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9296a7d..e37421bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,14 +126,3 @@ jobs: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} flags: start_ursim - - #check_links: - #runs-on: ubuntu-latest - #steps: - #- uses: actions/checkout@v4 - #- name: Check URLs - #run: | - #.github/helpers/check_urls.sh \ - #-d ".git build CMakeModules debian" \ - #-f "package.xml urcl_architecture.svg trajectory_interface.svg dataflow.graphml start_ursim.sh ursim_docker.rst" \ - #-p "vnc\.html opensource\.org\/licenses\/BSD-3-Clause kernel\.org\/pub\/linux\/kernel" From 45768df51fed96527d6ffeab2e10d97683929034 Mon Sep 17 00:00:00 2001 From: Felix Exner Date: Wed, 14 May 2025 11:42:09 +0200 Subject: [PATCH 5/5] Reorder command arguments --- .github/workflows/check_links.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_links.yml b/.github/workflows/check_links.yml index eecf026b..ca9f549b 100644 --- a/.github/workflows/check_links.yml +++ b/.github/workflows/check_links.yml @@ -29,11 +29,11 @@ jobs: --cache --cache-exclude-status '429, 403, 404, 502' --max-cache-age 2d - './**/*.md' './**/*.html' './**/*.rst' './**/*.cpp' './**/*.h' './**/*.py' --exclude '^http://192\.168\.56\.101' --exclude 'kernel\.org\/pub\/linux\/kernel' --exclude 'releases/download/v\$%7BURCAP_VERSION%7D/externalcontrol-\$%7BURCAP_VERSION%7D\.jar' --max-concurrency 1 + './**/*.md' './**/*.html' './**/*.rst' './**/*.cpp' './**/*.h' './**/*.py' - name: Save lychee cache uses: actions/cache/save@v4 if: always()