Skip to content

Commit e6f9494

Browse files
JohTJohT
committed
Add input parameter for source repository cloning
1 parent b0983a6 commit e6f9494

File tree

5 files changed

+313
-18
lines changed

5 files changed

+313
-18
lines changed

‎.github/workflows/internal-java-code-analysis.yml‎

Lines changed: 7 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,9 @@ jobs:
4646
runs-on: ubuntu-latest
4747
outputs:
4848
analysis-name: ${{ steps.set-analysis-name.outputs.analysis-name }}
49-
sources-upload-name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
5049
artifacts-upload-name: ${{ steps.set-artifacts-upload-name.outputs.artifacts-upload-name }}
5150
additional-maven-artifacts: ${{ steps.set-additional-maven-artifacts.outputs.additional-maven-artifacts }}
51+
source-repository-branch: ${{ steps.set-source-repository-branch.outputs.source-repository-branch }}
5252

5353
env:
5454
PROJECT_NAME: AxonFramework
@@ -87,10 +87,6 @@ jobs:
8787
- name: (Prepare Code to Analyze) Generate ARTIFACT_UPLOAD_ID
8888
run: echo "ARTIFACT_UPLOAD_ID=$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 10)" >> $GITHUB_ENV
8989

90-
- name: (Prepare Code to Analyze) Set sources-upload-name
91-
id: set-sources-upload-name
92-
run: echo "sources-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-sources_input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
93-
9490
- name: (Prepare Code to Analyze) Set output variable 'artifacts-upload-name'
9591
id: set-artifacts-upload-name
9692
run: echo "artifacts-upload-name=${{ steps.set-analysis-name.outputs.analysis-name }}-analysis-artifacts-input-${{ env.ARTIFACT_UPLOAD_ID }}" >> "$GITHUB_OUTPUT"
@@ -99,15 +95,9 @@ jobs:
9995
id: set-additional-maven-artifacts
10096
run: echo "additional-maven-artifacts=org.axonframework:axon-messaging:${{ env.AXON_FRAMEWORK_VERSION }},org.axonframework:axon-modelling:${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"
10197

102-
- name: (Prepare Code to Analyze) Upload sources to analyze
103-
if: success()
104-
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
105-
with:
106-
name: ${{ steps.set-sources-upload-name.outputs.sources-upload-name }}
107-
path: ./temp/${{ steps.set-analysis-name.outputs.analysis-name }}/source
108-
include-hidden-files: true
109-
if-no-files-found: error
110-
retention-days: 1
98+
- name: (Prepare Code to Analyze) Set output variable 'source-repository-branch'
99+
id: set-source-repository-branch
100+
run: echo "source-repository-branch=axon-${{ env.AXON_FRAMEWORK_VERSION }}" >> "$GITHUB_OUTPUT"
111101

112102
- name: (Prepare Code to Analyze) Upload artifacts to analyze
113103
if: success()
@@ -130,5 +120,7 @@ jobs:
130120
artifacts-upload-name: ${{ needs.prepare-code-to-analyze.outputs.artifacts-upload-name }}
131121
# Additional (duplicate) artifacts are only used here to test maven-artifacts parameter.
132122
maven-artifacts: ${{needs.prepare-code-to-analyze.outputs.additional-maven-artifacts}}
133-
sources-upload-name: ${{ needs.prepare-code-to-analyze.outputs.sources-upload-name }}
123+
source-repository: https://github.com/AxonFramework/AxonFramework.git
124+
source-repository-branch: ${{ needs.prepare-code-to-analyze.outputs.source-repository-branch}}
125+
source-repository-history-only: true
134126
jupyter-pdf: "false"

‎.github/workflows/public-analyze-code-graph.yml‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,28 @@ on:
3535
required: false
3636
type: string
3737
default: ''
38+
source-repository:
39+
description: >
40+
The URL of the source repository to analyze. For now, only GitHub repositories are supported.
41+
This can be used instead of 'sources-upload-name' to directly analyze a repository without uploading artifacts first.
42+
It can also be used in addition to 'sources-upload-name' to analyze both uploaded sources and a repository.
43+
required: false
44+
type: string
45+
default: ''
46+
source-repository-branch:
47+
description: >
48+
The branch, tag or SHA of the source repository to checkout.
49+
Default: default branch of the repository
50+
required: false
51+
type: string
52+
default: ''
53+
source-repository-history-only:
54+
description: >
55+
Whether to clone the source repository as a bare repository ("true") or not ("false", default).
56+
Bare repositories do not have a working directory and are useful for git history analysis when the sources are not needed.
57+
required: false
58+
type: string
59+
default: 'false'
3860
ref:
3961
description: >
4062
The branch, tag or SHA of the code-graph-analysis-pipeline to checkout.
@@ -95,9 +117,9 @@ jobs:
95117
python: 3.12
96118
miniforge: 24.9.0-0
97119
steps:
98-
- name: Assure that either artifacts-upload-name or maven-artifacts or sources-upload-name is set
99-
if: inputs.artifacts-upload-name == '' && inputs.maven-artifacts == '' && inputs.sources-upload-name == ''
100-
run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'maven-artifacts' or 'sources-upload-name'."; exit 1
120+
- name: Assure that either artifacts-upload-name or maven-artifacts or sources-upload-name or source-repository is set
121+
if: inputs.artifacts-upload-name == '' && inputs.maven-artifacts == '' && inputs.sources-upload-name == '' && inputs.source-repository == ''
122+
run: echo "Please specify either the input parameter 'artifacts-upload-name' or 'maven-artifacts' or 'sources-upload-name' or 'source-repository'."; exit 1
101123
- name: Verify analysis-name only consists of characters safe for folder names
102124
run: |
103125
if [[ ! "${{ inputs.analysis-name }}" =~ ^[A-Za-z0-9._-]+$ ]]; then
@@ -176,6 +198,11 @@ jobs:
176198
name: ${{ inputs.sources-upload-name }}
177199
path: temp/${{ inputs.analysis-name }}/source/${{ inputs.analysis-name }}
178200

201+
- name: (Code Analysis Setup) Clone source repository for analysis
202+
if: inputs.source-repository != ''
203+
working-directory: temp/${{ inputs.analysis-name }}
204+
run: ./../../scripts/cloneGitRepository.sh --url "${{ inputs.source-repository }}" --branch "${{ inputs.source-repository-branch }}" --history-only "${{ inputs.source-repository-history-only }}" --target "source/${{ inputs.analysis-name }}"
205+
179206
- name: (Code Analysis Setup) Download artifacts for analysis
180207
if: inputs.artifacts-upload-name != ''
181208
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5

‎INTEGRATION.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ The workflow parameters are as follows:
3737
- **sources-upload-name**: The name of the sources uploaded with [actions/upload-artifact](https://github.com/actions/upload-artifact/tree/65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08) containing the content of the 'source' directory for the analysis. It also supports sub-folders for multiple source code bases. This parameter is optional and defaults to an empty string.
3838
Please use 'include-hidden-files: true' if you also want to upload the git history.
3939
- **ref**: The branch, tag, or SHA of the code-graph-analysis-pipeline to checkout. This parameter is optional and defaults to "main".
40+
- **source-repository**: The URL of the source code repository to analyze. This parameter is optional and defaults to an empty string. If provided, it will be used to clone the repository for analysis instead of using the uploaded source code artifact. Currently. only public GitHub repositories are supported.
41+
- **source-repository-branch**: The branch of the source code repository to analyze. This parameter is optional and defaults to "main". It is only used if the 'source-repository' parameter is provided.
42+
- **source-repository-history-only**: If set to 'true', only the git history of the source code repository will be cloned for analysis. This parameter is optional and defaults to 'false'. It is only used if the 'source-repository' parameter is provided.
4043
- **analysis-arguments**: The arguments to pass to the analysis script. This parameter is optional and defaults to '--profile Neo4j-latest-low-memory'. You can find all available options in section [Command Line Options of COMMANDS.md/](./COMMANDS.md#command-line-options).
4144
- **typescript-scan-heap-memory**: The heap memory size in MB to use for the TypeScript code scans. This value is only used for the TypeScript code scans and is ignored for other scans. This parameter is optional and defaults to '4096'. It will set the environment variable `TYPESCRIPT_SCAN_HEAP_MEMORY` which leads to `NODE_OPTIONS` set to `--max-old-space-size=4096` for TypeScript scans. See [Questions and Answers of README.md](./README.md#thinking-questions--answers) for more information.
4245

‎scripts/cloneGitRepository.sh‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/usr/bin/env bash
2+
3+
# Provides safe-guarded (security checked parameters) git repository cloning.
4+
5+
# Note: This script needs the path to target directory to clone the git repository to. It defaults to SOURCE_DIRECTORY ("source").
6+
# Note: This script needs git to be installed.
7+
8+
# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
9+
set -o errexit -o pipefail
10+
11+
# Overrideable Defaults
12+
SOURCE_DIRECTORY=${SOURCE_DIRECTORY:-"source"} # Get the source repository directory (defaults to "source")
13+
14+
# Local constants
15+
SCRIPT_NAME=$(basename "${0}")
16+
17+
fail() {
18+
local ERROR_COLOR='\033[0;31m' # red
19+
local DEFAULT_COLOR='\033[0m'
20+
local errorMessage="${1}"
21+
echo -e "${ERROR_COLOR}${SCRIPT_NAME}: Error: ${errorMessage}${DEFAULT_COLOR}" >&2
22+
exit 1
23+
}
24+
25+
# Default and initial values for command line options
26+
url=""
27+
branch="main"
28+
history_only="false"
29+
target="${SOURCE_DIRECTORY}"
30+
dry_run="false"
31+
32+
# Read command line options
33+
USAGE="${SCRIPT_NAME}: Usage: $0 --url <github-repository-url> --branch <branch-name> [--history-only <true|false>] [--target <clone directory>(default=source)]"
34+
35+
while [ "$#" -gt "0" ]; do
36+
key="$1"
37+
case ${key} in
38+
--url)
39+
url="$2"
40+
shift
41+
;;
42+
--branch)
43+
branch="$2"
44+
shift
45+
;;
46+
--history-only)
47+
history_only="$2"
48+
shift
49+
;;
50+
--target)
51+
target="$2"
52+
shift
53+
;;
54+
--dry-run)
55+
dry_run="true"
56+
;;
57+
*)
58+
fail "Unknown option: ${key}"
59+
echo "${USAGE}" >&2
60+
exit 1
61+
esac
62+
shift
63+
done
64+
65+
# --- Validate URL (mandatory)
66+
if [ -z "${url}" ] ; then
67+
fail "The git repository URL (--url) must be provided."
68+
echo "${USAGE}" >&2
69+
exit 1
70+
fi
71+
case "${url}" in
72+
https://github.com/*/*|https://github.com/*/*.git)
73+
;;
74+
*)
75+
fail "The source repository (--url) must be a valid GitHub repository URL."
76+
;;
77+
esac
78+
79+
# --- Validate branch (mandatory, defaults to "main")
80+
if [ -z "${branch}" ] ; then
81+
fail "The git repository branch (--branch) must be provided."
82+
echo "${USAGE}" >&2
83+
exit 1
84+
fi
85+
case "${branch}" in
86+
*[\ ~^:?*[\]\\]*)
87+
fail "The source repository branch contains invalid characters."
88+
;;
89+
esac
90+
91+
# --- Validate history-only (mandatory, defaults to "false")
92+
case "${history_only}" in
93+
true|false)
94+
;;
95+
*)
96+
fail "The source repository history-only option must be either 'true' or 'false'."
97+
echo "${USAGE}" >&2
98+
;;
99+
esac
100+
101+
# --- Validate target directory (mandatory, defaults to SOURCE_DIRECTORY)
102+
if [ -z "${target}" ] ; then
103+
fail "The target directory (--target) ${target} must be provided." >&2
104+
echo "${USAGE}" >&2
105+
exit 1
106+
else
107+
mkdir -p "${target}"
108+
fi
109+
110+
if [ ${dry_run} = "true" ] ; then
111+
echo "Dry run mode enabled. The following command(s) would be executed:" >&2
112+
fi
113+
114+
# --- Clone the git repository
115+
bare_option=""
116+
bare_folder=""
117+
if [ "${history_only}" = "true" ]; then
118+
bare_option="--bare"
119+
bare_folder="/.git" # bare clones need the .git folder to be used as target
120+
fi
121+
122+
if [ ${dry_run} = "true" ] ; then
123+
echo "git clone ${bare_option} --single-branch ${url} --branch ${branch} ${target}${bare_folder}"
124+
exit 0
125+
else
126+
git clone ${bare_option} --single-branch "${url}" --branch "${branch}" "${target}${bare_folder}"
127+
fi

‎scripts/testCloneGitRepository.sh‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
#!/usr/bin/env bash
2+
3+
# Tests "cloneGitRepository.sh".
4+
5+
# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
6+
set -o errexit -o pipefail
7+
8+
# Local constants
9+
SCRIPT_NAME=$(basename "${0}")
10+
COLOR_ERROR='\033[0;31m' # red
11+
COLOR_DE_EMPHASIZED='\033[0;90m' # dark gray
12+
COLOR_SUCCESSFUL="\033[0;32m" # green
13+
COLOR_DEFAULT='\033[0m'
14+
15+
## Get this "scripts" directory if not already set
16+
# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution.
17+
# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
18+
# This way non-standard tools like readlink aren't needed.
19+
SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
20+
21+
tearDown() {
22+
# echo "${SCRIPT_NAME}: Tear down tests...."
23+
rm -rf "${temporaryTestDirectory}"
24+
}
25+
26+
successful() {
27+
echo ""
28+
echo -e "${COLOR_DE_EMPHASIZED}${SCRIPT_NAME}:${COLOR_DEFAULT} ${COLOR_SUCCESSFUL}✅ Tests finished successfully.${COLOR_DEFAULT}"
29+
tearDown
30+
}
31+
32+
info() {
33+
local infoMessage="${1}"
34+
echo -e "${COLOR_DE_EMPHASIZED}${SCRIPT_NAME}:${COLOR_DEFAULT} ${infoMessage}"
35+
}
36+
37+
fail() {
38+
local errorMessage="${1}"
39+
echo -e "${COLOR_DE_EMPHASIZED}${SCRIPT_NAME}: ${COLOR_ERROR}${errorMessage}${COLOR_DEFAULT}"
40+
tearDown
41+
return 1
42+
}
43+
44+
printTestLogFileContent() {
45+
local logFileName="${temporaryTestDirectory}/${SCRIPT_NAME}-${test_case_number}.log"
46+
if [ -f "${logFileName}" ]; then
47+
local logFileContent=$( cat "${logFileName}" )
48+
# Remove color codes from the output for better readability in test logs
49+
logFileContent=$(echo -e "${logFileContent}" | sed -r "s/\x1B\[[0-9;]*[mK]//g")
50+
echo -e "${COLOR_DE_EMPHASIZED}${logFileContent}${COLOR_DEFAULT}"
51+
else
52+
echo -e "${COLOR_ERROR}No log file found at expected location: ${logFileName}${COLOR_DEFAULT}"
53+
fi
54+
}
55+
56+
cloneGitRepositoryExpectingSuccessUnderTest() {
57+
local COLOR_DE_EMPHASIZED='\033[0;90m' # dark gray
58+
(
59+
cd "${temporaryTestDirectory}";
60+
source "${SCRIPTS_DIR}/cloneGitRepository.sh" "$@" > "${temporaryTestDirectory}/${SCRIPT_NAME}-${test_case_number}.log"
61+
)
62+
exitCode=$?
63+
if [ ${exitCode} -ne 0 ]; then
64+
fail "❌ Test failed: Script exited with non-zero exit code ${exitCode}."
65+
fi
66+
printTestLogFileContent
67+
}
68+
69+
cloneGitRepositoryExpectingFailureUnderTest() {
70+
set +o errexit
71+
(
72+
cd "${temporaryTestDirectory}";
73+
source "${SCRIPTS_DIR}/cloneGitRepository.sh" "$@" > "${temporaryTestDirectory}/${SCRIPT_NAME}-${test_case_number}.log" 2>&1
74+
exitCode=$?
75+
if [ ${exitCode} -eq 0 ]; then
76+
fail "❌ Test failed: Script exited with zero exit code but was expected to fail."
77+
fi
78+
)
79+
set -o errexit
80+
printTestLogFileContent
81+
}
82+
83+
info "Starting tests...."
84+
85+
# Create testing resources
86+
temporaryTestDirectory=$(mktemp -d 2>/dev/null || mktemp -d -t 'temporaryTestDirectory_${SCRIPT_NAME}')
87+
mkdir -p "${temporaryTestDirectory}"
88+
89+
# ------- Integration Test Case
90+
test_case_number=1
91+
echo ""
92+
info "${test_case_number}.) Should clone a valid GitHub Repository successfully (real-run/integration)."
93+
94+
cloneGitRepositoryExpectingSuccessUnderTest --url "https://github.com/JohT/livecoding.git" --branch "main" --target "${temporaryTestDirectory}/livecoding"
95+
if [ ! -f "${temporaryTestDirectory}/livecoding/README.md" ]; then
96+
fail "${test_case_number}.) Test failed: Expected 'README.md' in cloned repository 'livecoding'."
97+
fi
98+
99+
# ------- Unit Test Case
100+
test_case_number=2
101+
echo ""
102+
info "${test_case_number}.) Should fail when an unknown option is used (dry-run)."
103+
cloneGitRepositoryExpectingFailureUnderTest --non-existing-parameter --dry-run
104+
105+
# ------- Unit Test Case
106+
test_case_number=3
107+
echo ""
108+
info "${test_case_number}.) Should fail when --url is from a different domain than GitHub (dry-run)."
109+
cloneGitRepositoryExpectingFailureUnderTest --url "https://example.com/JohT/livecoding.git" --dry-run
110+
111+
# ------- Unit Test Case
112+
test_case_number=4
113+
echo ""
114+
info "${test_case_number}.) Should fail when --branch is empty (dry-run)."
115+
cloneGitRepositoryExpectingFailureUnderTest --url "https://github.com/JohT/livecoding.git" --branch "" --dry-run
116+
117+
# ------- Unit Test Case
118+
test_case_number=5
119+
echo ""
120+
info "${test_case_number}.) Should fail when --branch contains invalid characters (dry-run)."
121+
cloneGitRepositoryExpectingFailureUnderTest --url "https://github.com/JohT/livecoding.git" --branch "main;" --dry-run
122+
123+
# ------- Unit Test Case
124+
test_case_number=6
125+
echo ""
126+
info "${test_case_number}.) Should fail when --history-only is neither true nor false (dry-run)."
127+
cloneGitRepositoryExpectingFailureUnderTest --url "https://github.com/JohT/livecoding.git" --history-only "invalid" --dry-run
128+
129+
# ------- Unit Test Case
130+
test_case_number=7
131+
echo ""
132+
info "${test_case_number}.) Should fail when --target is empty (dry-run)."
133+
cloneGitRepositoryExpectingFailureUnderTest --url "https://github.com/JohT/livecoding.git" --target "" --dry-run
134+
135+
# ------- Unit Test Case
136+
test_case_number=8
137+
echo ""
138+
info "${test_case_number}.) Should include the bare option in git clone when --history-option is true (dry-run)."
139+
output=$(cloneGitRepositoryExpectingSuccessUnderTest --url "https://github.com/JohT/livecoding.git" --history-only "true" --target "${temporaryTestDirectory}/livecoding" --dry-run)
140+
141+
if ! echo "${output}" | grep -q "git clone --bare"; then
142+
fail "${test_case_number}.) Test failed: Expected '--bare' option in git clone command."
143+
fi
144+
145+
successful
146+
return 0

0 commit comments

Comments
 (0)