Skip to content

Commit b555d25

Browse files
Grishma123-Engadivinho
Grishma123-Eng
and
adivinho
authored
adding tests for trivy and updating minitest setup (#3666)
Co-authored-by: Vadim Yalovets <vadim.yalovets@percona.com>
1 parent 10858a2 commit b555d25

File tree

2 files changed

+355
-43
lines changed

2 files changed

+355
-43
lines changed

‎ps/jenkins/percona-server-for-mysql-8.0.groovy‎

Lines changed: 221 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -157,41 +157,54 @@ def installDependencies(def nodeName) {
157157
}
158158

159159
def runPlaybook(def nodeName) {
160-
161-
try {
162-
def playbook = "ps_lts_innovation.yml"
163-
def playbook_path = "package-testing/playbooks/${playbook}"
164-
165-
sh '''
166-
set -xe
167-
git clone --depth 1 https://github.com/Percona-QA/package-testing
168-
'''
169-
sh """
170-
set -xe
171-
export install_repo="\${install_repo}"
172-
export client_to_test="ps80"
173-
export check_warning="\${check_warnings}"
174-
export install_mysql_shell="\${install_mysql_shell}"
175-
ansible-playbook \
176-
--connection=local \
177-
--inventory 127.0.0.1, \
178-
--limit 127.0.0.1 \
179-
${playbook_path}
180-
"""
181-
} catch (Exception e) {
182-
slackNotify("${SLACKNOTIFY}", "#FF0000", "[${JOB_NAME}]: Mini Package Testing for ${nodeName} at ${BRANCH} FAILED !!!")
183-
mini_test_error="True"
160+
script {
161+
env.PS_RELEASE = sh(returnStdout: true, script: "echo ${BRANCH} | sed 's/release-//g'").trim()
162+
echo "PS_RELEASE : ${env.PS_RELEASE}"
163+
env.PS_VERSION_SHORT_KEY= sh(script: """echo ${PS_RELEASE} | awk -F'.' '{print \$1 \".\" \$2}'""", returnStdout: true).trim()
164+
echo "Version is : ${env.PS_VERSION_SHORT_KEY}"
165+
env.PS_VERSION_SHORT = "PS${env.PS_VERSION_SHORT_KEY.replace('.', '')}"
166+
echo "Value is : ${env.PS_VERSION_SHORT}"
167+
echo "Using PS_VERSION_SHORT in another function: ${env.PS_VERSION_SHORT}"
168+
def playbook
169+
if (env.PS_VERSION_SHORT == 'PS80') {
170+
playbook = "ps_80.yml"
171+
} else {
172+
playbook = "ps_84.yml"
173+
}
174+
def client_to_test = PS_VERSION_SHORT
175+
def playbook_path = "package-testing/playbooks/${playbook}"
176+
sh '''
177+
set -xe
178+
git clone --depth 1 https://github.com/Percona-QA/package-testing
179+
'''
180+
def exitCode = sh(
181+
script: """
182+
set -xe
183+
export install_repo="\${install_repo}"
184+
export client_to_test="ps80"
185+
export check_warning="\${check_warnings}"
186+
export install_mysql_shell="${env.INSTALL_MYSQL_SHELL}"
187+
ansible-playbook \
188+
--connection=local \
189+
--inventory 127.0.0.1, \
190+
--limit 127.0.0.1 \
191+
${playbook_path}
192+
""",
193+
returnStatus: true
194+
)
195+
if (exitCode != 0) {
196+
error "Ansible playbook failed on ${nodeName} with exit code ${exitCode}"
197+
}
198+
}
184199
}
185-
}
186-
187200
def minitestNodes = [ "min-bullseye-x64",
188201
"min-bookworm-x64",
189202
"min-ol-8-x64",
190-
"min-focal-x64",
191203
"min-amazon-2-x64",
192204
"min-jammy-x64",
193205
"min-noble-x64",
194-
"min-ol-9-x64" ]
206+
"min-ol-9-x64" ,
207+
]
195208

196209
def package_tests_ps80(def nodes) {
197210
def stepsForParallel = [:]
@@ -208,20 +221,170 @@ def package_tests_ps80(def nodes) {
208221
}
209222
parallel stepsForParallel
210223
}
224+
def docker_test() {
225+
def stepsForParallel = [:]
226+
stepsForParallel['Run for ARM64'] = {
227+
node('docker-32gb-aarch64') {
228+
stage("Docker tests for ARM64") {
229+
script{
230+
sh '''
231+
echo "running test for ARM"
232+
export DOCKER_PLATFORM=linux/arm64
233+
# disable THP on the host for TokuDB
234+
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" > disable_thp.sh
235+
echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> disable_thp.sh
236+
chmod +x disable_thp.sh
237+
sudo ./disable_thp.sh
238+
# run test
239+
export PATH=${PATH}:~/.local/bin
240+
sudo yum install -y python3 python3-pip
241+
rm -rf package-testing
242+
git clone https://github.com/Percona-QA/package-testing.git --depth 1
243+
cd package-testing/docker-image-tests/ps-arm
244+
pip3 install --user -r requirements.txt
245+
export PS_VERSION="${PS_RELEASE}-arm64"
246+
export PS_REVISION="${PS_REVISION}"
247+
export DOCKER_ACC="${DOCKER_ACC}"
248+
echo "printing variables: \$DOCKER_ACC , \$PS_VERSION , \$PS_REVISION "
249+
./run.sh
250+
'''
251+
}
252+
}
253+
stage('Docker image version check for ARM64'){
254+
script{
255+
sh '''
256+
export PS_VERSION="${PS_RELEASE}-arm64"
257+
fetched_docker_version=$(docker run -i --rm -e MYSQL_ROOT_PASSWORD=asdasd ${DOCKER_ACC}/percona-server:${PS_VERSION} \
258+
bash -c "mysql --version" | awk '{print $3}')
259+
echo "fetching docker version: \$fetched_docker_version"
260+
if [[ "$PS_RELEASE" == "$fetched_docker_version" ]]; then
261+
echo "Run succesfully for arm"
262+
else
263+
echo "Failed for arm"
264+
fi
265+
'''
266+
}
267+
}
268+
stage('Run trivy analyzer ARM64') {
269+
script{
270+
sh """
271+
sudo yum install -y curl wget git
272+
TRIVY_VERSION=\$(curl --silent 'https://api.github.com/repos/aquasecurity/trivy/releases/latest' | grep '"tag_name":' | tr -d '"' | sed -E 's/.*v(.+),.*/\\1/')
273+
ARCH=\$(uname -m)
274+
if [[ "\$ARCH" == "aarch64" ]]; then
275+
ARCH_NAME="ARM64"
276+
elif [[ "\$ARCH" == "x86_64" ]]; then
277+
ARCH_NAME="64bit"
278+
else
279+
echo "Unsupported architecture: \$ARCH"
280+
exit 1
281+
fi
282+
echo "Detected architecture: \$ARCH, using Trivy for Linux-\$ARCH_NAME"
283+
wget https://github.com/aquasecurity/trivy/releases/download/v\${TRIVY_VERSION}/trivy_\${TRIVY_VERSION}_Linux-\${ARCH_NAME}.tar.gz
284+
sudo tar zxvf trivy_\${TRIVY_VERSION}_Linux-\${ARCH_NAME}.tar.gz -C /usr/local/bin/
285+
wget https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/junit.tpl
286+
/usr/local/bin/trivy image --format template --template @junit.tpl -o trivy-hight-junit.xml \
287+
--timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL ${DOCKER_ACC}/percona-server:${PS_RELEASE}-arm64 || true
288+
echo "Ran succesfully for arm"
289+
"""
290+
}
291+
}
292+
}
293+
}
294+
stepsForParallel['Run for AMD'] = {
295+
node ( 'docker' ) {
296+
stage("Docker image version check for AMD") {
297+
script {
298+
sh '''
299+
echo "running the test for AMD"
300+
# disable THP on the host for TokuDB
301+
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" > disable_thp.sh
302+
echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> disable_thp.sh
303+
chmod +x disable_thp.sh
304+
sudo ./disable_thp.sh
305+
# run test
306+
export PATH=${PATH}:~/.local/bin
307+
sudo yum install -y python3 python3-pip
308+
rm -rf package-testing
309+
git clone https://github.com/Percona-QA/package-testing.git --depth 1
310+
cd package-testing/docker-image-tests/ps
311+
pip3 install --user -r requirements.txt
312+
export PS_VERSION="${PS_RELEASE}-amd64"
313+
export PS_REVISION="${PS_REVISION}"
314+
export DOCKER_ACC="${DOCKER_ACC}"
315+
echo "printing variables: \$DOCKER_ACC , \$PS_VERSION , \$PS_REVISION "
316+
./run.sh
317+
'''
318+
}
319+
}
320+
stage ("Docker image version check for amd64") {
321+
script{
322+
sh '''
323+
export PS_VERSION="${PS_RELEASE}-amd64"
324+
fetched_docker_version=$(docker run -i --rm -e MYSQL_ROOT_PASSWORD=asdasd ${DOCKER_ACC}/percona-server:${PS_VERSION} \
325+
bash -c "mysql --version" | awk '{print $3}')
326+
echo "fetching docker version: \$fetched_docker_version"
327+
if [[ "$PS_RELEASE" == "$fetched_docker_version" ]]; then
328+
echo "Run succesfully for amd"
329+
else
330+
echo "Failed for amd"
331+
fi
332+
'''
333+
}
334+
}
335+
stage ('Run trivy analyzer for AMD') {
336+
script {
337+
sh """
338+
sudo yum install -y curl wget git
339+
TRIVY_VERSION=\$(curl --silent 'https://api.github.com/repos/aquasecurity/trivy/releases/latest' | grep '"tag_name":' | tr -d '"' | sed -E 's/.*v(.+),.*/\\1/')
340+
wget https://github.com/aquasecurity/trivy/releases/download/v\${TRIVY_VERSION}/trivy_\${TRIVY_VERSION}_Linux-64bit.tar.gz
341+
sudo tar zxvf trivy_\${TRIVY_VERSION}_Linux-64bit.tar.gz -C /usr/local/bin/
342+
wget https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/junit.tpl
343+
/usr/local/bin/trivy image --format template --template @junit.tpl -o trivy-hight-junit.xml \
344+
--timeout 10m0s --ignore-unfixed --exit-code 1 --severity HIGH,CRITICAL ${DOCKER_ACC}/percona-server:${PS_RELEASE}-amd64 || true
345+
echo "ran succesfully for amd docker trivy"
346+
"""
347+
}
348+
}
349+
}
350+
}
351+
parallel stepsForParallel
352+
}
211353

212354
@Field def mini_test_error = "False"
213355
def AWS_STASH_PATH
214-
def PS8_RELEASE_VERSION
215-
def product_to_test = 'innovation-lts'
356+
def product_to_test = ''
216357
def install_repo = 'testing'
217358
def action_to_test = 'install'
218359
def check_warnings = 'yes'
219360
def install_mysql_shell = 'no'
361+
def BRANCH_NAME = env.BRANCH ?: "release-8.0.43-34"
362+
def PS_RELEASE = BRANCH_NAME.replaceAll("release-", "")
363+
def PS_VERSION_SHORT_KEY = PS_RELEASE.tokenize('.')[0..1].join('.')
364+
def PS_VERSION_SHORT = "PS${PS_VERSION_SHORT_KEY.replace('.', '')}"
365+
def DOCKER_ACC = "perconalab"
366+
product_to_test = (PS_VERSION_SHORT == 'PS84') ? 'ps_84' : 'ps_80'
367+
env.PS_RELEASE = PS_RELEASE
368+
env.PS_VERSION_SHORT_KEY = PS_VERSION_SHORT_KEY
369+
env.PS_VERSION_SHORT = PS_VERSION_SHORT
370+
env.DOCKER_ACC = DOCKER_ACC
371+
env.product_to_test = product_to_test
372+
373+
void notifyBuildSuccess() {
374+
if (env.FIPSMODE == 'YES') {
375+
slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: PRO -> build finished successfully for ${BRANCH} - [${BUILD_URL}]")
376+
} else {
377+
slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: build finished successfully for ${BRANCH} - [${BUILD_URL}]")
378+
}
379+
}
220380

221381
pipeline {
222382
agent {
223383
label params.CLOUD == 'Hetzner' ? 'docker-x64-min' : 'docker'
224384
}
385+
environment {
386+
INSTALL_MYSQL_SHELL = "${params.install_mysql_shell ?: 'no'}"
387+
}
225388
parameters {
226389
choice(
227390
choices: [ 'Hetzner','AWS' ],
@@ -263,7 +426,6 @@ parameters {
263426
timestamps ()
264427
}
265428
stages {
266-
267429
stage('Create PS source tarball') {
268430
agent {
269431
label params.CLOUD == 'Hetzner' ? 'deb12-x64' : 'min-focal-x64'
@@ -296,7 +458,7 @@ parameters {
296458
pushArtifactFolder(params.CLOUD, "source_tarball/", AWS_STASH_PATH)
297459
uploadTarballfromAWS(params.CLOUD, "source_tarball/", AWS_STASH_PATH, 'source')
298460
}
299-
}
461+
}
300462
stage('Build PS generic source packages') {
301463
parallel {
302464
stage('Build PS generic source rpm') {
@@ -1015,27 +1177,43 @@ parameters {
10151177
wait: false
10161178
}
10171179
}
1018-
}
1019-
}
1180+
}
1181+
}
10201182
post {
10211183
success {
10221184
script {
1023-
if (env.FIPSMODE == 'YES') {
1024-
slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: PRO -> build has been finished successfully for ${BRANCH} - [${BUILD_URL}]")
1185+
notifyBuildSuccess()
1186+
unstash 'properties'
1187+
// Extract PS_REVISION from properties file
1188+
def PS_REVISION = ''
1189+
if (fileExists('test/percona-server-8.0.properties')) {
1190+
PS_REVISION = sh(returnStdout: true, script: "grep REVISION test/percona-server-8.0.properties | awk -F '=' '{ print\$2 }'").trim()
1191+
echo "PS_REVISION extracted: ${PS_REVISION}"
1192+
env.PS_REVISION = PS_REVISION
10251193
} else {
1026-
slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: build has been finished successfully for ${BRANCH} - [${BUILD_URL}]")
1194+
error "Properties file not found: test/percona-server-8.0.properties"
10271195
}
1196+
MinitestPostSucess(
1197+
product_to_test: product_to_test,
1198+
PS_RELEASE: PS_RELEASE,
1199+
PS_VERSION_SHORT: PS_VERSION_SHORT,
1200+
PS_VERSION_SHORT_KEY: PS_VERSION_SHORT_KEY,
1201+
minitestNodes: minitestNodes,
1202+
SLACKNOTIFY: SLACKNOTIFY,
1203+
BRANCH: BRANCH,
1204+
DOCKER_ACC: DOCKER_ACC,
1205+
packageTestsClosure: { nodes -> package_tests_ps80(nodes) },
1206+
dockerTestClosure: { -> docker_test() }
1207+
)
10281208
}
1029-
slackNotify("${SLACKNOTIFY}", "#00FF00", "[${JOB_NAME}]: Triggering Builds for Package Testing for ${BRANCH} - [${BUILD_URL}]")
1030-
deleteDir()
10311209
}
1210+
10321211
failure {
10331212
deleteDir()
10341213
}
1214+
10351215
always {
1036-
sh '''
1037-
sudo rm -rf ./*
1038-
'''
1216+
sh 'sudo rm -rf ./*'
10391217
script {
10401218
if (env.FIPSMODE == 'YES') {
10411219
currentBuild.description = "Pro -> Build on ${BRANCH}"
@@ -1046,4 +1224,4 @@ parameters {
10461224
deleteDir()
10471225
}
10481226
}
1049-
}
1227+
}

0 commit comments

Comments
 (0)