Merge remote-tracking branch 'origin/6.0/stage' into 6.0/release

Author: Delphix User

.gitignore

@@ -1,2 +1,3 @@
 tmp/
 /artifacts/
+.idea

buildpkg.sh

@@ -39,6 +39,7 @@ function usage() {
 	echo "    -c  also run package's checkstyle hook."
 	echo "    -r  override default revision for package."
 	echo "    -h  display this message and exit."
+	echo "    -l  use locally-built dependencies instead of s3 versions."
 	echo ""
 	exit 2
 }
@@ -48,13 +49,15 @@ unset PARAM_PACKAGE_GIT_BRANCH
 unset PARAM_PACKAGE_REVISION
 
 do_checkstyle=false
-while getopts ':b:cg:hr:' c; do
+source="s3"
+while getopts ':b:cg:hlr:' c; do
 	case "$c" in
 	g) export PARAM_PACKAGE_GIT_URL="$OPTARG" ;;
 	b) export PARAM_PACKAGE_GIT_BRANCH="$OPTARG" ;;
 	r) export PARAM_PACKAGE_REVISION="$OPTARG" ;;
 	c) do_checkstyle=true ;;
 	h) usage >&2 ;;
+	l) source="local" ;;
 	*) usage "illegal option -- $OPTARG" >&2 ;;
 	esac
 done
@@ -86,7 +89,7 @@ logmust cd "$WORKDIR"
 stage fetch
 
 logmust cd "$WORKDIR"
-stage fetch_dependencies
+stage fetch_dependencies $source
 
 logmust cd "$WORKDIR"
 stage prepare

default-package-config.sh

@@ -155,6 +155,14 @@ function kernel_build() {
 	local build_deps_tool="apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes"
 	logmust sudo mk-build-deps --install debian/control --tool "${build_deps_tool}"
 
+	#
+	# Here we update the configs used to control the kernel's build
+	# system. This is useful as it allows us to override various
+	# kernel config options via an OVERRIDES file, which we use to
+	# disable varous kernel modules that we don't need or want.
+	#
+	logmust fakeroot debian/rules updateconfigs "${debian_rules_args[@]}"
+
 	logmust fakeroot debian/rules "binary" "${debian_rules_args[@]}"
 
 	logmust cd "$WORKDIR"

lib/common.sh

@@ -214,14 +214,15 @@ function check_git_ref() {
 #
 function stage() {
 	typeset hook=$1
+	shift 1
 
 	check_env PACKAGE
 	local stage_start=$SECONDS
 
 	echo ""
 	if type -t "$hook" >/dev/null; then
 		echo_bold "PACKAGE $PACKAGE: STAGE $hook STARTED"
-		logmust "$hook"
+		logmust "$hook" "$@"
 		echo_bold "PACKAGE $PACKAGE: STAGE $hook COMPLETED in" \
 			"$((SECONDS - stage_start)) seconds"
 	else
@@ -574,8 +575,16 @@ function list_linux_kernel_packages() {
 
 function install_shfmt() {
 	if [[ ! -f /usr/local/bin/shfmt ]]; then
+		local arch
+		arch=$(dpkg-architecture -q DEB_HOST_ARCH)
+
+		# The release names for shfmt don't use the actual
+		# architecture strings, unfortunately.
+		if [[ "$arch" == "arm64" ]]; then
+			arch="arm"
+		fi
 		logmust sudo wget -nv -O /usr/local/bin/shfmt \
-			https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_amd64
+			https://github.com/mvdan/sh/releases/download/v2.4.0/shfmt_v2.4.0_linux_$arch
 		logmust sudo chmod +x /usr/local/bin/shfmt
 	fi
 	echo "shfmt version $(shfmt -version) is installed."
@@ -699,6 +708,7 @@ function get_package_dependency_s3_url() {
 # is defined in the package's config.
 #
 function fetch_dependencies() {
+	local source="$1"
 	export DEPDIR="$WORKDIR/dependencies"
 	logmust mkdir "$DEPDIR"
 	logmust cd "$DEPDIR"
@@ -712,22 +722,31 @@ function fetch_dependencies() {
 	for dep in $PACKAGE_DEPENDENCIES; do
 		echo "Fetching artifacts for dependency '$dep' ..."
 		get_package_prefix "$dep"
-		s3urlvar="${_RET}_S3_URL"
-		if [[ -n "${!s3urlvar}" ]]; then
-			s3url="${!s3urlvar}"
-			echo "S3 URL of package dependency '$dep' provided" \
-				"externally"
-			echo "$s3urlvar=$s3url"
-		else
-			logmust get_package_dependency_s3_url "$dep"
-			s3url="$_RET"
-		fi
-		[[ "$s3url" != */ ]] && s3url="$s3url/"
-		logmust mkdir "$dep"
-		logmust aws s3 ls "$s3url"
-		logmust aws s3 cp --only-show-errors --recursive "$s3url" "$dep/"
-		echo_bold "Fetched artifacts for '$dep' from $s3url"
-		PACKAGE_DEPENDENCIES_METADATA="${PACKAGE_DEPENDENCIES_METADATA}$dep: $s3url\\n"
+		case "$source" in
+		"local")
+			logmust cp -r "$WORKDIR/../../$dep/tmp/artifacts/ $dep/"
+			;;
+		"s3")
+			s3urlvar="${_RET}_S3_URL"
+			if [[ -n "${!s3urlvar}" ]]; then
+				s3url="${!s3urlvar}"
+				echo "S3 URL of package dependency '$dep' provided externally"
+				echo "$s3urlvar=$s3url"
+			else
+				logmust get_package_dependency_s3_url "$dep"
+				s3url="$_RET"
+			fi
+			[[ "$s3url" != */ ]] && s3url="$s3url/"
+			logmust mkdir "$dep"
+			logmust aws s3 ls "$s3url"
+			logmust aws s3 cp --only-show-errors --recursive "$s3url" "$dep/"
+			echo_bold "Fetched artifacts for '$dep' from $s3url"
+			PACKAGE_DEPENDENCIES_METADATA="${PACKAGE_DEPENDENCIES_METADATA}$dep: $s3url\\n"
+			;;
+		*)
+			die "invalid source parameter specified: '$source'"
+			;;
+		esac
 	done
 }
 

packages/adoptopenjdk/config.sh

@@ -19,21 +19,36 @@
 DEFAULT_PACKAGE_GIT_URL=none
 PACKAGE_DEPENDENCIES="make-jpkg"
 
-if [[ "$UPSTREAM_PRODUCT_BRANCH" == "master" ]]; then
-	_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u342b07.tar.gz"
-	_tarfile_sha256="8252c0e11d542ea0f9ce6b8f147d1a2bea4a17e9fc299da270288f5a4f35b1f3"
+case $(dpkg-architecture -q DEB_HOST_ARCH 2>/dev/null || echo "none") in
+amd64)
+	_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u345b01.tar.gz"
+	_tarfile_sha256="ed6c9db3719895584fb1fd69fc79c29240977675f26631911c5a1dbce07b7d58"
 	_jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
-else
-	_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u332b09.tar.gz"
-	_tarfile_sha256="adc13a0a0540d77f0a3481b48f10d61eb203e5ad4914507d489c2de3bd3d83da"
-	_jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
-fi
+	;;
+arm64)
+	_tarfile="OpenJDK8U-jdk_aarch64_linux_hotspot_8u345b01.tar.gz"
+	_tarfile_sha256="c1965fb24dded7d7944e2da36cd902adf3b7b1d327aaa21ea507cff00a5a0090"
+	_jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-arm64"
+	;;
+*) ;;
+
+esac
 
 function prepare() {
 	logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb
 }
 
 function fetch() {
+	# We exit here rather than above in the architecture detection logic
+	# to deal with the fact that this file can, during test runs, be
+	# sourced on platforms where builds are not happening. list-packages
+	# sources the file to gather information about the package, and this
+	# is performed on jenkins and macos during test runs. Having the exit
+	# occur above causes those runs to fail.
+	if [[ -z "$_tarfile" ]]; then
+		echo "Invalid architecture detected" >&2
+		exit 1
+	fi
 	logmust cd "$WORKDIR/"
 
 	local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile"
@@ -42,6 +57,10 @@ function fetch() {
 }
 
 function build() {
+	if [[ -z "$_tarfile" ]]; then
+		echo "Invalid architecture detected" >&2
+		exit 1
+	fi
 	logmust cd "$WORKDIR/"
 
 	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$_tarfile" <<<y

packages/challenge-response/config.sh

@@ -28,5 +28,5 @@ function prepare() {
 function build() {
 	logmust cd "$WORKDIR/repo/challenge_response"
 	logmust make package
-	logmust mv ./x86_64/*deb "$WORKDIR/artifacts/"
+	logmust mv "./$(dpkg-architecture -q DEB_HOST_GNU_CPU)"/*deb "$WORKDIR/artifacts/"
 }

packages/containerized-masking/config.sh

@@ -40,6 +40,32 @@ function build() {
 
 	logmust cd "$WORKDIR/repo"
 
+	if [[ "$SECRET_DB_AWS_ENDPOINT" ]]; then
+		export SECRET_DB_AWS_ENDPOINT="$SECRET_DB_AWS_ENDPOINT"
+	fi
+
+	# Using secrets proxy
+	if [[ "$SECRET_DB_USE_JUMPBOX" ]]; then
+		export SECRET_DB_USE_JUMPBOX="$SECRET_DB_USE_JUMPBOX"
+	fi
+	if [[ "$SECRET_DB_JUMP_BOX_HOST" ]]; then
+		export SECRET_DB_JUMP_BOX_HOST="$SECRET_DB_JUMP_BOX_HOST"
+	fi
+	if [[ "$SECRET_DB_JUMP_BOX_USER" ]]; then
+		export SECRET_DB_JUMP_BOX_USER="$SECRET_DB_JUMP_BOX_USER"
+	fi
+	if [[ "$SECRET_DB_JUMP_BOX_PRIVATE_KEY" ]]; then
+		export SECRET_DB_JUMP_BOX_PRIVATE_KEY="$SECRET_DB_JUMP_BOX_PRIVATE_KEY"
+	fi
+
+	# Using master/eng-secret-user
+	if [[ "$SECRET_DB_AWS_PROFILE" ]]; then
+		export SECRET_DB_AWS_PROFILE="$SECRET_DB_AWS_PROFILE"
+	fi
+	if [[ "$SECRET_DB_AWS_REGION" ]]; then
+		export SECRET_DB_AWS_REGION="$SECRET_DB_AWS_REGION"
+	fi
+
 	logmust ./gradlew --no-daemon --stacktrace \
 		-Porg.gradle.configureondemand=false \
 		-PenvironmentName=linuxappliance \

packages/masking/config.sh

@@ -43,6 +43,32 @@ function build() {
 		'{ "dms-core-gate" : { "git-hash" : $h, "date": $d }}' \
 		>"$WORKDIR/artifacts/metadata.json"
 
+	if [[ "$SECRET_DB_AWS_ENDPOINT" ]]; then
+		export SECRET_DB_AWS_ENDPOINT="$SECRET_DB_AWS_ENDPOINT"
+	fi
+
+	# Using secrets proxy
+	if [[ "$SECRET_DB_USE_JUMPBOX" ]]; then
+		export SECRET_DB_USE_JUMPBOX="$SECRET_DB_USE_JUMPBOX"
+	fi
+	if [[ "$SECRET_DB_JUMP_BOX_HOST" ]]; then
+		export SECRET_DB_JUMP_BOX_HOST="$SECRET_DB_JUMP_BOX_HOST"
+	fi
+	if [[ "$SECRET_DB_JUMP_BOX_USER" ]]; then
+		export SECRET_DB_JUMP_BOX_USER="$SECRET_DB_JUMP_BOX_USER"
+	fi
+	if [[ "$SECRET_DB_JUMP_BOX_PRIVATE_KEY" ]]; then
+		export SECRET_DB_JUMP_BOX_PRIVATE_KEY="$SECRET_DB_JUMP_BOX_PRIVATE_KEY"
+	fi
+
+	# Using master/eng-secret-user
+	if [[ "$SECRET_DB_AWS_PROFILE" ]]; then
+		export SECRET_DB_AWS_PROFILE="$SECRET_DB_AWS_PROFILE"
+	fi
+	if [[ "$SECRET_DB_AWS_REGION" ]]; then
+		export SECRET_DB_AWS_REGION="$SECRET_DB_AWS_REGION"
+	fi
+
 	logmust ./gradlew --no-daemon --stacktrace \
 		-Porg.gradle.configureondemand=false \
 		-PenvironmentName=linuxappliance \

packages/virtualization/config.sh

@@ -56,14 +56,57 @@ function build() {
 	# Build the virtualization package
 	#
 	logmust cd "$WORKDIR/repo/appliance"
+
+	local args=()
+
+	# Here we check for whether the environment variables are set and pass them along. We check for
+	# existence instead of emptiness to avoid adding a layer of interpretation.
+
+	# We use parameter expansion in the form of ${variable+nothing} which evaluates to the variable
+	# 'nothing' if 'variable' is not set. Because 'nothing' is not defined it evaluates to "" when 'variable'
+	# is not set. So [[ "" ]] is what is actually evaluated when 'variable' is not set.
+
+	if [[ ${SECRET_DB_USE_JUMPBOX+nothing} ]]; then
+		args+=("-DSECRET_DB_USE_JUMPBOX=$SECRET_DB_USE_JUMPBOX")
+	fi
+
+	if [[ ${SECRET_DB_JUMP_BOX_HOST+nothing} ]]; then
+		args+=("-DSECRET_DB_JUMP_BOX_HOST=$SECRET_DB_JUMP_BOX_HOST")
+	fi
+
+	if [[ ${SECRET_DB_JUMP_BOX_USER+nothing} ]]; then
+		args+=("-DSECRET_DB_JUMP_BOX_USER=$SECRET_DB_JUMP_BOX_USER")
+	fi
+
+	if [[ ${SECRET_DB_JUMP_BOX_PRIVATE_KEY+nothing} ]]; then
+		if [[ ! -f "$SECRET_DB_JUMP_BOX_PRIVATE_KEY" ]]; then
+			die "Jumpbox private key not found."
+		fi
+		args+=("-DSECRET_DB_JUMP_BOX_PRIVATE_KEY=$SECRET_DB_JUMP_BOX_PRIVATE_KEY")
+	fi
+
+	if [[ ${SECRET_DB_AWS_ENDPOINT+nothing} ]]; then
+		args+=("-DSECRET_DB_AWS_ENDPOINT=$SECRET_DB_AWS_ENDPOINT")
+	fi
+
+	if [[ ${SECRET_DB_AWS_PROFILE+nothing} ]]; then
+		args+=("-DSECRET_DB_AWS_PROFILE=$SECRET_DB_AWS_PROFILE")
+	fi
+
+	if [[ ${SECRET_DB_AWS_REGION+nothing} ]]; then
+		args+=("-DSECRET_DB_AWS_REGION=$SECRET_DB_AWS_REGION")
+	fi
+
+	args+=("-Ddockerize=true")
+	args+=("-DbuildJni=true")
+
 	if [[ -n "$DELPHIX_RELEASE_VERSION" ]]; then
-		logmust ant -Ddockerize=true -DbuildJni=true \
-			-DhotfixGenDlpxVersion="$DELPHIX_RELEASE_VERSION" \
-			-Dbuild.legacy.resources.war=true all package
-	else
-		logmust ant -Ddockerize=true -DbuildJni=true all package
+		args+=("-DhotfixGenDlpxVersion=$DELPHIX_RELEASE_VERSION")
+		args+=("-Dbuild.legacy.resources.war=true")
 	fi
 
+	logmust ant "${args[@]}" all-secrets package
+
 	#
 	# Publish the virtualization package artifacts
 	#