Skip to content

Commit da49351

Browse files
TheBlueMattTheBlueMatt
committed
Add script to build deterministic android aar
1 parent aaaa51a commit da49351

File tree

4 files changed

+166
-11
lines changed

4 files changed

+166
-11
lines changed

‎.github/workflows/build.yml‎

Lines changed: 87 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ jobs:
7171
- name: Checkout latest MacOS binaries
7272
run: |
7373
export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(cat .git_ver)"
74-
git clone https://git.bitcoin.ninja/ldk-java-bins
74+
git clone --depth 1 https://git.bitcoin.ninja/ldk-java-bins
7575
mkdir -p src/main/resources/
7676
cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjni_MacOSX-"* src/main/resources/
7777
- name: Build Java/TS Release Bindings
@@ -82,7 +82,7 @@ jobs:
8282
run: ./build-release-jar.sh
8383
- name: Run Java Tests against built release jar
8484
run: |
85-
mvn install:install-file -Dfile=target/ldk-java-1.0-SNAPSHOT.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
85+
mvn install:install-file -Dfile=ldk-java.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
8686
cd javatester
8787
mvn package
8888
java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar
@@ -93,9 +93,91 @@ jobs:
9393
- name: Check latest library and jars are in bins repo
9494
run: |
9595
export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(cat .git_ver)"
96+
rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"*.jar
97+
rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"liblightningjni_Linux*.nativelib
9698
cp src/main/resources/liblightningjni_Linux-amd64.nativelib "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
97-
cp target/ldk-java-1.0-SNAPSHOT.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
98-
cp target/ldk-java-1.0-SNAPSHOT-sources.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
99+
cp ldk-java-sources.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
100+
cp ldk-java.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
101+
cp ldk-java-classes.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
102+
cd ldk-java-bins
103+
git diff --exit-code
104+
105+
android:
106+
runs-on: ubuntu-latest
107+
# Frankly, I'm not really sure why debian and ubuntu differ in the results here, they really shouldn't
108+
container: debian:bullseye
109+
env:
110+
TOOLCHAIN: stable
111+
steps:
112+
- name: Install rust targets
113+
run: |
114+
apt-get update
115+
apt-get -y dist-upgrade
116+
apt-get -y install git g++ clang faketime zip unzip curl openjdk-11-jdk
117+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rustup.sh
118+
chmod +x ./rustup.sh
119+
./rustup.sh -y
120+
. $HOME/.cargo/env
121+
rustup target add armv7-linux-androideabi
122+
rustup target add aarch64-linux-android
123+
rustup target add i686-linux-android
124+
rustup target add x86_64-linux-android
125+
- name: Checkout source code
126+
uses: actions/checkout@v2
127+
with:
128+
fetch-depth: 0
129+
- name: Install android NDK compilers
130+
run: |
131+
curl https://dl.google.com/android/repository/android-ndk-r22b-linux-x86_64.zip > android-ndk-r22b-linux-x86_64.zip
132+
if [ "$(sha256sum android-ndk-r22b-linux-x86_64.zip | awk '{ print $1 }')" != "ac3a0421e76f71dd330d0cd55f9d99b9ac864c4c034fc67e0d671d022d4e806b" ]; then
133+
echo "Bad hash"
134+
exit 1
135+
fi
136+
unzip android-ndk-r22b-linux-x86_64.zip
137+
- name: Install cbindgen
138+
run: |
139+
. $HOME/.cargo/env
140+
cargo install --force cbindgen
141+
- name: Checkout Rust-Lightning and LDK-C-Bindings git
142+
run: |
143+
git config --global user.email "ldk-ci@example.com"
144+
git config --global user.name "LDK CI"
145+
# Note this is a different endpoint, as we need one non-upstream commit!
146+
git clone https://git.bitcoin.ninja/rust-lightning
147+
cd rust-lightning
148+
git checkout origin/2021-03-java-bindings-base
149+
cd ..
150+
git clone https://github.com/lightningdevkit/ldk-c-bindings
151+
- name: Detect current git version
152+
run: |
153+
# We assume the top commit is just a bindings update commit, so we
154+
# check out the previous commit to use as the commit we git describe.
155+
# If the top commit is a merge commit, we need to get the last merge
156+
# head and assume the latest bindings are built against its parent.
157+
COMMIT_PARENTS=$(git show -s --pretty=format:%P HEAD)
158+
if [ "${#COMMIT_PARENTS}" = 40 ]; then
159+
export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag HEAD^1)"
160+
else
161+
MERGE_HEAD=$(git show --pretty=format:%P HEAD | ( for last in $(cat /dev/stdin); do true; done; echo $last ))
162+
export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(git describe --tag $MERGE_HEAD^1)"
163+
fi
164+
echo "Using $LDK_GARBAGECOLLECTED_GIT_OVERRIDE as git version"
165+
echo "$LDK_GARBAGECOLLECTED_GIT_OVERRIDE" > .git_ver
166+
- name: Checkout Android AAR binaries and artifacts
167+
run: git clone --depth 1 https://git.bitcoin.ninja/ldk-java-bins
168+
- name: Build Android aar
169+
run: |
170+
. $HOME/.cargo/env
171+
export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(cat .git_ver)"
172+
cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-classes.jar" ./
173+
export ANDROID_TOOLCHAIN="$(pwd)/android-ndk-r22b/toolchains/llvm/prebuilt/linux-x86_64"
174+
export PATH="$PATH:$ANDROID_TOOLCHAIN/bin"
175+
./android-build.sh ./rust-lightning ./ldk-c-bindings/ ./ldk-java-bins/android-artifacts
176+
- name: Check latest library and jars are in bins repo
177+
run: |
178+
export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(cat .git_ver)"
179+
rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar"
180+
cp LDK-release.aar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
99181
cd ldk-java-bins
100182
git diff --exit-code
101183
@@ -172,7 +254,7 @@ jobs:
172254
- name: Checkout latest Linux binaries
173255
run: |
174256
export LDK_GARBAGECOLLECTED_GIT_OVERRIDE="$(cat .git_ver)"
175-
git clone https://git.bitcoin.ninja/ldk-java-bins
257+
git clone --depth 1 https://git.bitcoin.ninja/ldk-java-bins
176258
mkdir -p src/main/resources/
177259
cp "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/liblightningjni_Linux-"* src/main/resources/
178260
- name: Build Java/TS Release Bindings

‎android-build.sh‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/bin/bash
2+
if [ ! -x "$ANDROID_TOOLCHAIN/bin/clang" ]; then
3+
echo "Please set ANDROID_TOOLCHAIN to the path to NDK" > /dev/stderr
4+
exit 1
5+
fi
6+
7+
if [ "$1" = "" -o ! -f "$1/lightning/Cargo.toml" ]; then
8+
echo "Please set first argument to the path to rust-lightning" > /dev/stderr
9+
exit 1
10+
fi
11+
12+
if [ "$2" = "" -o ! -d "$2/lightning-c-bindings" ]; then
13+
echo "Please set second argument to the path to ldk-c-bindings" > /dev/stderr
14+
exit 1
15+
fi
16+
17+
if [ ! -d "$3" -o ! -f "$3/AndroidManifest.xml" ]; then
18+
echo "Please set third argument to the path to ldk-java-bins/android-artifacts" > /dev/stderr
19+
exit 1
20+
fi
21+
22+
set -e
23+
set -x
24+
25+
LDK_C_BINDINGS="$(realpath $2)"
26+
RUST_LIGHTNING="$(realpath $1)"
27+
pushd "$2"
28+
export LDK_C_BINDINGS_EXTRA_TARGETS="x86_64-linux-android i686-linux-android armv7-linux-androideabi aarch64-linux-android"
29+
export LDK_C_BINDINGS_EXTRA_TARGET_CCS="x86_64-linux-android21-clang i686-linux-android21-clang armv7a-linux-androideabi21-clang aarch64-linux-android21-clang"
30+
./genbindings.sh "$RUST_LIGHTNING" true
31+
popd
32+
33+
export PATH=$PATH:$ANDROID_TOOLCHAIN/bin
34+
export SYSROOT=$ANDROID_TOOLCHAIN/sysroot/
35+
36+
# Remove any non-Android libraries installed locally
37+
rm -fr src/main/resources
38+
39+
EXTRA_TARGETS=( $LDK_C_BINDINGS_EXTRA_TARGETS )
40+
EXTRA_TARGET_CCS=( $LDK_C_BINDINGS_EXTRA_TARGET_CCS )
41+
TARGET_CPUS=( "sandybridge" "generic" "generic" "generic" )
42+
STRIPS=( "x86_64-linux-android-strip" "i686-linux-android-strip" "arm-linux-androideabi-strip" "aarch64-linux-android-strip" )
43+
for IDX in ${!EXTRA_TARGETS[@]}; do
44+
export CC="${EXTRA_TARGET_CCS[$IDX]}"
45+
export LDK_TARGET="${EXTRA_TARGETS[$IDX]}"
46+
export LDK_TARGET_CPU="${TARGET_CPUS[$IDX]}"
47+
# Note that we expect to often fail here if we don't have wasm32 bins in the C bindings dir
48+
./genbindings.sh "$LDK_C_BINDINGS" "-lm -llog -I$SYSROOT/usr/include/" false true || echo
49+
${STRIPS[$IDX]} liblightningjni_release_${LDK_TARGET}.so
50+
done
51+
52+
export LANG=C
53+
54+
echo "Need local deterministic ldk-java-classes.jar"
55+
ls ldk-java-classes.jar
56+
57+
rm -rf aar
58+
mkdir aar
59+
cp -r "$3/"* ./aar/
60+
mkdir -p ./aar/jni/{arm64,armeabi,x86,x86_64}
61+
62+
cp liblightningjni_release_aarch64-linux-android.so ./aar/jni/arm64/liblightningjni.so
63+
cp liblightningjni_release_armv7-linux-androideabi.so ./aar/jni/armeabi/liblightningjni.so
64+
cp liblightningjni_release_i686-linux-android.so ./aar/jni/x86/liblightningjni.so
65+
cp liblightningjni_release_x86_64-linux-android.so ./aar/jni/x86_64/liblightningjni.so
66+
cp ldk-java-classes.jar ./aar/classes.jar
67+
68+
rm -f LDK-release.aar
69+
cd ./aar
70+
find . | sort > ../sources-zip-files.txt
71+
touch -d "2021-01-01 00:00 UTC" $(cat ../sources-zip-files.txt)
72+
cat ../sources-zip-files.txt | zip -X@ ../LDK-release.aar
73+
cd ..
74+
rm -r aar

‎build-release-jar.sh‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,17 @@ cd ziptmp
1919
unzip ../target/ldk-java-1.0-SNAPSHOT-sources.jar | grep inflating | awk '{ print $2 }' | sort > ../sources-zip-files.txt
2020
sed -i 's/#Fri Jan 01 00:.* UTC 2021/#Fri Jan 01 00:00:00 UTC 2021/' META-INF/maven/org.ldk/ldk-java/pom.properties
2121
touch -d "2021-01-01 00:00 UTC" $(cat ../sources-zip-files.txt)
22-
cat ../sources-zip-files.txt | zip -X@ ../ldk-java-1.0-SNAPSHOT-sources-deterministic.jar
22+
cat ../sources-zip-files.txt | zip -X@ ../ldk-java-sources.jar
2323
cd ..
2424
rm -r ziptmp
25-
mv ldk-java-1.0-SNAPSHOT-sources-deterministic.jar target/ldk-java-1.0-SNAPSHOT-sources.jar
2625

2726
mkdir ziptmp
2827
cd ziptmp
2928
unzip ../target/ldk-java-1.0-SNAPSHOT.jar | grep inflating | awk '{ print $2 }' | sort > ../classes-zip-files.txt
3029
sed -i 's/#Fri Jan 01 00:.* UTC 2021/#Fri Jan 01 00:00:00 UTC 2021/' META-INF/maven/org.ldk/ldk-java/pom.properties
3130
sed -i 's/Built-By: .*/Built-By: ldk-deterministic-builds/' META-INF/MANIFEST.MF
3231
touch -d "2021-01-01 00:00 UTC" $(cat ../classes-zip-files.txt)
33-
cat ../classes-zip-files.txt | zip -X@ ../ldk-java-1.0-SNAPSHOT-deterministic.jar
32+
cat ../classes-zip-files.txt | zip -X@ ../ldk-java.jar
33+
cat ../classes-zip-files.txt | grep -v nativelib | zip -X@ ../ldk-java-classes.jar
3434
cd ..
3535
rm -r ziptmp
36-
mv ldk-java-1.0-SNAPSHOT-deterministic.jar target/ldk-java-1.0-SNAPSHOT.jar

‎genbindings.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ if [ "$3" = "true" ]; then
8585
$COMPILE -o liblightningjni_debug$LDK_TARGET_SUFFIX.so -g -fsanitize=address -shared-libasan -rdynamic -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c "$1"/lightning-c-bindings/target/$LDK_TARGET/debug/libldk.a -lm
8686
else
8787
LDK_LIB="$1"/lightning-c-bindings/target/$LDK_TARGET/release/libldk.a
88-
if [ "$IS_MAC" = "false" ]; then
88+
if [ "$IS_MAC" = "false" -a "$4" = "false" ]; then
8989
COMPILE="$COMPILE -Wl,--version-script=libcode.version -fuse-ld=lld"
9090
# __cxa_thread_atexit_impl is used to more effeciently cleanup per-thread local storage by rust libstd.
9191
# However, it is not available on glibc versions 2.17 or earlier, and rust libstd has a null-check and
@@ -128,7 +128,7 @@ else
128128
LDK_LIB="tmp/libldk.bc tmp/libldk.a"
129129
fi
130130
$COMPILE -o liblightningjni_release$LDK_TARGET_SUFFIX.so -flto -O3 -I"$1"/lightning-c-bindings/include/ $2 src/main/jni/bindings.c $LDK_LIB
131-
if [ "$IS_MAC" = "false" ]; then
131+
if [ "$IS_MAC" = "false" -a "$4" = "false" ]; then
132132
GLIBC_SYMBS="$(objdump -T liblightningjni_release$LDK_TARGET_SUFFIX.so | grep GLIBC_ | grep -v "GLIBC_2\.2\." | grep -v "GLIBC_2\.3\(\.\| \)" | grep -v "GLIBC_2.\(14\|17\) " || echo)"
133133
if [ "$GLIBC_SYMBS" != "" ]; then
134134
echo "Unexpected glibc version dependency! Some users need glibc 2.17 support, symbols for newer glibcs cannot be included."

0 commit comments

Comments
 (0)