Merge pull request #33 from TheBlueMatt/main

Expose supertrait implemetations to the Java API

Author: TheBlueMatt

.github/workflows/build.yml

@@ -10,6 +10,8 @@ jobs:
     container: debian:bullseye
     env:
       TOOLCHAIN: stable
+    strategy:
+      fail-fast: false
     steps:
       - name: Install native Rust toolchain, Valgrind, and build utilitis
         run: |
@@ -32,6 +34,8 @@ jobs:
           git checkout origin/2021-03-java-bindings-base
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
+          cd ldk-c-bindings
+          git checkout 0.0.99
       - name: Rebuild C bindings without STD for WASM
         run: |
           cd ldk-c-bindings
@@ -80,13 +84,6 @@ jobs:
           ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" false false
       - name: Build deterministic release jar
         run: ./build-release-jar.sh
-      - name: Run Java Tests against built release jar
-        run: |
-          mvn install:install-file -Dfile=ldk-java.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
-          cd javatester
-          mvn package
-          java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar
-          cd ..
       - name: Check latest headers are in git
         run: |
           git diff --exit-code
@@ -100,14 +97,31 @@ jobs:
           cp ldk-java.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
           cp ldk-java-classes.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
           cd ldk-java-bins
-          git diff --exit-code
+          if ! git diff --exit-code; then
+            mv "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"* ./
+            git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
+            apt-get -y install diffoscope
+            diffoscope ldk-java-sources.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-sources.jar"
+            diffoscope ldk-java-classes.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-classes.jar"
+            diffoscope ldk-java.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java.jar"
+            exit 1
+          fi
+      - name: Run Java Tests against built release jar
+        run: |
+          mvn install:install-file -Dfile=ldk-java.jar -DgroupId=org.ldk -DartifactId=ldk-java -Dversion=1.0-SNAPSHOT -Dpackaging=jar
+          cd javatester
+          mvn package
+          java -ea -jar target/ldk-java-tests-1.0-SNAPSHOT-jar-with-dependencies.jar
+          cd ..
 
   android:
     runs-on: ubuntu-latest
     # Frankly, I'm not really sure why debian and ubuntu differ in the results here, they really shouldn't
     container: debian:bullseye
     env:
       TOOLCHAIN: stable
+    strategy:
+      fail-fast: false
     steps:
       - name: Install rust targets
         run: |
@@ -148,6 +162,8 @@ jobs:
           git checkout origin/2021-03-java-bindings-base
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
+          cd ldk-c-bindings
+          git checkout 0.0.99
       - name: Detect current git version
         run: |
           # We assume the top commit is just a bindings update commit, so we
@@ -179,14 +195,21 @@ jobs:
           rm "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar"
           cp LDK-release.aar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
           cd ldk-java-bins
-          git diff --exit-code
+          if ! git diff --exit-code; then
+            mv "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar" ./
+            git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar"
+            apt-get -y install diffoscope
+            diffoscope LDK-release.aar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/LDK-release.aar"
+            exit 1
+          fi
 
   osx:
     strategy:
       matrix:
         include:
           - platform: macos-10.15
           - platform: macos-11
+      fail-fast: false
     runs-on: ${{ matrix.platform }}
     env:
       TOOLCHAIN: stable
@@ -213,13 +236,14 @@ jobs:
         run: |
           git config --global user.email "ldk-ci@example.com"
           git config --global user.name "LDK CI"
-          git clone https://github.com/rust-bitcoin/rust-lightning
+          # Note this is a different endpoint, as we need one non-upstream commit!
+          git clone https://git.bitcoin.ninja/rust-lightning
           cd rust-lightning
-          git remote add matt https://git.bitcoin.ninja/rust-lightning
-          git fetch matt
-          git merge matt/2021-03-java-bindings-base
+          git checkout origin/2021-03-java-bindings-base
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
+          cd ldk-c-bindings
+          git checkout 0.0.99
       - name: Rebuild C bindings with upstream clang, and check the sample app builds + links
         run: |
           cd ldk-c-bindings

bindingstypes.py

@@ -65,6 +65,8 @@ def __init__(self, ty_info, arg_name, arg_conv, arg_conv_name, arg_conv_cleanup,
         self.to_hu_conv = to_hu_conv
         self.to_hu_conv_name = to_hu_conv_name
         self.from_hu_conv = from_hu_conv
+        # This is set based on docstrings in various contexts:
+        self.nullable = False
 
 class TraitMethInfo:
     def __init__(self, fn_name, self_is_const, ret_ty_info, args_ty, docs):

gen_type_mapping.py

@@ -87,38 +87,45 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
                     ty_info.subty.requires_clone = False
                 subty = self.map_type_with_info(ty_info.subty, False, None, is_free, holds_ref)
                 arg_conv = ty_info.rust_obj + " " + arr_name + "_constr;\n"
-                arg_conv = arg_conv + arr_name + "_constr." + arr_len + " = " + self.consts.get_native_arr_len_call[0] + arr_name + self.consts.get_native_arr_len_call[1] + ";\n"
-                arg_conv = arg_conv + "if (" + arr_name + "_constr." + arr_len + " > 0)\n"
+                pf = ""
+                if ty_info.is_ptr:
+                    pf = "\t"
+                    arg_conv += ty_info.rust_obj + " *" + arr_name + "_ptr = NULL;\n"
+                    arg_conv += "if (" + self.consts.is_arr_some_check[0] + arr_name + self.consts.is_arr_some_check[1] + ") {\n"
+                arg_conv += pf + arr_name + "_constr." + arr_len + " = " + self.consts.get_native_arr_len_call[0] + arr_name + self.consts.get_native_arr_len_call[1] + ";\n"
+                arg_conv += pf + "if (" + arr_name + "_constr." + arr_len + " > 0)\n"
                 if subty.is_native_primitive:
                     szof = subty.c_ty
                 else:
                     szof = subty.rust_obj
-                arg_conv = arg_conv + "\t" + arr_name + "_constr." + ty_info.arr_access + " = MALLOC(" + arr_name + "_constr." + arr_len + " * sizeof(" + szof + "), \"" + ty_info.rust_obj + " Elements\");\n"
-                arg_conv = arg_conv + "else\n"
-                arg_conv = arg_conv + "\t" + arr_name + "_constr." + ty_info.arr_access + " = NULL;\n"
+                arg_conv += pf + "\t" + arr_name + "_constr." + ty_info.arr_access + " = MALLOC(" + arr_name + "_constr." + arr_len + " * sizeof(" + szof + "), \"" + ty_info.rust_obj + " Elements\");\n"
+                arg_conv += pf + "else\n"
+                arg_conv += pf + "\t" + arr_name + "_constr." + ty_info.arr_access + " = NULL;\n"
                 get_arr = self.consts.get_native_arr_contents(arr_name, "NO_DEST", arr_name + "_constr." + arr_len, ty_info, False)
                 if get_arr != None:
-                    arg_conv = arg_conv + subty.c_ty + "* " + arr_name + "_vals = " + get_arr + ";\n"
-                arg_conv = arg_conv + "for (size_t " + idxc + " = 0; " + idxc + " < " + arr_name + "_constr." + arr_len + "; " + idxc + "++) {\n"
+                    arg_conv += pf + subty.c_ty + "* " + arr_name + "_vals = " + get_arr + ";\n"
+                arg_conv += pf + "for (size_t " + idxc + " = 0; " + idxc + " < " + arr_name + "_constr." + arr_len + "; " + idxc + "++) {\n"
                 if get_arr != None:
-                    arg_conv = arg_conv + "\t" + subty.c_ty + " " + conv_name + " = " + arr_name + "_vals[" + idxc + "];"
+                    arg_conv += pf + "\t" + subty.c_ty + " " + conv_name + " = " + arr_name + "_vals[" + idxc + "];"
                     if subty.arg_conv is not None:
-                        arg_conv = arg_conv + "\n\t" + subty.arg_conv.replace("\n", "\n\t")
+                        arg_conv += "\n\t" + pf + subty.arg_conv.replace("\n", "\n\t" + pf)
                 else:
-                    arg_conv = arg_conv + "\t" + subty.c_ty + " " + conv_name + " = " + self.consts.get_native_arr_elem(arr_name, idxc, ty_info) + ";\n"
-                    arg_conv = arg_conv + "\t" + subty.arg_conv.replace("\n", "\n\t")
-                arg_conv = arg_conv + "\n\t" + arr_name + "_constr." + ty_info.arr_access + "[" + idxc + "] = " + subty.arg_conv_name + ";\n}"
+                    arg_conv += pf + "\t" + subty.c_ty + " " + conv_name + " = " + self.consts.get_native_arr_elem(arr_name, idxc, ty_info) + ";\n"
+                    arg_conv += pf + "\t" + subty.arg_conv.replace("\n", "\n\t" + pf)
+                arg_conv += "\n\t" + pf + arr_name + "_constr." + ty_info.arr_access + "[" + idxc + "] = " + subty.arg_conv_name + ";\n" + pf + "}"
                 if get_arr != None:
                     cleanup = self.consts.cleanup_native_arr_ref_contents(arr_name, arr_name + "_vals", arr_name + "_constr." + arr_len, ty_info)
                     if cleanup is not None:
-                        arg_conv = arg_conv + "\n" + cleanup + ";"
+                        arg_conv += "\n" + pf + cleanup + ";"
                 if ty_info.is_ptr:
-                    arg_conv_name = "&" + arr_name + "_constr"
+                    arg_conv_name = arr_name + "_ptr"
                 else:
                     arg_conv_name = arr_name + "_constr"
                 arg_conv_cleanup = None
                 if ty_info.is_ptr:
-                    arg_conv_cleanup = "FREE(" + arr_name + "_constr." + ty_info.arr_access + ");"
+                    arg_conv_cleanup = "if (" + arr_name + "_ptr != NULL) { FREE(" + arr_name + "_constr." + ty_info.arr_access + "); }"
+                if ty_info.is_ptr:
+                    arg_conv += "\n\t" + arr_name + "_ptr = &" + arr_name + "_constr;\n}"
 
                 ret_conv = (ty_info.rust_obj + " " + arr_name + "_var = ", "")
                 if subty.ret_conv is None:
@@ -159,11 +166,11 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
                 from_hu_conv = None
                 if subty.from_hu_conv is not None:
                     if subty.java_ty == "long" and subty.java_hu_ty != "long":
-                        from_hu_conv = ("Arrays.stream(" + arr_name + ").mapToLong(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray()", "/* TODO 2 " + subty.java_hu_ty + "  */")
+                        from_hu_conv = (arr_name + " != null ? Arrays.stream(" + arr_name + ").mapToLong(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray() : null", "/* TODO 2 " + subty.java_hu_ty + "  */")
                     elif subty.java_ty == "long":
-                        from_hu_conv = ("Arrays.stream(" + arr_name + ").map(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray()", "/* TODO 2 " + subty.java_hu_ty + "  */")
+                        from_hu_conv = (arr_name + " != null ? Arrays.stream(" + arr_name + ").map(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray() : null", "/* TODO 2 " + subty.java_hu_ty + "  */")
                     else:
-                        from_hu_conv = ("Arrays.stream(" + arr_name + ").map(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray(" + ty_info.java_ty + "::new)", "/* TODO 2 " + subty.java_hu_ty + "  */")
+                        from_hu_conv = (arr_name + " != null ? Arrays.stream(" + arr_name + ").map(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray(" + ty_info.java_ty + "::new) : null", "/* TODO 2 " + subty.java_hu_ty + "  */")
 
                 return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                     arg_conv = arg_conv, arg_conv_name = arg_conv_name, arg_conv_cleanup = arg_conv_cleanup,
@@ -183,10 +190,13 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
                     ret_conv_name = ty_info.var_name + "_conv",
                     to_hu_conv = None, to_hu_conv_name = None, from_hu_conv = None)
             else:
+                free_str = ""
+                if not holds_ref:
+                    free_str = "\nStr_free(" + ty_info.var_name + "_str);"
                 return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                     arg_conv = arg_conv, arg_conv_name = arg_conv_name, arg_conv_cleanup = None,
                     ret_conv = ("LDKStr " + ty_info.var_name + "_str = ",
-                        ";\njstring " + ty_info.var_name + "_conv = " + self.consts.str_ref_to_native_call(ty_info.var_name + "_str." + ty_info.arr_access, ty_info.var_name + "_str." + ty_info.arr_len) + ";"),
+                        ";\njstring " + ty_info.var_name + "_conv = " + self.consts.str_ref_to_native_call(ty_info.var_name + "_str." + ty_info.arr_access, ty_info.var_name + "_str." + ty_info.arr_len) + ";" + free_str),
                     ret_conv_name = ty_info.var_name + "_conv", to_hu_conv = None, to_hu_conv_name = None, from_hu_conv = None)
         elif ty_info.var_name == "" and not print_void:
             # We don't have a parameter name, and want one, just call it arg
@@ -402,7 +412,7 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
                         arg_conv = "", arg_conv_name = "(LDKu5){ ._0 = " + ty_info.var_name + " }", arg_conv_cleanup = None,
                         ret_conv = ("uint8_t " + ty_info.var_name + "_val = ", "._0;"), ret_conv_name = ty_info.var_name + "_val",
                         to_hu_conv = ty_info.java_hu_ty + " " + ty_info.var_name + "_conv = new " + ty_info.java_hu_ty + "(" + ty_info.var_name + ");",
-                        to_hu_conv_name = ty_info.var_name + "_conv", from_hu_conv = (ty_info.var_name + ".ptr", ""))
+                        to_hu_conv_name = ty_info.var_name + "_conv", from_hu_conv = (ty_info.var_name + ".getVal()", ""))
 
                 assert ty_info.rust_obj == "LDKTxOut"
                 if not ty_info.is_ptr and not holds_ref: