Merge pull request #45 from TheBlueMatt/main

Various fixes + cut 0.0.101

Author: TheBlueMatt

.github/workflows/build.yml

@@ -35,7 +35,7 @@ jobs:
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.100
+          git checkout 0.0.101
       - name: Rebuild C bindings without STD for WASM
         run: |
           cd ldk-c-bindings
@@ -98,14 +98,16 @@ jobs:
           cp ldk-java-classes.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
           cp ldk-java-javadoc.jar "ldk-java-bins/${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
           cd ldk-java-bins
+          # ldk-java-leaktracking is only for debug purposes and we don't bother with determinism
+          git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-leaktracking.jar"
           if ! git diff --exit-code; then
             mv "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"* ./
             git checkout "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/"
             apt-get -y install diffoscope
-            diffoscope ldk-java-sources.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-sources.jar"
-            diffoscope ldk-java-javadoc.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-javadoc.jar"
-            diffoscope ldk-java-classes.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-classes.jar"
-            diffoscope ldk-java.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java.jar"
+            diffoscope ldk-java-sources.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-sources.jar" || echo
+            diffoscope ldk-java-javadoc.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-javadoc.jar" || echo
+            diffoscope ldk-java-classes.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java-classes.jar" || echo
+            diffoscope ldk-java.jar "${LDK_GARBAGECOLLECTED_GIT_OVERRIDE}/ldk-java.jar" || echo
             exit 1
           fi
       - name: Run Java Tests against built release jar
@@ -165,7 +167,7 @@ jobs:
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.100
+          git checkout 0.0.101
       - name: Detect current git version
         run: |
           # We assume the top commit is just a bindings update commit, so we
@@ -245,7 +247,7 @@ jobs:
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
           cd ldk-c-bindings
-          git checkout 0.0.100
+          git checkout 0.0.101
       - name: Rebuild C bindings with upstream clang, and check the sample app builds + links
         run: |
           cd ldk-c-bindings

build-release-jar.sh

@@ -11,6 +11,8 @@ ls src/main/resources/liblightningjni_MacOSX-aarch64.nativelib
 
 export LANG=C
 
+# We need to fetch dependencies first as faketime will break PKI cert checks!
+mvn -DskipTests=true -Dorg.lightningdevkit.skipdocs=false package
 mvn clean
 faketime 2021-01-01 mvn -DskipTests=true -Dorg.lightningdevkit.skipdocs=false package
 

gen_type_mapping.py

@@ -157,20 +157,29 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
                 to_hu_conv = None
                 to_hu_conv_name = None
                 if subty.to_hu_conv is not None:
-                    to_hu_conv = ty_info.java_hu_ty + " " + conv_name + "_arr = new " + ty_info.subty.java_hu_ty.split("<")[0] + "[" + arr_name + ".length];\n"
-                    to_hu_conv = to_hu_conv + "for (int " + idxc + " = 0; " + idxc + " < " + arr_name + ".length; " + idxc + "++) {\n"
-                    to_hu_conv = to_hu_conv + "\t" + subty.java_ty + " " + conv_name + " = " + arr_name + "[" + idxc + "];\n"
-                    to_hu_conv = to_hu_conv + "\t" + subty.to_hu_conv.replace("\n", "\n\t") + "\n"
-                    to_hu_conv = to_hu_conv + "\t" + conv_name + "_arr[" + idxc + "] = " + subty.to_hu_conv_name + ";\n}"
+                    base_ty = ty_info.subty.java_hu_ty.split("[")[0].split("<")[0]
+                    to_hu_conv = ty_info.java_hu_ty + " " + conv_name + "_arr = new " + base_ty + "[" + arr_name + ".length]"
+                    if "[" in ty_info.subty.java_hu_ty.split("<")[0]:
+                        # Do a bit of a dance to move any excess [] to the end
+                        to_hu_conv += "[" + ty_info.subty.java_hu_ty.split("<")[0].split("[")[1]
+                    to_hu_conv += ";\nfor (int " + idxc + " = 0; " + idxc + " < " + arr_name + ".length; " + idxc + "++) {\n"
+                    to_hu_conv += "\t" + subty.java_ty + " " + conv_name + " = " + arr_name + "[" + idxc + "];\n"
+                    to_hu_conv += "\t" + subty.to_hu_conv.replace("\n", "\n\t") + "\n"
+                    to_hu_conv += "\t" + conv_name + "_arr[" + idxc + "] = " + subty.to_hu_conv_name + ";\n}"
                     to_hu_conv_name = conv_name + "_arr"
                 from_hu_conv = None
                 if subty.from_hu_conv is not None:
+                    hu_conv_b = ""
+                    if subty.from_hu_conv[1] != "":
+                        hu_conv_b = "for (" + subty.java_hu_ty + " " + conv_name + ": " + arr_name + ") { " + subty.from_hu_conv[1] + "; }"
                     if subty.java_ty == "long" and subty.java_hu_ty != "long":
-                        from_hu_conv = (arr_name + " != null ? Arrays.stream(" + arr_name + ").mapToLong(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray() : null", "/* TODO 2 " + subty.java_hu_ty + "  */")
+                        from_hu_conv = (arr_name + " != null ? Arrays.stream(" + arr_name + ").mapToLong(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray() : null",
+                            hu_conv_b)
                     elif subty.java_ty == "long":
                         from_hu_conv = (arr_name + " != null ? Arrays.stream(" + arr_name + ").map(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray() : null", "/* TODO 2 " + subty.java_hu_ty + "  */")
                     else:
-                        from_hu_conv = (arr_name + " != null ? Arrays.stream(" + arr_name + ").map(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray(" + ty_info.java_ty + "::new) : null", "/* TODO 2 " + subty.java_hu_ty + "  */")
+                        from_hu_conv = (arr_name + " != null ? Arrays.stream(" + arr_name + ").map(" + conv_name + " -> " + subty.from_hu_conv[0] + ").toArray(" + ty_info.java_ty + "::new) : null",
+                            hu_conv_b)
 
                 return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                     arg_conv = arg_conv, arg_conv_name = arg_conv_name, arg_conv_cleanup = arg_conv_cleanup,
@@ -288,12 +297,13 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
                         ret_conv_name = ty_info.var_name + "_conv", to_hu_conv = None, to_hu_conv_name = None, from_hu_conv = None)
                 base_conv = ty_info.rust_obj + " " + ty_info.var_name + "_conv = *(" + ty_info.rust_obj + "*)(((uint64_t)" + ty_info.var_name + ") & ~1);"
                 if ty_info.rust_obj in self.trait_structs:
-                    ret_conv = (ty_info.rust_obj + "* ret = MALLOC(sizeof(" + ty_info.rust_obj + "), \"" + ty_info.rust_obj + "\");\n*ret = ", ";")
+                    ret_conv = (ty_info.rust_obj + "* " + ty_info.var_name + "_ret =MALLOC(sizeof(" + ty_info.rust_obj + "), \"" + ty_info.rust_obj + "\");\n*" + ty_info.var_name + "_ret = ", ";")
                     if holds_ref:
                         if (ty_info.rust_obj.replace("LDK", "") + "_clone") in self.clone_fns:
                             ret_conv = (ret_conv[0] + ty_info.rust_obj.replace("LDK", "") + "_clone(&", ");")
                         else:
-                            ret_conv = (ret_conv[0], "; // Warning: We likely need to clone here, but no clone is available for " + ty_info.rust_obj)
+                            ret_conv = (ret_conv[0], ";\n// Warning: We likely need to clone here, but no clone is available, so we just do it for Java instances")
+                            ret_conv = (ret_conv[0], ret_conv[1] + "" + self.consts.trait_struct_inc_refcnt(ty_info).replace(ty_info.var_name + "_conv", "(*" + ty_info.var_name + "_ret)"))
                     if not is_free:
                         needs_full_clone = not is_free and (not ty_info.is_ptr and not holds_ref or ty_info.requires_clone == True) and ty_info.requires_clone != False
                         if needs_full_clone and (ty_info.rust_obj.replace("LDK", "") + "_clone") in self.clone_fns:
@@ -306,11 +316,11 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
                         base_conv = base_conv + "\n" + "FREE((void*)" + ty_info.var_name + ");"
                     return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                         arg_conv = base_conv, arg_conv_name = ty_info.var_name + "_conv", arg_conv_cleanup = None,
-                        ret_conv = ret_conv, ret_conv_name = "(uint64_t)ret",
+                        ret_conv = ret_conv, ret_conv_name = "(uint64_t)" + ty_info.var_name + "_ret",
                         to_hu_conv = ty_info.java_hu_ty + " ret_hu_conv = new " + ty_info.java_hu_ty + "(null, " + ty_info.var_name + ");\nret_hu_conv.ptrs_to.add(this);",
                         to_hu_conv_name = "ret_hu_conv",
                         from_hu_conv = (ty_info.var_name + " == null ? 0 : " + ty_info.var_name + ".ptr", "this.ptrs_to.add(" + ty_info.var_name + ")"))
-                needs_full_clone = not is_free and ((not ty_info.is_ptr and not holds_ref) or ty_info.requires_clone == True) and ty_info.requires_clone != False
+                needs_full_clone = not is_free and (not ty_info.is_ptr or ty_info.requires_clone == True) and ty_info.requires_clone != False
                 if needs_full_clone:
                     if "res" in ty_info.var_name: # XXX: This is a stupid hack
                         needs_full_clone = False
@@ -323,6 +333,18 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
                     # underlying unlike Vecs, and it gives Java more freedom.
                     base_conv = base_conv + "\nFREE((void*)" + ty_info.var_name + ");"
                 if ty_info.rust_obj in self.complex_enums:
+                    if needs_full_clone and (ty_info.rust_obj.replace("LDK", "") + "_clone") not in self.clone_fns:
+                        # We really need a full clone here, but for now we just implement
+                        # a manual clone explicitly for Option<Trait>s
+                        if ty_info.rust_obj.startswith("LDKCOption"):
+                            optional_ty = ty_info.rust_obj[11:-1]
+                            if "LDK" + optional_ty in self.trait_structs:
+                                base_conv += "\nif (" + ty_info.var_name + "_conv.tag == " + ty_info.rust_obj + "_Some) {"
+                                base_conv += "\n\t// Manually implement clone for Java trait instances"
+                                optional_ty_info = self.java_c_types("LDK" + optional_ty + " " + ty_info.var_name, None)
+                                base_conv += self.consts.trait_struct_inc_refcnt(optional_ty_info).\
+                                    replace("\n", "\n\t").replace(ty_info.var_name + "_conv", ty_info.var_name + "_conv.some")
+                                base_conv += "\n}"
                     ret_conv = ("uint64_t " + ty_info.var_name + "_ref = ((uint64_t)&", ") | 1;")
                     if not holds_ref:
                         ret_conv = (ty_info.rust_obj + " *" + ty_info.var_name + "_copy = MALLOC(sizeof(" + ty_info.rust_obj + "), \"" + ty_info.rust_obj + "\");\n", "")
@@ -385,6 +407,8 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
                             from_hu_conv = from_hu_conv + conv_map.from_hu_conv[0].replace(ty_info.var_name + "_" + chr(idx + ord("a")), ty_info.var_name + "." + chr(idx + ord("a")))
                             if conv_map.from_hu_conv[1] != "":
                                 from_hu_conv_sfx = from_hu_conv_sfx + conv_map.from_hu_conv[1].replace(conv.var_name, ty_info.var_name + "." + chr(idx + ord("a")))
+                                if idx != len(self.tuple_types[ty_info.rust_obj][0]) - 1:
+                                    from_hu_conv_sfx += "; "
                         else:
                             from_hu_conv = from_hu_conv + ty_info.var_name + "." + chr(idx + ord("a"))
 
@@ -427,10 +451,15 @@ def map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, holds_re
             elif ty_info.is_ptr:
                 assert(not is_free)
                 if ty_info.rust_obj in self.complex_enums:
+                    ret_conv = ("uint64_t ret_" + ty_info.var_name + " = (uint64_t)", " | 1; // Warning: We should clone here!")
+                    if ty_info.rust_obj.replace("LDK", "") + "_clone" in self.clone_fns:
+                        ret_conv_pfx = ty_info.rust_obj + " *ret_" + ty_info.var_name + " = MALLOC(sizeof(" + ty_info.rust_obj + "), \"" + ty_info.rust_obj + " ret conversion\");\n"
+                        ret_conv_pfx += "*ret_" + ty_info.var_name + " = " + ty_info.rust_obj.replace("LDK", "") + "_clone("
+                        ret_conv = (ret_conv_pfx, ");")
                     return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                         arg_conv = ty_info.rust_obj + "* " + ty_info.var_name + "_conv = (" + ty_info.rust_obj + "*)" + ty_info.var_name + ";",
                         arg_conv_name = ty_info.var_name + "_conv", arg_conv_cleanup = None,
-                        ret_conv = ("uint64_t ret_" + ty_info.var_name + " = (uint64_t)", ";"), ret_conv_name = "ret_" + ty_info.var_name,
+                        ret_conv = ret_conv, ret_conv_name = "(uint64_t)ret_" + ty_info.var_name,
                         to_hu_conv = ty_info.java_hu_ty + " " + ty_info.var_name + "_hu_conv = " + ty_info.java_hu_ty + ".constr_from_ptr(" + ty_info.var_name + ");",
                         to_hu_conv_name = ty_info.var_name + "_hu_conv",
                         from_hu_conv = (ty_info.var_name + " == null ? 0 : " + ty_info.var_name + ".ptr & ~1", "this.ptrs_to.add(" + ty_info.var_name + ")"))

genbindings.py

@@ -459,8 +459,23 @@ def map_fn(line, re_match, ret_arr_len, c_call_string, doc_comment):
                     takes_self_ptr = True
             elif argument_conversion_info.arg_name in params_nullable:
                 argument_conversion_info.nullable = True
+                if argument_conversion_info.arg_conv is not None and "Warning" in argument_conversion_info.arg_conv:
+                    arg_ty_info = java_c_types(argument, None)
+                    print("WARNING: Remapping argument " + arg_ty_info.var_name + " of function " + method_name + " to a reference")
+                    print("    The argument appears to require a move, or not clonable, and is nullable.")
+                    print("    Normally for arguments that require a move and are not clonable, we split")
+                    print("    the argument into the type's constructor's arguments and just use those to")
+                    print("    construct a new object on the fly.")
+                    print("    However, because the object is nullable, doing so would mean we can no")
+                    print("    longer allow the user to pass null, as we now have an argument list instead.")
+                    print("    Thus, we blindly assume its really an Option<&Type> instead of an Option<Type>.")
+                    print("    It may or may not actually be a reference, but its the simplest mapping option")
+                    print("    and also the only use of this code today.")
+                    arg_ty_info.requires_clone = False
+                    argument_conversion_info = type_mapping_generator.map_type_with_info(arg_ty_info, False, None, is_free, True)
+                    assert argument_conversion_info.arg_conv is not None and "Warning" not in argument_conversion_info.arg_conv
+
             if argument_conversion_info.arg_conv is not None and "Warning" in argument_conversion_info.arg_conv:
-                assert not argument_conversion_info.arg_name in params_nullable
                 if argument_conversion_info.rust_obj in constructor_fns:
                     assert not is_free
                     for explode_arg in constructor_fns[argument_conversion_info.rust_obj].split(','):