Switch 32-bit platforms to using 64-bit "pointers"

TheBlueMatt · TheBlueMatt · commit 7cbcc5ef17b8 · 2022-08-13T00:39:56.000Z
This much more cleanly fixes the issue described in the previous
commit by simply using 64-bit integers to represent pointers on
32-bit platforms, letting us store the required flag bit in the top
32-bits that are always free.

It does, however, imply changing the entire call semantics for
TypeScript to use BigInts for pointers, rather than numbers.
diff --git a/gen_type_mapping.py b/gen_type_mapping.py
@@ -282,7 +282,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                 ty_info.var_name = "ret"
 
             if ty_info.rust_obj in self.opaque_structs:
-                from_hu_conv = (ty_info.var_name + " == null ? 0 : " + self.consts.get_ptr(ty_info.var_name), self.consts.add_ref("this", ty_info.var_name))
+                from_hu_conv = (ty_info.var_name + " == null ? " + self.consts.native_zero_ptr + " : " + self.consts.get_ptr(ty_info.var_name), self.consts.add_ref("this", ty_info.var_name))
                 opaque_arg_conv = ty_info.rust_obj + " " + ty_info.var_name + "_conv;\n"
                 opaque_arg_conv = opaque_arg_conv + ty_info.var_name + "_conv.inner = untag_ptr(" + ty_info.var_name + ");\n"
                 opaque_arg_conv += ty_info.var_name + "_conv.is_owned = ptr_is_owned(" + ty_info.var_name + ");\n"
@@ -298,7 +298,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                         if holds_ref:
                             opaque_arg_conv += "\n" + ty_info.var_name + "_conv = " + ty_info.rust_obj.replace("LDK", "") + "_clone(&" + ty_info.var_name + "_conv);"
                         else:
-                            from_hu_conv = (ty_info.var_name + " == null ? 0 : " + ty_info.var_name + ".clone_ptr()", "")
+                            from_hu_conv = (ty_info.var_name + " == null ? " + self.consts.native_zero_ptr + " : " + ty_info.var_name + ".clone_ptr()", "")
                     elif ty_info.passed_as_ptr:
                         opaque_arg_conv += "\n// WARNING: we need a move here but no clone is available for " + ty_info.rust_obj + "\n"
                         # TODO: Once we support features cloning (which just isn't in C yet), we can make this a compile error instead!
@@ -380,7 +380,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                             if holds_ref:
                                 base_conv += "\n" + ty_info.var_name + "_conv = " + ty_info.rust_obj.replace("LDK", "") + "_clone(&" + ty_info.var_name + "_conv);"
                             else:
-                                from_hu_conv = (ty_info.var_name + " == null ? 0 : " + ty_info.var_name + ".clone_ptr()", "")
+                                from_hu_conv = (ty_info.var_name + " == null ? " + self.consts.native_zero_ptr + " : " + ty_info.var_name + ".clone_ptr()", "")
                                 base_conv += "\n" + "FREE(untag_ptr(" + ty_info.var_name + "));"
                         else:
                             base_conv = base_conv + self.consts.trait_struct_inc_refcnt(ty_info)
@@ -389,7 +389,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                     else:
                         base_conv = base_conv + "\n" + "FREE(untag_ptr(" + ty_info.var_name + "));"
                     if from_hu_conv is None:
-                        from_hu_conv = (ty_info.var_name + " == null ? 0 : " + self.consts.get_ptr(ty_info.var_name), "")
+                        from_hu_conv = (ty_info.var_name + " == null ? " + self.consts.native_zero_ptr + " : " + self.consts.get_ptr(ty_info.var_name), "")
                     from_hu_conv = (from_hu_conv[0], self.consts.add_ref("this", ty_info.var_name))
                     return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                         arg_conv = base_conv, arg_conv_name = ty_info.var_name + "_conv", arg_conv_cleanup = None,
@@ -408,7 +408,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                         if holds_ref:
                             base_conv += "\n" + ty_info.var_name + "_conv = " + ty_info.rust_obj.replace("LDK", "") + "_clone((" + ty_info.rust_obj + "*)untag_ptr(" + ty_info.var_name + "));"
                         else:
-                            from_hu_conv = (ty_info.var_name + " == null ? 0 : " + ty_info.var_name + ".clone_ptr()", "")
+                            from_hu_conv = (ty_info.var_name + " == null ? " + self.consts.native_zero_ptr + " : " + ty_info.var_name + ".clone_ptr()", "")
                             base_conv += "\n" + "FREE(untag_ptr(" + ty_info.var_name + "));"
                     elif needs_full_clone:
                         base_conv = base_conv + "\n// WARNING: we may need a move here but no clone is available for " + ty_info.rust_obj
@@ -456,7 +456,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                     else:
                         ret_conv = (ty_info.rust_obj + "* " + ty_info.var_name + "_conv = MALLOC(sizeof(" + ty_info.rust_obj + "), \"" + ty_info.rust_obj + "\");\n*" + ty_info.var_name + "_conv = ", ";")
                     if from_hu_conv is None:
-                        from_hu_conv = (ty_info.var_name + " != null ? " + self.consts.get_ptr(ty_info.var_name) + " : 0", "")
+                        from_hu_conv = (ty_info.var_name + " != null ? " + self.consts.get_ptr(ty_info.var_name) + " : " + self.consts.native_zero_ptr, "")
                     return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                         arg_conv = base_conv, arg_conv_name = ty_info.var_name + "_conv", arg_conv_cleanup = None,
                         ret_conv = ret_conv, ret_conv_name = "tag_ptr(" + ty_info.var_name + "_conv, true)",
@@ -482,7 +482,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                     else:
                         to_hu_conv_sfx = ""
                     if from_hu_conv is None:
-                        from_hu_conv = (ty_info.var_name + " != null ? " + self.consts.get_ptr(ty_info.var_name) + " : 0", "")
+                        from_hu_conv = (ty_info.var_name + " != null ? " + self.consts.get_ptr(ty_info.var_name) + " : " + self.consts.native_zero_ptr, "")
                     return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                         arg_conv = base_conv, arg_conv_name = ty_info.var_name + "_conv", arg_conv_cleanup = None,
                         ret_conv = ret_conv, ret_conv_name = ret_conv_name,
@@ -535,7 +535,7 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                         ret_conv = ret_conv, ret_conv_name = "ref_" + ty_info.var_name,
                         to_hu_conv = self.consts.var_decl_statement(ty_info.java_hu_ty, ty_info.var_name + "_hu_conv", ty_info.java_hu_ty + ".constr_from_ptr(" + ty_info.var_name + ")") + ";",
                         to_hu_conv_name = ty_info.var_name + "_hu_conv",
-                        from_hu_conv = (ty_info.var_name + " == null ? 0 : " + self.consts.get_ptr(ty_info.var_name), from_hu_sfx))
+                        from_hu_conv = (ty_info.var_name + " == null ? " + self.consts.native_zero_ptr + " : " + self.consts.get_ptr(ty_info.var_name), from_hu_sfx))
                 elif ty_info.rust_obj in self.trait_structs:
                     if ty_info.nonnull_ptr:
                         arg_conv = "void* " + ty_info.var_name + "_ptr = untag_ptr(" + ty_info.var_name + ");\n"
@@ -564,15 +564,15 @@ def _do_map_type_with_info(self, ty_info, print_void, ret_arr_len, is_free, hold
                             ret_conv_name = "tag_ptr(" + ty_info.var_name + "_clone, true)",
                             to_hu_conv = self.consts.var_decl_statement(ty_info.java_hu_ty, "ret_hu_conv", "new " + ty_info.java_hu_ty + "(null, " + ty_info.var_name + ")") + ";\n" + self.consts.add_ref("ret_hu_conv", "this") + ";",
                             to_hu_conv_name = "ret_hu_conv",
-                            from_hu_conv = (ty_info.var_name + " == null ? 0 : " + self.consts.get_ptr(ty_info.var_name), ""))
+                            from_hu_conv = (ty_info.var_name + " == null ? " + self.consts.native_zero_ptr + " : " + self.consts.get_ptr(ty_info.var_name), ""))
                     else:
                         return ConvInfo(ty_info = ty_info, arg_name = ty_info.var_name,
                             arg_conv = arg_conv, arg_conv_name = arg_conv_name, arg_conv_cleanup = None,
                             ret_conv = ("// WARNING: This object doesn't live past this scope, needs clone!\n" + self.consts.ptr_c_ty + " ret_" + ty_info.var_name + " = tag_ptr(", ", false);"),
                             ret_conv_name = "ret_" + ty_info.var_name,
                             to_hu_conv = self.consts.var_decl_statement(ty_info.java_hu_ty, "ret_hu_conv", "new " + ty_info.java_hu_ty + "(null, " + ty_info.var_name + ")") + ";\n" + self.consts.add_ref("ret_hu_conv", "this") + ";",
                             to_hu_conv_name = "ret_hu_conv",
-                            from_hu_conv = (ty_info.var_name + " == null ? 0 : " + self.consts.get_ptr(ty_info.var_name), self.consts.add_ref("this", ty_info.var_name)))
+                            from_hu_conv = (ty_info.var_name + " == null ? " + self.consts.native_zero_ptr + " : " + self.consts.get_ptr(ty_info.var_name), self.consts.add_ref("this", ty_info.var_name)))
                 ret_conv = (self.consts.ptr_c_ty + " ret_" + ty_info.var_name + " = tag_ptr(", ", true);")
                 if holds_ref:
                     ret_conv = (ret_conv[0], ", false);")
diff --git a/genbindings.py b/genbindings.py
@@ -302,8 +302,8 @@ def java_c_types(fn_arg, ret_arr_len):
             arr_ty = "uint64_t"
             fn_arg = fn_arg[8:].strip()
         else:
-            java_ty = consts.ptr_native_ty
-            c_ty = consts.ptr_c_ty
+            java_ty = consts.usize_native_ty
+            c_ty = consts.usize_c_ty
             arr_ty = "uintptr_t"
             rust_obj = "uintptr_t"
             fn_arg = fn_arg[9:].strip()
@@ -437,10 +437,11 @@ def java_c_types(fn_arg, ret_arr_len):
 write_c("static inline struct LDKThirtyTwoBytes ThirtyTwoBytes_clone(const struct LDKThirtyTwoBytes *orig) { struct LDKThirtyTwoBytes ret; memcpy(ret.data, orig->data, 32); return ret; }\n\n")
 
 
-write_c("static inline void* untag_ptr(uintptr_t ptr) {\n")
+write_c("static inline void* untag_ptr(uint64_t ptr) {\n")
 write_c("\tif (ptr < 4096) return (void*)ptr;\n")
 write_c("\tif (sizeof(void*) == 4) {\n")
-write_c("\t\treturn (void*)(ptr & ~1);\n")
+write_c("\t\t// For 32-bit systems, store pointers as 64-bit ints and use the 31st bit\n")
+write_c("\t\treturn (void*)(uintptr_t)ptr;\n")
 write_c("\t} else {\n")
 write_c("\t\t// For 64-bit systems, assume the top byte is used for tagging, then\n")
 write_c("\t\t// use bit 9 ^ bit 10.\n")
@@ -456,10 +457,10 @@ def java_c_types(fn_arg, ret_arr_len):
 write_c("\t}\n")
 write_c("}\n")
 
-write_c("static inline bool ptr_is_owned(uintptr_t ptr) {\n")
+write_c("static inline bool ptr_is_owned(uint64_t ptr) {\n")
 write_c("\tif(ptr < 4096) return true;\n")
 write_c("\tif (sizeof(void*) == 4) {\n")
-write_c("\t\treturn (ptr & 1) ? true : false;\n")
+write_c("\t\treturn ptr & (1ULL << 32);\n")
 write_c("\t} else {\n")
 write_c("\t\tuintptr_t ninth_bit = (((uintptr_t)ptr) & (1ULL << 55)) >> 55;\n")
 write_c("\t\tuintptr_t tenth_bit = (((uintptr_t)ptr) & (1ULL << 54)) >> 54;\n")
@@ -472,10 +473,10 @@ def java_c_types(fn_arg, ret_arr_len):
 write_c("\t}\n")
 write_c("}\n")
 
-write_c("static inline uintptr_t tag_ptr(const void* ptr, bool is_owned) {\n")
+write_c("static inline uint64_t tag_ptr(const void* ptr, bool is_owned) {\n")
 write_c("\tif ((uintptr_t)ptr < 4096) return (uint64_t)ptr;\n")
 write_c("\tif (sizeof(void*) == 4) {\n")
-write_c("\t\treturn ((uintptr_t)ptr) | (is_owned ? 1 : 0);\n")
+write_c("\t\treturn (((uint64_t)ptr) | ((is_owned ? 1ULL : 0) << 32));\n")
 write_c("\t} else {\n")
 write_c("\t\tCHECK(sizeof(uintptr_t) == 8);\n")
 write_c("\t\tuintptr_t tenth_bit = (((uintptr_t)ptr) & (1ULL << 54)) >> 54;\n")
@@ -538,13 +539,13 @@ def map_fn_with_ref_option(line, re_match, ret_arr_len, c_call_string, doc_comme
             return_type_info = type_mapping_generator.map_type(method_return_type.strip() + " ret", True, ret_arr_len, False, force_holds_ref)
 
         if method_name.endswith("_clone") and expected_struct not in unitary_enums:
-            meth_line = "uintptr_t " + expected_struct.replace("LDK", "") + "_clone_ptr(" + expected_struct + " *NONNULL_PTR arg)"
+            meth_line = "uint64_t " + expected_struct.replace("LDK", "") + "_clone_ptr(" + expected_struct + " *NONNULL_PTR arg)"
             write_c("static inline " + meth_line + " {\n")
             write_c("\t" + return_type_info.ret_conv[0].replace("\n", "\n\t"))
             write_c(method_name + "(arg)")
             write_c(return_type_info.ret_conv[1].replace("\n", "\n\t"))
             write_c("\n\treturn " + return_type_info.ret_conv_name + ";\n}\n")
-            map_fn(meth_line + ";\n", re.compile("(uintptr_t) ([A-Za-z_0-9]*)\((.*)\)").match(meth_line), None, None, None)
+            map_fn(meth_line + ";\n", re.compile("(uint64_t) ([A-Za-z_0-9]*)\((.*)\)").match(meth_line), None, None, None)
 
         argument_types = []
         default_constructor_args = {}
diff --git a/java_strings.py b/java_strings.py
@@ -555,6 +555,9 @@ class CommonBase {
         self.file_ext = ".java"
         self.ptr_c_ty = "int64_t"
         self.ptr_native_ty = "long"
+        self.usize_c_ty = "int64_t"
+        self.usize_native_ty = "long"
+        self.native_zero_ptr = "0"
         self.result_c_ty = "jclass"
         self.ptr_arr = "jobjectArray"
         self.is_arr_some_check = ("", " != NULL")
diff --git a/ts/js-wasm.h b/ts/js-wasm.h
@@ -39,8 +39,14 @@ typedef uint32_t JSValue;
 
 DECL_IMPORT(u, u, u, u, u, u)
 DECL_IMPORT(b, u, u, u, u, u)
+DECL_IMPORT(b, b, u, u, u, u)
+DECL_IMPORT(b, b, b, u, u, u)
+DECL_IMPORT(b, b, b, b, u, u)
 DECL_IMPORT(b, b, b, b, b, b)
+DECL_IMPORT(u, b, u, u, b, u)
 DECL_IMPORT(u, b, u, u, u, u)
+DECL_IMPORT(u, b, b, u, u, u)
 DECL_IMPORT(u, u, b, u, u, u)
+DECL_IMPORT(u, u, b, u, b, u)
 
 #endif
diff --git a/typescript_strings.py b/typescript_strings.py
