Merge pull request #16 from TheBlueMatt/2021-03-background-manager-persister

Adapt ChannelManagerConstructor to persist ChannelManager + handle events

Author: TheBlueMatt

.github/workflows/build.yml

@@ -30,7 +30,7 @@ jobs:
           cd ..
           git clone https://github.com/lightningdevkit/ldk-c-bindings
       - name: Rebuild C bindings, and check the sample app builds + links
-        run: cd ldk-c-bindings && ./genbindings.sh ../rust-lightning && cd ..
+        run: cd ldk-c-bindings && ./genbindings.sh ../rust-lightning true && cd ..
       - name: Build Java/TS Debug Bindings
         run: ./genbindings.sh ./ldk-c-bindings/ "-I/usr/lib/jvm/java-11-openjdk-amd64/include/ -I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" true false
       - name: Run Java Tests against Debug Bindings

liblightningjni_debug.so

@@ -34,7 +34,7 @@ public static class InvalidSerializedDataException extends Exception {}
      */
     public final TwoTuple<ChannelMonitor, byte[]>[] channel_monitors;
 
-    private final Watch chain_watch;
+    private final ChainMonitor chain_monitor;
 
     /**
      * Deserializes a channel manager and a set of channel monitors from the given serialized copies and interface implementations
@@ -44,7 +44,7 @@ public static class InvalidSerializedDataException extends Exception {}
      *               outputs will be loaded when chain_sync_completed is called.
      */
     public ChannelManagerConstructor(byte[] channel_manager_serialized, byte[][] channel_monitors_serialized,
-                                     KeysInterface keys_interface, FeeEstimator fee_estimator, Watch chain_watch, @Nullable Filter filter,
+                                     KeysInterface keys_interface, FeeEstimator fee_estimator, ChainMonitor chain_monitor, @Nullable Filter filter,
                                      BroadcasterInterface tx_broadcaster, Logger logger) throws InvalidSerializedDataException {
         final ChannelMonitor[] monitors = new ChannelMonitor[channel_monitors_serialized.length];
         this.channel_monitors = new TwoTuple[monitors.length];
@@ -57,14 +57,14 @@ public ChannelManagerConstructor(byte[] channel_manager_serialized, byte[][] cha
             this.channel_monitors[i] = new TwoTuple<>(monitors[i], ((Result_C2Tuple_BlockHashChannelMonitorZDecodeErrorZ.Result_C2Tuple_BlockHashChannelMonitorZDecodeErrorZ_OK)res).res.a);
         }
         Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ res =
-                UtilMethods.constructor_BlockHashChannelManagerZ_read(channel_manager_serialized, keys_interface, fee_estimator, chain_watch, tx_broadcaster,
+                UtilMethods.constructor_BlockHashChannelManagerZ_read(channel_manager_serialized, keys_interface, fee_estimator, chain_monitor.as_Watch(), tx_broadcaster,
                         logger, UserConfig.constructor_default(), monitors);
         if (res instanceof Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ.Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ_Err) {
             throw new InvalidSerializedDataException();
         }
         this.channel_manager = ((Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ.Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ_OK)res).res.b;
         this.channel_manager_latest_block_hash = ((Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ.Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ_OK)res).res.a;
-        this.chain_watch = chain_watch;
+        this.chain_monitor = chain_monitor;
         if (filter != null) {
             for (ChannelMonitor monitor : monitors) {
                 monitor.load_outputs_to_watch(filter);
@@ -76,21 +76,74 @@ public ChannelManagerConstructor(byte[] channel_manager_serialized, byte[][] cha
      * Constructs a channel manager from the given interface implementations
      */
     public ChannelManagerConstructor(LDKNetwork network, UserConfig config, byte[] current_blockchain_tip_hash, int current_blockchain_tip_height,
-                                     KeysInterface keys_interface, FeeEstimator fee_estimator, Watch chain_watch,
+                                     KeysInterface keys_interface, FeeEstimator fee_estimator, ChainMonitor chain_monitor,
                                      BroadcasterInterface tx_broadcaster, Logger logger) throws InvalidSerializedDataException {
         channel_monitors = new TwoTuple[0];
         channel_manager_latest_block_hash = null;
-        this.chain_watch = chain_watch;
-        channel_manager = ChannelManager.constructor_new(fee_estimator, chain_watch, tx_broadcaster, logger, keys_interface, config, network, current_blockchain_tip_hash, current_blockchain_tip_height);
+        this.chain_monitor = chain_monitor;
+        channel_manager = ChannelManager.constructor_new(fee_estimator, chain_monitor.as_Watch(), tx_broadcaster, logger, keys_interface, config, network, current_blockchain_tip_hash, current_blockchain_tip_height);
     }
 
+    /**
+     * Abstract interface which should handle Events and persist the ChannelManager. When you call chain_sync_completed
+     * a background thread is started which will automatically call these methods for you when events occur.
+     */
+    public interface ChannelManagerPersister {
+        void handle_events(Event[] events);
+        void persist_manager(byte[] channel_manager_bytes);
+    }
+
+    Thread persister_thread = null;
+    volatile boolean shutdown = false;
+
     /**
      * Utility which adds all of the deserialized ChannelMonitors to the chain watch so that further updates from the
      * ChannelManager are processed as normal.
+     *
+     * This also spawns a background thread which will call the appropriate methods on the provided
+     * ChannelManagerPersister as required.
      */
-    public void chain_sync_completed() {
+    public void chain_sync_completed(ChannelManagerPersister persister) {
+        if (persister_thread != null) { return; }
         for (TwoTuple<ChannelMonitor, byte[]> monitor: channel_monitors) {
-            this.chain_watch.watch_channel(monitor.a.get_funding_txo().a, monitor.a);
+            this.chain_monitor.as_Watch().watch_channel(monitor.a.get_funding_txo().a, monitor.a);
         }
+        persister_thread = new Thread(() -> {
+            long lastTimerTick = System.currentTimeMillis();
+            while (true) {
+                boolean need_persist = this.channel_manager.await_persistable_update_timeout(1);
+                Event[] events = this.channel_manager.as_EventsProvider().get_and_clear_pending_events();
+                if (events.length != 0) {
+                    persister.handle_events(events);
+                    need_persist = true;
+                }
+                events = this.chain_monitor.as_EventsProvider().get_and_clear_pending_events();
+                if (events.length != 0) {
+                    persister.handle_events(events);
+                    need_persist = true;
+                }
+                if (need_persist) {
+                    persister.persist_manager(this.channel_manager.write());
+                }
+                if (shutdown) {
+                    return;
+                }
+                if (lastTimerTick < System.currentTimeMillis() - 60 * 1000) {
+                    this.channel_manager.timer_chan_freshness_every_min();
+                    lastTimerTick = System.currentTimeMillis();
+                }
+            }
+        }, "NioPeerHandler NIO Thread");
+        persister_thread.start();
+    }
+
+    /**
+     * Interrupt the background thread, stopping the background handling of
+     */
+    public void interrupt() {
+        shutdown = true;
+        try {
+            persister_thread.join();
+        } catch (InterruptedException ignored) { }
     }
 }

liblightningjni_release.so

@@ -2163,6 +2163,8 @@ public interface LDKSocketDescriptor {
 	public static native void ChannelManager_block_connected(long this_arg, byte[] header, long[] txdata, int height);
 	// void ChannelManager_block_disconnected(const struct LDKChannelManager *NONNULL_PTR this_arg, const uint8_t (*header)[80]);
 	public static native void ChannelManager_block_disconnected(long this_arg, byte[] header);
+	// MUST_USE_RES bool ChannelManager_await_persistable_update_timeout(const struct LDKChannelManager *NONNULL_PTR this_arg, uint64_t max_wait);
+	public static native boolean ChannelManager_await_persistable_update_timeout(long this_arg, long max_wait);
 	// void ChannelManager_await_persistable_update(const struct LDKChannelManager *NONNULL_PTR this_arg);
 	public static native void ChannelManager_await_persistable_update(long this_arg);
 	// struct LDKChannelMessageHandler ChannelManager_as_ChannelMessageHandler(const struct LDKChannelManager *NONNULL_PTR this_arg);

src/main/java/org/ldk/batteries/ChannelManagerConstructor.java

@@ -394,6 +394,18 @@ public void block_disconnected(byte[] header) {
 		bindings.ChannelManager_block_disconnected(this.ptr, header);
 	}
 
+	/**
+	 * Blocks until ChannelManager needs to be persisted or a timeout is reached. It returns a bool
+	 * indicating whether persistence is necessary. Only one listener on
+	 * `await_persistable_update` or `await_persistable_update_timeout` is guaranteed to be woken
+	 * up.
+	 * Note that the feature `allow_wallclock_use` must be enabled to use this function.
+	 */
+	public boolean await_persistable_update_timeout(long max_wait) {
+		boolean ret = bindings.ChannelManager_await_persistable_update_timeout(this.ptr, max_wait);
+		return ret;
+	}
+
 	/**
 	 * Blocks until ChannelManager needs to be persisted. Only one listener on
 	 * `await_persistable_update` or `await_persistable_update_timeout` is guaranteed to be woken

src/main/java/org/ldk/impl/bindings.java

@@ -12722,6 +12722,14 @@ JNIEXPORT void JNICALL Java_org_ldk_impl_bindings_ChannelManager_1block_1disconn
 	ChannelManager_block_disconnected(&this_arg_conv, header_ref);
 }
 
+JNIEXPORT jboolean JNICALL Java_org_ldk_impl_bindings_ChannelManager_1await_1persistable_1update_1timeout(JNIEnv *env, jclass clz, int64_t this_arg, int64_t max_wait) {
+	LDKChannelManager this_arg_conv;
+	this_arg_conv.inner = (void*)(this_arg & (~1));
+	this_arg_conv.is_owned = false;
+	jboolean ret_val = ChannelManager_await_persistable_update_timeout(&this_arg_conv, max_wait);
+	return ret_val;
+}
+
 JNIEXPORT void JNICALL Java_org_ldk_impl_bindings_ChannelManager_1await_1persistable_1update(JNIEnv *env, jclass clz, int64_t this_arg) {
 	LDKChannelManager this_arg_conv;
 	this_arg_conv.inner = (void*)(this_arg & (~1));

src/main/java/org/ldk/structs/ChannelManager.java

@@ -1,10 +1,8 @@
 package org.ldk;
 
 import org.bitcoinj.core.*;
-import org.bitcoinj.core.Transaction;
 import org.bitcoinj.script.Script;
 import org.junit.jupiter.api.Test;
-import org.ldk.batteries.ChannelManagerConstructor;
 import org.ldk.batteries.NioPeerHandler;
 import org.ldk.enums.LDKNetwork;
 import org.ldk.impl.bindings;
@@ -287,38 +285,26 @@ private void bind_nio() {
         Object ptr_to;
         Peer(Peer orig) {
             this(null, orig.seed);
-            if (!break_cross_peer_refs) {
-                ChannelMonitor[] monitors = new ChannelMonitor[1];
-                synchronized (monitors) {
-                    assert orig.monitors.size() == 1;
+            // TODO: Optionally test ChannelManagerConstructor
+            ChannelMonitor[] monitors = new ChannelMonitor[1];
+            synchronized (monitors) {
+                assert orig.monitors.size() == 1;
+                if (!break_cross_peer_refs) {
                     monitors[0] = orig.monitors.values().stream().iterator().next();
-                }
-                byte[] serialized = orig.chan_manager.write();
-                Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ read_res =
-                        UtilMethods.constructor_BlockHashChannelManagerZ_read(serialized, this.keys_interface, this.fee_estimator, this.chain_watch, this.tx_broadcaster, this.logger, UserConfig.constructor_default(), monitors);
-                assert read_res instanceof Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ.Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ_OK;
-                this.chan_manager = ((Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ.Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ_OK) read_res).res.b;
-                this.chain_watch.watch_channel(monitors[0].get_funding_txo().a, monitors[0]);
-            } else {
-                final ArrayList<byte[]> channel_monitors = new ArrayList();
-                synchronized (monitors) {
-                    assert orig.monitors.size() == 1;
-                    channel_monitors.add(orig.monitors.values().stream().iterator().next().write());
-                }
-                byte[] serialized = orig.chan_manager.write();
-                try {
-                    ChannelManagerConstructor constructed = new ChannelManagerConstructor(serialized, channel_monitors.toArray(new byte[1][]), this.keys_interface, this.fee_estimator, this.chain_watch, this.filter, this.tx_broadcaster, this.logger);
-                    this.chan_manager = constructed.channel_manager;
-                    constructed.chain_sync_completed();
-                    if (use_filter && !use_manual_watch) {
-                        // With a manual watch we don't actually use the filter object at all.
-                        assert this.filter_additions.containsAll(orig.filter_additions) &&
-                                orig.filter_additions.containsAll(this.filter_additions);
-                    }
-                } catch (ChannelManagerConstructor.InvalidSerializedDataException e) {
-                    assert false;
+                } else {
+                    byte[] serialized = orig.monitors.values().stream().iterator().next().write();
+                    Result_C2Tuple_BlockHashChannelMonitorZDecodeErrorZ res =
+                            UtilMethods.constructor_BlockHashChannelMonitorZ_read(serialized, this.keys_interface);
+                    assert res instanceof Result_C2Tuple_BlockHashChannelMonitorZDecodeErrorZ.Result_C2Tuple_BlockHashChannelMonitorZDecodeErrorZ_OK;
+                    monitors[0] = ((Result_C2Tuple_BlockHashChannelMonitorZDecodeErrorZ.Result_C2Tuple_BlockHashChannelMonitorZDecodeErrorZ_OK) res).res.b;
                 }
             }
+            byte[] serialized = orig.chan_manager.write();
+            Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ read_res =
+                    UtilMethods.constructor_BlockHashChannelManagerZ_read(serialized, this.keys_interface, this.fee_estimator, this.chain_watch, this.tx_broadcaster, this.logger, UserConfig.constructor_default(), monitors);
+            assert read_res instanceof Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ.Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ_OK;
+            this.chan_manager = ((Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ.Result_C2Tuple_BlockHashChannelManagerZDecodeErrorZ_OK) read_res).res.b;
+            this.chain_watch.watch_channel(monitors[0].get_funding_txo().a, monitors[0]);
             if (!break_cross_peer_refs && (use_manual_watch || use_km_wrapper)) {
                 // When we pass monitors[0] into chain_watch.watch_channel we create a reference from the new Peer to a
                 // field in the old peer, preventing freeing of the original Peer until the new Peer is freed. Thus, we

src/main/jni/bindings.c

@@ -11492,6 +11492,14 @@ void  __attribute__((visibility("default"))) TS_ChannelManager_block_disconnecte
 	ChannelManager_block_disconnected(&this_arg_conv, header_ref);
 }
 
+jboolean  __attribute__((visibility("default"))) TS_ChannelManager_await_persistable_update_timeout(uint32_t this_arg, int64_t max_wait) {
+	LDKChannelManager this_arg_conv;
+	this_arg_conv.inner = (void*)(this_arg & (~1));
+	this_arg_conv.is_owned = false;
+	jboolean ret_val = ChannelManager_await_persistable_update_timeout(&this_arg_conv, max_wait);
+	return ret_val;
+}
+
 void  __attribute__((visibility("default"))) TS_ChannelManager_await_persistable_update(uint32_t this_arg) {
 	LDKChannelManager this_arg_conv;
 	this_arg_conv.inner = (void*)(this_arg & (~1));