diff --git a/Makefile b/Makefile index 54a3bbeab51f..f6cd7c1a0b71 100644 --- a/Makefile +++ b/Makefile @@ -603,7 +603,7 @@ check-tmpctx: @if git grep -n 'tal_free[(]tmpctx)' | grep -Ev '^ccan/|/test/|^common/setup.c:|^common/utils.c:'; then echo "Don't free tmpctx!">&2; exit 1; fi check-discouraged-functions: - @if git grep -E "[^a-z_/](fgets|fputs|gets|scanf|sprintf)\(" -- "*.c" "*.h" ":(exclude)ccan/" ":(exclude)contrib/"; then exit 1; fi + @if git grep -nE "[^a-z_/](fgets|fputs|gets|scanf|sprintf|randombytes_buf|time_now)\(" -- "*.c" "*.h" ":(exclude)ccan/" ":(exclude)contrib/" | grep -Fv '/* discouraged:'; then exit 1; fi check-bad-sprintf: @if git grep -n "%[*]\.s"; then exit 1; fi diff --git a/bitcoin/script.c b/bitcoin/script.c index 1c703dbbfd26..2422f612b0e0 100644 --- a/bitcoin/script.c +++ b/bitcoin/script.c @@ -6,8 +6,8 @@ #include #include #include +#include #include -#include #include /* To push 0-75 bytes onto stack. */ diff --git a/bitcoin/short_channel_id.c b/bitcoin/short_channel_id.c index cc1de68c4377..98fe4748b714 100644 --- a/bitcoin/short_channel_id.c +++ b/bitcoin/short_channel_id.c @@ -1,7 +1,7 @@ #include "config.h" #include #include -#include +#include #include #include @@ -104,6 +104,6 @@ struct short_channel_id fromwire_short_channel_id(const u8 **cursor, size_t *max struct short_channel_id random_scid(void) { struct short_channel_id scid; - randombytes_buf(&scid, sizeof(scid)); + randbytes(&scid, sizeof(scid)); return scid; } diff --git a/channeld/test/Makefile b/channeld/test/Makefile index 0bb50c47ab6c..f683ec31b26f 100644 --- a/channeld/test/Makefile +++ b/channeld/test/Makefile @@ -27,6 +27,7 @@ channeld/test/run-full_channel: \ common/msg_queue.o \ common/permute_tx.o \ common/pseudorand.o \ + common/randbytes.o \ common/setup.o \ common/utils.o diff --git a/channeld/test/run-commit_tx.c b/channeld/test/run-commit_tx.c index b57e50a36cc8..cc0a8cbe3a8a 100644 --- a/channeld/test/run-commit_tx.c +++ b/channeld/test/run-commit_tx.c @@ -16,6 +16,7 @@ static bool print_superverbose; #include #include #include +#include #include #include diff --git a/channeld/test/run-full_channel.c b/channeld/test/run-full_channel.c index a852d19279fa..1bb406a785c7 100644 --- a/channeld/test/run-full_channel.c +++ b/channeld/test/run-full_channel.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/common/Makefile b/common/Makefile index 91e95e28d08a..333e3321cb6a 100644 --- a/common/Makefile +++ b/common/Makefile @@ -21,6 +21,7 @@ COMMON_SRC_NOGEN := \ common/channel_config.c \ common/channel_id.c \ common/channel_type.c \ + common/clock_time.c \ common/close_tx.c \ common/codex32.c \ common/coin_mvt.c \ @@ -86,6 +87,7 @@ COMMON_SRC_NOGEN := \ common/psbt_keypath.c \ common/psbt_open.c \ common/pseudorand.c \ + common/randbytes.c \ common/random_select.c \ common/read_peer_msg.c \ common/route.c \ diff --git a/common/blindedpath.h b/common/blindedpath.h index 7a5238f15341..96a55edce0f8 100644 --- a/common/blindedpath.h +++ b/common/blindedpath.h @@ -22,7 +22,7 @@ struct tlv_encrypted_data_tlv_payment_relay; * @next_path_privkey: (out) e(i+1), the next blinding secret (optional) * @node_alias: (out) the blinded pubkey of the node to tell the recipient. * - * You create a blinding secret using randombytes_buf(), then call this + * You create a blinding secret using randbytes(), then call this * iteratively for each node in the path. */ u8 *encrypt_tlv_encrypted_data(const tal_t *ctx, diff --git a/common/clock_time.c b/common/clock_time.c new file mode 100644 index 000000000000..e3a982d77e4f --- /dev/null +++ b/common/clock_time.c @@ -0,0 +1,36 @@ +#include "config.h" +#include +#include + +static bool used = false; +static struct timeabs dev_override; + +bool clock_time_overridden(void) +{ + return dev_override.ts.tv_sec != 0; +} + +struct timeabs clock_time(void) +{ + used = true; + if (!clock_time_overridden()) + return time_now(); /* discouraged: use clock_time so we can override */ + + return dev_override; +} + +struct timeabs clock_time_progresses_(u64 *progress) +{ + if (!clock_time_overridden()) + return clock_time(); + + return timeabs_add(dev_override, time_from_sec((*progress)++)); +} + +void dev_override_clock_time(struct timeabs now) +{ + assert(!used); + + dev_override = now; + assert(clock_time_overridden()); +} diff --git a/common/clock_time.h b/common/clock_time.h new file mode 100644 index 000000000000..a267abf461d9 --- /dev/null +++ b/common/clock_time.h @@ -0,0 +1,19 @@ +#ifndef LIGHTNING_COMMON_CLOCK_TIME_H +#define LIGHTNING_COMMON_CLOCK_TIME_H +#include "config.h" +#include +#include + +/* We use this instead of time_now, for overriding when we want reproducibility */ +struct timeabs clock_time(void); + +/* If you need a clock that progresses even when reproducible, use this. */ +#define clock_time_progresses() ({static u64 progress; clock_time_progresses_(&progress);}) +struct timeabs clock_time_progresses_(u64 *progress); + +/* dev setting to override time */ +void dev_override_clock_time(struct timeabs now); + +/* Did someone override time? */ +bool clock_time_overridden(void); +#endif /* LIGHTNING_COMMON_CLOCK_TIME_H */ diff --git a/common/coin_mvt.c b/common/coin_mvt.c index e797758e697b..21550d61c120 100644 --- a/common/coin_mvt.c +++ b/common/coin_mvt.c @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -155,6 +155,12 @@ static enum mvt_tag mvt_tag_in_db(enum mvt_tag mvt_tag) abort(); } +/* This puts the coin movements in order */ +u64 coinmvt_current_time(void) +{ + return clock_time_progresses().ts.tv_sec; +} + const char *mvt_tag_str(enum mvt_tag tag) { assert((unsigned)tag < NUM_MVT_TAGS); @@ -338,7 +344,7 @@ static struct chain_coin_mvt *new_chain_coin_mvt_sat(const tal_t *ctx, assert(ok); return new_chain_coin_mvt(ctx, channel, account_name, - time_now().ts.tv_sec, tx_txid, + coinmvt_current_time(), tx_txid, outpoint, payment_hash, blockheight, tags, direction, amt_msat, /* All amounts that are sat are @@ -393,7 +399,7 @@ struct chain_coin_mvt *new_coin_channel_close(const tal_t *ctx, tags = mk_mvt_tags(MVT_CHANNEL_CLOSE); mvt = new_chain_coin_mvt(ctx, channel, alt_account, - time_now().ts.tv_sec, txid, + coinmvt_current_time(), txid, out, NULL, blockheight, tags, COIN_DEBIT, amount, @@ -421,7 +427,7 @@ struct chain_coin_mvt *new_coin_channel_open_proposed(const tal_t *ctx, if (is_leased) mvt_tag_set(&tags, MVT_LEASED); - mvt = new_chain_coin_mvt(ctx, channel, NULL, time_now().ts.tv_sec, + mvt = new_chain_coin_mvt(ctx, channel, NULL, coinmvt_current_time(), NULL, out, NULL, 0, tags, COIN_CREDIT, amount, output_val, 0); @@ -474,7 +480,7 @@ struct chain_coin_mvt *new_coin_channel_open(const tal_t *ctx, bool is_leased) { return new_coin_channel_open_general(ctx, channel, NULL, - time_now().ts.tv_sec, + coinmvt_current_time(), out, peer_id, blockheight, amount, output_val, is_opener, is_leased); } @@ -516,7 +522,7 @@ struct chain_coin_mvt *new_coin_external_spend(const tal_t *ctx, struct mvt_tags tags) { return new_chain_coin_mvt(ctx, NULL, ACCOUNT_NAME_EXTERNAL, - time_now().ts.tv_sec, txid, + coinmvt_current_time(), txid, outpoint, NULL, blockheight, tags, COIN_CREDIT, AMOUNT_MSAT(0), amount, 0); @@ -584,7 +590,7 @@ struct channel_coin_mvt *new_coin_channel_push(const tal_t *ctx, struct mvt_tags tags) { return new_coin_channel_push_general(ctx, channel, NULL, - time_now().ts.tv_sec, + coinmvt_current_time(), direction, amount, tags); } @@ -731,6 +737,10 @@ void fromwire_chain_coin_mvt(const u8 **cursor, size_t *max, struct chain_coin_m } else mvt->peer_id = NULL; mvt->timestamp = fromwire_u64(cursor, max); + + /* Align onchaind's timestamps with ours if we're deterministic */ + if (clock_time_overridden()) + mvt->timestamp = coinmvt_current_time(); } struct mvt_tags mk_mvt_tags_(enum mvt_tag tag, ...) diff --git a/common/coin_mvt.h b/common/coin_mvt.h index 822e6e821b22..b85eaedab360 100644 --- a/common/coin_mvt.h +++ b/common/coin_mvt.h @@ -346,4 +346,7 @@ bool mvt_tag_parse(const char *buf, size_t len, enum mvt_tag *tag); void towire_chain_coin_mvt(u8 **pptr, const struct chain_coin_mvt *mvt); void fromwire_chain_coin_mvt(const u8 **cursor, size_t *max, struct chain_coin_mvt *mvt); +/* Time helper for deterministic timestamps: always moves forwards */ +u64 coinmvt_current_time(void); + #endif /* LIGHTNING_COMMON_COIN_MVT_H */ diff --git a/common/daemon.c b/common/daemon.c index 6515828386be..822b715adc68 100644 --- a/common/daemon.c +++ b/common/daemon.c @@ -8,8 +8,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -201,6 +203,7 @@ void daemon_shutdown(void) bool daemon_developer_mode(char *argv[]) { bool developer = false, debug = false; + const char *entropy_override, *time_override; for (int i = 1; argv[i]; i++) { if (streq(argv[i], "--dev-debug-self")) @@ -225,6 +228,21 @@ bool daemon_developer_mode(char *argv[]) kill(getpid(), SIGSTOP); } + /* We can override cryptographic randomness with this var in development + * mode, for reproducible results */ + entropy_override = getenv("CLN_DEV_ENTROPY_SEED"); + if (entropy_override) + dev_override_randbytes(argv[0], atol(entropy_override)); + + /* We can also control TIME ITSELF! */ + time_override = getenv("CLN_DEV_SET_TIME"); + if (time_override) { + struct timeabs t; + t.ts.tv_nsec = 0; + t.ts.tv_sec = atol(time_override); + dev_override_clock_time(t); + } + /* This checks for any tal_steal loops, but it's not free: * only use if we're already using the fairly heavy memleak * detection. */ diff --git a/common/onion_message.c b/common/onion_message.c index 34faf004ca86..25e83dfb3a73 100644 --- a/common/onion_message.c +++ b/common/onion_message.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -66,7 +67,7 @@ struct blinded_path *blinded_path_from_encdata_tlvs(const tal_t *ctx, assert(nhops > 0); assert(tal_count(ids) > 0); - randombytes_buf(&first_blinding, sizeof(first_blinding)); + randbytes(&first_blinding, sizeof(first_blinding)); if (!pubkey_from_privkey(&first_blinding, &path->first_path_key)) abort(); sciddir_or_pubkey_from_pubkey(&path->first_node_id, &ids[0]); diff --git a/common/pseudorand.c b/common/pseudorand.c index b29aeee0621e..52da1e61ecdf 100644 --- a/common/pseudorand.c +++ b/common/pseudorand.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include static struct isaac64_ctx isaac64; static struct siphash_seed siphashseed; @@ -16,7 +16,7 @@ static void init_if_needed(void) unsigned char seedbuf[16]; struct sha256 sha; - randombytes_buf(seedbuf, sizeof(seedbuf)); + randbytes(seedbuf, sizeof(seedbuf)); memcpy(&siphashseed, seedbuf, sizeof(siphashseed)); /* In case isaac is reversible, don't leak seed. */ @@ -26,25 +26,42 @@ static void init_if_needed(void) } } -uint64_t pseudorand(uint64_t max) +uint64_t pseudorand_(uint64_t max, uint64_t *offset) { init_if_needed(); assert(max); + + /* We try to avoid being order-dependent here. */ + if (randbytes_overridden()) { + uint64_t rand; + randbytes_(&rand, sizeof(rand), offset); + return rand % max; + } return isaac64_next_uint(&isaac64, max); } -uint64_t pseudorand_u64(void) +uint64_t pseudorand_u64_(uint64_t *offset) { init_if_needed(); + if (randbytes_overridden()) { + uint64_t rand; + randbytes_(&rand, sizeof(rand), offset); + return rand; + } return isaac64_next_uint64(&isaac64); } -double pseudorand_double(void) +double pseudorand_double_(uint64_t *offset) { init_if_needed(); + if (randbytes_overridden()) { + uint64_t rand; + randbytes_(&rand, sizeof(rand), offset); + return rand / (double)UINT64_MAX; + } return isaac64_next_double(&isaac64); } diff --git a/common/pseudorand.h b/common/pseudorand.h index ebe9e0b59ab1..0e5ce6c56c31 100644 --- a/common/pseudorand.h +++ b/common/pseudorand.h @@ -8,18 +8,21 @@ /** * pseudorand - pseudo (guessable!) random number between 0 and max-1. */ -uint64_t pseudorand(uint64_t max); +#define pseudorand(max) ({static uint64_t offset; pseudorand_((max), &offset);}) +uint64_t pseudorand_(uint64_t max, uint64_t *offset); /** - * pseudorand - pseudo (guessable!) random number between 0 and UINT64_MAX. + * pseudorand_u64 - pseudo (guessable!) random number between 0 and UINT64_MAX. */ -uint64_t pseudorand_u64(void); +#define pseudorand_u64() ({static uint64_t offset; pseudorand_u64_(&offset);}) +uint64_t pseudorand_u64_(uint64_t *offset); /** * pseudorand - pseudo (guessable!) random number between 0 (inclusive) and 1 * (exclusive). */ -double pseudorand_double(void); +#define pseudorand_double() ({static uint64_t offset; pseudorand_double_(&offset);}) +double pseudorand_double_(uint64_t *offset); /** * Get the siphash seed for hash tables. diff --git a/common/randbytes.c b/common/randbytes.c new file mode 100644 index 000000000000..d78b5202490e --- /dev/null +++ b/common/randbytes.c @@ -0,0 +1,62 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool used = false; +static u64 dev_seed = 0; + +bool randbytes_overridden(void) +{ + return dev_seed != 0; +} + +void randbytes_(void *bytes, size_t num_bytes, u64 *offset) +{ + static u64 offset_init; + be64 pattern; + + used = true; + if (!randbytes_overridden()) { + randombytes_buf(bytes, num_bytes); /* discouraged: use randbytes() */ + return; + } + + /* First time, start callers at different offsets */ + if (*offset == 0) { + *offset = offset_init; + offset_init += 1000; + } + + /* Somewhat recognizable pattern */ + pattern = cpu_to_be64(dev_seed + (*offset)++); + for (size_t i = 0; i < num_bytes; i += sizeof(pattern)) { + size_t copy = num_bytes - i; + if (copy > sizeof(pattern)) + copy = sizeof(pattern); + + memcpy((u8 *)bytes + i, &pattern, copy); + } +} + +/* We want different seeds for each plugin (hence argv0), and for each + * lightmingd instance, (hence seed from environment) */ +void dev_override_randbytes(const char *argv0, long int seed) +{ + struct siphash_seed hashseed; + assert(!used); + + hashseed.u.u64[0] = seed; + hashseed.u.u64[1] = 0; + + dev_seed = siphash24(&hashseed, argv0, strlen(argv0)); + assert(randbytes_overridden()); +} diff --git a/common/randbytes.h b/common/randbytes.h new file mode 100644 index 000000000000..18aa485eecf3 --- /dev/null +++ b/common/randbytes.h @@ -0,0 +1,20 @@ +#ifndef LIGHTNING_COMMON_RANDBYTES_H +#define LIGHTNING_COMMON_RANDBYTES_H +#include "config.h" +#include +#include +#include + +/* Usually the libsodium routine randombytes_buf, but dev options can make this deterministic */ +#define randbytes(bytes, num_bytes) \ + do { \ + static u64 offset; \ + randbytes_((bytes), (num_bytes), &offset); \ + } while(0) + +void randbytes_(void *bytes, size_t num_bytes, u64 *offset); + +void dev_override_randbytes(const char *argv0, long int seed); + +bool randbytes_overridden(void); +#endif /* LIGHTNING_COMMON_RANDBYTES_H */ diff --git a/common/sphinx.c b/common/sphinx.c index dd405a05f3e0..6ac128d9a7a1 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -5,13 +5,13 @@ #include #include #include +#include #include #include #include #include -#include #define BLINDING_FACTOR_SIZE 32 @@ -566,7 +566,7 @@ struct onionpacket *create_onionpacket( if (sp->session_key == NULL) { sp->session_key = tal(sp, struct secret); - randombytes_buf(sp->session_key, sizeof(struct secret)); + randbytes(sp->session_key, sizeof(struct secret)); } params = generate_hop_params(ctx, sp->session_key->data, sp); diff --git a/common/test/Makefile b/common/test/Makefile index 8ae6a41366fb..ab192ce61dc5 100644 --- a/common/test/Makefile +++ b/common/test/Makefile @@ -5,6 +5,8 @@ COMMON_TEST_PROGRAMS := $(COMMON_TEST_OBJS:.o=) COMMON_TEST_COMMON_OBJS := \ common/autodata.o \ + common/randbytes.o \ + common/clock_time.o \ common/setup.o \ common/utils.o @@ -119,6 +121,7 @@ common/test/run-trace: \ common/amount.o \ common/memleak.o \ common/pseudorand.o \ + common/randbytes.o \ common/trace.o \ wire/fromwire.o \ wire/towire.o diff --git a/common/test/run-amount.c b/common/test/run-amount.c index 0e9f295dc0c2..f76484ef3436 100644 --- a/common/test/run-amount.c +++ b/common/test/run-amount.c @@ -1,5 +1,6 @@ #include "config.h" #include "../amount.c" +#include #include #include diff --git a/common/test/run-base64.c b/common/test/run-base64.c index c36c08bb3fc3..e71371c9873b 100644 --- a/common/test/run-base64.c +++ b/common/test/run-base64.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include diff --git a/common/test/run-bigsize.c b/common/test/run-bigsize.c index 8ff40e5497a6..1e167fd7c559 100644 --- a/common/test/run-bigsize.c +++ b/common/test/run-bigsize.c @@ -4,6 +4,7 @@ #include #include #include +#include #include static const char *reason; diff --git a/common/test/run-blindedpath_enctlv.c b/common/test/run-blindedpath_enctlv.c index c9408cba2dd5..d2552ad4e727 100644 --- a/common/test/run-blindedpath_enctlv.c +++ b/common/test/run-blindedpath_enctlv.c @@ -3,6 +3,7 @@ #include "../blinding.c" #include "../hmac.c" #include +#include #include #include diff --git a/common/test/run-bolt11.c b/common/test/run-bolt11.c index e4fbbd797cbd..bd8c2c662db2 100644 --- a/common/test/run-bolt11.c +++ b/common/test/run-bolt11.c @@ -6,11 +6,12 @@ #include "../features.c" #include "../node_id.c" #include "../hash_u5.c" -#include "../memleak.c" #include "../wire/fromwire.c" #include "../wire/towire.c" #include +#include #include +#include /* AUTOGENERATED MOCKS START */ /* AUTOGENERATED MOCKS END */ diff --git a/common/test/run-bolt12-encode-test.c b/common/test/run-bolt12-encode-test.c index e7de25fdd381..5ca641f93012 100644 --- a/common/test/run-bolt12-encode-test.c +++ b/common/test/run-bolt12-encode-test.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/common/test/run-bolt12-format-string-test.c b/common/test/run-bolt12-format-string-test.c index e120af65f6f1..f2a5e1124f56 100644 --- a/common/test/run-bolt12-format-string-test.c +++ b/common/test/run-bolt12-format-string-test.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include diff --git a/common/test/run-bolt12-offer-decode.c b/common/test/run-bolt12-offer-decode.c index 6febbb3637b5..329a13970ac4 100644 --- a/common/test/run-bolt12-offer-decode.c +++ b/common/test/run-bolt12-offer-decode.c @@ -14,6 +14,7 @@ #include #include #include +#include #include /* AUTOGENERATED MOCKS START */ diff --git a/common/test/run-bolt12_decode.c b/common/test/run-bolt12_decode.c index 9e5333b33f3b..e4473ada7b64 100644 --- a/common/test/run-bolt12_decode.c +++ b/common/test/run-bolt12_decode.c @@ -8,6 +8,7 @@ #include #include #include +#include #include /* AUTOGENERATED MOCKS START */ diff --git a/common/test/run-bolt12_merkle-json.c b/common/test/run-bolt12_merkle-json.c index 3c7d5a047661..fefac76cc2cd 100644 --- a/common/test/run-bolt12_merkle-json.c +++ b/common/test/run-bolt12_merkle-json.c @@ -12,6 +12,7 @@ #include #include #include +#include #include /* AUTOGENERATED MOCKS START */ diff --git a/common/test/run-bolt12_merkle.c b/common/test/run-bolt12_merkle.c index 43265ef4d74f..4d2f0bc6d639 100644 --- a/common/test/run-bolt12_merkle.c +++ b/common/test/run-bolt12_merkle.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include diff --git a/common/test/run-bolt12_period.c b/common/test/run-bolt12_period.c index e4daf72f5d90..6b92f9cb63f0 100644 --- a/common/test/run-bolt12_period.c +++ b/common/test/run-bolt12_period.c @@ -5,6 +5,7 @@ #include #include #include +#include #include /* AUTOGENERATED MOCKS START */ diff --git a/common/test/run-channel_type.c b/common/test/run-channel_type.c index 46edef12b3ee..2cac84ac8f41 100644 --- a/common/test/run-channel_type.c +++ b/common/test/run-channel_type.c @@ -2,6 +2,7 @@ #include "../channel_type.c" #include "../features.c" #include +#include #include #include #include diff --git a/common/test/run-codex32.c b/common/test/run-codex32.c index 3b32582f931c..276bd02775b0 100644 --- a/common/test/run-codex32.c +++ b/common/test/run-codex32.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include diff --git a/common/test/run-coin_mvt.c b/common/test/run-coin_mvt.c index e7512c089b55..99651ccbb54e 100644 --- a/common/test/run-coin_mvt.c +++ b/common/test/run-coin_mvt.c @@ -1,6 +1,7 @@ #include "config.h" #include "../coin_mvt.c" #include +#include #include #include diff --git a/common/test/run-cryptomsg.c b/common/test/run-cryptomsg.c index 9b7619d95bfe..0284f9da1f94 100644 --- a/common/test/run-cryptomsg.c +++ b/common/test/run-cryptomsg.c @@ -1,5 +1,6 @@ #include "config.h" #include +#include #include #include #include diff --git a/common/test/run-deprecation.c b/common/test/run-deprecation.c index fe1a4cd0579c..fa1378c8648b 100644 --- a/common/test/run-deprecation.c +++ b/common/test/run-deprecation.c @@ -6,6 +6,7 @@ static const char *test_next_version; #include "../deprecation.c" #include +#include #include #include #include diff --git a/common/test/run-derive_basepoints.c b/common/test/run-derive_basepoints.c index 861a65d63531..cb896ad7795a 100644 --- a/common/test/run-derive_basepoints.c +++ b/common/test/run-derive_basepoints.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include diff --git a/common/test/run-features.c b/common/test/run-features.c index 9d2d37d47d2e..efbea7f9fbe6 100644 --- a/common/test/run-features.c +++ b/common/test/run-features.c @@ -3,6 +3,7 @@ #include "../memleak.c" #include #include +#include #include /* AUTOGENERATED MOCKS START */ diff --git a/common/test/run-gossmap-fp16.c b/common/test/run-gossmap-fp16.c index 274eccf3a3fd..539b129b5293 100644 --- a/common/test/run-gossmap-fp16.c +++ b/common/test/run-gossmap-fp16.c @@ -1,5 +1,6 @@ #include "config.h" #include "../fp16.c" +#include #include #include #include diff --git a/common/test/run-htable.c b/common/test/run-htable.c index 64ac0b568555..75164ce204fe 100644 --- a/common/test/run-htable.c +++ b/common/test/run-htable.c @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include #include #include diff --git a/common/test/run-ip_port_parsing.c b/common/test/run-ip_port_parsing.c index 026a06f8f5d4..c7c5ba226530 100644 --- a/common/test/run-ip_port_parsing.c +++ b/common/test/run-ip_port_parsing.c @@ -4,6 +4,7 @@ #include #include +#include #include /* AUTOGENERATED MOCKS START */ diff --git a/common/test/run-json.c b/common/test/run-json.c index 529d96c62c47..18130c236945 100644 --- a/common/test/run-json.c +++ b/common/test/run-json.c @@ -4,7 +4,9 @@ #include #include #include +#include #include +#include #include #include #include diff --git a/common/test/run-json_filter.c b/common/test/run-json_filter.c index 54138cbe6999..d682b82130b3 100644 --- a/common/test/run-json_filter.c +++ b/common/test/run-json_filter.c @@ -6,6 +6,7 @@ #include "../json_stream.c" #include #include +#include #include #include diff --git a/common/test/run-json_remove.c b/common/test/run-json_remove.c index f1c16a28cbe4..8a15653ebfae 100644 --- a/common/test/run-json_remove.c +++ b/common/test/run-json_remove.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include diff --git a/common/test/run-json_scan.c b/common/test/run-json_scan.c index ab3e487ca37e..877bce6be077 100644 --- a/common/test/run-json_scan.c +++ b/common/test/run-json_scan.c @@ -2,6 +2,7 @@ #include "../json_parse.c" #include "../json_parse_simple.c" #include +#include #include #include #include diff --git a/common/test/run-json_stream-filter.c b/common/test/run-json_stream-filter.c index 1e484b475868..6f6fe060f834 100644 --- a/common/test/run-json_stream-filter.c +++ b/common/test/run-json_stream-filter.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/common/test/run-jsonrpc_io.c b/common/test/run-jsonrpc_io.c index 4d5e604bb1f7..d0f75ff2fa89 100644 --- a/common/test/run-jsonrpc_io.c +++ b/common/test/run-jsonrpc_io.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/common/test/run-key_derive.c b/common/test/run-key_derive.c index f30fcdfb2e18..6b158f74413d 100644 --- a/common/test/run-key_derive.c +++ b/common/test/run-key_derive.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/common/test/run-lease_rates.c b/common/test/run-lease_rates.c index 31da81c7d317..f995eb88c2f6 100644 --- a/common/test/run-lease_rates.c +++ b/common/test/run-lease_rates.c @@ -1,6 +1,7 @@ #include "config.h" #include "../amount.c" #include "../lease_rates.c" +#include #include #include diff --git a/common/test/run-marginal_feerate.c b/common/test/run-marginal_feerate.c index e0514cfdd83f..097852ceabc8 100644 --- a/common/test/run-marginal_feerate.c +++ b/common/test/run-marginal_feerate.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include "../fee_states.c" diff --git a/common/test/run-param.c b/common/test/run-param.c index 3cb94bf854c7..2f73a70e5ced 100644 --- a/common/test/run-param.c +++ b/common/test/run-param.c @@ -7,6 +7,9 @@ #include #include #include +#include +#include +#include #include #include diff --git a/common/test/run-psbt_diff.c b/common/test/run-psbt_diff.c index 6d355697792d..1c107851ac66 100644 --- a/common/test/run-psbt_diff.c +++ b/common/test/run-psbt_diff.c @@ -1,4 +1,5 @@ #include "config.h" +#include #include #include #include "../amount.c" @@ -37,9 +38,9 @@ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) /* Generated stub for fromwire_u8_array */ void fromwire_u8_array(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, u8 *arr UNNEEDED, size_t num UNNEEDED) { fprintf(stderr, "fromwire_u8_array called!\n"); abort(); } -/* Generated stub for pseudorand_u64 */ -uint64_t pseudorand_u64(void) -{ fprintf(stderr, "pseudorand_u64 called!\n"); abort(); } +/* Generated stub for pseudorand_u64_ */ +uint64_t pseudorand_u64_(uint64_t *offset UNNEEDED) +{ fprintf(stderr, "pseudorand_u64_ called!\n"); abort(); } /* Generated stub for towire */ void towire(u8 **pptr UNNEEDED, const void *data UNNEEDED, size_t len UNNEEDED) { fprintf(stderr, "towire called!\n"); abort(); } diff --git a/common/test/run-route-infloop.c b/common/test/run-route-infloop.c index c2e532fe5191..b52921e1d9a9 100644 --- a/common/test/run-route-infloop.c +++ b/common/test/run-route-infloop.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/common/test/run-route-specific.c b/common/test/run-route-specific.c index 93f23de72c19..6e73f459d0ee 100644 --- a/common/test/run-route-specific.c +++ b/common/test/run-route-specific.c @@ -12,6 +12,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/common/test/run-route.c b/common/test/run-route.c index 6b06dab66143..53c7359181de 100644 --- a/common/test/run-route.c +++ b/common/test/run-route.c @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/common/test/run-route_blinding_test.c b/common/test/run-route_blinding_test.c index 049e2a585f88..cbd30ad4712f 100644 --- a/common/test/run-route_blinding_test.c +++ b/common/test/run-route_blinding_test.c @@ -10,8 +10,10 @@ #include #include #include +#include #include #include +#include #include #include #include diff --git a/common/test/run-shutdown_scriptpubkey.c b/common/test/run-shutdown_scriptpubkey.c index db543a9d8d60..c98610d28277 100644 --- a/common/test/run-shutdown_scriptpubkey.c +++ b/common/test/run-shutdown_scriptpubkey.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include diff --git a/common/test/run-splice_script.c b/common/test/run-splice_script.c index c3bd63fcb2fc..082f46ceab49 100644 --- a/common/test/run-splice_script.c +++ b/common/test/run-splice_script.c @@ -6,6 +6,7 @@ #include "../json_stream.c" #include "../json_parse_simple.c" #include +#include #include #include #include diff --git a/common/test/run-tlv_span.c b/common/test/run-tlv_span.c index 4055ec9bcaae..eb3ede4b6cb4 100644 --- a/common/test/run-tlv_span.c +++ b/common/test/run-tlv_span.c @@ -2,6 +2,7 @@ #include "../bolt12.c" #include "../bigsize.c" #include "../../wire/fromwire.c" +#include #include #include diff --git a/common/test/run-tlv_unknown.c b/common/test/run-tlv_unknown.c index fdecc469965d..071b10432e1d 100644 --- a/common/test/run-tlv_unknown.c +++ b/common/test/run-tlv_unknown.c @@ -5,6 +5,7 @@ #include "../../wire/towire.c" #include "../bigsize.c" #include "../bolt12.c" +#include #include #include diff --git a/common/test/run-version.c b/common/test/run-version.c index 0e04b4cb061a..3b46f56e2289 100644 --- a/common/test/run-version.c +++ b/common/test/run-version.c @@ -1,9 +1,13 @@ #include "config.h" #include "../version.c" +#include #include #include #include +/* AUTOGENERATED MOCKS START */ +/* AUTOGENERATED MOCKS END */ + int main(int argc, char *argv[]) { common_setup(argv[0]); diff --git a/common/test/run-wireaddr.c b/common/test/run-wireaddr.c index faa48e3c7c46..f5dd2ff92806 100644 --- a/common/test/run-wireaddr.c +++ b/common/test/run-wireaddr.c @@ -1,6 +1,7 @@ #include "config.h" #include #include +#include #include #include diff --git a/common/trace.c b/common/trace.c index e3018a00e67a..793c72e21777 100644 --- a/common/trace.c +++ b/common/trace.c @@ -75,7 +75,7 @@ static void init_span(struct span *s, const char *name, struct span *parent) { - struct timeabs now = time_now(); + struct timeabs now = time_now(); /* discouraged: but tracing wants non-dev time */ s->key = key; s->id = pseudorand_u64(); @@ -366,7 +366,7 @@ void trace_span_end(const void *key) trace_check_tree(); - struct timeabs now = time_now(); + struct timeabs now = time_now(); /* discouraged: but tracing wants non-dev time */ s->end_time = (now.ts.tv_sec * 1000000) + now.ts.tv_nsec / 1000; DTRACE_PROBE1(lightningd, span_end, s->id); if (trace_to_file) { diff --git a/connectd/connectd.c b/connectd/connectd.c index 761b6dd23092..f1c9a1b7b5e8 100644 --- a/connectd/connectd.c +++ b/connectd/connectd.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -130,7 +132,7 @@ static struct peer *new_peer(struct daemon *daemon, peer->draining_state = NOT_DRAINING; peer->peer_in_lastmsg = -1; peer->peer_outq = msg_queue_new(peer, false); - peer->last_recv_time = time_now(); + peer->last_recv_time = time_mono(); peer->is_websocket = is_websocket; peer->dev_writes_enabled = NULL; peer->dev_read_enabled = true; @@ -1560,7 +1562,7 @@ setup_listeners(const tal_t *ctx, if (sodium_mlock(&random, sizeof(random)) != 0) status_failed(STATUS_FAIL_INTERNAL_ERROR, "Could not lock the random prf key memory."); - randombytes_buf((void * const)&random, 32); + randbytes((void * const)&random, 32); /* generate static tor node address, take first 32 bytes from secret of node_id plus 32 random bytes from sodiom */ struct sha256 sha; struct secret ss; @@ -1585,7 +1587,7 @@ setup_listeners(const tal_t *ctx, localaddr, 0); /* get rid of blob data on our side of tor and add jitter */ - randombytes_buf((void * const)proposed_wireaddr[i].u.torservice.blob, TOR_V3_BLOBLEN); + randbytes((void * const)proposed_wireaddr[i].u.torservice.blob, TOR_V3_BLOBLEN); if (!(proposed_listen_announce[i] & ADDR_ANNOUNCE)) { continue; @@ -2259,7 +2261,7 @@ static void dev_report_fds(struct daemon *daemon, const u8 *msg) void update_recent_timestamp(struct daemon *daemon, struct gossmap *gossmap) { /* 2 hours allows for some clock drift, not too much gossip */ - u32 recent = time_now().ts.tv_sec - 7200; + u32 recent = clock_time().ts.tv_sec - 7200; /* Only update every minute */ if (daemon->gossip_recent_time + 60 > recent) diff --git a/connectd/connectd.h b/connectd/connectd.h index fd7509631961..cae8b149e116 100644 --- a/connectd/connectd.h +++ b/connectd/connectd.h @@ -105,7 +105,7 @@ struct peer { struct oneshot *ping_timer; /* Last time we received traffic */ - struct timeabs last_recv_time; + struct timemono last_recv_time; /* How long have we been ignoring peer input? */ struct timemono peer_in_lasttime; diff --git a/connectd/handshake.c b/connectd/handshake.c index 09e44bfa988d..3382b50d5479 100644 --- a/connectd/handshake.c +++ b/connectd/handshake.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,7 +16,6 @@ #include #include #include -#include #ifndef SUPERVERBOSE #define SUPERVERBOSE(...) @@ -199,7 +199,7 @@ static struct keypair generate_key(void) struct keypair k; do { - randombytes_buf(k.priv.secret.data, sizeof(k.priv.secret.data)); + randbytes(k.priv.secret.data, sizeof(k.priv.secret.data)); } while (!secp256k1_ec_pubkey_create(secp256k1_ctx, &k.pub.pubkey, k.priv.secret.data)); return k; diff --git a/connectd/multiplex.c b/connectd/multiplex.c index a06a2107979d..5c6fc4fe5651 100644 --- a/connectd/multiplex.c +++ b/connectd/multiplex.c @@ -634,8 +634,8 @@ static void send_ping(struct peer *peer) /* If it's still sending us traffic, maybe ping reply is backed up? * That's OK, ping is just to make sure it's still alive, and clearly * it is. */ - if (time_before(peer->last_recv_time, - timeabs_sub(time_now(), time_from_sec(60)))) { + if (timemono_before(peer->last_recv_time, + timemono_sub(time_mono(), time_from_sec(60)))) { /* Already have a ping in flight? */ if (peer->expecting_pong != PONG_UNEXPECTED) { status_peer_debug(&peer->id, "Last ping unreturned: hanging up"); @@ -1258,7 +1258,7 @@ static struct io_plan *read_body_from_peer_done(struct io_conn *peer_conn, } /* We got something! */ - peer->last_recv_time = time_now(); + peer->last_recv_time = time_mono(); /* Don't process packets while we're closing */ if (peer->draining_state != NOT_DRAINING) diff --git a/connectd/test/run-crc32_of_update.c b/connectd/test/run-crc32_of_update.c index 4555c12daf04..85521395ff6d 100644 --- a/connectd/test/run-crc32_of_update.c +++ b/connectd/test/run-crc32_of_update.c @@ -7,6 +7,7 @@ int unused_main(int argc, char *argv[]); #include #include #include +#include #include #include #include diff --git a/connectd/test/run-initiator-success.c b/connectd/test/run-initiator-success.c index 6a652ca5c78e..fd59a040bfeb 100644 --- a/connectd/test/run-initiator-success.c +++ b/connectd/test/run-initiator-success.c @@ -17,10 +17,10 @@ /* AUTOGENERATED MOCKS END */ /* No randomness please, we want to replicate test vectors. */ -#include +#include -static void seed_randomness(u8 *secret, size_t len); -#define randombytes_buf(secret, len) seed_randomness((secret), (len)) +static void seed_randomness(u8 *secret, size_t len, u64 *offset); +#define randbytes_(secret, len, offset) seed_randomness((secret), (len), (offset)) struct handshake; static struct io_plan *test_write(struct io_conn *conn, @@ -91,7 +91,7 @@ const void *trc; static struct pubkey rs_pub, ls_pub, e_pub; static struct privkey ls_priv, e_priv; -static void seed_randomness(u8 *secret, size_t len) +static void seed_randomness(u8 *secret, size_t len, u64 *offset) { assert(len == sizeof(e_priv)); memcpy(secret, &e_priv, len); diff --git a/connectd/test/run-netaddress.c b/connectd/test/run-netaddress.c index 889ef887d401..a24b0d93c250 100644 --- a/connectd/test/run-netaddress.c +++ b/connectd/test/run-netaddress.c @@ -1,6 +1,7 @@ #include "config.h" #include #include +#include #include #include #include diff --git a/connectd/test/run-responder-success.c b/connectd/test/run-responder-success.c index c4c2e7339feb..9e93f802b78b 100644 --- a/connectd/test/run-responder-success.c +++ b/connectd/test/run-responder-success.c @@ -17,10 +17,10 @@ /* AUTOGENERATED MOCKS END */ /* No randomness please, we want to replicate test vectors. */ -#include +#include -static void seed_randomness(u8 *secret, size_t len); -#define randombytes_buf(secret, len) seed_randomness((secret), (len)) +static void seed_randomness(u8 *secret, size_t len, u64 *offset); +#define randbytes_(secret, len, offset) seed_randomness((secret), (len), (offset)) struct handshake; static struct io_plan *test_write(struct io_conn *conn, @@ -90,7 +90,7 @@ extern secp256k1_context *secp256k1_ctx; static struct pubkey ls_pub, e_pub; static struct privkey ls_priv, e_priv; -static void seed_randomness(u8 *secret, size_t len) +static void seed_randomness(u8 *secret, size_t len, u64 *offset) { assert(len == sizeof(e_priv)); memcpy(secret, &e_priv, len); diff --git a/connectd/test/run-websocket.c b/connectd/test/run-websocket.c index 809618dfc66c..d31e889b2533 100644 --- a/connectd/test/run-websocket.c +++ b/connectd/test/run-websocket.c @@ -1,6 +1,7 @@ #include "config.h" #include #include +#include #include #include #include diff --git a/contrib/pyln-testing/pyln/testing/fixtures.py b/contrib/pyln-testing/pyln/testing/fixtures.py index 694189bc8abd..db4206a1279c 100644 --- a/contrib/pyln-testing/pyln/testing/fixtures.py +++ b/contrib/pyln-testing/pyln/testing/fixtures.py @@ -125,36 +125,40 @@ def node_cls(): @pytest.fixture -def bitcoind(directory, teardown_checks): +def bitcoind(request, directory, teardown_checks): chaind = network_daemons[env('TEST_NETWORK', 'regtest')] bitcoind = chaind(bitcoin_dir=directory) - try: - bitcoind.start() - except Exception: - bitcoind.stop() - raise - - info = bitcoind.rpc.getnetworkinfo() - - # FIXME: include liquid-regtest in this check after elementsd has been - # updated - if info['version'] < 200100 and env('TEST_NETWORK') != 'liquid-regtest': - bitcoind.rpc.stop() - raise ValueError("bitcoind is too old. At least version 20100 (v0.20.1)" - " is needed, current version is {}".format(info['version'])) - elif info['version'] < 160000: - bitcoind.rpc.stop() - raise ValueError("elementsd is too old. At least version 160000 (v0.16.0)" - " is needed, current version is {}".format(info['version'])) - - info = bitcoind.rpc.getblockchaininfo() - # Make sure we have some spendable funds - if info['blocks'] < 101: - bitcoind.generate_block(101 - info['blocks']) - elif bitcoind.rpc.getwalletinfo()['balance'] < 1: - logging.debug("Insufficient balance, generating 1 block") - bitcoind.generate_block(1) + # @pytest.mark.parametrize('bitcoind', [False], indirect=True) if you don't + # want bitcoind started! + if getattr(request, 'param', True): + try: + bitcoind.start() + except Exception: + bitcoind.stop() + raise + + info = bitcoind.rpc.getnetworkinfo() + + # FIXME: include liquid-regtest in this check after elementsd has been + # updated + if info['version'] < 200100 and env('TEST_NETWORK') != 'liquid-regtest': + bitcoind.rpc.stop() + raise ValueError("bitcoind is too old. At least version 20100 (v0.20.1)" + " is needed, current version is {}".format(info['version'])) + elif info['version'] < 160000: + bitcoind.rpc.stop() + raise ValueError("elementsd is too old. At least version 160000 (v0.16.0)" + " is needed, current version is {}".format(info['version'])) + + info = bitcoind.rpc.getblockchaininfo() + + # Make sure we have some spendable funds + if info['blocks'] < 101: + bitcoind.generate_block(101 - info['blocks']) + elif bitcoind.rpc.getwalletinfo()['balance'] < 1: + logging.debug("Insufficient balance, generating 1 block") + bitcoind.generate_block(1) yield bitcoind diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index cfcb4b3aad21..b47da3e8e71e 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -411,6 +411,7 @@ def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None): self.bitcoin_dir = bitcoin_dir self.rpcport = rpcport self.prefix = 'bitcoind' + self.canned_blocks = None regtestdir = os.path.join(bitcoin_dir, 'regtest') if not os.path.exists(regtestdir): @@ -446,15 +447,18 @@ def __del__(self): if self.reserved_rpcport is not None: drop_unused_port(self.reserved_rpcport) - def start(self): + def start(self, wallet_file=None): TailableProc.start(self) self.wait_for_log("Done loading", timeout=TIMEOUT) logging.info("BitcoinD started") - try: - self.rpc.createwallet("lightningd-tests") - except JSONRPCError: - self.rpc.loadwallet("lightningd-tests") + if wallet_file: + self.rpc.restorewallet("lightningd-tests", wallet_file) + else: + try: + self.rpc.createwallet("lightningd-tests") + except JSONRPCError: + self.rpc.loadwallet("lightningd-tests") def stop(self): for p in self.proxies: @@ -468,6 +472,10 @@ def get_proxy(self): proxy.start() return proxy + def set_canned_blocks(self, blocks): + """Set blocks as an array of blocks to "generate", or None to reset""" + self.canned_blocks = blocks + # wait_for_mempool can be used to wait for the mempool before generating blocks: # True := wait for at least 1 transation # int > 0 := wait for at least N transactions @@ -482,6 +490,16 @@ def generate_block(self, numblocks=1, wait_for_mempool=0, to_addr=None, needfeer else: wait_for(lambda: len(self.rpc.getrawmempool()) >= wait_for_mempool) + # Use canned blocks if we have them (fails if we run out!). + if self.canned_blocks is not None: + ret = [] + while numblocks > 0: + self.rpc.submitblock(self.canned_blocks[0]) + ret.append(self.rpc.getbestblockhash()) + numblocks -= 1 + del self.canned_blocks[0] + return ret + mempool = self.rpc.getrawmempool(True) logging.debug("Generating {numblocks}, confirming {lenmempool} transactions: {mempool}".format( numblocks=numblocks, @@ -509,6 +527,21 @@ def generate_block(self, numblocks=1, wait_for_mempool=0, to_addr=None, needfeer return self.rpc.generatetoaddress(numblocks, to_addr) + def send_and_mine_block(self, addr, sats): + """Sometimes we want the txid. We assume it's the first tx for canned blocks""" + if self.canned_blocks: + self.generate_block(1) + # Find which non-coinbase txs sent to this address: return txid + for txid in self.rpc.getblock(self.rpc.getbestblockhash())['tx'][1:]: + for out in self.rpc.getrawtransaction(txid, 1)['vout']: + if out['scriptPubKey'].get('address') == addr: + return txid + assert False, f"No address {addr} in block {self.rpc.getblock(self.rpc.getbestblockhash())}" + + txid = self.rpc.sendtoaddress(addr, sats / 10**8) + self.generate_block(1) + return txid + def simple_reorg(self, height, shift=0): """ Reorganize chain by creating a fork at height=[height] and re-mine all mempool @@ -526,6 +559,7 @@ def simple_reorg(self, height, shift=0): forward to h1. 2. Set [height]=h2 and [shift]= h1-h2 """ + assert self.canned_blocks is None hashes = [] fee_delta = 1000000 orig_len = self.rpc.getblockcount() @@ -559,7 +593,7 @@ def save_blocks(self): """Bundle up blocks into an array, for restore_blocks""" blocks = [] numblocks = self.rpc.getblockcount() - for bnum in range(1, numblocks): + for bnum in range(1, numblocks + 1): bhash = self.rpc.getblockhash(bnum) blocks.append(self.rpc.getblock(bhash, False)) return blocks @@ -892,6 +926,10 @@ def __init__(self, node_id, lightning_dir, bitcoind, executor, valgrind, may_fai self.daemon.opts['grpc-port'] = grpc_port self.grpc_port = grpc_port or 9736 + # If bitcoind is serving canned blocks, it will keep initialblockdownload on true! + if self.bitcoin.canned_blocks is not None: + self.daemon.opts['dev-ignore-ibd'] = True + def _create_rpc(self, jsonschemas): """Prepares anything related to the RPC. """ @@ -987,10 +1025,12 @@ def openchannel(self, remote_node, capacity=FUNDAMOUNT, addrtype="bech32", confi def fundwallet(self, sats, addrtype="bech32", mine_block=True): addr = self.rpc.newaddr(addrtype)[addrtype] - txid = self.bitcoin.rpc.sendtoaddress(addr, sats / 10**8) if mine_block: - self.bitcoin.generate_block(1) + txid = self.bitcoin.send_and_mine_block(addr, sats) self.daemon.wait_for_log('Owning output .* txid {} CONFIRMED'.format(txid)) + else: + txid = self.bitcoin.rpc.sendtoaddress(addr, sats / 10**8) + return addr, txid def fundbalancedchannel(self, remote_node, total_capacity=FUNDAMOUNT, announce=True): @@ -1118,8 +1158,7 @@ def has_funds_on_addr(addr): # We should not have funds on that address yet, we just generated it. assert not has_funds_on_addr(addr) - self.bitcoin.rpc.sendtoaddress(addr, (amount + 1000000) / 10**8) - self.bitcoin.generate_block(1) + self.bitcoin.send_and_mine_block(addr, amount + 1000000) # Now we should. wait_for(lambda: has_funds_on_addr(addr)) @@ -1134,10 +1173,18 @@ def has_funds_on_addr(addr): **kwargs) blockid = self.bitcoin.generate_block(1, wait_for_mempool=res['txid'])[0] + txnum = None for i, txid in enumerate(self.bitcoin.rpc.getblock(blockid)['tx']): if txid == res['txid']: txnum = i + if txnum is None: + print(f"mempool = {self.bitcoin.rpc.getrawmempool()}") + print("txs:") + for txid in self.bitcoin.rpc.getblock(blockid)['tx'][1:]: + print(f"txid {txid}: {self.bitcoin.rpc.getrawtransaction(txid)} {self.bitcoin.rpc.getrawtransaction(txid, 1)}") + assert False, f"txid {res['txid']} not found" + scid = "{}x{}x{}".format(self.bitcoin.rpc.getblockcount(), txnum, res['outnum']) diff --git a/devtools/bolt11-cli.c b/devtools/bolt11-cli.c index 02204fb15b44..17a7fa6b3110 100644 --- a/devtools/bolt11-cli.c +++ b/devtools/bolt11-cli.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -109,7 +110,7 @@ static void encode(const tal_t *ctx, struct pubkey me; bool explicit_n = false; - b11->timestamp = time_now().ts.tv_sec; + b11->timestamp = clock_time().ts.tv_sec; b11->chain = chainparams_for_network("regtest"); b11->expiry = 3600; b11->min_final_cltv_expiry = DEFAULT_FINAL_CLTV_DELTA; diff --git a/devtools/gossipwith.c b/devtools/gossipwith.c index 84b396828904..7f85e8eecd1b 100644 --- a/devtools/gossipwith.c +++ b/devtools/gossipwith.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -209,7 +210,7 @@ static struct io_plan *handshake_success(struct io_conn *conn, msg = towire_gossip_timestamp_filter(NULL, &chainparams->genesis_blockhash, all_gossip ? 0 - : no_gossip ? 0xFFFFFFFF : time_now().ts.tv_sec, + : no_gossip ? 0xFFFFFFFF : clock_time().ts.tv_sec, 0xFFFFFFFF); sync_crypto_write(peer_fd, cs, take(msg)); } diff --git a/gossipd/gossip_store.c b/gossipd/gossip_store.c index 21436251407f..eff86c679555 100644 --- a/gossipd/gossip_store.c +++ b/gossipd/gossip_store.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -194,7 +195,7 @@ static int gossip_store_compact(struct daemon *daemon, struct gossip_hdr hdr; u8 oldversion, version = GOSSIP_STORE_VER; struct stat st; - struct timeabs start = time_now(); + struct timemono start = time_mono(); const char *bad; *populated = false; @@ -341,7 +342,7 @@ static int gossip_store_compact(struct daemon *daemon, /* If we have any contents, and the file is less than 1 hour * old, say "seems good" */ - if (st.st_mtime > time_now().ts.tv_sec - 3600 && *total_len > 1) { + if (st.st_mtime > clock_time().ts.tv_sec - 3600 && *total_len > 1) { *populated = true; } @@ -360,7 +361,7 @@ static int gossip_store_compact(struct daemon *daemon, } status_debug("Store compact time: %"PRIu64" msec", - time_to_msec(time_between(time_now(), start))); + time_to_msec(timemono_between(time_mono(), start))); status_debug("gossip_store: Read %zu/%zu/%zu/%zu cannounce/cupdate/nannounce/delete from store in %"PRIu64" bytes, now %"PRIu64" bytes (populated=%s)", cannounces, cupdates, nannounces, deleted, old_len, *total_len, diff --git a/gossipd/gossipd.c b/gossipd/gossipd.c index a3661912d745..15bd4a46fee5 100644 --- a/gossipd/gossipd.c +++ b/gossipd/gossipd.c @@ -12,6 +12,7 @@ */ #include "config.h" #include +#include #include #include #include @@ -367,19 +368,11 @@ static void master_or_connectd_gone(struct daemon_conn *dc UNUSED) exit(2); } -struct timeabs gossip_time_now(const struct daemon *daemon) -{ - if (daemon->dev_gossip_time) - return *daemon->dev_gossip_time; - - return time_now(); -} - /* We don't check this when loading from the gossip_store: that would break * our canned tests, and usually old gossip is better than no gossip */ bool timestamp_reasonable(const struct daemon *daemon, u32 timestamp) { - u64 now = gossip_time_now(daemon).ts.tv_sec; + u64 now = clock_time().ts.tv_sec; /* More than one day ahead? */ if (timestamp > now + 24*60*60) @@ -393,27 +386,16 @@ bool timestamp_reasonable(const struct daemon *daemon, u32 timestamp) /*~ Parse init message from lightningd: starts the daemon properly. */ static void gossip_init(struct daemon *daemon, const u8 *msg) { - u32 *dev_gossip_time; - if (!fromwire_gossipd_init(daemon, msg, &chainparams, &daemon->our_features, &daemon->id, - &dev_gossip_time, &daemon->dev_fast_gossip, &daemon->dev_fast_gossip_prune, &daemon->autoconnect_seeker_peers)) { master_badmsg(WIRE_GOSSIPD_INIT, msg); } - if (dev_gossip_time) { - assert(daemon->developer); - daemon->dev_gossip_time = tal(daemon, struct timeabs); - daemon->dev_gossip_time->ts.tv_sec = *dev_gossip_time; - daemon->dev_gossip_time->ts.tv_nsec = 0; - tal_free(dev_gossip_time); - } - /* Gossmap manager starts up */ daemon->gm = gossmap_manage_new(daemon, daemon); @@ -481,18 +463,6 @@ static void dev_gossip_memleak(struct daemon *daemon, const u8 *msg) found_leak))); } -static void dev_gossip_set_time(struct daemon *daemon, const u8 *msg) -{ - u32 time; - - if (!fromwire_gossipd_dev_set_time(msg, &time)) - master_badmsg(WIRE_GOSSIPD_DEV_SET_TIME, msg); - if (!daemon->dev_gossip_time) - daemon->dev_gossip_time = tal(daemon, struct timeabs); - daemon->dev_gossip_time->ts.tv_sec = time; - daemon->dev_gossip_time->ts.tv_nsec = 0; -} - /*~ lightningd tells us when about a gossip message directly, when told to by * the addgossip RPC call. That's usually used when a plugin gets an update * returned in an payment error. */ @@ -584,12 +554,6 @@ static struct io_plan *recv_req(struct io_conn *conn, goto done; } /* fall thru */ - case WIRE_GOSSIPD_DEV_SET_TIME: - if (daemon->developer) { - dev_gossip_set_time(daemon, msg); - goto done; - } - /* fall thru */ /* We send these, we don't receive them */ case WIRE_GOSSIPD_INIT_CUPDATE: @@ -623,7 +587,6 @@ int main(int argc, char *argv[]) daemon = tal(NULL, struct daemon); daemon->developer = developer; - daemon->dev_gossip_time = NULL; daemon->peers = new_htable(daemon, peer_node_id_map); daemon->deferred_txouts = tal_arr(daemon, struct short_channel_id, 0); daemon->current_blockheight = 0; /* i.e. unknown */ @@ -634,7 +597,7 @@ int main(int argc, char *argv[]) /* Note the use of time_mono() here. That's a monotonic clock, which * is really useful: it can only be used to measure relative events * (there's no correspondence to time-since-Ken-grew-a-beard or - * anything), but unlike time_now(), this will never jump backwards by + * anything), but unlike time_now, this will never jump backwards by * half a second and leave me wondering how my tests failed CI! */ timers_init(&daemon->timers, time_mono()); diff --git a/gossipd/gossipd.h b/gossipd/gossipd.h index 9ee193e55aef..d558f58f3a26 100644 --- a/gossipd/gossipd.h +++ b/gossipd/gossipd.h @@ -66,9 +66,6 @@ struct daemon { /* Features lightningd told us to set. */ struct feature_set *our_features; - /* Override local time for gossip messages */ - struct timeabs *dev_gossip_time; - /* Speed up gossip. */ bool dev_fast_gossip; @@ -164,13 +161,6 @@ void tell_lightningd_peer_update(struct daemon *daemon, struct amount_msat htlc_minimum, struct amount_msat htlc_maximum); -/** - * Get the local time. - * - * This gets overridden in dev mode so we can use canned (stale) gossip. - */ -struct timeabs gossip_time_now(const struct daemon *daemon); - /** * Is this gossip timestamp reasonable? */ diff --git a/gossipd/gossipd_wire.csv b/gossipd/gossipd_wire.csv index c5868eb74843..1721b3aeed4f 100644 --- a/gossipd/gossipd_wire.csv +++ b/gossipd/gossipd_wire.csv @@ -9,7 +9,6 @@ msgtype,gossipd_init,3000 msgdata,gossipd_init,chainparams,chainparams, msgdata,gossipd_init,our_features,feature_set, msgdata,gossipd_init,id,node_id, -msgdata,gossipd_init,dev_gossip_time,?u32, msgdata,gossipd_init,dev_fast_gossip,bool, msgdata,gossipd_init,dev_fast_gossip_prune,bool, msgdata,gossipd_init,autoconnect_seeker_peers,u32, @@ -27,10 +26,6 @@ msgdata,gossipd_init_nannounce,nannounce,u8,len msgtype,gossipd_init_reply,3100 -# In developer mode, we can mess with time. -msgtype,gossipd_dev_set_time,3001 -msgdata,gossipd_dev_set_time,dev_gossip_time,u32, - # Gossipd->master get this tx output please. msgtype,gossipd_get_txout,3018 msgdata,gossipd_get_txout,short_channel_id,short_channel_id, diff --git a/gossipd/gossmap_manage.c b/gossipd/gossmap_manage.c index bf4faee561aa..3227465da5ae 100644 --- a/gossipd/gossmap_manage.c +++ b/gossipd/gossmap_manage.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -357,7 +358,7 @@ static bool channel_already_dying(const struct chan_dying dying_channels[], /* Every half a week we look for dead channels (faster in dev) */ static void prune_network(struct gossmap_manage *gm) { - u64 now = gossip_time_now(gm->daemon).ts.tv_sec; + u64 now = clock_time().ts.tv_sec; /* Anything below this highwater mark ought to be pruned */ const s64 highwater = now - GOSSIP_PRUNE_INTERVAL(gm->daemon->dev_fast_gossip_prune); const struct gossmap_node *me; diff --git a/gossipd/seeker.c b/gossipd/seeker.c index 4ba862e93c6b..c632e8b8b6d5 100644 --- a/gossipd/seeker.c +++ b/gossipd/seeker.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -242,7 +243,7 @@ static void enable_gossip_stream(struct seeker *seeker, struct peer *peer, start = 0; } else { /* Just in case they care */ - start = time_now().ts.tv_sec - GOSSIP_SEEKER_INTERVAL(seeker) * 10; + start = clock_time().ts.tv_sec - GOSSIP_SEEKER_INTERVAL(seeker) * 10; } status_peer_debug(&peer->id, "seeker: starting gossip (%s)", diff --git a/gossipd/test/run-check_channel_announcement.c b/gossipd/test/run-check_channel_announcement.c index b16b887fec4a..9de5a730621a 100644 --- a/gossipd/test/run-check_channel_announcement.c +++ b/gossipd/test/run-check_channel_announcement.c @@ -33,8 +33,10 @@ In particular, we set feature bit 19. The spec says we should set feature bit 1 #include #include #include +#include #include #include +#include #include #include #include diff --git a/gossipd/test/run-extended-info.c b/gossipd/test/run-extended-info.c index 417c0eb8b139..c5f3752b3f47 100644 --- a/gossipd/test/run-extended-info.c +++ b/gossipd/test/run-extended-info.c @@ -5,9 +5,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include diff --git a/gossipd/test/run-txout_failure.c b/gossipd/test/run-txout_failure.c index 4d3df88e2664..d8d54545efb6 100644 --- a/gossipd/test/run-txout_failure.c +++ b/gossipd/test/run-txout_failure.c @@ -3,9 +3,11 @@ #include "../common/timeout.c" #include #include +#include #include #include #include +#include #include #include #include diff --git a/hsmd/hsmd.c b/hsmd/hsmd.c index 708fa84b388b..8391164f9314 100644 --- a/hsmd/hsmd.c +++ b/hsmd/hsmd.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -310,7 +311,7 @@ static void create_hsm(int fd, const char *passphrase) /* Initialize wally tal context for libwally operations */ /* Generate random entropy for new mnemonic */ - randombytes_buf(entropy, sizeof(entropy)); + randbytes(entropy, sizeof(entropy)); /* Generate mnemonic from entropy */ tal_wally_start(); diff --git a/lightningd/channel.c b/lightningd/channel.c index 798c7ad7a35e..622c866df059 100644 --- a/lightningd/channel.c +++ b/lightningd/channel.c @@ -1,9 +1,11 @@ #include "config.h" #include #include +#include #include #include #include +#include #include #include #include @@ -1005,7 +1007,7 @@ void channel_set_state(struct channel *channel, struct channel_state_change *change; change = new_channel_state_change(channel->state_changes, - time_now(), + clock_time(), old_state, state, reason, diff --git a/lightningd/channel.h b/lightningd/channel.h index 3606d0f98941..1281c61e9a06 100644 --- a/lightningd/channel.h +++ b/lightningd/channel.h @@ -295,7 +295,7 @@ struct channel { u32 feerate_base, feerate_ppm; /* But allow these feerates/htlcs up until this time. */ - struct timeabs old_feerate_timeout; + struct timemono old_feerate_timeout; u32 old_feerate_base, old_feerate_ppm; struct amount_msat old_htlc_minimum_msat, old_htlc_maximum_msat; diff --git a/lightningd/channel_gossip.c b/lightningd/channel_gossip.c index 79b12ffc5f02..6cc925200fd8 100644 --- a/lightningd/channel_gossip.c +++ b/lightningd/channel_gossip.c @@ -1,6 +1,7 @@ #include "config.h" #include #include +#include #include #include #include @@ -580,7 +581,7 @@ static void arm_refresh_timer(struct channel *channel) { struct lightningd *ld = channel->peer->ld; struct channel_gossip *cg = channel->channel_gossip; - struct timeabs now = time_now(), due; + struct timeabs now = clock_time(), due; u32 timestamp; if (!channel_update_details(cg->cupdate, ×tamp, NULL)) { @@ -1186,7 +1187,7 @@ void channel_gossip_init_done(struct lightningd *ld) static void channel_reestablished_stable(struct channel *channel) { channel->stable_conn_timer = NULL; - channel->last_stable_connection = time_now().ts.tv_sec; + channel->last_stable_connection = clock_time().ts.tv_sec; wallet_channel_save(channel->peer->ld->wallet, channel); } diff --git a/lightningd/coin_mvts.c b/lightningd/coin_mvts.c index dcbc30594f89..a33037e0e1c8 100644 --- a/lightningd/coin_mvts.c +++ b/lightningd/coin_mvts.c @@ -1,5 +1,6 @@ #include "config.h" #include +#include #include #include #include @@ -10,7 +11,7 @@ struct channel_coin_mvt *new_channel_mvt_invoice_hin(const tal_t *ctx, const struct htlc_in *hin, const struct channel *channel) { - return new_channel_coin_mvt(ctx, channel, time_now().ts.tv_sec, + return new_channel_coin_mvt(ctx, channel, coinmvt_current_time(), &hin->payment_hash, NULL, NULL, COIN_CREDIT, hin->msat, mk_mvt_tags(MVT_INVOICE), @@ -30,7 +31,7 @@ struct channel_coin_mvt *new_channel_mvt_routed_hin(const tal_t *ctx, hin->payload->amt_to_forward)) return NULL; - return new_channel_coin_mvt(ctx, channel, time_now().ts.tv_sec, + return new_channel_coin_mvt(ctx, channel, coinmvt_current_time(), &hin->payment_hash, NULL, NULL, COIN_CREDIT, hin->msat, mk_mvt_tags(MVT_ROUTED), @@ -41,7 +42,7 @@ struct channel_coin_mvt *new_channel_mvt_invoice_hout(const tal_t *ctx, const struct htlc_out *hout, const struct channel *channel) { - return new_channel_coin_mvt(ctx, channel, time_now().ts.tv_sec, + return new_channel_coin_mvt(ctx, channel, coinmvt_current_time(), &hout->payment_hash, &hout->partid, &hout->groupid, @@ -54,7 +55,7 @@ struct channel_coin_mvt *new_channel_mvt_routed_hout(const tal_t *ctx, const struct htlc_out *hout, const struct channel *channel) { - return new_channel_coin_mvt(ctx, channel, time_now().ts.tv_sec, + return new_channel_coin_mvt(ctx, channel, coinmvt_current_time(), &hout->payment_hash, NULL, NULL, COIN_DEBIT, hout->msat, mk_mvt_tags(MVT_ROUTED), @@ -66,7 +67,7 @@ struct channel_coin_mvt *new_channel_mvt_penalty_adj(const tal_t *ctx, struct amount_msat amount, enum coin_mvt_dir direction) { - return new_channel_coin_mvt(ctx, channel, time_now().ts.tv_sec, + return new_channel_coin_mvt(ctx, channel, coinmvt_current_time(), NULL, NULL, NULL, direction, amount, mk_mvt_tags(MVT_PENALTY_ADJ), @@ -107,7 +108,7 @@ void send_account_balance_snapshot(struct lightningd *ld) struct peer_node_id_map_iter it; snap->blockheight = get_block_height(ld->topology); - snap->timestamp = time_now().ts.tv_sec; + snap->timestamp = coinmvt_current_time(); snap->node_id = &ld->our_nodeid; /* Add the 'wallet' account balance */ diff --git a/lightningd/dual_open_control.c b/lightningd/dual_open_control.c index 1dc4c6a4b671..cb7c28b4776d 100644 --- a/lightningd/dual_open_control.c +++ b/lightningd/dual_open_control.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include #include +#include #include struct commit_rcvd { @@ -1333,7 +1335,7 @@ wallet_update_channel_commit(struct lightningd *ld, &channel->peer->id, &channel->cid, channel->scid, - time_now(), + clock_time(), DUALOPEND_OPEN_COMMIT_READY, DUALOPEND_OPEN_COMMITTED, REASON_REMOTE, @@ -1431,7 +1433,7 @@ wallet_commit_channel(struct lightningd *ld, &channel->peer->id, &channel->cid, channel->scid, - time_now(), + clock_time(), DUALOPEND_OPEN_INIT, DUALOPEND_OPEN_COMMIT_READY, REASON_REMOTE, @@ -4086,6 +4088,35 @@ static void dualopen_errmsg(struct channel *channel, err_for_them ? "sent" : "received", desc); } +/* This is a hack for CLN_DEV_ENTROPY_SEED. We cannot actually use + * the same seed for each dualopend, or they choose the same ids, and we + * clash when combining the PSBTs (this is phenomenally unlikey normally). + * So we set it (for the child) to an incrementing value. */ +static const char *dev_setup_dualopend_seed(const tal_t *ctx, struct lightningd *ld) +{ + static u64 seed_incr = 0; + char seedstr[STR_MAX_CHARS(u64)]; + const char *old_seed; + + if (!ld->developer) + return NULL; + + old_seed = getenv("CLN_DEV_ENTROPY_SEED"); + if (!old_seed) + return NULL; + + old_seed = tal_strdup(tmpctx, old_seed); + seed_incr++; + snprintf(seedstr, sizeof(seedstr), "%"PRIu64, atol(old_seed) + seed_incr); + setenv("CLN_DEV_ENTROPY_SEED", seedstr, 1); + return old_seed; +} + +static void dev_restore_seed(const char *old_seed) +{ + if (old_seed) + setenv("CLN_DEV_ENTROPY_SEED", old_seed, 1); +} bool peer_start_dualopend(struct peer *peer, struct peer_fd *peer_fd, @@ -4095,6 +4126,7 @@ bool peer_start_dualopend(struct peer *peer, u32 max_to_self_delay; struct amount_msat min_effective_htlc_capacity; const u8 *msg; + const char *dev_old_seed; hsmfd = hsm_get_client_fd(peer->ld, &peer->id, channel->unsaved_dbid, HSM_PERM_COMMITMENT_POINT @@ -4108,6 +4140,7 @@ bool peer_start_dualopend(struct peer *peer, return false; } + dev_old_seed = dev_setup_dualopend_seed(tmpctx, peer->ld); channel->owner = new_channel_subd(channel, peer->ld, "lightning_dualopend", @@ -4120,6 +4153,7 @@ bool peer_start_dualopend(struct peer *peer, channel_set_billboard, take(&peer_fd->fd), take(&hsmfd), NULL); + dev_restore_seed(dev_old_seed); if (!channel->owner) { channel_internal_error(channel, @@ -4173,6 +4207,7 @@ bool peer_restart_dualopend(struct peer *peer, int hsmfd; u32 *local_shutdown_script_wallet_index; u8 *msg; + const char *dev_old_seed; if (channel_state_uncommitted(channel->state)) return peer_start_dualopend(peer, peer_fd, channel); @@ -4192,6 +4227,7 @@ bool peer_restart_dualopend(struct peer *peer, return false; } + dev_old_seed = dev_setup_dualopend_seed(tmpctx, peer->ld); channel_set_owner(channel, new_channel_subd(channel, peer->ld, "lightning_dualopend", @@ -4204,6 +4240,8 @@ bool peer_restart_dualopend(struct peer *peer, channel_set_billboard, take(&peer_fd->fd), take(&hsmfd), NULL)); + dev_restore_seed(dev_old_seed); + if (!channel->owner) { log_broken(channel->log, "Could not subdaemon channel: %s", strerror(errno)); diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 0043de9682df..0f013b762540 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -192,7 +192,6 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIPD_GET_TXOUT_REPLY: case WIRE_GOSSIPD_OUTPOINTS_SPENT: case WIRE_GOSSIPD_DEV_MEMLEAK: - case WIRE_GOSSIPD_DEV_SET_TIME: case WIRE_GOSSIPD_NEW_BLOCKHEIGHT: case WIRE_GOSSIPD_ADDGOSSIP: /* This is a reply, so never gets through to here. */ @@ -319,7 +318,6 @@ void gossip_init(struct lightningd *ld, int connectd_fd) chainparams, ld->our_features, &ld->our_nodeid, - ld->dev_gossip_time ? &ld->dev_gossip_time: NULL, ld->dev_fast_gossip, ld->dev_fast_gossip_prune, ld->autoconnect_seeker_peers); @@ -488,29 +486,3 @@ static const struct json_command dev_set_max_scids_encode_size = { .dev_only = true, }; AUTODATA(json_command, &dev_set_max_scids_encode_size); - -static struct command_result *json_dev_gossip_set_time(struct command *cmd, - const char *buffer, - const jsmntok_t *obj UNNEEDED, - const jsmntok_t *params) -{ - u8 *msg; - u32 *time; - - if (!param(cmd, buffer, params, - p_req("time", param_number, &time), - NULL)) - return command_param_failed(); - - msg = towire_gossipd_dev_set_time(NULL, *time); - subd_send_msg(cmd->ld->gossip, take(msg)); - - return command_success(cmd, json_stream_success(cmd)); -} - -static const struct json_command dev_gossip_set_time = { - "dev-gossip-set-time", - json_dev_gossip_set_time, - .dev_only = true, -}; -AUTODATA(json_command, &dev_gossip_set_time); diff --git a/lightningd/gossip_generation.c b/lightningd/gossip_generation.c index 7a59f0e9d213..433f4fb85dcd 100644 --- a/lightningd/gossip_generation.c +++ b/lightningd/gossip_generation.c @@ -1,6 +1,7 @@ #include "config.h" #include #include +#include #include #include #include @@ -119,7 +120,7 @@ u8 *unsigned_channel_update(const tal_t *ctx, message_flags |= ROUTING_OPT_DONT_FORWARD; /* Make sure timestamp changes! */ - timestamp = time_now().ts.tv_sec; + timestamp = clock_time().ts.tv_sec; /* FIXME: @endothermicdev points out that our clock could be * wrong once, and now we'll keep producing future timestamps. * We could sanity check that old_timestamp is within 2 weeks and @@ -412,7 +413,7 @@ u8 *unsigned_node_announcement(const tal_t *ctx, { secp256k1_ecdsa_signature sig; const struct wireaddr *addrs; - u32 timestamp = time_now().ts.tv_sec; + u32 timestamp = clock_time().ts.tv_sec; addrs = gather_addresses(tmpctx, ld); /* Even if we're quick, don't duplicate timestamps! */ diff --git a/lightningd/htlc_end.c b/lightningd/htlc_end.c index e2a06170dd4f..82912aeb3ee8 100644 --- a/lightningd/htlc_end.c +++ b/lightningd/htlc_end.c @@ -1,6 +1,7 @@ #include "config.h" #include #include +#include #include #include #include @@ -158,7 +159,7 @@ struct htlc_in *new_htlc_in(const tal_t *ctx, hin->we_filled = NULL; hin->payload = NULL; - hin->received_time = time_now(); + hin->received_time = clock_time(); return htlc_in_check(hin, "new_htlc_in"); } diff --git a/lightningd/invoice.c b/lightningd/invoice.c index 49933a932cc3..dc8239199b14 100644 --- a/lightningd/invoice.c +++ b/lightningd/invoice.c @@ -9,7 +9,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -20,7 +22,6 @@ #include #include #include -#include #include #include @@ -1180,8 +1181,8 @@ static struct command_result *json_invoice(struct command *cmd, info->payment_preimage = *preimage; else /* Generate random secret preimage. */ - randombytes_buf(&info->payment_preimage, - sizeof(info->payment_preimage)); + randbytes(&info->payment_preimage, + sizeof(info->payment_preimage)); /* Generate preimage hash. */ sha256(&rhash, &info->payment_preimage, sizeof(info->payment_preimage)); /* Generate payment secret. */ @@ -1189,7 +1190,7 @@ static struct command_result *json_invoice(struct command *cmd, info->b11 = new_bolt11(info, msatoshi_val); info->b11->chain = chainparams; - info->b11->timestamp = time_now().ts.tv_sec; + info->b11->timestamp = clock_time().ts.tv_sec; info->b11->payment_hash = rhash; info->b11->receiver_id = cmd->ld->our_nodeid; info->b11->min_final_cltv_expiry = *cltv; @@ -1620,7 +1621,7 @@ static void add_stub_blindedpath(const tal_t *ctx, path = tal(NULL, struct blinded_path); sciddir_or_pubkey_from_pubkey(&path->first_node_id, &ld->our_pubkey); - randombytes_buf(&path_key, sizeof(path_key)); + randbytes(&path_key, sizeof(path_key)); if (!pubkey_from_privkey(&path_key, &path->first_path_key)) abort(); path->path = tal_arr(path, struct blinded_path_hop *, 1); diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index e5e3dedd274e..7e0bf26cc49c 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -125,7 +125,6 @@ static struct lightningd *new_lightningd(const tal_t *ctx) ld->dev_disconnect_fd = -1; ld->dev_subdaemon_fail = false; ld->dev_allow_localhost = false; - ld->dev_gossip_time = 0; ld->dev_fast_gossip = false; ld->dev_fast_gossip_prune = false; ld->dev_throttle_gossip = false; @@ -1189,8 +1188,6 @@ int main(int argc, char *argv[]) bool try_reexec; size_t num_channels; - trace_span_start("lightningd/startup", argv); - /*~ What happens in strange locales should stay there. */ setup_locale(); @@ -1214,6 +1211,10 @@ int main(int argc, char *argv[]) * backtraces when we crash (if supported on this platform). */ daemon_setup(argv[0], log_backtrace_print, log_backtrace_exit); + /*~ We enable trace as early as possible, but it uses support functions + * (particularly if we're avoid entropy) so do it after daemon_setup. */ + trace_span_start("lightningd/startup", argv); + /*~ There's always a battle between what a constructor like this * should do, and what should be added later by the caller. In * general, because we use valgrind heavily for testing, we prefer not diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index 6415e6598903..2d3f028dcd9a 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -305,9 +305,6 @@ struct lightningd { /* Allow and accept localhost node_announcement addresses */ bool dev_allow_localhost; - /* Timestamp to use for gossipd, iff non-zero */ - u32 dev_gossip_time; - /* Speedup gossip propagation, for testing. */ bool dev_fast_gossip; bool dev_fast_gossip_prune; diff --git a/lightningd/log.c b/lightningd/log.c index 5b5bcd5ae272..8e12a6a13511 100644 --- a/lightningd/log.c +++ b/lightningd/log.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -398,7 +399,7 @@ struct log_book *new_log_book(struct lightningd *ld, size_t max_mem) log_book->prefix = tal_strdup(log_book, ""); list_head_init(&log_book->print_filters); list_head_init(&log_book->loggers); - log_book->init_time = time_now(); + log_book->init_time = clock_time(); log_book->ld = ld; log_book->cache = tal(log_book, struct node_id_map); node_id_map_init(log_book->cache); @@ -523,7 +524,7 @@ static struct log_entry *new_log_entry(struct logger *log, enum log_level level, tal_resize(&log->log_book->log, tal_count(log->log_book->log) * 2); l = &log->log_book->log[log->log_book->num_entries]; - l->time = time_now(); + l->time = clock_time(); l->level = level; l->skipped = 0; l->prefix = log_prefix_get(log->prefix); @@ -1012,7 +1013,7 @@ void log_backtrace_exit(void) int fd; char timebuf[sizeof("YYYYmmddHHMMSS")]; char logfile[sizeof("/tmp/lightning-crash.log.") + sizeof(timebuf)]; - struct timeabs time = time_now(); + struct timeabs time = clock_time(); strftime(timebuf, sizeof(timebuf), "%Y%m%d%H%M%S", gmtime(&time.ts.tv_sec)); diff --git a/lightningd/opening_control.c b/lightningd/opening_control.c index 8570160a1987..eaa806caabcb 100644 --- a/lightningd/opening_control.c +++ b/lightningd/opening_control.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -240,7 +241,7 @@ wallet_commit_channel(struct lightningd *ld, wallet_channel_insert(ld->wallet, channel); /* Notify that channel state changed (from non existant to existant) */ - timestamp = time_now(); + timestamp = clock_time(); notify_channel_state_changed(ld, &channel->peer->id, &channel->cid, channel->scid, /* NULL */ diff --git a/lightningd/options.c b/lightningd/options.c index 2175d434196b..02b84877c7cb 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -824,10 +824,6 @@ static void dev_register_opts(struct lightningd *ld) opt_set_bool, &ld->dev_fast_gossip_prune, "Make gossip pruning 120 seconds"); - clnopt_witharg("--dev-gossip-time", OPT_DEV|OPT_SHOWINT, - opt_set_u32, opt_show_u32, - &ld->dev_gossip_time, - "UNIX time to override gossipd to use."); clnopt_witharg("--dev-force-privkey", OPT_DEV, opt_force_privkey, NULL, ld, "Force HSM to use this as node private key"); diff --git a/lightningd/pay.c b/lightningd/pay.c index b38b8e1e9f15..17a41a2ed081 100644 --- a/lightningd/pay.c +++ b/lightningd/pay.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -1147,7 +1148,7 @@ send_payment_core(struct lightningd *ld, payment = wallet_add_payment(cmd, ld->wallet, - time_now().ts.tv_sec, + clock_time().ts.tv_sec, NULL, rhash, partid, @@ -1461,7 +1462,7 @@ static struct command_result *self_payment(struct lightningd *ld, payment = wallet_add_payment(tmpctx, ld->wallet, - time_now().ts.tv_sec, + clock_time().ts.tv_sec, NULL, rhash, partid, @@ -1791,7 +1792,7 @@ static void register_payment_and_waiter(struct command *cmd, { wallet_add_payment(cmd, cmd->ld->wallet, - time_now().ts.tv_sec, + clock_time().ts.tv_sec, NULL, payment_hash, partid, @@ -2018,7 +2019,7 @@ static struct command_result *json_injectpaymentonion(struct command *cmd, if (amount_msat_greater(*msat, next->htlc_maximum_msat) || amount_msat_less(*msat, next->htlc_minimum_msat)) { /* Are we in old-range grace-period? */ - if (!time_before(time_now(), next->old_feerate_timeout) + if (!timemono_before(time_mono(), next->old_feerate_timeout) || amount_msat_less(*msat, next->old_htlc_minimum_msat) || amount_msat_greater(*msat, next->old_htlc_maximum_msat)) { return command_fail(cmd, JSONRPC2_INVALID_PARAMS, diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index f5b064ca81ce..4b44384d9ee8 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -3207,7 +3207,7 @@ static void set_channel_config(struct command *cmd, struct channel *channel, || (htlc_max && amount_msat_less(*htlc_max, channel->htlc_maximum_msat))) { channel->old_feerate_timeout - = timeabs_add(time_now(), time_from_sec(delaysecs)); + = timemono_add(time_mono(), time_from_sec(delaysecs)); channel->old_feerate_base = channel->feerate_base; channel->old_feerate_ppm = channel->feerate_ppm; channel->old_htlc_minimum_msat = channel->htlc_minimum_msat; diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 7c818782bed0..548a15c82517 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -848,7 +848,7 @@ static void forward_htlc(struct htlc_in *hin, * - If it creates a new `channel_update` with updated channel parameters: * - SHOULD keep accepting the previous channel parameters for 10 minutes */ - if (!time_before(time_now(), next->old_feerate_timeout) + if (!timemono_before(time_mono(), next->old_feerate_timeout) || !check_fwd_amount(hin, amt_to_forward, hin->msat, next->old_feerate_base, next->old_feerate_ppm)) { @@ -864,7 +864,7 @@ static void forward_htlc(struct htlc_in *hin, if (amount_msat_greater(amt_to_forward, next->htlc_maximum_msat) || amount_msat_less(amt_to_forward, next->htlc_minimum_msat)) { /* Are we in old-range grace-period? */ - if (!time_before(time_now(), next->old_feerate_timeout) + if (!timemono_before(time_mono(), next->old_feerate_timeout) || amount_msat_less(amt_to_forward, next->old_htlc_minimum_msat) || amount_msat_greater(amt_to_forward, next->old_htlc_maximum_msat)) { failmsg = towire_temporary_channel_failure(tmpctx, diff --git a/lightningd/runes.c b/lightningd/runes.c index 70821bcb9c0b..36a4b10dfe31 100644 --- a/lightningd/runes.c +++ b/lightningd/runes.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -939,7 +940,7 @@ static struct command_result *json_checkrune(struct command *cmd, cinfo.buf = buffer; cinfo.method = method; cinfo.params = methodparams; - cinfo.now = time_now(); + cinfo.now = clock_time(); strmap_init(&cinfo.cached_params); err = rune_is_ours(cmd->ld, ras->rune); diff --git a/onchaind/test/run-grind_feerate-bug.c b/onchaind/test/run-grind_feerate-bug.c index ba71992cc5e8..e57aca2a457e 100644 --- a/onchaind/test/run-grind_feerate-bug.c +++ b/onchaind/test/run-grind_feerate-bug.c @@ -4,6 +4,7 @@ */ #include "config.h" #include +#include #include #include @@ -171,6 +172,12 @@ void *notleak_(void *ptr UNNEEDED, bool plus_children UNNEEDED) /* Generated stub for peer_billboard */ void peer_billboard(bool perm UNNEEDED, const char *fmt UNNEEDED, ...) { fprintf(stderr, "peer_billboard called!\n"); abort(); } +/* Generated stub for randbytes_ */ +void randbytes_(void *bytes UNNEEDED, size_t num_bytes UNNEEDED, u64 *offset UNNEEDED) +{ fprintf(stderr, "randbytes_ called!\n"); abort(); } +/* Generated stub for randbytes_overridden */ +bool randbytes_overridden(void) +{ fprintf(stderr, "randbytes_overridden called!\n"); abort(); } /* Generated stub for shachain_get_secret */ bool shachain_get_secret(const struct shachain *shachain UNNEEDED, u64 commit_num UNNEEDED, diff --git a/plugins/askrene/askrene.c b/plugins/askrene/askrene.c index d76afa68d153..d25fc4aed78f 100644 --- a/plugins/askrene/askrene.c +++ b/plugins/askrene/askrene.c @@ -9,7 +9,7 @@ #include "config.h" #include #include -#include +#include #include #include #include @@ -1061,7 +1061,7 @@ static struct command_result *json_askrene_inform_channel(struct command *cmd, *amount = AMOUNT_MSAT(0); if (command_check_only(cmd)) return command_check_done(cmd); - c = layer_add_constraint(layer, scidd, time_now().ts.tv_sec, + c = layer_add_constraint(layer, scidd, clock_time().ts.tv_sec, NULL, amount); goto output; case INFORM_UNCONSTRAINED: @@ -1069,7 +1069,7 @@ static struct command_result *json_askrene_inform_channel(struct command *cmd, * that no reserves were used) */ if (command_check_only(cmd)) return command_check_done(cmd); - c = layer_add_constraint(layer, scidd, time_now().ts.tv_sec, + c = layer_add_constraint(layer, scidd, clock_time().ts.tv_sec, amount, NULL); goto output; case INFORM_SUCCEEDED: @@ -1130,7 +1130,7 @@ static struct command_result *json_askrene_bias_channel(struct command *cmd, plugin_log(cmd->plugin, LOG_TRACE, "%s called: %.*s", __func__, json_tok_full_len(params), json_tok_full(buffer, params)); - timestamp = time_now().ts.tv_sec; + timestamp = clock_time().ts.tv_sec; b = layer_set_bias(layer, scidd, description, *bias, *relative, timestamp); response = jsonrpc_stream_success(cmd); @@ -1167,7 +1167,7 @@ static struct command_result *json_askrene_bias_node(struct command *cmd, plugin_log(cmd->plugin, LOG_TRACE, "%s called: %.*s", __func__, json_tok_full_len(params), json_tok_full(buffer, params)); - timestamp = time_now().ts.tv_sec; + timestamp = clock_time().ts.tv_sec; b = layer_set_node_bias(layer, node, description, *bias, *relative, *out_dir, timestamp); response = jsonrpc_stream_success(cmd); diff --git a/plugins/askrene/layer.c b/plugins/askrene/layer.c index 043b9168eea2..2ef27d8f17b5 100644 --- a/plugins/askrene/layer.c +++ b/plugins/askrene/layer.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -696,7 +697,7 @@ static void load_channel_bias(struct plugin *plugin, s8 bias_factor; /* If we read an old version without timestamp, just put the current * time. */ - u64 timestamp = time_now().ts.tv_sec; + u64 timestamp = clock_time().ts.tv_sec; fromwire_short_channel_id_dir(cursor, len, &scidd); bias_factor = fromwire_s8(cursor, len); diff --git a/plugins/autoclean.c b/plugins/autoclean.c index e4601cb0246e..70aa12f4f360 100644 --- a/plugins/autoclean.c +++ b/plugins/autoclean.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -546,7 +547,7 @@ static struct command_result *list_done(struct command *cmd, const struct subsystem_ops *ops = get_subsystem_ops(subsystem); const jsmntok_t *t, *inv = json_get_member(buf, result, ops->arr_name); size_t i; - u64 now = time_now().ts.tv_sec; + u64 now = clock_time().ts.tv_sec; json_for_each_arr(i, t, inv) { struct per_variant *variant; diff --git a/plugins/bcli.c b/plugins/bcli.c index 7646b34801ab..51ff4942baa7 100644 --- a/plugins/bcli.c +++ b/plugins/bcli.c @@ -66,6 +66,9 @@ struct bitcoind { /* Override in case we're developer mode for testing*/ bool dev_no_fake_fees; + + /* Override initialblockdownload (using canned blocks sets this) */ + bool dev_ignore_ibd; }; static struct bitcoind *bitcoind; @@ -77,7 +80,7 @@ struct bitcoin_cli { pid_t pid; const char **args; const char **stdinargs; - struct timeabs start; + struct timemono start; enum bitcoind_prio prio; char *output; size_t output_bytes; @@ -260,7 +263,7 @@ static void bcli_finished(struct io_conn *conn UNUSED, struct bitcoin_cli *bcli) int ret, status; struct command_result *res; enum bitcoind_prio prio = bcli->prio; - u64 msec = time_to_msec(time_between(time_now(), bcli->start)); + u64 msec = time_to_msec(timemono_between(time_mono(), bcli->start)); /* If it took over 10 seconds, that's rather strange. */ if (msec > 10000) @@ -336,7 +339,7 @@ static void next_bcli(enum bitcoind_prio prio) } close(in); - bcli->start = time_now(); + bcli->start = time_mono(); bitcoind->num_requests[prio]++; @@ -477,6 +480,9 @@ static struct command_result *process_getblockchaininfo(struct bitcoin_cli *bcli if (err) return command_err_bcli_badjson(bcli, err); + if (bitcoind->dev_ignore_ibd) + ibd = false; + response = jsonrpc_stream_success(bcli->cmd); json_add_string(response, "chain", chain); json_add_u32(response, "headercount", headers); @@ -1176,6 +1182,7 @@ static struct bitcoind *new_bitcoind(const tal_t *ctx) although normal rpcclienttimeout default value is 900. */ bitcoind->rpcclienttimeout = 60; bitcoind->dev_no_fake_fees = false; + bitcoind->dev_ignore_ibd = false; return bitcoind; } @@ -1227,5 +1234,9 @@ int main(int argc, char *argv[]) "bool", "Suppress fee faking for regtest", bool_option, NULL, &bitcoind->dev_no_fake_fees), + plugin_option_dev("dev-ignore-ibd", + "bool", + "Never tell lightningd we're doing initial block download", + bool_option, NULL, &bitcoind->dev_ignore_ibd), NULL); } diff --git a/plugins/bkpr/bookkeeper.c b/plugins/bkpr/bookkeeper.c index bba69efbafee..7521b210c016 100644 --- a/plugins/bkpr/bookkeeper.c +++ b/plugins/bkpr/bookkeeper.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/plugins/bkpr/incomestmt.c b/plugins/bkpr/incomestmt.c index d9041f56ba71..2b5cb334d62c 100644 --- a/plugins/bkpr/incomestmt.c +++ b/plugins/bkpr/incomestmt.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -438,7 +439,7 @@ const char *csv_filename(const tal_t *ctx, const struct csv_fmt *fmt) { return tal_fmt(ctx, "cln_incomestmt_%s_%lu.csv", fmt->fmt_name, - (unsigned long)time_now().ts.tv_sec); + (unsigned long)clock_time().ts.tv_sec); } static void cointrack_header(FILE *csvf) diff --git a/plugins/bkpr/test/run-recorder.c b/plugins/bkpr/test/run-recorder.c index bc2177e72105..848593499cc7 100644 --- a/plugins/bkpr/test/run-recorder.c +++ b/plugins/bkpr/test/run-recorder.c @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/plugins/bkpr/test/run-sql.c b/plugins/bkpr/test/run-sql.c index 28da98b7bf5a..665c4d0e874b 100644 --- a/plugins/bkpr/test/run-sql.c +++ b/plugins/bkpr/test/run-sql.c @@ -3,6 +3,7 @@ #include "plugins/bkpr/sql.c" #include "plugins/libplugin.c" +#include #include #include #include diff --git a/plugins/chanbackup.c b/plugins/chanbackup.c index 09afcb750e8e..3b0d128606a9 100644 --- a/plugins/chanbackup.c +++ b/plugins/chanbackup.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -137,7 +138,7 @@ static void write_scb(struct plugin *p, struct modern_scb_chan **scb_chan_arr) { const struct chanbackup *cb = chanbackup(p); - u32 timestamp = time_now().ts.tv_sec; + u32 timestamp = clock_time().ts.tv_sec; u8 *decrypted_scb = towire_static_chan_backup_with_tlvs(tmpctx, VERSION, diff --git a/plugins/fetchinvoice.c b/plugins/fetchinvoice.c index a18b82e18035..7a71eb444a10 100644 --- a/plugins/fetchinvoice.c +++ b/plugins/fetchinvoice.c @@ -5,17 +5,18 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include #include -#include static LIST_HEAD(sent_list); @@ -452,7 +453,7 @@ static struct blinded_path *make_reply_path(const tal_t *ctx, assert(tal_count(path) > 0); - randombytes_buf(reply_secret, sizeof(struct secret)); + randbytes(reply_secret, sizeof(struct secret)); if (sent->dev_reply_path) { ids = sent->dev_reply_path; @@ -744,7 +745,7 @@ static struct command_result *invreq_done(struct command *cmd, } if (base) { - u64 period_start, period_end, now = time_now().ts.tv_sec; + u64 period_start, period_end, now = clock_time().ts.tv_sec; offer_period_paywindow(recurrence, sent->invreq->offer_recurrence_paywindow, sent->invreq->offer_recurrence_base, @@ -942,7 +943,7 @@ struct command_result *json_fetchinvoice(struct command *cmd, * (i.e. continuing an existing offer with recurrence is ok) */ if (sent->offer->offer_absolute_expiry - && time_now().ts.tv_sec > *sent->offer->offer_absolute_expiry + && clock_time().ts.tv_sec > *sent->offer->offer_absolute_expiry && (!recurrence_counter || *recurrence_counter == 0)) { return command_fail(cmd, OFFER_EXPIRED, "Offer expired"); } @@ -1080,8 +1081,8 @@ struct command_result *json_fetchinvoice(struct command *cmd, * bytes. */ invreq->invreq_metadata = tal_arr(invreq, u8, 16); - randombytes_buf(invreq->invreq_metadata, - tal_bytelen(invreq->invreq_metadata)); + randbytes(invreq->invreq_metadata, + tal_bytelen(invreq->invreq_metadata)); } } @@ -1581,7 +1582,7 @@ struct command_result *json_sendinvoice(struct command *cmd, * `invreq_chain`. */ sent->inv->invoice_created_at = tal(sent->inv, u64); - *sent->inv->invoice_created_at = time_now().ts.tv_sec; + *sent->inv->invoice_created_at = clock_time().ts.tv_sec; /* FIXME: Support blinded paths, in which case use fake nodeid */ @@ -1589,7 +1590,7 @@ struct command_result *json_sendinvoice(struct command *cmd, * - MUST set `invoice_payment_hash` to the SHA256 hash of the * `payment_preimage` that will be given in return for payment. */ - randombytes_buf(&sent->inv_preimage, sizeof(sent->inv_preimage)); + randbytes(&sent->inv_preimage, sizeof(sent->inv_preimage)); sent->inv->invoice_payment_hash = tal(sent->inv, struct sha256); sha256(sent->inv->invoice_payment_hash, &sent->inv_preimage, sizeof(sent->inv_preimage)); diff --git a/plugins/keysend.c b/plugins/keysend.c index 58c66cbc994e..56b3b368a40f 100644 --- a/plugins/keysend.c +++ b/plugins/keysend.c @@ -3,14 +3,15 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include -#include #define PREIMAGE_TLV_TYPE 5482373484 #define KEYSEND_FEATUREBIT 55 @@ -49,7 +50,7 @@ static struct keysend_data *keysend_init(struct payment *p) * and populate the preimage field in the keysend_data and the * payment_hash in the payment. */ d = tal(p, struct keysend_data); - randombytes_buf(&d->preimage, sizeof(d->preimage)); + randbytes(&d->preimage, sizeof(d->preimage)); ccan_sha256(&payment_hash, &d->preimage, sizeof(d->preimage)); p->payment_hash = tal_dup(p, struct sha256, &payment_hash); d->extra_tlvs = NULL; @@ -239,7 +240,7 @@ static struct command_result *json_keysend(struct command *cmd, const char *buf, p->invstring_used = true; p->why = "Initial attempt"; p->constraints.cltv_budget = *maxdelay; - p->deadline = timeabs_add(time_now(), time_from_sec(*retryfor)); + p->deadline = timemono_add(time_mono(), time_from_sec(*retryfor)); p->getroute->riskfactorppm = 10000000; if (node_id_eq(&my_id, p->route_destination)) { @@ -446,7 +447,8 @@ static struct command_result *htlc_accepted_call(struct command *cmd, bigsize_t s; struct keysend_in *ki; struct out_req *req; - struct timeabs now = time_now(); + /* Even with CLN_DEV_SET_TIME, we need this to change */ + struct timeabs now = clock_time_progresses(); const char *err; u64 *allowed; size_t err_off; diff --git a/plugins/libplugin-pay.c b/plugins/libplugin-pay.c index 919ca7de3527..58deb4d0aaea 100644 --- a/plugins/libplugin-pay.c +++ b/plugins/libplugin-pay.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -88,7 +89,7 @@ struct payment *payment_new(tal_t *ctx, struct command *cmd, p->modifiers = mods; p->cmd = cmd; p->finished = false; - p->start_time = time_now(); + p->start_time = clock_time(); p->result = NULL; p->why = NULL; p->getroute = tal(p, struct getroute_request); @@ -429,7 +430,7 @@ static void channel_hints_update(struct payment *p, /* Local channels must have an HTLC budget */ assert(!local || htlc_budget != NULL); - channel_hint_set_add(root->hints, time_now().ts.tv_sec, scidd, enabled, + channel_hint_set_add(root->hints, clock_time().ts.tv_sec, scidd, enabled, estimated_capacity, overall_capacity, htlc_budget); hint = channel_hint_set_find(root->hints, scidd); @@ -1638,7 +1639,7 @@ payment_waitsendpay_finished(struct command *cmd, assert(p->route != NULL); - p->end_time = time_now(); + p->end_time = clock_time(); p->result = tal_sendpay_result_from_json(p, buffer, toks); if (p->result == NULL) { @@ -2333,7 +2334,7 @@ void payment_set_step(struct payment *p, enum payment_step newstep) /* Any final state needs an end_time */ if (p->step >= PAYMENT_STEP_SPLIT) - p->end_time = time_now(); + p->end_time = clock_time(); } struct command_result *payment_continue(struct payment *p) @@ -2392,7 +2393,7 @@ struct command_result *payment_abort(struct payment *p, enum jsonrpc_errcode cod va_list ap; struct payment *root = payment_root(p); payment_set_step(p, PAYMENT_STEP_FAILED); - p->end_time = time_now(); + p->end_time = clock_time(); /* We can fail twice, it seems. */ tal_free(p->failreason); @@ -2419,7 +2420,7 @@ struct command_result *payment_abort(struct payment *p, enum jsonrpc_errcode cod struct command_result *payment_fail(struct payment *p, const char *fmt, ...) { va_list ap; - p->end_time = time_now(); + p->end_time = clock_time(); payment_set_step(p, PAYMENT_STEP_FAILED); /* We can fail twice, it seems. */ tal_free(p->failreason); @@ -2526,12 +2527,12 @@ static struct command_result *retry_step_cb(struct retry_mod_data *rd, { struct payment *subpayment, *root = payment_root(p); struct retry_mod_data *rdata = payment_mod_retry_get_data(p); - struct timeabs now = time_now(); + struct timemono now = time_mono(); if (p->step != PAYMENT_STEP_FAILED) return payment_continue(p); - if (time_after(now, p->deadline)) { + if (timemono_after(now, p->deadline)) { paymod_log( p, LOG_INFORM, "Payment deadline expired, not retrying (partial-)payment " @@ -2645,7 +2646,7 @@ local_channel_hints_listpeerchannels(struct command *cmd, * observations, and should re-enable some channels that would * otherwise start out as excluded and remain so until * forever. */ - channel_hint_set_update(payment_root(p)->hints, time_now()); + channel_hint_set_update(payment_root(p)->hints, clock_time()); p->mods = gossmods_from_listpeerchannels( p, p->local_id, buffer, toks, true, gossmod_add_localchan, NULL); diff --git a/plugins/libplugin-pay.h b/plugins/libplugin-pay.h index 1498f0fb432a..50bd00008501 100644 --- a/plugins/libplugin-pay.h +++ b/plugins/libplugin-pay.h @@ -200,7 +200,7 @@ struct payment { u32 start_block; struct timeabs start_time, end_time; - struct timeabs deadline; + struct timemono deadline; /* Constraints the state machine and modifiers needs to maintain. */ struct payment_constraints constraints; diff --git a/plugins/offers.c b/plugins/offers.c index 383beff08e1e..735ba87b69b0 100644 --- a/plugins/offers.c +++ b/plugins/offers.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -1321,7 +1322,7 @@ static void json_add_rune(struct command *cmd, struct json_stream *js, const str u64 t = atol(alt->value); if (t) { - u64 diff, now = time_now().ts.tv_sec; + u64 diff, now = clock_time().ts.tv_sec; /* Need a non-const during construction */ char *v; diff --git a/plugins/offers_inv_hook.c b/plugins/offers_inv_hook.c index 3e72fae0fe4a..b1f859d4f7fc 100644 --- a/plugins/offers_inv_hook.c +++ b/plugins/offers_inv_hook.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -312,7 +313,7 @@ struct command_result *handle_invoice(struct command *cmd, invexpiry = *inv->inv->invoice_created_at + *inv->inv->invoice_relative_expiry; else invexpiry = *inv->inv->invoice_created_at + BOLT12_DEFAULT_REL_EXPIRY; - if (time_now().ts.tv_sec > invexpiry) + if (clock_time().ts.tv_sec > invexpiry) return fail_inv(cmd, inv, "Expired invoice"); /* BOLT #12: diff --git a/plugins/offers_invreq_hook.c b/plugins/offers_invreq_hook.c index 153def675135..6f8bcc1032db 100644 --- a/plugins/offers_invreq_hook.c +++ b/plugins/offers_invreq_hook.c @@ -5,16 +5,17 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include -#include /* We need to keep the reply path around so we can reply with invoice */ struct invreq { @@ -890,7 +891,7 @@ static struct command_result *listoffers_done(struct command *cmd, if (ir->invreq->offer_absolute_expiry && (!ir->invreq->invreq_recurrence_counter || *ir->invreq->invreq_recurrence_counter == 0) - && time_now().ts.tv_sec >= *ir->invreq->offer_absolute_expiry) { + && clock_time().ts.tv_sec >= *ir->invreq->offer_absolute_expiry) { return fail_invreq(cmd, ir, "Offer expired"); } @@ -1000,13 +1001,13 @@ static struct command_result *listoffers_done(struct command *cmd, * Midnight 1 January 1970, UTC when the invoice was created. */ ir->inv->invoice_created_at = tal(ir->inv, u64); - *ir->inv->invoice_created_at = time_now().ts.tv_sec; + *ir->inv->invoice_created_at = clock_time().ts.tv_sec; /* BOLT #12: * - MUST set `invoice_payment_hash` to the SHA256 hash of the * `payment_preimage` that will be given in return for payment. */ - randombytes_buf(&ir->preimage, sizeof(ir->preimage)); + randbytes(&ir->preimage, sizeof(ir->preimage)); ir->inv->invoice_payment_hash = tal(ir->inv, struct sha256); sha256(ir->inv->invoice_payment_hash, &ir->preimage, sizeof(ir->preimage)); diff --git a/plugins/offers_offer.c b/plugins/offers_offer.c index 34b6844d4adc..53c88f05afbd 100644 --- a/plugins/offers_offer.c +++ b/plugins/offers_offer.c @@ -9,9 +9,9 @@ #include #include #include +#include #include #include -#include static bool msat_or_any(const char *buffer, const jsmntok_t *tok, @@ -706,8 +706,8 @@ struct command_result *json_invoicerequest(struct command *cmd, * - MUST set `invreq_metadata` to an unpredictable series of bytes. */ invreq->invreq_metadata = tal_arr(invreq, u8, 16); - randombytes_buf(invreq->invreq_metadata, - tal_bytelen(invreq->invreq_metadata)); + randbytes(invreq->invreq_metadata, + tal_bytelen(invreq->invreq_metadata)); /* BOLT #12: * - otherwise (not responding to an offer): diff --git a/plugins/pay.c b/plugins/pay.c index acb532c3a78a..a0ef430a73c0 100644 --- a/plugins/pay.c +++ b/plugins/pay.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -143,7 +144,7 @@ static void paystatus_add_payment(struct json_stream *s, const struct payment *p json_add_string(s, "strategy", p->why); json_add_string(s, "start_time", timestr); json_add_u64(s, "age_in_seconds", - time_to_sec(time_between(time_now(), p->start_time))); + time_to_sec(time_between(clock_time(), p->start_time))); /* Any final state will have an end time. */ if (p->step >= PAYMENT_STEP_SPLIT) { @@ -1417,7 +1418,7 @@ static struct command_result *json_pay(struct command *cmd, p->payment_secret = NULL; } - if (time_now().ts.tv_sec > invexpiry) + if (clock_time().ts.tv_sec > invexpiry) return command_fail(cmd, PAY_INVOICE_EXPIRED, "Invoice expired"); if (invmsat) { @@ -1471,7 +1472,7 @@ static struct command_result *json_pay(struct command *cmd, p->why = "Initial attempt"; p->constraints.cltv_budget = *maxdelay; tal_free(maxdelay); - p->deadline = timeabs_add(time_now(), time_from_sec(*retryfor)); + p->deadline = timemono_add(time_mono(), time_from_sec(*retryfor)); tal_free(retryfor); p->getroute->riskfactorppm = *riskfactor_millionths; tal_free(riskfactor_millionths); @@ -1540,7 +1541,7 @@ static struct command_result *handle_channel_hint_update(struct command *cmd, fmt_amount_msat(tmpctx, hint->estimated_capacity), fmt_amount_msat(tmpctx, hint->capacity) ); - channel_hint_set_add(global_hints, time_now().ts.tv_sec, &hint->scid, + channel_hint_set_add(global_hints, clock_time().ts.tv_sec, &hint->scid, hint->enabled, &hint->estimated_capacity, hint->capacity, NULL); tal_free(hint); diff --git a/plugins/renepay/main.c b/plugins/renepay/main.c index 8d92ea78e856..21790f26f6fd 100644 --- a/plugins/renepay/main.c +++ b/plugins/renepay/main.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -312,7 +313,7 @@ static struct command_result *json_renepay(struct command *cmd, const char *buf, /* === Is it expired? === */ - const u64 now_sec = time_now().ts.tv_sec; + const u64 now_sec = clock_time().ts.tv_sec; if (now_sec > invexpiry) return command_fail(cmd, PAY_INVOICE_EXPIRED, "Invoice expired"); diff --git a/plugins/renepay/mods.c b/plugins/renepay/mods.c index 34e505816538..30d6314a2e6c 100644 --- a/plugins/renepay/mods.c +++ b/plugins/renepay/mods.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -917,7 +918,7 @@ REGISTER_PAYMENT_MODIFIER(end, end_cb); static struct command_result *checktimeout_cb(struct payment *payment) { - if (time_after(time_now(), payment->payment_info.stop_time)) { + if (time_after(clock_time(), payment->payment_info.stop_time)) { return payment_fail(payment, PAY_STOPPED_RETRYING, "Timed out"); } return payment_continue(payment); @@ -1129,7 +1130,7 @@ REGISTER_PAYMENT_MODIFIER(pendingsendpays, pendingsendpays_cb); static struct command_result *knowledgerelax_cb(struct payment *payment) { - const u64 now_sec = time_now().ts.tv_sec; + const u64 now_sec = clock_time().ts.tv_sec; enum renepay_errorcode err = uncertainty_relax( pay_plugin->uncertainty, now_sec - pay_plugin->last_time); if (err) diff --git a/plugins/renepay/payment.c b/plugins/renepay/payment.c index 02566b01cdff..c732ecca0ae4 100644 --- a/plugins/renepay/payment.c +++ b/plugins/renepay/payment.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -96,7 +97,7 @@ bool payment_refresh(struct payment *p){ p->retry = false; p->waitresult_timer = tal_free(p->waitresult_timer); - pinfo->start_time = time_now(); + pinfo->start_time = clock_time(); pinfo->stop_time = timeabs_add(pinfo->start_time, time_from_sec(pinfo->retryfor)); diff --git a/plugins/renepay/test/run-bottleneck.c b/plugins/renepay/test/run-bottleneck.c index 6e8bda76240a..eff7f81d088f 100644 --- a/plugins/renepay/test/run-bottleneck.c +++ b/plugins/renepay/test/run-bottleneck.c @@ -14,9 +14,10 @@ #include #include #include +#include +#include #include #include -#include /* AUTOGENERATED MOCKS START */ /* AUTOGENERATED MOCKS END */ @@ -221,7 +222,7 @@ int main(int argc, char *argv[]) pinfo.maxdelay = 100; pinfo.final_cltv = 5; - pinfo.start_time = time_now(); + pinfo.start_time = clock_time(); pinfo.stop_time = timeabs_add(pinfo.start_time, time_from_sec(10000)); pinfo.base_fee_penalty = 1e-5; @@ -231,7 +232,7 @@ int main(int argc, char *argv[]) pinfo.base_prob_success = 1.0; pinfo.use_shadow = false; - randombytes_buf(&preimage, sizeof(preimage)); + randbytes(&preimage, sizeof(preimage)); sha256(&pinfo.payment_hash, &preimage, sizeof(preimage)); // char hex_preimage[600], hex_sha256[600]; diff --git a/plugins/topology.c b/plugins/topology.c index 944180de5f0f..5990bbe752f2 100644 --- a/plugins/topology.c +++ b/plugins/topology.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -459,6 +460,33 @@ static struct amount_msat peer_capacity(const struct gossmap *gossmap, return capacity; } +/* For deterministic results with bolt12/11 routes, we only return a + * single candidate: choose the one with most capacity */ +static size_t best_candidate(const struct gossmap *gossmap, + const struct gossmap_node *me) +{ + struct amount_msat best_cap = AMOUNT_MSAT(0); + size_t best_num = 0; + for (size_t i = 0; i < me->num_chans; i++) { + int dir; + struct gossmap_chan *ourchan; + struct amount_msat cap; + struct gossmap_node *peer; + + ourchan = gossmap_nth_chan(gossmap, me, i, &dir); + if (ourchan->cupdate_off[!dir] == 0) + continue; + + peer = gossmap_nth_node(gossmap, ourchan, !dir); + cap = peer_capacity(gossmap, me, peer, ourchan); + if (amount_msat_greater(cap, best_cap)) { + best_num = i; + best_cap = cap; + } + } + return best_num; +} + static struct command_result * listpeerchannels_listincoming_done(struct command *cmd, const char *method, @@ -470,6 +498,7 @@ listpeerchannels_listincoming_done(struct command *cmd, struct gossmap_node *me; struct gossmap *gossmap; struct gossmap_localmods *mods; + size_t deterministic_candidate = 0; /* Get local knowledge */ mods = gossmods_from_listpeerchannels(tmpctx, &local_id, @@ -487,6 +516,9 @@ listpeerchannels_listincoming_done(struct command *cmd, if (!me) goto done; + if (randbytes_overridden()) + deterministic_candidate = best_candidate(gossmap, me); + for (size_t i = 0; i < me->num_chans; i++) { struct node_id peer_id; int dir; @@ -499,6 +531,10 @@ listpeerchannels_listincoming_done(struct command *cmd, /* Entirely missing? Ignore. */ if (ourchan->cupdate_off[!dir] == 0) continue; + + if (randbytes_overridden() && i != deterministic_candidate) + continue; + /* We used to ignore if the peer said it was disabled, * but we have a report of LND telling us our unannounced * channel is disabled, so we still use them. */ diff --git a/plugins/xpay/xpay.c b/plugins/xpay/xpay.c index 56dde1422c06..aa59feae702d 100644 --- a/plugins/xpay/xpay.c +++ b/plugins/xpay/xpay.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1947,7 +1948,7 @@ static struct command_result *xpay_core(struct command *cmd, payment->requests = tal_arr(payment, struct out_req *, 0); payment->prior_results = tal_strdup(payment, ""); payment->deadline = timemono_add(time_mono(), time_from_sec(retryfor)); - payment->start_time = time_now(); + payment->start_time = clock_time(); payment->start_blockheight = xpay->blockheight; payment->pay_compat = as_pay; payment->invstring = tal_strdup(payment, invstring); @@ -2056,7 +2057,7 @@ static struct command_result *xpay_core(struct command *cmd, invexpiry = b11->timestamp + b11->expiry; } - now = time_now().ts.tv_sec; + now = clock_time().ts.tv_sec; if (now > invexpiry) return command_fail(cmd, PAY_INVOICE_EXPIRED, "Invoice expired %"PRIu64" seconds ago", @@ -2173,7 +2174,7 @@ static struct command_result *age_layer(struct command *timer_cmd, void *unused) plugin_broken_cb, NULL); json_add_string(req->js, "layer", "xpay"); - json_add_u64(req->js, "cutoff", time_now().ts.tv_sec - 3600); + json_add_u64(req->js, "cutoff", clock_time().ts.tv_sec - 3600); return send_outreq(req); } diff --git a/tests/fuzz/connectd_handshake.h b/tests/fuzz/connectd_handshake.h index 139ee25d4516..8145bd63039f 100644 --- a/tests/fuzz/connectd_handshake.h +++ b/tests/fuzz/connectd_handshake.h @@ -1,6 +1,6 @@ /* This header contains globals and helper functions used by all the * fuzz-connectd-handshake-act* fuzz targets. It also takes care of intercepting - * io_read(), io_write(), and randombytes_buf(), so that the actual fuzz targets + * io_read(), io_write(), and randbytes(), so that the actual fuzz targets * only need to implement the test_read() and test_write() interceptors and the * run() function. */ @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -19,9 +20,9 @@ /* No randomness please, we want to replicate test vectors. */ #include - +#undef randbytes static void seeded_randombytes_buf(u8 *secret, size_t len); -#define randombytes_buf(secret, len) seeded_randombytes_buf((secret), (len)) +#define randbytes(secret, len) seeded_randombytes_buf((secret), (len)) struct handshake; diff --git a/tests/fuzz/libfuzz.c b/tests/fuzz/libfuzz.c index aada6f8ceee4..00e4b84d0614 100644 --- a/tests/fuzz/libfuzz.c +++ b/tests/fuzz/libfuzz.c @@ -21,7 +21,7 @@ int LLVMFuzzerInitialize(int *argc, char ***argv); /* Provide a non-random pseudo-random function to speed fuzzing. */ static isaac64_ctx isaac64; -uint64_t pseudorand(uint64_t max) +uint64_t pseudorand_(uint64_t max, uint64_t *offset) { assert(max); return isaac64_next_uint(&isaac64, max); diff --git a/tools/bench-gossipd.sh b/tools/bench-gossipd.sh index 9b4ff5269b44..4bea1552ce22 100755 --- a/tools/bench-gossipd.sh +++ b/tools/bench-gossipd.sh @@ -78,7 +78,7 @@ if ! bitcoin-cli -regtest ping >/dev/null 2>&1; then while ! bitcoin-cli -regtest ping >/dev/null 2>&1; do sleep 1; done fi -LIGHTNINGD="./lightningd/lightningd --developer --network=regtest --dev-gossip-time=1550513768" +LIGHTNINGD="CLN_DEV_SET_TIME=1550513768 ./lightningd/lightningd --developer --network=regtest" LCLI1="./cli/lightning-cli --lightning-dir=$DIR -R" if [ -z "$DIR" ]; then diff --git a/wallet/invoices.c b/wallet/invoices.c index 0376f179cef6..085618c77ecd 100644 --- a/wallet/invoices.c +++ b/wallet/invoices.c @@ -1,5 +1,6 @@ #include "config.h" #include +#include #include #include #include @@ -178,7 +179,7 @@ static u64 *expired_ids(const tal_t *ctx, static void trigger_expiration(struct invoices *invoices) { u64 *inv_dbids; - u64 now = time_now().ts.tv_sec; + u64 now = clock_time().ts.tv_sec; struct db_stmt *stmt; /* Free current expiration timer */ @@ -213,7 +214,7 @@ static void install_expiration_timer(struct invoices *invoices) struct db_stmt *stmt; struct timerel rel; struct timeabs expiry; - struct timeabs now = time_now(); + struct timeabs now = clock_time(); assert(!invoices->expiration_timer); @@ -274,7 +275,7 @@ bool invoices_create(struct invoices *invoices, { struct db_stmt *stmt; u64 expiry_time; - u64 now = time_now().ts.tv_sec; + u64 now = clock_time().ts.tv_sec; if (invoices_find_by_label(invoices, inv_dbid, label)) { if (taken(msat)) @@ -608,7 +609,7 @@ bool invoices_resolve(struct invoices *invoices, /* Assign a pay-index. */ pay_index = get_next_pay_index(invoices->wallet->db); - paid_timestamp = time_now().ts.tv_sec; + paid_timestamp = clock_time().ts.tv_sec; /* Update database. */ stmt = db_prepare_v2(invoices->wallet->db, SQL("UPDATE invoices" diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index 05a1a0ce6683..617e967593f8 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -1926,6 +1926,7 @@ int main(int argc, const char *argv[]) ld = tal(tmpctx, struct lightningd); ld->config = test_config; ld->hsm_capabilities = NULL; + ld->developer = true; memset(&ld->indexes[WAIT_SUBSYSTEM_HTLCS], 0, sizeof(ld->indexes[WAIT_SUBSYSTEM_HTLCS])); diff --git a/wallet/wallet.c b/wallet/wallet.c index 780aba244676..07ffaddf8c9b 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -1,12 +1,15 @@ #include "config.h" #include #include +#include #include #include #include #include +#include #include #include +#include #include #include #include @@ -24,6 +27,7 @@ #include #include #include +#include #define SQLITE_MAX_UINT 0x7FFFFFFFFFFFFFFF #define DIRECTION_INCOMING 0 @@ -449,7 +453,23 @@ bool wallet_update_output_status(struct wallet *w, return changes > 0; } -static struct utxo **gather_utxos(const tal_t *ctx, struct db_stmt *stmt STEALS) +static int cmp_utxo(struct utxo *const *a, + struct utxo *const *b, + void *unused) +{ + int ret = memcmp(&(*a)->outpoint.txid, &(*b)->outpoint.txid, + sizeof((*a)->outpoint.txid)); + if (ret) + return ret; + if ((*a)->outpoint.n < (*b)->outpoint.n) + return -1; + else if ((*a)->outpoint.n > (*b)->outpoint.n) + return 1; + return 0; +} + +static struct utxo **gather_utxos(const tal_t *ctx, + struct db_stmt *stmt STEALS) { struct utxo **results; @@ -461,6 +481,10 @@ static struct utxo **gather_utxos(const tal_t *ctx, struct db_stmt *stmt STEALS) } tal_free(stmt); + /* Make sure these are in order if we're trying to remove entropy */ + if (randbytes_overridden()) + asort(results, tal_count(results), cmp_utxo, NULL); + return results; } @@ -807,27 +831,53 @@ struct utxo *wallet_find_utxo(const tal_t *ctx, struct wallet *w, struct db_stmt *stmt; struct utxo *utxo; - stmt = db_prepare_v2(w->db, SQL("SELECT" - " prev_out_tx" - ", prev_out_index" - ", value" - ", type" - ", status" - ", keyindex" - ", channel_id" - ", peer_id" - ", commitment_point" - ", option_anchor_outputs" - ", confirmation_height" - ", spend_height" - ", scriptpubkey " - ", reserved_til" - ", csv_lock" - ", is_in_coinbase" - " FROM outputs" - " WHERE status = ?" - " OR (status = ? AND reserved_til <= ?)" - "ORDER BY RANDOM();")); + /* Make sure these are in order if we're trying to remove entropy! */ + if (w->ld->developer && getenv("CLN_DEV_ENTROPY_SEED")) { + stmt = db_prepare_v2(w->db, SQL("SELECT" + " prev_out_tx" + ", prev_out_index" + ", value" + ", type" + ", status" + ", keyindex" + ", channel_id" + ", peer_id" + ", commitment_point" + ", option_anchor_outputs" + ", confirmation_height" + ", spend_height" + ", scriptpubkey " + ", reserved_til" + ", csv_lock" + ", is_in_coinbase" + " FROM outputs" + " WHERE status = ?" + " OR (status = ? AND reserved_til <= ?)" + "ORDER BY prev_out_tx, prev_out_index;")); + } else { + stmt = db_prepare_v2(w->db, SQL("SELECT" + " prev_out_tx" + ", prev_out_index" + ", value" + ", type" + ", status" + ", keyindex" + ", channel_id" + ", peer_id" + ", commitment_point" + ", option_anchor_outputs" + ", confirmation_height" + ", spend_height" + ", scriptpubkey " + ", reserved_til" + ", csv_lock" + ", is_in_coinbase" + " FROM outputs" + " WHERE status = ?" + " OR (status = ? AND reserved_til <= ?)" + "ORDER BY RANDOM();")); + } + db_bind_int(stmt, output_status_in_db(OUTPUT_STATE_AVAILABLE)); db_bind_int(stmt, output_status_in_db(OUTPUT_STATE_RESERVED)); db_bind_u64(stmt, current_blockheight); @@ -4195,7 +4245,7 @@ void wallet_payment_set_status(struct wallet *wallet, u32 completed_at = 0; if (newstatus != PAYMENT_PENDING) - completed_at = time_now().ts.tv_sec; + completed_at = clock_time().ts.tv_sec; stmt = db_prepare_v2(wallet->db, SQL("UPDATE payments SET status=?, completed_at=?, updated_index=? " @@ -5350,7 +5400,7 @@ void wallet_forwarded_payment_add(struct wallet *w, const struct htlc_in *in, if (state == FORWARD_SETTLED || state == FORWARD_FAILED) { resolved_time = tal(tmpctx, struct timeabs); - *resolved_time = time_now(); + *resolved_time = clock_time(); } else { resolved_time = NULL; } @@ -7584,7 +7634,7 @@ void wallet_save_network_event(struct lightningd *ld, db_bind_u64(stmt, id); db_bind_node_id(stmt, peer_id); db_bind_int(stmt, network_event_in_db(etype)); - db_bind_u64(stmt, time_now().ts.tv_sec); + db_bind_u64(stmt, clock_time().ts.tv_sec); if (reason) db_bind_text(stmt, reason); else @@ -7786,7 +7836,7 @@ void migrate_setup_coinmoves(struct lightningd *ld, struct db *db) { struct utxo **utxos = db_get_unspent_utxos(tmpctx, db); struct db_stmt *stmt; - u64 base_timestamp = time_now().ts.tv_sec - 2; + u64 base_timestamp = clock_time().ts.tv_sec - 2; for (size_t i = 0; i < tal_count(utxos); i++) { struct chain_coin_mvt *mvt; diff --git a/wire/test/run-peer-wire.c b/wire/test/run-peer-wire.c index 894920440a88..cc688645673b 100644 --- a/wire/test/run-peer-wire.c +++ b/wire/test/run-peer-wire.c @@ -12,6 +12,8 @@ #include #include +#include +#include #include #include #include @@ -20,6 +22,12 @@ extern secp256k1_context *secp256k1_ctx; /* AUTOGENERATED MOCKS START */ +/* Generated stub for randbytes_ */ +void randbytes_(void *bytes UNNEEDED, size_t num_bytes UNNEEDED, u64 *offset UNNEEDED) +{ fprintf(stderr, "randbytes_ called!\n"); abort(); } +/* Generated stub for randbytes_overridden */ +bool randbytes_overridden(void) +{ fprintf(stderr, "randbytes_overridden called!\n"); abort(); } /* AUTOGENERATED MOCKS END */ /* memsetting pubkeys doesn't work */ diff --git a/wire/test/run-tlvstream.c b/wire/test/run-tlvstream.c index 68420fb900ed..5aa609336299 100644 --- a/wire/test/run-tlvstream.c +++ b/wire/test/run-tlvstream.c @@ -9,7 +9,9 @@ static const char *reason; #include #include #include +#include #include +#include #include #include @@ -28,6 +30,12 @@ int chainparams_get_ln_port(const struct chainparams *params UNNEEDED) bool fromwire_channel_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct channel_id *channel_id UNNEEDED) { fprintf(stderr, "fromwire_channel_id called!\n"); abort(); } +/* Generated stub for randbytes_ */ +void randbytes_(void *bytes UNNEEDED, size_t num_bytes UNNEEDED, u64 *offset UNNEEDED) +{ fprintf(stderr, "randbytes_ called!\n"); abort(); } +/* Generated stub for randbytes_overridden */ +bool randbytes_overridden(void) +{ fprintf(stderr, "randbytes_overridden called!\n"); abort(); } /* Generated stub for towire_channel_id */ void towire_channel_id(u8 **pptr UNNEEDED, const struct channel_id *channel_id UNNEEDED) { fprintf(stderr, "towire_channel_id called!\n"); abort(); }