From 32b29ccbe8583abb7d740d7c6ab32f01198b5fe1 Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Mon, 12 Oct 2026 11:04:16 +0100 Subject: [PATCH 1/9] xpay: add onion budget helpers Add onion budget and payload constructors to the internal xpay API. Onion budget computation can be used to know a realistic number of hops that constraints our payment before calling askrene. Changelog-None Signed-off-by: Lagrang3 --- plugins/xpay/Makefile | 8 +- plugins/xpay/onion_budget.c | 152 +++++++++++ plugins/xpay/onion_budget.h | 97 +++++++ plugins/xpay/test/Makefile | 15 ++ plugins/xpay/test/run-onion_budget.c | 379 +++++++++++++++++++++++++++ plugins/xpay/xpay.c | 77 ++---- 6 files changed, 671 insertions(+), 57 deletions(-) create mode 100644 plugins/xpay/onion_budget.c create mode 100644 plugins/xpay/onion_budget.h create mode 100644 plugins/xpay/test/Makefile create mode 100644 plugins/xpay/test/run-onion_budget.c diff --git a/plugins/xpay/Makefile b/plugins/xpay/Makefile index 33f22eb117b2..8d06f585ce3b 100644 --- a/plugins/xpay/Makefile +++ b/plugins/xpay/Makefile @@ -1,8 +1,12 @@ PLUGIN_XPAY_SRC := \ plugins/xpay/listpays.c \ + plugins/xpay/onion_budget.c \ plugins/xpay/xpay.c -PLUGIN_XPAY_HDRS := +PLUGIN_XPAY_HDRS := \ + plugins/xpay/listpays.h \ + plugins/xpay/onion_budget.h \ + plugins/xpay/xpay.h PLUGIN_XPAY_OBJS := $(PLUGIN_XPAY_SRC:.c=.o) @@ -14,3 +18,5 @@ ALL_C_HEADERS += $(PLUGIN_XPAY_HDRS) $(PLUGIN_XPAY_OBJS): $(PLUGIN_XPAY_HDRS) $(PLUGIN_LIB_HEADER) plugins/xpay/Makefile plugins/cln-xpay: $(PLUGIN_XPAY_OBJS) $(PLUGIN_LIB_OBJS) libcommon.a + +include plugins/xpay/test/Makefile diff --git a/plugins/xpay/onion_budget.c b/plugins/xpay/onion_budget.c new file mode 100644 index 000000000000..ce0a09f80d8e --- /dev/null +++ b/plugins/xpay/onion_budget.c @@ -0,0 +1,152 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include + +/* create_onionpacket(ctx, sp, fixed_size, path_secrets) fails if + * sphinx_path_payloads_size(sp) > fixed_size, where the fixed_size is + * ROUTING_INFO_SIZE (1300) and sphinx_path_payloads_size is the sum of + * tal_bytelen(hop->raw_payload) + HMAC_SIZE (32) for every hop. */ + +/* hop has size prepended, so this isn't a simple "append" */ +static void hop_append(u8 **hop, const u8 *appendme) +{ + bigsize_t prepended_len; + const u8 *cursor = *hop; + size_t tlvs_len = tal_bytelen(*hop); + u8 *result; + + /* Pull off length */ + prepended_len = fromwire_bigsize(&cursor, &tlvs_len); + assert(cursor); + assert(prepended_len == tlvs_len); + + /* Rewrite: length + contents */ + result = tal_arr(tal_parent(*hop), u8, 0); + towire_bigsize(&result, tlvs_len + tal_bytelen(appendme)); + towire(&result, cursor, tlvs_len); + towire(&result, appendme, tal_bytelen(appendme)); + + tal_free(*hop); + *hop = result; +} + +u8 *onion_final_payload(const tal_t *ctx, + struct amount_msat deliver, + u32 final_cltv, + struct amount_msat total_msat, + const struct secret *payment_secret, + const u8 *payment_metadata, + const u8 *extra_tlvs) +{ + u8 *final = onion_final_hop(ctx, deliver, final_cltv, total_msat, + payment_secret, payment_metadata); + hop_append(&final, extra_tlvs); + return final; +} + +u8 *onion_blinded_payload(const tal_t *ctx, + const struct blinded_path *path, + size_t i, + const struct amount_msat *deliver, + const struct amount_msat *total_msat, + const u32 *final_cltv) +{ + bool first = (i == 0); + bool final = (i == tal_count(path->path) - 1); + + /* BOLT #4: + * - For every node inside a blinded route: + * - MUST include the `encrypted_recipient_data` provided by the + * recipient + * - For the first node in the blinded route: + * - MUST include the `path_key` provided by the + * recipient in `current_path_key` + * - If it is the final node: + * - MUST include `amt_to_forward`, `outgoing_cltv_value` and `total_amount_msat`. + *... + * - MUST NOT include any other tlv field. + */ + return onion_blinded_hop(ctx, + final ? deliver : NULL, + final ? total_msat : NULL, + final ? final_cltv : NULL, + path->path[i]->encrypted_recipient_data, + first ? &path->first_path_key : NULL); +} + +size_t onion_forward_hop_size(struct amount_msat amount_out, u32 outgoing_cltv) +{ + /* The scid is fixed width, so any value will do. */ + const struct short_channel_id scid = {.u64 = 0}; + return onion_hop_size(take( + onion_forward_payload(tmpctx, &scid, amount_out, outgoing_cltv))); +} + +size_t onion_final_hop_size(struct amount_msat deliver, + u32 final_cltv, + struct amount_msat total_msat, + const struct secret *payment_secret, + const u8 *payment_metadata, + const u8 *extra_tlvs) +{ + return onion_hop_size(take( + onion_final_payload(tmpctx, deliver, final_cltv, total_msat, + payment_secret, payment_metadata, extra_tlvs))); +} + +size_t onion_blinded_tail_size(const struct blinded_path *path, + struct amount_msat deliver, + struct amount_msat total_msat, + u32 final_cltv) +{ + size_t total = 0; + for (size_t i = 0; i < tal_count(path->path); i++) + total += onion_hop_size(take(onion_blinded_payload( + tmpctx, path, i, &deliver, &total_msat, &final_cltv))); + return total; +} + +u32 onion_budget(struct amount_msat amount, + struct amount_msat maxfee, + u32 blockheight, + u32 maxdelay, + const struct blinded_path *blinded_path, + const struct secret *payment_secret, + const u8 *payment_metadata, + const u8 *extra_tlvs, + size_t *tail_size) +{ + struct amount_msat worst_amount; + u32 worst_cltv; + size_t final_size, per_hop_size; + + if(!amount_msat_add(&worst_amount, amount, maxfee)) + worst_amount = AMOUNT_MSAT(UINT64_MAX); + + if(!assign_overflow_u32(&worst_cltv, maxdelay + (u64)blockheight)) + worst_cltv = UINT32_MAX; + + per_hop_size = onion_forward_hop_size(worst_amount, worst_cltv); + + if (blinded_path) + final_size = onion_blinded_tail_size(blinded_path, amount, + amount, worst_cltv); + else + final_size = onion_final_hop_size(amount, worst_cltv, amount, + payment_secret, + payment_metadata, extra_tlvs); + + if (tail_size) + *tail_size = final_size; + /* In a case like this we could make a bolt11 self-payment, but + * cannot start a blinded path payment even if the sender is the first + * node. */ + if(final_size > ROUTING_INFO_SIZE) + return UINT32_MAX; + return (ROUTING_INFO_SIZE - final_size) / per_hop_size; +} diff --git a/plugins/xpay/onion_budget.h b/plugins/xpay/onion_budget.h new file mode 100644 index 000000000000..33825f07b74a --- /dev/null +++ b/plugins/xpay/onion_budget.h @@ -0,0 +1,97 @@ +#ifndef LIGHTNING_PLUGINS_XPAY_ONION_BUDGET_H +#define LIGHTNING_PLUGINS_XPAY_ONION_BUDGET_H +#include "config.h" +#include +#include +#include +#include +#include + +struct blinded_path; +struct secret; +struct short_channel_id; + +/* Payload for a forwarding node. */ +static inline u8 *onion_forward_payload(const tal_t *ctx, + const struct short_channel_id *scid, + struct amount_msat amount_out, + u32 outgoing_cltv) +{ + return onion_nonfinal_hop(ctx, scid, amount_out, outgoing_cltv); +} + +/* Payload for the final node. */ +u8 *onion_final_payload(const tal_t *ctx, + struct amount_msat deliver, + u32 final_cltv, + struct amount_msat total_msat, + const struct secret *payment_secret, + const u8 *payment_metadata, + const u8 *extra_tlvs); + +/* Payload for the i-th node in a blinded path. */ +u8 *onion_blinded_payload(const tal_t *ctx, + const struct blinded_path *path, + size_t i, + const struct amount_msat *deliver, + const struct amount_msat *total_msat, + const u32 *final_cltv); + +/* Bytes (payload + HMAC) a forwarding hop takes. The encoding of amounts + * and cltvs is truncated, so it never shrinks as the values grow: pass + * upper bounds to get a worst case. */ +size_t onion_forward_hop_size(struct amount_msat amount_out, u32 outgoing_cltv); + +/* Bytes (payloads + HMACs) of the tail. */ +size_t onion_final_hop_size(struct amount_msat deliver, + u32 final_cltv, + struct amount_msat total_msat, + const struct secret *payment_secret, + const u8 *payment_metadata, + const u8 *extra_tlvs); + +/* Bytes (payloads + HMACs) of the tail. */ +size_t onion_blinded_tail_size(const struct blinded_path *path, + struct amount_msat deliver, + struct amount_msat total_msat, + u32 final_cltv); + +/* How many forwarding hops we can afford to encode in a payment onion packet. + * This number depends slightly on the payment amount blockheight, but mostly on + * the contents of the tail, either a final node (bolt11) or a blinded path + * (bolt12). xpay uses injectpaymentonion primitive therefore down the line + * this means usually that the sender is included. + * + * eg. for n=3: + * (sender) -> (node 1) -> (node 2) -> (final node) + * or + * (sender) -> (node 1) -> (node 2) -> (first node in blinded path) -> ... + * + * eg. for n=0 + * (sender = final node) // we could pay ourselves using injectpaymentonion + * or + * (sender = first node in blinded path) -> ... + * + * errors: + * If the tail packet is too big to fit ROUTING_INFO_SIZE this returns + * UINT32_MAX. + * */ +u32 onion_budget(struct amount_msat amount, + struct amount_msat maxfee, + u32 blockheight, + u32 maxdelay, + const struct blinded_path *blinded_path, + const struct secret *payment_secret, + const u8 *payment_metadata, + const u8 *extra_tlvs, + size_t *tail_size); + +/* Exactly what sphinx_hop_size() counts for each hop. */ +static inline size_t onion_hop_size(const u8 *payload TAKES) +{ + size_t len = tal_bytelen(payload) + HMAC_SIZE; + tal_free_if_taken(payload); + return len; +} + +#endif /* LIGHTNING_PLUGINS_XPAY_ONION_BUDGET_H */ diff --git a/plugins/xpay/test/Makefile b/plugins/xpay/test/Makefile new file mode 100644 index 000000000000..72fdd25e38d7 --- /dev/null +++ b/plugins/xpay/test/Makefile @@ -0,0 +1,15 @@ +# Note that these actually #include everything they need, except ccan/, bitcoin/ and wire/. +PLUGIN_XPAY_TEST_SRC := $(wildcard plugins/xpay/test/run-*.c) +PLUGIN_XPAY_TEST_OBJS := $(PLUGIN_XPAY_TEST_SRC:.c=.o) +PLUGIN_XPAY_TEST_PROGRAMS := $(PLUGIN_XPAY_TEST_OBJS:.o=) + +ALL_C_SOURCES += $(PLUGIN_XPAY_TEST_SRC) +ALL_TEST_PROGRAMS += $(PLUGIN_XPAY_TEST_PROGRAMS) + +$(PLUGIN_XPAY_TEST_OBJS): $(PLUGIN_XPAY_SRC) $(PLUGIN_XPAY_HDRS) plugins/xpay/test/Makefile + +$(PLUGIN_XPAY_TEST_PROGRAMS): libcommon.a + +check-xpay: $(PLUGIN_XPAY_TEST_PROGRAMS:%=unittest/%) + +check-units: check-xpay diff --git a/plugins/xpay/test/run-onion_budget.c b/plugins/xpay/test/run-onion_budget.c new file mode 100644 index 000000000000..e517be1a31d3 --- /dev/null +++ b/plugins/xpay/test/run-onion_budget.c @@ -0,0 +1,379 @@ +#include "config.h" +#include "../onion_budget.c" +#include +#include +#include +#include + +/* A valid, distinct pubkey for each index. */ +static struct pubkey test_pubkey(size_t i) +{ + struct privkey priv; + struct pubkey pub; + + memset(&priv, 0, sizeof(priv)); + priv.secret.data[30] = (i + 1) >> 8; + priv.secret.data[31] = (i + 1) & 0xFF; + if (!pubkey_from_privkey(&priv, &pub)) + abort(); + return pub; +} + +static struct blinded_path *test_blinded_path(const tal_t *ctx, + size_t num_hops, + size_t enc_len) +{ + struct blinded_path *path = tal(ctx, struct blinded_path); + + path->first_node_id.is_pubkey = true; + path->first_node_id.pubkey = test_pubkey(1000); + path->first_path_key = test_pubkey(1001); + path->path = tal_arr(path, struct blinded_path_hop *, num_hops); + for (size_t i = 0; i < num_hops; i++) { + path->path[i] = tal(path->path, struct blinded_path_hop); + path->path[i]->blinded_node_id = test_pubkey(1100 + i); + path->path[i]->encrypted_recipient_data + = tal_arrz(path->path[i], u8, enc_len); + } + return path; +} + +/* Random value no greater than max, biased towards all encoding widths. */ +static u64 random_upto(u64 max) +{ + u64 v = pseudorand_u64(); + int bits = pseudorand(65); + + if (bits < 64) + v &= (1ULL << bits) - 1; + return v > max ? max : v; +} + +/* Everything onion_budget() needs to know about a payment. */ +struct payment_params { + struct amount_msat amount; + struct amount_msat maxfee; + u32 blockheight; + u32 maxdelay; + const struct blinded_path *path; + const struct secret *payment_secret; + const u8 *payment_metadata; + const u8 *extra_tlvs; +}; + +static u32 params_budget(const struct payment_params *p, size_t *tail_size) +{ + return onion_budget(p->amount, p->maxfee, p->blockheight, p->maxdelay, + p->path, p->payment_secret, p->payment_metadata, + p->extra_tlvs, tail_size); +} + +/* The upper bounds onion_budget() promises to cover (saturating, like it). */ +static struct amount_msat max_out(const struct payment_params *p) +{ + struct amount_msat out; + if (!amount_msat_add(&out, p->amount, p->maxfee)) + return AMOUNT_MSAT(UINT64_MAX); + return out; +} + +static u32 max_cltv(const struct payment_params *p) +{ + u64 cltv = (u64)p->blockheight + p->maxdelay; + return cltv > UINT32_MAX ? UINT32_MAX : cltv; +} + +/* Build an onion like create_onion() does: n forwarding payloads (amounts + * and cltvs at or below the bounds, or exactly at them if worst), then the + * tail. */ +static struct sphinx_path *build_path(const tal_t *ctx, + const struct payment_params *p, + size_t n, bool worst) +{ + struct sphinx_path *sp = sphinx_path_new(ctx, NULL, 0); + struct pubkey node = test_pubkey(0); + struct amount_msat deliver; + u32 tail_cltv; + + for (size_t i = 0; i < n; i++) { + const struct short_channel_id scid = {.u64 = pseudorand_u64()}; + struct amount_msat amt; + u32 cltv; + + if (worst) { + amt = max_out(p); + cltv = max_cltv(p); + } else { + amt = amount_msat(random_upto(max_out(p).millisatoshis)); /* Raw: test */ + cltv = random_upto(max_cltv(p)); + } + sphinx_add_hop_has_length(sp, &node, + take(onion_forward_payload(NULL, &scid, + amt, cltv))); + node = test_pubkey(i + 1); + } + + if (worst) { + deliver = p->amount; + tail_cltv = max_cltv(p); + } else { + deliver = amount_msat(random_upto(p->amount.millisatoshis)); /* Raw: test */ + tail_cltv = random_upto(max_cltv(p)); + } + + if (p->path) { + for (size_t i = 0; i < tal_count(p->path->path); i++) + sphinx_add_hop_has_length(sp, + i == 0 ? &p->path->first_node_id.pubkey + : &p->path->path[i]->blinded_node_id, + take(onion_blinded_payload(NULL, p->path, i, + &deliver, + &p->amount, + &tail_cltv))); + } else { + /* Without payment_secret, total_msat must equal deliver. */ + sphinx_add_hop_has_length(sp, &node, + take(onion_final_payload(NULL, deliver, + tail_cltv, + p->payment_secret + ? p->amount : deliver, + p->payment_secret, + p->payment_metadata, + p->extra_tlvs))); + } + return sp; +} + +static bool onion_fits(struct sphinx_path *sp) +{ + struct secret *secrets; + struct onionpacket *packet; + bool ok = (sphinx_path_payloads_size(sp) <= ROUTING_INFO_SIZE); + + if (ok) { + /* we should be able to build the build the onion packet */ + packet = + create_onionpacket(tmpctx, sp, ROUTING_INFO_SIZE, &secrets); + assert(packet); + tal_free(packet); + } + return ok; +} + +/* The size helpers agree exactly with what sphinx counts. */ +static void check_sizes_exact(void) +{ + struct pubkey node = test_pubkey(0); + struct secret secret; + struct blinded_path *path = test_blinded_path(tmpctx, 3, 77); + const u8 *metadata = tal_arrz(tmpctx, u8, 300); + const u8 *extra = tal_arrz(tmpctx, u8, 40); + + memset(&secret, 7, sizeof(secret)); + + for (size_t iter = 0; iter < 200; iter++) { + struct sphinx_path *sp = sphinx_path_new(tmpctx, NULL, 0); + const struct short_channel_id scid = {.u64 = pseudorand_u64()}; + struct amount_msat amt = amount_msat(random_upto(UINT64_MAX)); + struct amount_msat total = amount_msat(random_upto(UINT64_MAX)); + u32 cltv = random_upto(UINT32_MAX); + size_t expect; + + sphinx_add_hop_has_length(sp, &node, + take(onion_forward_payload(NULL, &scid, + amt, cltv))); + expect = onion_forward_hop_size(amt, cltv); + assert(sphinx_path_payloads_size(sp) == expect); + + /* bolt11, keysend, with and without metadata */ + sphinx_add_hop_has_length(sp, &node, + take(onion_final_payload(NULL, amt, cltv, + total, &secret, + iter % 2 ? metadata : NULL, + NULL))); + expect += onion_final_hop_size(amt, cltv, total, &secret, + iter % 2 ? metadata : NULL, NULL); + assert(sphinx_path_payloads_size(sp) == expect); + + sphinx_add_hop_has_length(sp, &node, + take(onion_final_payload(NULL, amt, cltv, + amt, NULL, NULL, + extra))); + expect += onion_final_hop_size(amt, cltv, amt, NULL, NULL, extra); + assert(sphinx_path_payloads_size(sp) == expect); + + for (size_t i = 0; i < tal_count(path->path); i++) + sphinx_add_hop_has_length(sp, &node, + take(onion_blinded_payload(NULL, path, + i, &amt, + &total, + &cltv))); + expect += onion_blinded_tail_size(path, amt, total, cltv); + assert(sphinx_path_payloads_size(sp) == expect); + } +} + +/* The builders can't notice themselves dropping a field: compare against + * the raw encoder. */ +static void check_against_raw_encoder(void) +{ + struct secret secret; + const u8 *extra = tal_arrz(tmpctx, u8, 40); + const u8 *raw; + + memset(&secret, 7, sizeof(secret)); + + /* keysend: extra tlvs are appended (payload stays < 253 bytes, so + * the length prefix stays 1 byte). */ + raw = onion_final_hop(tmpctx, AMOUNT_MSAT(1000), 100, AMOUNT_MSAT(1000), + NULL, NULL); + assert(onion_final_hop_size(AMOUNT_MSAT(1000), 100, AMOUNT_MSAT(1000), + NULL, NULL, extra) + == tal_bytelen(raw) + tal_bytelen(extra) + HMAC_SIZE); + + /* No payment_secret means no payment_data: 32 + 2 + total bytes less. */ + assert(onion_final_hop_size(AMOUNT_MSAT(1000), 100, AMOUNT_MSAT(1000), + &secret, NULL, NULL) + > onion_final_hop_size(AMOUNT_MSAT(1000), 100, AMOUNT_MSAT(1000), + NULL, NULL, NULL) + 32); + + /* Forwarding payload: 1 len + amt (2+3) + cltv (2+3) + scid 10 + hmac */ + assert(onion_forward_hop_size(AMOUNT_MSAT(0xff1234), 0xffabcd) == 53); + /* Worst case is 27 bytes plus HMAC. */ + assert(onion_forward_hop_size(AMOUNT_MSAT(UINT64_MAX), UINT32_MAX) == 59); +} + +/* max_hops always fits at the bounds; one more never does. */ +static void check_budget(const struct payment_params *p) +{ + size_t tail; + u32 n = params_budget(p, &tail); + + if (n == UINT32_MAX) { + assert(tail > ROUTING_INFO_SIZE); + assert(!onion_fits(build_path(tmpctx, p, 0, true))); + return; + } + assert(tail <= ROUTING_INFO_SIZE); + + /* At the bounds, and below them. */ + assert(onion_fits(build_path(tmpctx, p, n, true))); + for (size_t i = 0; i < 5; i++) + assert(onion_fits(build_path(tmpctx, p, n, false))); + + /* The bound is tight. */ + assert(!onion_fits(build_path(tmpctx, p, n + 1, true))); +} + +int main(int argc, char *argv[]) +{ + struct payment_params p; + struct secret secret; + size_t tail; + u32 n; + + common_setup(argv[0]); + + /* seed the pRNG */ + dev_override_randbytes("run-onion_budget.c", 42); + + memset(&secret, 7, sizeof(secret)); + + check_sizes_exact(); + check_against_raw_encoder(); + + /* Plain bolt11, 100k sat. */ + p.amount = AMOUNT_MSAT(100000000); + p.maxfee = AMOUNT_MSAT(5000000); + p.blockheight = 900000; + p.maxdelay = 2016; + p.path = NULL; + p.payment_secret = &secret; + p.payment_metadata = NULL; + p.extra_tlvs = NULL; + check_budget(&p); + n = params_budget(&p, &tail); + assert(n >= 20); + + /* Huge amounts. */ + p.amount = AMOUNT_MSAT(0xFFFFFFFFFFFF0000ULL); + p.maxfee = AMOUNT_MSAT(0xFFFF); + check_budget(&p); + /* maxfee overflow: treated as UINT64_MAX */ + p.maxfee = AMOUNT_MSAT(0x10000000); + check_budget(&p); + p.amount = AMOUNT_MSAT(100000000); + p.maxfee = AMOUNT_MSAT(5000000); + + /* cltv overflow: treated as UINT32_MAX */ + p.blockheight = UINT32_MAX - 10; + check_budget(&p); + p.blockheight = 900000; + + /* Large payment_metadata eats into the budget... */ + p.payment_metadata = tal_arrz(tmpctx, u8, 600); + check_budget(&p); + assert(params_budget(&p, NULL) < 20); + /* ...and too large doesn't fit at all. */ + p.payment_metadata = tal_arrz(tmpctx, u8, 1300); + check_budget(&p); + assert(params_budget(&p, &tail) == UINT32_MAX); + assert(tail > ROUTING_INFO_SIZE); + p.payment_metadata = NULL; + + /* keysend: no secret, extra tlvs. */ + p.payment_secret = NULL; + p.extra_tlvs = tal_arrz(tmpctx, u8, 42); + check_budget(&p); + p.extra_tlvs = NULL; + p.payment_secret = &secret; + + /* Blinded paths: small, large, and too large to ever use. */ + p.path = test_blinded_path(tmpctx, 1, 50); + check_budget(&p); + + p.path = test_blinded_path(tmpctx, 3, 300); + check_budget(&p); + assert(params_budget(&p, NULL) < 6); + + /* Exactly zero hops left. */ + p.path = test_blinded_path(tmpctx, 4, 260); + check_budget(&p); + assert(params_budget(&p, &tail) == 0); + assert(tail <= ROUTING_INFO_SIZE); + + p.path = test_blinded_path(tmpctx, 4, 320); + check_budget(&p); + assert(params_budget(&p, NULL) == UINT32_MAX); + + /* A tail of exactly ROUTING_INFO_SIZE fits (0 hops), one more byte + * doesn't. */ + { + bool found = false; + p.path = NULL; + for (size_t len = 1100; len < 1300 && !found; len++) { + p.payment_metadata = tal_arrz(tmpctx, u8, len); + n = params_budget(&p, &tail); + if (tail != ROUTING_INFO_SIZE) + continue; + found = true; + assert(n == 0); + check_budget(&p); + p.payment_metadata = tal_arrz(tmpctx, u8, len + 1); + assert(params_budget(&p, &tail) == UINT32_MAX); + assert(tail == ROUTING_INFO_SIZE + 1); + } + assert(found); + p.payment_metadata = NULL; + } + + /* Sweep enc_len: budget must be tight everywhere. */ + for (size_t len = 0; len < 1300; len += 61) { + p.path = test_blinded_path(tmpctx, 2, len); + check_budget(&p); + clean_tmpctx(); + } + + common_shutdown(); + return 0; +} diff --git a/plugins/xpay/xpay.c b/plugins/xpay/xpay.c index 2267f25ccd3c..736f556f6971 100644 --- a/plugins/xpay/xpay.c +++ b/plugins/xpay/xpay.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -1349,58 +1350,20 @@ static void append_blinded_payloads(struct sphinx_path *sp, const struct amount_msat mpp_amount = attempt_mpp_amount(attempt); for (size_t i = 0; i < tal_count(path->path); i++) { - bool first = (i == 0); - bool final = (i == tal_count(path->path) - 1); - const u8 *payload; - - /* BOLT #4: - * - For every node inside a blinded route: - * - MUST include the `encrypted_recipient_data` provided by the - * recipient - * - For the first node in the blinded route: - * - MUST include the `path_key` provided by the - * recipient in `current_path_key` - * - If it is the final node: - * - MUST include `amt_to_forward`, `outgoing_cltv_value` and `total_amount_msat`. - *... - * - MUST NOT include any other tlv field. - */ - payload = onion_blinded_hop(NULL, - final ? &deliver : NULL, - final ? &mpp_amount : NULL, - final ? &final_cltv : NULL, - path->path[i]->encrypted_recipient_data, - first ? &path->first_path_key : NULL); + const u8 *payload = onion_blinded_payload(tmpctx, + path, + i, + &deliver, + &mpp_amount, + &final_cltv); sphinx_add_hop_has_length(sp, - first ? &path->first_node_id.pubkey - : &path->path[i]->blinded_node_id, + i == 0 + ? &path->first_node_id.pubkey + : &path->path[i]->blinded_node_id, take(payload)); } } -/* hop has size prepended, so this isn't a simple "append" */ -static void hop_append(u8 **hop, const u8 *appendme) -{ - bigsize_t prepended_len; - const u8 *cursor = *hop; - size_t tlvs_len = tal_bytelen(*hop); - u8 *result; - - /* Pull off length */ - prepended_len = fromwire_bigsize(&cursor, &tlvs_len); - assert(cursor); - assert(prepended_len == tlvs_len); - - /* Rewrite: length + contents */ - result = tal_arr(tal_parent(*hop), u8, 0); - towire_bigsize(&result, tlvs_len + tal_bytelen(appendme)); - towire(&result, cursor, tlvs_len); - towire(&result, appendme, tal_bytelen(appendme)); - - tal_free(*hop); - *hop = result; -} - static const u8 *create_onion(const tal_t *ctx, struct attempt *attempt, u32 effective_bheight) @@ -1433,8 +1396,10 @@ static const u8 *create_onion(const tal_t *ctx, break; } /* We tell it how much to send *out* */ - payload = onion_nonfinal_hop(NULL, &hop->scidd.scid, hop->amount_out, - hop->cltv_value_out + effective_bheight); + payload = onion_forward_payload(tmpctx, + &hop->scidd.scid, + hop->amount_out, + hop->cltv_value_out + effective_bheight); sphinx_add_hop_has_length(sp, node, take(payload)); node = &hop->next_node; } @@ -1442,13 +1407,13 @@ static const u8 *create_onion(const tal_t *ctx, /* If we use a blinded path, final has to be special, so * that's done in append_blinded_payloads. */ if (!blinded_path) { - u8 *final = onion_final_hop(NULL, - attempt_deliver(attempt), - attempt->payment->final_cltv + effective_bheight, - mpp_amount, - attempt->payment->payment_secret, - attempt->payment->payment_metadata); - hop_append(&final, attempt->payment->extra_tlvs); + u8 *final = onion_final_payload(tmpctx, + attempt_deliver(attempt), + attempt->payment->final_cltv + effective_bheight, + mpp_amount, + attempt->payment->payment_secret, + attempt->payment->payment_metadata, + attempt->payment->extra_tlvs); sphinx_add_hop_has_length(sp, node, take(final)); } From 0d9e9f2f4d6ff9fb7ff6a328de8cbfc80ad0df72 Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Wed, 21 Oct 2026 12:31:36 +0100 Subject: [PATCH 2/9] xpay: use onion budget to detect early failures Changelog-Changed: xpay: computes a budget for the number of hops in the payment depending on the final payload and blinded paths, it can fail early if the onion packet doesn't fit Signed-off-by: Lagrang3 --- plugins/xpay/xpay.c | 147 +++++++++++++++++++++-- tests/plugins/accept_payment_metadata.py | 32 +++++ 2 files changed, 171 insertions(+), 8 deletions(-) create mode 100755 tests/plugins/accept_payment_metadata.py diff --git a/plugins/xpay/xpay.c b/plugins/xpay/xpay.c index 736f556f6971..2c70458cd7d8 100644 --- a/plugins/xpay/xpay.c +++ b/plugins/xpay/xpay.c @@ -1580,6 +1580,16 @@ static void add_cltv_shadow(struct payment *payment, } } +static const struct pubkey * +get_askrene_destination(const struct payment *payment) +{ + struct xpay *xpay = xpay_of(payment->plugin); + if (payment->paths) + return &xpay->fakenode; + else + return &payment->destination; +} + static struct command_result *getroutes_done(struct command *aux_cmd, const char *method, const char *buf, @@ -1789,11 +1799,7 @@ static struct command_result *getroutes_for(struct command *aux_cmd, /* If we get injectpaymentonion responses, they can wait */ payment->amount_being_routed = deliver; - - if (payment->paths) - dst = &xpay->fakenode; - else - dst = &payment->destination; + dst = get_askrene_destination(payment); /* Self-pay? Shortcut all this */ if (pubkey_eq(&xpay->local_id, dst)) { @@ -2108,6 +2114,9 @@ static struct command_result *populate_private_layer(struct command *cmd, return batch_done(aux_cmd, batch); } +static struct command_result *check_onion_budget(struct command *cmd, + struct payment *payment); + static struct command_result * preapproveinvoice_succeed(struct command *cmd, const char *method, @@ -2120,7 +2129,7 @@ preapproveinvoice_succeed(struct command *cmd, return command_check_done(cmd); } - return populate_private_layer(cmd, payment); + return check_onion_budget(cmd, payment); } /* If it returns NULL, *authorized is the most we agreed to pay. */ @@ -3014,6 +3023,129 @@ static struct command_result *age_done(struct command *timer_cmd, return timer_complete(timer_cmd); } +static struct command_result *check_onion_budget(struct command *cmd, + struct payment *payment) +{ + struct xpay *xpay = xpay_of(payment->plugin); + const u32 blockheight_slack = 100; + u32 max_hops; + size_t tail_size, smallest_tail_size = SIZE_MAX; + char *error; + const struct pubkey *dst; + bool this_node_front = false; + + if (payment->paths) { + /* Among all possible blinded routes assume the worst case. */ + max_hops = UINT32_MAX; + if (tal_count(payment->paths) == 0) { + error = + tal_fmt(tmpctx, "Weird! There are no blinded paths " + "left to build routes to."); + payment_log(payment, LOG_UNUSUAL, "%s", error); + return command_fail(cmd, PAY_ROUTE_NOT_FOUND, "%s", + error); + } + for (size_t i = 0; i < tal_count(payment->paths); i++) { + u32 this_hops = onion_budget(payment->mpp_amount, + payment->maxfee, + /* give some blockheight + * slack */ + payment->start_blockheight + blockheight_slack, + payment->maxdelay, + payment->paths[i], + payment->payment_secret, + payment->payment_metadata, + payment->extra_tlvs, + &tail_size); + + if (smallest_tail_size > tail_size) + smallest_tail_size = tail_size; + + if (this_hops == UINT32_MAX) { + payment_log(payment, LOG_UNUSUAL, + "discarding blinded path: requires " + "%zu bytes", + tail_size); + /* Remove blinded paths that cannot be used. */ + tal_arr_remove(&payment->paths, i); + tal_arr_remove(&payment->payinfos, i); + i--; + continue; + } + + /* Using the worst case here to make all blinded paths + * equivalently *good* down the line, which is simpler. + * A more complicated approach could be: "try with this + * number, if no path is found try the next worst case + * and remove the blinded paths under this number." */ + if (this_hops < max_hops) + max_hops = this_hops; + + if (payment->paths[i]->first_node_id.is_pubkey && + pubkey_eq(&xpay->local_id, + &payment->paths[i]->first_node_id.pubkey)) + this_node_front = true; + } + if(tal_count(payment->paths) == 0) { + error = tal_fmt( + tmpctx, + "Cannot use any blinded path, the smallest one " + "needs at least %zu byte onion packet", + smallest_tail_size); + payment_log(payment, LOG_INFORM, "%s", error); + return command_fail(cmd, PAY_ROUTE_NOT_FOUND, "%s", + error); + } + } else { + max_hops = onion_budget(payment->mpp_amount, + payment->maxfee, + /* give some blockheight slack */ + payment->start_blockheight + blockheight_slack, + payment->maxdelay, + NULL, + payment->payment_secret, + payment->payment_metadata, + payment->extra_tlvs, + &tail_size); + if (max_hops == UINT32_MAX) { + error = tal_fmt(tmpctx, + "The final payload is too big to " + "fit an onion packet: %zu bytes", + tail_size); + payment_log(payment, LOG_INFORM, "%s", error); + return command_fail(cmd, PAY_ROUTE_NOT_FOUND, "%s", + error); + } + } + + payment_log(payment, LOG_DBG, "onion budget is %" PRIu32 ".", max_hops); + + dst = get_askrene_destination(payment); + /* This is bolt11 payment with 0 hops and we are not the destination. */ + if (max_hops == 0 && !payment->paths && + !pubkey_eq(&xpay->local_id, dst)) { + error = tal_fmt( + tmpctx, + "onion budget is 0 and we are not the destination node, " + "this payment is impossible"); + payment_log(payment, LOG_INFORM, "%s", error); + return command_fail(cmd, PAY_ROUTE_NOT_FOUND, "%s", error); + } + + /* It is a payment through a blinded path and we are not the front node. + */ + if (max_hops == 0 && payment->paths && + pubkey_eq(&xpay->fakenode, dst) && !this_node_front) { + error = tal_fmt(tmpctx, "onion budget is 0 and we are not a " + "front node in any blinded path, " + "this payment is impossible"); + payment_log(payment, LOG_INFORM, "%s", error); + return command_fail(cmd, PAY_ROUTE_NOT_FOUND, "%s", error); + } + + return populate_private_layer(cmd, payment); +} + static struct command_result *age_layer(struct command *timer_cmd, void *unused) { struct out_req *req; @@ -3057,8 +3189,7 @@ preapprovekeysend_succeed(struct command *cmd, return command_check_done(cmd); } - /* Actually we don't need a private layer, but unification is easy. */ - return populate_private_layer(cmd, payment); + return check_onion_budget(cmd, payment); } static struct command_result *json_xkeysend(struct command *cmd, diff --git a/tests/plugins/accept_payment_metadata.py b/tests/plugins/accept_payment_metadata.py new file mode 100755 index 000000000000..5c09678aa242 --- /dev/null +++ b/tests/plugins/accept_payment_metadata.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Settle HTLCs which carry payment_metadata, using a preimage we were told. + +lightningd itself rejects any payment_metadata ("Unexpected payment_metadata"), +so this lets a test check that an onion with a large final payload arrives +intact. +""" +from pyln.client import Plugin + +plugin = Plugin() + + +@plugin.method("setmetadatapreimage") +def setmetadatapreimage(plugin, preimage, metadata): + """Settle HTLCs carrying {metadata} with {preimage}.""" + plugin.expect = (preimage, metadata) + return {} + + +@plugin.hook("htlc_accepted") +def on_htlc_accepted(onion, plugin, **kwargs): + if plugin.expect is None or 'payment_metadata' not in onion: + return {'result': 'continue'} + preimage, metadata = plugin.expect + if onion['payment_metadata'] != metadata: + plugin.log(f"Unexpected metadata {onion['payment_metadata']}", level='broken') + return {'result': 'continue'} + return {'result': 'resolve', 'payment_key': preimage} + + +plugin.expect = None +plugin.run() From b9a3eefae6aba46beab63a2f68237d0923fa1316 Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Thu, 22 Oct 2026 07:17:52 +0100 Subject: [PATCH 3/9] askrene: add constant probability cost per hop Added an homogeneous and constant cost to every hop in the MCF solver. test_real_data shows a slight decrease of paths lengths by looking at the path length histograms SLOW_MACHINE=0 histogram before {2: 1, 3: 8, 4: 34, 5: 77, 6: 45, 7: 23, 8: 4} histogram after {2: 2, 3: 59, 4: 110, 5: 5, 6: 5} SLOW_MACHINE=1 histograms before {2: 1, 3: 3, 4: 11, 5: 15, 6: 6, 7: 6, 8: 3} histogram after {2: 2, 3: 16, 4: 18, 5: 1, 6: 5} the most frequent path length moves from 5 to 4 and there aren't more than 6 hops in any given path. Another consequence of the additional per hop cost is a slight reduction in the number paths (or payment parts) per route query. Even though our linear approximation gives equal cost to 3 paths of length 1 than to 1 path of length 3. in test_real_data this is shown by the expected_first_success (the number of success computations in the first loop) and expected_total_paths (the total number of parts for those successes, also the sum of values in the histogram). For SLOW_MACHINE=0 we get a mean parts per payment of expected_total_paths / expected_first_success = 192 / 95 = 2.02 that becomes expected_total_paths / expected_first_success = 181 / 95 = 1.90 For SLOW_MACHINE=1 we get a mean parts per payment of expected_total_paths / expected_first_success = 45 / 24 = 1.87 that becomes expected_total_paths / expected_first_success = 42 / 24 = 1.75 Changelog-Changed: askrene-getroutes: penalizes every extra hop in every route with an homogeneous and constant probability of failure of 1%, as a consequence askrene will slightly prefer smaller routes and fewer payment parts. Signed-off-by: Lagrang3 --- plugins/askrene/child/mcf.c | 111 ++++++++++++++++++++++++++++++------ tests/test_askrene.py | 8 +-- 2 files changed, 99 insertions(+), 20 deletions(-) diff --git a/plugins/askrene/child/mcf.c b/plugins/askrene/child/mcf.c index dde97f02250e..59a965cee39d 100644 --- a/plugins/askrene/child/mcf.c +++ b/plugins/askrene/child/mcf.c @@ -301,6 +301,10 @@ struct pay_parameters { double delay_feefactor; double base_fee_penalty; + + /* An homogeneous (same for every channel) constant (does not depend on + * the amount we forward) probability of success. */ + double const_success_probability; }; /* Helper function. @@ -386,15 +390,64 @@ static void linearize_channel(const struct pay_parameters *params, params->accuracy), amount_msat_ratio_ceil(params->amount, params->accuracy)); + const double flow_units = amount_msat_ratio(params->amount, params->accuracy); + const double k = 1000 * flow_units; + + assert(params->const_success_probability > 0); + assert(params->const_success_probability <= 1.0); + /* Moving the entire payment through */ + const double linear_const_cost = + -log(params->const_success_probability) / flow_units; + set_capacity(&capacity[0], a, &cap_on_capacity); - cost[0]=0; + cost[0] = k * linear_const_cost; for(size_t i=1;icap_fraction[i]*(b-a), &cap_on_capacity); - cost[i] = params->cost_fraction[i] * 1000 - * amount_msat_ratio(params->amount, params->accuracy) - / (b - a); + /* We have + * cost(x) = - k log P_pickhardt(x) + * where k is an arbitrary constant and P_pickhardt(x) is the probability + * of success. In the range a <= x < b + * P_pickhardt(x) = 1 - (x-a)/(b-a) = 1 - t + * t is the probability of failure and a convenient variable to + * write down the piecewise linearization of -log P. In terms of + * t the linearization of -log P in the ranges [0, 0.5, 0.8, 0.95] + * produces the t-space slopes [0, 1.38, 3.05, 9.24], therefore the + * slopes for cost(x) are + * m_0 = 0 + * m_1 = 1.38 * k / (b-a) + * m_2 = 3.05 * k / (b-a) + * m_3 = 9.24 * k / (b-a) + * + * In the formula below cost_fraction[] corresponds to the + * t-space slopes and k = T * 1000, being T the total payment + * amount. + * + * The choice of k is based on the fact that cost[] is integer. + * Simply letting k=1 makes the slopes zero for most of useful + * arcs, eg. 1.38/(b-a) = 0. Since the slope has inverse units + * of msats the natural choice of k is to have equivalent msats + * units. With k = T * 1000 we neutralize the units of the slope + * and we lift from zero a whole range of useful arc costs. + * A linear arc will have a zero cost if (b-a) > 1000 * T + * that is roughly an arc that can carry 1000T above the `a` + * lower bound, therefore a probability of failure less than 1 + * in 1000. + * + * Introducing a constant probability of success factor works as + * follows: + * P_success(x) = P_const * P_pickhardt(x) + * + * cost(x) = -k log P_success(x) + * = -k log P_const - k log P_pickhardt(x) + * + * A simple linearization strategy for a constant factor f is + * simply consider linear_cost(x) = x * f / T, therefore + * linear_cost(x) = (-k log P_const / T) * x + linear_pickhardt(x) + * */ + cost[i] = k * (params->cost_fraction[i] / (b - a) + + linear_const_cost); } } @@ -951,7 +1004,8 @@ static struct flow **minflow(const tal_t *ctx, const struct gossmap_node *target, struct amount_msat amount, u32 mu, - double delay_feefactor) + double delay_feefactor, + double per_hop_probability) { struct flow **flow_paths; /* We allocate everything off this, and free it at the end, @@ -986,6 +1040,8 @@ static struct flow **minflow(const tal_t *ctx, params->delay_feefactor = delay_feefactor; params->base_fee_penalty = base_fee_penalty_estimate(amount); + assert(per_hop_probability > 0 && per_hop_probability <= 1); + params->const_success_probability = per_hop_probability; // build the uncertainty network with linearization and residual arcs struct graph *graph; @@ -1110,6 +1166,11 @@ static void init_linear_network_single_path( if (amount_msat_greater_eq(params->amount, maxcap)) continue; + const double probability = + pickhardt_richter_probability(mincap, maxcap, + params->amount) * + params->const_success_probability; + const u32 chan_id = gossmap_chan_idx(gossmap, c); const struct gossmap_node *next = @@ -1128,9 +1189,7 @@ static void init_linear_network_single_path( node_obj(next_id)); (*arc_capacity)[arc.idx] = 1; - (*arc_prob_cost)[arc.idx] = - (-1.0) * log(pickhardt_richter_probability( - mincap, maxcap, params->amount)); + (*arc_prob_cost)[arc.idx] = (-1.0) * log(probability); struct amount_msat fee; if (!amount_msat_fee(&fee, params->amount, @@ -1164,7 +1223,8 @@ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query const struct gossmap_node *source, const struct gossmap_node *target, struct amount_msat amount, u32 mu, - double delay_feefactor) + double delay_feefactor, + double per_hop_probability) { struct flow **flow_paths; /* We allocate everything off this, and free it at the end, @@ -1181,6 +1241,8 @@ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query params->accuracy = amount; params->delay_feefactor = delay_feefactor; params->base_fee_penalty = base_fee_penalty_estimate(amount); + assert(per_hop_probability > 0 && per_hop_probability <= 1); + params->const_success_probability = per_hop_probability; struct graph *graph; double *arc_prob_cost; @@ -1347,7 +1409,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, struct flow **(*solver)(const tal_t *, const struct route_query *, const struct gossmap_node *, const struct gossmap_node *, - struct amount_msat, u32, double)) + struct amount_msat, u32, double, double)) { const tal_t *working_ctx = tal(ctx, tal_t); const char *error_message; @@ -1361,6 +1423,13 @@ linear_routes(const tal_t *ctx, struct route_query *rq, /* we start at 1e-6 and increase it exponentially (x2) up to 10. */ double delay_feefactor = 1e-6; + /* An homogeneous 1% probability of failure is assumed per hop. We + * don't have a proper treatment of *activation costs* on arcs right + * now, but we use a simple linearization instead. The result is still + * a reduction of the mean path length. Shorter paths are more likely + * but also be more msat expensive. */ + double per_hop_probability = 0.99; + struct flow **new_flows = NULL; struct amount_msat all_deliver; @@ -1390,13 +1459,23 @@ linear_routes(const tal_t *ctx, struct route_query *rq, * amount. */ if (amount_msat_less_eq(amount_to_deliver, SINGLE_PATH_THRESHOLD)) { - new_flows = single_path_flow(working_ctx, rq, srcnode, - dstnode, amount_to_deliver, - mu, delay_feefactor); + new_flows = single_path_flow(working_ctx, + rq, + srcnode, + dstnode, + amount_to_deliver, + mu, + delay_feefactor, + per_hop_probability); } else { - new_flows = - solver(working_ctx, rq, srcnode, dstnode, - amount_to_deliver, mu, delay_feefactor); + new_flows = solver(working_ctx, + rq, + srcnode, + dstnode, + amount_to_deliver, + mu, + delay_feefactor, + per_hop_probability); } if (!new_flows) { diff --git a/tests/test_askrene.py b/tests/test_askrene.py index e5216ed89444..1a6001ff6ca0 100644 --- a/tests/test_askrene.py +++ b/tests/test_askrene.py @@ -1601,10 +1601,10 @@ def test_real_data(node_factory, bitcoind, executor): # CI, it's slow. if SLOW_MACHINE: limit = 25 - expected = (9, 24, 1935647, 219066, 89) + expected = (11, 24, 4663500, 221912, 96) else: limit = 100 - expected = (11, 95, 8026484, 925406, 89) + expected = (11, 95, 16558980, 944568, 95) # 0.5% is the norm MAX_FEE = AMOUNT // 200 @@ -1711,10 +1711,10 @@ def test_real_biases(node_factory, bitcoind, executor): # CI, it's slow. if SLOW_MACHINE: limit = 25 - expected = ({1: 6, 2: 7, 4: 12, 8: 13, 16: 18, 32: 23, 64: 24, 100: 24}, 0) + expected = ({1: 5, 2: 7, 4: 13, 8: 16, 16: 22, 32: 24, 64: 24, 100: 24}, 0) else: limit = 100 - expected = ({1: 26, 2: 33, 4: 48, 8: 57, 16: 77, 32: 90, 64: 95, 100: 95}, 0) + expected = ({1: 24, 2: 34, 4: 52, 8: 65, 16: 89, 32: 95, 64: 95, 100: 95}, 0) l1.rpc.askrene_create_layer('biases') num_changed = {} From c1dff07f9cb7f47084d553ba0217bdda9ba680f8 Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Tue, 20 Oct 2026 08:44:59 +0100 Subject: [PATCH 4/9] askrene: mcf: all feature costs are double Treat all cost features as the same type of variable. Changelog-None Signed-off-by: Lagrang3 --- plugins/askrene/child/mcf.c | 74 +++++++++++++++++-------------------- tests/test_askrene.py | 4 +- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/plugins/askrene/child/mcf.c b/plugins/askrene/child/mcf.c index 59a965cee39d..f5432e9a1524 100644 --- a/plugins/askrene/child/mcf.c +++ b/plugins/askrene/child/mcf.c @@ -451,15 +451,6 @@ static void linearize_channel(const struct pay_parameters *params, } } -static int cmp_u64(const u64 *a, const u64 *b, void *unused) -{ - if (*a < *b) - return -1; - if (*a > *b) - return 1; - return 0; -} - static int cmp_double(const double *a, const double *b, void *unused) { if (*a < *b) @@ -469,14 +460,13 @@ static int cmp_double(const double *a, const double *b, void *unused) return 0; } -static double get_median_ratio(const tal_t *working_ctx, - const struct graph *graph, - const double *arc_prob_cost, - const s64 *arc_fee_cost) +static double get_median(const tal_t *working_ctx, + const struct graph *graph, + const double *arc_cost) { const size_t max_num_arcs = graph_max_num_arcs(graph); - u64 *u64_arr = tal_arr(working_ctx, u64, max_num_arcs); - double *double_arr = tal_arr(working_ctx, double, max_num_arcs); + double *cost_arr = tal_arr(working_ctx, double, max_num_arcs); + double result; size_t n = 0; for (struct arc arc = {.idx=0};arc.idx < max_num_arcs; ++arc.idx) { @@ -484,31 +474,35 @@ static double get_median_ratio(const tal_t *working_ctx, if (arc_is_dual(graph, arc) || !arc_enabled(graph, arc)) continue; assert(n < max_num_arcs/2); - u64_arr[n] = arc_fee_cost[arc.idx]; - double_arr[n] = arc_prob_cost[arc.idx]; + cost_arr[n] = arc_cost[arc.idx]; n++; } - asort(u64_arr, n, cmp_u64, NULL); - asort(double_arr, n, cmp_double, NULL); - /* Empty network, or tiny probability, nobody cares */ - if (n == 0 || double_arr[n/2] < 0.001) - return 1; + asort(cost_arr, n, cmp_double, NULL); + + /* Empty network or too small value. */ + if (n == 0) + result = 1; + else + result = cost_arr[n / 2]; + + if (result < 1e-6) + result = 1; - /* You need to scale arc_prob_cost by this to match arc_fee_cost */ - return u64_arr[n/2] / double_arr[n/2]; + tal_free(cost_arr); + return result; } static void combine_cost_function(const tal_t *working_ctx, const struct graph *graph, const double *arc_prob_cost, - const s64 *arc_fee_cost, const s8 *biases, + const double *arc_fee_cost, const s8 *biases, s64 mu, s64 *arc_cost) { /* probabilty and fee costs are not directly comparable! * Scale by ratio of (positive) medians. */ - const double k = - get_median_ratio(working_ctx, graph, arc_prob_cost, arc_fee_cost); + const double k = get_median(working_ctx, graph, arc_fee_cost) / + get_median(working_ctx, graph, arc_prob_cost); const double ln_30 = log(30); const size_t max_num_arcs = graph_max_num_arcs(graph); @@ -518,13 +512,13 @@ static void combine_cost_function(const tal_t *working_ctx, continue; const double pcost = arc_prob_cost[arc.idx]; - const s64 fcost = arc_fee_cost[arc.idx]; + const double fcost = arc_fee_cost[arc.idx]; double combined; u32 chanidx; int chandir; s32 bias; - assert(fcost != INFINITE); + assert(fcost != DBL_MAX); assert(pcost != DBL_MAX); combined = fcost*mu + (MU_MAX-mu)*pcost*k; @@ -562,11 +556,11 @@ static void combine_cost_function(const tal_t *working_ctx, * use `base_fee_penalty` to weight the base fee and `delay_feefactor` to * weight the CLTV delay. * */ -static s64 linear_fee_cost(u32 base_fee, u32 proportional_fee, u16 cltv_delta, +static double linear_fee_cost(u32 base_fee, u32 proportional_fee, u16 cltv_delta, double base_fee_penalty, double delay_feefactor) { - s64 pfee = proportional_fee, + double pfee = proportional_fee, bfee = base_fee, delay = cltv_delta; @@ -584,7 +578,7 @@ static double base_fee_penalty_estimate(struct amount_msat amount) static void init_linear_network(const tal_t *ctx, const struct pay_parameters *params, struct graph **graph, double **arc_prob_cost, - s64 **arc_fee_cost, s64 **arc_capacity) + double **arc_fee_cost, s64 **arc_capacity) { const struct gossmap *gossmap = params->rq->gossmap; const size_t max_num_chans = gossmap_max_chan_idx(gossmap); @@ -596,9 +590,9 @@ static void init_linear_network(const tal_t *ctx, for (size_t i = 0; i < max_num_arcs; ++i) (*arc_prob_cost)[i] = DBL_MAX; - *arc_fee_cost = tal_arr(ctx, s64, max_num_arcs); + *arc_fee_cost = tal_arr(ctx, double, max_num_arcs); for (size_t i = 0; i < max_num_arcs; ++i) - (*arc_fee_cost)[i] = INT64_MAX; + (*arc_fee_cost)[i] = DBL_MAX; *arc_capacity = tal_arrz(ctx, s64, max_num_arcs); @@ -641,7 +635,7 @@ static void init_linear_network(const tal_t *ctx, linearize_channel(params, c, half, capacity, prob_cost); /* linear fee_cost per unit of flow */ - const s64 fee_cost = linear_fee_cost( + const double fee_cost = linear_fee_cost( c->half[half].base_fee, c->half[half].proportional_fee, c->half[half].delay, @@ -1046,7 +1040,7 @@ static struct flow **minflow(const tal_t *ctx, // build the uncertainty network with linearization and residual arcs struct graph *graph; double *arc_prob_cost; - s64 *arc_fee_cost; + double *arc_fee_cost; s64 *arc_capacity; init_linear_network(working_ctx, params, &graph, &arc_prob_cost, &arc_fee_cost, &arc_capacity); @@ -1113,7 +1107,7 @@ static struct flow **minflow(const tal_t *ctx, /* Initialize the data vectors for the single-path solver. */ static void init_linear_network_single_path( const tal_t *ctx, const struct pay_parameters *params, struct graph **graph, - double **arc_prob_cost, s64 **arc_fee_cost, s64 **arc_capacity) + double **arc_prob_cost, double **arc_fee_cost, s64 **arc_capacity) { const size_t max_num_chans = gossmap_max_chan_idx(params->rq->gossmap); const size_t max_num_arcs = max_num_chans * ARCS_PER_CHANNEL; @@ -1124,9 +1118,9 @@ static void init_linear_network_single_path( for (size_t i = 0; i < max_num_arcs; ++i) (*arc_prob_cost)[i] = DBL_MAX; - *arc_fee_cost = tal_arr(ctx, s64, max_num_arcs); + *arc_fee_cost = tal_arr(ctx, double, max_num_arcs); for (size_t i = 0; i < max_num_arcs; ++i) - (*arc_fee_cost)[i] = INT64_MAX; + (*arc_fee_cost)[i] = DBL_MAX; *arc_capacity = tal_arrz(ctx, s64, max_num_arcs); const struct gossmap *gossmap = params->rq->gossmap; @@ -1246,7 +1240,7 @@ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query struct graph *graph; double *arc_prob_cost; - s64 *arc_fee_cost; + double *arc_fee_cost; s64 *arc_capacity; init_linear_network_single_path(working_ctx, params, &graph, diff --git a/tests/test_askrene.py b/tests/test_askrene.py index 1a6001ff6ca0..b63b475c52c9 100644 --- a/tests/test_askrene.py +++ b/tests/test_askrene.py @@ -1601,10 +1601,10 @@ def test_real_data(node_factory, bitcoind, executor): # CI, it's slow. if SLOW_MACHINE: limit = 25 - expected = (11, 24, 4663500, 221912, 96) + expected = (11, 24, 4663500, 221543, 96) else: limit = 100 - expected = (11, 95, 16558980, 944568, 95) + expected = (11, 95, 16558980, 941774, 95) # 0.5% is the norm MAX_FEE = AMOUNT // 200 From 9f6ed92d96226bbd6c672b41bf32067661fa7fe8 Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Tue, 20 Oct 2026 10:48:46 +0100 Subject: [PATCH 5/9] askrene: mcf: introduce cost dials Use cost dials normalized to 1 to combine the cost of different features. This is a generalization of the mu factor combination with medians ratio applied to more than 2 features: fees and delays. Just like that fee constraining got slightly better. Before mu (the corresponding fee dial) started at 1% and increased linearly in chunks of 10%. Now every dial is increased by a factor of 2 per iteration and it approximates 1 asymptotically as the dials are renormalized at each iteration. We have a cut-off at 1 - 1e-6, that means 27 iterations until mu reaches the cut-off. Changelog-None Signed-off-by: Lagrang3 --- plugins/askrene/child/mcf.c | 173 ++++++++++++++++++++++++------------ tests/test_askrene.py | 6 +- 2 files changed, 117 insertions(+), 62 deletions(-) diff --git a/plugins/askrene/child/mcf.c b/plugins/askrene/child/mcf.c index f5432e9a1524..7f0e874fba25 100644 --- a/plugins/askrene/child/mcf.c +++ b/plugins/askrene/child/mcf.c @@ -168,7 +168,6 @@ static const double CHANNEL_PIVOTS[]={0,0.5,0.8,0.95}; static const s64 INFINITE = INT64_MAX; -static const s64 MU_MAX = 100; /* every payment under 1000sat will be routed through a single path */ static const struct amount_msat SINGLE_PATH_THRESHOLD = AMOUNT_MSAT(1000000); @@ -299,7 +298,6 @@ struct pay_parameters { double cap_fraction[CHANNEL_PARTS], cost_fraction[CHANNEL_PARTS]; - double delay_feefactor; double base_fee_penalty; /* An homogeneous (same for every channel) constant (does not depend on @@ -496,13 +494,25 @@ static double get_median(const tal_t *working_ctx, static void combine_cost_function(const tal_t *working_ctx, const struct graph *graph, const double *arc_prob_cost, - const double *arc_fee_cost, const s8 *biases, - s64 mu, s64 *arc_cost) + const double *arc_fee_cost, + const double *arc_delay_cost, + const s8 *biases, + double fee_dial, + double delay_dial, + double prob_dial, + s64 *arc_cost) { + assert(prob_dial >= 0); + assert(fee_dial >= 0); + assert(delay_dial >= 0); + const double total_dial = prob_dial + fee_dial + delay_dial; + assert(fabs(total_dial - 1.0) < 1e-9); + /* probabilty and fee costs are not directly comparable! * Scale by ratio of (positive) medians. */ - const double k = get_median(working_ctx, graph, arc_fee_cost) / - get_median(working_ctx, graph, arc_prob_cost); + const double median_fee = get_median(working_ctx, graph, arc_fee_cost); + const double median_prob = get_median(working_ctx, graph, arc_prob_cost); + const double median_delay = get_median(working_ctx, graph, arc_delay_cost); const double ln_30 = log(30); const size_t max_num_arcs = graph_max_num_arcs(graph); @@ -513,6 +523,7 @@ static void combine_cost_function(const tal_t *working_ctx, const double pcost = arc_prob_cost[arc.idx]; const double fcost = arc_fee_cost[arc.idx]; + const double dcost = arc_delay_cost[arc.idx]; double combined; u32 chanidx; int chandir; @@ -520,7 +531,18 @@ static void combine_cost_function(const tal_t *working_ctx, assert(fcost != DBL_MAX); assert(pcost != DBL_MAX); - combined = fcost*mu + (MU_MAX-mu)*pcost*k; + combined = fcost / median_fee * fee_dial + + dcost / median_delay * delay_dial + + pcost / median_prob * prob_dial; + + /* The median costs corresponds to the normalize value 1.0. + * An arbitrary up-scale is used here to capture values below + * the median when converting costs from double to u64. + * To match the rough values of the previous version + * (see commit 2a0f09fc2d79c636817d610e529c3160b531d51e) + * we use the median of the fee times the old MU_MAX (100). + * */ + combined *= median_fee * 100; /* Bias is in human scale, where "bigger is better" */ arc_to_parts(arc, &chanidx, &chandir, NULL, NULL); @@ -552,19 +574,14 @@ static void combine_cost_function(const tal_t *working_ctx, * into * * fee = c_fee/10^6 * x - * - * use `base_fee_penalty` to weight the base fee and `delay_feefactor` to - * weight the CLTV delay. * */ -static double linear_fee_cost(u32 base_fee, u32 proportional_fee, u16 cltv_delta, - double base_fee_penalty, - double delay_feefactor) +static double linear_fee_cost(u32 base_fee, u32 proportional_fee, + double base_fee_penalty) { double pfee = proportional_fee, - bfee = base_fee, - delay = cltv_delta; + bfee = base_fee; - return pfee + bfee* base_fee_penalty+ delay*delay_feefactor; + return pfee + bfee* base_fee_penalty; } /* This is inversely proportional to the amount we expect to send. Let's @@ -578,7 +595,9 @@ static double base_fee_penalty_estimate(struct amount_msat amount) static void init_linear_network(const tal_t *ctx, const struct pay_parameters *params, struct graph **graph, double **arc_prob_cost, - double **arc_fee_cost, s64 **arc_capacity) + double **arc_fee_cost, + double **arc_delay_cost, + s64 **arc_capacity) { const struct gossmap *gossmap = params->rq->gossmap; const size_t max_num_chans = gossmap_max_chan_idx(gossmap); @@ -594,6 +613,10 @@ static void init_linear_network(const tal_t *ctx, for (size_t i = 0; i < max_num_arcs; ++i) (*arc_fee_cost)[i] = DBL_MAX; + *arc_delay_cost = tal_arr(ctx, double, max_num_arcs); + for (size_t i = 0; i < max_num_arcs; ++i) + (*arc_delay_cost)[i] = DBL_MAX; + *arc_capacity = tal_arrz(ctx, s64, max_num_arcs); for(struct gossmap_node *node = gossmap_first_node(gossmap); @@ -638,9 +661,7 @@ static void init_linear_network(const tal_t *ctx, const double fee_cost = linear_fee_cost( c->half[half].base_fee, c->half[half].proportional_fee, - c->half[half].delay, - params->base_fee_penalty, - params->delay_feefactor); + params->base_fee_penalty); // let's subscribe the 4 parts of the channel direction // (c,half), the dual of these guys will be subscribed @@ -660,6 +681,7 @@ static void init_linear_network(const tal_t *ctx, (*arc_capacity)[arc.idx] = capacity[k]; (*arc_prob_cost)[arc.idx] = prob_cost[k]; (*arc_fee_cost)[arc.idx] = fee_cost; + (*arc_delay_cost)[arc.idx] = c->half[half].delay; // + the respective dual struct arc dual = arc_dual(*graph, arc); @@ -667,6 +689,7 @@ static void init_linear_network(const tal_t *ctx, (*arc_capacity)[dual.idx] = 0; (*arc_prob_cost)[dual.idx] = -prob_cost[k]; (*arc_fee_cost)[dual.idx] = -fee_cost; + (*arc_delay_cost)[dual.idx] = -c->half[half].delay; } } } @@ -997,8 +1020,9 @@ static struct flow **minflow(const tal_t *ctx, const struct gossmap_node *source, const struct gossmap_node *target, struct amount_msat amount, - u32 mu, - double delay_feefactor, + double mu, + double delay_dial, + double prob_dial, double per_hop_probability) { struct flow **flow_paths; @@ -1032,7 +1056,6 @@ static struct flow **minflow(const tal_t *ctx, /params->cap_fraction[i]; } - params->delay_feefactor = delay_feefactor; params->base_fee_penalty = base_fee_penalty_estimate(amount); assert(per_hop_probability > 0 && per_hop_probability <= 1); params->const_success_probability = per_hop_probability; @@ -1041,9 +1064,12 @@ static struct flow **minflow(const tal_t *ctx, struct graph *graph; double *arc_prob_cost; double *arc_fee_cost; + double *arc_delay_cost; s64 *arc_capacity; init_linear_network(working_ctx, params, &graph, &arc_prob_cost, - &arc_fee_cost, &arc_capacity); + &arc_fee_cost, + &arc_delay_cost, + &arc_capacity); const size_t max_num_arcs = graph_max_num_arcs(graph); const size_t max_num_nodes = graph_max_num_nodes(graph); @@ -1071,7 +1097,12 @@ static struct flow **minflow(const tal_t *ctx, goto fail; } combine_cost_function(working_ctx, graph, arc_prob_cost, arc_fee_cost, - rq->biases, mu, arc_cost); + arc_delay_cost, + rq->biases, + mu, + delay_dial, + prob_dial, + arc_cost); /* We solve a linear MCF problem. */ if (!mcf_refinement(working_ctx, @@ -1107,7 +1138,8 @@ static struct flow **minflow(const tal_t *ctx, /* Initialize the data vectors for the single-path solver. */ static void init_linear_network_single_path( const tal_t *ctx, const struct pay_parameters *params, struct graph **graph, - double **arc_prob_cost, double **arc_fee_cost, s64 **arc_capacity) + double **arc_prob_cost, double **arc_fee_cost, double **arc_delay_cost, + s64 **arc_capacity) { const size_t max_num_chans = gossmap_max_chan_idx(params->rq->gossmap); const size_t max_num_arcs = max_num_chans * ARCS_PER_CHANNEL; @@ -1121,6 +1153,10 @@ static void init_linear_network_single_path( *arc_fee_cost = tal_arr(ctx, double, max_num_arcs); for (size_t i = 0; i < max_num_arcs; ++i) (*arc_fee_cost)[i] = DBL_MAX; + + *arc_delay_cost = tal_arr(ctx, double, max_num_arcs); + for (size_t i = 0; i < max_num_arcs; ++i) + (*arc_delay_cost)[i] = DBL_MAX; *arc_capacity = tal_arrz(ctx, s64, max_num_arcs); const struct gossmap *gossmap = params->rq->gossmap; @@ -1190,9 +1226,8 @@ static void init_linear_network_single_path( c->half[half].base_fee, c->half[half].proportional_fee)) abort(); - (*arc_fee_cost)[arc.idx] = - fee.millisatoshis + /* Raw: fee cost */ - params->delay_feefactor * c->half[half].delay; + (*arc_fee_cost)[arc.idx] = fee.millisatoshis; /* Raw: fee cost */ + (*arc_delay_cost)[arc.idx] = c->half[half].delay; } } } @@ -1204,20 +1239,19 @@ static void init_linear_network_single_path( * @source: the source to start from * @target: the target to pay * @amount: the amount we want to reach @target - * @mu: 0 = corresponds to only probabilities, 100 corresponds to only fee. - * @delay_feefactor: convert 1 block delay into msat. - * - * @delay_feefactor converts 1 block delay into msat, as if it were an additional - * fee. So if a CLTV delay on a node is 5 blocks, that's treated as if it - * were a fee of 5 * @delay_feefactor. + * @mu: 0 = corresponds to only probabilities, 1 corresponds to only fee. + * @delay_dial: same as mu but for delays. + * @prob_dial: same as mu but for probabilities. * * Returns an array with one flow which deliver amount to target, or NULL. */ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query *rq, const struct gossmap_node *source, const struct gossmap_node *target, - struct amount_msat amount, u32 mu, - double delay_feefactor, + struct amount_msat amount, + double mu, + double delay_dial, + double prob_dial, double per_hop_probability) { struct flow **flow_paths; @@ -1233,7 +1267,6 @@ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query /* for the single-path solver the accuracy does not detriment * performance */ params->accuracy = amount; - params->delay_feefactor = delay_feefactor; params->base_fee_penalty = base_fee_penalty_estimate(amount); assert(per_hop_probability > 0 && per_hop_probability <= 1); params->const_success_probability = per_hop_probability; @@ -1241,10 +1274,12 @@ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query struct graph *graph; double *arc_prob_cost; double *arc_fee_cost; + double *arc_delay_cost; s64 *arc_capacity; init_linear_network_single_path(working_ctx, params, &graph, &arc_prob_cost, &arc_fee_cost, + &arc_delay_cost, &arc_capacity); const struct node dst = {.idx = gossmap_node_idx(rq->gossmap, target)}; @@ -1259,7 +1294,8 @@ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query struct arc *prev = tal_arrz(working_ctx, struct arc, max_num_nodes); combine_cost_function(working_ctx, graph, arc_prob_cost, arc_fee_cost, - rq->biases, mu, arc_cost); + arc_delay_cost, rq->biases, mu, delay_dial, + prob_dial, arc_cost); /* We solve a linear cost flow problem. */ if (!dijkstra_path(working_ctx, graph, src, dst, @@ -1384,6 +1420,18 @@ static bool check_htlc_max_limits(struct route_query *rq, struct flow **flows) return true; } +/* Renormalize the dials so that their sum is always 1. + * All values assumed to be positive. */ +static void renormalize_dials(double *fee_dial, + double *delay_dial, + double *prob_dial) +{ + double sum = *fee_dial + *delay_dial + *prob_dial; + *fee_dial /= sum; + *delay_dial /= sum; + *prob_dial /= sum; +} + /* FIXME: add extra constraint maximum route length, use an activation * probability cost for each channel. Recall that every activation cost, eg. * base fee and activation probability can only be properly added modifying the @@ -1403,19 +1451,25 @@ linear_routes(const tal_t *ctx, struct route_query *rq, struct flow **(*solver)(const tal_t *, const struct route_query *, const struct gossmap_node *, const struct gossmap_node *, - struct amount_msat, u32, double, double)) + struct amount_msat, + double, + double, + double, + double)) { + /* Almost 1 to limit the dials' iterations. */ + const double near_one = 1. - 1e-6; const tal_t *working_ctx = tal(ctx, tal_t); const char *error_message; struct amount_msat amount_to_deliver = amount; struct amount_msat feebudget = maxfee; - /* FIXME: mu is an integer from 0 to MU_MAX that we use to combine fees - * and probability costs, but I think we can make it a real number from - * 0 to 1. */ - u32 mu = 1; - /* we start at 1e-6 and increase it exponentially (x2) up to 10. */ - double delay_feefactor = 1e-6; + /* These are the cost dials for the feature costs we combine. + * Each one must be a number between 0 and 1, and the sum of all must be + * equal to 1. */ + double mu = 0.01; + double delay_dial = 0.001; + double prob_dial = 0.989; /* An homogeneous 1% probability of failure is assumed per hop. We * don't have a proper treatment of *activation costs* on arcs right @@ -1459,7 +1513,8 @@ linear_routes(const tal_t *ctx, struct route_query *rq, dstnode, amount_to_deliver, mu, - delay_feefactor, + delay_dial, + prob_dial, per_hop_probability); } else { new_flows = solver(working_ctx, @@ -1468,7 +1523,8 @@ linear_routes(const tal_t *ctx, struct route_query *rq, dstnode, amount_to_deliver, mu, - delay_feefactor, + delay_dial, + prob_dial, per_hop_probability); } @@ -1532,18 +1588,16 @@ linear_routes(const tal_t *ctx, struct route_query *rq, goto fail; } if (amount_msat_greater(all_fees, partial_feebudget)) { - if (mu < MU_MAX) { + if (mu < near_one) { /* all_fees exceed the strong budget limit, try * to fix it increasing mu. */ - if (mu == 1) - mu = 10; - else - mu += 10; - mu = MIN(mu, MU_MAX); + mu *= 2; + renormalize_dials(&mu, &delay_dial, + &prob_dial); child_log( tmpctx, LOG_INFORM, "The flows had a fee of %s, greater than " - "max of %s, retrying with mu of %u%%...", + "max of %s, retrying with mu of %.6lf...", fmt_amount_msat(tmpctx, all_fees), fmt_amount_msat(tmpctx, partial_feebudget), mu); @@ -1574,7 +1628,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, /* Too much delay? */ if (finalcltv + flows_worst_delay(new_flows) > maxdelay) { - if (delay_feefactor > 10) { + if (delay_dial >= near_one) { error_message = child_log(ctx, LOG_UNUSUAL, "Could not find route without " @@ -1583,12 +1637,13 @@ linear_routes(const tal_t *ctx, struct route_query *rq, goto fail; } - delay_feefactor *= 2; + delay_dial *= 2; + renormalize_dials(&mu, &delay_dial, &prob_dial); child_log(tmpctx, LOG_INFORM, "The worst flow delay is %" PRIu64 - " (> %i), retrying with delay_feefactor %f...", + " (> %i), retrying with delay_dial %f...", flows_worst_delay(new_flows), maxdelay - finalcltv, - delay_feefactor); + delay_dial); continue; } diff --git a/tests/test_askrene.py b/tests/test_askrene.py index b63b475c52c9..4efaca038e19 100644 --- a/tests/test_askrene.py +++ b/tests/test_askrene.py @@ -1601,10 +1601,10 @@ def test_real_data(node_factory, bitcoind, executor): # CI, it's slow. if SLOW_MACHINE: limit = 25 - expected = (11, 24, 4663500, 221543, 96) + expected = (14, 24, 4663500, 209635, 96) else: limit = 100 - expected = (11, 95, 16558980, 941774, 95) + expected = (18, 95, 16558980, 911656, 95) # 0.5% is the norm MAX_FEE = AMOUNT // 200 @@ -1714,7 +1714,7 @@ def test_real_biases(node_factory, bitcoind, executor): expected = ({1: 5, 2: 7, 4: 13, 8: 16, 16: 22, 32: 24, 64: 24, 100: 24}, 0) else: limit = 100 - expected = ({1: 24, 2: 34, 4: 52, 8: 65, 16: 89, 32: 95, 64: 95, 100: 95}, 0) + expected = ({1: 23, 2: 34, 4: 52, 8: 64, 16: 89, 32: 95, 64: 95, 100: 95}, 0) l1.rpc.askrene_create_layer('biases') num_changed = {} From 21d10cbe16399be8660d013d4f8e8c831879d09d Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Thu, 22 Oct 2026 21:44:17 +0100 Subject: [PATCH 6/9] askrene: getroutes: add maxhops option No updates to the RPC, no need to run as developer. We inject the maxhops value through a layer name. Changelog-None Signed-off-by: Lagrang3 --- plugins/askrene/askrene.c | 63 +++++++++++++++++++++++++++- plugins/askrene/child/child.c | 11 +++-- plugins/askrene/child/child.h | 5 ++- plugins/askrene/child/flow.c | 11 +++++ plugins/askrene/child/flow.h | 3 ++ plugins/askrene/child/mcf.c | 78 ++++++++++++++++++++++++++--------- plugins/askrene/child/mcf.h | 9 +++- tests/test_askrene.py | 6 +-- 8 files changed, 154 insertions(+), 32 deletions(-) diff --git a/plugins/askrene/askrene.c b/plugins/askrene/askrene.c index d06d2ceea804..f7f31ececf92 100644 --- a/plugins/askrene/askrene.c +++ b/plugins/askrene/askrene.c @@ -29,6 +29,8 @@ #include #include #include +#include +#include #include #include #include @@ -91,6 +93,44 @@ static struct command_result *param_direction(struct command *cmd, return NULL; } +static bool is_maxhops(const char *s) +{ + return strncmp(s, "maxhops=", strlen("maxhops=")) == 0; +} + +static bool parse_maxhops(const char *s, u32 *v) +{ + if(!is_maxhops(s)) + return false; + + const char *num = s + strlen("maxhops="); + char *end; + unsigned long n; + + if (*num < '0' || *num > '9') + return false; + errno = 0; + n = strtoul(num, &end, 10); + if (*end != '\0' || errno == ERANGE || n > UINT32_MAX) + return false; + *v = n; + return true; +} + +static u32 extract_maxhops(const char ***layers, u32 default_maxhops) +{ + u32 maxhops = default_maxhops; + for (size_t i = 0; i < tal_count(*layers); i++) { + u32 v; + if (parse_maxhops((*layers)[i], &v)) { + maxhops = v; + tal_arr_remove(layers, i); + i--; + } + } + return maxhops; +} + /* Valid, known layers */ static struct command_result *param_layer_names(struct command *cmd, const char *name, @@ -112,6 +152,15 @@ static struct command_result *param_layer_names(struct command *cmd, "should be a string"); (*arr)[i] = json_strdup(*arr, buffer, t); + if (is_maxhops((*arr)[i])) { + u32 v; + if (!parse_maxhops((*arr)[i], &v)) + return command_fail_badparam( + cmd, name, buffer, t, + "maxhops must be a u32"); + continue; + } + /* Must be a known layer name */ if (streq((*arr)[i], "auto.localchans") || streq((*arr)[i], "auto.sourcefree") @@ -359,6 +408,7 @@ struct getroutes_info { /* Non-NULL if we are told to use "auto.localchans" */ struct layer *local_layer; u32 maxparts; + u32 maxhops; }; static void add_layer(const struct layer ***layers, @@ -716,7 +766,11 @@ static struct command_result *do_getroutes(struct command *cmd, take(capacities), info->dev_algo == ALGO_SINGLE_PATH, deadline, srcnode, dstnode, info->amount, - info->maxfee, info->finalcltv, info->maxdelay, info->maxparts, + info->maxfee, + info->finalcltv, + info->maxdelay, + info->maxparts, + info->maxhops, include_fees, cmd->id, cmd->filter, include_next_node_id, @@ -900,7 +954,7 @@ static struct command_result *json_getroutes(struct command *cmd, /* FIXME: Typo in spec for CLTV in descripton! But it breaks our spelling check, so we omit it above */ const u32 maxdelay_allowed = 2016; struct askrene *askrene = get_askrene(cmd->plugin); - const u32 default_maxparts = 100; + const u32 default_maxparts = 100, default_maxhops = 100; struct getroutes_info *info = tal(cmd, struct getroutes_info); /* param functions require pointers */ struct node_id *source, *dest; @@ -962,6 +1016,11 @@ static struct command_result *json_getroutes(struct command *cmd, info->additional_costs = new_htable(info, additional_cost_htable); info->maxparts = *maxparts; + /* A trick to pass maxhops without introducing a new RPC or requiring + * developer option. We pass maxhops as a layer string, eg. + * "maxhops=10" */ + info->maxhops = extract_maxhops(&info->layers, default_maxhops); + if (askrene->num_live_requests >= askrene->max_children) { cmd_log(tmpctx, cmd, LOG_INFORM, "Too many running at once (%zu vs %u): waiting", diff --git a/plugins/askrene/child/child.c b/plugins/askrene/child/child.c index 72aecd68dfd8..d140af9bdddb 100644 --- a/plugins/askrene/child/child.c +++ b/plugins/askrene/child/child.c @@ -219,8 +219,11 @@ void run_child(const struct gossmap *gossmap, const struct gossmap_node *srcnode, const struct gossmap_node *dstnode, struct amount_msat amount, struct amount_msat maxfee, - u32 finalcltv, u32 maxdelay, size_t maxparts, - bool include_fees, + u32 finalcltv, + u32 maxdelay, + size_t maxparts, + u32 maxhops, + bool include_fees, const char *cmd_id, struct json_filter *cmd_filter, bool include_next_node_id, @@ -243,11 +246,11 @@ void run_child(const struct gossmap *gossmap, if (single_path) { err = single_path_routes(rq, rq, deadline, srcnode, dstnode, amount, maxfee, finalcltv, - maxdelay, &flows, &probability, &ecode); + maxdelay, maxhops, &flows, &probability, &ecode); } else { err = default_routes(rq, rq, deadline, srcnode, dstnode, amount, maxfee, finalcltv, maxdelay, - maxparts, &flows, &probability, &ecode); + maxparts, maxhops, &flows, &probability, &ecode); } if (err) { write_all(replyfd, &ecode, sizeof(ecode)); diff --git a/plugins/askrene/child/child.h b/plugins/askrene/child/child.h index 5cb6688e52be..46068e0606c7 100644 --- a/plugins/askrene/child/child.h +++ b/plugins/askrene/child/child.h @@ -27,7 +27,10 @@ void run_child(const struct gossmap *gossmap, const struct gossmap_node *srcnode, const struct gossmap_node *dstnode, struct amount_msat amount, struct amount_msat maxfee, - u32 finalcltv, u32 maxdelay, size_t maxparts, + u32 finalcltv, + u32 maxdelay, + size_t maxparts, + u32 maxhops, bool include_fees, const char *cmd_id, struct json_filter *cmd_filter, diff --git a/plugins/askrene/child/flow.c b/plugins/askrene/child/flow.c index 5b21c27319cf..426f32b1014c 100644 --- a/plugins/askrene/child/flow.c +++ b/plugins/askrene/child/flow.c @@ -173,6 +173,17 @@ u64 flows_worst_delay(struct flow **flows) return maxdelay; } +u32 flows_worst_length(struct flow **flows) +{ + u32 maxlen = 0; + for (size_t i = 0; i < tal_count(flows); i++) { + u32 len = tal_count(flows[i]->path); + if (len > maxlen) + maxlen = len; + } + return maxlen; +} + const char *fmt_flows_step_scid(const tal_t *ctx, const struct route_query *rq, const struct flow *flow, size_t i) diff --git a/plugins/askrene/child/flow.h b/plugins/askrene/child/flow.h index eeec5a96fb44..45b8ccc8cb1b 100644 --- a/plugins/askrene/child/flow.h +++ b/plugins/askrene/child/flow.h @@ -57,6 +57,9 @@ u64 flow_delay(const struct flow *flow); /* Max CLTV any of these flows requires */ u64 flows_worst_delay(struct flow **flows); +/* Max number of hops any of these flows requires */ +u32 flows_worst_length(struct flow **flows); + const char *fmt_flows_step_scid(const tal_t *ctx, const struct route_query *rq, const struct flow *flow, size_t i); diff --git a/plugins/askrene/child/mcf.c b/plugins/askrene/child/mcf.c index 7f0e874fba25..1c7a472010f0 100644 --- a/plugins/askrene/child/mcf.c +++ b/plugins/askrene/child/mcf.c @@ -499,13 +499,15 @@ static void combine_cost_function(const tal_t *working_ctx, const s8 *biases, double fee_dial, double delay_dial, + double per_hop_dial, double prob_dial, s64 *arc_cost) { assert(prob_dial >= 0); assert(fee_dial >= 0); assert(delay_dial >= 0); - const double total_dial = prob_dial + fee_dial + delay_dial; + assert(per_hop_dial >= 0); + const double total_dial = prob_dial + fee_dial + delay_dial + per_hop_dial; assert(fabs(total_dial - 1.0) < 1e-9); /* probabilty and fee costs are not directly comparable! @@ -533,6 +535,8 @@ static void combine_cost_function(const tal_t *working_ctx, assert(pcost != DBL_MAX); combined = fcost / median_fee * fee_dial + dcost / median_delay * delay_dial + + /* per-hop cost is 1 for every arc */ + 1.0 * per_hop_dial + pcost / median_prob * prob_dial; /* The median costs corresponds to the normalize value 1.0. @@ -1022,6 +1026,7 @@ static struct flow **minflow(const tal_t *ctx, struct amount_msat amount, double mu, double delay_dial, + double per_hop_dial, double prob_dial, double per_hop_probability) { @@ -1101,6 +1106,7 @@ static struct flow **minflow(const tal_t *ctx, rq->biases, mu, delay_dial, + per_hop_dial, prob_dial, arc_cost); @@ -1251,6 +1257,7 @@ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query struct amount_msat amount, double mu, double delay_dial, + double per_hop_dial, double prob_dial, double per_hop_probability) { @@ -1295,7 +1302,7 @@ static struct flow **single_path_flow(const tal_t *ctx, const struct route_query combine_cost_function(working_ctx, graph, arc_prob_cost, arc_fee_cost, arc_delay_cost, rq->biases, mu, delay_dial, - prob_dial, arc_cost); + per_hop_dial, prob_dial, arc_cost); /* We solve a linear cost flow problem. */ if (!dijkstra_path(working_ctx, graph, src, dst, @@ -1424,21 +1431,16 @@ static bool check_htlc_max_limits(struct route_query *rq, struct flow **flows) * All values assumed to be positive. */ static void renormalize_dials(double *fee_dial, double *delay_dial, + double *per_hop_dial, double *prob_dial) { - double sum = *fee_dial + *delay_dial + *prob_dial; + double sum = *fee_dial + *delay_dial + *per_hop_dial + *prob_dial; *fee_dial /= sum; *delay_dial /= sum; + *per_hop_dial /= sum; *prob_dial /= sum; } -/* FIXME: add extra constraint maximum route length, use an activation - * probability cost for each channel. Recall that every activation cost, eg. - * base fee and activation probability can only be properly added modifying the - * graph topology by creating an activation node for every half channel. */ -/* FIXME: add extra constraint maximum number of routes, fixes issue 8331. */ -/* FIXME: add a boolean option to make recipient pay for fees, fixes issue 8353. - */ static const char * linear_routes(const tal_t *ctx, struct route_query *rq, struct timemono deadline, @@ -1446,6 +1448,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, const struct gossmap_node *dstnode, struct amount_msat amount, struct amount_msat maxfee, u32 finalcltv, u32 maxdelay, size_t maxparts, + u32 maxhops, struct flow ***flows, double *probability, enum jsonrpc_errcode *ecode, struct flow **(*solver)(const tal_t *, const struct route_query *, @@ -1455,6 +1458,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, double, double, double, + double, double)) { /* Almost 1 to limit the dials' iterations. */ @@ -1469,7 +1473,8 @@ linear_routes(const tal_t *ctx, struct route_query *rq, * equal to 1. */ double mu = 0.01; double delay_dial = 0.001; - double prob_dial = 0.989; + double per_hop_dial = 0.001; + double prob_dial = 0.988; /* An homogeneous 1% probability of failure is assumed per hop. We * don't have a proper treatment of *activation costs* on arcs right @@ -1514,6 +1519,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, amount_to_deliver, mu, delay_dial, + per_hop_dial, prob_dial, per_hop_probability); } else { @@ -1524,6 +1530,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, amount_to_deliver, mu, delay_dial, + per_hop_dial, prob_dial, per_hop_probability); } @@ -1593,6 +1600,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, * to fix it increasing mu. */ mu *= 2; renormalize_dials(&mu, &delay_dial, + &per_hop_dial, &prob_dial); child_log( tmpctx, LOG_INFORM, @@ -1638,7 +1646,8 @@ linear_routes(const tal_t *ctx, struct route_query *rq, } delay_dial *= 2; - renormalize_dials(&mu, &delay_dial, &prob_dial); + renormalize_dials(&mu, &delay_dial, &per_hop_dial, + &prob_dial); child_log(tmpctx, LOG_INFORM, "The worst flow delay is %" PRIu64 " (> %i), retrying with delay_dial %f...", @@ -1647,6 +1656,30 @@ linear_routes(const tal_t *ctx, struct route_query *rq, continue; } + /* Too many hops? */ + u32 count_hops = flows_worst_length(new_flows); + if (count_hops > maxhops) { + if (per_hop_dial >= near_one) { + error_message = + child_log(ctx, LOG_UNUSUAL, + "Could not find route without " + "excessive hop count"); + *ecode = PAY_ROUTE_NOT_FOUND; + goto fail; + } + + per_hop_dial *= 2; + renormalize_dials(&mu, &delay_dial, &per_hop_dial, + &prob_dial); + + child_log(tmpctx, LOG_INFORM, + "The worst flow number of hops is %" PRIu32 + " (> %" PRIu32 + "), retrying with per_hop_dial %f ...", + count_hops, maxhops, per_hop_dial); + continue; + } + all_fees = AMOUNT_MSAT(0); all_deliver = AMOUNT_MSAT(0); /* add the new flows to the final solution */ @@ -1760,14 +1793,17 @@ const char *default_routes(const tal_t *ctx, struct route_query *rq, const struct gossmap_node *srcnode, const struct gossmap_node *dstnode, struct amount_msat amount, struct amount_msat maxfee, - u32 finalcltv, u32 maxdelay, size_t maxparts, + u32 finalcltv, + u32 maxdelay, + size_t maxparts, + u32 maxhops, struct flow ***flows, double *probability, enum jsonrpc_errcode *ecode) { - return linear_routes(ctx, rq, deadline, srcnode, dstnode, amount, maxfee, - finalcltv, maxdelay, maxparts, flows, probability, ecode, - minflow); + return linear_routes(ctx, rq, deadline, srcnode, dstnode, amount, + maxfee, finalcltv, maxdelay, maxparts, maxhops, + flows, probability, ecode, minflow); } const char *single_path_routes(const tal_t *ctx, struct route_query *rq, @@ -1776,11 +1812,13 @@ const char *single_path_routes(const tal_t *ctx, struct route_query *rq, const struct gossmap_node *dstnode, struct amount_msat amount, struct amount_msat maxfee, u32 finalcltv, - u32 maxdelay, struct flow ***flows, + u32 maxdelay, + u32 maxhops, + struct flow ***flows, double *probability, enum jsonrpc_errcode *ecode) { - return linear_routes(ctx, rq, deadline, srcnode, dstnode, amount, maxfee, - finalcltv, maxdelay, 1, flows, probability, ecode, - single_path_flow); + return linear_routes(ctx, rq, deadline, srcnode, dstnode, amount, + maxfee, finalcltv, maxdelay, 1, maxhops, flows, + probability, ecode, single_path_flow); } diff --git a/plugins/askrene/child/mcf.h b/plugins/askrene/child/mcf.h index 9cceac32af31..f2dd2a8d3bd8 100644 --- a/plugins/askrene/child/mcf.h +++ b/plugins/askrene/child/mcf.h @@ -18,7 +18,10 @@ const char *default_routes(const tal_t *ctx, struct route_query *rq, const struct gossmap_node *dstnode, struct amount_msat amount, struct amount_msat maxfee, u32 finalcltv, - u32 maxdelay, size_t maxparts, struct flow ***flows, + u32 maxdelay, + size_t maxparts, + u32 maxhops, + struct flow ***flows, double *probability, enum jsonrpc_errcode *ecode); @@ -29,7 +32,9 @@ const char *single_path_routes(const tal_t *ctx, struct route_query *rq, const struct gossmap_node *dstnode, struct amount_msat amount, struct amount_msat maxfee, u32 finalcltv, - u32 maxdelay, struct flow ***flows, + u32 maxdelay, + u32 maxhops, + struct flow ***flows, double *probability, enum jsonrpc_errcode *ecode); diff --git a/tests/test_askrene.py b/tests/test_askrene.py index 4efaca038e19..96daa094cf48 100644 --- a/tests/test_askrene.py +++ b/tests/test_askrene.py @@ -1604,7 +1604,7 @@ def test_real_data(node_factory, bitcoind, executor): expected = (14, 24, 4663500, 209635, 96) else: limit = 100 - expected = (18, 95, 16558980, 911656, 95) + expected = (17, 95, 16558980, 911656, 95) # 0.5% is the norm MAX_FEE = AMOUNT // 200 @@ -1711,10 +1711,10 @@ def test_real_biases(node_factory, bitcoind, executor): # CI, it's slow. if SLOW_MACHINE: limit = 25 - expected = ({1: 5, 2: 7, 4: 13, 8: 16, 16: 22, 32: 24, 64: 24, 100: 24}, 0) + expected = ({1: 5, 2: 6, 4: 13, 8: 16, 16: 22, 32: 24, 64: 24, 100: 24}, 0) else: limit = 100 - expected = ({1: 23, 2: 34, 4: 52, 8: 64, 16: 89, 32: 95, 64: 95, 100: 95}, 0) + expected = ({1: 23, 2: 32, 4: 52, 8: 64, 16: 89, 32: 95, 64: 95, 100: 95}, 0) l1.rpc.askrene_create_layer('biases') num_changed = {} From 739f948fea70d40877a0c14ef1ec9dc29f2cfae4 Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Thu, 22 Oct 2026 06:34:42 +0100 Subject: [PATCH 7/9] askrene: getroutes: avoid dial races Fail early if we see detect a suggestion that a pair of cost dials are racing each other. We are not likely to find a solution and we would just waste cycles. This usually means that our method is not able to find a solution that satisfies all given constraints. Changelog-None Signed-off-by: Lagrang3 --- plugins/askrene/child/mcf.c | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/plugins/askrene/child/mcf.c b/plugins/askrene/child/mcf.c index 1c7a472010f0..99708729442e 100644 --- a/plugins/askrene/child/mcf.c +++ b/plugins/askrene/child/mcf.c @@ -1476,6 +1476,11 @@ linear_routes(const tal_t *ctx, struct route_query *rq, double per_hop_dial = 0.001; double prob_dial = 0.988; + /* Counters to keep a record of how many times the dials have been + * updated. */ + int count_mu_updates = 0, count_delay_updates = 0, + count_hop_updates = 0; + /* An homogeneous 1% probability of failure is assumed per hop. We * don't have a proper treatment of *activation costs* on arcs right * now, but we use a simple linearization instead. The result is still @@ -1500,6 +1505,36 @@ linear_routes(const tal_t *ctx, struct route_query *rq, goto fail; } + /* If we see any indication of a race between any pair of dials + * we can't guarantee converge so we stop. */ + if (count_mu_updates > 10 && count_delay_updates > 10) { + error_message = + child_log(ctx, LOG_UNUSUAL, + "%s: can't satisfy both maxfee and " + "maxdelay constraints", + __func__); + *ecode = PAY_ROUTE_NOT_FOUND; + goto fail; + } + if (count_mu_updates > 10 && count_hop_updates > 10) { + error_message = + child_log(ctx, LOG_UNUSUAL, + "%s: can't satisfy both maxfee and " + "maxhops constraints", + __func__); + *ecode = PAY_ROUTE_NOT_FOUND; + goto fail; + } + if (count_hop_updates > 10 && count_delay_updates > 10) { + error_message = + child_log(ctx, LOG_UNUSUAL, + "%s: can't satisfy both maxhops and " + "maxdelay constraints", + __func__); + *ecode = PAY_ROUTE_NOT_FOUND; + goto fail; + } + new_flows = tal_free(new_flows); /* If the amount_to_deliver is very small we better use a single @@ -1599,6 +1634,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, /* all_fees exceed the strong budget limit, try * to fix it increasing mu. */ mu *= 2; + count_mu_updates++; renormalize_dials(&mu, &delay_dial, &per_hop_dial, &prob_dial); @@ -1646,6 +1682,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, } delay_dial *= 2; + count_delay_updates++; renormalize_dials(&mu, &delay_dial, &per_hop_dial, &prob_dial); child_log(tmpctx, LOG_INFORM, @@ -1669,6 +1706,7 @@ linear_routes(const tal_t *ctx, struct route_query *rq, } per_hop_dial *= 2; + count_hop_updates++; renormalize_dials(&mu, &delay_dial, &per_hop_dial, &prob_dial); From 925c8a13dc1f7b6ca89ede0a0dbd533be6ef0358 Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Wed, 21 Oct 2026 10:02:19 +0100 Subject: [PATCH 8/9] xpay: call askrene with maxhops Changelog-Fixed: xpay: Clamp the length of the payment routes using askrene's maxhops option aiming at producing routes that fit the data limits in the onion packet. Signed-off-by: Lagrang3 --- plugins/xpay/xpay.c | 54 ++++++++++++++ tests/test_xpay.py | 171 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) diff --git a/plugins/xpay/xpay.c b/plugins/xpay/xpay.c index 2c70458cd7d8..76799eb5c64d 100644 --- a/plugins/xpay/xpay.c +++ b/plugins/xpay/xpay.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -157,6 +158,9 @@ struct payment { /* Are we to add a shadow route? */ bool use_shadow; + + /* Maximum number of forward hops we can afford for this payment. */ + u32 onion_budget; }; /* One step in a path. */ @@ -1580,6 +1584,24 @@ static void add_cltv_shadow(struct payment *payment, } } +static u32 get_askrene_onion_budget(const struct payment *payment) +{ + if (payment->paths) { + /* For bolt12 payments we try to route through a blinded route, + * the destination is hidden behind the fake channels + * representing the blinded tail of the payment. + * Therefore in askrene's terms the onion_budget is the + * real maximum forwarding hops including the sender plus one + * more hop that accounts for the blinded path. */ + return payment->onion_budget + 1; + } else { + /* For bolt11 payments we try to hit the bare destination so the + * onion_budget is the real maximum forwarding hops including + * the sender. */ + return payment->onion_budget; + } +} + static const struct pubkey * get_askrene_destination(const struct payment *payment) { @@ -1600,6 +1622,7 @@ static struct command_result *getroutes_done(struct command *aux_cmd, size_t i; struct amount_msat needs_routing, was_routing; struct gossmap *gossmap = get_gossmap(xpay_of(payment->plugin)); + u32 askrene_onion_budget = get_askrene_onion_budget(payment); payment_log(payment, LOG_DBG, "getroutes_done: %s", payment->cmd ? "continuing" : "ignoring"); @@ -1650,6 +1673,25 @@ static struct command_result *getroutes_done(struct command *aux_cmd, json_to_msat(buf, json_get_member(buf, t, "amount_msat"), &delivers); path = json_get_member(buf, t, "path"); + if (path->type != JSMN_ARRAY) + plugin_err(aux_cmd->plugin, + "Malformed getroutes result: " + "routes[%zu].path should be an array", + i); + if(path->size > askrene_onion_budget){ + char *error = tal_fmt( + tmpctx, + "We asked for maxhops=%" PRIu32 + ", but getroutes gave us a route with %d hops", + askrene_onion_budget, path->size); + + payment_log(payment, LOG_BROKEN, "%s", error); + + payment_give_up(aux_cmd, payment, + PAY_UNSPECIFIED_ERROR, "%s", error); + return command_still_pending(aux_cmd); + } + hops = tal_arr(NULL, struct hop, path->size); json_for_each_arr(j, hoptok, path) { const char *err; @@ -1790,6 +1832,7 @@ static struct command_result *getroutes_for(struct command *aux_cmd, struct out_req *req; const struct pubkey *dst; struct amount_msat maxfee; + u32 askrene_onion_budget = get_askrene_onion_budget(payment); /* I would normally assert here, but we have reports of this happening... */ if (amount_msat_is_zero(deliver)) { @@ -1851,6 +1894,13 @@ static struct command_result *getroutes_for(struct command *aux_cmd, : fmt_pubkey(tmpctx, &payment->destination)); json_add_amount_msat(req->js, "amount_msat", deliver); json_array_start(req->js, "layers"); + + /* Sending maxhops to askrene through a layer name hack. Only for 26.06 + * backport, don't do try at home (master I mean)! */ + json_add_string( + req->js, NULL, + tal_fmt(tmpctx, "maxhops=%" PRIu32, askrene_onion_budget)); + /* Add local channels */ json_add_string(req->js, NULL, "auto.localchans"); /* For the MCF computation we must discard the cost of routing through @@ -2564,6 +2614,9 @@ static struct payment *new_payment(const tal_t *ctx, payment->unique_id = xpay->counter++; payment->private_layer = tal_fmt(payment, "xpay-%"PRIu64, payment->unique_id); + /* If we don't compute the onion budget ahead this is still a reference + * value to keep in mind. */ + payment->onion_budget = ROUTING_MAX_HOPS; list_add_tail(&xpay->payments, &payment->list); tal_add_destructor(payment, destroy_payment); @@ -3119,6 +3172,7 @@ static struct command_result *check_onion_budget(struct command *cmd, } payment_log(payment, LOG_DBG, "onion budget is %" PRIu32 ".", max_hops); + payment->onion_budget = max_hops; dst = get_askrene_destination(payment); /* This is bolt11 payment with 0 hops and we are not the destination. */ diff --git a/tests/test_xpay.py b/tests/test_xpay.py index d0e2abe63e5b..c73af628c20d 100644 --- a/tests/test_xpay.py +++ b/tests/test_xpay.py @@ -1486,3 +1486,174 @@ def test_sendamount_bip353(node_factory): ret = l2.rpc.sendamount("fake@fake.com", "100sat") assert ret["successful_parts"] == 1 assert ret["amount_sent_msat"] == 100000 + + +# l2's private key (node #2 always gets this hsm_secret, see test_pay_bolt11_metadata) +L2_PRIVKEY = "0c633a7c17c701a0980158f5483035e01fa8bd091b47fadf2e86e589a9f93fca" + + +def bolt11_with_metadata(node, chainparams, inv, metadata): + """Re-encode inv (made by l2) with payment metadata attached.""" + dec = node.rpc.decode(inv["bolt11"]) + return ( + subprocess.check_output( + [ + "devtools/bolt11-cli", + "encode", + L2_PRIVKEY, + f"currency={chainparams['bip173_prefix']}", + f"p={inv['payment_hash']}", + f"s={inv['payment_secret']}", + f"d={dec['description']}", + f"amount={dec['amount_msat']}msat", + f"x={dec['expiry']}", + f"c={dec['min_final_cltv_expiry']}", + f"9={dec['features']}", + "m=" + metadata.hex(), + ] + ) + .decode("utf-8") + .strip() + ) + + +@unittest.skipIf(TEST_NETWORK != "regtest", "Invoice is network specific") +def test_xpay_onion_budget_bolt11_metadata(node_factory, chainparams): + """Large payment_metadata is carried intact, and reduces the hop budget. + + BOLT11 limits a field to 1023 5-bit units, so metadata can be at most + 639 bytes: with that, there's still room for 11 forwarding payloads.""" + plugin = os.path.join( + os.path.dirname(__file__), "plugins/accept_payment_metadata.py" + ) + l1, l2, l3 = node_factory.get_nodes(3, opts=[{}, {"plugin": plugin}, {}]) + node_factory.join_nodes([l1, l3, l2], wait_for_announce=True) + + metadata = bytes([7]) * 639 + inv = l2.rpc.invoice(10000, "big", "big", preimage="02" * 32) + b11 = bolt11_with_metadata(l1, chainparams, inv, metadata) + assert l1.rpc.decode(b11)["payment_metadata"] == metadata.hex() + l2.rpc.setmetadatapreimage(preimage="02" * 32, metadata=metadata.hex()) + + ret = l1.rpc.xpay(b11) + assert ret["payment_preimage"] == "02" * 32 + assert ret["failed_parts"] == 0 + l1.daemon.wait_for_log(r"onion budget is 11\.") + + +@unittest.skipIf(TEST_NETWORK != "regtest", "Invoice is network specific") +def test_xpay_onion_budget_keysend(node_factory): + """xkeysend with large extratlvs is limited by the onion budget too.""" + l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True) + + # Fits through l2. + l1.rpc.xkeysend(l3.info["id"], 1000, extratlvs={133773310: "00" * 900}) + + # Way too big. + with pytest.raises( + RpcError, match=r"The final payload is too big to fit an onion packet" + ): + l1.rpc.xkeysend(l3.info["id"], 1000, extratlvs={133773310: "00" * 1300}) + + # Fits only with zero forwarding hops, but we're not the destination. + with pytest.raises( + RpcError, match=r"onion budget is 0 and we are not the destination node" + ): + l1.rpc.xkeysend(l3.info["id"], 1000, extratlvs={133773310: "00" * 1180}) + + +@pytest.mark.slow_test +@unittest.skipIf(TEST_NETWORK != "regtest", "Invoice is network specific") +def test_xpay_maxhops_bolt11(node_factory, chainparams): + """A bolt11 final payload that no longer fits the cheap route forces the short one. + + The cheap route is longer than a 639-byte payment_metadata onion budget, + and still fits a plain bolt11. The other route is 2 hops and much more + expensive, so it is used only when the long one does not fit. + + The expensive hop is not the first one. xpay applies auto.sourcefree, so + a direct channel would look free and always win. + """ + plugin = os.path.join( + os.path.dirname(__file__), "plugins/accept_payment_metadata.py" + ) + # payee is node #2, so bolt11-cli can sign with L2_PRIVKEY. + l1, payee = node_factory.get_nodes(2, opts=[{}, {"plugin": plugin}]) + # 12 hops: one more than the budget left by a max-size bolt11 metadata field. + mids = node_factory.get_nodes(11) + line = [l1] + mids + [payee] + long_hops = len(line) - 1 + short_hops = 2 + node_factory.join_nodes(line, wait_for_announce=True) + # Expensive shortcut: l1 -> mids[0] -> payee. + node_factory.join_nodes([mids[0], payee], wait_for_announce=True) + + cheap_feeppm = 1 + # High enough that the per-hop success penalty does not prefer these 2 hops + # over the cheap 12, but still inside maxfee. + pricey_feeppm = 500_000 + for a, b in zip(line, line[1:]): + a.rpc.setchannel( + first_scid(a, b), feebase=0, feeppm=cheap_feeppm, htlcmin=1, enforcedelay=0 + ) + mids[0].rpc.setchannel( + first_scid(mids[0], payee), + feebase=0, + feeppm=pricey_feeppm, + htlcmin=1, + enforcedelay=0, + ) + + def fees_visible(): + chans = l1.rpc.listchannels()["channels"] + cheap = sum( + 1 + for c in chans + if c["fee_per_millionth"] == cheap_feeppm + and c["base_fee_millisatoshi"] == 0 + ) + pricey = sum(1 for c in chans if c["fee_per_millionth"] == pricey_feeppm) + return cheap >= long_hops - 1 and pricey >= 1 + + wait_for(fees_visible) + + # Last hop of the long route. The expensive shortcut never reaches it. + def long_forwards(): + return len(mids[-1].rpc.listforwards(status="settled")["forwards"]) + + def budget_of(line): + return int(re.search(r"onion budget is ([0-9]+)\.", line).group(1)) + + # payment_log prints "N: onion budget is X." A second copy of the same + # line is also forwarded as a notify, so match the numbered one only. + + amount = 100_000 + + def pay(label, preimage, metadata=None): + inv = payee.rpc.invoice(amount, label, label, preimage=preimage) + if metadata is None: + bolt = inv["bolt11"] + else: + payee.rpc.setmetadatapreimage(preimage=preimage, metadata=metadata.hex()) + bolt = bolt11_with_metadata(l1, chainparams, inv, metadata) + return l1.rpc.xpay(bolt, maxfee="80000msat") + + ret = pay("plain", "11" * 32) + assert ret["payment_preimage"] == "11" * 32 + assert ret["failed_parts"] == 0 + plain_budget = budget_of(l1.daemon.wait_for_log(r"\d+: onion budget is [0-9]+\.")) + assert plain_budget >= long_hops + # Long cheap route: the last mid forwarded, and the payer paid almost no fee. + assert long_forwards() == 1 + assert ret["amount_sent_msat"] - amount < 1000 + + # Bolt-11 field limit: 1023 5-bit units, so metadata is at most 639 bytes. + metadata = bytes([7]) * 639 + ret = pay("big", "22" * 32, metadata) + assert ret["payment_preimage"] == "22" * 32 + assert ret["failed_parts"] == 0 + meta_budget = budget_of(l1.daemon.wait_for_log(r"\d+: onion budget is [0-9]+\.")) + assert short_hops <= meta_budget < long_hops + # Expensive short route: the last mid saw nothing more, and the payer paid the high fee. + assert long_forwards() == 1 + assert ret["amount_sent_msat"] - amount > 1000 From d5d2044e0cff3e0c4b54c71f67dceabc42d980eb Mon Sep 17 00:00:00 2001 From: Lagrang3 Date: Thu, 22 Oct 2026 18:55:44 +0100 Subject: [PATCH 9/9] xpay: better error message when create_onion fails Signed-off-by: Lagrang3 --- plugins/xpay/xpay.c | 67 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/plugins/xpay/xpay.c b/plugins/xpay/xpay.c index 76799eb5c64d..2ee73b0404f3 100644 --- a/plugins/xpay/xpay.c +++ b/plugins/xpay/xpay.c @@ -1368,17 +1368,32 @@ static void append_blinded_payloads(struct sphinx_path *sp, } } -static const u8 *create_onion(const tal_t *ctx, - struct attempt *attempt, - u32 effective_bheight) +struct xpay_onion { + /* Serialized onion. */ + const u8 *raw_onion; + + /* Number of forwarding hops before the tail including the sender. */ + u32 num_forwards; + + /* Bytes (payloads + HMACs) used by forward hops. */ + size_t forward_bytes; + + /* Bytes in the final payload plus blinded hops if present. */ + size_t tail_bytes; +}; + +static struct xpay_onion *create_onion(const tal_t *ctx, + struct attempt *attempt, + u32 effective_bheight) { struct xpay *xpay = xpay_of(attempt->payment->plugin); bool blinded_path = false; struct onionpacket *packet; struct sphinx_path *sp; - const u8 *payload, *ret; + const u8 *payload; const struct pubkey *node; const struct amount_msat mpp_amount = attempt_mpp_amount(attempt); + struct xpay_onion *xpay_onion = talz(ctx, struct xpay_onion); sp = sphinx_path_new(ctx, attempt->payment->payment_hash.u.u8, sizeof(attempt->payment->payment_hash.u.u8)); @@ -1392,6 +1407,7 @@ static const u8 *create_onion(const tal_t *ctx, if (pubkey_eq(&hop->next_node, &xpay->fakenode) && hop->scidd.scid.u64 < tal_count(attempt->payment->paths)) { blinded_path = true; + xpay_onion->forward_bytes = sphinx_path_payloads_size(sp); append_blinded_payloads(sp, attempt, effective_bheight, hop->scidd.scid.u64); /* This must be at the end, unless they put the fake nodeid @@ -1405,12 +1421,14 @@ static const u8 *create_onion(const tal_t *ctx, hop->amount_out, hop->cltv_value_out + effective_bheight); sphinx_add_hop_has_length(sp, node, take(payload)); + xpay_onion->num_forwards++; node = &hop->next_node; } /* If we use a blinded path, final has to be special, so * that's done in append_blinded_payloads. */ if (!blinded_path) { + xpay_onion->forward_bytes = sphinx_path_payloads_size(sp); u8 *final = onion_final_payload(tmpctx, attempt_deliver(attempt), attempt->payment->final_cltv + effective_bheight, @@ -1420,32 +1438,51 @@ static const u8 *create_onion(const tal_t *ctx, attempt->payment->extra_tlvs); sphinx_add_hop_has_length(sp, node, take(final)); } + xpay_onion->tail_bytes = + sphinx_path_payloads_size(sp) - xpay_onion->forward_bytes; /* Fails if would be too long */ packet = create_onionpacket(attempt, sp, ROUTING_INFO_SIZE, &attempt->shared_secrets); - if (!packet) - return NULL; + if (packet) + xpay_onion->raw_onion = + serialize_onionpacket(xpay_onion, packet); - ret = serialize_onionpacket(ctx, packet); + tal_free(sp); tal_free(packet); - return ret; + return xpay_onion; } static struct command_result *do_inject(struct command *aux_cmd, struct attempt *attempt) { struct out_req *req; - const u8 *onion; + const struct xpay_onion *xpay_onion; struct xpay *xpay = xpay_of(attempt->payment->plugin); /* In case a block comes in, we give CLTVs an extra 1. */ u32 effective_bheight = xpay->blockheight + 1; - onion = create_onion(tmpctx, attempt, effective_bheight); - /* FIXME: Handle this better! */ - if (!onion) { - payment_give_up(aux_cmd, attempt->payment, PAY_UNSPECIFIED_ERROR, - "Could not create payment onion: path too long!"); + xpay_onion = create_onion(tmpctx, attempt, effective_bheight); + if (!xpay_onion->raw_onion) { + char *error; + size_t total = + xpay_onion->forward_bytes + xpay_onion->tail_bytes; + if (total > ROUTING_INFO_SIZE) + error = tal_fmt( + tmpctx, + "%zu bytes needed (%" PRIu32 + " hops using %zu bytes, plus %zu bytes for the " + "final payload or blinded path)", + total, xpay_onion->num_forwards, + xpay_onion->forward_bytes, xpay_onion->tail_bytes); + else + error = tal_fmt(tmpctx, "failed to derive shared secrets"); + + payment_log(attempt->payment, LOG_BROKEN, + "create_onion failed: %s", error); + payment_give_up(aux_cmd, attempt->payment, + PAY_UNSPECIFIED_ERROR, + "Could not create payment onion: %s", error); return command_still_pending(aux_cmd); } @@ -1457,7 +1494,7 @@ static struct command_result *do_inject(struct command *aux_cmd, injectpaymentonion_succeeded, injectpaymentonion_failed, attempt); - json_add_hex_talarr(req->js, "onion", onion); + json_add_hex_talarr(req->js, "onion", xpay_onion->raw_onion); json_add_sha256(req->js, "payment_hash", &attempt->payment->payment_hash); /* If no route, its the same as delivery (self-pay) */ json_add_amount_msat(req->js, "amount_msat", inject_amount(attempt));