Skip to content

Gossmap: compaction support #8869

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rustyrussell wants to merge 21 commits into
base: master
Choose a base branch
from
Open

Gossmap: compaction support #8869

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
3f5a607
devtools: enhance dump-gossipstore to show some details of messages.
rustyrussell Feb 5, 2026
350a401
pytest: fix bogus test_gossip_store_compact_noappend test.
rustyrussell Feb 5, 2026
c00bd97
devtools/gossmap-compress: generate better scids.
rustyrussell Feb 5, 2026
b2312ce
gossmap: reduce load times by 20%
rustyrussell Feb 5, 2026
87fb544
tools: delete gossip_store of needed for downgrade even if db hasn't …
rustyrussell Feb 5, 2026
1208ba4
gossip_store: add UUID entry at front of the store.
rustyrussell Feb 5, 2026
5d29a57
common/gossmap: use the UUID record on reopen.
rustyrussell Feb 5, 2026
f63df60
gossipd: write uuid record on startup.
rustyrussell Feb 5, 2026
93ea855
common: move gossip_store_wire.csv into common/ from gossipd/
rustyrussell Feb 5, 2026
a2af73b
gossmap: add callback for gossipd to see dying messages.
rustyrussell Feb 5, 2026
27ad646
gossmap: keep stats on live/deleted records.
rustyrussell Feb 5, 2026
5673e16
gossipd: use gossmap to load the dying entries.
rustyrussell Feb 5, 2026
c24e582
gossipd: don't gather dying channels during compaction.
rustyrussell Feb 5, 2026
28a7ae2
gossipd: don't compact on startup.
rustyrussell Feb 5, 2026
d33bc9b
devtools/gossmap-compress: create latest gossip_store version
rustyrussell Feb 5, 2026
f0c0970
gossipd: put the last_writes array inside struct gossip_store.
rustyrussell Feb 5, 2026
33641ed
common: expose gossip_store "header and type" single-read struct.
rustyrussell Feb 5, 2026
632c99a
gossipd: lightningd/lightning_gossip_compactd
rustyrussell Feb 5, 2026
077c889
gossipd: code to invoke compactd and reopen store.
rustyrussell Feb 5, 2026
a511469
gossipd: compact when gossip store is 80% deleted records.
rustyrussell Feb 5, 2026
5ad16e1
gossipd: dev-compact-gossip-store to manually invoke compaction.
rustyrussell Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,7 @@ PKGLIBEXEC_PROGRAMS = \
lightningd/lightning_connectd \
lightningd/lightning_dualopend \
lightningd/lightning_gossipd \
lightningd/lightning_gossip_compactd \
lightningd/lightning_hsmd \
lightningd/lightning_onchaind \
lightningd/lightning_openingd \
Expand All @@ -489,9 +490,14 @@ mkdocs.yml: $(MANPAGES:=.md)
# Every single object file.
ALL_OBJS := $(ALL_C_SOURCES:.c=.o)

WIREGEN_FILES := $(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(ALL_C_HEADERS) $(ALL_C_SOURCES))

# Always make wiregen files before any object file
$(ALL_OBJS): $(WIREGEN_FILES)

# We always regen wiregen and printgen files, since SHA256STAMP protects against
# spurious rebuilds.
$(filter %printgen.h %printgen.c %wiregen.h %wiregen.c, $(ALL_C_HEADERS) $(ALL_C_SOURCES)): $(FORCE)
$(WIREGEN_FILES): $(FORCE)

ifneq ($(TEST_GROUP_COUNT),)
PYTEST_OPTS += --test-group=$(TEST_GROUP) --test-group-count=$(TEST_GROUP_COUNT)
Expand Down
6 changes: 2 additions & 4 deletions common/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ COMMON_SRC_NOGEN := \
common/wire_error.c


COMMON_SRC_GEN := common/status_wiregen.c common/peer_status_wiregen.c common/scb_wiregen.c
COMMON_SRC_GEN := common/status_wiregen.c common/peer_status_wiregen.c common/scb_wiregen.c common/gossip_store_wiregen.c

COMMON_HEADERS_NOGEN := $(COMMON_SRC_NOGEN:.c=.h) \
common/closing_fee.h \
Expand All @@ -124,7 +124,7 @@ COMMON_HEADERS_NOGEN := $(COMMON_SRC_NOGEN:.c=.h) \
common/jsonrpc_errors.h \
common/overflows.h

COMMON_HEADERS_GEN := common/htlc_state_names_gen.h common/status_wiregen.h common/peer_status_wiregen.h common/scb_wiregen.h
COMMON_HEADERS_GEN := common/htlc_state_names_gen.h common/status_wiregen.h common/peer_status_wiregen.h common/scb_wiregen.h common/gossip_store_wiregen.h

COMMON_HEADERS := $(COMMON_HEADERS_GEN) $(COMMON_HEADERS_NOGEN)
COMMON_SRC := $(COMMON_SRC_NOGEN) $(COMMON_SRC_GEN)
Expand Down Expand Up @@ -157,8 +157,6 @@ ALL_C_SOURCES += $(COMMON_SRC)
common/htlc_state_names_gen.h: common/htlc_state.h ccan/ccan/cdump/tools/cdump-enumstr
ccan/ccan/cdump/tools/cdump-enumstr common/htlc_state.h > $@

common/gossip_store.o: gossipd/gossip_store_wiregen.h

check-source-bolt: $(COMMON_SRC_NOGEN:%=bolt-check/%) $(COMMON_HEADERS:%=bolt-check/%)
check-whitespace: $(COMMON_SRC_NOGEN:%=check-whitespace/%) $(COMMON_HEADERS:%=check-whitespace/%)

Expand Down
16 changes: 4 additions & 12 deletions common/gossip_store.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,19 @@
#include "config.h"
#include <common/gossip_store.h>
#include <gossipd/gossip_store_wiregen.h>
#include <common/gossip_store_wiregen.h>
#include <unistd.h>

/* We cheat and read first two bytes of message too. */
struct hdr_and_type {
struct gossip_hdr hdr;
be16 type;
};
/* Beware padding! */
#define HDR_AND_TYPE_SIZE (sizeof(struct gossip_hdr) + sizeof(u16))

bool gossip_store_readhdr(int gossip_store_fd, size_t off,
size_t *len,
u32 *timestamp,
u16 *flags,
u16 *type)
{
struct hdr_and_type buf;
struct gossip_hdr_and_type buf;
int r;

r = pread(gossip_store_fd, &buf, HDR_AND_TYPE_SIZE, off);
if (r != HDR_AND_TYPE_SIZE)
r = pread(gossip_store_fd, &buf, GOSSIP_HDR_AND_TYPE_SIZE, off);
if (r != GOSSIP_HDR_AND_TYPE_SIZE)
return false;
if (!(buf.hdr.flags & CPU_TO_BE16(GOSSIP_STORE_COMPLETED_BIT)))
return false;
Expand Down
17 changes: 11 additions & 6 deletions common/gossip_store.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ struct gossip_rcvd_filter;
/* First byte of file is the version.
*
* Top three bits mean incompatible change.
* As of this writing, major == 0, minor == 15.
* As of this writing, major == 0, minor == 16.
*/
#define GOSSIP_STORE_MAJOR_VERSION_MASK 0xE0
#define GOSSIP_STORE_MINOR_VERSION_MASK 0x1F
Expand Down Expand Up @@ -50,6 +50,14 @@ struct gossip_hdr {
beint32_t timestamp; /* timestamp of msg. */
};

/* Useful to read gossip_hdr and type of msg. */
struct gossip_hdr_and_type {
struct gossip_hdr hdr;
be16 type;
};
/* Beware padding! */
#define GOSSIP_HDR_AND_TYPE_SIZE (sizeof(struct gossip_hdr) + sizeof(u16))

/**
* Direct store accessor: read gossip msg hdr from store.
* @gossip_store_fd: the readable file descriptor
Expand All @@ -59,11 +67,8 @@ struct gossip_hdr {
* @flags (out): if non-NULL, set to the flags.
* @type (out): if non-NULL, set to the msg type.
*
* Returns false if there are no more gossip msgs. If you
* want to read the message, use gossip_store_next, if you
* want to skip, simply add sizeof(gossip_hdr) + *len to *off.
* Note: it's possible that entire record isn't there yet,
* so gossip_store_next can fail.
* Returns false if there are no more gossip msgs, or message
* is incomplete. To iterate, simply add sizeof(gossip_hdr) + *len to off.
*/
bool gossip_store_readhdr(int gossip_store_fd, size_t off,
size_t *len,
Expand Down
4 changes: 4 additions & 0 deletions gossipd/gossip_store_wire.csv → common/gossip_store_wire.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@ msgdata,gossip_store_delete_chan,scid,short_channel_id,

msgtype,gossip_store_ended,4105
msgdata,gossip_store_ended,equivalent_offset,u64,
msgdata,gossip_store_ended,uuid,u8,32

msgtype,gossip_store_chan_dying,4106
msgdata,gossip_store_chan_dying,scid,short_channel_id,
msgdata,gossip_store_chan_dying,blockheight,u32,

msgtype,gossip_store_uuid,4107
msgdata,gossip_store_uuid,uuid,u8,32
Loading
Loading