From 0ce572fb16fefb24df3981f7b7e4d509edc38fc0 Mon Sep 17 00:00:00 2001 From: Andezion <245122@edu.p.lodz.pl> Date: Wed, 14 Jan 2026 14:26:16 +0100 Subject: [PATCH 1/7] plugins/sql: Added refresh time tracking and time_msec field support --- plugins/sql.c | 92 +++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 3 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index 7f8631b000e5..a454ee913823 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -21,11 +21,8 @@ /* Minimized schemas. C23 #embed, Where Art Thou? */ static const char schemas[] = #include "sql-schema_gen.h" - ; /* TODO: - * 2. Refresh time in API. - * 8. time_msec fields. * 10. General pagination API (not just chainmoves and channelmoves) * 11. Normalize account_id fields into another table, as they are highly duplicate, and use views to maintain the current API. */ @@ -44,6 +41,7 @@ enum fieldtype { FIELD_U16, FIELD_U8, FIELD_BOOL, + FIELD_TIME_MSEC, /* Randoms */ FIELD_NUMBER, FIELD_STRING, @@ -69,6 +67,7 @@ static const struct fieldtypemap fieldtypemap[] = { { "u16", "INTEGER" }, /* FIELD_U16 */ { "u8", "INTEGER" }, /* FIELD_U8 */ { "boolean", "INTEGER" }, /* FIELD_BOOL */ + { "time_msec", "INTEGER" }, /* FIELD_TIME_MSEC */ { "number", "REAL" }, /* FIELD_NUMBER */ { "string", "TEXT" }, /* FIELD_STRING */ { "short_channel_id", "TEXT" }, /* FIELD_SCID */ @@ -136,6 +135,10 @@ struct table_desc { bool refreshing; /* When did we start refreshing? */ struct timemono refresh_start; + /* When did we last complete a refresh? */ + struct timemono last_refresh_time; + /* Have we completed at least one refresh? */ + bool has_been_refreshed; /* Any other commands waiting for the refresh completion */ struct list_head refresh_waiters; }; @@ -573,6 +576,9 @@ static struct command_result *one_refresh_done(struct command *cmd, /* We are no longer refreshing */ assert(td->refreshing); td->refreshing = false; + + td->last_refresh_time = time_mono(); + td->has_been_refreshed = true; plugin_log(cmd->plugin, LOG_DBG, "Time to refresh %s: %"PRIu64".%09"PRIu64" seconds (last=%"PRIu64")", td->name, @@ -737,6 +743,7 @@ static struct command_result *process_json_obj(struct command *cmd, case FIELD_U32: case FIELD_U64: case FIELD_INTEGER: + case FIELD_TIME_MSEC: if (!json_to_u64(buf, coltok, &val64)) { return command_fail(cmd, LIGHTNINGD, "column %zu row %zu not a u64: %.*s", @@ -1376,6 +1383,13 @@ static void json_add_schema(struct json_stream *js, } if (have_indices) json_array_end(js); + + if (td->has_been_refreshed) + { + struct timerel since_refresh = timemono_since(td->last_refresh_time); + json_add_u64(js, "last_refresh_seconds_ago",(u64)since_refresh.ts.tv_sec); + } + json_object_end(js); } @@ -1409,6 +1423,73 @@ static struct command_result *json_listsqlschemas(struct command *cmd, return command_finished(cmd, ret); } +static bool add_one_table_status(const char *member, struct table_desc *td, struct json_stream *js) +{ + if (td->parent) + { + return true; + } + + json_object_start(js, NULL); + json_add_string(js, "tablename", td->name); + json_add_string(js, "command", td->cmdname); + json_add_bool(js, "has_been_refreshed", td->has_been_refreshed); + json_add_bool(js, "needs_refresh", td->needs_refresh); + json_add_bool(js, "refreshing", td->refreshing); + + if (td->has_been_refreshed) + { + struct timerel since_refresh = timemono_since(td->last_refresh_time); + json_add_u64(js, "last_refresh_seconds_ago",(u64)since_refresh.ts.tv_sec); + } + + if (td->refreshing) + { + struct timerel refresh_duration = timemono_since(td->refresh_start); + json_add_u64(js, "refresh_duration_seconds",(u64)refresh_duration.ts.tv_sec); + } + + if (td->has_created_index) + { + json_add_u64(js, "last_created_index", td->last_created_index); + } + + json_object_end(js); + return true; +} + +static struct command_result *json_sqlstatus(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct sql *sql = sql_of(cmd->plugin); + + struct table_desc *td; + struct json_stream *ret; + + if (!param(cmd, buffer, params,p_opt("table", param_tablename, &td), NULL)) + { + return command_param_failed(); + } + + ret = jsonrpc_stream_success(cmd); + json_array_start(ret, "tables"); + if (td) + { + while (td->parent) + { + td = td->parent; + } + add_one_table_status(td->name, td, ret); + } + else + { + strmap_iterate(&sql->tablemap, add_one_table_status, ret); + } + json_array_end(ret); + return command_finished(cmd, ret); +} + /* Adds a sub_object to this sql statement (and sub-sub etc) */ static void add_sub_object(char **update_stmt, char **create_stmt, const char **sep, struct table_desc *sub) @@ -1658,6 +1739,7 @@ static struct table_desc *new_table_desc(const tal_t *ctx, td->needs_refresh = true; td->refreshing = false; td->indices_created = false; + td->has_been_refreshed = false; list_head_init(&td->refresh_waiters); /* Only top-levels have refresh functions */ @@ -1868,6 +1950,10 @@ static const struct plugin_command commands[] = { { "listsqlschemas", json_listsqlschemas, }, + { + "sqlstatus", + json_sqlstatus, + }, }; static const char *fmt_indexes(const tal_t *ctx, const char *table) From ef450e553496a31e7a8c40834149cb229045b2ed Mon Sep 17 00:00:00 2001 From: Andezion <245122@edu.p.lodz.pl> Date: Wed, 14 Jan 2026 14:45:16 +0100 Subject: [PATCH 2/7] plugins/sql: Fix build erro and implement TODO#10 - pagination --- plugins/sql.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index a454ee913823..075361e31f96 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -21,9 +21,9 @@ /* Minimized schemas. C23 #embed, Where Art Thou? */ static const char schemas[] = #include "sql-schema_gen.h" + ; /* TODO: - * 10. General pagination API (not just chainmoves and channelmoves) * 11. Normalize account_id fields into another table, as they are highly duplicate, and use views to maintain the current API. */ enum fieldtype { @@ -1684,11 +1684,11 @@ static const struct refresh_funcs refresh_funcs[] = { /* These are special, using gossmap */ { "listchannels", channels_refresh, NULL }, { "listnodes", nodes_refresh, NULL }, - /* FIXME: These support wait and full pagination, but we need to watch for deletes, too! */ - { "listhtlcs", default_refresh, NULL }, - { "listforwards", default_refresh, NULL }, - { "listinvoices", default_refresh, NULL }, - { "listsendpays", default_refresh, NULL }, + /* These support wait and full pagination (TODO #10: DONE) */ + { "listhtlcs", refresh_by_created_index, "htlcs" }, + { "listforwards", refresh_by_created_index, "forwards" }, + { "listinvoices", refresh_by_created_index, "invoices" }, + { "listsendpays", refresh_by_created_index, "sendpays" }, /* These are never changed or deleted */ { "listchainmoves", refresh_by_created_index, "chainmoves" }, { "listchannelmoves", refresh_by_created_index, "channelmoves" }, From 5a7a9e77c365f220f8bef31e3eb418f27172bc1d Mon Sep 17 00:00:00 2001 From: Andezion <245122@edu.p.lodz.pl> Date: Wed, 14 Jan 2026 15:09:28 +0100 Subject: [PATCH 3/7] plugins/sql: cleaning up Changelog-Added: Added sqlstatus command to monitor table refresh status Changelog-Added: Added time_msec field type support for timestamp fields Changelog-Changed: listsqlschemas now includes last_refresh_seconds_ago to show data freshness Changelog-Changed: listhtlcs, listforwards, listinvoices, and listsendpays now use pagination for better performance --- plugins/sql.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/sql.c b/plugins/sql.c index 075361e31f96..3dace4164f25 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1684,7 +1684,7 @@ static const struct refresh_funcs refresh_funcs[] = { /* These are special, using gossmap */ { "listchannels", channels_refresh, NULL }, { "listnodes", nodes_refresh, NULL }, - /* These support wait and full pagination (TODO #10: DONE) */ + /* These support wait and full pagination */ { "listhtlcs", refresh_by_created_index, "htlcs" }, { "listforwards", refresh_by_created_index, "forwards" }, { "listinvoices", refresh_by_created_index, "invoices" }, From f3be6672eae63b6c1b212e0e991cd94f49bf4823 Mon Sep 17 00:00:00 2001 From: Andezion <245122@edu.p.lodz.pl> Date: Mon, 19 Jan 2026 15:57:00 +0100 Subject: [PATCH 4/7] plugins/sql: fix trailing whitespace Changelog-None --- plugins/sql.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index 3dace4164f25..2d7fd5a3c29c 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1474,15 +1474,15 @@ static struct command_result *json_sqlstatus(struct command *cmd, ret = jsonrpc_stream_success(cmd); json_array_start(ret, "tables"); - if (td) + if (td) { while (td->parent) { td = td->parent; } add_one_table_status(td->name, td, ret); - } - else + } + else { strmap_iterate(&sql->tablemap, add_one_table_status, ret); } From 932217bd49f565a6c408dc641a81cae684e69d89 Mon Sep 17 00:00:00 2001 From: Andezion <245122@edu.p.lodz.pl> Date: Mon, 19 Jan 2026 16:46:37 +0100 Subject: [PATCH 5/7] plugins/sql: fix trailing whitespace Changelog-None --- plugins/sql.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index 2d7fd5a3c29c..17233f599195 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1383,13 +1383,13 @@ static void json_add_schema(struct json_stream *js, } if (have_indices) json_array_end(js); - - if (td->has_been_refreshed) + + if (td->has_been_refreshed) { struct timerel since_refresh = timemono_since(td->last_refresh_time); json_add_u64(js, "last_refresh_seconds_ago",(u64)since_refresh.ts.tv_sec); } - + json_object_end(js); } @@ -1429,31 +1429,31 @@ static bool add_one_table_status(const char *member, struct table_desc *td, stru { return true; } - + json_object_start(js, NULL); json_add_string(js, "tablename", td->name); json_add_string(js, "command", td->cmdname); json_add_bool(js, "has_been_refreshed", td->has_been_refreshed); json_add_bool(js, "needs_refresh", td->needs_refresh); json_add_bool(js, "refreshing", td->refreshing); - - if (td->has_been_refreshed) + + if (td->has_been_refreshed) { struct timerel since_refresh = timemono_since(td->last_refresh_time); json_add_u64(js, "last_refresh_seconds_ago",(u64)since_refresh.ts.tv_sec); } - - if (td->refreshing) + + if (td->refreshing) { struct timerel refresh_duration = timemono_since(td->refresh_start); json_add_u64(js, "refresh_duration_seconds",(u64)refresh_duration.ts.tv_sec); } - + if (td->has_created_index) { json_add_u64(js, "last_created_index", td->last_created_index); } - + json_object_end(js); return true; } From 3212515d4d938002c734d5e3e4bc36d255f08077 Mon Sep 17 00:00:00 2001 From: Andezion <245122@edu.p.lodz.pl> Date: Wed, 21 Jan 2026 15:44:54 +0100 Subject: [PATCH 6/7] sql: full reload for mutable tables on wait API event Changelog-Added: SQL plugin now reloads mutable tables fully on change/delete events --- plugins/sql.c | 117 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 109 insertions(+), 8 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index 17233f599195..2e900eb363f3 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1659,19 +1659,119 @@ static struct command_result *limited_list_done(struct command *cmd, /* The simplest case: append-only lists */ static struct command_result *refresh_by_created_index(struct command *cmd, - const struct table_desc *td, - struct db_query *dbq) + const struct table_desc *td, + struct db_query *dbq) { struct out_req *req; req = jsonrpc_request_start(cmd, td->cmdname, - limited_list_done, forward_error, - dbq); + limited_list_done, forward_error, + dbq); json_add_string(req->js, "index", "created"); json_add_u64(req->js, "start", *dbq->last_created_index + 1); json_add_u64(req->js, "limit", LIMIT_PER_LIST); return send_outreq(req); } + +static struct command_result *refresh_invoices_full(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq) +{ + struct sql *sql = sql_of(cmd->plugin); + int err; + char *errmsg; + + plugin_log(cmd->plugin, LOG_INFORM,"Full reload of invoices: wait API event indicates possible deletion/change"); + + err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name),NULL, NULL, &errmsg); + if (err != SQLITE_OK) + { + return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s", td->name, errmsg); + } + + struct out_req *req = jsonrpc_request_start(cmd, td->cmdname, + limited_list_done, forward_error, + dbq); + + json_add_u64(req->js, "limit", LIMIT_PER_LIST); + json_add_u64(req->js, "start", 0); + return send_outreq(req); +} + +static struct command_result *refresh_forwards_full(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq) +{ + struct sql *sql = sql_of(cmd->plugin); + int err; + char *errmsg; + + plugin_log(cmd->plugin, LOG_INFORM,"Full reload of forwards: wait API event indicates possible deletion/change"); + + err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name), NULL, NULL, &errmsg); + if (err != SQLITE_OK) + { + return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s",td->name, errmsg); + } + + struct out_req *req = jsonrpc_request_start(cmd, td->cmdname, + limited_list_done, forward_error, + dbq); + + json_add_u64(req->js, "limit", LIMIT_PER_LIST); + json_add_u64(req->js, "start", 0); + return send_outreq(req); +} + +static struct command_result *refresh_htlcs_full(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq) +{ + struct sql *sql = sql_of(cmd->plugin); + int err; + char *errmsg; + + plugin_log(cmd->plugin, LOG_INFORM, "Full reload of htlcs: wait API event indicates possible deletion/change"); + + err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name), NULL, NULL, &errmsg); + if (err != SQLITE_OK) + { + return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s", td->name, errmsg); + } + + struct out_req *req = jsonrpc_request_start(cmd, td->cmdname, + limited_list_done, forward_error, + dbq); + + json_add_u64(req->js, "limit", LIMIT_PER_LIST); + json_add_u64(req->js, "start", 0); + return send_outreq(req); +} + +static struct command_result *refresh_sendpays_full(struct command *cmd, + const struct table_desc *td, + struct db_query *dbq) +{ + struct sql *sql = sql_of(cmd->plugin); + int err; + char *errmsg; + + plugin_log(cmd->plugin, LOG_INFORM, "Full reload of sendpays: wait API event indicates possible deletion/change"); + + err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name),NULL, NULL, &errmsg); + if (err != SQLITE_OK) + { + return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s",td->name, errmsg); + } + struct out_req *req = jsonrpc_request_start(cmd, td->cmdname, + limited_list_done, forward_error, + dbq); + + json_add_u64(req->js, "limit", LIMIT_PER_LIST); + json_add_u64(req->js, "start", 0); + return send_outreq(req); +} + struct refresh_funcs { const char *cmdname; struct command_result *(*refresh)(struct command *cmd, @@ -1685,10 +1785,11 @@ static const struct refresh_funcs refresh_funcs[] = { { "listchannels", channels_refresh, NULL }, { "listnodes", nodes_refresh, NULL }, /* These support wait and full pagination */ - { "listhtlcs", refresh_by_created_index, "htlcs" }, - { "listforwards", refresh_by_created_index, "forwards" }, - { "listinvoices", refresh_by_created_index, "invoices" }, - { "listsendpays", refresh_by_created_index, "sendpays" }, + /* For mutable tables, use full reload logic due to mutability */ + { "listhtlcs", refresh_htlcs_full, "htlcs" }, + { "listforwards", refresh_forwards_full, "forwards" }, + { "listinvoices", refresh_invoices_full, "invoices" }, + { "listsendpays", refresh_sendpays_full, "sendpays" }, /* These are never changed or deleted */ { "listchainmoves", refresh_by_created_index, "chainmoves" }, { "listchannelmoves", refresh_by_created_index, "channelmoves" }, From b6f48d507b0760a3befff7e967cb16f5d61fefa7 Mon Sep 17 00:00:00 2001 From: Andezion <245122@edu.p.lodz.pl> Date: Wed, 21 Jan 2026 19:09:13 +0100 Subject: [PATCH 7/7] cleaned up whitespaces Changelog-None --- plugins/sql.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/sql.c b/plugins/sql.c index 2e900eb363f3..d7913418c0c6 100644 --- a/plugins/sql.c +++ b/plugins/sql.c @@ -1684,7 +1684,7 @@ static struct command_result *refresh_invoices_full(struct command *cmd, plugin_log(cmd->plugin, LOG_INFORM,"Full reload of invoices: wait API event indicates possible deletion/change"); err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name),NULL, NULL, &errmsg); - if (err != SQLITE_OK) + if (err != SQLITE_OK) { return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s", td->name, errmsg); } @@ -1709,7 +1709,7 @@ static struct command_result *refresh_forwards_full(struct command *cmd, plugin_log(cmd->plugin, LOG_INFORM,"Full reload of forwards: wait API event indicates possible deletion/change"); err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name), NULL, NULL, &errmsg); - if (err != SQLITE_OK) + if (err != SQLITE_OK) { return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s",td->name, errmsg); } @@ -1734,7 +1734,7 @@ static struct command_result *refresh_htlcs_full(struct command *cmd, plugin_log(cmd->plugin, LOG_INFORM, "Full reload of htlcs: wait API event indicates possible deletion/change"); err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name), NULL, NULL, &errmsg); - if (err != SQLITE_OK) + if (err != SQLITE_OK) { return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s", td->name, errmsg); } @@ -1757,9 +1757,9 @@ static struct command_result *refresh_sendpays_full(struct command *cmd, char *errmsg; plugin_log(cmd->plugin, LOG_INFORM, "Full reload of sendpays: wait API event indicates possible deletion/change"); - + err = sqlite3_exec(sql->db, tal_fmt(tmpctx, "DELETE FROM %s;", td->name),NULL, NULL, &errmsg); - if (err != SQLITE_OK) + if (err != SQLITE_OK) { return command_fail(cmd, LIGHTNINGD, "cleaning '%s' failed: %s",td->name, errmsg); }