From a50c9ac3fb50562dab5d89b91338d0e0c0075277 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Wed, 8 Oct 2025 12:50:33 +0100 Subject: [PATCH 1/8] ns-api: fix scraping bug when operator specifies custom node HTTP API port in bond --- ...afa2525da6c0b871633aad80ad555db9cf47c.json | 32 ---------------- ...342dee177abc1c92e4a89147de3c22d3d1a5.json} | 10 ++++- ...105614dea7fd301f0ec38bfe85bfe546dad40.json | 12 ------ ...a317aabce063a650be362d3a8ed83cc7c3549.json | 12 ------ ...9f166ad7f57681f76a1d0c7723d007c1f2c1e.json | 12 ------ ...bd97845350f4fb9f806c60b93c7cebd5e410d.json | 20 ---------- ...91d2a2af57656b405212af414d765b2263347.json | 26 ------------- ...2bd71d1cfe5f5265e6f161b3122d1317a421.json} | 10 ++++- ...11e5732ece0bf84ea98f2328b20add8f2b5ef.json | 12 ------ ...7fca89d80869484f2f3c66cabb898f0298c62.json | 12 ------ ...57f91a11f5057273cb70bd0e629712d17dd41.json | 12 ------ ...13ab15fed747fb0cee1c9f949fb58461b3f79.json | 32 ---------------- ...6a811c593aea8febf1f891117e5e84213f147.json | 12 ------ ...c9f560017b4da6b0303ea0397d9568229e167.json | 38 ------------------- ...abd806877ce914b514d4f7cd6be318c4debe6.json | 12 ------ ...8f633a94d18a19b7f0f96935a62560def7d0f.json | 12 ------ ...52f2bfed2d918b894ec0f588e38dd5e8ad726.json | 12 ------ ...6a2e314787586a46cffd074abb67f2f4d109e.json | 38 ------------------- ...18a1f62fa67dfe8f130deb876ebee11bf1602.json | 20 ---------- ...1008000000_nym_nodes_add_http_api_port.sql | 2 + .../nym-node-status-api/src/db/models.rs | 28 ++++++++++++-- .../src/db/queries/nym_nodes.rs | 6 ++- .../src/db/queries/scraper.rs | 13 ++++--- .../nym-node-status-api/src/db/tests.rs | 29 ++++++++++++-- 24 files changed, 80 insertions(+), 344 deletions(-) delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c.json rename nym-node-status-api/nym-node-status-api/.sqlx/{query-283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c.json => query-0b51df277ed66c6553f66af9b135342dee177abc1c92e4a89147de3c22d3d1a5.json} (83%) delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347.json rename nym-node-status-api/nym-node-status-api/.sqlx/{query-c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc.json => query-3ddc12cc4e1796b787a50c40560d2bd71d1cfe5f5265e6f161b3122d1317a421.json} (85%) delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e.json delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602.json create mode 100644 nym-node-status-api/nym-node-status-api/migrations_pg/20251008000000_nym_nodes_add_http_api_port.sql diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c.json deleted file mode 100644 index 068bc54b4b9..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n SELECT epoch_id as \"epoch_id: u32\", serialised_key, serialization_revision as \"serialization_revision: u8\"\n FROM master_verification_key WHERE epoch_id = ?\n ", - "describe": { - "columns": [ - { - "name": "epoch_id: u32", - "ordinal": 0, - "type_info": "Integer" - }, - { - "name": "serialised_key", - "ordinal": 1, - "type_info": "Blob" - }, - { - "name": "serialization_revision: u8", - "ordinal": 2, - "type_info": "Integer" - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-0b51df277ed66c6553f66af9b135342dee177abc1c92e4a89147de3c22d3d1a5.json similarity index 83% rename from nym-node-status-api/nym-node-status-api/.sqlx/query-283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c.json rename to nym-node-status-api/nym-node-status-api/.sqlx/query-0b51df277ed66c6553f66af9b135342dee177abc1c92e4a89147de3c22d3d1a5.json index af701d00586..a5cc377447c 100644 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c.json +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-0b51df277ed66c6553f66af9b135342dee177abc1c92e4a89147de3c22d3d1a5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n node_id,\n ed25519_identity_pubkey,\n total_stake,\n ip_addresses as \"ip_addresses!: serde_json::Value\",\n mix_port,\n x25519_sphinx_pubkey,\n node_role as \"node_role: serde_json::Value\",\n supported_roles as \"supported_roles: serde_json::Value\",\n entry as \"entry: serde_json::Value\",\n performance,\n self_described as \"self_described: serde_json::Value\",\n bond_info as \"bond_info: serde_json::Value\"\n FROM\n nym_nodes\n WHERE\n self_described IS NOT NULL\n AND\n bond_info IS NOT NULL\n ", + "query": "SELECT\n node_id,\n ed25519_identity_pubkey,\n total_stake,\n ip_addresses as \"ip_addresses!: serde_json::Value\",\n mix_port,\n x25519_sphinx_pubkey,\n node_role as \"node_role: serde_json::Value\",\n supported_roles as \"supported_roles: serde_json::Value\",\n entry as \"entry: serde_json::Value\",\n performance,\n self_described as \"self_described: serde_json::Value\",\n bond_info as \"bond_info: serde_json::Value\",\n http_api_port\n FROM\n nym_nodes\n WHERE\n self_described IS NOT NULL\n AND\n bond_info IS NOT NULL\n ", "describe": { "columns": [ { @@ -62,6 +62,11 @@ "ordinal": 11, "name": "bond_info: serde_json::Value", "type_info": "Jsonb" + }, + { + "ordinal": 12, + "name": "http_api_port", + "type_info": "Int4" } ], "parameters": { @@ -79,8 +84,9 @@ true, false, true, + true, true ] }, - "hash": "283f49a65c7d70bf271702ff6a5c7ad6e68c81932d295ff18ed198c54706a57c" + "hash": "0b51df277ed66c6553f66af9b135342dee177abc1c92e4a89147de3c22d3d1a5" } diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40.json deleted file mode 100644 index cdc057df57e..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n INSERT INTO ecash_deposit_usage (deposit_id, ticketbooks_requested_on, client_pubkey, request_uuid)\n VALUES (?, ?, ?, ?)\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - }, - "hash": "1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549.json deleted file mode 100644 index 32f3f9958c4..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n DELETE FROM blinded_shares WHERE created < ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e.json deleted file mode 100644 index 9ef7d96cca1..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n INSERT INTO global_expiration_date_signatures(expiration_date, epoch_id, serialised_signatures, serialization_revision)\n VALUES (?, ?, ?, ?)\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 4 - }, - "nullable": [] - }, - "hash": "2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d.json deleted file mode 100644 index 944c86e33e9..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n SELECT error_message\n FROM blinded_shares\n WHERE id = ?;\n ", - "describe": { - "columns": [ - { - "name": "error_message", - "ordinal": 0, - "type_info": "Text" - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - true - ] - }, - "hash": "396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347.json deleted file mode 100644 index 9d8cff13268..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n SELECT serialised_signatures, serialization_revision as \"serialization_revision: u8\"\n FROM global_expiration_date_signatures\n WHERE expiration_date = ? AND epoch_id = ?\n ", - "describe": { - "columns": [ - { - "name": "serialised_signatures", - "ordinal": 0, - "type_info": "Blob" - }, - { - "name": "serialization_revision: u8", - "ordinal": 1, - "type_info": "Integer" - } - ], - "parameters": { - "Right": 2 - }, - "nullable": [ - false, - false - ] - }, - "hash": "3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-3ddc12cc4e1796b787a50c40560d2bd71d1cfe5f5265e6f161b3122d1317a421.json similarity index 85% rename from nym-node-status-api/nym-node-status-api/.sqlx/query-c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc.json rename to nym-node-status-api/nym-node-status-api/.sqlx/query-3ddc12cc4e1796b787a50c40560d2bd71d1cfe5f5265e6f161b3122d1317a421.json index 7954f28f312..2e19afef0da 100644 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc.json +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-3ddc12cc4e1796b787a50c40560d2bd71d1cfe5f5265e6f161b3122d1317a421.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n node_id,\n ed25519_identity_pubkey,\n total_stake,\n ip_addresses as \"ip_addresses!: serde_json::Value\",\n mix_port,\n x25519_sphinx_pubkey,\n node_role as \"node_role: serde_json::Value\",\n supported_roles as \"supported_roles: serde_json::Value\",\n entry as \"entry: serde_json::Value\",\n performance,\n self_described as \"self_described: serde_json::Value\",\n bond_info as \"bond_info: serde_json::Value\"\n FROM\n nym_nodes\n ORDER BY\n node_id\n ", + "query": "SELECT\n node_id,\n ed25519_identity_pubkey,\n total_stake,\n ip_addresses as \"ip_addresses!: serde_json::Value\",\n mix_port,\n x25519_sphinx_pubkey,\n node_role as \"node_role: serde_json::Value\",\n supported_roles as \"supported_roles: serde_json::Value\",\n entry as \"entry: serde_json::Value\",\n performance,\n self_described as \"self_described: serde_json::Value\",\n bond_info as \"bond_info: serde_json::Value\",\n http_api_port\n FROM\n nym_nodes\n ORDER BY\n node_id\n ", "describe": { "columns": [ { @@ -62,6 +62,11 @@ "ordinal": 11, "name": "bond_info: serde_json::Value", "type_info": "Jsonb" + }, + { + "ordinal": 12, + "name": "http_api_port", + "type_info": "Int4" } ], "parameters": { @@ -79,8 +84,9 @@ true, false, true, + true, true ] }, - "hash": "c48d04fc3de59dd484f0a63d40336ced54e08785f77e9ef85f3157d004ec85dc" + "hash": "3ddc12cc4e1796b787a50c40560d2bd71d1cfe5f5265e6f161b3122d1317a421" } diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef.json deleted file mode 100644 index 77a99816d49..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n DELETE FROM partial_blinded_wallet_failure WHERE created < ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62.json deleted file mode 100644 index 2f22ed93fa9..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT INTO master_verification_key(epoch_id, serialised_key, serialization_revision) VALUES (?, ?, ?)", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - }, - "hash": "70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41.json deleted file mode 100644 index 84e8ba9f160..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n INSERT INTO partial_blinded_wallet_failure(corresponding_deposit, epoch_id, expiration_date, node_id, created, failure_message)\n VALUES (?, ?, ?, ?, ?, ?)\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - }, - "hash": "97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79.json deleted file mode 100644 index 6cb8138cc52..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n SELECT epoch_id as \"epoch_id: u32\", serialised_signatures, serialization_revision as \"serialization_revision: u8\"\n FROM global_coin_index_signatures WHERE epoch_id = ?\n ", - "describe": { - "columns": [ - { - "name": "epoch_id: u32", - "ordinal": 0, - "type_info": "Integer" - }, - { - "name": "serialised_signatures", - "ordinal": 1, - "type_info": "Blob" - }, - { - "name": "serialization_revision: u8", - "ordinal": 2, - "type_info": "Integer" - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147.json deleted file mode 100644 index 03efdc4734f..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n DELETE FROM partial_blinded_wallet WHERE created < ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167.json deleted file mode 100644 index 6b70481c757..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n SELECT\n t1.node_id as \"node_id!\",\n t1.blinded_signature as \"blinded_signature!\",\n t1.epoch_id as \"epoch_id!\",\n t1.expiration_date as \"expiration_date!: Date\"\n FROM partial_blinded_wallet as t1\n JOIN ecash_deposit_usage as t2\n on t1.corresponding_deposit = t2.deposit_id\n JOIN blinded_shares as t3\n ON t2.request_uuid = t3.request_uuid\n WHERE t3.device_id = ? AND t3.credential_id = ?;\n ", - "describe": { - "columns": [ - { - "name": "node_id!", - "ordinal": 0, - "type_info": "Integer" - }, - { - "name": "blinded_signature!", - "ordinal": 1, - "type_info": "Blob" - }, - { - "name": "epoch_id!", - "ordinal": 2, - "type_info": "Integer" - }, - { - "name": "expiration_date!: Date", - "ordinal": 3, - "type_info": "Date" - } - ], - "parameters": { - "Right": 2 - }, - "nullable": [ - true, - true, - true, - true - ] - }, - "hash": "c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6.json deleted file mode 100644 index 9de4c7f2301..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT INTO global_coin_index_signatures(epoch_id, serialised_signatures, serialization_revision) VALUES (?, ?, ?)", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - }, - "hash": "d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f.json deleted file mode 100644 index e5dd7c84131..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n INSERT INTO partial_blinded_wallet(corresponding_deposit, epoch_id, expiration_date, node_id, created, blinded_signature)\n VALUES (?, ?, ?, ?, ?, ?)\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 6 - }, - "nullable": [] - }, - "hash": "db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726.json deleted file mode 100644 index 340f83a2806..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n UPDATE ecash_deposit_usage\n SET ticketbook_request_error = ?\n WHERE deposit_id = ?\n ", - "describe": { - "columns": [], - "parameters": { - "Right": 2 - }, - "nullable": [] - }, - "hash": "e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e.json deleted file mode 100644 index 449bc6869fe..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n SELECT t1.node_id, t1.blinded_signature, t1.epoch_id, t1.expiration_date as \"expiration_date!: Date\"\n FROM partial_blinded_wallet as t1\n JOIN ecash_deposit_usage as t2\n on t1.corresponding_deposit = t2.deposit_id\n JOIN blinded_shares as t3\n ON t2.request_uuid = t3.request_uuid\n WHERE t3.id = ?;\n ", - "describe": { - "columns": [ - { - "name": "node_id", - "ordinal": 0, - "type_info": "Integer" - }, - { - "name": "blinded_signature", - "ordinal": 1, - "type_info": "Blob" - }, - { - "name": "epoch_id", - "ordinal": 2, - "type_info": "Integer" - }, - { - "name": "expiration_date!: Date", - "ordinal": 3, - "type_info": "Date" - } - ], - "parameters": { - "Right": 1 - }, - "nullable": [ - false, - false, - false, - false - ] - }, - "hash": "e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602.json deleted file mode 100644 index c78ff30f506..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "\n SELECT error_message\n FROM blinded_shares\n WHERE device_id = ? AND credential_id = ?;\n ", - "describe": { - "columns": [ - { - "name": "error_message", - "ordinal": 0, - "type_info": "Text" - } - ], - "parameters": { - "Right": 2 - }, - "nullable": [ - true - ] - }, - "hash": "ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602" -} diff --git a/nym-node-status-api/nym-node-status-api/migrations_pg/20251008000000_nym_nodes_add_http_api_port.sql b/nym-node-status-api/nym-node-status-api/migrations_pg/20251008000000_nym_nodes_add_http_api_port.sql new file mode 100644 index 00000000000..3b7d2ab8fee --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/migrations_pg/20251008000000_nym_nodes_add_http_api_port.sql @@ -0,0 +1,2 @@ +ALTER TABLE nym_nodes + ADD COLUMN http_api_port INTEGER; \ No newline at end of file diff --git a/nym-node-status-api/nym-node-status-api/src/db/models.rs b/nym-node-status-api/nym-node-status-api/src/db/models.rs index 17c0425fd88..54913d8b39b 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/models.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/models.rs @@ -381,7 +381,7 @@ impl ScrapeNodeKind { pub(crate) struct ScraperNodeInfo { pub node_kind: ScrapeNodeKind, pub hosts: Vec, - pub http_api_port: i64, + pub http_api_port: Option, } impl ScraperNodeInfo { @@ -395,8 +395,21 @@ impl ScraperNodeInfo { format!("http://{}", host), ]); - if self.http_api_port != DEFAULT_NYM_NODE_HTTP_PORT as i64 { - urls.insert(0, format!("http://{}:{}", host, self.http_api_port)); + if let Some(custom_http_api_port) = self.http_api_port { + urls = Vec::new(); + for host in &self.hosts { + urls.append(&mut vec![format!( + "http://{}:{}", + host, custom_http_api_port + )]); + } + + // do not fall back to default ports, if the operator sets a custom http api port + // in their bond, use it and error out if it's not available + // this will correctly handle cases where some operators run multiple nodes + // on a single IP address and assign different custom http port apis at bond time + + // urls.insert(0, format!("http://{}:{}", host, custom_http_api_port)); } } @@ -423,6 +436,7 @@ pub(crate) struct NymNodeDto { pub performance: String, pub self_described: Option, pub bond_info: Option, + pub http_api_port: Option, } #[allow(dead_code)] // it's not dead code but clippy doesn't detect usage in sqlx macros @@ -440,6 +454,7 @@ pub(crate) struct NymNodeInsertRecord { pub entry: Option, pub self_described: Option, pub bond_info: Option, + pub http_api_port: Option, pub last_updated_utc: i64, } @@ -456,6 +471,12 @@ impl NymNodeInsertRecord { .map(|info| decimal_to_i64(info.total_stake())) .unwrap_or(0); let entry = serialize_opt_to_value!(skimmed_node.entry)?; + let http_api_port = bond_info.and_then(|bond| { + bond.bond_information + .node + .custom_http_port + .map(|port| port as i32) + }); let bond_info = serialize_opt_to_value!(bond_info)?; let self_described = serialize_opt_to_value!(self_described)?; @@ -472,6 +493,7 @@ impl NymNodeInsertRecord { entry, self_described, bond_info, + http_api_port, last_updated_utc: now, }; diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs index b42f1301a8f..737df830862 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs @@ -35,7 +35,8 @@ pub(crate) async fn get_all_nym_nodes(pool: &DbPool) -> anyhow::Result Result Some(node), + nodes_dto.into_iter().filter_map(|node_dto| { + let node_id = node_dto.node_id; + let http_api_port = node_dto.http_api_port; + match SkimmedNode::try_from(node_dto) { + Ok(node) => Some((node, http_api_port)), Err(e) => { tracing::error!("Failed to decode node_id={}: {}", node_id, e); None @@ -33,7 +34,7 @@ pub(crate) async fn get_nodes_for_scraping(pool: &DbPool) -> Result Result>(), - http_api_port: node.mix_port.into(), + http_api_port: http_api_port.map(|port| port as u16), }) }); diff --git a/nym-node-status-api/nym-node-status-api/src/db/tests.rs b/nym-node-status-api/nym-node-status-api/src/db/tests.rs index 0c3ae935555..a80bdc1bdd1 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/tests.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/tests.rs @@ -138,6 +138,7 @@ mod db_tests { performance: "1.0".to_string(), self_described: None, bond_info: None, + http_api_port: None, }; let skimmed_node: nym_validator_client::nym_api::SkimmedNode = @@ -362,22 +363,42 @@ fn test_scraper_node_info_contact_addresses() { let node_info = ScraperNodeInfo { node_kind: ScrapeNodeKind::MixingNymNode { node_id: 123 }, hosts: vec!["1.1.1.1".to_string(), "example.com".to_string()], - http_api_port: 8080, + http_api_port: None, }; let addresses = node_info.contact_addresses(); // Should generate multiple URLs for each host - // Custom port (8080) should be inserted at the beginning + // When no custom port is specified only default ports should be used assert!(addresses.contains(&"http://1.1.1.1:8080".to_string())); - assert!(addresses.contains(&"http://example.com:8080".to_string())); assert!(addresses.contains(&"http://1.1.1.1:8000".to_string())); assert!(addresses.contains(&"https://1.1.1.1".to_string())); + assert!(addresses.contains(&"http://1.1.1.1".to_string())); assert!(addresses.contains(&"http://example.com:8000".to_string())); // Check that URLs follow the expected pattern assert!(addresses.len() >= 8); // At least 4 URLs per host } +#[test] +fn test_scraper_node_info_contact_addresses_with_custom_http_api_port() { + use crate::db::models::{ScrapeNodeKind, ScraperNodeInfo}; + + let node_info = ScraperNodeInfo { + node_kind: ScrapeNodeKind::MixingNymNode { node_id: 123 }, + hosts: vec!["1.1.1.1".to_string(), "example.com".to_string()], + http_api_port: Some(4444), + }; + + let addresses = node_info.contact_addresses(); + + // Should generate multiple URLs for each host + // Custom port (4444) should be the only port in the list + assert!(addresses.contains(&"http://1.1.1.1:4444".to_string())); + assert!(addresses.contains(&"http://example.com:4444".to_string())); + // Check that URLs follow the expected pattern + assert!(addresses.len() >= 2); // At least 4 URLs per host +} + #[test] fn test_scrape_node_kind_node_id() { use crate::db::models::ScrapeNodeKind; @@ -414,6 +435,7 @@ fn test_nym_node_dto_with_invalid_keys() { performance: "1.0".to_string(), self_described: None, bond_info: None, + http_api_port: None, }; let result: Result = nym_node_dto.try_into(); @@ -451,6 +473,7 @@ fn test_nym_node_dto_with_invalid_performance() { performance: "invalid_percent".to_string(), self_described: None, bond_info: None, + http_api_port: None, }; let result: Result = nym_node_dto.try_into(); From e18e64bf21afd8f4a5a20dfb89de0fa49ed728ac Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 10 Oct 2025 15:13:47 +0100 Subject: [PATCH 2/8] wip --- ...afa2525da6c0b871633aad80ad555db9cf47c.json | 32 ++++++++++++++++ ...105614dea7fd301f0ec38bfe85bfe546dad40.json | 12 ++++++ ...a317aabce063a650be362d3a8ed83cc7c3549.json | 12 ++++++ ...9f166ad7f57681f76a1d0c7723d007c1f2c1e.json | 12 ++++++ ...bd97845350f4fb9f806c60b93c7cebd5e410d.json | 20 ++++++++++ ...91d2a2af57656b405212af414d765b2263347.json | 26 +++++++++++++ ...11e5732ece0bf84ea98f2328b20add8f2b5ef.json | 12 ++++++ ...7fca89d80869484f2f3c66cabb898f0298c62.json | 12 ++++++ ...57f91a11f5057273cb70bd0e629712d17dd41.json | 12 ++++++ ...13ab15fed747fb0cee1c9f949fb58461b3f79.json | 32 ++++++++++++++++ ...6a811c593aea8febf1f891117e5e84213f147.json | 12 ++++++ ...c9f560017b4da6b0303ea0397d9568229e167.json | 38 +++++++++++++++++++ ...abd806877ce914b514d4f7cd6be318c4debe6.json | 12 ++++++ ...8f633a94d18a19b7f0f96935a62560def7d0f.json | 12 ++++++ ...52f2bfed2d918b894ec0f588e38dd5e8ad726.json | 12 ++++++ ...6a2e314787586a46cffd074abb67f2f4d109e.json | 38 +++++++++++++++++++ ...18a1f62fa67dfe8f130deb876ebee11bf1602.json | 20 ++++++++++ .../nym-node-status-api/src/cli/mod.rs | 15 +++++++- .../nym-node-status-api/src/main.rs | 13 +++++++ .../src/node_scraper/helpers.rs | 11 ++++++ 20 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602.json diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c.json new file mode 100644 index 00000000000..068bc54b4b9 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c.json @@ -0,0 +1,32 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT epoch_id as \"epoch_id: u32\", serialised_key, serialization_revision as \"serialization_revision: u8\"\n FROM master_verification_key WHERE epoch_id = ?\n ", + "describe": { + "columns": [ + { + "name": "epoch_id: u32", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "serialised_key", + "ordinal": 1, + "type_info": "Blob" + }, + { + "name": "serialization_revision: u8", + "ordinal": 2, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "0112296b190328a3856d1adf51aafa2525da6c0b871633aad80ad555db9cf47c" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40.json new file mode 100644 index 00000000000..cdc057df57e --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n INSERT INTO ecash_deposit_usage (deposit_id, ticketbooks_requested_on, client_pubkey, request_uuid)\n VALUES (?, ?, ?, ?)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 4 + }, + "nullable": [] + }, + "hash": "1fc72f8ba24039548047e1766c9105614dea7fd301f0ec38bfe85bfe546dad40" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549.json new file mode 100644 index 00000000000..32f3f9958c4 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n DELETE FROM blinded_shares WHERE created < ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + }, + "hash": "28681fcd8e2d4326f628681b8f2a317aabce063a650be362d3a8ed83cc7c3549" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e.json new file mode 100644 index 00000000000..9ef7d96cca1 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n INSERT INTO global_expiration_date_signatures(expiration_date, epoch_id, serialised_signatures, serialization_revision)\n VALUES (?, ?, ?, ?)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 4 + }, + "nullable": [] + }, + "hash": "2930ca6e3875c74acb7abb9ad889f166ad7f57681f76a1d0c7723d007c1f2c1e" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d.json new file mode 100644 index 00000000000..944c86e33e9 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT error_message\n FROM blinded_shares\n WHERE id = ?;\n ", + "describe": { + "columns": [ + { + "name": "error_message", + "ordinal": 0, + "type_info": "Text" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + true + ] + }, + "hash": "396f40c33f0f62796eb7449d640bd97845350f4fb9f806c60b93c7cebd5e410d" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347.json new file mode 100644 index 00000000000..9d8cff13268 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347.json @@ -0,0 +1,26 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT serialised_signatures, serialization_revision as \"serialization_revision: u8\"\n FROM global_expiration_date_signatures\n WHERE expiration_date = ? AND epoch_id = ?\n ", + "describe": { + "columns": [ + { + "name": "serialised_signatures", + "ordinal": 0, + "type_info": "Blob" + }, + { + "name": "serialization_revision: u8", + "ordinal": 1, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false, + false + ] + }, + "hash": "3cc446220668fb3e02f0578104291d2a2af57656b405212af414d765b2263347" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef.json new file mode 100644 index 00000000000..77a99816d49 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n DELETE FROM partial_blinded_wallet_failure WHERE created < ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + }, + "hash": "52b378e282d93db941eff53b5b311e5732ece0bf84ea98f2328b20add8f2b5ef" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62.json new file mode 100644 index 00000000000..2f22ed93fa9 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO master_verification_key(epoch_id, serialised_key, serialization_revision) VALUES (?, ?, ?)", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + }, + "hash": "70d8f240ad6edda6b8c7f2e800e7fca89d80869484f2f3c66cabb898f0298c62" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41.json new file mode 100644 index 00000000000..84e8ba9f160 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n INSERT INTO partial_blinded_wallet_failure(corresponding_deposit, epoch_id, expiration_date, node_id, created, failure_message)\n VALUES (?, ?, ?, ?, ?, ?)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + }, + "hash": "97d97ebb6bc8f4114fdea9ebc9f57f91a11f5057273cb70bd0e629712d17dd41" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79.json new file mode 100644 index 00000000000..6cb8138cc52 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79.json @@ -0,0 +1,32 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT epoch_id as \"epoch_id: u32\", serialised_signatures, serialization_revision as \"serialization_revision: u8\"\n FROM global_coin_index_signatures WHERE epoch_id = ?\n ", + "describe": { + "columns": [ + { + "name": "epoch_id: u32", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "serialised_signatures", + "ordinal": 1, + "type_info": "Blob" + }, + { + "name": "serialization_revision: u8", + "ordinal": 2, + "type_info": "Integer" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "a8b7ce0fe4755c28b96d1e503e313ab15fed747fb0cee1c9f949fb58461b3f79" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147.json new file mode 100644 index 00000000000..03efdc4734f --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n DELETE FROM partial_blinded_wallet WHERE created < ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 1 + }, + "nullable": [] + }, + "hash": "b8257a0832d0124f0a8aaaf81dc6a811c593aea8febf1f891117e5e84213f147" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167.json new file mode 100644 index 00000000000..6b70481c757 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167.json @@ -0,0 +1,38 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT\n t1.node_id as \"node_id!\",\n t1.blinded_signature as \"blinded_signature!\",\n t1.epoch_id as \"epoch_id!\",\n t1.expiration_date as \"expiration_date!: Date\"\n FROM partial_blinded_wallet as t1\n JOIN ecash_deposit_usage as t2\n on t1.corresponding_deposit = t2.deposit_id\n JOIN blinded_shares as t3\n ON t2.request_uuid = t3.request_uuid\n WHERE t3.device_id = ? AND t3.credential_id = ?;\n ", + "describe": { + "columns": [ + { + "name": "node_id!", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "blinded_signature!", + "ordinal": 1, + "type_info": "Blob" + }, + { + "name": "epoch_id!", + "ordinal": 2, + "type_info": "Integer" + }, + { + "name": "expiration_date!: Date", + "ordinal": 3, + "type_info": "Date" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + true, + true, + true, + true + ] + }, + "hash": "c2b841762bdb963fff337ef5c8ec9f560017b4da6b0303ea0397d9568229e167" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6.json new file mode 100644 index 00000000000..9de4c7f2301 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO global_coin_index_signatures(epoch_id, serialised_signatures, serialization_revision) VALUES (?, ?, ?)", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + }, + "hash": "d3510846941fa2525926b9bfbcdabd806877ce914b514d4f7cd6be318c4debe6" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f.json new file mode 100644 index 00000000000..e5dd7c84131 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n INSERT INTO partial_blinded_wallet(corresponding_deposit, epoch_id, expiration_date, node_id, created, blinded_signature)\n VALUES (?, ?, ?, ?, ?, ?)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + }, + "hash": "db176e98198fe594d88eb860d918f633a94d18a19b7f0f96935a62560def7d0f" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726.json new file mode 100644 index 00000000000..340f83a2806 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n UPDATE ecash_deposit_usage\n SET ticketbook_request_error = ?\n WHERE deposit_id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 2 + }, + "nullable": [] + }, + "hash": "e584253e3856355899537eb8fc152f2bfed2d918b894ec0f588e38dd5e8ad726" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e.json new file mode 100644 index 00000000000..449bc6869fe --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e.json @@ -0,0 +1,38 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT t1.node_id, t1.blinded_signature, t1.epoch_id, t1.expiration_date as \"expiration_date!: Date\"\n FROM partial_blinded_wallet as t1\n JOIN ecash_deposit_usage as t2\n on t1.corresponding_deposit = t2.deposit_id\n JOIN blinded_shares as t3\n ON t2.request_uuid = t3.request_uuid\n WHERE t3.id = ?;\n ", + "describe": { + "columns": [ + { + "name": "node_id", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "blinded_signature", + "ordinal": 1, + "type_info": "Blob" + }, + { + "name": "epoch_id", + "ordinal": 2, + "type_info": "Integer" + }, + { + "name": "expiration_date!: Date", + "ordinal": 3, + "type_info": "Date" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "e77ffab19b099b84470fe5611716a2e314787586a46cffd074abb67f2f4d109e" +} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602.json new file mode 100644 index 00000000000..c78ff30f506 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT error_message\n FROM blinded_shares\n WHERE device_id = ? AND credential_id = ?;\n ", + "describe": { + "columns": [ + { + "name": "error_message", + "ordinal": 0, + "type_info": "Text" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + true + ] + }, + "hash": "ef60c2683211cc4ec2d3e46392518a1f62fa67dfe8f130deb876ebee11bf1602" +} diff --git a/nym-node-status-api/nym-node-status-api/src/cli/mod.rs b/nym-node-status-api/nym-node-status-api/src/cli/mod.rs index 2f66449f232..4883e951395 100644 --- a/nym-node-status-api/nym-node-status-api/src/cli/mod.rs +++ b/nym-node-status-api/nym-node-status-api/src/cli/mod.rs @@ -1,5 +1,5 @@ use crate::ticketbook_manager::TicketbookManagerConfig; -use clap::Parser; +use clap::{Parser, Subcommand}; use nym_bin_common::bin_info; use nym_credential_proxy_lib::shared_state::ecash_state::TicketType; use reqwest::Url; @@ -105,6 +105,19 @@ pub(crate) struct Cli { #[clap(flatten)] pub(crate) ticketbook: TicketbookArgs, + + #[command(subcommand)] + pub(crate) command: Option, +} + +#[derive(Subcommand, Debug)] +pub(crate) enum Commands { + /// Scrape a single node and output detailed debug logs + ScrapeNode { + /// The id of the node to scrape + #[arg(long)] + node_id: i64, + }, } #[derive(Debug, Parser)] diff --git a/nym-node-status-api/nym-node-status-api/src/main.rs b/nym-node-status-api/nym-node-status-api/src/main.rs index 452d395b1d9..987c9b4bf29 100644 --- a/nym-node-status-api/nym-node-status-api/src/main.rs +++ b/nym-node-status-api/nym-node-status-api/src/main.rs @@ -1,4 +1,6 @@ +use crate::cli::Commands; use crate::monitor::DelegationsCache; +use crate::node_scraper::helpers::scrape_and_store_description_by_node_id; use crate::ticketbook_manager::TicketbookManager; use crate::ticketbook_manager::state::TicketbookManagerState; use clap::Parser; @@ -45,6 +47,17 @@ async fn main() -> anyhow::Result<()> { let storage = db::Storage::init(connection_url, args.sqlx_busy_timeout_s).await?; let db_pool = storage.pool_owned(); + match args.command { + Some(Commands::ScrapeNode { node_id }) => { + tracing::info!("Scraping node with id {node_id}..."); + scrape_and_store_description_by_node_id(&db_pool, node_id).await?; + return Ok(()); + } + None => { + // default behaviour + } + } + // Start the node scraper let scraper = node_scraper::DescriptionScraper::new(storage.pool_owned()); shutdown_manager.spawn_with_shutdown(async move { diff --git a/nym-node-status-api/nym-node-status-api/src/node_scraper/helpers.rs b/nym-node-status-api/nym-node-status-api/src/node_scraper/helpers.rs index 4ec1e5d49bb..664a763e382 100644 --- a/nym-node-status-api/nym-node-status-api/src/node_scraper/helpers.rs +++ b/nym-node-status-api/nym-node-status-api/src/node_scraper/helpers.rs @@ -118,6 +118,17 @@ pub fn sanitize_description( } } +pub async fn scrape_and_store_description_by_node_id(pool: &DbPool, node_id: i64) -> Result<()> { + let nodes = crate::db::queries::get_nodes_for_scraping(pool).await?; + match nodes.iter().find(|n| *n.node_kind.node_id() == node_id) { + Some(node) => Ok(scrape_and_store_description(pool, node.clone()).await?), + None => { + error!("Could not find node with id {node_id}"); + Err(anyhow!("Could not find node with id {node_id}")) + } + } +} + pub async fn scrape_and_store_description(pool: &DbPool, node: ScraperNodeInfo) -> Result<()> { let client = build_client()?; let urls = node.contact_addresses(); From de8030d85a91c588c2a6a0f323bc23b2941c46b0 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 10 Oct 2025 16:07:22 +0100 Subject: [PATCH 3/8] allow NS API to run once for scraping for troubleshooting and debugging --- ...8b2abb437cccdb6ebd2e1087cc822ed737b0e.json | 26 ------------ ...fa3d5c939e7122ba7c88b78a353f00b271ec2.json | 27 +++++++++++++ .../nym-node-status-api/src/main.rs | 40 +++++++++++++------ .../nym-node-status-api/src/monitor/mod.rs | 34 +++++++++++++++- 4 files changed, 87 insertions(+), 40 deletions(-) delete mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-b010fb91828f7e4f0b72bdfe3b58b2abb437cccdb6ebd2e1087cc822ed737b0e.json create mode 100644 nym-node-status-api/nym-node-status-api/.sqlx/query-dde9aff827c34086077927bbe33fa3d5c939e7122ba7c88b78a353f00b271ec2.json diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-b010fb91828f7e4f0b72bdfe3b58b2abb437cccdb6ebd2e1087cc822ed737b0e.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-b010fb91828f7e4f0b72bdfe3b58b2abb437cccdb6ebd2e1087cc822ed737b0e.json deleted file mode 100644 index 5379205f8d1..00000000000 --- a/nym-node-status-api/nym-node-status-api/.sqlx/query-b010fb91828f7e4f0b72bdfe3b58b2abb437cccdb6ebd2e1087cc822ed737b0e.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO nym_nodes\n (node_id, ed25519_identity_pubkey,\n total_stake,\n ip_addresses, mix_port,\n x25519_sphinx_pubkey, node_role,\n supported_roles, entry,\n self_described,\n bond_info,\n performance, last_updated_utc\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)\n ON CONFLICT(node_id) DO UPDATE SET\n ed25519_identity_pubkey=excluded.ed25519_identity_pubkey,\n ip_addresses=excluded.ip_addresses,\n mix_port=excluded.mix_port,\n x25519_sphinx_pubkey=excluded.x25519_sphinx_pubkey,\n node_role=excluded.node_role,\n supported_roles=excluded.supported_roles,\n entry=excluded.entry,\n self_described=excluded.self_described,\n bond_info=excluded.bond_info,\n performance=excluded.performance,\n last_updated_utc=excluded.last_updated_utc\n ;", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "Varchar", - "Int8", - "Jsonb", - "Int4", - "Varchar", - "Jsonb", - "Jsonb", - "Jsonb", - "Jsonb", - "Jsonb", - "Varchar", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "b010fb91828f7e4f0b72bdfe3b58b2abb437cccdb6ebd2e1087cc822ed737b0e" -} diff --git a/nym-node-status-api/nym-node-status-api/.sqlx/query-dde9aff827c34086077927bbe33fa3d5c939e7122ba7c88b78a353f00b271ec2.json b/nym-node-status-api/nym-node-status-api/.sqlx/query-dde9aff827c34086077927bbe33fa3d5c939e7122ba7c88b78a353f00b271ec2.json new file mode 100644 index 00000000000..0b380650c51 --- /dev/null +++ b/nym-node-status-api/nym-node-status-api/.sqlx/query-dde9aff827c34086077927bbe33fa3d5c939e7122ba7c88b78a353f00b271ec2.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO nym_nodes\n (node_id, ed25519_identity_pubkey,\n total_stake,\n ip_addresses, mix_port,\n x25519_sphinx_pubkey, node_role,\n supported_roles, entry,\n self_described,\n bond_info,\n performance, last_updated_utc, http_api_port\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)\n ON CONFLICT(node_id) DO UPDATE SET\n ed25519_identity_pubkey=excluded.ed25519_identity_pubkey,\n ip_addresses=excluded.ip_addresses,\n mix_port=excluded.mix_port,\n x25519_sphinx_pubkey=excluded.x25519_sphinx_pubkey,\n node_role=excluded.node_role,\n supported_roles=excluded.supported_roles,\n entry=excluded.entry,\n self_described=excluded.self_described,\n bond_info=excluded.bond_info,\n performance=excluded.performance,\n last_updated_utc=excluded.last_updated_utc,\n http_api_port=excluded.http_api_port\n ;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Varchar", + "Int8", + "Jsonb", + "Int4", + "Varchar", + "Jsonb", + "Jsonb", + "Jsonb", + "Jsonb", + "Jsonb", + "Varchar", + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "dde9aff827c34086077927bbe33fa3d5c939e7122ba7c88b78a353f00b271ec2" +} diff --git a/nym-node-status-api/nym-node-status-api/src/main.rs b/nym-node-status-api/nym-node-status-api/src/main.rs index 987c9b4bf29..ebb76373aca 100644 --- a/nym-node-status-api/nym-node-status-api/src/main.rs +++ b/nym-node-status-api/nym-node-status-api/src/main.rs @@ -42,13 +42,40 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Registered {} agent keys", agent_key_list.len()); let connection_url = args.database_url.clone(); - tracing::debug!("Using config:\n{:#?}", args); + if std::env::var("SHOW_CONFIG").ok().is_some() { + tracing::debug!("Using config:\n{:#?}", args); + } let storage = db::Storage::init(connection_url, args.sqlx_busy_timeout_s).await?; let db_pool = storage.pool_owned(); + // node geocache is shared between node monitor and HTTP server + let geocache = moka::future::Cache::builder() + .time_to_live(args.geodata_ttl) + .build(); + let delegations_cache = DelegationsCache::new(); + + let client_config = nym_validator_client::nyxd::Config::try_from_nym_network_details( + &nym_network_defaults::NymNetworkDetails::new_from_env(), + )?; + let nyxd_client = NyxdClient::connect(client_config.clone(), args.nyxd_addr.as_str()) + .map_err(|err| anyhow::anyhow!("Couldn't connect: {}", err))?; + match args.command { Some(Commands::ScrapeNode { node_id }) => { + if std::env::var("RUN_ONCE_INIT_NODES").ok().is_some() { + let geocache_clone = geocache.clone(); + let delegations_cache_clone = Arc::clone(&delegations_cache); + monitor::run_once( + db_pool.clone(), + args.nym_api_client_timeout, + nyxd_client, + args.ipinfo_api_token, + geocache_clone, + delegations_cache_clone, + ) + .await?; + } tracing::info!("Scraping node with id {node_id}..."); scrape_and_store_description_by_node_id(&db_pool, node_id).await?; return Ok(()); @@ -71,20 +98,9 @@ async fn main() -> anyhow::Result<()> { scraper.start().await; }); - // node geocache is shared between node monitor and HTTP server - let geocache = moka::future::Cache::builder() - .time_to_live(args.geodata_ttl) - .build(); - let delegations_cache = DelegationsCache::new(); - // Start the monitor let geocache_clone = geocache.clone(); let delegations_cache_clone = Arc::clone(&delegations_cache); - let client_config = nym_validator_client::nyxd::Config::try_from_nym_network_details( - &nym_network_defaults::NymNetworkDetails::new_from_env(), - )?; - let nyxd_client = NyxdClient::connect(client_config.clone(), args.nyxd_addr.as_str()) - .map_err(|err| anyhow::anyhow!("Couldn't connect: {}", err))?; shutdown_manager.spawn_with_shutdown(async move { monitor::run_in_background( diff --git a/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs b/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs index 688e644af59..17117de68cc 100644 --- a/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs +++ b/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs @@ -68,7 +68,7 @@ pub(crate) async fn run_in_background( loop { tracing::info!("Refreshing node info..."); - if let Err(e) = monitor.run().await { + if let Err(e) = monitor.run(false).await { tracing::error!( "Monitor run failed: {e}, retrying in {}s...", MONITOR_FAILURE_RETRY_DELAY.as_secs() @@ -84,8 +84,33 @@ pub(crate) async fn run_in_background( } } +#[instrument(level = "debug", name = "data_monitor", skip_all)] +pub(crate) async fn run_once( + db_pool: DbPool, + nym_api_client_timeout: Duration, + nyxd_client: nym_validator_client::QueryHttpRpcNyxdClient, + ipinfo_api_token: String, + geocache: NodeGeoCache, + node_delegations: Arc>, +) -> anyhow::Result<()> { + let ipinfo = IpInfoClient::new(ipinfo_api_token.clone()); + + let mut monitor = Monitor { + db_pool, + network_details: nym_network_defaults::NymNetworkDetails::new_from_env(), + nym_api_client_timeout, + nyxd_client, + ipinfo, + geocache, + node_delegations, + }; + + tracing::info!("Refreshing node info..."); + Ok(monitor.run(true).await?) +} + impl Monitor { - async fn run(&mut self) -> anyhow::Result<()> { + async fn run(&mut self, exit_early: bool) -> anyhow::Result<()> { self.check_ipinfo_bandwidth().await; let default_api_url = self @@ -153,6 +178,11 @@ impl Monitor { tracing::debug!("{} nym nodes written to DB!", inserted); })?; + // stop here if running once + if exit_early { + return Ok(()); + } + // refresh geodata for all nodes for node_description in described_nodes.values() { self.location_cached(node_description).await; From 6bbb14f12f231118c9252339d131854755fda74b Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 10 Oct 2025 16:07:43 +0100 Subject: [PATCH 4/8] save custom_http_port to db --- .../nym-node-status-api/src/db/queries/nym_nodes.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs b/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs index 737df830862..ae53e041c23 100644 --- a/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs +++ b/nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs @@ -117,9 +117,9 @@ pub(crate) async fn update_nym_nodes( supported_roles, entry, self_described, bond_info, - performance, last_updated_utc + performance, last_updated_utc, http_api_port ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT(node_id) DO UPDATE SET ed25519_identity_pubkey=excluded.ed25519_identity_pubkey, ip_addresses=excluded.ip_addresses, @@ -131,7 +131,8 @@ pub(crate) async fn update_nym_nodes( self_described=excluded.self_described, bond_info=excluded.bond_info, performance=excluded.performance, - last_updated_utc=excluded.last_updated_utc + last_updated_utc=excluded.last_updated_utc, + http_api_port=excluded.http_api_port ;", record.node_id, record.ed25519_identity_pubkey, @@ -146,6 +147,7 @@ pub(crate) async fn update_nym_nodes( record.bond_info, record.performance, record.last_updated_utc as i32, + record.http_api_port, ) .execute(&mut *tx) .await From cd77b1032f5864c3cbb57959e06cf8fab03a49d7 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 10 Oct 2025 16:07:54 +0100 Subject: [PATCH 5/8] clippy --- nym-node-status-api/nym-node-status-api/src/monitor/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs b/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs index 17117de68cc..983d75d1faf 100644 --- a/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs +++ b/nym-node-status-api/nym-node-status-api/src/monitor/mod.rs @@ -106,7 +106,7 @@ pub(crate) async fn run_once( }; tracing::info!("Refreshing node info..."); - Ok(monitor.run(true).await?) + monitor.run(true).await } impl Monitor { From ceeeb6211b0909e7d83901d18d90e2657daa59f3 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 10 Oct 2025 17:52:44 +0100 Subject: [PATCH 6/8] add tracing output --- .../nym-node-status-api/src/node_scraper/helpers.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nym-node-status-api/nym-node-status-api/src/node_scraper/helpers.rs b/nym-node-status-api/nym-node-status-api/src/node_scraper/helpers.rs index 664a763e382..0fd5b0b2381 100644 --- a/nym-node-status-api/nym-node-status-api/src/node_scraper/helpers.rs +++ b/nym-node-status-api/nym-node-status-api/src/node_scraper/helpers.rs @@ -163,7 +163,13 @@ pub async fn scrape_and_store_description(pool: &DbPool, node: ScraperNodeInfo) anyhow::anyhow!("Failed to fetch description from any URL: {}", err_msg) })?; - let sanitized_description = sanitize_description(description, *node.node_id()); + let sanitized_description = sanitize_description(description.clone(), *node.node_id()); + + trace!("tried_url_list = {tried_url_list:?}"); + trace!("ndoe_id = {}", node.node_id()); + trace!("description = {:?}", description); + trace!("sanitized_description = {:?}", sanitized_description); + insert_scraped_node_description(pool, &node.node_kind, &sanitized_description).await?; Ok(()) From fef7e42eb47058b5687e699be9340148a00d8ef1 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 10 Oct 2025 16:09:11 +0100 Subject: [PATCH 7/8] bump version to rc --- Cargo.lock | 6 +++--- nym-node-status-api/nym-node-status-api/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04f9da4092b..3464d234d97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,9 +133,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "ammonia" -version = "4.1.1" +version = "4.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b346764dd0814805de8abf899fe03065bcee69bb1a4771c785817e39f3978f" +checksum = "17e913097e1a2124b46746c980134e8c954bc17a6a59bb3fde96f088d126dde6" dependencies = [ "cssparser", "html5ever", @@ -6546,7 +6546,7 @@ dependencies = [ [[package]] name = "nym-node-status-api" -version = "4.0.10" +version = "4.0.11-rc0" dependencies = [ "ammonia", "anyhow", diff --git a/nym-node-status-api/nym-node-status-api/Cargo.toml b/nym-node-status-api/nym-node-status-api/Cargo.toml index a1fefcdcc5b..3b12aa34fc8 100644 --- a/nym-node-status-api/nym-node-status-api/Cargo.toml +++ b/nym-node-status-api/nym-node-status-api/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-node-status-api" -version = "4.0.10" +version = "4.0.11-rc0" authors.workspace = true repository.workspace = true homepage.workspace = true From 17d11f201e2c5dda9911c21fd6f079f9da50ac72 Mon Sep 17 00:00:00 2001 From: Mark Sinclair Date: Fri, 17 Oct 2025 15:04:53 +0100 Subject: [PATCH 8/8] change migration and bump version --- Cargo.lock | 2 +- nym-node-status-api/nym-node-status-api/Cargo.toml | 2 +- .../20251008000000_nym_nodes_add_http_api_port.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3464d234d97..d02273aa8c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6546,7 +6546,7 @@ dependencies = [ [[package]] name = "nym-node-status-api" -version = "4.0.11-rc0" +version = "4.0.11-rc1" dependencies = [ "ammonia", "anyhow", diff --git a/nym-node-status-api/nym-node-status-api/Cargo.toml b/nym-node-status-api/nym-node-status-api/Cargo.toml index 3b12aa34fc8..c92ebb4dd9d 100644 --- a/nym-node-status-api/nym-node-status-api/Cargo.toml +++ b/nym-node-status-api/nym-node-status-api/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "nym-node-status-api" -version = "4.0.11-rc0" +version = "4.0.11-rc1" authors.workspace = true repository.workspace = true homepage.workspace = true diff --git a/nym-node-status-api/nym-node-status-api/migrations_pg/20251008000000_nym_nodes_add_http_api_port.sql b/nym-node-status-api/nym-node-status-api/migrations_pg/20251008000000_nym_nodes_add_http_api_port.sql index 3b7d2ab8fee..7dbdd644beb 100644 --- a/nym-node-status-api/nym-node-status-api/migrations_pg/20251008000000_nym_nodes_add_http_api_port.sql +++ b/nym-node-status-api/nym-node-status-api/migrations_pg/20251008000000_nym_nodes_add_http_api_port.sql @@ -1,2 +1,2 @@ ALTER TABLE nym_nodes - ADD COLUMN http_api_port INTEGER; \ No newline at end of file + ADD COLUMN IF NOT EXISTS http_api_port INTEGER; \ No newline at end of file