ns-api: fix scraping bug when operator specifies custom node HTTP API port in bond

nym-node-status-api/nym-node-status-api/Cargo.toml

@@ -3,7 +3,7 @@
 
 [package]
 name = "nym-node-status-api"
-version = "4.0.10"
+version = "4.0.11-rc1"
 authors.workspace = true
 repository.workspace = true
 homepage.workspace = true

nym-node-status-api/nym-node-status-api/migrations_pg/20251008000000_nym_nodes_add_http_api_port.sql

@@ -0,0 +1,2 @@
+ALTER TABLE nym_nodes
+    ADD COLUMN IF NOT EXISTS http_api_port INTEGER;

nym-node-status-api/nym-node-status-api/src/cli/mod.rs

@@ -1,5 +1,5 @@
 use crate::ticketbook_manager::TicketbookManagerConfig;
-use clap::Parser;
+use clap::{Parser, Subcommand};
 use nym_bin_common::bin_info;
 use nym_credential_proxy_lib::shared_state::ecash_state::TicketType;
 use reqwest::Url;
@@ -105,6 +105,19 @@ pub(crate) struct Cli {
 
     #[clap(flatten)]
     pub(crate) ticketbook: TicketbookArgs,
+
+    #[command(subcommand)]
+    pub(crate) command: Option<Commands>,
+}
+
+#[derive(Subcommand, Debug)]
+pub(crate) enum Commands {
+    /// Scrape a single node and output detailed debug logs
+    ScrapeNode {
+        /// The id of the node to scrape
+        #[arg(long)]
+        node_id: i64,
+    },
 }
 
 #[derive(Debug, Parser)]

nym-node-status-api/nym-node-status-api/src/db/models.rs

@@ -381,7 +381,7 @@ impl ScrapeNodeKind {
 pub(crate) struct ScraperNodeInfo {
     pub node_kind: ScrapeNodeKind,
     pub hosts: Vec<String>,
-    pub http_api_port: i64,
+    pub http_api_port: Option<u16>,
 }
 
 impl ScraperNodeInfo {
@@ -395,8 +395,21 @@ impl ScraperNodeInfo {
                 format!("http://{}", host),
             ]);
 
-            if self.http_api_port != DEFAULT_NYM_NODE_HTTP_PORT as i64 {
-                urls.insert(0, format!("http://{}:{}", host, self.http_api_port));
+            if let Some(custom_http_api_port) = self.http_api_port {
+                urls = Vec::new();
+                for host in &self.hosts {
+                    urls.append(&mut vec![format!(
+                        "http://{}:{}",
+                        host, custom_http_api_port
+                    )]);
+                }
+
+                // do not fall back to default ports, if the operator sets a custom http api port
+                // in their bond, use it and error out if it's not available
+                // this will correctly handle cases where some operators run multiple nodes
+                // on a single IP address and assign different custom http port apis at bond time
+
+                // urls.insert(0, format!("http://{}:{}", host, custom_http_api_port));
             }
         }
 
@@ -423,6 +436,7 @@ pub(crate) struct NymNodeDto {
     pub performance: String,
     pub self_described: Option<serde_json::Value>,
     pub bond_info: Option<serde_json::Value>,
+    pub http_api_port: Option<i32>,
 }
 
 #[allow(dead_code)] // it's not dead code but clippy doesn't detect usage in sqlx macros
@@ -440,6 +454,7 @@ pub(crate) struct NymNodeInsertRecord {
     pub entry: Option<serde_json::Value>,
     pub self_described: Option<serde_json::Value>,
     pub bond_info: Option<serde_json::Value>,
+    pub http_api_port: Option<i32>,
     pub last_updated_utc: i64,
 }
 
@@ -456,6 +471,12 @@ impl NymNodeInsertRecord {
             .map(|info| decimal_to_i64(info.total_stake()))
             .unwrap_or(0);
         let entry = serialize_opt_to_value!(skimmed_node.entry)?;
+        let http_api_port = bond_info.and_then(|bond| {
+            bond.bond_information
+                .node
+                .custom_http_port
+                .map(|port| port as i32)
+        });
         let bond_info = serialize_opt_to_value!(bond_info)?;
         let self_described = serialize_opt_to_value!(self_described)?;
 
@@ -472,6 +493,7 @@ impl NymNodeInsertRecord {
             entry,
             self_described,
             bond_info,
+            http_api_port,
             last_updated_utc: now,
         };
 

nym-node-status-api/nym-node-status-api/src/db/queries/nym_nodes.rs

@@ -35,7 +35,8 @@ pub(crate) async fn get_all_nym_nodes(pool: &DbPool) -> anyhow::Result<Vec<NymNo
             entry as "entry: serde_json::Value",
             performance,
             self_described as "self_described: serde_json::Value",
-            bond_info as "bond_info: serde_json::Value"
+            bond_info as "bond_info: serde_json::Value",
+            http_api_port
         FROM
             nym_nodes
         ORDER BY
@@ -72,7 +73,8 @@ pub(crate) async fn get_described_bonded_nym_nodes(
             entry as "entry: serde_json::Value",
             performance,
             self_described as "self_described: serde_json::Value",
-            bond_info as "bond_info: serde_json::Value"
+            bond_info as "bond_info: serde_json::Value",
+            http_api_port
         FROM
             nym_nodes
         WHERE
@@ -115,9 +117,9 @@ pub(crate) async fn update_nym_nodes(
                     supported_roles, entry,
                     self_described,
                     bond_info,
-                    performance, last_updated_utc
+                    performance, last_updated_utc, http_api_port
                 )
-                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+                VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
                 ON CONFLICT(node_id) DO UPDATE SET
                 ed25519_identity_pubkey=excluded.ed25519_identity_pubkey,
                 ip_addresses=excluded.ip_addresses,
@@ -129,7 +131,8 @@ pub(crate) async fn update_nym_nodes(
                 self_described=excluded.self_described,
                 bond_info=excluded.bond_info,
                 performance=excluded.performance,
-                last_updated_utc=excluded.last_updated_utc
+                last_updated_utc=excluded.last_updated_utc,
+                http_api_port=excluded.http_api_port
                 ;",
             record.node_id,
             record.ed25519_identity_pubkey,
@@ -144,6 +147,7 @@ pub(crate) async fn update_nym_nodes(
             record.bond_info,
             record.performance,
             record.last_updated_utc as i32,
+            record.http_api_port,
         )
         .execute(&mut *tx)
         .await

nym-node-status-api/nym-node-status-api/src/db/queries/scraper.rs

@@ -21,10 +21,11 @@ pub(crate) async fn get_nodes_for_scraping(pool: &DbPool) -> Result<Vec<ScraperN
     let skimmed_nodes = queries::get_described_bonded_nym_nodes(pool)
         .await
         .map(|nodes_dto| {
-            nodes_dto.into_iter().filter_map(|node| {
-                let node_id = node.node_id;
-                match SkimmedNode::try_from(node) {
-                    Ok(node) => Some(node),
+            nodes_dto.into_iter().filter_map(|node_dto| {
+                let node_id = node_dto.node_id;
+                let http_api_port = node_dto.http_api_port;
+                match SkimmedNode::try_from(node_dto) {
+                    Ok(node) => Some((node, http_api_port)),
                     Err(e) => {
                         tracing::error!("Failed to decode node_id={}: {}", node_id, e);
                         None
@@ -33,7 +34,7 @@ pub(crate) async fn get_nodes_for_scraping(pool: &DbPool) -> Result<Vec<ScraperN
             })
         })?;
 
-    skimmed_nodes.for_each(|node| {
+    skimmed_nodes.for_each(|(node, http_api_port)| {
         // TODO: relies on polyfilling: Nym nodes table might contain legacy mixnodes
         // as well. Categorize them here.
         let node_kind = if gateway_keys.contains(&node.ed25519_identity_pubkey.to_base58_string()) {
@@ -54,7 +55,7 @@ pub(crate) async fn get_nodes_for_scraping(pool: &DbPool) -> Result<Vec<ScraperN
                 .into_iter()
                 .map(|ip| ip.to_string())
                 .collect::<Vec<_>>(),
-            http_api_port: node.mix_port.into(),
+            http_api_port: http_api_port.map(|port| port as u16),
         })
     });