From 6164da48fb52e889331ce9d4fc4012f3671e615b Mon Sep 17 00:00:00 2001 From: baishen Date: Thu, 13 Nov 2025 15:08:49 +0800 Subject: [PATCH 1/3] fix(query): Fix copy into Variant field panic with infinite number --- Cargo.lock | 72 +++++++++++-------- Cargo.toml | 8 +-- tests/data/ndjson/numbers.ndjson | 6 +- .../stage/formats/ndjson/ndjson_cast.test | 23 ++++-- .../stage/formats/parquet/infer_schema.test | 3 +- 5 files changed, 69 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c3bc4ea6dae6a..d00eb34e1a5b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3055,7 +3055,7 @@ dependencies = [ "logos", "nom 7.1.3", "nom-rule", - "ordered-float 5.0.0", + "ordered-float 5.1.0", "percent-encoding", "pratt", "pretty_assertions", @@ -5452,7 +5452,7 @@ dependencies = [ "match-template", "num-traits", "num_cpus", - "ordered-float 5.0.0", + "ordered-float 5.1.0", "parking_lot 0.12.3", "parquet", "rand 0.8.5", @@ -6356,9 +6356,9 @@ dependencies = [ [[package]] name = "ethnum" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0939f82868b77ef93ce3c3c3daf2b3c526b456741da5a1a4559e590965b6026b" +checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" dependencies = [ "serde", ] @@ -6457,7 +6457,7 @@ dependencies = [ "fastrace-macro", "parking_lot 0.12.3", "pin-project", - "rand 0.9.1", + "rand 0.9.2", "rtrb", "serde", ] @@ -8370,7 +8370,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.1", + "rand 0.9.2", "ring", "thiserror 2.0.12", "tinyvec", @@ -8392,7 +8392,7 @@ dependencies = [ "moka", "once_cell", "parking_lot 0.12.3", - "rand 0.9.1", + "rand 0.9.2", "resolv-conf", "smallvec", "thiserror 2.0.12", @@ -9410,9 +9410,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02000660d30638906021176af16b17498bd0d12813dbfe7b276d8bc7f3c0806" +checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" dependencies = [ "jiff-static", "jiff-tzdb", @@ -9420,15 +9420,15 @@ dependencies = [ "log", "portable-atomic", "portable-atomic-util", - "serde", + "serde_core", "windows-sys 0.59.0", ] [[package]] name = "jiff-static" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c30758ddd7188629c6713fc45d1188af4f44c90582311d0c8d8c9907f60c48" +checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" dependencies = [ "proc-macro2", "quote", @@ -9473,7 +9473,7 @@ dependencies = [ [[package]] name = "jsonb" version = "0.5.4" -source = "git+https://github.com/databendlabs/jsonb?rev=425aae9#425aae9cbf567dd7e523d4657e8290268480953c" +source = "git+https://github.com/b41sh/jsonb?rev=f44ac3ede8d576054968993b9a0e27985269b64d#f44ac3ede8d576054968993b9a0e27985269b64d" dependencies = [ "byteorder", "ethnum", @@ -9482,8 +9482,8 @@ dependencies = [ "jiff", "nom 8.0.0", "num-traits", - "ordered-float 5.0.0", - "rand 0.9.1", + "ordered-float 5.1.0", + "rand 0.9.2", "ryu", "serde", "serde_json", @@ -10919,7 +10919,7 @@ dependencies = [ "parking_lot 0.12.3", "percent-encoding", "quick-xml 0.37.5", - "rand 0.9.1", + "rand 0.9.2", "reqwest", "ring", "rustls-pemfile 2.2.0", @@ -11021,7 +11021,7 @@ dependencies = [ "futures", "maplit", "openraft-macros", - "rand 0.9.1", + "rand 0.9.2", "serde", "thiserror 1.0.69", "tokio", @@ -11270,7 +11270,7 @@ dependencies = [ "glob", "opentelemetry 0.29.1", "percent-encoding", - "rand 0.9.1", + "rand 0.9.2", "serde_json", "thiserror 2.0.12", "tokio", @@ -11325,9 +11325,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2c1f9f56e534ac6a9b8a4600bdf0f530fb393b5f393e7b4d03489c3cf0c3f01" +checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" dependencies = [ "num-traits", ] @@ -12588,7 +12588,7 @@ dependencies = [ "bytes", "getrandom 0.3.3", "lru-slab", - "rand 0.9.1", + "rand 0.9.2", "ring", "rustc-hash 2.1.1", "rustls 0.23.27", @@ -12672,9 +12672,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", @@ -13090,7 +13090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4466c6958bbbd3024ace15426f7b5369222c0ad517167d10056fd5cc76aec99b" dependencies = [ "hickory-resolver", - "rand 0.9.1", + "rand 0.9.2", "reqwest", ] @@ -13784,10 +13784,11 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -13828,11 +13829,20 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -15175,7 +15185,7 @@ source = "git+https://github.com/datafuse-extras/tantivy?rev=9065a4d#9065a4de248 dependencies = [ "fnv", "nom 7.1.3", - "ordered-float 5.0.0", + "ordered-float 5.1.0", "serde", "serde_json", ] @@ -16231,7 +16241,7 @@ checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ "getrandom 0.3.3", "js-sys", - "rand 0.9.1", + "rand 0.9.2", "serde", "wasm-bindgen", ] @@ -16374,7 +16384,7 @@ dependencies = [ "nix 0.29.0", "once_cell", "pin-project", - "rand 0.9.1", + "rand 0.9.2", "socket2", "thiserror 2.0.12", "tokio", diff --git a/Cargo.toml b/Cargo.toml index d3ec524ecc2f7..c271207e820d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -296,7 +296,7 @@ enquote = "1.1.0" enum-as-inner = "0.6" enum_dispatch = "0.3.13" enumflags2 = { version = "0.7.7", features = ["serde"] } -ethnum = { version = "1.5.1", features = ["serde", "macros"] } +ethnum = { version = "1.5.2", features = ["serde", "macros"] } faststr = "0.2" feature-set = { version = "0.1.1" } feistel-permutation-rs = "0.1.1" @@ -352,7 +352,7 @@ jaq-core = "1.5.1" jaq-interpret = "1.5.0" jaq-parse = "1.0.3" jaq-std = "1.6.0" -jiff = { version = "0.2.10", features = ["serde", "tzdb-bundle-always"] } +jiff = { version = "0.2.16", features = ["serde", "tzdb-bundle-always"] } jsonb = "0.5.4" jwt-simple = { version = "0.12.10", default-features = false, features = ["pure-rust"] } lenient_semver = "0.4.2" @@ -412,7 +412,7 @@ openraft = { version = "0.10.0", features = [ ] } opensrv-mysql = { git = "https://github.com/databendlabs/opensrv.git", tag = "v0.10.0", features = ["tls"] } orc-rust = "0.6.0" -ordered-float = { version = "5.0.0", default-features = false } +ordered-float = { version = "5.1.0", default-features = false } ordq = "0.2.0" p256 = "0.13" parking_lot = "0.12.1" @@ -651,7 +651,7 @@ async-recursion = { git = "https://github.com/datafuse-extras/async-recursion.gi backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "72265be" } color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" } deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "9954bff" } -jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "425aae9" } +jsonb = { git = "https://github.com/b41sh/jsonb", rev = "f44ac3ede8d576054968993b9a0e27985269b64d" } map-api = { git = "https://github.com/databendlabs/map-api", tag = "v0.4.2" } openraft = { git = "https://github.com/databendlabs/openraft", tag = "v0.10.0-alpha.11" } orc-rust = { git = "https://github.com/datafuse-extras/orc-rust", rev = "d82aa6d" } diff --git a/tests/data/ndjson/numbers.ndjson b/tests/data/ndjson/numbers.ndjson index aecddc3762d07..be43d173ee9a8 100644 --- a/tests/data/ndjson/numbers.ndjson +++ b/tests/data/ndjson/numbers.ndjson @@ -1,3 +1,3 @@ -{"id": 1, "value": 100} -{"id": 2, "value": 200} -{"id": 3, "value": 300} +{"id": 1, "value": 100, "score": 1.0e38} +{"id": 2, "value": 200, "score": -1.0e77} +{"id": 3, "value": 300, "score": 1.0e999999} diff --git a/tests/sqllogictests/suites/stage/formats/ndjson/ndjson_cast.test b/tests/sqllogictests/suites/stage/formats/ndjson/ndjson_cast.test index c286e68577e8c..eb9f98f3093ab 100644 --- a/tests/sqllogictests/suites/stage/formats/ndjson/ndjson_cast.test +++ b/tests/sqllogictests/suites/stage/formats/ndjson/ndjson_cast.test @@ -4,12 +4,12 @@ drop table if exists cast_ndjson statement ok CREATE TABLE cast_ndjson (name String, tags Map(String, String)) -query +query TIBTI copy into cast_ndjson from @data/ndjson/cast_sample.ndjson file_format = (type = NDJSON) ON_ERROR=continue ---- ndjson/cast_sample.ndjson 3 0 NULL NULL -query +query TT select * from cast_ndjson ---- data1 {'env':'test1','length':'ok'} @@ -19,14 +19,29 @@ data3 {'env':'test3','length':'10'} statement ok CREATE OR REPLACE TABLE cast_ts_ndjson (t timestamp) -query +query TIBTI copy into cast_ts_ndjson from @data/ndjson/ts.ndjson file_format = (type = NDJSON) ON_ERROR=continue ---- ndjson/ts.ndjson 3 0 NULL NULL -query +query T select * from cast_ts_ndjson order by t ---- 2025-01-08 03:11:04.000000 2025-01-08 03:11:05.000000 2025-01-08 03:11:06.000000 + +statement ok +CREATE OR REPLACE TABLE number (id Int32, value Int64, score Variant) + +query TIBTI +COPY INTO number FROM @data/ndjson/numbers.ndjson file_format = (type = NDJSON) ON_ERROR=continue +---- +ndjson/numbers.ndjson 3 0 NULL NULL + +query IIT +SELECT * FROM number +---- +1 100 1e38 +2 200 -1e77 +3 300 null diff --git a/tests/sqllogictests/suites/stage/formats/parquet/infer_schema.test b/tests/sqllogictests/suites/stage/formats/parquet/infer_schema.test index 9113d03729c9e..def47b99e62c2 100644 --- a/tests/sqllogictests/suites/stage/formats/parquet/infer_schema.test +++ b/tests/sqllogictests/suites/stage/formats/parquet/infer_schema.test @@ -175,6 +175,7 @@ select * from infer_schema(location => '@data/ndjson/numbers.ndjson', file_forma ---- id BIGINT 1 ndjson/numbers.ndjson 0 value BIGINT 1 ndjson/numbers.ndjson 1 +score DOUBLE 1 ndjson/numbers.ndjson 2 query TTBTI select * from infer_schema(location => '@data/ndjson/ragged.ndjson', file_format => 'NDJSON'); @@ -252,4 +253,4 @@ select CASE from infer_schema(location => '@data/ndjson/max_file_count/', file_format => 'NDJSON', max_file_count => 2); ---- Y -Y \ No newline at end of file +Y From c8606b869a9204ebab1737ecb6499c552051a9f2 Mon Sep 17 00:00:00 2001 From: baishen Date: Thu, 13 Nov 2025 16:39:35 +0800 Subject: [PATCH 2/3] fix --- Cargo.lock | 30 ++++++++++-------------------- Cargo.toml | 4 ++-- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d00eb34e1a5b8..05a7f86bc1fae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9410,9 +9410,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.16" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" +checksum = "f02000660d30638906021176af16b17498bd0d12813dbfe7b276d8bc7f3c0806" dependencies = [ "jiff-static", "jiff-tzdb", @@ -9420,15 +9420,15 @@ dependencies = [ "log", "portable-atomic", "portable-atomic-util", - "serde_core", + "serde", "windows-sys 0.59.0", ] [[package]] name = "jiff-static" -version = "0.2.16" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" +checksum = "f3c30758ddd7188629c6713fc45d1188af4f44c90582311d0c8d8c9907f60c48" dependencies = [ "proc-macro2", "quote", @@ -9473,7 +9473,7 @@ dependencies = [ [[package]] name = "jsonb" version = "0.5.4" -source = "git+https://github.com/b41sh/jsonb?rev=f44ac3ede8d576054968993b9a0e27985269b64d#f44ac3ede8d576054968993b9a0e27985269b64d" +source = "git+https://github.com/b41sh/jsonb?rev=11402b1debafd49faf4ffebb2636058b1050d279#11402b1debafd49faf4ffebb2636058b1050d279" dependencies = [ "byteorder", "ethnum", @@ -13784,11 +13784,10 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.228" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ - "serde_core", "serde_derive", ] @@ -13829,20 +13828,11 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index c271207e820d9..58f5ed03ca448 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -352,7 +352,7 @@ jaq-core = "1.5.1" jaq-interpret = "1.5.0" jaq-parse = "1.0.3" jaq-std = "1.6.0" -jiff = { version = "0.2.16", features = ["serde", "tzdb-bundle-always"] } +jiff = { version = "0.2.10", features = ["serde", "tzdb-bundle-always"] } jsonb = "0.5.4" jwt-simple = { version = "0.12.10", default-features = false, features = ["pure-rust"] } lenient_semver = "0.4.2" @@ -651,7 +651,7 @@ async-recursion = { git = "https://github.com/datafuse-extras/async-recursion.gi backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "72265be" } color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" } deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "9954bff" } -jsonb = { git = "https://github.com/b41sh/jsonb", rev = "f44ac3ede8d576054968993b9a0e27985269b64d" } +jsonb = { git = "https://github.com/b41sh/jsonb", rev = "11402b1debafd49faf4ffebb2636058b1050d279" } map-api = { git = "https://github.com/databendlabs/map-api", tag = "v0.4.2" } openraft = { git = "https://github.com/databendlabs/openraft", tag = "v0.10.0-alpha.11" } orc-rust = { git = "https://github.com/datafuse-extras/orc-rust", rev = "d82aa6d" } From 23dd08f20d5b9366e722129746f91d61736d40fe Mon Sep 17 00:00:00 2001 From: baishen Date: Fri, 14 Nov 2025 11:31:35 +0800 Subject: [PATCH 3/3] bump jsonb v0.5.5 --- Cargo.lock | 5 +++-- Cargo.toml | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05a7f86bc1fae..0c9a5fe5619c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9472,8 +9472,9 @@ dependencies = [ [[package]] name = "jsonb" -version = "0.5.4" -source = "git+https://github.com/b41sh/jsonb?rev=11402b1debafd49faf4ffebb2636058b1050d279#11402b1debafd49faf4ffebb2636058b1050d279" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a901f06163d352fbe41c3c2ff5e08b75330a003cc941e988fb501022f5421e6" dependencies = [ "byteorder", "ethnum", diff --git a/Cargo.toml b/Cargo.toml index 58f5ed03ca448..0e07205297c75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -353,7 +353,7 @@ jaq-interpret = "1.5.0" jaq-parse = "1.0.3" jaq-std = "1.6.0" jiff = { version = "0.2.10", features = ["serde", "tzdb-bundle-always"] } -jsonb = "0.5.4" +jsonb = "0.5.5" jwt-simple = { version = "0.12.10", default-features = false, features = ["pure-rust"] } lenient_semver = "0.4.2" levenshtein_automata = "0.2.1" @@ -651,7 +651,6 @@ async-recursion = { git = "https://github.com/datafuse-extras/async-recursion.gi backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "72265be" } color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" } deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "9954bff" } -jsonb = { git = "https://github.com/b41sh/jsonb", rev = "11402b1debafd49faf4ffebb2636058b1050d279" } map-api = { git = "https://github.com/databendlabs/map-api", tag = "v0.4.2" } openraft = { git = "https://github.com/databendlabs/openraft", tag = "v0.10.0-alpha.11" } orc-rust = { git = "https://github.com/datafuse-extras/orc-rust", rev = "d82aa6d" }