From fa5444c4ad73c33d76fb3008e4f65435cf87a9fe Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Sun, 15 Nov 2026 14:46:16 +0300 Subject: [PATCH] fix(rust): broaden Postgres cell decode and cut stream reallocations (#1979) Cache per-column decode strategy from describe, decode arrays/time/ interval/money/inet/mac/geometry/ranges/bits, and replace batch buffers with capacity instead of take+reserve churn. --- apps/tauri/src-tauri/Cargo.lock | 47 ++ apps/tauri/src-tauri/Cargo.toml | 7 + .../src/connection/postgres_query.rs | 489 +++++++++++++++--- 3 files changed, 462 insertions(+), 81 deletions(-) diff --git a/apps/tauri/src-tauri/Cargo.lock b/apps/tauri/src-tauri/Cargo.lock index b7e43402..fba2763f 100644 --- a/apps/tauri/src-tauri/Cargo.lock +++ b/apps/tauri/src-tauri/Cargo.lock @@ -297,6 +297,12 @@ dependencies = [ "bit-vec 0.10.2", ] +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bit-vec" version = "0.8.0" @@ -2036,6 +2042,15 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +[[package]] +name = "ipnetwork" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" +dependencies = [ + "serde", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -2369,6 +2384,16 @@ version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" +[[package]] +name = "mac_address" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0581a75c45969c63afd2bb87b1d8b25cfcc556c7a918cf9bb13b0fe8d2381" +dependencies = [ + "nix", + "windows-sys 0.61.2", +] + [[package]] name = "markup5ever" version = "0.39.0" @@ -2510,6 +2535,19 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -3208,10 +3246,13 @@ dependencies = [ name = "querya_desktop_tauri" version = "0.1.0" dependencies = [ + "bit-vec 0.6.3", "chrono", "dirs 5.0.1", "futures-util", + "ipnetwork", "keyring", + "mac_address", "rusqlite", "rust_decimal", "serde", @@ -4090,6 +4131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ "base64 0.22.1", + "bit-vec 0.6.3", "bytes", "chrono", "crc", @@ -4103,7 +4145,9 @@ dependencies = [ "hashbrown 0.15.5", "hashlink 0.10.0", "indexmap 2.14.2", + "ipnetwork", "log", + "mac_address", "memchr", "once_cell", "percent-encoding", @@ -4211,6 +4255,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64 0.22.1", + "bit-vec 0.6.3", "bitflags 2.13.2", "byteorder", "chrono", @@ -4224,8 +4269,10 @@ dependencies = [ "hkdf", "hmac", "home", + "ipnetwork", "itoa", "log", + "mac_address", "md-5", "memchr", "once_cell", diff --git a/apps/tauri/src-tauri/Cargo.toml b/apps/tauri/src-tauri/Cargo.toml index 49e5ad29..f2161fd6 100644 --- a/apps/tauri/src-tauri/Cargo.toml +++ b/apps/tauri/src-tauri/Cargo.toml @@ -29,12 +29,19 @@ sqlx = { version = "0.8", default-features = false, features = [ "chrono", "rust_decimal", "uuid", + "ipnetwork", + "mac_address", + "bit-vec", ] } futures-util = "0.3" chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } keyring = { version = "3", features = ["apple-native", "windows-native", "linux-native-sync-persistent"] } sqlparser = "0.64" rust_decimal = { version = "1", features = ["serde"] } +# Pin to sqlx 0.8's dependency versions so Decode impls match. +ipnetwork = "0.20" +mac_address = "1.1" +bit-vec = "0.6.3" [profile.release] codegen-units = 1 diff --git a/apps/tauri/src-tauri/src/connection/postgres_query.rs b/apps/tauri/src-tauri/src/connection/postgres_query.rs index b7b68881..91793120 100644 --- a/apps/tauri/src-tauri/src/connection/postgres_query.rs +++ b/apps/tauri/src-tauri/src/connection/postgres_query.rs @@ -5,10 +5,15 @@ use super::id::DriverKind; use super::postgres::PgPool; use super::query::QueryEvent; use super::runtime; -use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; +use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Utc}; use futures_util::TryStreamExt; +use sqlx::postgres::types::{ + Oid, PgBox, PgCircle, PgInterval, PgLine, PgLSeg, PgMoney, PgPath, PgPoint, PgPolygon, PgRange, + PgTimeTz, +}; +use sqlx::postgres::{PgTypeInfo, PgValueFormat, PgValueRef}; use sqlx::postgres::PgRow; -use sqlx::{Column, Executor, Row, TypeInfo, ValueRef}; +use sqlx::{Column, Decode, Executor, Row, TypeInfo, ValueRef}; use std::time::Instant; const DEFAULT_BATCH: usize = 256; @@ -57,8 +62,12 @@ pub fn execute_streaming( return Ok(()); } + // Resolve decode strategy once per column (not per cell). + let decoders: Vec = columns + .iter() + .map(|c| CellKind::from_type_info(c.type_info())) + .collect(); let names: Vec = columns.iter().map(|c| c.name().to_owned()).collect(); - let col_count = names.len(); emit(QueryEvent::Columns { names })?; let mut stream = sqlx::query(sql).fetch(&mut *conn); @@ -76,17 +85,17 @@ pub fn execute_streaming( truncated = true; break; } - batch.push(pg_row_to_json(&row, col_count)); + batch.push(pg_row_to_json(&row, &decoders)); row_count += 1; if batch.len() >= batch_size { - let rows_out = std::mem::take(&mut batch); + // Replace keeps capacity discipline without take→reserve churn. + let rows_out = std::mem::replace(&mut batch, Vec::with_capacity(batch_size)); let n = rows_out.len() as u64; emit(QueryEvent::Rows { rows: rows_out, offset, })?; offset += n; - batch.reserve(batch_size); } } @@ -107,11 +116,119 @@ pub fn execute_streaming( }) } -fn pg_row_to_json(row: &PgRow, col_count: usize) -> Vec { - (0..col_count).map(|i| pg_cell(row, i)).collect() +#[derive(Clone, Copy, Debug)] +enum CellKind { + Bool, + Int2, + Int4, + Int8, + Float4, + Float8, + Numeric, + Money, + Text, + Json, + Timestamptz, + Timestamp, + Date, + Time, + Timetz, + Interval, + Bytea, + Uuid, + Oid, + Inet, + Macaddr, + Bit, + Point, + Line, + Lseg, + Box, + Path, + Polygon, + Circle, + Int4Range, + Int8Range, + NumRange, + TsRange, + TstzRange, + DateRange, + Int4Array, + Int8Array, + TextArray, + UuidArray, + BoolArray, + JsonbArray, + NumericArray, + Unknown, } -fn pg_cell(row: &PgRow, i: usize) -> serde_json::Value { +impl CellKind { + fn from_type_info(info: &PgTypeInfo) -> Self { + Self::from_name(info.name()) + } + + fn from_name(name: &str) -> Self { + match name { + "BOOL" => Self::Bool, + "INT2" => Self::Int2, + "INT4" => Self::Int4, + "INT8" => Self::Int8, + "FLOAT4" => Self::Float4, + "FLOAT8" => Self::Float8, + "NUMERIC" => Self::Numeric, + "MONEY" => Self::Money, + "TEXT" | "VARCHAR" | "CHAR" | "NAME" | "BPCHAR" | "CITEXT" | "XML" | "xml" => { + Self::Text + } + "JSON" | "JSONB" => Self::Json, + "TIMESTAMPTZ" => Self::Timestamptz, + "TIMESTAMP" => Self::Timestamp, + "DATE" => Self::Date, + "TIME" => Self::Time, + "TIMETZ" => Self::Timetz, + "INTERVAL" => Self::Interval, + "BYTEA" => Self::Bytea, + "UUID" => Self::Uuid, + "OID" => Self::Oid, + "INET" | "CIDR" => Self::Inet, + "MACADDR" => Self::Macaddr, + // MACADDR8 (EUI-64) has no sqlx Decode in 0.8 — falls through to textish. + "BIT" | "VARBIT" => Self::Bit, + "POINT" => Self::Point, + "LINE" => Self::Line, + "LSEG" => Self::Lseg, + "BOX" => Self::Box, + "PATH" => Self::Path, + "POLYGON" => Self::Polygon, + "CIRCLE" => Self::Circle, + "INT4RANGE" => Self::Int4Range, + "INT8RANGE" => Self::Int8Range, + "NUMRANGE" => Self::NumRange, + "TSRANGE" => Self::TsRange, + "TSTZRANGE" => Self::TstzRange, + "DATERANGE" => Self::DateRange, + "INT2[]" | "INT4[]" => Self::Int4Array, + "INT8[]" => Self::Int8Array, + "TEXT[]" | "VARCHAR[]" | "CHAR[]" | "NAME[]" | "BPCHAR[]" => Self::TextArray, + "UUID[]" => Self::UuidArray, + "BOOL[]" => Self::BoolArray, + "JSON[]" | "JSONB[]" => Self::JsonbArray, + "NUMERIC[]" => Self::NumericArray, + _ => Self::Unknown, + } + } +} + +fn pg_row_to_json(row: &PgRow, decoders: &[CellKind]) -> Vec { + let mut out = Vec::with_capacity(decoders.len()); + for (i, kind) in decoders.iter().enumerate() { + out.push(pg_cell(row, i, *kind)); + } + out +} + +fn pg_cell(row: &PgRow, i: usize, kind: CellKind) -> serde_json::Value { let Ok(raw) = row.try_get_raw(i) else { return serde_json::Value::Null; }; @@ -119,90 +236,201 @@ fn pg_cell(row: &PgRow, i: usize) -> serde_json::Value { return serde_json::Value::Null; } - // sqlx matches Decode by exact Postgres type — INT4≠i64, FLOAT4≠f64, etc. - let type_info = raw.type_info(); - let type_name = type_info.name(); - match type_name { - "BOOL" => { - if let Ok(v) = row.try_get::(i) { - return serde_json::Value::Bool(v); - } + match decode_cell(raw.clone(), kind) { + Some(v) => v, + None => { + let type_info = raw.type_info(); + textish_fallback(row, i, type_info.name()) } - "INT2" => { - if let Ok(v) = row.try_get::(i) { - return serde_json::json!(v); - } - } - "INT4" | "OID" => { - if let Ok(v) = row.try_get::(i) { - return serde_json::json!(v); - } - } - "INT8" => { - if let Ok(v) = row.try_get::(i) { - return serde_json::json!(v); - } - } - "FLOAT4" => { - if let Ok(v) = row.try_get::(i) { - return f64_json(f64::from(v)); - } - } - "FLOAT8" => { - if let Ok(v) = row.try_get::(i) { - return f64_json(v); - } - } - "NUMERIC" => { - if let Ok(v) = row.try_get::(i) { - // Keep full precision as string (JSON number can't hold all NUMERIC). - return serde_json::Value::String(v.normalize().to_string()); - } - } - "JSON" | "JSONB" => { - if let Ok(v) = row.try_get::(i) { - return v; - } - } - "TIMESTAMPTZ" => { - if let Ok(v) = row.try_get::, _>(i) { - return serde_json::Value::String(v.to_rfc3339()); - } + } +} + +fn decode_cell(raw: PgValueRef<'_>, kind: CellKind) -> Option { + match kind { + CellKind::Bool => decode::(raw).map(serde_json::Value::Bool), + CellKind::Int2 => decode::(raw).map(|v| json_i64(i64::from(v))), + CellKind::Int4 => decode::(raw).map(|v| json_i64(i64::from(v))), + CellKind::Int8 => decode::(raw).map(json_i64), + CellKind::Float4 => decode::(raw).map(|v| f64_json(f64::from(v))), + CellKind::Float8 => decode::(raw).map(f64_json), + CellKind::Numeric => decode::(raw) + .map(|v| serde_json::Value::String(v.normalize().to_string())), + CellKind::Money => decode::(raw).map(|v| { + serde_json::Value::String(v.to_decimal(2).normalize().to_string()) + }), + CellKind::Text => decode::(raw).map(serde_json::Value::String), + CellKind::Json => decode::(raw), + CellKind::Timestamptz => decode::>(raw) + .map(|v| serde_json::Value::String(v.to_rfc3339())), + CellKind::Timestamp => decode::(raw).map(|v| { + serde_json::Value::String(v.format("%Y-%m-%dT%H:%M:%S%.f").to_string()) + }), + CellKind::Date => { + decode::(raw).map(|v| serde_json::Value::String(v.to_string())) } - "TIMESTAMP" => { - if let Ok(v) = row.try_get::(i) { - return serde_json::Value::String( - v.format("%Y-%m-%dT%H:%M:%S%.f").to_string(), - ); - } + CellKind::Time => { + decode::(raw).map(|v| serde_json::Value::String(v.to_string())) } - "DATE" => { - if let Ok(v) = row.try_get::(i) { - return serde_json::Value::String(v.to_string()); - } + CellKind::Timetz => decode::>(raw).map(|v| { + serde_json::Value::String(format!("{}{}", v.time, v.offset)) + }), + CellKind::Interval => decode::(raw).map(|v| { + serde_json::Value::String(format_interval(&v)) + }), + CellKind::Bytea => decode::>(raw).map(|v| serde_json::Value::String(bytes_to_hex(&v))), + CellKind::Uuid => { + decode::(raw).map(|v| serde_json::Value::String(v.to_string())) } - "BYTEA" => { - if let Ok(v) = row.try_get::, _>(i) { - return serde_json::Value::String(bytes_to_hex(&v)); + CellKind::Oid => decode::(raw).map(|v| json_i64(i64::from(v.0))), + CellKind::Inet => decode::(raw) + .map(|v| serde_json::Value::String(v.to_string())), + CellKind::Macaddr => decode::(raw) + .map(|v| serde_json::Value::String(v.to_string())), + CellKind::Bit => decode::(raw).map(|v| { + let mut s = String::with_capacity(v.len()); + for bit in v.iter() { + s.push(if bit { '1' } else { '0' }); } - } - "UUID" => { - if let Ok(v) = row.try_get::(i) { - return serde_json::Value::String(v.to_string()); - } - } - _ => {} + serde_json::Value::String(s) + }), + CellKind::Point => decode::(raw).map(|v| { + serde_json::Value::String(format!("({},{})", v.x, v.y)) + }), + CellKind::Line => decode::(raw).map(|v| { + serde_json::Value::String(format!("{{{},{},{}}}", v.a, v.b, v.c)) + }), + CellKind::Lseg => decode::(raw).map(|v| { + serde_json::Value::String(format!( + "[({},{}),({},{})]", + v.start_x, v.start_y, v.end_x, v.end_y + )) + }), + CellKind::Box => decode::(raw).map(|v| { + serde_json::Value::String(format!( + "({},{}),({},{})", + v.upper_right_x, v.upper_right_y, v.lower_left_x, v.lower_left_y + )) + }), + CellKind::Path => decode::(raw).map(|v| { + let body = format_points(&v.points); + serde_json::Value::String(if v.closed { + format!("({body})") + } else { + format!("[{body}]") + }) + }), + CellKind::Polygon => decode::(raw).map(|v| { + serde_json::Value::String(format!("(({}))", format_points(&v.points))) + }), + CellKind::Circle => decode::(raw).map(|v| { + serde_json::Value::String(format!("<({},{}),{}>", v.x, v.y, v.radius)) + }), + CellKind::Int4Range => decode::>(raw) + .map(|v| serde_json::Value::String(v.to_string())), + CellKind::Int8Range => decode::>(raw) + .map(|v| serde_json::Value::String(v.to_string())), + CellKind::NumRange => decode::>(raw) + .map(|v| serde_json::Value::String(v.to_string())), + CellKind::TsRange => decode::>(raw) + .map(|v| serde_json::Value::String(v.to_string())), + CellKind::TstzRange => decode::>>(raw) + .map(|v| serde_json::Value::String(v.to_string())), + CellKind::DateRange => decode::>(raw) + .map(|v| serde_json::Value::String(v.to_string())), + CellKind::Int4Array => decode::>(raw).map(|v| { + serde_json::Value::Array(v.into_iter().map(|n| json_i64(i64::from(n))).collect()) + }), + CellKind::Int8Array => decode::>(raw).map(|v| { + serde_json::Value::Array(v.into_iter().map(json_i64).collect()) + }), + CellKind::TextArray => decode::>(raw).map(|v| { + serde_json::Value::Array(v.into_iter().map(serde_json::Value::String).collect()) + }), + CellKind::UuidArray => decode::>(raw).map(|v| { + serde_json::Value::Array( + v.into_iter() + .map(|u| serde_json::Value::String(u.to_string())) + .collect(), + ) + }), + CellKind::BoolArray => decode::>(raw).map(|v| { + serde_json::Value::Array(v.into_iter().map(serde_json::Value::Bool).collect()) + }), + CellKind::JsonbArray => decode::>(raw).map(serde_json::Value::Array), + CellKind::NumericArray => decode::>(raw).map(|v| { + serde_json::Value::Array( + v.into_iter() + .map(|n| serde_json::Value::String(n.normalize().to_string())) + .collect(), + ) + }), + CellKind::Unknown => None, } +} + +fn decode<'r, T>(raw: PgValueRef<'r>) -> Option +where + T: Decode<'r, sqlx::Postgres>, +{ + T::decode(raw).ok() +} +fn textish_fallback(row: &PgRow, i: usize, type_name: &str) -> serde_json::Value { if let Ok(v) = row.try_get::(i) { return serde_json::Value::String(v); } + if let Ok(raw) = row.try_get_raw(i) { + if let Ok(s) = raw_as_utf8(raw) { + return serde_json::Value::String(s); + } + } if let Ok(v) = row.try_get::, _>(i) { return serde_json::Value::String(bytes_to_hex(&v)); } serde_json::Value::String(format!("<{type_name}>")) } +fn raw_as_utf8(raw: PgValueRef<'_>) -> Result { + match raw.format() { + PgValueFormat::Text => raw.as_str().map(|s| s.to_owned()).map_err(|_| ()), + PgValueFormat::Binary => { + let bytes = raw.as_bytes().map_err(|_| ())?; + std::str::from_utf8(bytes) + .map(|s| s.to_owned()) + .map_err(|_| ()) + } + } +} + +fn format_points(points: &[PgPoint]) -> String { + let mut out = String::with_capacity(points.len().saturating_mul(16)); + for (i, p) in points.iter().enumerate() { + if i > 0 { + out.push(','); + } + use std::fmt::Write; + let _ = write!(out, "({},{})", p.x, p.y); + } + out +} + +fn format_interval(v: &PgInterval) -> String { + let secs = v.microseconds / 1_000_000; + let micros = (v.microseconds % 1_000_000).abs(); + if micros == 0 { + format!("{} mon {} days {} seconds", v.months, v.days, secs) + } else { + format!( + "{} mon {} days {}.{:06} seconds", + v.months, v.days, secs, micros + ) + } +} + +fn json_i64(v: i64) -> serde_json::Value { + serde_json::Value::Number(v.into()) +} + fn f64_json(v: f64) -> serde_json::Value { serde_json::Number::from_f64(v) .map(serde_json::Value::Number) @@ -295,7 +523,6 @@ mod tests { .sum(); assert_eq!(rows, 5); - // INT4 must decode to numbers — not the `` fallback placeholder. let first_id = events.iter().find_map(|e| match e { QueryEvent::Rows { rows, .. } => rows.first().and_then(|r| r.first()), _ => None, @@ -343,11 +570,111 @@ mod tests { matches!(&cells[1], serde_json::Value::String(s) if !s.starts_with('<')), "total NUMERIC: {cells:?}" ); + session.disconnect().unwrap(); + } + + #[test] + fn live_postgres_decodes_text_array_tags() { + let Some(params) = live_params() else { + return; + }; + let session = match open_postgres(params) { + Ok(s) => s, + Err(_) => return, + }; + let mut events = Vec::new(); + session + .execute_streaming( + "SELECT tags FROM shop.products WHERE tags IS NOT NULL LIMIT 1", + Some(1), + |ev| { + events.push(ev); + Ok(()) + }, + ) + .expect("stream"); + let cell = events.iter().find_map(|e| match e { + QueryEvent::Rows { rows, .. } => rows.first().and_then(|r| r.first()).cloned(), + _ => None, + }); + match cell { + Some(serde_json::Value::Array(items)) => { + assert!(!items.is_empty()); + assert!(items.iter().all(|v| v.is_string())); + } + other => panic!("expected TEXT[] as JSON array, got {other:?}"), + } + session.disconnect().unwrap(); + } + + #[test] + fn live_postgres_showcase_has_no_angle_placeholders() { + let Some(params) = live_params() else { + return; + }; + let session = match open_postgres(params) { + Ok(s) => s, + Err(_) => return, + }; + let tables = [ + "all_numerics", + "all_datetime", + "all_identifiers_and_network", + "all_binary_and_bits", + "all_geometric", + "all_ranges", + "all_arrays", + "all_strings", + "all_json_xml", + ]; + let mut placeholders = Vec::new(); + for table in tables { + let sql = format!("SELECT * FROM types_showcase.{table} LIMIT 1"); + let mut events = Vec::new(); + if session + .execute_streaming(&sql, Some(1), |ev| { + events.push(ev); + Ok(()) + }) + .is_err() + { + continue; + } + let cols = events.iter().find_map(|e| match e { + QueryEvent::Columns { names } => Some(names.clone()), + _ => None, + }); + let row = events.iter().find_map(|e| match e { + QueryEvent::Rows { rows, .. } => rows.first().cloned(), + _ => None, + }); + let (Some(cols), Some(cells)) = (cols, row) else { + continue; + }; + for (name, cell) in cols.iter().zip(cells.iter()) { + // Multiranges are still unsupported — allow only those. + if name.contains("multirange") { + continue; + } + if let serde_json::Value::String(s) = cell { + if s.starts_with('<') && s.ends_with('>') && !s.contains('(') { + // geometric circle uses <(x,y),r> — not a type placeholder + if s.chars().nth(1).is_some_and(|c| c.is_ascii_uppercase()) + || s.chars().nth(1) == Some('_') + || name.contains("multirange") + { + placeholders.push(format!("{table}.{name}={s}")); + } + } + } + } + } + session.disconnect().ok(); assert!( - !matches!(&cells[1], serde_json::Value::String(s) if s == ""), - "total must not be type placeholder: {cells:?}" + placeholders.is_empty(), + "unexpected type placeholders:\n{}", + placeholders.join("\n") ); - session.disconnect().unwrap(); } #[test]