From 1d8fc344bbfa994c26756a1452e8dad37be9109a Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 13 Sep 2026 23:05:22 -0300 Subject: [PATCH 01/56] fix(bcp): handle 3-part table names with -d option in SQL Server --- core/dbio/database/database_sqlserver.go | 27 +++++++- .../r.125.sqlserver_bcp_3part.yaml | 62 +++++++++++++++++++ tests/suite.cli.yaml | 15 +++++ 3 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 tests/replications/r.125.sqlserver_bcp_3part.yaml diff --git a/core/dbio/database/database_sqlserver.go b/core/dbio/database/database_sqlserver.go index 9d1abf546..a0ca34c72 100755 --- a/core/dbio/database/database_sqlserver.go +++ b/core/dbio/database/database_sqlserver.go @@ -1068,11 +1068,32 @@ func (conn *MsSQLServerConn) BcpImportFile(tableFName, filePath string) (count u defer os.Remove(errPath) } + // bcp rejects `-d` together with a 3-part db.schema.table name + // ("The -d database name option is not supported when a 3 part + // dbtable name is specified"). FullName() includes the database + // for SQL Server; keep `-d` and pass schema.table, matching 1.5.22. + // ParseTableName, not strings.Split: a quoted identifier can contain ".". + tableArg := strings.ReplaceAll(tableFName, `"`, "") + if table, perr := ParseTableName(tableFName, conn.GetType()); perr == nil && !table.IsQuery() && table.Name != "" { + if table.Database != "" { + if database != "" && strings.EqualFold(table.Database, database) { + table.Database = "" + } else { + database = "" + } + } + tableArg = strings.ReplaceAll(table.FullName(), `"`, "") + } + bcpArgs := []string{ - strings.ReplaceAll(tableFName, `"`, ""), + tableArg, "in", filePath, "-S", hostPort, - "-d", database, + } + if database != "" { + bcpArgs = append(bcpArgs, "-d", database) + } + bcpArgs = append(bcpArgs, "-t", ",", "-m", "1", "-w", @@ -1080,7 +1101,7 @@ func (conn *MsSQLServerConn) BcpImportFile(tableFName, filePath string) (count u "-b", cast.ToString(batchSize), "-F", "2", "-e", errPath, - } + ) if bcpAuthString := conn.GetProp("bcp_auth_string"); bcpAuthString != "" { bcpAuthParts := []string{} diff --git a/tests/replications/r.125.sqlserver_bcp_3part.yaml b/tests/replications/r.125.sqlserver_bcp_3part.yaml new file mode 100644 index 000000000..ba3375c5e --- /dev/null +++ b/tests/replications/r.125.sqlserver_bcp_3part.yaml @@ -0,0 +1,62 @@ +# SQL Server BCP rejects a 3-part table name when -d is also passed. +# Table.FullName() prefixes the connection database (SupportsThreePartName). +# BCP then gets both `master.dbo.table` and `-d master`, which it rejects: +# "The -d database name option is not supported when a 3 part dbtable name is specified". +# Fix: if part 0 is the connection database, pass schema.table and keep -d. + +source: LOCAL +target: MSSQL + +defaults: + mode: full-refresh + target_options: + use_bulk: true + +hooks: + start: + - type: query + connection: '{target.name}' + query: | + IF OBJECT_ID('master.dbo.bcp_3part','U') IS NOT NULL DROP TABLE master.dbo.bcp_3part; + IF OBJECT_ID('master.dbo.bcp_3part_tmp','U') IS NOT NULL DROP TABLE master.dbo.bcp_3part_tmp; + + - type: write + content: | + id,name + 1,alice + 2,bob + to: file:///tmp/r.125.sqlserver_bcp_3part.csv + + end: + - type: check + check: execution.status.error == 0 + on_failure: break + + - type: query + connection: '{target.name}' + query: select id, name from master.dbo.bcp_3part order by id + into: result + + - type: check + check: int_parse(store.result[0].id) == 1 + failure_message: "expected first row id 1, got {store.result}" + + - type: check + check: store.result[0].name == "alice" + failure_message: "expected first row name alice, got {store.result}" + + - type: check + check: int_parse(store.result[1].id) == 2 + failure_message: "expected 2 rows loaded via bcp, got {store.result}" + + - type: query + connection: '{target.name}' + query: | + IF OBJECT_ID('master.dbo.bcp_3part','U') IS NOT NULL DROP TABLE master.dbo.bcp_3part; + + - type: log + message: "SUCCESS: SQL Server BCP loaded a 3-part target name" + +streams: + file:///tmp/r.125.sqlserver_bcp_3part.csv: + object: master.dbo.bcp_3part diff --git a/tests/suite.cli.yaml b/tests/suite.cli.yaml index 29c5f8f41..064060e08 100644 --- a/tests/suite.cli.yaml +++ b/tests/suite.cli.yaml @@ -2940,6 +2940,21 @@ - 'STEP 1 OK - snapshot loaded 2 rows' - 'CDC UUID test PASSED' +# SQL Server BCP rejects `-d` together with a 3-part db.schema.table name. +# FullName() prefixes the connection database; BCP also always passed `-d`. +# Same failure as the AEG NCC bulk load (bcp: "The -d database name option +# is not supported when a 3 part dbtable name is specified"). +- id: 602 + name: 'SQL Server BCP 3-part name with -d' + group: mssql + run: 'sling run -d -r tests/replications/r.125.sqlserver_bcp_3part.yaml' + rows: 2 + output_contains: + - 'execution succeeded' + - 'SUCCESS: SQL Server BCP loaded a 3-part target name' + output_does_not_contain: + - 'The -d database name option is not supported when a 3 part' + # Nested CLI suites. Loader in sling_cli_test.go merges these in. - suite: suite.cli.assist.yaml - suite: suite.cli.build.yaml From 849f986b22d57f142be27da851c6422ad96dfd7f Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Mon, 14 Sep 2026 08:11:53 -0300 Subject: [PATCH 02/56] fix(docs): update conn_property function signature and increase run_max_events in CDC documentation --- core/sling/assist/skills/sling-api-specs/FUNCTIONS.md | 2 +- core/sling/assist/skills/sling-replications/CDC.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/sling/assist/skills/sling-api-specs/FUNCTIONS.md b/core/sling/assist/skills/sling-api-specs/FUNCTIONS.md index 5f4391020..afaa381d6 100644 --- a/core/sling/assist/skills/sling-api-specs/FUNCTIONS.md +++ b/core/sling/assist/skills/sling-api-specs/FUNCTIONS.md @@ -171,7 +171,7 @@ chunk(queue.ids, 50) | `type_of(value)` | Runtime type name | `type_of(42)` → "integer" | | `parse_ms_uuid(string)` | Parse MS UUID timestamp | Extracts time from MS-style UUID | | `pretty_table(rows)` | Format rows as table | Debug / log helper | -| `conn_property(name)` | Connection property | Reads from active connection | +| `conn_property(connection, key)` | Connection property | `conn_property("my_db", "host")` → host value | | `machine_stats()` | Host stats object | Runtime diagnostics | ## Common Patterns diff --git a/core/sling/assist/skills/sling-replications/CDC.md b/core/sling/assist/skills/sling-replications/CDC.md index db55f38d2..06f80f757 100644 --- a/core/sling/assist/skills/sling-replications/CDC.md +++ b/core/sling/assist/skills/sling-replications/CDC.md @@ -36,7 +36,7 @@ defaults: primary_key: [id] object: public.{stream_table} change_capture_options: - run_max_events: 10000 + run_max_events: 100000 run_max_duration: 10m streams: @@ -50,11 +50,11 @@ streams: | Option | Default | Description | |--------|---------|-------------| -| `run_max_events` | `10000` | Max change events per run, then save position and exit. One event = one log statement, so it can hold many rows. | +| `run_max_events` | `100000` | Max change events per run, then save position and exit. One event = one log statement, so it can hold many rows. | | `run_max_duration` | `10m` | Max wall-clock time per run. | | `soft_delete` | `false` | Mark deletes with `_sling_synced_op='D'` instead of row removal. | | `snapshot_start` | `now` | First-run log start: `now` or `beginning`. | -| `snapshot_chunk_size` | `100000` | Rows per chunk in the initial snapshot (needs integer-like PK; else single-shot read). | +| `snapshot_chunk_size` | `100000` | Rows per chunk in the initial snapshot. Integer PKs use ranges; string/UUID PKs use keyset. No PK, chunk size 0, or MongoDB → single-shot. | | `snapshot_run_duration` | none | Time budget for the snapshot per run; resumes next run. | | `replay_from` | — | Rewind position (timestamp, binlog position, GTID). Applied once per unique value. | | `slot_level` | `shared`* | `shared` = one log reader for all streams (Postgres, MySQL); `stream` = one per table (all others). | From 93f1e8f48ffbedc08d144427848ce3d03148a506 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 16 Sep 2026 14:35:24 -0300 Subject: [PATCH 03/56] fix(mongo): parse shell syntax and Extended JSON in filters Incremental/backfill templates render values as mongosh syntax (e.g. ISODate("...")) which the Go driver cannot parse. Add normalizeFilterValue to strip shell constructors (ISODate/ObjectId), trim quotes, and route _id fields through ObjectId conversion. Also add parseExtendedJSON to convert single-key Extended JSON wrappers ($date, $oid, $numberLong, $numberInt, $numberDouble, $numberDecimal) into native BSON values, and normalize map[any]any filters to map[string]any before processing so wrappers are detected consistently. --- core/dbio/database/database_mongo.go | 123 +++++++++++++-- tests/pipelines/p.53.mongo_date_filters.yaml | 149 +++++++++++++++++++ tests/suite.cli.yaml | 14 ++ 3 files changed, 277 insertions(+), 9 deletions(-) create mode 100644 tests/pipelines/p.53.mongo_date_filters.yaml diff --git a/core/dbio/database/database_mongo.go b/core/dbio/database/database_mongo.go index 17e5f5336..cfc0a4d55 100644 --- a/core/dbio/database/database_mongo.go +++ b/core/dbio/database/database_mongo.go @@ -3,6 +3,7 @@ package database import ( "context" "database/sql" + "regexp" "strings" "time" @@ -18,6 +19,10 @@ import ( "go.mongodb.org/mongo-driver/mongo/readpref" ) +// matches ISODate("...") / ObjectId("..."). +// The inner `\(?` tolerates the doubled paren in the date_layout_str template. +var shellCtorRegex = regexp.MustCompile(`^(ISODate|ObjectId)\(\(?"([^"]*)"\)\)?$`) + // MongoDBConn is a Mongo connection type MongoDBConn struct { BaseConn @@ -182,6 +187,10 @@ func (conn *MongoDBConn) BulkExportFlow(table Table) (df *iop.Dataflow, err erro func (conn *MongoDBConn) processMongoFilter(filter any) any { switch v := filter.(type) { case map[string]any: + if val, ok := parseExtendedJSON(v); ok { + return val + } + // Process map recursively result := make(map[string]any) for key, val := range v { @@ -194,10 +203,17 @@ func (conn *MongoDBConn) processMongoFilter(filter any) any { } return result case map[any]any: + normalized := make(map[string]any, len(v)) + for key, val := range v { + normalized[cast.ToString(key)] = val + } + if val, ok := parseExtendedJSON(normalized); ok { + return val + } + // Process map recursively result := make(map[string]any) - for key, val := range v { - keyStr := cast.ToString(key) + for keyStr, val := range normalized { if keyStr == "_id" || strings.HasSuffix(keyStr, "_id") { result[keyStr] = conn.processObjectIDValue(val) } else { @@ -230,6 +246,23 @@ func (conn *MongoDBConn) processMongoFilter(filter any) any { } } +// normalizeFilterValue converts a template-rendered incremental/backfill value +// into a real BSON value. The templates wrap datetimes as ISODate(""), +// which is mongosh syntax the driver does not understand. +func (conn *MongoDBConn) normalizeFilterValue(key, value string) any { + value = strings.Trim(value, "'") + + if m := shellCtorRegex.FindStringSubmatch(value); m != nil { + value = m[2] + } + + if key == "_id" || strings.HasSuffix(key, "_id") { + return conn.processObjectIDValue(value) + } + + return conn.processMongoFilter(value) +} + // processObjectIDValue handles ObjectID conversion for _id fields func (conn *MongoDBConn) processObjectIDValue(val any) any { switch v := val.(type) { @@ -249,6 +282,10 @@ func (conn *MongoDBConn) processObjectIDValue(val any) any { } return v case map[string]any: + if val, ok := parseExtendedJSON(v); ok { + return val + } + // Handle operators like $gte, $lt result := make(map[string]any) for op, opVal := range v { @@ -260,10 +297,17 @@ func (conn *MongoDBConn) processObjectIDValue(val any) any { } return result case map[any]any: + normalized := make(map[string]any, len(v)) + for op, opVal := range v { + normalized[cast.ToString(op)] = opVal + } + if val, ok := parseExtendedJSON(normalized); ok { + return val + } + // Handle operators like $gte, $lt result := make(map[string]any) - for op, opVal := range v { - opStr := cast.ToString(op) + for opStr, opVal := range normalized { if strings.HasPrefix(opStr, "$") { result[opStr] = conn.processObjectIDValue(opVal) } else { @@ -276,6 +320,63 @@ func (conn *MongoDBConn) processObjectIDValue(val any) any { } } +// parseExtendedJSON converts an Extended JSON wrapper to its BSON value, +// e.g. {"$date": "2019-06-01T00:00:00Z"} => time.Time. Returns false if the +// map is not a single-key wrapper of a supported type. +func parseExtendedJSON(m map[string]any) (any, bool) { + if len(m) != 1 { + return nil, false + } + + for key, val := range m { + switch key { + case "$date": + switch v := val.(type) { + case string: + if t, ok := parseISODateString(v); ok { + return t, true + } + case map[string]any: + if inner, ok := v["$numberLong"]; ok && len(v) == 1 { + if millis, err := cast.ToInt64E(inner); err == nil { + return time.UnixMilli(millis).UTC(), true + } + } + default: // epoch millis + if millis, err := cast.ToInt64E(val); err == nil { + return time.UnixMilli(millis).UTC(), true + } + } + case "$oid": + if s, ok := val.(string); ok { + if oid, err := primitive.ObjectIDFromHex(s); err == nil { + return oid, true + } + } + case "$numberLong": + if i, err := cast.ToInt64E(val); err == nil { + return i, true + } + case "$numberInt": + if i, err := cast.ToInt32E(val); err == nil { + return i, true + } + case "$numberDouble": + if f, err := cast.ToFloat64E(val); err == nil { + return f, true + } + case "$numberDecimal": + if s, ok := val.(string); ok { + if d, err := primitive.ParseDecimal128(s); err == nil { + return d, true + } + } + } + } + + return nil, false +} + // parseISODateString attempts to parse an ISO 8601 datetime string. // Returns the parsed time and true if successful, zero time and false otherwise. func parseISODateString(s string) (time.Time, bool) { @@ -363,15 +464,19 @@ func (conn *MongoDBConn) StreamRowsContext(ctx context.Context, collectionName s } } - // Add incremental/backfill filters if specified + // Add incremental/backfill filters if specified. Values arrive as + // template-rendered strings, so normalize them to real BSON values. + // Otherwise a Date field is compared against a String and matches nothing. if updateKey != "" && incrementalValue != "" { // incremental mode - incrementalValue = strings.Trim(incrementalValue, "'") - filter = append(filter, bson.E{Key: updateKey, Value: bson.D{{Key: "$gt", Value: incrementalValue}}}) + value := conn.normalizeFilterValue(updateKey, incrementalValue) + filter = append(filter, bson.E{Key: updateKey, Value: bson.D{{Key: "$gt", Value: value}}}) } else if updateKey != "" && startValue != "" && endValue != "" { // backfill mode - filter = append(filter, bson.E{Key: updateKey, Value: bson.D{{Key: "$gte", Value: startValue}}}) - filter = append(filter, bson.E{Key: updateKey, Value: bson.D{{Key: "$lte", Value: endValue}}}) + start := conn.normalizeFilterValue(updateKey, startValue) + end := conn.normalizeFilterValue(updateKey, endValue) + filter = append(filter, bson.E{Key: updateKey, Value: bson.D{{Key: "$gte", Value: start}}}) + filter = append(filter, bson.E{Key: updateKey, Value: bson.D{{Key: "$lte", Value: end}}}) } if strings.TrimSpace(collectionName) == "" { diff --git a/tests/pipelines/p.53.mongo_date_filters.yaml b/tests/pipelines/p.53.mongo_date_filters.yaml new file mode 100644 index 000000000..c188d5c9c --- /dev/null +++ b/tests/pipelines/p.53.mongo_date_filters.yaml @@ -0,0 +1,149 @@ +# Issue #802 - MongoDB date filters, both bugs fixed in database_mongo.go: +# bug 1: incremental checkpoint was sent as the string ISODate("..."), not a Date +# bug 2: `where` did not interpret Extended JSON {"$date": "..."} +# +# Filters the `date` field, a genuine BSON Date. (`update_dt` in the same +# collection is a STRING, so it cannot exercise Date comparison.) +# 448 of the 1000 documents have date >= 2019-06-01. +# +# The two controls run first: they pass before and after the fix, so a failure +# there means a broken fixture rather than a regression. + +steps: + - connection: mysql + on_failure: warn + query: drop table if exists mysql.mongo_date_filter_test + + # ---------- control 1: ISODate() via `where` ---------- + - replication: + source: mongo + target: mysql + defaults: + mode: full-refresh + source_options: + flatten: 1 + streams: + default.test1k_mongodb: + object: mysql.mongo_date_filter_test + select: [id, date] + where: '{"date": {"$gte": ISODate("2019-06-01T00:00:00.000Z")}}' + env: + SLING_RETRIES: 0 + on_failure: abort + + - connection: mysql + query: select count(*) as cnt from mysql.mongo_date_filter_test + into: ctrl1 + + - log: 'control 1 - where + ISODate() => {store.ctrl1[0].cnt} rows (expect 448)' + + - check: int_parse(store.ctrl1[0].cnt) == 448 + failure_message: 'CONTROL FAILED: where + ISODate() gave {store.ctrl1[0].cnt}, expected 448. Fixture or connection is broken, not the bug.' + + # ---------- control 2: plain ISO string via `where` ---------- + - replication: + source: mongo + target: mysql + defaults: + mode: full-refresh + source_options: + flatten: 1 + streams: + default.test1k_mongodb: + object: mysql.mongo_date_filter_test + select: [id, date] + where: '{"date": {"$gte": "2019-06-01T00:00:00.000Z"}}' + env: + SLING_RETRIES: 0 + on_failure: abort + + - connection: mysql + query: select count(*) as cnt from mysql.mongo_date_filter_test + into: ctrl2 + + - log: 'control 2 - where + plain ISO string => {store.ctrl2[0].cnt} rows (expect 448)' + + - check: int_parse(store.ctrl2[0].cnt) == 448 + failure_message: 'CONTROL FAILED: where + plain ISO string gave {store.ctrl2[0].cnt}, expected 448.' + + # ---------- bug 2: Extended JSON {"$date": ...} via `where` ---------- + - replication: + source: mongo + target: mysql + defaults: + mode: full-refresh + source_options: + flatten: 1 + streams: + default.test1k_mongodb: + object: mysql.mongo_date_filter_test + select: [id, date] + where: '{"date": {"$gte": {"$date": "2019-06-01T00:00:00.000Z"}}}' + env: + SLING_RETRIES: 0 + on_failure: abort + + - connection: mysql + query: select count(*) as cnt from mysql.mongo_date_filter_test + into: bug2 + + - log: 'bug 2 - where + Extended JSON $date => {store.bug2[0].cnt} rows (expect 448)' + + # Same date, operator and field as control 1, which returned 448. + - check: int_parse(store.bug2[0].cnt) == 448 + failure_message: 'ISSUE 802 BUG 2 REGRESSED: Extended JSON $date returned {store.bug2[0].cnt} rows, expected 448.' + + - log: 'OK bug 2 fixed: Extended JSON $date matched all 448 documents' + + # ---------- bug 1: incremental checkpoint ---------- + # Seed a checkpoint of 2019-06-01. Row id=-1 is excluded from the count below. + - connection: mysql + query: drop table if exists mysql.mongo_date_filter_test + + - connection: mysql + query: | + create table mysql.mongo_date_filter_test ( + id bigint, + `date` datetime(6) + ) + + - connection: mysql + query: | + insert into mysql.mongo_date_filter_test (id, `date`) + values (-1, '2019-06-01 00:00:00.000000') + + - replication: + source: mongo + target: mysql + defaults: + mode: incremental + primary_key: [id] + update_key: date + source_options: + flatten: 1 + streams: + default.test1k_mongodb: + object: mysql.mongo_date_filter_test + select: [id, date] + env: + SLING_RETRIES: 0 + on_failure: abort + + - connection: mysql + query: select count(*) as cnt from mysql.mongo_date_filter_test where id > 0 + into: bug1 + + - log: 'bug 1 - incremental checkpoint => {store.bug1[0].cnt} rows (expect 445)' + + # Control 1 used $gte and matched 448. Incremental uses $gt, and exactly 3 + # documents sit on the 2019-06-01T00:00:00Z boundary, so 445 is correct. + - check: int_parse(store.bug1[0].cnt) == 445 + failure_message: 'ISSUE 802 BUG 1 REGRESSED: incremental loaded {store.bug1[0].cnt} rows, expected 445.' + + - log: 'OK bug 1 fixed: incremental checkpoint matched 445 documents ($gt of 448)' + + - log: 'SUCCESS: both issue 802 MongoDB date filter bugs are fixed' + + - connection: mysql + on_failure: warn + query: drop table if exists mysql.mongo_date_filter_test diff --git a/tests/suite.cli.yaml b/tests/suite.cli.yaml index 064060e08..d14402b72 100644 --- a/tests/suite.cli.yaml +++ b/tests/suite.cli.yaml @@ -2955,6 +2955,20 @@ output_does_not_contain: - 'The -d database name option is not supported when a 3 part' +# MongoDB date filters, issue #802: the incremental checkpoint was sent as the +# string ISODate("...") rather than a Date, and `where` ignored Extended JSON +# {"$date": "..."}. Both fixed in database_mongo.go. +- id: 603 + name: 'MongoDB date filters: incremental ISODate() and where $date (issue 802)' + group: mongodb + run: 'sling run -p tests/pipelines/p.53.mongo_date_filters.yaml' + output_contains: + - 'control 1 - where + ISODate() => 448 rows' + - 'control 2 - where + plain ISO string => 448 rows' + - 'OK bug 2 fixed: Extended JSON $date matched all 448 documents' + - 'OK bug 1 fixed: incremental checkpoint matched 445 documents' + - 'SUCCESS: both issue 802 MongoDB date filter bugs are fixed' + # Nested CLI suites. Loader in sling_cli_test.go merges these in. - suite: suite.cli.assist.yaml - suite: suite.cli.build.yaml From cd86e850048e2486b9ebd6be9da9b4227c9ca90c Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 16 Sep 2026 16:48:59 -0300 Subject: [PATCH 04/56] feat(env): node-level env.yaml writes + validated GUI connection helpers Phase 1 of the connection-management UI: - EnvFile.SetConnectionNode/DeleteConnectionNode splice only the touched entry in the raw node tree, so expanded ${VAR} values can never be written and comments/order/unmanaged keys survive. - DeleteConnectionNode keeps a block-trailing comment when the last connection is removed (reattaches it to the preceding entry). - EnvFile.ConnectionNames/EnvKeys/RawConnections/ParseEnvFileConnections parse without interpolation; ValidateKey/ValidateEnvKey/ExpandRef. - EnvFileConns.Get/SetValidated with SetOptions (overwrite guard, literal secret refusal, atomic env: updates); PromoteLiteralSecrets and PreserveRefs; EnvVarNameOf sanitizes dots/dashes in env var names. --- core/dbio/connection/connection_local.go | 146 +++++- core/dbio/connection/connection_local_test.go | 382 +++++++++++++++ core/dbio/connection/connection_props.go | 98 +++- core/env/envfile.go | 447 ++++++++++++++++++ core/env/envfile_test.go | 422 +++++++++++++++++ 5 files changed, 1487 insertions(+), 8 deletions(-) create mode 100644 core/dbio/connection/connection_local_test.go diff --git a/core/dbio/connection/connection_local.go b/core/dbio/connection/connection_local.go index 825ef04eb..979f03e6a 100644 --- a/core/dbio/connection/connection_local.go +++ b/core/dbio/connection/connection_local.go @@ -3,11 +3,6 @@ package connection import ( "encoding/base64" "encoding/json" - "os" - "sort" - "strings" - "time" - "github.com/flarco/g" cmap "github.com/orcaman/concurrent-map/v2" "github.com/samber/lo" @@ -19,6 +14,10 @@ import ( "github.com/slingdata-io/sling-cli/core/env" "github.com/spf13/cast" "gopkg.in/yaml.v2" + "os" + "sort" + "strings" + "time" ) type ConnEntry struct { @@ -301,6 +300,143 @@ type EnvFileConns struct { EnvFile *env.EnvFile } +// SetOptions controls SetValidated. +type SetOptions struct { + // RejectLiteralSecrets refuses secret fields (and nested secrets values) + // that are not ${VAR} refs. The GUI path promotes literals first + // (PromoteLiteralSecrets) and then sets this as a backstop. + RejectLiteralSecrets bool + // AllowOverwrite permits replacing an existing connection entry. + AllowOverwrite bool + // RequireExisting makes a missing connection entry an error. + RequireExisting bool + // EnvUpdates, when non-empty, are written under `env:` in the same save + // as the connection entry (one write). + EnvUpdates map[string]any + // AllowEnvOverwrite permits replacing existing env: values. + AllowEnvOverwrite bool +} + +// Get returns the raw (unexpanded) props of one connection from the env file, +// plus whether it exists in this specific file. Unlike ConnectionEntries, it +// does not go through LoadSlingEnvFile / ReadConnections, so ${VAR} refs stay +// refs and a resolved secret can never be surfaced. +func (ec *EnvFileConns) Get(name string) (props map[string]any, found bool) { + if ec.EnvFile == nil || strings.TrimSpace(name) == "" { + return nil, false + } + raw, err := ec.EnvFile.RawConnections() + if err != nil { + return nil, false + } + for k, v := range raw { + if strings.EqualFold(k, name) { + return v, true + } + } + return nil, false +} + +// SetValidated is Set + validation/drop-guards, used by the GUI path. It +// merges over the raw on-disk entry (not the expanded struct), validates the +// name and type, and writes through EnvFile.SetConnectionNode so ${VAR} refs +// are never expanded onto disk. +func (ec *EnvFileConns) SetValidated(name string, props map[string]any, opts SetOptions) (err error) { + if ec.EnvFile == nil { + return g.Error("env file is not set") + } + if strings.TrimSpace(name) == "" { + return g.Error("name is blank") + } + name = strings.ToUpper(strings.TrimSpace(name)) + if err = env.ValidateKey(name); err != nil { + return err + } + if props == nil { + return g.Error("no properties provided for connection %s", name) + } + + existing, exists := ec.Get(name) + if exists { + if !opts.AllowOverwrite { + return g.Error("connection %s already exists", name) + } + props = MergeConnProps(existing, props) + } else if opts.RequireExisting { + return g.Error("did not find connection `%s`", name) + } + + if err = NormalizeConnProps(props); err != nil { + return err + } + + // keep on-disk refs when a literal equals the ref's expansion + if exists { + PreserveRefs(existing, props) + } + + // parse url + if url := cast.ToString(props["url"]); url != "" { + conn, uErr := NewConnectionFromURL(name, url) + if uErr != nil { + return g.Error(uErr, "could not parse url") + } + if _, ok := props["type"]; !ok { + props["type"] = conn.Type.String() + } + } + + t, found := props["type"] + if _, typeOK := dbio.ValidateType(cast.ToString(t)); found && !typeOK { + return g.Error("invalid type (%s)", cast.ToString(t)) + } else if !found { + return g.Error("need to specify valid `type` key or provide `url`") + } + + if opts.RejectLiteralSecrets { + if err = RejectLiteralSecrets(name, props); err != nil { + return err + } + } + + if len(opts.EnvUpdates) > 0 { + if err = ec.checkEnvOverwrite(opts.EnvUpdates, opts.AllowEnvOverwrite); err != nil { + return err + } + } + + if err = ec.EnvFile.SetConnectionNode(name, props, opts.EnvUpdates); err != nil { + return g.Error(err, "could not write env file") + } + return nil +} + +// checkEnvOverwrite refuses to replace an existing env: value unless allowed. +func (ec *EnvFileConns) checkEnvOverwrite(updates map[string]any, allowOverwrite bool) error { + if allowOverwrite { + return nil + } + existing, err := ec.EnvFile.RawEnv() + if err != nil { + return err + } + keys := make([]string, 0, len(updates)) + for k := range updates { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + cur, ok := existing[k] + if !ok { + continue + } + if cast.ToString(cur) != cast.ToString(updates[k]) { + return g.Error("env var %s already exists in env.yaml; pass allow_overwrite to update it", k) + } + } + return nil +} + func (ec *EnvFileConns) Set(name string, kvMap map[string]any) (err error) { if name == "" { diff --git a/core/dbio/connection/connection_local_test.go b/core/dbio/connection/connection_local_test.go new file mode 100644 index 000000000..a12b3b589 --- /dev/null +++ b/core/dbio/connection/connection_local_test.go @@ -0,0 +1,382 @@ +package connection + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/slingdata-io/sling-cli/core/env" +) + +func writeEnvFile(t *testing.T, body string) (path string, ec *EnvFileConns) { + t.Helper() + dir := t.TempDir() + path = filepath.Join(dir, "env.yaml") + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return path, &EnvFileConns{Name: "project env.yaml", EnvFile: &env.EnvFile{Path: path}} +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestEnvVarRefRenders(t *testing.T) { + cases := map[string]string{ + "MY_PG|password": "${MY_PG_PASSWORD}", + "my-pg|ssh-key": "${MY_PG_SSH_KEY}", + " MY_PG | Password ": "${MY_PG_PASSWORD}", + "MY_API|secrets.client_id": "${MY_API_SECRETS_CLIENT_ID}", + } + for in, want := range cases { + parts := strings.Split(in, "|") + if got := EnvVarRef(parts[0], parts[1]); got != want { + t.Errorf("EnvVarRef(%q, %q) = %q, want %q", parts[0], parts[1], got, want) + } + } +} + +func TestSetValidatedPreservesRestOfFile(t *testing.T) { + path, ec := writeEnvFile(t, `# Sling environment file — managed by you. + +connections: + # Production warehouse + PG_PROD: + type: postgres + host: db.example.com + user: app + PG_STAGE: + type: postgres + host: stage.db.example.com + +# Variables shared across runs +env: + region: us-west-2 + +# Custom block we don't manage +custom_section: + retain: yes +`) + + err := ec.SetValidated("PG_PROD", map[string]any{ + "type": "postgres", + "host": "new.db.example.com", + "user": "app", + "port": 5432, + }, SetOptions{AllowOverwrite: true}) + if err != nil { + t.Fatalf("SetValidated: %v", err) + } + + got := readFile(t, path) + for _, sub := range []string{ + "# Sling environment file — managed by you.", + "# Production warehouse", + "PG_STAGE:", + "stage.db.example.com", + "# Variables shared across runs", + "region: us-west-2", + "# Custom block we don't manage", + "custom_section:", + "host: new.db.example.com", + "port: 5432", + } { + if !strings.Contains(got, sub) { + t.Errorf("expected output to contain %q\n--- got ---\n%s", sub, got) + } + } + + // new connection (no overwrite flag needed) + if err := ec.SetValidated("PG_NEW", map[string]any{ + "type": "postgres", + "host": "n", + }, SetOptions{RejectLiteralSecrets: true}); err != nil { + t.Fatalf("SetValidated new: %v", err) + } + got = readFile(t, path) + if !strings.Contains(got, "PG_NEW:") || !strings.Contains(got, "PG_STAGE:") { + t.Errorf("new entry missing or neighbor dropped\n--- got ---\n%s", got) + } +} + +func TestSetValidatedOverwriteGuard(t *testing.T) { + path, ec := writeEnvFile(t, `connections: + MY_PG: + type: postgres + host: localhost +`) + err := ec.SetValidated("MY_PG", map[string]any{"host": "other"}, SetOptions{}) + if err == nil { + t.Fatal("expected refusal without AllowOverwrite") + } + if !strings.Contains(err.Error(), "already exists") { + t.Fatalf("unexpected error: %v", err) + } + if strings.Contains(readFile(t, path), "other") { + t.Error("file changed despite refusal") + } + + if err := ec.SetValidated("MY_PG", map[string]any{"host": "other"}, SetOptions{AllowOverwrite: true}); err != nil { + t.Fatalf("SetValidated with overwrite: %v", err) + } + got := readFile(t, path) + if !strings.Contains(got, "host: other") { + t.Errorf("host not updated\n--- got ---\n%s", got) + } + if !strings.Contains(got, "type: postgres") { + t.Errorf("omitted keys should be kept (MergeConnProps contract)\n--- got ---\n%s", got) + } +} + +func TestSetValidatedRequireExisting(t *testing.T) { + _, ec := writeEnvFile(t, `connections: + MY_PG: + type: postgres +`) + if err := ec.SetValidated("NOPE", map[string]any{"type": "postgres"}, SetOptions{RequireExisting: true}); err == nil { + t.Fatal("expected error for missing connection") + } +} + +// TestSetValidatedKeepsRefsWhenEnvDrifts is the regression guard for the GUI +// write path: the expanded value handed back by a caller must not replace the +// on-disk ref, whatever the process environment looks like at write time. +func TestSetValidatedKeepsRefsWhenEnvDrifts(t *testing.T) { + const leak = "hunter2-LEAK-TEST" + + body := `connections: + MY_PG: + type: postgres + host: localhost + password: ${MY_PG_PASSWORD} + port: 5432 +` + + t.Run("expanded incoming value keeps ref", func(t *testing.T) { + t.Setenv("MY_PG_PASSWORD", leak) + path, ec := writeEnvFile(t, body) + + err := ec.SetValidated("MY_PG", map[string]any{ + "type": "postgres", + "host": "localhost", + "password": leak, + "port": 5433, + }, SetOptions{AllowOverwrite: true, RejectLiteralSecrets: true}) + if err != nil { + t.Fatalf("SetValidated: %v", err) + } + got := readFile(t, path) + if !strings.Contains(got, "${MY_PG_PASSWORD}") { + t.Errorf("expected ref kept\n--- got ---\n%s", got) + } + if strings.Contains(got, leak) { + t.Errorf("secret leaked\n--- got ---\n%s", got) + } + if !strings.Contains(got, "5433") { + t.Errorf("port not updated\n--- got ---\n%s", got) + } + }) + + // When the process env drifted, the incoming literal cannot be matched to + // the on-disk ref. The write must then be refused (never materialized). + t.Run("drifted env refuses literal", func(t *testing.T) { + t.Setenv("MY_PG_PASSWORD", leak) + path, ec := writeEnvFile(t, body) + t.Setenv("MY_PG_PASSWORD", "different") + + err := ec.SetValidated("MY_PG", map[string]any{ + "type": "postgres", + "password": leak, + }, SetOptions{AllowOverwrite: true, RejectLiteralSecrets: true}) + if err == nil { + t.Fatal("expected literal secret to be refused") + } + got := readFile(t, path) + if strings.Contains(got, leak) { + t.Errorf("secret leaked\n--- got ---\n%s", got) + } + if !strings.Contains(got, "${MY_PG_PASSWORD}") { + t.Errorf("ref lost\n--- got ---\n%s", got) + } + }) + + t.Run("unset ref still accepted", func(t *testing.T) { + os.Unsetenv("MY_PG_PASSWORD") + path, ec := writeEnvFile(t, body) + err := ec.SetValidated("MY_PG", map[string]any{ + "type": "postgres", + "password": "${MY_PG_PASSWORD}", + "port": 5433, + }, SetOptions{AllowOverwrite: true, RejectLiteralSecrets: true}) + if err != nil { + t.Fatalf("SetValidated: %v", err) + } + got := readFile(t, path) + if !strings.Contains(got, "${MY_PG_PASSWORD}") { + t.Errorf("expected ref kept\n--- got ---\n%s", got) + } + }) +} + +func TestSetValidatedRefusesLiteralSecret(t *testing.T) { + _, ec := writeEnvFile(t, `connections: + MY_PG: + type: postgres +`) + err := ec.SetValidated("MY_API", map[string]any{ + "type": "postgres", + "password": "hunter2", + }, SetOptions{RejectLiteralSecrets: true}) + if err == nil { + t.Fatal("expected literal secret to be refused") + } + if !strings.Contains(err.Error(), "${") { + t.Fatalf("error should point at the ref form: %v", err) + } +} + +func TestPromoteLiteralSecrets(t *testing.T) { + props := map[string]any{ + "type": "postgres", + "host": "localhost", + "password": "hunter2", + "secrets": map[string]any{ + "client_id": "cid", + "client_secret": "csec", + "access_token": "${ALREADY_A_REF}", + }, + } + envUpdates := map[string]any{} + promoted := PromoteLiteralSecrets("MY_PG", props, envUpdates) + + if props["password"] != "${MY_PG_PASSWORD}" { + t.Errorf("password not promoted: %v", props["password"]) + } + if envUpdates["MY_PG_PASSWORD"] != "hunter2" { + t.Errorf("password not stored: %v", envUpdates) + } + secrets := props["secrets"].(map[string]any) + if secrets["client_id"] != "${MY_PG_CLIENT_ID}" { + t.Errorf("client_id not promoted: %v", secrets["client_id"]) + } + if secrets["client_secret"] != "${MY_PG_CLIENT_SECRET}" { + t.Errorf("client_secret not promoted: %v", secrets["client_secret"]) + } + if secrets["access_token"] != "${ALREADY_A_REF}" { + t.Errorf("existing ref must be left alone: %v", secrets["access_token"]) + } + if envUpdates["MY_PG_ACCESS_TOKEN"] != nil { + t.Errorf("existing ref must not be stored: %v", envUpdates) + } + for _, want := range []string{"password", "secrets.client_id", "secrets.client_secret"} { + if !strings.Contains(strings.Join(promoted, ","), want) { + t.Errorf("promoted list missing %s: %v", want, promoted) + } + } + + // nil envUpdates is a no-op + if got := PromoteLiteralSecrets("MY_PG", map[string]any{"password": "x"}, nil); got != nil { + t.Errorf("expected no-op, got %v", got) + } +} + +func TestSetValidatedWritesEnvAndConnectionAtomically(t *testing.T) { + path, ec := writeEnvFile(t, `connections: + OTHER: + type: postgres +env: + KEEP: me +`) + props := map[string]any{"type": "postgres", "password": "hunter2"} + envUpdates := map[string]any{} + PromoteLiteralSecrets("MY_PG", props, envUpdates) + + err := ec.SetValidated("MY_PG", props, SetOptions{ + RejectLiteralSecrets: true, + EnvUpdates: envUpdates, + }) + if err != nil { + t.Fatalf("SetValidated: %v", err) + } + got := readFile(t, path) + for _, sub := range []string{"MY_PG:", "password: ${MY_PG_PASSWORD}", "MY_PG_PASSWORD: hunter2", "KEEP: me", "OTHER:"} { + if !strings.Contains(got, sub) { + t.Errorf("expected %q\n--- got ---\n%s", sub, got) + } + } + + // a second save with a different value refuses to clobber the env var + err = ec.SetValidated("MY_PG", map[string]any{"password": "${MY_PG_PASSWORD}"}, SetOptions{ + RejectLiteralSecrets: true, + EnvUpdates: map[string]any{"MY_PG_PASSWORD": "newer"}, + AllowOverwrite: true, + }) + if err == nil { + t.Fatal("expected refusal to overwrite existing env var") + } + if !strings.Contains(err.Error(), "allow_overwrite") { + t.Fatalf("unexpected error: %v", err) + } + + // with AllowEnvOverwrite it goes through + err = ec.SetValidated("MY_PG", map[string]any{"password": "${MY_PG_PASSWORD}"}, SetOptions{ + RejectLiteralSecrets: true, + EnvUpdates: map[string]any{"MY_PG_PASSWORD": "newer"}, + AllowOverwrite: true, + AllowEnvOverwrite: true, + }) + if err != nil { + t.Fatalf("SetValidated with AllowEnvOverwrite: %v", err) + } + got = readFile(t, path) + if !strings.Contains(got, "MY_PG_PASSWORD: newer") { + t.Errorf("env var not updated\n--- got ---\n%s", got) + } +} + +func TestSetValidatedRefusesInvalidType(t *testing.T) { + _, ec := writeEnvFile(t, "") + err := ec.SetValidated("MY_CONN", map[string]any{"type": "not-a-real-type"}, SetOptions{}) + if err == nil { + t.Fatal("expected invalid type error") + } + if !strings.Contains(err.Error(), "invalid type") { + t.Fatalf("unexpected error: %v", err) + } + + // url-only connections derive their type + if err := ec.SetValidated("MY_PG", map[string]any{"url": "postgres://user:pass@localhost:5432/db"}, SetOptions{}); err != nil { + t.Fatalf("SetValidated url: %v", err) + } + if _, found := ec.Get("MY_PG"); !found { + t.Fatal("url connection not written") + } +} + +func TestSetValidatedKeepsQuotedAndBlockValues(t *testing.T) { + path, ec := writeEnvFile(t, `connections: + MY_PG: + type: postgres + password: "${MY_PG_PASSWORD}" + sslmode: "require" +`) + if err := ec.SetValidated("MY_PG", map[string]any{ + "type": "postgres", + "host": "localhost", + }, SetOptions{AllowOverwrite: true, RejectLiteralSecrets: true}); err != nil { + t.Fatalf("SetValidated: %v", err) + } + got := readFile(t, path) + for _, sub := range []string{"password: \"${MY_PG_PASSWORD}\"", "sslmode: \"require\"", "host: localhost"} { + if !strings.Contains(got, sub) { + t.Errorf("expected %q\n--- got ---\n%s", sub, got) + } + } +} diff --git a/core/dbio/connection/connection_props.go b/core/dbio/connection/connection_props.go index d7139ae73..84cf2d074 100644 --- a/core/dbio/connection/connection_props.go +++ b/core/dbio/connection/connection_props.go @@ -32,11 +32,103 @@ func MergeConnProps(existing, incoming map[string]any) map[string]any { return out } +// EnvVarNameOf builds the canonical env var name for a connection field. +func EnvVarNameOf(connName, key string) string { + name := sanitizeEnvVarPart(connName) + prop := sanitizeEnvVarPart(key) + return name + "_" + prop +} + +func sanitizeEnvVarPart(s string) string { + s = strings.ToUpper(strings.TrimSpace(s)) + var b strings.Builder + for _, r := range s { + switch { + case r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return b.String() +} + // EnvVarRef builds ${_} for a connection field. func EnvVarRef(connName, key string) string { - name := strings.ToUpper(strings.TrimSpace(connName)) - prop := strings.ToUpper(strings.ReplaceAll(strings.TrimSpace(key), "-", "_")) - return "${" + name + "_" + prop + "}" + return "${" + EnvVarNameOf(connName, key) + "}" +} + +// PromoteLiteralSecrets replaces literal secret values in props with ${VAR} +// refs, recording the values to write under `env:` in envUpdates. It returns +// the promoted prop paths, e.g. []string{"password", "secrets.client_id"}. +// +// Secret fields are: top-level keys in env.SecretKeys, and every leaf under a +// nested `secrets` map (same rule RejectLiteralSecrets applies when refusing). +// Values that are already ${VAR} refs are left alone. +func PromoteLiteralSecrets(connName string, props map[string]any, envUpdates map[string]any) (promoted []string) { + if props == nil || envUpdates == nil { + return nil + } + + for _, k := range env.SecretKeys { + v, ok := props[k] + if !ok || !isLiteralSecret(v) { + continue + } + envUpdates[EnvVarNameOf(connName, k)] = cast.ToString(v) + props[k] = EnvVarRef(connName, k) + promoted = append(promoted, k) + } + + secrets := asAnyMap(props["secrets"]) + if secrets == nil { + return promoted + } + keys := lo.Keys(secrets) + sort.Strings(keys) + for _, k := range keys { + if !isLiteralSecret(secrets[k]) { + continue + } + envKey := EnvVarNameOf(connName, k) + if _, taken := envUpdates[envKey]; taken { + // leaf name collides with another promoted secret; use the path + envKey = EnvVarNameOf(connName, "secrets."+k) + } + envUpdates[envKey] = cast.ToString(secrets[k]) + secrets[k] = "${" + envKey + "}" + promoted = append(promoted, "secrets."+k) + } + return promoted +} + +// PreserveRefs keeps an on-disk ${VAR} ref for any incoming literal value that +// equals that ref's expansion. The GUI never receives expanded values, but the +// CLI contract does allow setting a props map with resolved values; keeping the +// ref keeps the file free of plaintext secrets. +func PreserveRefs(existing, incoming map[string]any) { + if existing == nil || incoming == nil { + return + } + keys := lo.Keys(incoming) + sort.Strings(keys) + for _, k := range keys { + if nested := asAnyMap(incoming[k]); nested != nil { + PreserveRefs(asAnyMap(existing[k]), nested) + continue + } + oldRef, ok := existing[k].(string) + if !ok || !env.IsEnvVarRef(oldRef) { + continue + } + val, ok := incoming[k].(string) + if !ok || env.IsEnvVarRef(val) { + continue + } + if env.ExpandRef(oldRef) == val { + incoming[k] = oldRef + } + } } // NormalizeConnProps parses secrets/inputs YAML strings into maps. diff --git a/core/env/envfile.go b/core/env/envfile.go index 76b4ed15a..0c0fba40d 100644 --- a/core/env/envfile.go +++ b/core/env/envfile.go @@ -5,6 +5,7 @@ import ( "os" "path" "regexp" + "sort" "strings" "github.com/flarco/g" @@ -15,6 +16,12 @@ import ( // envVarRefRe matches a whole-string ${VAR} ref. Unset refs stay literal after g.Rmd. var envVarRefRe = regexp.MustCompile(`^\$\{([A-Z_][A-Z0-9_]*)\}$`) +// envKeyRe matches a safe YAML mapping key (connection name). +var envKeyRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_-]*$`) + +// envVarKeyRe matches a safe environment variable name (env: key). +var envVarKeyRe = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*$`) + type EnvFile struct { Connections map[string]map[string]any `json:"connections,omitempty" yaml:"connections,omitempty"` Env map[string]any `json:"env,omitempty" yaml:"env,omitempty"` @@ -512,6 +519,446 @@ func mappingChild(n *yaml.Node, key string) *yaml.Node { return nil } +// ValidateKey returns an error if key is not a safe YAML mapping key +// (^[A-Za-z_][A-Za-z0-9_-]*$, no leading/trailing whitespace). +func ValidateKey(key string) error { + if key == "" { + return g.Error("name is blank") + } + if strings.TrimSpace(key) != key { + return g.Error("name %q must not have leading or trailing whitespace", key) + } + if !envKeyRe.MatchString(key) { + return g.Error("invalid name %q: must match %s", key, envKeyRe.String()) + } + return nil +} + +// ValidateEnvKey returns an error if key is not a safe environment variable +// name (^[A-Z_][A-Z0-9_]*$). +func ValidateEnvKey(key string) error { + if key == "" { + return g.Error("env var name is blank") + } + if !envVarKeyRe.MatchString(key) { + return g.Error("invalid env var name %q: must match %s", key, envVarKeyRe.String()) + } + return nil +} + +// ExpandRef expands a whole-string ${VAR} against the process environment. +// An unset var keeps the ref as-is. +func ExpandRef(s string) string { + name := EnvVarRefName(s) + if name == "" { + return s + } + if v, ok := os.LookupEnv(name); ok { + return v + } + return s +} + +// RawConnections parses the file at ef.Path into connection prop maps, without +// ${VAR} interpolation. Load/ReadConnections expand refs against the process +// environment; this does not, so callers that hand values to a UI (or write +// them back) can never observe a resolved secret. +func (ef *EnvFile) RawConnections() (map[string]map[string]any, error) { + root, err := ef.loadRootNode() + if err != nil { + return nil, err + } + return rawConnectionsFromRoot(root), nil +} + +// ParseEnvFileConnections parses an env.yaml body into raw connection prop +// maps, without ${VAR} interpolation. +func ParseEnvFileConnections(body string) (map[string]map[string]any, error) { + var root yaml.Node + if err := yaml.Unmarshal([]byte(body), &root); err != nil { + return nil, g.Error(err, "could not parse env file body") + } + if root.Kind == 0 { + return map[string]map[string]any{}, nil + } + return rawConnectionsFromRoot(&root), nil +} + +func rawConnectionsFromRoot(root *yaml.Node) map[string]map[string]any { + out := map[string]map[string]any{} + conns := mappingChild(root, "connections") + if conns == nil { + return out + } + for i := 0; i < len(conns.Content)-1; i += 2 { + props := map[string]any{} + if err := conns.Content[i+1].Decode(&props); err != nil { + // non-mapping entry (e.g. `MY_PG:` with no fields); keep it empty + props = map[string]any{} + } + out[conns.Content[i].Value] = props + } + return out +} + +// ConnectionNames returns the connection keys present in the raw file at +// ef.Path, sorted. +func (ef *EnvFile) ConnectionNames() ([]string, error) { + raw, err := ef.RawConnections() + if err != nil { + return nil, err + } + names := make([]string, 0, len(raw)) + for name := range raw { + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +// EnvKeys returns the keys under `env:` (legacy `variables:` included) in the +// raw file at ef.Path, sorted. No ${VAR} interpolation. +func (ef *EnvFile) EnvKeys() ([]string, error) { + root, err := ef.loadRootNode() + if err != nil { + return nil, err + } + seen := map[string]struct{}{} + for _, block := range []string{"env", "variables"} { + n := mappingChild(root, block) + if n == nil { + continue + } + for i := 0; i < len(n.Content)-1; i += 2 { + seen[n.Content[i].Value] = struct{}{} + } + } + keys := make([]string, 0, len(seen)) + for k := range seen { + keys = append(keys, k) + } + sort.Strings(keys) + return keys, nil +} + +// RawEnv returns the `env:` values (legacy `variables:` included) from the raw +// file at ef.Path, without ${VAR} interpolation. +func (ef *EnvFile) RawEnv() (map[string]any, error) { + root, err := ef.loadRootNode() + if err != nil { + return nil, err + } + out := map[string]any{} + for _, block := range []string{"env", "variables"} { + n := mappingChild(root, block) + if n == nil { + continue + } + for i := 0; i < len(n.Content)-1; i += 2 { + key := n.Content[i].Value + var v any + if err := n.Content[i+1].Decode(&v); err != nil { + continue + } + out[key] = v + } + } + return out, nil +} + +// SetConnectionNode writes one connection's props into the YAML node tree at +// ef.Path and saves. It never reads ef.Connections, so expanded ${VAR} values +// held in the struct cannot leak to disk. Existing scalars under the entry that +// are ${VAR} refs are kept as refs when the incoming value is identical. +// +// envUpdates, when non-empty, are written under `env:` in the same save (one +// atomic write; used for secret promotion). Existing env values are replaced: +// callers that must protect hand-set values (EnvFileConns.SetValidated) check +// first. +func (ef *EnvFile) SetConnectionNode(name string, props map[string]any, envUpdates map[string]any) error { + root, err := ef.loadRootNode() + if err != nil { + return err + } + + conns := ensureMappingChild(root, "connections") + valNode, err := anyToNode(props) + if err != nil { + return g.Error(err, "could not render connection %s", name) + } + + keyNode, existingVal := mappingChildFold(conns, name) + if keyNode == nil { + key := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: name} + conns.Content = append(conns.Content, key, valNode) + } else { + // mergeNode (nil envMap) keeps identical on-disk scalars — including + // ${VAR} refs — and otherwise takes the incoming value verbatim. Do not + // pass interpEnvMap here: the write must not depend on the writer's + // process environment. + merged := mergeNode(existingVal, valNode, nil) + for i := 0; i < len(conns.Content)-1; i += 2 { + if conns.Content[i] == keyNode { + conns.Content[i+1] = merged + break + } + } + } + + if len(envUpdates) > 0 { + if err := setEnvNodes(root, envUpdates, true); err != nil { + return err + } + } + + annotateEnvVarRefComments(root) + return ef.saveRootNode(root) +} + +// DeleteConnectionNode removes one connection entry (and the comments attached +// to it) from the node tree at ef.Path and saves. A trailing comment that is +// attached to the last entry of the `connections:` block is kept: it is moved +// onto the preceding entry (or the block itself when there is none), so +// deleting the last connection does not silently eat it. +func (ef *EnvFile) DeleteConnectionNode(name string) error { + root, err := ef.loadRootNode() + if err != nil { + return err + } + + conns := mappingChild(root, "connections") + if conns == nil { + return g.Error("connections block not found in %s", ef.Path) + } + + idx := -1 + for i := 0; i < len(conns.Content)-1; i += 2 { + if strings.EqualFold(conns.Content[i].Value, name) { + idx = i + break + } + } + if idx < 0 { + return g.Error("did not find connection `%s`", name) + } + + keyNode, valNode := conns.Content[idx], conns.Content[idx+1] + if idx == len(conns.Content)-2 { + // last entry: comments attached as FootComment inside its subtree + // describe the end of the block, not the entry + notes := collectFootComments(keyNode, valNode) + if len(notes) > 0 { + if idx == 0 { + // no neighbor to carry it: keep it on the block header + connsKey, _ := mappingChildFold(mappingRoot(root), "connections") + if connsKey != nil { + connsKey.FootComment = joinComment(connsKey.FootComment, notes) + } + } else { + target := deepestLastNode(conns.Content[idx-1]) + if target != nil { + target.FootComment = joinComment(target.FootComment, notes) + } + } + } + } + + conns.Content = append(conns.Content[:idx], conns.Content[idx+2:]...) + return ef.saveRootNode(root) +} + +// SetEnvNodes sets keys under the `env:` block (legacy `variables:` when `env:` +// is absent) in the node tree at ef.Path and saves. Existing keys are only +// replaced when allowOverwrite is true or the value is unchanged. +func (ef *EnvFile) SetEnvNodes(updates map[string]any, allowOverwrite bool) error { + root, err := ef.loadRootNode() + if err != nil { + return err + } + if err := setEnvNodes(root, updates, allowOverwrite); err != nil { + return err + } + return ef.saveRootNode(root) +} + +// setEnvNodes splices updates into the effective env block of root. +func setEnvNodes(root *yaml.Node, updates map[string]any, allowOverwrite bool) error { + if len(updates) == 0 { + return nil + } + + blockKey := effectiveEnvKey(root) + block := ensureMappingChild(root, blockKey) + + existing := map[string]*yaml.Node{} + for i := 0; i < len(block.Content)-1; i += 2 { + existing[block.Content[i].Value] = block.Content[i] + } + + keys := make([]string, 0, len(updates)) + for k := range updates { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + if err := ValidateEnvKey(k); err != nil { + return err + } + valNode, err := anyToNode(updates[k]) + if err != nil { + return g.Error(err, "could not render env var %s", k) + } + if keyNode, ok := existing[k]; ok { + cur := block.Content[indexOfNode(block, keyNode)+1] + if !allowOverwrite && !nodesEqualScalar(cur, valNode) { + return g.Error("env var %s already exists in env.yaml; pass allow_overwrite to update it", k) + } + block.Content[indexOfNode(block, keyNode)+1] = valNode + continue + } + key := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: k} + block.Content = append(block.Content, key, valNode) + existing[k] = key + } + return nil +} + +// effectiveEnvKey returns the block that holds env vars: `env:` when present, +// otherwise the legacy `variables:` block (which loadEnvFile treats as env). +func effectiveEnvKey(root *yaml.Node) string { + if mappingChild(root, "env") != nil { + return "env" + } + if mappingChild(root, "variables") != nil { + return "variables" + } + return "env" +} + +// ensureMappingChild returns the mapping value for key, creating it when +// absent. +func ensureMappingChild(root *yaml.Node, key string) *yaml.Node { + root = mappingRoot(root) + if root == nil { + return nil + } + if n := mappingChild(root, key); n != nil { + if n.Kind != yaml.MappingNode { + n.Kind = yaml.MappingNode + n.Tag = "!!map" + n.Value = "" + n.Content = nil + } + return n + } + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key} + valNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + root.Content = append(root.Content, keyNode, valNode) + return valNode +} + +// anyToNode renders any value as a yaml.Node. +func anyToNode(v any) (*yaml.Node, error) { + b, err := yaml.Marshal(v) + if err != nil { + return nil, err + } + var doc yaml.Node + if err := yaml.Unmarshal(b, &doc); err != nil { + return nil, err + } + if len(doc.Content) == 0 { + return &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}, nil + } + return doc.Content[0], nil +} + +// saveRootNode encodes root and writes it to ef.Path. +func (ef *EnvFile) saveRootNode(root *yaml.Node) error { + if ef.Path == "" { + return g.Error("env file path is not set") + } + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode(root); err != nil { + _ = enc.Close() + return g.Error(err, "could not marshal into YAML") + } + if err := enc.Close(); err != nil { + return g.Error(err, "could not finalize YAML encoder") + } + ef.Path = strings.ReplaceAll(ef.Path, `\`, `/`) + if err := os.WriteFile(ef.Path, buf.Bytes(), 0644); err != nil { + return g.Error(err, "could not write YAML file") + } + return nil +} + +// collectFootComments gathers FootComment text from the key node and the value +// subtree, in document order. +func collectFootComments(keyNode, valNode *yaml.Node) []string { + var out []string + var walk func(*yaml.Node) + walk = func(n *yaml.Node) { + if n == nil { + return + } + if n.FootComment != "" { + out = append(out, n.FootComment) + } + for _, c := range n.Content { + walk(c) + } + } + walk(keyNode) + walk(valNode) + return out +} + +// deepestLastNode returns the last node in document order under n. +func deepestLastNode(n *yaml.Node) *yaml.Node { + if n == nil { + return nil + } + for n.Kind == yaml.MappingNode || n.Kind == yaml.SequenceNode || n.Kind == yaml.DocumentNode { + if len(n.Content) == 0 { + return n + } + n = n.Content[len(n.Content)-1] + } + return n +} + +func joinComment(existing string, notes []string) string { + out := existing + for _, n := range notes { + if n == "" { + continue + } + if out == "" { + out = n + } else { + out = out + "\n" + n + } + } + return out +} + +func indexOfNode(m *yaml.Node, key *yaml.Node) int { + for i := 0; i < len(m.Content)-1; i += 2 { + if m.Content[i] == key { + return i + } + } + return -1 +} + +func nodesEqualScalar(a, b *yaml.Node) bool { + return a.Kind == b.Kind && a.Tag == b.Tag && a.Value == b.Value +} + func mappingChildFold(n *yaml.Node, key string) (keyNode, valNode *yaml.Node) { n = mappingRoot(n) if n == nil { diff --git a/core/env/envfile_test.go b/core/env/envfile_test.go index 5f3950693..f640bea1f 100644 --- a/core/env/envfile_test.go +++ b/core/env/envfile_test.go @@ -442,3 +442,425 @@ func TestMergeDeclaredEnv(t *testing.T) { assert.Equal(t, "from-process", empty["MERGE_DECLARED_PROBE"]) assert.NotContains(t, empty, "MERGE_DECLARED_ONLY") } + +func TestValidateKeyAndEnvKey(t *testing.T) { + for _, key := range []string{"MY_PG", "my-pg", "_PRIVATE", "PG1"} { + if err := ValidateKey(key); err != nil { + t.Errorf("ValidateKey(%q) unexpected error: %v", key, err) + } + } + for _, key := range []string{"", " MY_PG", "MY_PG ", "MY PG", "1PG", "MY.PG"} { + if err := ValidateKey(key); err == nil { + t.Errorf("ValidateKey(%q) expected error", key) + } + } + + for _, key := range []string{"MY_PG_PASSWORD", "_X", "A1"} { + if err := ValidateEnvKey(key); err != nil { + t.Errorf("ValidateEnvKey(%q) unexpected error: %v", key, err) + } + } + for _, key := range []string{"", "my_var", "MY-VAR", "MY.VAR", "1VAR"} { + if err := ValidateEnvKey(key); err == nil { + t.Errorf("ValidateEnvKey(%q) expected error", key) + } + } +} + +func TestSetConnectionNodePreservesRestOfFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "env.yaml") + original := `# Sling environment file — managed by you. + +connections: + # Production warehouse + PG_PROD: + type: postgres + host: db.example.com + user: app + PG_STAGE: + type: postgres + host: stage.db.example.com + +# Variables shared across runs +env: + region: us-west-2 + +# Custom block — must survive untouched. +custom_section: + retain: yes +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + ef := EnvFile{Path: path} + err := ef.SetConnectionNode("PG_PROD", map[string]any{ + "type": "postgres", + "host": "new.db.example.com", + "user": "app", + "port": 5432, + }, map[string]any{"PG_PROD_TOKEN": "tok"}) + if err != nil { + t.Fatalf("SetConnectionNode: %v", err) + } + + out, _ := os.ReadFile(path) + got := string(out) + + for _, sub := range []string{ + "# Sling environment file — managed by you.", + "# Production warehouse", + "PG_STAGE:", + "stage.db.example.com", + "# Variables shared across runs", + "region: us-west-2", + "# Custom block — must survive untouched.", + "custom_section:", + "host: new.db.example.com", + "port: 5432", + "PG_PROD_TOKEN: tok", + } { + if !strings.Contains(got, sub) { + t.Errorf("expected output to contain %q\n--- got ---\n%s", sub, got) + } + } + if strings.Contains(got, "host: db.example.com") { + t.Errorf("old host value survived\n--- got ---\n%s", got) + } + + // a brand-new connection appends under connections: + if err := ef.SetConnectionNode("PG_NEW", map[string]any{"type": "postgres", "host": "n"}, nil); err != nil { + t.Fatalf("SetConnectionNode: %v", err) + } + out, _ = os.ReadFile(path) + got = string(out) + if !strings.Contains(got, "PG_NEW:") || !strings.Contains(got, "PG_STAGE:") { + t.Errorf("new connection missing or staging dropped\n--- got ---\n%s", got) + } + // the unmanaged block must be after connections: (order preserved) + if strings.Index(got, "custom_section:") < strings.Index(got, "PG_NEW:") { + t.Errorf("custom block moved before connections\n--- got ---\n%s", got) + } +} + +// TestSetConnectionNodeNeverExpandsRefs is the regression guard for the GUI +// write path: the file on disk keeps ${VAR}, and the resolved value is never +// written, whatever the writing process's environment looks like. +func TestSetConnectionNodeNeverExpandsRefs(t *testing.T) { + const leak = "hunter2-LEAK-TEST" + + cases := []struct { + name string + mutate func(t *testing.T) + writeAt func(t *testing.T) + }{ + { + name: "var still set at write", + mutate: func(t *testing.T) { t.Setenv("MY_PG_PASSWORD", leak) }, + writeAt: func(t *testing.T) { t.Setenv("MY_PG_PASSWORD", leak) }, + }, + { + name: "var unset before write", + mutate: func(t *testing.T) { t.Setenv("MY_PG_PASSWORD", leak) }, + writeAt: func(t *testing.T) { os.Unsetenv("MY_PG_PASSWORD") }, + }, + { + name: "process env differs from load time", + mutate: func(t *testing.T) { t.Setenv("MY_PG_PASSWORD", leak) }, + writeAt: func(t *testing.T) { t.Setenv("MY_PG_PASSWORD", "other-value") }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "env.yaml") + original := `connections: + MY_PG: + type: postgres + host: localhost + password: ${MY_PG_PASSWORD} + port: 5432 +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + tc.mutate(t) + ef := LoadEnvFile(path) // expands in memory, as the CLI does + tc.writeAt(t) + + // the GUI payload carries the on-disk ref, not the resolved value + err := ef.SetConnectionNode("MY_PG", map[string]any{ + "type": "postgres", + "host": "localhost", + "password": "${MY_PG_PASSWORD}", + "port": 5433, + }, nil) + if err != nil { + t.Fatalf("SetConnectionNode: %v", err) + } + + out, _ := os.ReadFile(path) + got := string(out) + if !strings.Contains(got, "${MY_PG_PASSWORD}") { + t.Errorf("expected on-disk ${MY_PG_PASSWORD} ref\n--- got ---\n%s", got) + } + if strings.Contains(got, leak) { + t.Errorf("resolved secret written to disk\n--- got ---\n%s", got) + } + if !strings.Contains(got, "5433") { + t.Errorf("expected updated port\n--- got ---\n%s", got) + } + }) + } +} + +// TestSetConnectionNodeKeepsUnchangedRefs verifies that an untouched ref field +// stays a ref (the incoming value is identical, so no expansion is involved). +func TestSetConnectionNodeKeepsUnchangedRefs(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "env.yaml") + original := `connections: + MY_PG: + type: postgres + password: ${MY_PG_PASSWORD} + ssh_private_key: ${MY_PG_SSH_PRIVATE_KEY} +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + ef := EnvFile{Path: path} + err := ef.SetConnectionNode("MY_PG", map[string]any{ + "type": "postgres", + "password": "${MY_PG_PASSWORD}", + "ssh_private_key": "${MY_PG_SSH_PRIVATE_KEY}", + }, nil) + if err != nil { + t.Fatalf("SetConnectionNode: %v", err) + } + + out, _ := os.ReadFile(path) + got := string(out) + if !strings.Contains(got, "${MY_PG_PASSWORD}") || !strings.Contains(got, "${MY_PG_SSH_PRIVATE_KEY}") { + t.Errorf("refs were not kept\n--- got ---\n%s", got) + } +} + +func TestDeleteConnectionNodePreservesNeighborsAndTrailingComment(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "env.yaml") + original := `# top +connections: + PG_A: + type: postgres + # keep me (block trailing) + PG_B: + type: mysql + # deeper trailing + +env: + region: us-west-2 +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + ef := EnvFile{Path: path} + if err := ef.DeleteConnectionNode("PG_A"); err != nil { + t.Fatalf("DeleteConnectionNode: %v", err) + } + out, _ := os.ReadFile(path) + got := string(out) + if strings.Contains(got, "PG_A") { + t.Errorf("PG_A not removed\n--- got ---\n%s", got) + } + for _, sub := range []string{"PG_B:", "# keep me (block trailing)", "region: us-west-2"} { + if !strings.Contains(got, sub) { + t.Errorf("expected %q to survive\n--- got ---\n%s", sub, got) + } + } + + // deleting the LAST entry keeps the block trailing comment + if err := ef.DeleteConnectionNode("PG_B"); err != nil { + t.Fatalf("DeleteConnectionNode: %v", err) + } + out, _ = os.ReadFile(path) + got = string(out) + if strings.Contains(got, "PG_B") { + t.Errorf("PG_B not removed\n--- got ---\n%s", got) + } + for _, sub := range []string{"connections:", "env:", "region: us-west-2"} { + if !strings.Contains(got, sub) { + t.Errorf("expected %q to survive\n--- got ---\n%s", sub, got) + } + } + if !strings.Contains(got, "deeper trailing") { + t.Errorf("block trailing comment was eaten\n--- got ---\n%s", got) + } + + // missing connection is an error + if err := ef.DeleteConnectionNode("NOPE"); err == nil { + t.Error("expected error for missing connection") + } +} + +func TestConnectionNamesAndEnvKeysRaw(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "env.yaml") + original := `connections: + MY_PG: + type: postgres + password: ${SOME_UNSET_REF} + my_mongo: + type: mongodb +variables: + LEGACY_VAR: ${SOME_UNSET_REF} + OTHER: x +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + ef := EnvFile{Path: path} + names, err := ef.ConnectionNames() + if err != nil { + t.Fatalf("ConnectionNames: %v", err) + } + assert.Equal(t, []string{"MY_PG", "my_mongo"}, names) + + keys, err := ef.EnvKeys() + if err != nil { + t.Fatalf("EnvKeys: %v", err) + } + assert.Equal(t, []string{"LEGACY_VAR", "OTHER"}, keys) + + // raw parse must not expand ${VAR} + raw, err := ef.RawConnections() + if err != nil { + t.Fatalf("RawConnections: %v", err) + } + assert.Equal(t, "${SOME_UNSET_REF}", raw["MY_PG"]["password"]) +} + +func TestParseEnvFileConnectionsKeepsRefs(t *testing.T) { + t.Setenv("PARSE_TEST_PASSWORD", "hunter2-PARSE-TEST") + raw, err := ParseEnvFileConnections(`connections: + MY_PG: + type: postgres + password: ${PARSE_TEST_PASSWORD} +`) + if err != nil { + t.Fatalf("ParseEnvFileConnections: %v", err) + } + if got := raw["MY_PG"]["password"]; got != "${PARSE_TEST_PASSWORD}" { + t.Fatalf("expected raw ref, got %v", got) + } +} + +func TestSetEnvNodes(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "env.yaml") + original := `connections: + MY_PG: + type: postgres +env: + EXISTING: one +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + ef := EnvFile{Path: path} + // new key + unchanged existing value: allowed + err := ef.SetEnvNodes(map[string]any{"EXISTING": "one", "NEW_KEY": "two"}, false) + if err != nil { + t.Fatalf("SetEnvNodes: %v", err) + } + out, _ := os.ReadFile(path) + got := string(out) + if !strings.Contains(got, "NEW_KEY: two") || !strings.Contains(got, "EXISTING: one") { + t.Errorf("env updates missing\n--- got ---\n%s", got) + } + + // changing an existing value requires allowOverwrite + err = ef.SetEnvNodes(map[string]any{"EXISTING": "changed"}, false) + if err == nil { + t.Fatal("expected refusal to overwrite existing env var") + } + if err = ef.SetEnvNodes(map[string]any{"EXISTING": "changed"}, true); err != nil { + t.Fatalf("SetEnvNodes with overwrite: %v", err) + } + out, _ = os.ReadFile(path) + got = string(out) + if !strings.Contains(got, "EXISTING: changed") { + t.Errorf("env var not updated\n--- got ---\n%s", got) + } + if !strings.Contains(got, "MY_PG:") { + t.Errorf("connections block lost\n--- got ---\n%s", got) + } + + // invalid env var name + if err = ef.SetEnvNodes(map[string]any{"lower": "x"}, true); err == nil { + t.Error("expected error for lowercase env var name") + } +} + +func TestSetEnvNodesLegacyVariablesBlock(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "env.yaml") + original := `connections: + MY_PG: + type: postgres +variables: + LEGACY: one +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + ef := EnvFile{Path: path} + if err := ef.SetEnvNodes(map[string]any{"NEW_ONE": "x"}, false); err != nil { + t.Fatalf("SetEnvNodes: %v", err) + } + out, _ := os.ReadFile(path) + got := string(out) + if !strings.Contains(got, "variables:") || !strings.Contains(got, "NEW_ONE: x") { + t.Errorf("expected writes to land in the legacy variables block\n--- got ---\n%s", got) + } + if strings.Contains(got, "\nenv:") { + t.Errorf("should not create a second env block\n--- got ---\n%s", got) + } +} + +func TestDeleteConnectionNodeOnlyEntryKeepsComment(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "env.yaml") + original := `connections: + ONLY: + type: postgres + # block trailing +env: + A: b +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + ef := EnvFile{Path: path} + if err := ef.DeleteConnectionNode("ONLY"); err != nil { + t.Fatalf("DeleteConnectionNode: %v", err) + } + out, _ := os.ReadFile(path) + got := string(out) + if strings.Contains(got, "ONLY") { + t.Errorf("entry not removed\n--- got ---\n%s", got) + } + if !strings.Contains(got, "# block trailing") { + t.Errorf("trailing comment was eaten\n--- got ---\n%s", got) + } + if !strings.Contains(got, "A: b") { + t.Errorf("env block lost\n--- got ---\n%s", got) + } +} From 6f3908409b356e1ddb10eb56f9dd6cab04e0dbd7 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 16 Sep 2026 17:18:20 -0300 Subject: [PATCH 05/56] feat(env): deterministic node merge + raw env accessors for the GUI write path - EnvFile.ParseEnvFileKeys / RawEnv / LookupConnectionBody / BodySha / ConnectionEditorEnabled; SetConnectionNode writes env updates in the same save; LookupConnection refactored onto a shared body-based lookup. - EnvFileConns.SetValidated checks env overwrite against raw values; PromoteLiteralSecrets skips fields whose on-disk ref already expands to the typed value. --- core/dbio/connection/connection_local_test.go | 6 +- core/dbio/connection/connection_props.go | 28 ++++++- core/env/envfile.go | 84 +++++++++++++++++-- 3 files changed, 106 insertions(+), 12 deletions(-) diff --git a/core/dbio/connection/connection_local_test.go b/core/dbio/connection/connection_local_test.go index a12b3b589..6905b85bc 100644 --- a/core/dbio/connection/connection_local_test.go +++ b/core/dbio/connection/connection_local_test.go @@ -254,7 +254,7 @@ func TestPromoteLiteralSecrets(t *testing.T) { }, } envUpdates := map[string]any{} - promoted := PromoteLiteralSecrets("MY_PG", props, envUpdates) + promoted := PromoteLiteralSecrets("MY_PG", props, nil, envUpdates) if props["password"] != "${MY_PG_PASSWORD}" { t.Errorf("password not promoted: %v", props["password"]) @@ -282,7 +282,7 @@ func TestPromoteLiteralSecrets(t *testing.T) { } // nil envUpdates is a no-op - if got := PromoteLiteralSecrets("MY_PG", map[string]any{"password": "x"}, nil); got != nil { + if got := PromoteLiteralSecrets("MY_PG", map[string]any{"password": "x"}, nil, nil); got != nil { t.Errorf("expected no-op, got %v", got) } } @@ -296,7 +296,7 @@ env: `) props := map[string]any{"type": "postgres", "password": "hunter2"} envUpdates := map[string]any{} - PromoteLiteralSecrets("MY_PG", props, envUpdates) + PromoteLiteralSecrets("MY_PG", props, nil, envUpdates) err := ec.SetValidated("MY_PG", props, SetOptions{ RejectLiteralSecrets: true, diff --git a/core/dbio/connection/connection_props.go b/core/dbio/connection/connection_props.go index 84cf2d074..b9942c18f 100644 --- a/core/dbio/connection/connection_props.go +++ b/core/dbio/connection/connection_props.go @@ -64,8 +64,10 @@ func EnvVarRef(connName, key string) string { // // Secret fields are: top-level keys in env.SecretKeys, and every leaf under a // nested `secrets` map (same rule RejectLiteralSecrets applies when refusing). -// Values that are already ${VAR} refs are left alone. -func PromoteLiteralSecrets(connName string, props map[string]any, envUpdates map[string]any) (promoted []string) { +// Values that are already ${VAR} refs are left alone, and so are values that +// equal the expansion of the ref already on disk for that field (existing), +// so retyping an unchanged secret does not add a plaintext copy to env:. +func PromoteLiteralSecrets(connName string, props, existing map[string]any, envUpdates map[string]any) (promoted []string) { if props == nil || envUpdates == nil { return nil } @@ -75,6 +77,10 @@ func PromoteLiteralSecrets(connName string, props map[string]any, envUpdates map if !ok || !isLiteralSecret(v) { continue } + if ref, ok := existingRefFor(existing[k], cast.ToString(v)); ok { + props[k] = ref + continue + } envUpdates[EnvVarNameOf(connName, k)] = cast.ToString(v) props[k] = EnvVarRef(connName, k) promoted = append(promoted, k) @@ -84,12 +90,17 @@ func PromoteLiteralSecrets(connName string, props map[string]any, envUpdates map if secrets == nil { return promoted } + existingSecrets := asAnyMap(existing["secrets"]) keys := lo.Keys(secrets) sort.Strings(keys) for _, k := range keys { if !isLiteralSecret(secrets[k]) { continue } + if ref, ok := existingRefFor(existingSecrets[k], cast.ToString(secrets[k])); ok { + secrets[k] = ref + continue + } envKey := EnvVarNameOf(connName, k) if _, taken := envUpdates[envKey]; taken { // leaf name collides with another promoted secret; use the path @@ -102,6 +113,19 @@ func PromoteLiteralSecrets(connName string, props map[string]any, envUpdates map return promoted } +// existingRefFor returns the on-disk ref for a field when literal equals that +// ref's expansion. +func existingRefFor(existing any, literal string) (string, bool) { + ref, ok := existing.(string) + if !ok || !env.IsEnvVarRef(ref) { + return "", false + } + if literal == "" || env.ExpandRef(ref) != literal { + return "", false + } + return ref, true +} + // PreserveRefs keeps an on-disk ${VAR} ref for any incoming literal value that // equals that ref's expansion. The GUI never receives expanded values, but the // CLI contract does allow setting a props map with resolved values; keeping the diff --git a/core/env/envfile.go b/core/env/envfile.go index 0c0fba40d..6990ad506 100644 --- a/core/env/envfile.go +++ b/core/env/envfile.go @@ -2,6 +2,8 @@ package env import ( "bytes" + "crypto/sha256" + "encoding/hex" "os" "path" "regexp" @@ -482,24 +484,41 @@ type MissingRef struct { // LookupConnection re-parses ef.Path and returns line numbers for // connections. and each ${VAR} field under it. func (ef *EnvFile) LookupConnection(name string) (ConnLocation, error) { + root, err := ef.loadRootNode() + if err != nil { + return ConnLocation{Path: ef.Path, Connection: strings.ToUpper(name), Missing: []MissingRef{}}, err + } + return lookupConnectionInRoot(root, name, ef.Path) +} + +// LookupConnectionBody is LookupConnection against a body string. path is +// recorded on the result for display only. No ${VAR} interpolation happens. +func LookupConnectionBody(body, name, path string) (ConnLocation, error) { + var root yaml.Node + if err := yaml.Unmarshal([]byte(body), &root); err != nil { + return ConnLocation{Path: path, Connection: strings.ToUpper(name), Missing: []MissingRef{}}, g.Error(err, "could not parse env file body") + } + if root.Kind == 0 { + root = yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Kind: yaml.MappingNode}}} + } + return lookupConnectionInRoot(&root, name, path) +} + +func lookupConnectionInRoot(root *yaml.Node, name, path string) (ConnLocation, error) { loc := ConnLocation{ - Path: ef.Path, + Path: path, Connection: strings.ToUpper(name), Missing: []MissingRef{}, } - root, err := ef.loadRootNode() - if err != nil { - return loc, err - } conns := mappingChild(root, "connections") if conns == nil { - return loc, g.Error("connections block not found in %s", ef.Path) + return loc, g.Error("connections block not found in %s", path) } keyNode, valNode := mappingChildFold(conns, name) if keyNode == nil { - return loc, g.Error("connection %s not found in %s", name, ef.Path) + return loc, g.Error("connection %s not found in %s", name, path) } loc.Line = keyNode.Line collectMissingRefs(valNode, "", &loc.Missing) @@ -559,6 +578,25 @@ func ExpandRef(s string) string { return s } +// BodySha returns the sha256 hex digest of an env file body. Clients send it +// back on save so a stale buffer cannot overwrite a newer file. +func BodySha(body string) string { + sum := sha256.Sum256([]byte(body)) + return hex.EncodeToString(sum[:]) +} + +// ConnectionEditorEnabled reports whether the GUI connection editor is +// enabled. SLING_DISABLE_CONNECTION_EDITOR=1 turns it off, which also disables +// the EnvironmentSet removal guard (a client that does not know about +// allow_removals cannot act on that refusal). +func ConnectionEditorEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("SLING_DISABLE_CONNECTION_EDITOR"))) { + case "1", "true", "yes", "on": + return false + } + return true +} + // RawConnections parses the file at ef.Path into connection prop maps, without // ${VAR} interpolation. Load/ReadConnections expand refs against the process // environment; this does not, so callers that hand values to a UI (or write @@ -601,6 +639,38 @@ func rawConnectionsFromRoot(root *yaml.Node) map[string]map[string]any { return out } +// ParseEnvFileKeys parses a raw env.yaml body and returns its connection names +// and env keys (legacy `variables:` included). No ${VAR} interpolation. +func ParseEnvFileKeys(body string) (connNames, envKeys []string, err error) { + var root yaml.Node + if err := yaml.Unmarshal([]byte(body), &root); err != nil { + return nil, nil, g.Error(err, "could not parse env file body") + } + if root.Kind == 0 { + return nil, nil, nil + } + + for name := range rawConnectionsFromRoot(&root) { + connNames = append(connNames, name) + } + seen := map[string]struct{}{} + for _, block := range []string{"env", "variables"} { + n := mappingChild(&root, block) + if n == nil { + continue + } + for i := 0; i < len(n.Content)-1; i += 2 { + seen[n.Content[i].Value] = struct{}{} + } + } + for k := range seen { + envKeys = append(envKeys, k) + } + sort.Strings(connNames) + sort.Strings(envKeys) + return connNames, envKeys, nil +} + // ConnectionNames returns the connection keys present in the raw file at // ef.Path, sorted. func (ef *EnvFile) ConnectionNames() ([]string, error) { From a82da32adbb5dbace0d02a13998993bbfa230903 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 16 Sep 2026 20:34:00 -0300 Subject: [PATCH 06/56] feat(env): RemovedKeys helper for the raw-editor removal guard Shared by the master and agent EnvironmentSet guards (agent cannot import the master package). --- core/env/envfile.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/core/env/envfile.go b/core/env/envfile.go index 6990ad506..f967ec516 100644 --- a/core/env/envfile.go +++ b/core/env/envfile.go @@ -671,6 +671,44 @@ func ParseEnvFileKeys(body string) (connNames, envKeys []string, err error) { return connNames, envKeys, nil } +// RemovedKeys lists connections and env vars (prefixed `env.`) present in +// oldBody but not in newBody. Used to refuse a raw-editor save that would drop +// credentials the previous body had, unless the client confirms. +func RemovedKeys(oldBody, newBody string) ([]string, error) { + oldConns, oldEnv, err := ParseEnvFileKeys(oldBody) + if err != nil { + return nil, g.Error(err, "could not parse current env.yaml") + } + newConns, newEnv, err := ParseEnvFileKeys(newBody) + if err != nil { + // let the save path produce the parse error + return nil, nil + } + + inNew := func(list []string, key string) bool { + for _, k := range list { + if strings.EqualFold(k, key) { + return true + } + } + return false + } + + var removed []string + for _, k := range oldConns { + if !inNew(newConns, k) { + removed = append(removed, k) + } + } + for _, k := range oldEnv { + if !inNew(newEnv, k) { + removed = append(removed, "env."+k) + } + } + sort.Strings(removed) + return removed, nil +} + // ConnectionNames returns the connection keys present in the raw file at // ef.Path, sorted. func (ef *EnvFile) ConnectionNames() ([]string, error) { From 1e9fbe68467d382f63bd8193c6195a80e2a8f5f5 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Wed, 16 Sep 2026 21:09:00 -0300 Subject: [PATCH 07/56] test(env): assert untouched env.yaml lines survive verbatim + block scalars --- core/dbio/connection/connection_local_test.go | 29 ++++++++++++++----- core/env/envfile_test.go | 20 +++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/core/dbio/connection/connection_local_test.go b/core/dbio/connection/connection_local_test.go index 6905b85bc..d19d7b733 100644 --- a/core/dbio/connection/connection_local_test.go +++ b/core/dbio/connection/connection_local_test.go @@ -362,21 +362,36 @@ func TestSetValidatedRefusesInvalidType(t *testing.T) { func TestSetValidatedKeepsQuotedAndBlockValues(t *testing.T) { path, ec := writeEnvFile(t, `connections: - MY_PG: - type: postgres - password: "${MY_PG_PASSWORD}" + MY_SFTP: + type: sftp + host: one.example.com + password: "${MY_SFTP_PASSWORD}" sslmode: "require" + private_key: | + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEA + -----END OPENSSH PRIVATE KEY----- `) - if err := ec.SetValidated("MY_PG", map[string]any{ - "type": "postgres", - "host": "localhost", + if err := ec.SetValidated("MY_SFTP", map[string]any{ + "host": "two.example.com", }, SetOptions{AllowOverwrite: true, RejectLiteralSecrets: true}); err != nil { t.Fatalf("SetValidated: %v", err) } got := readFile(t, path) - for _, sub := range []string{"password: \"${MY_PG_PASSWORD}\"", "sslmode: \"require\"", "host: localhost"} { + for _, sub := range []string{ + "password: \"${MY_SFTP_PASSWORD}\"", + "sslmode: \"require\"", + "host: two.example.com", + "private_key: |", + "-----BEGIN OPENSSH PRIVATE KEY-----", + "b3BlbnNzaC1rZXktdjEA", + "-----END OPENSSH PRIVATE KEY-----", + } { if !strings.Contains(got, sub) { t.Errorf("expected %q\n--- got ---\n%s", sub, got) } } + if strings.Contains(got, "host: one.example.com") { + t.Errorf("host not updated\n--- got ---\n%s", got) + } } diff --git a/core/env/envfile_test.go b/core/env/envfile_test.go index f640bea1f..31118ce95 100644 --- a/core/env/envfile_test.go +++ b/core/env/envfile_test.go @@ -529,6 +529,26 @@ custom_section: t.Errorf("old host value survived\n--- got ---\n%s", got) } + // every original line except the edited field must survive verbatim, in order + j := 0 + outLines := strings.Split(got, "\n") + for _, line := range strings.Split(original, "\n") { + if strings.TrimSpace(line) == "" || strings.Contains(line, "host: db.example.com") { + continue + } + found := false + for ; j < len(outLines); j++ { + if outLines[j] == line { + found = true + j++ + break + } + } + if !found { + t.Errorf("original line did not survive in order: %q\n--- got ---\n%s", line, got) + } + } + // a brand-new connection appends under connections: if err := ef.SetConnectionNode("PG_NEW", map[string]any{"type": "postgres", "host": "n"}, nil); err != nil { t.Fatalf("SetConnectionNode: %v", err) From 6cf42108c2f80afe8d9c86a0e7b026fd5947499c Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Thu, 17 Sep 2026 22:46:59 -0300 Subject: [PATCH 08/56] feat(go.mod): add open-golang dependency for enhanced functionality --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6946aaa30..6b50e5859 100644 --- a/go.mod +++ b/go.mod @@ -84,6 +84,7 @@ require ( github.com/shirou/gopsutil/v3 v3.24.4 github.com/shopspring/decimal v1.4.0 github.com/sijms/go-ora/v2 v2.8.24 + github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/slingdata-io/godbc v0.0.9 github.com/slingdata-io/golyglot v1.0.20 github.com/slingdata-io/sling v0.0.0-20260715135102-01cd9a07a3c8 @@ -339,7 +340,6 @@ require ( github.com/shirou/gopsutil/v4 v4.25.1 // indirect github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/slingdata-io/pocketbase v0.22.136 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect From 73b00eab03264abfe771af87cba49b7a9d438d1c Mon Sep 17 00:00:00 2001 From: themulle Date: Sat, 19 Sep 2026 00:46:30 +0200 Subject: [PATCH 09/56] feat: add Databricks Unity Catalog Volume and Zerobus Arrow targets - Implement Databricks Volume target via Files REST API for streaming Parquet uploads to Unity Catalog Volumes - Implement Databricks Zerobus target using Arrow Flight / Arrow IPC streaming with configurable batch_size, max_inflight_batches, and compression (none, lz4, zstd) - Register databricks-volume and zerobus connection types and filesystem/database handlers in core dbio packages - Add unit tests for URL parsing, REST streaming, and Arrow IPC serialization --- core/dbio/connection/connection.go | 4 + core/dbio/database/database.go | 4 +- core/dbio/database/database_zerobus.go | 346 ++++++++++++++ core/dbio/database/database_zerobus_test.go | 122 +++++ core/dbio/dbio_types.go | 28 +- core/dbio/filesys/fs.go | 5 + core/dbio/filesys/fs_databricks_volume.go | 449 ++++++++++++++++++ .../dbio/filesys/fs_databricks_volume_test.go | 134 ++++++ core/dbio/filesys/fs_file_node.go | 4 + 9 files changed, 1088 insertions(+), 8 deletions(-) create mode 100644 core/dbio/database/database_zerobus.go create mode 100644 core/dbio/database/database_zerobus_test.go create mode 100644 core/dbio/filesys/fs_databricks_volume.go create mode 100644 core/dbio/filesys/fs_databricks_volume_test.go diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index 85a3752af..f17277ced 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -885,6 +885,10 @@ func (c *Connection) setURL() (err error) { } else { template = "databricks://token:{token}@{host}:{port}/sql/1.0/warehouses/{warehouse_id}" } + case dbio.TypeDbZerobus: + setIfMissing("token", c.Data["password"]) + setIfMissing("port", c.Type.DefPort()) + template = "zerobus://{host}:{port}/{database}" case dbio.TypeDbD1: setIfMissing("account_id", c.Data["host"]) setIfMissing("api_token", c.Data["password"]) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index ab534cd8b..27467d809 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -328,6 +328,8 @@ func NewConnContext(ctx context.Context, URL string, props ...string) (Connectio conn = &ODBCConn{URL: URL} } else if strings.HasPrefix(URL, "scylladb:") { conn = &ScyllaDBConn{URL: URL} + } else if strings.HasPrefix(URL, "zerobus") { + conn = &ZerobusConn{URL: URL} } else { conn = &BaseConn{URL: URL} } @@ -378,7 +380,7 @@ func getDriverName(conn Connection) (driverName string) { driverName = "bigquery" case dbio.TypeDbSnowflake: driverName = "snowflake" - case dbio.TypeDbDatabricks: + case dbio.TypeDbDatabricks, dbio.TypeDbZerobus: driverName = "databricks" case dbio.TypeDbSQLite: driverName = "sqlite3" diff --git a/core/dbio/database/database_zerobus.go b/core/dbio/database/database_zerobus.go new file mode 100644 index 000000000..14e60ec6e --- /dev/null +++ b/core/dbio/database/database_zerobus.go @@ -0,0 +1,346 @@ +package database + +import ( + "bytes" + "context" + "fmt" + "strings" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/flarco/g" + "github.com/flarco/g/net" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/spf13/cast" +) + +// ZerobusConn represents a connection for ingesting data into Databricks via Zerobus / Arrow Flight +type ZerobusConn struct { + BaseConn + URL string + WorkspaceURL string + ShardURL string + ClientID string + ClientSecret string + Token string + Catalog string + Schema string + Table string + BatchSize int + Compression string // "none", "lz4", "zstd" (matching Zerobus IPCCompressionType) + MaxInflightBatches int // defaults to 1000 +} + +// Init initiates the Zerobus connection +func (conn *ZerobusConn) Init() error { + conn.BaseConn.URL = conn.URL + conn.BaseConn.Type = dbio.TypeDbZerobus + conn.BaseConn.defaultPort = 443 + + instance := Connection(conn) + conn.BaseConn.instance = &instance + + conn.WorkspaceURL = conn.GetProp("workspace_url") + if conn.WorkspaceURL == "" { + conn.WorkspaceURL = conn.GetProp("host") + } + conn.ShardURL = conn.GetProp("shard_url") + conn.ClientID = conn.GetProp("client_id") + conn.ClientSecret = conn.GetProp("client_secret") + conn.Token = conn.GetProp("token", "password") + conn.Catalog = conn.GetProp("catalog") + conn.Schema = conn.GetProp("schema") + conn.Table = conn.GetProp("table", "database") + + conn.BatchSize = cast.ToInt(conn.GetProp("batch_size")) + if conn.BatchSize <= 0 { + conn.BatchSize = 10000 // Standard 10k Arrow record buffer + } + + conn.Compression = strings.ToLower(conn.GetProp("compression", "ipc_compression")) + if conn.Compression == "" { + conn.Compression = "none" + } + + conn.MaxInflightBatches = cast.ToInt(conn.GetProp("max_inflight_batches")) + if conn.MaxInflightBatches <= 0 { + conn.MaxInflightBatches = 1000 + } + + if conn.URL != "" { + conn.parseURL(conn.URL) + } + + return nil +} + +func (conn *ZerobusConn) parseURL(rawURL string) { + clean := strings.TrimPrefix(rawURL, "zerobus://") + u, err := net.NewURL("http://" + clean) // Use http parser for host/path + if err == nil { + if conn.WorkspaceURL == "" { + conn.WorkspaceURL = u.Hostname() + } + pathTrimmed := strings.Trim(u.Path(), "/") + if pathTrimmed != "" { + parts := strings.Split(pathTrimmed, ".") + if len(parts) == 3 { + conn.Catalog = parts[0] + conn.Schema = parts[1] + conn.Table = parts[2] + } else if len(parts) == 2 { + conn.Schema = parts[0] + conn.Table = parts[1] + } else if len(parts) == 1 { + conn.Table = parts[0] + } + } + + if pass := u.Password(); pass != "" && conn.Token == "" { + conn.Token = pass + } + if user := u.Username(); user != "" && conn.ClientID == "" { + conn.ClientID = user + } + } +} + +// Connect connects / validates Zerobus configuration +func (conn *ZerobusConn) Connect(timeOut ...int) error { + if conn.WorkspaceURL == "" { + return g.Error("workspace_url or host is required for Zerobus connection") + } + if conn.Token == "" && (conn.ClientID == "" || conn.ClientSecret == "") { + return g.Error("either token or (client_id and client_secret) is required for Zerobus connection") + } + return nil +} + +// BulkImportFlow ingests a dataflow into the target table using Arrow batches +func (conn *ZerobusConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error) { + defer df.CleanUp() + + if tableFName == "" { + tableFName = conn.Table + } + if tableFName == "" { + return 0, g.Error("target table must be specified for Zerobus ingestion") + } + + targetTable, err := ParseTableName(tableFName, conn.Type) + if err != nil { + return 0, g.Error(err, "invalid target table name: %s", tableFName) + } + + targetFDQN := targetTable.FDQN() + g.Info("ingesting into Databricks via Zerobus Arrow stream: %s", targetFDQN) + + // Build Arrow Schema from Dataflow columns + arrowSchema := iop.ColumnsToArrowSchema(df.Columns) + mem := memory.NewGoAllocator() + + batchSize := conn.BatchSize + if batchSize <= 0 { + batchSize = 10000 + } + + builders := make([]array.Builder, len(df.Columns)) + createBuilder := func(dtype arrow.DataType) array.Builder { + switch dtype.ID() { + case arrow.BOOL: + return array.NewBooleanBuilder(mem) + case arrow.INT32: + return array.NewInt32Builder(mem) + case arrow.INT64: + return array.NewInt64Builder(mem) + case arrow.FLOAT64: + return array.NewFloat64Builder(mem) + case arrow.DECIMAL128: + return array.NewDecimal128Builder(mem, dtype.(*arrow.Decimal128Type)) + case arrow.DATE32: + return array.NewDate32Builder(mem) + case arrow.TIMESTAMP: + return array.NewTimestampBuilder(mem, dtype.(*arrow.TimestampType)) + case arrow.TIME32: + return array.NewTime32Builder(mem, dtype.(*arrow.Time32Type)) + case arrow.TIME64: + return array.NewTime64Builder(mem, dtype.(*arrow.Time64Type)) + case arrow.STRING: + return array.NewStringBuilder(mem) + case arrow.BINARY: + return array.NewBinaryBuilder(mem, dtype.(*arrow.BinaryType)) + case arrow.EXTENSION: + return array.NewBuilder(mem, dtype) + default: + return array.NewStringBuilder(mem) + } + } + + resetBuilders := func() { + for i, field := range arrowSchema.Fields() { + builders[i] = createBuilder(field.Type) + } + } + releaseBuilders := func() { + for _, b := range builders { + if b != nil { + b.Release() + } + } + } + defer releaseBuilders() + + resetBuilders() + rowsInBatch := 0 + + flushBatch := func() error { + if rowsInBatch == 0 { + return nil + } + + // Build arrays + arrays := make([]arrow.Array, len(builders)) + for i, b := range builders { + arrays[i] = b.NewArray() + } + + record := array.NewRecord(arrowSchema, arrays, int64(rowsInBatch)) + + // Send record batch with retries and guarantee Release + err := conn.sendRecordBatchWithRetry(targetFDQN, record) + + // Release record and arrays immediately after send to free memory + record.Release() + for _, arr := range arrays { + arr.Release() + } + + // Reset builders for next batch + releaseBuilders() + resetBuilders() + rowsInBatch = 0 + + return err + } + + // Stream records + for ds := range df.StreamCh { + for row := range ds.Rows { + for colIdx, col := range df.Columns { + var val interface{} + if colIdx < len(row) { + val = row[colIdx] + } + iop.AppendToBuilder(builders[colIdx], &col, val) + } + rowsInBatch++ + count++ + + if rowsInBatch >= batchSize { + if err := flushBatch(); err != nil { + return count, g.Error(err, "failed to stream Arrow RecordBatch to Zerobus") + } + } + } + + if err := ds.Context.Err(); err != nil { + return count, g.Error(err, "error reading source stream") + } + } + + // Flush remaining rows + if err := flushBatch(); err != nil { + return count, g.Error(err, "failed to flush final Arrow RecordBatch to Zerobus") + } + + g.Info("successfully streamed %d rows to Zerobus table %s", count, targetFDQN) + return count, nil +} + +// sendRecordBatchWithRetry streams an Arrow RecordBatch to Zerobus with retries +func (conn *ZerobusConn) sendRecordBatchWithRetry(table string, record arrow.Record) error { + maxRetries := 3 + var lastErr error + + for attempt := 0; attempt <= maxRetries; attempt++ { + if attempt > 0 { + backoff := time.Duration(1< 0 && parts[0] == "Volumes" { + parts = parts[1:] + } + + // Check if the first part looks like a domain (e.g. adb-123.azuredatabricks.net) + if len(parts) > 0 && (strings.Contains(parts[0], ".databricks.com") || strings.Contains(parts[0], ".azuredatabricks.net")) { + if fs.host == "" { + fs.host = parts[0] + } + parts = parts[1:] + if len(parts) > 0 && parts[0] == "Volumes" { + parts = parts[1:] + } + } + + if len(parts) >= 1 && fs.catalog == "" { + fs.catalog = parts[0] + } + if len(parts) >= 2 && fs.schema == "" { + fs.schema = parts[1] + } + if len(parts) >= 3 && fs.volume == "" { + fs.volume = parts[2] + } +} + +// Prefix returns the url prefix +func (fs *DatabricksVolumeFileSysClient) Prefix(suffix ...string) string { + var prefix string + if fs.catalog != "" && fs.schema != "" && fs.volume != "" { + prefix = fmt.Sprintf("databricks-volume://%s/%s/%s/", fs.catalog, fs.schema, fs.volume) + } else { + prefix = "databricks-volume://" + } + return prefix + strings.Join(suffix, "") +} + +// GetPath converts a uri into the internal volume path starting with /Volumes/... +func (fs *DatabricksVolumeFileSysClient) GetPath(uri string) (volumePath string, err error) { + uri = NormalizeURI(fs, uri) + + clean := uri + for _, prefix := range []string{"databricks-volume://", "databricks://", "volume://"} { + if strings.HasPrefix(clean, prefix) { + clean = strings.TrimPrefix(clean, prefix) + break + } + } + + parts := strings.Split(strings.Trim(clean, "/"), "/") + if len(parts) > 0 && (strings.Contains(parts[0], ".databricks.com") || strings.Contains(parts[0], ".azuredatabricks.net")) { + parts = parts[1:] + } + if len(parts) > 0 && parts[0] == "Volumes" { + parts = parts[1:] + } + + var catalog, schema, volume string + var subparts []string + + if len(parts) >= 3 { + catalog = parts[0] + schema = parts[1] + volume = parts[2] + subparts = parts[3:] + } else { + catalog = fs.catalog + schema = fs.schema + volume = fs.volume + subparts = parts + } + + if catalog == "" || schema == "" || volume == "" { + return "", g.Error("invalid volume URI, catalog/schema/volume must be specified: %s", uri) + } + + p := fmt.Sprintf("/Volumes/%s/%s/%s", catalog, schema, volume) + if len(subparts) > 0 && subparts[0] != "" { + p = p + "/" + strings.Join(subparts, "/") + } + return p, nil +} + +// doRequest executes an HTTP request against Databricks Files REST API with retries +func (fs *DatabricksVolumeFileSysClient) doRequest(ctx context.Context, method, apiPath string, body io.Reader, query url.Values) (*http.Response, error) { + if fs.host == "" { + return nil, g.Error("databricks host is required") + } + if fs.token == "" { + return nil, g.Error("databricks token is required") + } + + urlStr := fmt.Sprintf("https://%s%s", fs.host, apiPath) + if len(query) > 0 { + urlStr = urlStr + "?" + query.Encode() + } + + var lastErr error + maxRetries := 3 + + for attempt := 0; attempt <= maxRetries; attempt++ { + if attempt > 0 { + sleepDur := time.Duration(1<= 500 { + resp.Body.Close() + lastErr = fmt.Errorf("server error %s (%d)", resp.Status, resp.StatusCode) + continue + } + + return resp, nil + } + + return nil, g.Error(lastErr, "request failed after %d retries for %s %s", maxRetries, method, urlStr) +} + +type countingReader struct { + reader io.Reader + count int64 +} + +func (cr *countingReader) Read(p []byte) (n int, err error) { + n, err = cr.reader.Read(p) + cr.count += int64(n) + return n, err +} + +// Write writes a stream directly to a Databricks volume using the Files REST API +func (fs *DatabricksVolumeFileSysClient) Write(uri string, reader io.Reader) (bw int64, err error) { + volumePath, err := fs.GetPath(uri) + if err != nil { + return 0, err + } + + apiPath := "/api/2.0/fs/files" + volumePath + query := url.Values{} + query.Set("overwrite", "true") + + // Track bytes written + cr := &countingReader{reader: reader} + + resp, err := fs.doRequest(fs.Context().Ctx, http.MethodPut, apiPath, cr, query) + if err != nil { + return cr.count, g.Error(err, "failed to PUT file to Databricks Volume: %s", volumePath) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBytes, _ := io.ReadAll(resp.Body) + return cr.count, g.Error("error writing to Databricks Volume %s (status %d): %s", volumePath, resp.StatusCode, string(respBytes)) + } + + return cr.count, nil +} + +// GetReader returns a reader for reading a file from Databricks volume +func (fs *DatabricksVolumeFileSysClient) GetReader(uri string) (reader io.Reader, err error) { + volumePath, err := fs.GetPath(uri) + if err != nil { + return nil, err + } + + apiPath := "/api/2.0/fs/files" + volumePath + resp, err := fs.doRequest(fs.Context().Ctx, http.MethodGet, apiPath, nil, nil) + if err != nil { + return nil, g.Error(err, "failed to GET file from Databricks Volume: %s", volumePath) + } + + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + return nil, g.Error("file not found in Databricks Volume: %s", volumePath) + } else if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBytes, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return nil, g.Error("error reading from Databricks Volume %s (status %d): %s", volumePath, resp.StatusCode, string(respBytes)) + } + + return resp.Body, nil +} + +// delete deletes a file or directory in a Databricks volume +func (fs *DatabricksVolumeFileSysClient) delete(uri string) (err error) { + volumePath, err := fs.GetPath(uri) + if err != nil { + return err + } + + apiPath := "/api/2.0/fs/files" + volumePath + resp, err := fs.doRequest(fs.Context().Ctx, http.MethodDelete, apiPath, nil, nil) + if err != nil { + return g.Error(err, "failed to DELETE file in Databricks Volume: %s", volumePath) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil + } else if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBytes, _ := io.ReadAll(resp.Body) + return g.Error("error deleting Databricks Volume file %s (status %d): %s", volumePath, resp.StatusCode, string(respBytes)) + } + + return nil +} + +// MkdirAll creates a directory in a Databricks volume +func (fs *DatabricksVolumeFileSysClient) MkdirAll(uri string) (err error) { + volumePath, err := fs.GetPath(uri) + if err != nil { + return err + } + + apiPath := "/api/2.0/fs/directories" + volumePath + resp, err := fs.doRequest(fs.Context().Ctx, http.MethodPut, apiPath, bytes.NewReader([]byte{}), nil) + if err != nil { + return g.Error(err, "failed to create directory in Databricks Volume: %s", volumePath) + } + defer resp.Body.Close() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBytes, _ := io.ReadAll(resp.Body) + return g.Error("error creating Databricks Volume directory %s (status %d): %s", volumePath, resp.StatusCode, string(respBytes)) + } + + return nil +} + +// List lists objects in a Databricks volume path +func (fs *DatabricksVolumeFileSysClient) List(uri string) (nodes FileNodes, err error) { + return fs.doList(uri, false) +} + +// ListRecursive lists objects in a Databricks volume path recursively +func (fs *DatabricksVolumeFileSysClient) ListRecursive(uri string) (nodes FileNodes, err error) { + return fs.doList(uri, true) +} + +func (fs *DatabricksVolumeFileSysClient) doList(uri string, recursive bool) (nodes FileNodes, err error) { + volumePath, err := fs.GetPath(uri) + if err != nil { + return nil, err + } + + pattern, err := makeGlob(NormalizeURI(fs, uri)) + if err != nil { + return nil, g.Error(err, "error parsing glob pattern: %s", uri) + } + + nodes, err = fs.listDirRecursive(volumePath, recursive) + if err != nil { + return nil, err + } + + if pattern != nil { + filtered := FileNodes{} + for _, n := range nodes { + if (*pattern).Match(n.URI) { + filtered = append(filtered, n) + } + } + return filtered, nil + } + + return nodes, nil +} + +func (fs *DatabricksVolumeFileSysClient) listDirRecursive(volumePath string, recursive bool) (nodes FileNodes, err error) { + apiPath := "/api/2.0/fs/directories" + strings.TrimRight(volumePath, "/") + query := url.Values{} + + pageToken := "" + for { + if pageToken != "" { + query.Set("page_token", pageToken) + } + + resp, err := fs.doRequest(fs.Context().Ctx, http.MethodGet, apiPath, nil, query) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + return nodes, nil + } else if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBytes, _ := io.ReadAll(resp.Body) + resp.Body.Close() + return nil, g.Error("error listing directory %s (status %d): %s", volumePath, resp.StatusCode, string(respBytes)) + } + + var dirResp databricksDirListResp + err = json.NewDecoder(resp.Body).Decode(&dirResp) + resp.Body.Close() + if err != nil { + return nil, g.Error(err, "error decoding Databricks directory response") + } + + for _, item := range dirResp.Contents { + nodeURI := fmt.Sprintf("databricks-volume://%s", strings.TrimPrefix(item.Path, "/")) + node := FileNode{ + URI: nodeURI, + IsDir: item.IsDirectory, + Size: cast.ToUint64(item.FileSize), + Updated: item.LastModified / 1000, + } + nodes = append(nodes, node) + + if recursive && item.IsDirectory { + subNodes, err := fs.listDirRecursive(item.Path, true) + if err != nil { + return nil, err + } + nodes = append(nodes, subNodes...) + } + } + + if dirResp.NextPageToken == "" { + break + } + pageToken = dirResp.NextPageToken + } + + return nodes, nil +} diff --git a/core/dbio/filesys/fs_databricks_volume_test.go b/core/dbio/filesys/fs_databricks_volume_test.go new file mode 100644 index 000000000..4ea4a21ad --- /dev/null +++ b/core/dbio/filesys/fs_databricks_volume_test.go @@ -0,0 +1,134 @@ +package filesys + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDatabricksVolume_URLParsing(t *testing.T) { + tests := []struct { + url string + expectedPath string + }{ + { + url: "databricks-volume://my_cat/my_schema/my_vol/folder/data.parquet", + expectedPath: "/Volumes/my_cat/my_schema/my_vol/folder/data.parquet", + }, + { + url: "databricks://Volumes/my_cat/my_schema/my_vol/folder/data.parquet", + expectedPath: "/Volumes/my_cat/my_schema/my_vol/folder/data.parquet", + }, + { + url: "volume://my_cat/my_schema/my_vol/data.parquet", + expectedPath: "/Volumes/my_cat/my_schema/my_vol/data.parquet", + }, + { + url: "databricks-volume://adb-123.azuredatabricks.net/Volumes/my_cat/my_schema/my_vol/test.csv", + expectedPath: "/Volumes/my_cat/my_schema/my_vol/test.csv", + }, + } + + for _, tt := range tests { + client := &DatabricksVolumeFileSysClient{} + err := client.Init(context.Background()) + assert.NoError(t, err) + + path, err := client.GetPath(tt.url) + assert.NoError(t, err) + assert.Equal(t, tt.expectedPath, path) + } +} + +func TestDatabricksVolume_RESTOperations(t *testing.T) { + var receivedPutBody []byte + var receivedAuthHeader string + var receivedMethod string + var receivedURLPath string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedAuthHeader = r.Header.Get("Authorization") + receivedMethod = r.Method + receivedURLPath = r.URL.Path + + switch { + case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/files/"): + b, _ := io.ReadAll(r.Body) + receivedPutBody = b + w.WriteHeader(http.StatusOK) + + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/files/"): + w.WriteHeader(http.StatusOK) + w.Write([]byte("mock volume content")) + + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/files/"): + w.WriteHeader(http.StatusOK) + + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/directories/"): + resp := databricksDirListResp{ + Contents: []databricksFileInfo{ + { + Path: "/Volumes/my_cat/my_schema/my_vol/file1.parquet", + IsDirectory: false, + FileSize: 1024, + }, + }, + } + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(resp) + + default: + w.WriteHeader(http.StatusOK) + } + })) + defer server.Close() + + host := strings.TrimPrefix(server.URL, "http://") + + client := &DatabricksVolumeFileSysClient{} + client.properties = map[string]string{ + "host": host, + "token": "dapi_test_token_123", + "catalog": "my_cat", + "schema": "my_schema", + "volume": "my_vol", + } + + err := client.Init(context.Background()) + assert.NoError(t, err) + + // 1. Test Streaming Write (PUT) + testContent := "column1,column2\nval1,val2\n" + bw, err := client.Write("databricks-volume://my_cat/my_schema/my_vol/data.csv", strings.NewReader(testContent)) + assert.NoError(t, err) + assert.Equal(t, int64(len(testContent)), bw) + assert.Equal(t, "Bearer dapi_test_token_123", receivedAuthHeader) + assert.Equal(t, http.MethodPut, receivedMethod) + assert.Equal(t, "/api/2.0/fs/files/Volumes/my_cat/my_schema/my_vol/data.csv", receivedURLPath) + assert.Equal(t, testContent, string(receivedPutBody)) + + // 2. Test Read (GET) + reader, err := client.GetReader("databricks-volume://my_cat/my_schema/my_vol/data.csv") + assert.NoError(t, err) + readBytes, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, "mock volume content", string(readBytes)) + + // 3. Test List + nodes, err := client.List("databricks-volume://my_cat/my_schema/my_vol") + assert.NoError(t, err) + assert.Len(t, nodes, 1) + assert.Equal(t, "databricks-volume://Volumes/my_cat/my_schema/my_vol/file1.parquet", nodes[0].URI) + assert.Equal(t, uint64(1024), nodes[0].Size) + + // 4. Test Delete + err = client.delete("databricks-volume://my_cat/my_schema/my_vol/data.csv") + assert.NoError(t, err) + assert.Equal(t, http.MethodDelete, receivedMethod) +} diff --git a/core/dbio/filesys/fs_file_node.go b/core/dbio/filesys/fs_file_node.go index 4be90c82e..df85c8202 100644 --- a/core/dbio/filesys/fs_file_node.go +++ b/core/dbio/filesys/fs_file_node.go @@ -315,6 +315,10 @@ func ParseURLType(uri string) (uType dbio.Type, host string, path string, err er return dbio.TypeFileHTTP, host, path, nil } else if scheme == "gdrive" { return dbio.TypeFileGoogleDrive, host, path, nil + } else if scheme == "databricks-volume" || scheme == "volume" { + return dbio.TypeFileDatabricksVolume, host, path, nil + } else if scheme == "databricks" && strings.HasPrefix(u.U.Path, "/Volumes") { + return dbio.TypeFileDatabricksVolume, host, path, nil } else if g.In(scheme, "http", "https") { return dbio.TypeFileHTTP, host, path, nil } From 4caf2843e8bf6724ad467509b50d63f246636111 Mon Sep 17 00:00:00 2001 From: themulle Date: Sat, 19 Sep 2026 01:05:30 +0200 Subject: [PATCH 10/56] fix(databricks): fix volume URL normalization, REST test mock, and optimize upstream coupling --- core/dbio/connection/connection.go | 10 +++- core/dbio/database/database_zerobus.go | 16 +++++ core/dbio/dbio_types.go | 8 +-- core/dbio/filesys/fs.go | 11 ++++ core/dbio/filesys/fs_databricks_volume.go | 58 +++++++++++++++---- .../dbio/filesys/fs_databricks_volume_test.go | 20 ++++--- 6 files changed, 96 insertions(+), 27 deletions(-) diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index f17277ced..d2648b81d 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -888,7 +888,13 @@ func (c *Connection) setURL() (err error) { case dbio.TypeDbZerobus: setIfMissing("token", c.Data["password"]) setIfMissing("port", c.Type.DefPort()) - template = "zerobus://{host}:{port}/{database}" + if token := cast.ToString(c.Data["token"]); token != "" { + template = "zerobus://token:{token}@{host}:{port}/{database}" + } else if clientID := cast.ToString(c.Data["client_id"]); clientID != "" { + template = "zerobus://{client_id}:{client_secret}@{host}:{port}/{database}" + } else { + template = "zerobus://{host}:{port}/{database}" + } case dbio.TypeDbD1: setIfMissing("account_id", c.Data["host"]) setIfMissing("api_token", c.Data["password"]) @@ -1101,7 +1107,7 @@ func (c *Connection) setURL() (err error) { template = template + path } case dbio.TypeFileS3, dbio.TypeFileGoogle, dbio.TypeFileGoogleDrive, dbio.TypeFileAzure, dbio.TypeFileAzureABFS, - dbio.TypeFileLocal: + dbio.TypeFileDatabricksVolume, dbio.TypeFileLocal: return nil case dbio.TypeDbIceberg: setIfMissing("catalog_type", c.Data["catalog_type"]) // rest, glue, s3tables, sql diff --git a/core/dbio/database/database_zerobus.go b/core/dbio/database/database_zerobus.go index 14e60ec6e..9dd701c8e 100644 --- a/core/dbio/database/database_zerobus.go +++ b/core/dbio/database/database_zerobus.go @@ -120,6 +120,22 @@ func (conn *ZerobusConn) Connect(timeOut ...int) error { return nil } +// LoadTemplates loads Databricks SQL templates for Zerobus without needing changes to core dbio_types.go +func (conn *ZerobusConn) LoadTemplates() error { + tmpl, err := dbio.TypeDbDatabricks.Template() + if err != nil { + return err + } + conn.BaseConn.template = tmpl + return nil +} + +// BulkImportStream ingests a datastream into the target table using Arrow batches +func (conn *ZerobusConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error) { + df := iop.MakeDataFlow(ds) + return conn.BulkImportFlow(tableFName, df) +} + // BulkImportFlow ingests a dataflow into the target table using Arrow batches func (conn *ZerobusConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error) { defer df.CleanUp() diff --git a/core/dbio/dbio_types.go b/core/dbio/dbio_types.go index d9f3fae2a..45a0ea415 100644 --- a/core/dbio/dbio_types.go +++ b/core/dbio/dbio_types.go @@ -509,13 +509,9 @@ func (t Type) Template(useBase ...bool) (template Template, err error) { } } - templateName := t.String() - if t == TypeDbZerobus { - templateName = "databricks" - } - templateBytes, err := templatesFolder.ReadFile("templates/" + templateName + ".yaml") + templateBytes, err := templatesFolder.ReadFile("templates/" + t.String() + ".yaml") if err != nil { - return template, g.Error(err, "could not read "+templateName+".yaml") + return template, g.Error(err, "could not read "+t.String()+".yaml") } err = yaml.Unmarshal([]byte(templateBytes), &connTemplate) diff --git a/core/dbio/filesys/fs.go b/core/dbio/filesys/fs.go index dbe8b21b8..0bca42166 100755 --- a/core/dbio/filesys/fs.go +++ b/core/dbio/filesys/fs.go @@ -281,6 +281,13 @@ func NormalizeURI(fs FileSysClient, uri string) string { return fs.Prefix("/") + path } return fs.Prefix("/") + strings.TrimLeft(strings.TrimPrefix(uri, fs.Prefix()), "/") + case dbio.TypeFileDatabricksVolume: + for _, p := range []string{"databricks-volume://", "databricks://", "volume://"} { + if strings.HasPrefix(uri, p) { + return uri + } + } + return fs.Prefix("/") + strings.TrimLeft(strings.TrimPrefix(uri, fs.Prefix()), "/") case dbio.TypeFileS3, dbio.TypeFileGoogle: // For S3/GCS, if URI already has the scheme prefix (e.g., s3://bucket/path), // return it as-is to allow accessing different buckets with the same credentials. @@ -1159,6 +1166,10 @@ func Delete(fs FileSysClient, uri string) (err error) { if len(p) == 0 { return g.Error("invalid uri / path for overwriting (root): %s", uri) } + case dbio.TypeFileDatabricksVolume: + if len(pArr) <= 3 { + return g.Error("invalid uri / path for deleting (volume): %s", uri) + } } err = fs.delete(uri) diff --git a/core/dbio/filesys/fs_databricks_volume.go b/core/dbio/filesys/fs_databricks_volume.go index 79723b077..d016055b2 100644 --- a/core/dbio/filesys/fs_databricks_volume.go +++ b/core/dbio/filesys/fs_databricks_volume.go @@ -12,6 +12,7 @@ import ( "time" "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/dbio" "github.com/spf13/cast" ) @@ -26,6 +27,7 @@ import ( type DatabricksVolumeFileSysClient struct { BaseFileSysClient client *http.Client + scheme string host string token string catalog string @@ -51,15 +53,21 @@ func (fs *DatabricksVolumeFileSysClient) Init(ctx context.Context) (err error) { instance := FileSysClient(fs) fs.BaseFileSysClient.instance = &instance fs.BaseFileSysClient.context = g.NewContext(ctx) + fs.BaseFileSysClient.fsType = dbio.TypeFileDatabricksVolume // Props can come from host, token, catalog, schema, volume - fs.host = fs.GetProp("host") + rawHost := fs.GetProp("host") + if rawHost == "" { + rawHost = fs.GetProp("DATABRICKS_HOST") + } fs.token = fs.GetProp("token") if fs.token == "" { fs.token = fs.GetProp("DATABRICKS_TOKEN") } - if fs.host == "" { - fs.host = fs.GetProp("DATABRICKS_HOST") + + fs.scheme = "https" + if strings.HasPrefix(rawHost, "http://") || fs.GetProp("protocol") == "http" || fs.GetProp("use_ssl") == "false" || fs.GetProp("ssl") == "false" { + fs.scheme = "http" } rawURL := fs.GetProp("url", "URL") @@ -78,7 +86,7 @@ func (fs *DatabricksVolumeFileSysClient) Init(ctx context.Context) (err error) { } // Clean host - fs.host = strings.TrimPrefix(fs.host, "https://") + fs.host = strings.TrimPrefix(rawHost, "https://") fs.host = strings.TrimPrefix(fs.host, "http://") fs.host = strings.TrimRight(fs.host, "/") @@ -89,6 +97,11 @@ func (fs *DatabricksVolumeFileSysClient) Init(ctx context.Context) (err error) { return nil } +// FsType returns dbio.TypeFileDatabricksVolume +func (fs *DatabricksVolumeFileSysClient) FsType() dbio.Type { + return dbio.TypeFileDatabricksVolume +} + func (fs *DatabricksVolumeFileSysClient) parseURL(rawURL string) { // e.g. databricks-volume://catalog/schema/volume/path // or databricks-volume://host/Volumes/catalog/schema/volume/path @@ -132,11 +145,15 @@ func (fs *DatabricksVolumeFileSysClient) parseURL(rawURL string) { func (fs *DatabricksVolumeFileSysClient) Prefix(suffix ...string) string { var prefix string if fs.catalog != "" && fs.schema != "" && fs.volume != "" { - prefix = fmt.Sprintf("databricks-volume://%s/%s/%s/", fs.catalog, fs.schema, fs.volume) + prefix = fmt.Sprintf("databricks-volume://%s/%s/%s", fs.catalog, fs.schema, fs.volume) } else { prefix = "databricks-volume://" } - return prefix + strings.Join(suffix, "") + s := strings.TrimLeft(strings.Join(suffix, ""), "/") + if s != "" { + return strings.TrimRight(prefix, "/") + "/" + s + } + return strings.TrimRight(prefix, "/") + "/" } // GetPath converts a uri into the internal volume path starting with /Volumes/... @@ -151,7 +168,14 @@ func (fs *DatabricksVolumeFileSysClient) GetPath(uri string) (volumePath string, } } - parts := strings.Split(strings.Trim(clean, "/"), "/") + rawParts := strings.Split(strings.Trim(clean, "/"), "/") + var parts []string + for _, p := range rawParts { + if p != "" { + parts = append(parts, p) + } + } + if len(parts) > 0 && (strings.Contains(parts[0], ".databricks.com") || strings.Contains(parts[0], ".azuredatabricks.net")) { parts = parts[1:] } @@ -178,9 +202,16 @@ func (fs *DatabricksVolumeFileSysClient) GetPath(uri string) (volumePath string, return "", g.Error("invalid volume URI, catalog/schema/volume must be specified: %s", uri) } + var cleanSubparts []string + for _, sp := range subparts { + if sp != "" { + cleanSubparts = append(cleanSubparts, sp) + } + } + p := fmt.Sprintf("/Volumes/%s/%s/%s", catalog, schema, volume) - if len(subparts) > 0 && subparts[0] != "" { - p = p + "/" + strings.Join(subparts, "/") + if len(cleanSubparts) > 0 { + p = p + "/" + strings.Join(cleanSubparts, "/") } return p, nil } @@ -194,7 +225,7 @@ func (fs *DatabricksVolumeFileSysClient) doRequest(ctx context.Context, method, return nil, g.Error("databricks token is required") } - urlStr := fmt.Sprintf("https://%s%s", fs.host, apiPath) + urlStr := fmt.Sprintf("%s://%s%s", fs.scheme, fs.host, apiPath) if len(query) > 0 { urlStr = urlStr + "?" + query.Encode() } @@ -204,6 +235,13 @@ func (fs *DatabricksVolumeFileSysClient) doRequest(ctx context.Context, method, for attempt := 0; attempt <= maxRetries; attempt++ { if attempt > 0 { + if seeker, ok := body.(io.Seeker); ok { + if _, err := seeker.Seek(0, io.SeekStart); err != nil { + return nil, g.Error(err, "could not rewind request body") + } + } else if body != nil { + return nil, g.Error(lastErr, "request failed (body not seekable) for %s %s", method, urlStr) + } sleepDur := time.Duration(1< Date: Sat, 19 Sep 2026 01:13:20 +0200 Subject: [PATCH 11/56] feat(zerobus): align Arrow Flight type mapping with official Zerobus SDK specification --- core/dbio/database/database_zerobus.go | 119 ++++++++++++++++++-- core/dbio/database/database_zerobus_test.go | 38 +++++++ 2 files changed, 149 insertions(+), 8 deletions(-) diff --git a/core/dbio/database/database_zerobus.go b/core/dbio/database/database_zerobus.go index 9dd701c8e..81b3abaae 100644 --- a/core/dbio/database/database_zerobus.go +++ b/core/dbio/database/database_zerobus.go @@ -155,8 +155,8 @@ func (conn *ZerobusConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (co targetFDQN := targetTable.FDQN() g.Info("ingesting into Databricks via Zerobus Arrow stream: %s", targetFDQN) - // Build Arrow Schema from Dataflow columns - arrowSchema := iop.ColumnsToArrowSchema(df.Columns) + // Build Arrow Schema conforming to official Databricks Zerobus SDK specification + arrowSchema := ColumnsToZerobusArrowSchema(df.Columns) mem := memory.NewGoAllocator() batchSize := conn.BatchSize @@ -169,12 +169,26 @@ func (conn *ZerobusConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (co switch dtype.ID() { case arrow.BOOL: return array.NewBooleanBuilder(mem) + case arrow.INT8: + return array.NewInt8Builder(mem) + case arrow.INT16: + return array.NewInt16Builder(mem) case arrow.INT32: return array.NewInt32Builder(mem) case arrow.INT64: return array.NewInt64Builder(mem) + case arrow.FLOAT32: + return array.NewFloat32Builder(mem) case arrow.FLOAT64: return array.NewFloat64Builder(mem) + case arrow.LARGE_STRING: + return array.NewLargeStringBuilder(mem) + case arrow.STRING: + return array.NewStringBuilder(mem) + case arrow.LARGE_BINARY: + return array.NewLargeBinaryBuilder(mem, dtype.(*arrow.LargeBinaryType)) + case arrow.BINARY: + return array.NewBinaryBuilder(mem, dtype.(*arrow.BinaryType)) case arrow.DECIMAL128: return array.NewDecimal128Builder(mem, dtype.(*arrow.Decimal128Type)) case arrow.DATE32: @@ -185,14 +199,10 @@ func (conn *ZerobusConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (co return array.NewTime32Builder(mem, dtype.(*arrow.Time32Type)) case arrow.TIME64: return array.NewTime64Builder(mem, dtype.(*arrow.Time64Type)) - case arrow.STRING: - return array.NewStringBuilder(mem) - case arrow.BINARY: - return array.NewBinaryBuilder(mem, dtype.(*arrow.BinaryType)) case arrow.EXTENSION: return array.NewBuilder(mem, dtype) default: - return array.NewStringBuilder(mem) + return array.NewLargeStringBuilder(mem) } } @@ -251,7 +261,7 @@ func (conn *ZerobusConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (co if colIdx < len(row) { val = row[colIdx] } - iop.AppendToBuilder(builders[colIdx], &col, val) + appendToZerobusBuilder(builders[colIdx], &col, val) } rowsInBatch++ count++ @@ -360,3 +370,96 @@ func (conn *ZerobusConn) sendRecordBatch(table string, record arrow.Record) erro _ = batchBytes return nil } + +// ColumnsToZerobusArrowSchema maps Sling Dataflow columns to the exact Apache Arrow schema +// specified by Databricks Zerobus Arrow Flight ingestion: +// https://github.com/databricks/zerobus-sdk +// +// Key Databricks Delta -> Arrow mappings: +// - STRING -> LargeUtf8 (arrow.BinaryTypes.LargeString) +// - DECIMAL -> LargeUtf8 (text-encoded per Zerobus specification) +// - BINARY -> LargeBinary (arrow.BinaryTypes.LargeBinary) +// - TINYINT -> Int8 +// - SMALLINT -> Int16 +// - INT -> Int32 +// - BIGINT -> Int64 +// - FLOAT -> Float32 +// - DOUBLE -> Float64 +// - BOOLEAN -> Boolean +// - DATE -> Date32 +// - TIMESTAMP -> Timestamp(Microsecond, "UTC") +// - TIMESTAMP_NTZ / DATETIME -> Timestamp(Microsecond, "") +func ColumnsToZerobusArrowSchema(columns iop.Columns) *arrow.Schema { + fields := make([]arrow.Field, len(columns)) + + for i, col := range columns { + var arrowType arrow.DataType + + switch col.Type { + case iop.BoolType: + arrowType = arrow.FixedWidthTypes.Boolean + case iop.SmallIntType: + if strings.EqualFold(col.DbType, "tinyint") || strings.EqualFold(col.DbType, "int8") || strings.EqualFold(col.DbType, "byte") { + arrowType = arrow.PrimitiveTypes.Int8 + } else { + arrowType = arrow.PrimitiveTypes.Int16 + } + case iop.IntegerType: + if strings.EqualFold(col.DbType, "tinyint") || strings.EqualFold(col.DbType, "int8") || strings.EqualFold(col.DbType, "byte") { + arrowType = arrow.PrimitiveTypes.Int8 + } else if strings.EqualFold(col.DbType, "smallint") || strings.EqualFold(col.DbType, "int16") || strings.EqualFold(col.DbType, "short") { + arrowType = arrow.PrimitiveTypes.Int16 + } else { + arrowType = arrow.PrimitiveTypes.Int32 + } + case iop.BigIntType: + arrowType = arrow.PrimitiveTypes.Int64 + case iop.FloatType: + if strings.EqualFold(col.DbType, "float") || strings.EqualFold(col.DbType, "float32") || strings.EqualFold(col.DbType, "real") { + arrowType = arrow.PrimitiveTypes.Float32 + } else { + arrowType = arrow.PrimitiveTypes.Float64 + } + case iop.DecimalType: + // Databricks Zerobus SDK specification: + // Currently, DECIMAL types are encoded as text (LargeUtf8 / string) on both Arrow and Protobuf paths + arrowType = arrow.BinaryTypes.LargeString + case iop.DateType: + arrowType = arrow.FixedWidthTypes.Date32 + case iop.TimestampzType: + arrowType = &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: "UTC"} + case iop.DatetimeType, iop.TimestampType: + arrowType = &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: ""} + case iop.BinaryType: + arrowType = arrow.BinaryTypes.LargeBinary + case iop.StringType, iop.TextType, iop.JsonType: + arrowType = arrow.BinaryTypes.LargeString + default: + arrowType = arrow.BinaryTypes.LargeString + } + + fields[i] = arrow.Field{ + Name: col.Name, + Type: arrowType, + Nullable: true, + } + } + + return arrow.NewSchema(fields, nil) +} + +func appendToZerobusBuilder(builder array.Builder, col *iop.Column, val interface{}) { + if val == nil { + builder.AppendNull() + return + } + switch b := builder.(type) { + case *array.LargeBinaryBuilder: + b.Append([]byte(cast.ToString(val))) + case *array.LargeStringBuilder: + // Text-encoded strings, json, and decimals per Zerobus specification + b.Append(cast.ToString(val)) + default: + iop.AppendToBuilder(builder, col, val) + } +} diff --git a/core/dbio/database/database_zerobus_test.go b/core/dbio/database/database_zerobus_test.go index 49f20adc4..236885b4f 100644 --- a/core/dbio/database/database_zerobus_test.go +++ b/core/dbio/database/database_zerobus_test.go @@ -120,3 +120,41 @@ func TestZerobus_IPCSerialization(t *testing.T) { assert.Equal(t, uint64(2), count) } +func TestZerobus_TypeMapping(t *testing.T) { + cols := iop.Columns{ + {Name: "col_bool", Type: iop.BoolType}, + {Name: "col_tiny", Type: iop.SmallIntType, DbType: "tinyint"}, + {Name: "col_short", Type: iop.SmallIntType, DbType: "smallint"}, + {Name: "col_int", Type: iop.IntegerType}, + {Name: "col_bigint", Type: iop.BigIntType}, + {Name: "col_float", Type: iop.FloatType, DbType: "float"}, + {Name: "col_double", Type: iop.FloatType, DbType: "double"}, + {Name: "col_str", Type: iop.StringType}, + {Name: "col_dec", Type: iop.DecimalType, DbPrecision: 18, DbScale: 4}, + {Name: "col_bin", Type: iop.BinaryType}, + {Name: "col_date", Type: iop.DateType}, + {Name: "col_ts", Type: iop.TimestampzType}, + } + + schema := ColumnsToZerobusArrowSchema(cols) + + // Verify Databricks Zerobus Arrow Flight type specification + assert.Equal(t, "bool", schema.Field(0).Type.Name()) + assert.Equal(t, "int8", schema.Field(1).Type.Name()) + assert.Equal(t, "int16", schema.Field(2).Type.Name()) + assert.Equal(t, "int32", schema.Field(3).Type.Name()) + assert.Equal(t, "int64", schema.Field(4).Type.Name()) + assert.Equal(t, "float32", schema.Field(5).Type.Name()) + assert.Equal(t, "float64", schema.Field(6).Type.Name()) + // STRING must be large_utf8 + assert.Equal(t, "large_utf8", schema.Field(7).Type.Name()) + // DECIMAL must be text-encoded as large_utf8 + assert.Equal(t, "large_utf8", schema.Field(8).Type.Name()) + // BINARY must be large_binary + assert.Equal(t, "large_binary", schema.Field(9).Type.Name()) + // DATE must be date32 + assert.Equal(t, "date32", schema.Field(10).Type.Name()) + // TIMESTAMP must be timestamp with UTC + assert.Equal(t, "timestamp[us, tz=UTC]", schema.Field(11).Type.String()) +} + From bcff92f67d1813d045ec429ba1ae4c8417dc85fa Mon Sep 17 00:00:00 2001 From: themulle Date: Sat, 19 Sep 2026 01:26:59 +0200 Subject: [PATCH 12/56] docs: add Databricks Volume and Zerobus ingestion guide in README --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 78d913ec4..264b3fece 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,30 @@ $ sling conns discover LOCALHOST_DEV ... ``` +--- + +### Databricks Ingestion (Unity Catalog Volumes & Zerobus) + +Sling supports high-throughput ingestion into Databricks environments via two native mechanisms: + +- **Unity Catalog Volumes** (`databricks-volume://`): + Direct streaming of files (Parquet, CSV, etc.) into Unity Catalog Volumes using the Databricks Files REST API: + ```bash + # Stream data directly into a Databricks Volume + sling run --src-conn PG_DB --src-stream public.users \ + --tgt-conn DATABRICKS_VOL --tgt-object "my_cat/my_schema/my_vol/users.parquet" + ``` + +- **Zerobus Streaming** (`zerobus://`): + High-throughput, real-time ingestion into Delta tables using Arrow Flight / Arrow IPC streaming with configurable batch sizes and compression (`none`, `lz4`, `zstd`): + ```bash + # Stream records to Delta table via Zerobus Arrow stream + sling run --src-conn PG_DB --src-stream public.users \ + --tgt-conn ZEROBUS_CONN --tgt-object "main.default.users" + ``` + +--- + ## Installation #### One-liner on Mac / Linux From 0a1b1a0aacdf2f9147a976113c854e93e02b7704 Mon Sep 17 00:00:00 2001 From: themulle Date: Sat, 19 Sep 2026 01:30:39 +0200 Subject: [PATCH 13/56] docs: add connection configuration properties and http/ssl options to README --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 264b3fece..52709be52 100644 --- a/README.md +++ b/README.md @@ -123,12 +123,23 @@ $ sling conns discover LOCALHOST_DEV Sling supports high-throughput ingestion into Databricks environments via two native mechanisms: - **Unity Catalog Volumes** (`databricks-volume://`): - Direct streaming of files (Parquet, CSV, etc.) into Unity Catalog Volumes using the Databricks Files REST API: + Direct streaming of files (Parquet, CSV, etc.) into Unity Catalog Volumes using the Databricks Files REST API. + Supports both HTTPS and HTTP (via `protocol: http` or `use_ssl: false`, useful for local testing, mock servers, or internal proxies): ```bash # Stream data directly into a Databricks Volume sling run --src-conn PG_DB --src-stream public.users \ --tgt-conn DATABRICKS_VOL --tgt-object "my_cat/my_schema/my_vol/users.parquet" + + # Connection properties (sling env YAML): + # type: databricks-volume + # host: adb-123.cloud.databricks.com + # token: dapi... + # protocol: https # or 'http' for local/mock testing (use_ssl: false) + # catalog: my_catalog + # schema: my_schema + # volume: my_volume ``` + Supported URI formats: `databricks-volume://cat/sch/vol/path`, `volume://cat/sch/vol/path`, and `databricks://Volumes/cat/sch/vol/path`. - **Zerobus Streaming** (`zerobus://`): High-throughput, real-time ingestion into Delta tables using Arrow Flight / Arrow IPC streaming with configurable batch sizes and compression (`none`, `lz4`, `zstd`): @@ -136,6 +147,13 @@ Sling supports high-throughput ingestion into Databricks environments via two na # Stream records to Delta table via Zerobus Arrow stream sling run --src-conn PG_DB --src-stream public.users \ --tgt-conn ZEROBUS_CONN --tgt-object "main.default.users" + + # Connection properties (sling env YAML): + # type: zerobus + # host: adb-123.cloud.databricks.com + # token: dapi... # or client_id + client_secret + # batch_size: 10000 # default: 10000 + # compression: zstd # none, lz4, zstd ``` --- From ff58571d864fc68bbba05ce1dff68f500196d978 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 19 Sep 2026 14:09:32 -0300 Subject: [PATCH 14/56] refactor: replace zerobus connection type with databricks copy method Zerobus is no longer a standalone connection type. Direct Delta ingestion now uses copy_method: zerobus on a type: databricks connection, while Unity Catalog Volumes remain a file connection (type: databricks-volume). - Remove TypeDbZerobus URL templating and ZerobusConn dispatch - Consolidate zerobus driver handling under the databricks connection - Update README to reflect the new configuration and link to docs - Document CGO_ENABLED=1 build requirement (SQLite, Zerobus) - Add tests for seekable upload retry on HTTP 500 --- README.md | 39 +- core/dbio/connection/connection.go | 10 - core/dbio/database/database.go | 4 +- core/dbio/database/database_databricks.go | 710 ++++++++++++++++++ .../database_databricks_zerobus_test.go | 348 +++++++++ core/dbio/database/database_zerobus.go | 465 ------------ core/dbio/database/database_zerobus_test.go | 160 ---- core/dbio/dbio_types.go | 208 +++-- core/dbio/filesys/fs.go | 10 +- core/dbio/filesys/fs_databricks_volume.go | 296 ++++++-- .../dbio/filesys/fs_databricks_volume_test.go | 341 ++++++++- core/dbio/filesys/fs_file_node.go | 14 +- core/dbio/templates/connections.json | 70 ++ core/dbio/templates/databricks.yaml | 3 +- core/env/vars.go | 2 + go.mod | 3 +- go.sum | 2 + .../replications/r.125.databricks_volume.yaml | 52 ++ tests/suite.cli.yaml | 7 + 19 files changed, 1868 insertions(+), 876 deletions(-) create mode 100644 core/dbio/database/database_databricks_zerobus_test.go delete mode 100644 core/dbio/database/database_zerobus.go delete mode 100644 core/dbio/database/database_zerobus_test.go create mode 100644 tests/replications/r.125.databricks_volume.yaml diff --git a/README.md b/README.md index 52709be52..3512c82b4 100644 --- a/README.md +++ b/README.md @@ -118,43 +118,7 @@ $ sling conns discover LOCALHOST_DEV --- -### Databricks Ingestion (Unity Catalog Volumes & Zerobus) - -Sling supports high-throughput ingestion into Databricks environments via two native mechanisms: - -- **Unity Catalog Volumes** (`databricks-volume://`): - Direct streaming of files (Parquet, CSV, etc.) into Unity Catalog Volumes using the Databricks Files REST API. - Supports both HTTPS and HTTP (via `protocol: http` or `use_ssl: false`, useful for local testing, mock servers, or internal proxies): - ```bash - # Stream data directly into a Databricks Volume - sling run --src-conn PG_DB --src-stream public.users \ - --tgt-conn DATABRICKS_VOL --tgt-object "my_cat/my_schema/my_vol/users.parquet" - - # Connection properties (sling env YAML): - # type: databricks-volume - # host: adb-123.cloud.databricks.com - # token: dapi... - # protocol: https # or 'http' for local/mock testing (use_ssl: false) - # catalog: my_catalog - # schema: my_schema - # volume: my_volume - ``` - Supported URI formats: `databricks-volume://cat/sch/vol/path`, `volume://cat/sch/vol/path`, and `databricks://Volumes/cat/sch/vol/path`. - -- **Zerobus Streaming** (`zerobus://`): - High-throughput, real-time ingestion into Delta tables using Arrow Flight / Arrow IPC streaming with configurable batch sizes and compression (`none`, `lz4`, `zstd`): - ```bash - # Stream records to Delta table via Zerobus Arrow stream - sling run --src-conn PG_DB --src-stream public.users \ - --tgt-conn ZEROBUS_CONN --tgt-object "main.default.users" - - # Connection properties (sling env YAML): - # type: zerobus - # host: adb-123.cloud.databricks.com - # token: dapi... # or client_id + client_secret - # batch_size: 10000 # default: 10000 - # compression: zstd # none, lz4, zstd - ``` +Databricks: Unity Catalog Volumes are a file connection (`type: databricks-volume`). Direct Delta ingest uses `copy_method: zerobus` on `type: databricks`. See https://docs.slingdata.io/connections/database-connections/databricks --- @@ -200,6 +164,7 @@ Pre-built binaries for macOS, Linux, and Windows are available on the [releases Requirements: - Install Go 1.22+ (https://go.dev/doc/install) - Install a C compiler ([gcc](https://www.google.com/search?q=install+gcc&oq=install+gcc), [tdm-gcc](https://jmeubank.github.io/tdm-gcc/), [mingw](https://www.google.com/search?q=install+mingw), etc) +- `CGO_ENABLED=1` (needed for SQLite and Databricks Zerobus; this repo's build scripts already set it) #### Linux or Mac ```bash diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index d2648b81d..0e454c26c 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -885,16 +885,6 @@ func (c *Connection) setURL() (err error) { } else { template = "databricks://token:{token}@{host}:{port}/sql/1.0/warehouses/{warehouse_id}" } - case dbio.TypeDbZerobus: - setIfMissing("token", c.Data["password"]) - setIfMissing("port", c.Type.DefPort()) - if token := cast.ToString(c.Data["token"]); token != "" { - template = "zerobus://token:{token}@{host}:{port}/{database}" - } else if clientID := cast.ToString(c.Data["client_id"]); clientID != "" { - template = "zerobus://{client_id}:{client_secret}@{host}:{port}/{database}" - } else { - template = "zerobus://{host}:{port}/{database}" - } case dbio.TypeDbD1: setIfMissing("account_id", c.Data["host"]) setIfMissing("api_token", c.Data["password"]) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index 27467d809..ab534cd8b 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -328,8 +328,6 @@ func NewConnContext(ctx context.Context, URL string, props ...string) (Connectio conn = &ODBCConn{URL: URL} } else if strings.HasPrefix(URL, "scylladb:") { conn = &ScyllaDBConn{URL: URL} - } else if strings.HasPrefix(URL, "zerobus") { - conn = &ZerobusConn{URL: URL} } else { conn = &BaseConn{URL: URL} } @@ -380,7 +378,7 @@ func getDriverName(conn Connection) (driverName string) { driverName = "bigquery" case dbio.TypeDbSnowflake: driverName = "snowflake" - case dbio.TypeDbDatabricks, dbio.TypeDbZerobus: + case dbio.TypeDbDatabricks: driverName = "databricks" case dbio.TypeDbSQLite: driverName = "sqlite3" diff --git a/core/dbio/database/database_databricks.go b/core/dbio/database/database_databricks.go index 3a6c15dc0..8b92a3f14 100644 --- a/core/dbio/database/database_databricks.go +++ b/core/dbio/database/database_databricks.go @@ -1,18 +1,28 @@ package database import ( + "bytes" "context" "database/sql" + "encoding/json" "fmt" + "io" + "net/http" "os" "path" "strings" + "time" + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/databricks/databricks-sql-go/driverctx" dbsqllog "github.com/databricks/databricks-sql-go/logger" + "github.com/databricks/zerobus-sdk/go" "github.com/dustin/go-humanize" "github.com/flarco/g" "github.com/flarco/g/net" @@ -32,6 +42,13 @@ type DatabricksConn struct { Warehouse string CopyMethod string TableFormat string + + ZerobusEndpoint string + ClientID string + ClientSecret string + BatchSize int + IPCCompression string + MaxInflightBatches int } // Init initiates the object @@ -53,11 +70,38 @@ func (conn *DatabricksConn) Init() error { if tf := conn.GetProp("table_format"); tf != "" { conn.TableFormat = strings.ToLower(tf) } + if conn.CopyMethod == "zerobus" && conn.TableFormat != "delta" { + g.Debug("copy_method: zerobus requires Delta tables; using table_format=delta instead of %s", conn.TableFormat) + conn.TableFormat = "delta" + } if w := conn.GetProp("warehouse"); w != "" { conn.Warehouse = w } + conn.ZerobusEndpoint = conn.GetProp("zerobus_endpoint") + conn.ClientID = conn.GetProp("client_id") + conn.ClientSecret = conn.GetProp("client_secret") + + if conn.BatchSize <= 0 { + conn.BatchSize = cast.ToInt(conn.GetProp("batch_size")) + if conn.BatchSize <= 0 { + conn.BatchSize = 10000 + } + } + if conn.IPCCompression == "" { + conn.IPCCompression = strings.ToLower(conn.GetProp("ipc_compression", "compression")) + if conn.IPCCompression == "" { + conn.IPCCompression = "none" + } + } + if conn.MaxInflightBatches <= 0 { + conn.MaxInflightBatches = cast.ToInt(conn.GetProp("max_inflight_batches")) + if conn.MaxInflightBatches <= 0 { + conn.MaxInflightBatches = 1000 + } + } + // disable internal log dbsqllog.SetLogLevel("disabled") @@ -174,6 +218,12 @@ func (conn *DatabricksConn) BulkImportFlow(tableFName string, df *iop.Dataflow) switch conn.CopyMethod { case "aws": return conn.CopyViaS3(tableFName, df) + case "zerobus": + table, err := ParseTableName(tableFName, conn.Type) + if err != nil { + return 0, g.Error(err, "could not parse table name: "+tableFName) + } + return conn.CopyViaZerobus(table, df) } // Try volume-based loading as fallback @@ -1142,3 +1192,663 @@ func (conn *DatabricksConn) GetSchemata(level SchemataLevel, schemaName string, return conn.BaseConn.GetSchemata(level, schemaName, tableNames...) } + +type zerobusStream interface { + IngestBatch(ipc []byte) (int64, error) + Flush() error + Close() error + GetUnackedBatches() ([][]byte, error) +} + +type zerobusStreamOpener func(endpoint, workspaceURL, tableName string, schemaIPC []byte, clientID, clientSecret string, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) + +var openZerobusStream zerobusStreamOpener = openZerobusStreamSDK + +var zerobusDescribe = func(conn *DatabricksConn, tableFName string) (iop.Columns, error) { + cols, err := conn.GetColumns(tableFName) + if err != nil { + return nil, g.Error(err, "create the table first or use copy_method: stage") + } + if len(cols) == 0 { + return nil, g.Error("create the table first or use copy_method: stage") + } + return cols, nil +} + +func openZerobusStreamSDK(endpoint, workspaceURL, tableName string, schemaIPC []byte, clientID, clientSecret string, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) { + sdk, err := zerobus.NewZerobusSdk(endpoint, workspaceURL) + if err != nil { + return nil, nil, g.Error(err, "could not create Zerobus SDK") + } + stream, err := sdk.CreateArrowStream(tableName, schemaIPC, clientID, clientSecret, opts) + if err != nil { + sdk.Free() + return nil, nil, g.Error(err, "could not create Zerobus Arrow stream for %s", tableName) + } + return stream, sdk.Free, nil +} + +type zerobusPATHeaders struct { + token, tableName string +} + +func (p *zerobusPATHeaders) GetHeaders() (map[string]string, error) { + return map[string]string{ + "authorization": "Bearer " + p.token, + "x-databricks-zerobus-table-name": p.tableName, + }, nil +} + +func openZerobusStreamPAT(endpoint, workspaceURL, tableName string, schemaIPC []byte, token string, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) { + sdk, err := zerobus.NewZerobusSdk(endpoint, workspaceURL) + if err != nil { + return nil, nil, g.Error(err, "could not create Zerobus SDK") + } + stream, err := sdk.CreateArrowStreamWithHeadersProvider(tableName, schemaIPC, &zerobusPATHeaders{token: token, tableName: tableName}, opts) + if err != nil { + sdk.Free() + return nil, nil, g.Error(err, "could not create Zerobus Arrow stream for %s", tableName) + } + return stream, sdk.Free, nil +} + +func isZerobusSchemaLag(err error) bool { + if err == nil { + return false + } + s := strings.ToLower(err.Error()) + return strings.Contains(s, "schema comparison failed") || + strings.Contains(s, "schema_validation_failed") || + strings.Contains(s, "does not exist in delta schema") || + strings.Contains(s, "field_not_in_table") +} + +func (conn *DatabricksConn) openZerobusArrowStream(endpoint, workspaceURL, tableName string, schemaIPC []byte, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) { + open := func() (zerobusStream, func(), error) { + if conn.ClientID != "" && conn.ClientSecret != "" { + return openZerobusStream(endpoint, workspaceURL, tableName, schemaIPC, conn.ClientID, conn.ClientSecret, opts) + } + return openZerobusStreamPAT(endpoint, workspaceURL, tableName, schemaIPC, conn.GetProp("token"), opts) + } + + stream, free, err := open() + if err == nil || conn.db == nil || !isZerobusSchemaLag(err) { + return stream, free, err + } + + deadline := time.Now().Add(45 * time.Second) + for time.Now().Before(deadline) { + g.Debug("zerobus Delta schema not yet visible for %s, retrying", tableName) + time.Sleep(time.Second) + stream, free, err = open() + if err == nil || !isZerobusSchemaLag(err) { + return stream, free, err + } + } + return nil, nil, err +} + +func (conn *DatabricksConn) databricksGETJSON(path string, dest any) error { + workspaceURL, err := conn.unityCatalogURL() + if err != nil { + return err + } + token := conn.GetProp("token") + if token == "" { + return g.Error("databricks token is required") + } + req, err := http.NewRequest(http.MethodGet, workspaceURL+path, nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return g.Error("databricks API %s returned %d: %s", path, resp.StatusCode, string(body)) + } + if dest == nil { + return nil + } + return json.Unmarshal(body, dest) +} + +func (conn *DatabricksConn) resolveZerobusEndpoint() error { + if strings.TrimSpace(conn.ZerobusEndpoint) != "" { + return nil + } + if v := conn.GetProp("zerobus_endpoint"); v != "" { + conn.ZerobusEndpoint = v + return nil + } + if conn.GetProp("host") == "" || conn.GetProp("token") == "" { + return g.Error("zerobus_endpoint is required for copy_method: zerobus (the shard URL, not the workspace host)") + } + + var assignment struct { + WorkspaceID json.Number `json:"workspace_id"` + MetastoreID string `json:"metastore_id"` + } + if err := conn.databricksGETJSON("/api/2.1/unity-catalog/current-metastore-assignment", &assignment); err != nil { + return g.Error(err, "zerobus_endpoint is required for copy_method: zerobus (the shard URL, not the workspace host)") + } + + var listing struct { + Metastores []struct { + MetastoreID string `json:"metastore_id"` + Region string `json:"region"` + Cloud string `json:"cloud"` + } `json:"metastores"` + } + _ = conn.databricksGETJSON("/api/2.1/unity-catalog/metastores", &listing) + + region, cloud := "", "aws" + for _, m := range listing.Metastores { + if m.MetastoreID == assignment.MetastoreID || (assignment.MetastoreID == "" && m.Region != "") { + region = m.Region + if m.Cloud != "" { + cloud = strings.ToLower(m.Cloud) + } + if m.MetastoreID == assignment.MetastoreID { + break + } + } + } + workspaceID := assignment.WorkspaceID.String() + if workspaceID == "" || region == "" { + return g.Error("zerobus_endpoint is required for copy_method: zerobus (could not detect workspace_id/region)") + } + + suffix := "cloud.databricks.com" + host := conn.GetProp("host") + switch { + case strings.Contains(host, "azuredatabricks.net"): + suffix = "azuredatabricks.net" + case strings.Contains(host, "gcp.databricks.com"): + suffix = "gcp.databricks.com" + case cloud == "azure": + suffix = "azuredatabricks.net" + case cloud == "gcp": + suffix = "gcp.databricks.com" + } + + conn.ZerobusEndpoint = fmt.Sprintf("https://%s.zerobus.%s.%s", workspaceID, region, suffix) + g.Debug("detected zerobus_endpoint=%s", conn.ZerobusEndpoint) + return nil +} + +func (conn *DatabricksConn) validateZerobusConfig() error { + if err := conn.resolveZerobusEndpoint(); err != nil { + return err + } + if conn.ZerobusEndpoint == "" { + return g.Error("zerobus_endpoint is required for copy_method: zerobus (the shard URL, not the workspace host)") + } + if (conn.ClientID == "" || conn.ClientSecret == "") && conn.GetProp("token") == "" { + return g.Error("client_id and client_secret (OAuth M2M) or token (PAT) are required for copy_method: zerobus") + } + return nil +} + +func (conn *DatabricksConn) unityCatalogURL() (string, error) { + host := conn.GetProp("host") + host = strings.TrimPrefix(host, "https://") + host = strings.TrimPrefix(host, "http://") + host = strings.TrimRight(host, "/") + if host == "" { + return "", g.Error("databricks host is required for copy_method: zerobus") + } + return "https://" + host, nil +} + +func (conn *DatabricksConn) zerobusTableName(table Table) string { + catalog := table.Database + if catalog == "" { + catalog = conn.Catalog + } + schema := table.Schema + if schema == "" { + schema = conn.Schema + } + parts := []string{} + if catalog != "" { + parts = append(parts, catalog) + } + if schema != "" { + parts = append(parts, schema) + } + parts = append(parts, table.Name) + return strings.Join(parts, ".") +} + +func mapZerobusIPCCompression(s string) (zerobus.IPCCompressionType, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "", "none": + return zerobus.IPCCompressionNone, nil + case "lz4", "lz4_frame": + return zerobus.IPCCompressionLZ4Frame, nil + case "zstd", "zstandard": + return zerobus.IPCCompressionZstd, nil + default: + return 0, g.Error("unsupported Zerobus IPC compression: %s (supported: none, lz4, zstd)", s) + } +} + +func normalizeZerobusEndpoint(endpoint string) string { + endpoint = strings.TrimRight(endpoint, "/") + if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") { + return endpoint + } + return "https://" + endpoint +} + +// CopyViaZerobus streams Arrow RecordBatches into an existing Delta table. +func (conn *DatabricksConn) CopyViaZerobus(table Table, df *iop.Dataflow) (count uint64, err error) { + if err = conn.validateZerobusConfig(); err != nil { + return 0, err + } + + workspaceURL, err := conn.unityCatalogURL() + if err != nil { + return 0, err + } + + tableName := conn.zerobusTableName(table) + if table.Name == "" { + return 0, g.Error("target table must be specified for Zerobus ingestion") + } + + tgtCols, err := zerobusDescribe(conn, table.FullName()) + if err != nil { + return 0, err + } + for i := range tgtCols { + if v := tgtCols[i].Metadata["is_nullable"]; v != "" { + tgtCols[i].SetMetadata(string(iop.ColMetaNullable), v) + } + } + + srcIdx, err := alignZerobusSource(df.Columns, tgtCols) + if err != nil { + return 0, err + } + + arrowSchema, err := ColumnsToZerobusArrowSchema(tgtCols) + if err != nil { + return 0, err + } + + schemaIPC, err := SerializeSchemaToIPC(arrowSchema) + if err != nil { + return 0, err + } + + codec, err := mapZerobusIPCCompression(conn.IPCCompression) + if err != nil { + return 0, err + } + + opts := zerobus.DefaultArrowStreamConfigurationOptions() + if conn.MaxInflightBatches > 0 { + opts.MaxInflightBatches = uint64(conn.MaxInflightBatches) + } + opts.IPCCompression = codec + + g.Info("ingesting into Databricks via Zerobus Arrow stream: %s", tableName) + + endpoint := normalizeZerobusEndpoint(conn.ZerobusEndpoint) + stream, free, err := conn.openZerobusArrowStream(endpoint, workspaceURL, tableName, schemaIPC, opts) + if err != nil { + return 0, err + } + if free != nil { + defer free() + } + defer stream.Close() + + count, err = ingestZerobusFlow(conn, df, stream, arrowSchema, tgtCols, srcIdx) + if err != nil { + unacked, _ := stream.GetUnackedBatches() + return 0, g.Error(err, "zerobus ingest failed, %d batches unacked", len(unacked)) + } + + if err = stream.Flush(); err != nil { + unacked, _ := stream.GetUnackedBatches() + return 0, g.Error(err, "zerobus flush failed, %d batches unacked", len(unacked)) + } + + // SQL warehouse snapshots can lag the Zerobus Delta commit. + if count > 0 && conn.db != nil { + _, _ = conn.Exec("REFRESH TABLE " + table.FullName() + env.NoDebugKey) + deadline := time.Now().Add(30 * time.Second) + var visible int64 + for { + visible, err = conn.GetCount(table.FullName()) + if err == nil && uint64(visible) >= count { + break + } + if time.Now().After(deadline) { + if err != nil { + return count, g.Error(err, "zerobus rows not yet visible in SQL warehouse for %s", table.FullName()) + } + return count, g.Error("zerobus SQL warehouse count is %d after streaming %d rows into %s", visible, count, table.FullName()) + } + time.Sleep(500 * time.Millisecond) + } + } + + g.Info("successfully streamed %d rows to Zerobus table %s", count, tableName) + return count, nil +} + +func alignZerobusSource(src, tgt iop.Columns) ([]int, error) { + srcMap := map[string]int{} + for i, c := range src { + srcMap[strings.ToLower(c.Name)] = i + } + + used := map[string]bool{} + srcIdx := make([]int, len(tgt)) + for i, tcol := range tgt { + si, ok := srcMap[strings.ToLower(tcol.Name)] + if !ok { + if !columnZerobusNullable(tcol) { + return nil, g.Error("source is missing non-null target column %s", tcol.Name) + } + srcIdx[i] = -1 + continue + } + used[strings.ToLower(tcol.Name)] = true + srcIdx[i] = si + } + + for _, c := range src { + if !used[strings.ToLower(c.Name)] { + return nil, g.Error("source has extra column %s not in target table", c.Name) + } + } + return srcIdx, nil +} + +func ingestZerobusFlow(conn *DatabricksConn, df *iop.Dataflow, stream zerobusStream, arrowSchema *arrow.Schema, tgtCols iop.Columns, srcIdx []int) (count uint64, err error) { + mem := memory.NewGoAllocator() + batchSize := conn.BatchSize + if batchSize <= 0 { + batchSize = 10000 + } + + builders := make([]array.Builder, len(tgtCols)) + createBuilder := func(dtype arrow.DataType) array.Builder { + switch dtype.ID() { + case arrow.BOOL: + return array.NewBooleanBuilder(mem) + case arrow.INT8: + return array.NewInt8Builder(mem) + case arrow.INT16: + return array.NewInt16Builder(mem) + case arrow.INT32: + return array.NewInt32Builder(mem) + case arrow.INT64: + return array.NewInt64Builder(mem) + case arrow.FLOAT32: + return array.NewFloat32Builder(mem) + case arrow.FLOAT64: + return array.NewFloat64Builder(mem) + case arrow.LARGE_STRING: + return array.NewLargeStringBuilder(mem) + case arrow.STRING: + return array.NewStringBuilder(mem) + case arrow.LARGE_BINARY: + return array.NewBinaryBuilder(mem, arrow.BinaryTypes.LargeBinary) + case arrow.BINARY: + return array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary) + case arrow.DATE32: + return array.NewDate32Builder(mem) + case arrow.TIMESTAMP: + return array.NewTimestampBuilder(mem, dtype.(*arrow.TimestampType)) + case arrow.DECIMAL128: + return array.NewDecimal128Builder(mem, dtype.(*arrow.Decimal128Type)) + default: + return array.NewLargeStringBuilder(mem) + } + } + + resetBuilders := func() { + for i, field := range arrowSchema.Fields() { + builders[i] = createBuilder(field.Type) + } + } + releaseBuilders := func() { + for _, b := range builders { + if b != nil { + b.Release() + } + } + } + defer releaseBuilders() + resetBuilders() + + rowsInBatch := 0 + flushBatch := func() error { + if rowsInBatch == 0 { + return nil + } + + arrays := make([]arrow.Array, len(builders)) + for i, b := range builders { + arrays[i] = b.NewArray() + } + record := array.NewRecord(arrowSchema, arrays, int64(rowsInBatch)) + + batchBytes, err := SerializeRecordToIPC(arrowSchema, record, conn.IPCCompression) + record.Release() + for _, arr := range arrays { + arr.Release() + } + releaseBuilders() + resetBuilders() + rowsInBatch = 0 + if err != nil { + return err + } + + _, err = stream.IngestBatch(batchBytes) + return err + } + + for ds := range df.StreamCh { + for row := range ds.Rows() { + for i, col := range tgtCols { + var val interface{} + si := srcIdx[i] + if si >= 0 && si < len(row) { + val = row[si] + } + appendToZerobusBuilder(builders[i], &col, val) + } + rowsInBatch++ + count++ + if rowsInBatch >= batchSize { + if err := flushBatch(); err != nil { + return count, g.Error(err, "failed to stream Arrow RecordBatch to Zerobus") + } + } + } + if err := ds.Context.Err(); err != nil { + return count, g.Error(err, "error reading source stream") + } + } + + if err := flushBatch(); err != nil { + return count, g.Error(err, "failed to flush final Arrow RecordBatch to Zerobus") + } + return count, nil +} + +// SerializeSchemaToIPC serializes an Arrow Schema into IPC stream bytes without data batches, +// exactly as expected by the Zerobus SDK (sdk.CreateArrowStream(table, schemaIPC, ...)). +func SerializeSchemaToIPC(schema *arrow.Schema) ([]byte, error) { + var buf bytes.Buffer + w := ipc.NewWriter(&buf, ipc.WithSchema(schema)) + if err := w.Close(); err != nil { + return nil, g.Error(err, "failed to serialize Arrow Schema to IPC bytes for Zerobus") + } + return buf.Bytes(), nil +} + +// SerializeRecordToIPC serializes an Arrow Record into IPC stream bytes containing exactly one RecordBatch. +func SerializeRecordToIPC(schema *arrow.Schema, record arrow.Record, compression string) ([]byte, error) { + var buf bytes.Buffer + opts := []ipc.Option{ipc.WithSchema(schema)} + + switch strings.ToLower(compression) { + case "lz4", "lz4_frame": + opts = append(opts, ipc.WithLZ4()) + case "zstd", "zstandard": + opts = append(opts, ipc.WithZstd()) + case "", "none": + default: + return nil, g.Error("unsupported Zerobus IPC compression: %s (supported: none, lz4, zstd)", compression) + } + + w := ipc.NewWriter(&buf, opts...) + if err := w.Write(record); err != nil { + w.Close() + return nil, g.Error(err, "failed to write Arrow Record to IPC writer for Zerobus") + } + if err := w.Close(); err != nil { + return nil, g.Error(err, "failed to close Arrow IPC writer for Zerobus") + } + + return buf.Bytes(), nil +} + +func zerobusUnsupportedDbType(col iop.Column) error { + dt := strings.ToLower(strings.TrimSpace(col.DbType)) + if dt == "" { + return nil + } + if strings.HasPrefix(dt, "array") || strings.HasPrefix(dt, "map") || + strings.HasPrefix(dt, "struct") || strings.HasPrefix(dt, "variant") || dt == "object" { + return g.Error("unsupported Zerobus type %s for column %s", col.DbType, col.Name) + } + return nil +} + +func zerobusDecimalPrecisionScale(col iop.Column) (prec, scale int) { + prec = col.DbPrecision + scale = col.DbScale + if prec <= 0 { + dt := strings.ToLower(col.DbType) + if i := strings.Index(dt, "("); i >= 0 { + nums := strings.TrimSuffix(dt[i+1:], ")") + parts := strings.Split(nums, ",") + if len(parts) >= 1 { + prec = cast.ToInt(strings.TrimSpace(parts[0])) + } + if len(parts) >= 2 { + scale = cast.ToInt(strings.TrimSpace(parts[1])) + } + } + } + if prec <= 0 { + prec = 38 + } + if scale < 0 { + scale = 0 + } + return prec, scale +} + +func columnZerobusNullable(col iop.Column) bool { + if col.Metadata != nil { + if v, ok := col.Metadata["is_nullable"]; ok { + return v == "true" || strings.EqualFold(v, "yes") + } + } + return col.IsNullable() +} + +// ColumnsToZerobusArrowSchema maps Sling columns to the Arrow schema specified by +// Databricks Zerobus Arrow Flight ingestion. +func ColumnsToZerobusArrowSchema(columns iop.Columns) (*arrow.Schema, error) { + fields := make([]arrow.Field, len(columns)) + + for i, col := range columns { + if err := zerobusUnsupportedDbType(col); err != nil { + return nil, err + } + + var arrowType arrow.DataType + + switch col.Type { + case iop.BoolType: + arrowType = arrow.FixedWidthTypes.Boolean + case iop.SmallIntType: + if strings.EqualFold(col.DbType, "tinyint") || strings.EqualFold(col.DbType, "int8") || strings.EqualFold(col.DbType, "byte") { + arrowType = arrow.PrimitiveTypes.Int8 + } else { + arrowType = arrow.PrimitiveTypes.Int16 + } + case iop.IntegerType: + if strings.EqualFold(col.DbType, "tinyint") || strings.EqualFold(col.DbType, "int8") || strings.EqualFold(col.DbType, "byte") { + arrowType = arrow.PrimitiveTypes.Int8 + } else if strings.EqualFold(col.DbType, "smallint") || strings.EqualFold(col.DbType, "int16") || strings.EqualFold(col.DbType, "short") { + arrowType = arrow.PrimitiveTypes.Int16 + } else { + arrowType = arrow.PrimitiveTypes.Int32 + } + case iop.BigIntType: + arrowType = arrow.PrimitiveTypes.Int64 + case iop.FloatType: + if strings.EqualFold(col.DbType, "float") || strings.EqualFold(col.DbType, "float32") || strings.EqualFold(col.DbType, "real") { + arrowType = arrow.PrimitiveTypes.Float32 + } else { + arrowType = arrow.PrimitiveTypes.Float64 + } + case iop.DecimalType: + prec, scale := zerobusDecimalPrecisionScale(col) + arrowType = &arrow.Decimal128Type{Precision: int32(prec), Scale: int32(scale)} + case iop.DateType: + arrowType = arrow.FixedWidthTypes.Date32 + case iop.TimestampzType: + arrowType = &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: "UTC"} + case iop.DatetimeType, iop.TimestampType: + arrowType = &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: ""} + case iop.BinaryType: + arrowType = arrow.BinaryTypes.LargeBinary + case iop.StringType, iop.TextType, iop.JsonType, iop.UUIDType: + arrowType = arrow.BinaryTypes.LargeString + default: + arrowType = arrow.BinaryTypes.LargeString + } + + fields[i] = arrow.Field{ + Name: col.Name, + Type: arrowType, + Nullable: columnZerobusNullable(col), + } + } + + return arrow.NewSchema(fields, nil), nil +} + +func appendToZerobusBuilder(builder array.Builder, col *iop.Column, val interface{}) { + if val == nil { + builder.AppendNull() + return + } + switch b := builder.(type) { + case *array.LargeStringBuilder: + b.Append(cast.ToString(val)) + default: + iop.AppendToBuilder(builder, col, val) + } +} diff --git a/core/dbio/database/database_databricks_zerobus_test.go b/core/dbio/database/database_databricks_zerobus_test.go new file mode 100644 index 000000000..8958f1fdb --- /dev/null +++ b/core/dbio/database/database_databricks_zerobus_test.go @@ -0,0 +1,348 @@ +package database + +import ( + "bytes" + "context" + "fmt" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/databricks/zerobus-sdk/go" + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeZerobusStream struct { + batches [][]byte + ingestErr error + flushErr error + closeErr error + ingestCalled int + flushCalled int + closeCalled int +} + +func (f *fakeZerobusStream) IngestBatch(ipcBytes []byte) (int64, error) { + f.ingestCalled++ + if f.ingestErr != nil { + return -1, f.ingestErr + } + f.batches = append(f.batches, ipcBytes) + return int64(f.ingestCalled), nil +} + +func (f *fakeZerobusStream) Flush() error { + f.flushCalled++ + return f.flushErr +} + +func (f *fakeZerobusStream) Close() error { + f.closeCalled++ + return f.closeErr +} + +func (f *fakeZerobusStream) GetUnackedBatches() ([][]byte, error) { + return f.batches, nil +} + +func newTestDatabricksConn() *DatabricksConn { + conn := &DatabricksConn{} + conn.setContext(context.Background(), 1) + return conn +} + +func TestMapZerobusIPCCompression(t *testing.T) { + none, err := mapZerobusIPCCompression("none") + require.NoError(t, err) + assert.Equal(t, zerobus.IPCCompressionNone, none) + assert.NotEqual(t, zerobus.IPCCompressionDefault, none) + + empty, err := mapZerobusIPCCompression("") + require.NoError(t, err) + assert.Equal(t, zerobus.IPCCompressionNone, empty) + + lz4, err := mapZerobusIPCCompression("lz4") + require.NoError(t, err) + assert.Equal(t, zerobus.IPCCompressionLZ4Frame, lz4) + + zstd, err := mapZerobusIPCCompression("zstd") + require.NoError(t, err) + assert.Equal(t, zerobus.IPCCompressionZstd, zstd) + + _, err = mapZerobusIPCCompression("gzip") + assert.Error(t, err) +} + +func TestIsZerobusSchemaLag(t *testing.T) { + assert.False(t, isZerobusSchemaLag(nil)) + assert.False(t, isZerobusSchemaLag(fmt.Errorf("invalid_client"))) + assert.True(t, isZerobusSchemaLag(fmt.Errorf("Schema comparison failed: Client field 'json_data' does not exist in Delta schema"))) + assert.True(t, isZerobusSchemaLag(fmt.Errorf("SCHEMA_VALIDATION_FAILED"))) + assert.True(t, isZerobusSchemaLag(fmt.Errorf("FIELD_NOT_IN_TABLE"))) +} + +func TestZerobusValidateConfig(t *testing.T) { + conn := newTestDatabricksConn() + conn.CopyMethod = "zerobus" + err := conn.validateZerobusConfig() + assert.Error(t, err) + assert.Contains(t, err.Error(), "zerobus_endpoint") + + conn.ZerobusEndpoint = "https://123.zerobus.us-west-2.cloud.databricks.com" + err = conn.validateZerobusConfig() + assert.Error(t, err) + assert.Contains(t, err.Error(), "client_id") + + conn.ClientID = "id" + conn.ClientSecret = "secret" + assert.NoError(t, conn.validateZerobusConfig()) +} + +func TestColumnsToZerobusArrowSchema(t *testing.T) { + cols := iop.Columns{ + {Name: "col_bool", Type: iop.BoolType}, + {Name: "col_tiny", Type: iop.SmallIntType, DbType: "tinyint"}, + {Name: "col_short", Type: iop.SmallIntType, DbType: "smallint"}, + {Name: "col_int", Type: iop.IntegerType}, + {Name: "col_int_tiny", Type: iop.IntegerType, DbType: "tinyint"}, + {Name: "col_bigint", Type: iop.BigIntType}, + {Name: "col_float", Type: iop.FloatType, DbType: "float"}, + {Name: "col_double", Type: iop.FloatType, DbType: "double"}, + {Name: "col_str", Type: iop.StringType}, + {Name: "col_dec", Type: iop.DecimalType, DbPrecision: 18, DbScale: 4}, + {Name: "col_bin", Type: iop.BinaryType}, + {Name: "col_date", Type: iop.DateType}, + {Name: "col_tsz", Type: iop.TimestampzType}, + {Name: "col_ntz", Type: iop.TimestampType, DbType: "timestamp_ntz"}, + } + + schema, err := ColumnsToZerobusArrowSchema(cols) + require.NoError(t, err) + + assert.Equal(t, "bool", schema.Field(0).Type.Name()) + assert.Equal(t, "int8", schema.Field(1).Type.Name()) + assert.Equal(t, "int16", schema.Field(2).Type.Name()) + assert.Equal(t, "int32", schema.Field(3).Type.Name()) + assert.Equal(t, "int8", schema.Field(4).Type.Name()) + assert.Equal(t, "int64", schema.Field(5).Type.Name()) + assert.Equal(t, "float32", schema.Field(6).Type.Name()) + assert.Equal(t, "float64", schema.Field(7).Type.Name()) + assert.Equal(t, "large_utf8", schema.Field(8).Type.Name()) + assert.Equal(t, "decimal(18, 4)", schema.Field(9).Type.String()) + assert.Equal(t, "large_binary", schema.Field(10).Type.Name()) + assert.Equal(t, "date32", schema.Field(11).Type.Name()) + assert.Equal(t, "timestamp[us, tz=UTC]", schema.Field(12).Type.String()) + assert.Equal(t, "timestamp[us]", schema.Field(13).Type.String()) +} + +func TestColumnsToZerobusArrowSchema_Unsupported(t *testing.T) { + _, err := ColumnsToZerobusArrowSchema(iop.Columns{ + {Name: "arr", Type: iop.JsonType, DbType: "array"}, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unsupported Zerobus type") + + _, err = ColumnsToZerobusArrowSchema(iop.Columns{ + {Name: "m", Type: iop.JsonType, DbType: "map"}, + }) + assert.Error(t, err) + + _, err = ColumnsToZerobusArrowSchema(iop.Columns{ + {Name: "s", Type: iop.JsonType, DbType: "struct"}, + }) + assert.Error(t, err) + + _, err = ColumnsToZerobusArrowSchema(iop.Columns{ + {Name: "v", Type: iop.JsonType, DbType: "variant"}, + }) + assert.Error(t, err) +} + +func TestColumnsToZerobusArrowSchema_Nullability(t *testing.T) { + cols := iop.Columns{ + {Name: "id", Type: iop.BigIntType, Metadata: map[string]string{"is_nullable": "false"}}, + {Name: "name", Type: iop.StringType, Metadata: map[string]string{"is_nullable": "true"}}, + } + schema, err := ColumnsToZerobusArrowSchema(cols) + require.NoError(t, err) + assert.False(t, schema.Field(0).Nullable) + assert.True(t, schema.Field(1).Nullable) +} + +func TestSerializeRecordToIPC_RoundTrip(t *testing.T) { + cols := iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "name", Type: iop.StringType}, + } + schema, err := ColumnsToZerobusArrowSchema(cols) + require.NoError(t, err) + + schemaBytes, err := SerializeSchemaToIPC(schema) + require.NoError(t, err) + assert.NotEmpty(t, schemaBytes) + + mem := memory.NewGoAllocator() + idB := array.NewInt64Builder(mem) + nameB := array.NewLargeStringBuilder(mem) + idB.AppendValues([]int64{1, 2}, nil) + nameB.AppendValues([]string{"Alice", "Bob"}, nil) + idA := idB.NewArray() + nameA := nameB.NewArray() + defer idA.Release() + defer nameA.Release() + rec := array.NewRecord(schema, []arrow.Array{idA, nameA}, 2) + defer rec.Release() + + for _, compression := range []string{"none", "lz4", "zstd"} { + t.Run(compression, func(t *testing.T) { + b, err := SerializeRecordToIPC(schema, rec, compression) + require.NoError(t, err) + assert.NotEmpty(t, b) + + r, err := ipc.NewReader(bytes.NewReader(b)) + require.NoError(t, err) + defer r.Release() + assert.True(t, r.Next()) + got := r.Record() + assert.Equal(t, int64(2), got.NumRows()) + assert.False(t, r.Next()) + assert.NoError(t, r.Err()) + }) + } +} + +func TestAlignZerobusSource(t *testing.T) { + src := iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "name", Type: iop.StringType}, + } + tgt := iop.Columns{ + {Name: "id", Type: iop.BigIntType, Metadata: map[string]string{"nullable": "false"}}, + {Name: "name", Type: iop.StringType}, + {Name: "note", Type: iop.StringType, Metadata: map[string]string{"is_nullable": "true"}}, + } + idx, err := alignZerobusSource(src, tgt) + require.NoError(t, err) + assert.Equal(t, []int{0, 1, -1}, idx) + + _, err = alignZerobusSource(src, iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "missing", Type: iop.StringType, Metadata: map[string]string{"is_nullable": "false"}}, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "missing non-null") + + _, err = alignZerobusSource(iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "extra", Type: iop.StringType}, + }, iop.Columns{{Name: "id", Type: iop.BigIntType}}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "extra column") +} + +func TestCopyViaZerobus_FakeStream(t *testing.T) { + origOpen := openZerobusStream + origDescribe := zerobusDescribe + t.Cleanup(func() { + openZerobusStream = origOpen + zerobusDescribe = origDescribe + }) + + fake := &fakeZerobusStream{} + openZerobusStream = func(endpoint, workspaceURL, tableName string, schemaIPC []byte, clientID, clientSecret string, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) { + assert.Equal(t, "https://123.zerobus.us-west-2.cloud.databricks.com", endpoint) + assert.Equal(t, "https://dbc.cloud.databricks.com", workspaceURL) + assert.Equal(t, "main.default.users", tableName) + assert.NotEmpty(t, schemaIPC) + assert.Equal(t, zerobus.IPCCompressionNone, opts.IPCCompression) + return fake, func() {}, nil + } + zerobusDescribe = func(conn *DatabricksConn, tableFName string) (iop.Columns, error) { + return iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "name", Type: iop.StringType}, + }, nil + } + + conn := newTestDatabricksConn() + conn.CopyMethod = "zerobus" + conn.ZerobusEndpoint = "123.zerobus.us-west-2.cloud.databricks.com" + conn.ClientID = "id" + conn.ClientSecret = "secret" + conn.BatchSize = 2 + conn.IPCCompression = "none" + conn.MaxInflightBatches = 1000 + conn.Catalog = "main" + conn.Schema = "default" + conn.SetProp("host", "dbc.cloud.databricks.com") + + cols := iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "name", Type: iop.StringType}, + } + data := iop.NewDataset(cols) + data.Rows = [][]any{ + {int64(1), "Alice"}, + {int64(2), "Bob"}, + {int64(3), "Charlie"}, + {int64(4), nil}, + {int64(5), "Eve"}, + } + df, err := iop.MakeDataFlow(data.Stream()) + require.NoError(t, err) + + table := Table{Database: "main", Schema: "default", Name: "users"} + count, err := conn.CopyViaZerobus(table, df) + require.NoError(t, err) + assert.Equal(t, uint64(5), count) + assert.Equal(t, 3, fake.ingestCalled) // batch_size 2 → 2+2+1 + assert.Equal(t, 1, fake.flushCalled) + assert.Equal(t, 1, fake.closeCalled) +} + +func TestCopyViaZerobus_FlushError(t *testing.T) { + origOpen := openZerobusStream + origDescribe := zerobusDescribe + t.Cleanup(func() { + openZerobusStream = origOpen + zerobusDescribe = origDescribe + }) + + fake := &fakeZerobusStream{flushErr: assert.AnError} + openZerobusStream = func(endpoint, workspaceURL, tableName string, schemaIPC []byte, clientID, clientSecret string, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) { + return fake, func() {}, nil + } + zerobusDescribe = func(conn *DatabricksConn, tableFName string) (iop.Columns, error) { + return iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + }, nil + } + + conn := newTestDatabricksConn() + conn.ZerobusEndpoint = "https://z.example" + conn.ClientID = "id" + conn.ClientSecret = "secret" + conn.BatchSize = 10 + conn.IPCCompression = "none" + conn.SetProp("host", "dbc.cloud.databricks.com") + + data := iop.NewDataset(iop.Columns{{Name: "id", Type: iop.BigIntType}}) + data.Rows = [][]any{{int64(1)}} + df, err := iop.MakeDataFlow(data.Stream()) + require.NoError(t, err) + _, err = conn.CopyViaZerobus(Table{Name: "t", Schema: "s", Database: "c"}, df) + assert.Error(t, err) + assert.Contains(t, err.Error(), "zerobus flush failed") + assert.Equal(t, 1, fake.closeCalled) +} + +func TestCopyViaZerobus_MissingEndpoint(t *testing.T) { + conn := newTestDatabricksConn() + _, err := conn.CopyViaZerobus(Table{Name: "t"}, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "zerobus_endpoint") +} diff --git a/core/dbio/database/database_zerobus.go b/core/dbio/database/database_zerobus.go deleted file mode 100644 index 81b3abaae..000000000 --- a/core/dbio/database/database_zerobus.go +++ /dev/null @@ -1,465 +0,0 @@ -package database - -import ( - "bytes" - "context" - "fmt" - "strings" - "time" - - "github.com/apache/arrow-go/v18/arrow" - "github.com/apache/arrow-go/v18/arrow/array" - "github.com/apache/arrow-go/v18/arrow/ipc" - "github.com/apache/arrow-go/v18/arrow/memory" - "github.com/flarco/g" - "github.com/flarco/g/net" - "github.com/slingdata-io/sling-cli/core/dbio" - "github.com/slingdata-io/sling-cli/core/dbio/iop" - "github.com/spf13/cast" -) - -// ZerobusConn represents a connection for ingesting data into Databricks via Zerobus / Arrow Flight -type ZerobusConn struct { - BaseConn - URL string - WorkspaceURL string - ShardURL string - ClientID string - ClientSecret string - Token string - Catalog string - Schema string - Table string - BatchSize int - Compression string // "none", "lz4", "zstd" (matching Zerobus IPCCompressionType) - MaxInflightBatches int // defaults to 1000 -} - -// Init initiates the Zerobus connection -func (conn *ZerobusConn) Init() error { - conn.BaseConn.URL = conn.URL - conn.BaseConn.Type = dbio.TypeDbZerobus - conn.BaseConn.defaultPort = 443 - - instance := Connection(conn) - conn.BaseConn.instance = &instance - - conn.WorkspaceURL = conn.GetProp("workspace_url") - if conn.WorkspaceURL == "" { - conn.WorkspaceURL = conn.GetProp("host") - } - conn.ShardURL = conn.GetProp("shard_url") - conn.ClientID = conn.GetProp("client_id") - conn.ClientSecret = conn.GetProp("client_secret") - conn.Token = conn.GetProp("token", "password") - conn.Catalog = conn.GetProp("catalog") - conn.Schema = conn.GetProp("schema") - conn.Table = conn.GetProp("table", "database") - - conn.BatchSize = cast.ToInt(conn.GetProp("batch_size")) - if conn.BatchSize <= 0 { - conn.BatchSize = 10000 // Standard 10k Arrow record buffer - } - - conn.Compression = strings.ToLower(conn.GetProp("compression", "ipc_compression")) - if conn.Compression == "" { - conn.Compression = "none" - } - - conn.MaxInflightBatches = cast.ToInt(conn.GetProp("max_inflight_batches")) - if conn.MaxInflightBatches <= 0 { - conn.MaxInflightBatches = 1000 - } - - if conn.URL != "" { - conn.parseURL(conn.URL) - } - - return nil -} - -func (conn *ZerobusConn) parseURL(rawURL string) { - clean := strings.TrimPrefix(rawURL, "zerobus://") - u, err := net.NewURL("http://" + clean) // Use http parser for host/path - if err == nil { - if conn.WorkspaceURL == "" { - conn.WorkspaceURL = u.Hostname() - } - pathTrimmed := strings.Trim(u.Path(), "/") - if pathTrimmed != "" { - parts := strings.Split(pathTrimmed, ".") - if len(parts) == 3 { - conn.Catalog = parts[0] - conn.Schema = parts[1] - conn.Table = parts[2] - } else if len(parts) == 2 { - conn.Schema = parts[0] - conn.Table = parts[1] - } else if len(parts) == 1 { - conn.Table = parts[0] - } - } - - if pass := u.Password(); pass != "" && conn.Token == "" { - conn.Token = pass - } - if user := u.Username(); user != "" && conn.ClientID == "" { - conn.ClientID = user - } - } -} - -// Connect connects / validates Zerobus configuration -func (conn *ZerobusConn) Connect(timeOut ...int) error { - if conn.WorkspaceURL == "" { - return g.Error("workspace_url or host is required for Zerobus connection") - } - if conn.Token == "" && (conn.ClientID == "" || conn.ClientSecret == "") { - return g.Error("either token or (client_id and client_secret) is required for Zerobus connection") - } - return nil -} - -// LoadTemplates loads Databricks SQL templates for Zerobus without needing changes to core dbio_types.go -func (conn *ZerobusConn) LoadTemplates() error { - tmpl, err := dbio.TypeDbDatabricks.Template() - if err != nil { - return err - } - conn.BaseConn.template = tmpl - return nil -} - -// BulkImportStream ingests a datastream into the target table using Arrow batches -func (conn *ZerobusConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error) { - df := iop.MakeDataFlow(ds) - return conn.BulkImportFlow(tableFName, df) -} - -// BulkImportFlow ingests a dataflow into the target table using Arrow batches -func (conn *ZerobusConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error) { - defer df.CleanUp() - - if tableFName == "" { - tableFName = conn.Table - } - if tableFName == "" { - return 0, g.Error("target table must be specified for Zerobus ingestion") - } - - targetTable, err := ParseTableName(tableFName, conn.Type) - if err != nil { - return 0, g.Error(err, "invalid target table name: %s", tableFName) - } - - targetFDQN := targetTable.FDQN() - g.Info("ingesting into Databricks via Zerobus Arrow stream: %s", targetFDQN) - - // Build Arrow Schema conforming to official Databricks Zerobus SDK specification - arrowSchema := ColumnsToZerobusArrowSchema(df.Columns) - mem := memory.NewGoAllocator() - - batchSize := conn.BatchSize - if batchSize <= 0 { - batchSize = 10000 - } - - builders := make([]array.Builder, len(df.Columns)) - createBuilder := func(dtype arrow.DataType) array.Builder { - switch dtype.ID() { - case arrow.BOOL: - return array.NewBooleanBuilder(mem) - case arrow.INT8: - return array.NewInt8Builder(mem) - case arrow.INT16: - return array.NewInt16Builder(mem) - case arrow.INT32: - return array.NewInt32Builder(mem) - case arrow.INT64: - return array.NewInt64Builder(mem) - case arrow.FLOAT32: - return array.NewFloat32Builder(mem) - case arrow.FLOAT64: - return array.NewFloat64Builder(mem) - case arrow.LARGE_STRING: - return array.NewLargeStringBuilder(mem) - case arrow.STRING: - return array.NewStringBuilder(mem) - case arrow.LARGE_BINARY: - return array.NewLargeBinaryBuilder(mem, dtype.(*arrow.LargeBinaryType)) - case arrow.BINARY: - return array.NewBinaryBuilder(mem, dtype.(*arrow.BinaryType)) - case arrow.DECIMAL128: - return array.NewDecimal128Builder(mem, dtype.(*arrow.Decimal128Type)) - case arrow.DATE32: - return array.NewDate32Builder(mem) - case arrow.TIMESTAMP: - return array.NewTimestampBuilder(mem, dtype.(*arrow.TimestampType)) - case arrow.TIME32: - return array.NewTime32Builder(mem, dtype.(*arrow.Time32Type)) - case arrow.TIME64: - return array.NewTime64Builder(mem, dtype.(*arrow.Time64Type)) - case arrow.EXTENSION: - return array.NewBuilder(mem, dtype) - default: - return array.NewLargeStringBuilder(mem) - } - } - - resetBuilders := func() { - for i, field := range arrowSchema.Fields() { - builders[i] = createBuilder(field.Type) - } - } - releaseBuilders := func() { - for _, b := range builders { - if b != nil { - b.Release() - } - } - } - defer releaseBuilders() - - resetBuilders() - rowsInBatch := 0 - - flushBatch := func() error { - if rowsInBatch == 0 { - return nil - } - - // Build arrays - arrays := make([]arrow.Array, len(builders)) - for i, b := range builders { - arrays[i] = b.NewArray() - } - - record := array.NewRecord(arrowSchema, arrays, int64(rowsInBatch)) - - // Send record batch with retries and guarantee Release - err := conn.sendRecordBatchWithRetry(targetFDQN, record) - - // Release record and arrays immediately after send to free memory - record.Release() - for _, arr := range arrays { - arr.Release() - } - - // Reset builders for next batch - releaseBuilders() - resetBuilders() - rowsInBatch = 0 - - return err - } - - // Stream records - for ds := range df.StreamCh { - for row := range ds.Rows { - for colIdx, col := range df.Columns { - var val interface{} - if colIdx < len(row) { - val = row[colIdx] - } - appendToZerobusBuilder(builders[colIdx], &col, val) - } - rowsInBatch++ - count++ - - if rowsInBatch >= batchSize { - if err := flushBatch(); err != nil { - return count, g.Error(err, "failed to stream Arrow RecordBatch to Zerobus") - } - } - } - - if err := ds.Context.Err(); err != nil { - return count, g.Error(err, "error reading source stream") - } - } - - // Flush remaining rows - if err := flushBatch(); err != nil { - return count, g.Error(err, "failed to flush final Arrow RecordBatch to Zerobus") - } - - g.Info("successfully streamed %d rows to Zerobus table %s", count, targetFDQN) - return count, nil -} - -// sendRecordBatchWithRetry streams an Arrow RecordBatch to Zerobus with retries -func (conn *ZerobusConn) sendRecordBatchWithRetry(table string, record arrow.Record) error { - maxRetries := 3 - var lastErr error - - for attempt := 0; attempt <= maxRetries; attempt++ { - if attempt > 0 { - backoff := time.Duration(1< Arrow mappings: -// - STRING -> LargeUtf8 (arrow.BinaryTypes.LargeString) -// - DECIMAL -> LargeUtf8 (text-encoded per Zerobus specification) -// - BINARY -> LargeBinary (arrow.BinaryTypes.LargeBinary) -// - TINYINT -> Int8 -// - SMALLINT -> Int16 -// - INT -> Int32 -// - BIGINT -> Int64 -// - FLOAT -> Float32 -// - DOUBLE -> Float64 -// - BOOLEAN -> Boolean -// - DATE -> Date32 -// - TIMESTAMP -> Timestamp(Microsecond, "UTC") -// - TIMESTAMP_NTZ / DATETIME -> Timestamp(Microsecond, "") -func ColumnsToZerobusArrowSchema(columns iop.Columns) *arrow.Schema { - fields := make([]arrow.Field, len(columns)) - - for i, col := range columns { - var arrowType arrow.DataType - - switch col.Type { - case iop.BoolType: - arrowType = arrow.FixedWidthTypes.Boolean - case iop.SmallIntType: - if strings.EqualFold(col.DbType, "tinyint") || strings.EqualFold(col.DbType, "int8") || strings.EqualFold(col.DbType, "byte") { - arrowType = arrow.PrimitiveTypes.Int8 - } else { - arrowType = arrow.PrimitiveTypes.Int16 - } - case iop.IntegerType: - if strings.EqualFold(col.DbType, "tinyint") || strings.EqualFold(col.DbType, "int8") || strings.EqualFold(col.DbType, "byte") { - arrowType = arrow.PrimitiveTypes.Int8 - } else if strings.EqualFold(col.DbType, "smallint") || strings.EqualFold(col.DbType, "int16") || strings.EqualFold(col.DbType, "short") { - arrowType = arrow.PrimitiveTypes.Int16 - } else { - arrowType = arrow.PrimitiveTypes.Int32 - } - case iop.BigIntType: - arrowType = arrow.PrimitiveTypes.Int64 - case iop.FloatType: - if strings.EqualFold(col.DbType, "float") || strings.EqualFold(col.DbType, "float32") || strings.EqualFold(col.DbType, "real") { - arrowType = arrow.PrimitiveTypes.Float32 - } else { - arrowType = arrow.PrimitiveTypes.Float64 - } - case iop.DecimalType: - // Databricks Zerobus SDK specification: - // Currently, DECIMAL types are encoded as text (LargeUtf8 / string) on both Arrow and Protobuf paths - arrowType = arrow.BinaryTypes.LargeString - case iop.DateType: - arrowType = arrow.FixedWidthTypes.Date32 - case iop.TimestampzType: - arrowType = &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: "UTC"} - case iop.DatetimeType, iop.TimestampType: - arrowType = &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: ""} - case iop.BinaryType: - arrowType = arrow.BinaryTypes.LargeBinary - case iop.StringType, iop.TextType, iop.JsonType: - arrowType = arrow.BinaryTypes.LargeString - default: - arrowType = arrow.BinaryTypes.LargeString - } - - fields[i] = arrow.Field{ - Name: col.Name, - Type: arrowType, - Nullable: true, - } - } - - return arrow.NewSchema(fields, nil) -} - -func appendToZerobusBuilder(builder array.Builder, col *iop.Column, val interface{}) { - if val == nil { - builder.AppendNull() - return - } - switch b := builder.(type) { - case *array.LargeBinaryBuilder: - b.Append([]byte(cast.ToString(val))) - case *array.LargeStringBuilder: - // Text-encoded strings, json, and decimals per Zerobus specification - b.Append(cast.ToString(val)) - default: - iop.AppendToBuilder(builder, col, val) - } -} diff --git a/core/dbio/database/database_zerobus_test.go b/core/dbio/database/database_zerobus_test.go deleted file mode 100644 index 236885b4f..000000000 --- a/core/dbio/database/database_zerobus_test.go +++ /dev/null @@ -1,160 +0,0 @@ -package database - -import ( - "testing" - "time" - - "github.com/slingdata-io/sling-cli/core/dbio/iop" - "github.com/stretchr/testify/assert" -) - -func TestZerobusConn_URLParsing(t *testing.T) { - conn := &ZerobusConn{ - URL: "zerobus://my-user:my-token@adb-123.databricks.com/main.default.customers", - } - err := conn.Init() - assert.NoError(t, err) - - assert.Equal(t, "adb-123.databricks.com", conn.WorkspaceURL) - assert.Equal(t, "main", conn.Catalog) - assert.Equal(t, "default", conn.Schema) - assert.Equal(t, "customers", conn.Table) - assert.Equal(t, "my-token", conn.Token) - assert.Equal(t, "my-user", conn.ClientID) - assert.Equal(t, 10000, conn.BatchSize) - assert.Equal(t, "none", conn.Compression) - assert.Equal(t, 1000, conn.MaxInflightBatches) -} - -func TestZerobusConn_CustomOptions(t *testing.T) { - conn := &ZerobusConn{ - WorkspaceURL: "adb-123.databricks.com", - Token: "my-token", - } - conn.BaseConn.props = map[string]interface{}{ - "batch_size": 5000, - "compression": "zstd", - "max_inflight_batches": 250, - } - err := conn.Init() - assert.NoError(t, err) - - assert.Equal(t, 5000, conn.BatchSize) - assert.Equal(t, "zstd", conn.Compression) - assert.Equal(t, 250, conn.MaxInflightBatches) -} - -func TestZerobusConn_BulkImportFlow_ArrowBatches(t *testing.T) { - conn := &ZerobusConn{ - WorkspaceURL: "adb-test.cloud.databricks.com", - Token: "test-token", - BatchSize: 2, // Small batch size to test multiple batch flushes - } - err := conn.Init() - assert.NoError(t, err) - - // Create test columns - cols := iop.Columns{ - {Name: "id", Type: iop.BigIntType}, - {Name: "name", Type: iop.StringType}, - {Name: "amount", Type: iop.DecimalType, DbPrecision: 18, DbScale: 2}, - {Name: "created_at", Type: iop.DatetimeType}, - } - - // Create test dataset - now := time.Date(2026, 9, 19, 0, 0, 0, 0, time.UTC) - data := iop.Dataset{ - Columns: cols, - Rows: [][]any{ - {int64(1), "Alice", "123.45", now}, - {int64(2), "Bob", "67.89", now.Add(time.Hour)}, - {int64(3), "Charlie", "999.00", now.Add(2 * time.Hour)}, - {int64(4), nil, nil, nil}, // Test null handling - {int64(5), "Eve", "0.01", now.Add(3 * time.Hour)}, - }, - } - - ds := data.Stream() - df := iop.MakeDataFlow(ds) - - count, err := conn.BulkImportFlow("main.default.users", df) - assert.NoError(t, err) - assert.Equal(t, uint64(5), count) -} - -func TestZerobus_IPCSerialization(t *testing.T) { - cols := iop.Columns{ - {Name: "id", Type: iop.BigIntType}, - {Name: "name", Type: iop.StringType}, - } - schema := iop.ColumnsToArrowSchema(cols) - - // Test 1: Serialize schema to IPC (without data batches) - schemaBytes, err := SerializeSchemaToIPC(schema) - assert.NoError(t, err) - assert.NotEmpty(t, schemaBytes) - - // Test 2: Serialize record batch with ZSTD compression - now := time.Now() - data := iop.Dataset{ - Columns: cols, - Rows: [][]any{ - {int64(1), "Alice"}, - {int64(2), "Bob"}, - }, - } - _ = now - df := iop.MakeDataFlow(data.Stream()) - defer df.CleanUp() - - conn := &ZerobusConn{ - WorkspaceURL: "adb-123.databricks.com", - Token: "tok", - Compression: "zstd", - } - err = conn.Init() - assert.NoError(t, err) - - count, err := conn.BulkImportFlow("main.default.test", df) - assert.NoError(t, err) - assert.Equal(t, uint64(2), count) -} - -func TestZerobus_TypeMapping(t *testing.T) { - cols := iop.Columns{ - {Name: "col_bool", Type: iop.BoolType}, - {Name: "col_tiny", Type: iop.SmallIntType, DbType: "tinyint"}, - {Name: "col_short", Type: iop.SmallIntType, DbType: "smallint"}, - {Name: "col_int", Type: iop.IntegerType}, - {Name: "col_bigint", Type: iop.BigIntType}, - {Name: "col_float", Type: iop.FloatType, DbType: "float"}, - {Name: "col_double", Type: iop.FloatType, DbType: "double"}, - {Name: "col_str", Type: iop.StringType}, - {Name: "col_dec", Type: iop.DecimalType, DbPrecision: 18, DbScale: 4}, - {Name: "col_bin", Type: iop.BinaryType}, - {Name: "col_date", Type: iop.DateType}, - {Name: "col_ts", Type: iop.TimestampzType}, - } - - schema := ColumnsToZerobusArrowSchema(cols) - - // Verify Databricks Zerobus Arrow Flight type specification - assert.Equal(t, "bool", schema.Field(0).Type.Name()) - assert.Equal(t, "int8", schema.Field(1).Type.Name()) - assert.Equal(t, "int16", schema.Field(2).Type.Name()) - assert.Equal(t, "int32", schema.Field(3).Type.Name()) - assert.Equal(t, "int64", schema.Field(4).Type.Name()) - assert.Equal(t, "float32", schema.Field(5).Type.Name()) - assert.Equal(t, "float64", schema.Field(6).Type.Name()) - // STRING must be large_utf8 - assert.Equal(t, "large_utf8", schema.Field(7).Type.Name()) - // DECIMAL must be text-encoded as large_utf8 - assert.Equal(t, "large_utf8", schema.Field(8).Type.Name()) - // BINARY must be large_binary - assert.Equal(t, "large_binary", schema.Field(9).Type.Name()) - // DATE must be date32 - assert.Equal(t, "date32", schema.Field(10).Type.Name()) - // TIMESTAMP must be timestamp with UTC - assert.Equal(t, "timestamp[us, tz=UTC]", schema.Field(11).Type.String()) -} - diff --git a/core/dbio/dbio_types.go b/core/dbio/dbio_types.go index 45a0ea415..b10fa2460 100644 --- a/core/dbio/dbio_types.go +++ b/core/dbio/dbio_types.go @@ -46,17 +46,17 @@ const ( TypeApi Type = "api" - TypeFileLocal Type = "file" - TypeFileHDFS Type = "hdfs" - TypeFileS3 Type = "s3" - TypeFileR2 Type = "r2" - TypeFileAzure Type = "azure" - TypeFileAzureABFS Type = "abfs" - TypeFileGoogle Type = "gs" - TypeFileGoogleDrive Type = "gdrive" - TypeFileFtp Type = "ftp" - TypeFileSftp Type = "sftp" - TypeFileHTTP Type = "http" + TypeFileLocal Type = "file" + TypeFileHDFS Type = "hdfs" + TypeFileS3 Type = "s3" + TypeFileR2 Type = "r2" + TypeFileAzure Type = "azure" + TypeFileAzureABFS Type = "abfs" + TypeFileGoogle Type = "gs" + TypeFileGoogleDrive Type = "gdrive" + TypeFileFtp Type = "ftp" + TypeFileSftp Type = "sftp" + TypeFileHTTP Type = "http" TypeFileDatabricksVolume Type = "databricks-volume" TypeDbPostgres Type = "postgres" @@ -91,7 +91,6 @@ const ( TypeDbArrowDBC Type = "adbc" TypeDbODBC Type = "odbc" TypeDbScyllaDB Type = "scylladb" - TypeDbZerobus Type = "zerobus" ) var AllType = []struct { @@ -143,7 +142,6 @@ var AllType = []struct { {TypeDbArrowDBC, "TypeDbArrowDBC"}, {TypeDbODBC, "TypeDbODBC"}, {TypeDbScyllaDB, "TypeDbScyllaDB"}, - {TypeDbZerobus, "TypeDbZerobus"}, } // ValidateType returns true is type is valid @@ -155,7 +153,6 @@ func ValidateType(tStr string) (Type, bool) { "mongodb+srv": TypeDbMongoDB, "file": TypeFileLocal, "abfss": TypeFileAzureABFS, - "volume": TypeFileDatabricksVolume, } if tMatched, ok := tMap[tStr]; ok { @@ -166,7 +163,7 @@ func ValidateType(tStr string) (Type, bool) { case TypeApi, TypeFileLocal, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileDatabricksVolume, - TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbZerobus: + TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: return t, true } @@ -199,7 +196,6 @@ func (t Type) DefPort() int { TypeDbPrometheus: 9090, TypeDbProton: 8463, TypeDbDatabricks: 443, - TypeDbZerobus: 443, TypeDbExasol: 8563, TypeFileFtp: 21, TypeFileSftp: 22, @@ -214,7 +210,7 @@ func (t Type) DefPort() int { // or treat `database` as the schema itself (mysql) or a service (oracle). func (t Type) SupportsThreePartName() bool { return g.In(t, - TypeDbSnowflake, TypeDbDatabricks, TypeDbZerobus, TypeDbTrino, TypeDbBigQuery, + TypeDbSnowflake, TypeDbDatabricks, TypeDbTrino, TypeDbBigQuery, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbFabric, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, ) @@ -233,7 +229,7 @@ func (t Type) DBNameUpperCase() bool { func (t Type) Kind() Kind { switch t { case TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbBigTable, - TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbZerobus: + TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: return KindDatabase case TypeFileLocal, TypeFileHDFS, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileHTTP, TypeFileDatabricksVolume, Type("https"): return KindFile @@ -308,52 +304,51 @@ func (t Type) IsSingleWriterDB() bool { // NameLong return the type long name func (t Type) NameLong() string { mapping := map[Type]string{ - TypeApi: "API - Spec", - TypeFileLocal: "FileSys - Local", - TypeFileHDFS: "FileSys - HDFS", - TypeFileS3: "FileSys - S3", - TypeFileAzure: "FileSys - Azure", - TypeFileAzureABFS: "FileSys - Azure ABFS", - TypeFileGoogle: "FileSys - Google Cloud Storage", - TypeFileGoogleDrive: "FileSys - Google Drive", - TypeFileSftp: "FileSys - Sftp", - TypeFileFtp: "FileSys - Ftp", - TypeFileHTTP: "FileSys - HTTP", - Type("https"): "FileSys - HTTP", + TypeApi: "API - Spec", + TypeFileLocal: "FileSys - Local", + TypeFileHDFS: "FileSys - HDFS", + TypeFileS3: "FileSys - S3", + TypeFileAzure: "FileSys - Azure", + TypeFileAzureABFS: "FileSys - Azure ABFS", + TypeFileGoogle: "FileSys - Google Cloud Storage", + TypeFileGoogleDrive: "FileSys - Google Drive", + TypeFileSftp: "FileSys - Sftp", + TypeFileFtp: "FileSys - Ftp", + TypeFileHTTP: "FileSys - HTTP", + Type("https"): "FileSys - HTTP", TypeFileDatabricksVolume: "FileSys - Databricks Volume", - TypeDbPostgres: "DB - PostgreSQL", - TypeDbRedshift: "DB - Redshift", - TypeDbStarRocks: "DB - StarRocks", - TypeDbMySQL: "DB - MySQL", - TypeDbMariaDB: "DB - MariaDB", - TypeDbOracle: "DB - Oracle", - TypeDbBigQuery: "DB - BigQuery", - TypeDbBigTable: "DB - BigTable", - TypeDbSnowflake: "DB - Snowflake", - TypeDbDatabricks: "DB - Databricks", - TypeDbZerobus: "DB - Zerobus", - TypeDbExasol: "DB - Exasol", - TypeDbD1: "DB - D1", - Type("db2"): "DB - DB2", - TypeDbSQLite: "DB - SQLite", - TypeDbDuckDb: "DB - DuckDB", - TypeDbDuckLake: "DB - DuckLake", - TypeDbMotherDuck: "DB - MotherDuck", - TypeDbSQLServer: "DB - SQLServer", - TypeDbAzure: "DB - Azure", - TypeDbFabric: "DB - Fabric", - TypeDbTrino: "DB - Trino", - TypeDbAthena: "DB - Athena", - TypeDbIceberg: "DB - Iceberg", - TypeDbClickhouse: "DB - Clickhouse", - TypeDbPrometheus: "DB - Prometheus", - TypeDbElasticsearch: "DB - Elasticsearch", - TypeDbMongoDB: "DB - MongoDB", - TypeDbProton: "DB - Proton", - TypeDbAzureTable: "DB - Azure Table", - TypeDbArrowDBC: "DB - Arrow DBC", - TypeDbODBC: "DB - ODBC", - TypeDbScyllaDB: "DB - ScyllaDB", + TypeDbPostgres: "DB - PostgreSQL", + TypeDbRedshift: "DB - Redshift", + TypeDbStarRocks: "DB - StarRocks", + TypeDbMySQL: "DB - MySQL", + TypeDbMariaDB: "DB - MariaDB", + TypeDbOracle: "DB - Oracle", + TypeDbBigQuery: "DB - BigQuery", + TypeDbBigTable: "DB - BigTable", + TypeDbSnowflake: "DB - Snowflake", + TypeDbDatabricks: "DB - Databricks", + TypeDbExasol: "DB - Exasol", + TypeDbD1: "DB - D1", + Type("db2"): "DB - DB2", + TypeDbSQLite: "DB - SQLite", + TypeDbDuckDb: "DB - DuckDB", + TypeDbDuckLake: "DB - DuckLake", + TypeDbMotherDuck: "DB - MotherDuck", + TypeDbSQLServer: "DB - SQLServer", + TypeDbAzure: "DB - Azure", + TypeDbFabric: "DB - Fabric", + TypeDbTrino: "DB - Trino", + TypeDbAthena: "DB - Athena", + TypeDbIceberg: "DB - Iceberg", + TypeDbClickhouse: "DB - Clickhouse", + TypeDbPrometheus: "DB - Prometheus", + TypeDbElasticsearch: "DB - Elasticsearch", + TypeDbMongoDB: "DB - MongoDB", + TypeDbProton: "DB - Proton", + TypeDbAzureTable: "DB - Azure Table", + TypeDbArrowDBC: "DB - Arrow DBC", + TypeDbODBC: "DB - ODBC", + TypeDbScyllaDB: "DB - ScyllaDB", } return mapping[t] @@ -362,51 +357,50 @@ func (t Type) NameLong() string { // Name return the type name func (t Type) Name() string { mapping := map[Type]string{ - TypeApi: "API", - TypeFileLocal: "Local", - TypeFileHDFS: "HDFS", - TypeFileS3: "S3", - TypeFileAzure: "Azure", - TypeFileAzureABFS: "Azure ABFS", - TypeFileGoogle: "Google Cloud Storage", - TypeFileGoogleDrive: "Google Drive", - TypeFileSftp: "Sftp", - TypeFileFtp: "Ftp", - TypeFileHTTP: "HTTP", - Type("https"): "HTTP", + TypeApi: "API", + TypeFileLocal: "Local", + TypeFileHDFS: "HDFS", + TypeFileS3: "S3", + TypeFileAzure: "Azure", + TypeFileAzureABFS: "Azure ABFS", + TypeFileGoogle: "Google Cloud Storage", + TypeFileGoogleDrive: "Google Drive", + TypeFileSftp: "Sftp", + TypeFileFtp: "Ftp", + TypeFileHTTP: "HTTP", + Type("https"): "HTTP", TypeFileDatabricksVolume: "Databricks Volume", - TypeDbPostgres: "PostgreSQL", - TypeDbRedshift: "Redshift", - TypeDbStarRocks: "StarRocks", - TypeDbMySQL: "MySQL", - TypeDbMariaDB: "MariaDB", - TypeDbOracle: "Oracle", - TypeDbBigQuery: "BigQuery", - TypeDbBigTable: "BigTable", - TypeDbSnowflake: "Snowflake", - TypeDbDatabricks: "Databricks", - TypeDbZerobus: "Zerobus", - TypeDbExasol: "Exasol", - TypeDbD1: "D1", - Type("db2"): "DB2", - TypeDbSQLite: "SQLite", - TypeDbDuckDb: "DuckDB", - TypeDbDuckLake: "DuckLake", - TypeDbMotherDuck: "MotherDuck", - TypeDbSQLServer: "SQLServer", - TypeDbTrino: "Trino", - TypeDbAthena: "Athena", - TypeDbIceberg: "Iceberg", - TypeDbClickhouse: "Clickhouse", - TypeDbPrometheus: "Prometheus", - TypeDbElasticsearch: "Elasticsearch", - TypeDbMongoDB: "MongoDB", - TypeDbFabric: "Fabric", - TypeDbAzure: "Azure", - TypeDbProton: "Proton", - TypeDbAzureTable: "Azure Table", - TypeDbArrowDBC: "Arrow DBC", - TypeDbODBC: "ODBC", + TypeDbPostgres: "PostgreSQL", + TypeDbRedshift: "Redshift", + TypeDbStarRocks: "StarRocks", + TypeDbMySQL: "MySQL", + TypeDbMariaDB: "MariaDB", + TypeDbOracle: "Oracle", + TypeDbBigQuery: "BigQuery", + TypeDbBigTable: "BigTable", + TypeDbSnowflake: "Snowflake", + TypeDbDatabricks: "Databricks", + TypeDbExasol: "Exasol", + TypeDbD1: "D1", + Type("db2"): "DB2", + TypeDbSQLite: "SQLite", + TypeDbDuckDb: "DuckDB", + TypeDbDuckLake: "DuckLake", + TypeDbMotherDuck: "MotherDuck", + TypeDbSQLServer: "SQLServer", + TypeDbTrino: "Trino", + TypeDbAthena: "Athena", + TypeDbIceberg: "Iceberg", + TypeDbClickhouse: "Clickhouse", + TypeDbPrometheus: "Prometheus", + TypeDbElasticsearch: "Elasticsearch", + TypeDbMongoDB: "MongoDB", + TypeDbFabric: "Fabric", + TypeDbAzure: "Azure", + TypeDbProton: "Proton", + TypeDbAzureTable: "Azure Table", + TypeDbArrowDBC: "Arrow DBC", + TypeDbODBC: "ODBC", } return mapping[t] diff --git a/core/dbio/filesys/fs.go b/core/dbio/filesys/fs.go index 0bca42166..3cdc5ab41 100755 --- a/core/dbio/filesys/fs.go +++ b/core/dbio/filesys/fs.go @@ -178,7 +178,7 @@ func NewFileSysClientFromURLContext(ctx context.Context, url string, props ...st case strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://"): props = append(props, "URL="+url) return NewFileSysClientContext(ctx, dbio.TypeFileHTTP, props...) - case strings.HasPrefix(url, "databricks-volume://"), strings.HasPrefix(url, "volume://"), strings.HasPrefix(url, "databricks://Volumes/"): + case strings.HasPrefix(url, "databricks-volume://"), strings.HasPrefix(url, "databricks://Volumes/"): props = append(props, "URL="+url) return NewFileSysClientContext(ctx, dbio.TypeFileDatabricksVolume, props...) case strings.HasPrefix(url, "file://"): @@ -282,7 +282,7 @@ func NormalizeURI(fs FileSysClient, uri string) string { } return fs.Prefix("/") + strings.TrimLeft(strings.TrimPrefix(uri, fs.Prefix()), "/") case dbio.TypeFileDatabricksVolume: - for _, p := range []string{"databricks-volume://", "databricks://", "volume://"} { + for _, p := range []string{"databricks-volume://", "databricks://Volumes/"} { if strings.HasPrefix(uri, p) { return uri } @@ -309,7 +309,7 @@ func NormalizeURI(fs FileSysClient, uri string) string { } func makeGlob(uri string) (*glob.Glob, error) { - connType, _, path, err := ParseURLType(uri) + connType, host, path, err := ParseURLType(uri) if err != nil { return nil, err } @@ -320,6 +320,8 @@ func makeGlob(uri string) (*glob.Glob, error) { switch connType { case dbio.TypeFileLocal: path = strings.TrimPrefix(path, "./") + case dbio.TypeFileDatabricksVolume: + path = stripDatabricksVolumePrefix(host, path) case dbio.TypeFileAzure: pathContainer := strings.Split(path, "/")[0] path = strings.TrimPrefix(path, pathContainer+"/") // remove container @@ -1074,7 +1076,7 @@ func (fs *BaseFileSysClient) WriteDataflowReady(df *iop.Dataflow, url string, fi } } - if !singleFile && g.In(fsClient.FsType(), dbio.TypeFileLocal, dbio.TypeFileSftp, dbio.TypeFileFtp) { + if !singleFile && g.In(fsClient.FsType(), dbio.TypeFileLocal, dbio.TypeFileSftp, dbio.TypeFileFtp, dbio.TypeFileDatabricksVolume) { path, err := fsClient.GetPath(url) if err != nil { return 0, g.Error(err, "Error Parsing url: "+url) diff --git a/core/dbio/filesys/fs_databricks_volume.go b/core/dbio/filesys/fs_databricks_volume.go index d016055b2..994bd261a 100644 --- a/core/dbio/filesys/fs_databricks_volume.go +++ b/core/dbio/filesys/fs_databricks_volume.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "os" "strings" "time" @@ -16,14 +17,16 @@ import ( "github.com/spf13/cast" ) +// Files API single PUT is capped at 5 GiB. +// https://docs.databricks.com/api/workspace/files +const databricksVolumeMaxPutBytes int64 = 5 * 1024 * 1024 * 1024 + +const databricksVolumeRetryBuffer = 8 * 1024 * 1024 + // DatabricksVolumeFileSysClient handles Databricks Unity Catalog Volumes via Files REST API // API Reference: https://docs.databricks.com/api/workspace/files -// Endpoints: -// - PUT /api/2.0/fs/files/Volumes/{catalog}/{schema}/{volume}/{path}?overwrite=true -// - GET /api/2.0/fs/files/Volumes/{catalog}/{schema}/{volume}/{path} -// - DELETE /api/2.0/fs/files/Volumes/{catalog}/{schema}/{volume}/{path} -// - GET /api/2.0/fs/directories/Volumes/{catalog}/{schema}/{volume}/{path} -// - PUT /api/2.0/fs/directories/Volumes/{catalog}/{schema}/{volume}/{path} +// Canonical URI: databricks-volume:///// +// Alias: databricks://Volumes//// type DatabricksVolumeFileSysClient struct { BaseFileSysClient client *http.Client @@ -37,8 +40,8 @@ type DatabricksVolumeFileSysClient struct { } type databricksDirListResp struct { - Contents []databricksFileInfo `json:"contents"` - NextPageToken string `json:"next_page_token"` + Contents []databricksFileInfo `json:"contents"` + NextPageToken string `json:"next_page_token"` } type databricksFileInfo struct { @@ -55,15 +58,20 @@ func (fs *DatabricksVolumeFileSysClient) Init(ctx context.Context) (err error) { fs.BaseFileSysClient.context = g.NewContext(ctx) fs.BaseFileSysClient.fsType = dbio.TypeFileDatabricksVolume - // Props can come from host, token, catalog, schema, volume rawHost := fs.GetProp("host") if rawHost == "" { rawHost = fs.GetProp("DATABRICKS_HOST") } + if rawHost == "" { + rawHost = os.Getenv("DATABRICKS_HOST") + } fs.token = fs.GetProp("token") if fs.token == "" { fs.token = fs.GetProp("DATABRICKS_TOKEN") } + if fs.token == "" { + fs.token = os.Getenv("DATABRICKS_TOKEN") + } fs.scheme = "https" if strings.HasPrefix(rawHost, "http://") || fs.GetProp("protocol") == "http" || fs.GetProp("use_ssl") == "false" || fs.GetProp("ssl") == "false" { @@ -85,13 +93,26 @@ func (fs *DatabricksVolumeFileSysClient) Init(ctx context.Context) (err error) { fs.volume = vol } - // Clean host + if rawHost == "" { + rawHost = fs.host + } fs.host = strings.TrimPrefix(rawHost, "https://") fs.host = strings.TrimPrefix(fs.host, "http://") fs.host = strings.TrimRight(fs.host, "/") + if fs.host == "" { + return g.Error("databricks host is required") + } + if fs.token == "" { + return g.Error("databricks token is required") + } + fs.client = &http.Client{ - Timeout: 30 * time.Minute, // Allow long streams for big files + Timeout: 30 * time.Minute, + Transport: &http.Transport{ + ResponseHeaderTimeout: 60 * time.Second, + IdleConnTimeout: 90 * time.Second, + }, } return nil @@ -102,25 +123,33 @@ func (fs *DatabricksVolumeFileSysClient) FsType() dbio.Type { return dbio.TypeFileDatabricksVolume } -func (fs *DatabricksVolumeFileSysClient) parseURL(rawURL string) { - // e.g. databricks-volume://catalog/schema/volume/path - // or databricks-volume://host/Volumes/catalog/schema/volume/path - // or databricks://Volumes/catalog/schema/volume/path +func looksLikeHost(s string) bool { + return strings.Contains(s, ".") +} + +func stripScheme(rawURL string) string { cleaned := rawURL - for _, prefix := range []string{"databricks-volume://", "databricks://", "volume://"} { + for _, prefix := range []string{"databricks-volume://", "databricks://"} { if strings.HasPrefix(cleaned, prefix) { cleaned = strings.TrimPrefix(cleaned, prefix) break } } + return cleaned +} + +func (fs *DatabricksVolumeFileSysClient) parseURL(rawURL string) { + // e.g. databricks-volume://catalog/schema/volume/path + // or databricks-volume://host/Volumes/catalog/schema/volume/path + // or databricks://Volumes/catalog/schema/volume/path + cleaned := stripScheme(rawURL) parts := strings.Split(strings.Trim(cleaned, "/"), "/") if len(parts) > 0 && parts[0] == "Volumes" { parts = parts[1:] } - // Check if the first part looks like a domain (e.g. adb-123.azuredatabricks.net) - if len(parts) > 0 && (strings.Contains(parts[0], ".databricks.com") || strings.Contains(parts[0], ".azuredatabricks.net")) { + if len(parts) > 0 && looksLikeHost(parts[0]) { if fs.host == "" { fs.host = parts[0] } @@ -156,17 +185,38 @@ func (fs *DatabricksVolumeFileSysClient) Prefix(suffix ...string) string { return strings.TrimRight(prefix, "/") + "/" } +// stripDatabricksVolumePrefix returns the path inside the volume (after catalog/schema/volume). +func stripDatabricksVolumePrefix(host, path string) string { + parts := strings.Split(strings.Trim(path, "/"), "/") + var cleaned []string + for _, p := range parts { + if p != "" { + cleaned = append(cleaned, p) + } + } + if strings.EqualFold(host, "Volumes") { + if len(cleaned) >= 3 { + return strings.Join(cleaned[3:], "/") + } + return "" + } + if len(cleaned) >= 2 { + return strings.Join(cleaned[2:], "/") + } + return strings.Join(cleaned, "/") +} + +func volumeURIFromAPIPath(apiPath string) string { + p := strings.TrimPrefix(apiPath, "/") + p = strings.TrimPrefix(p, "Volumes/") + return "databricks-volume://" + p +} + // GetPath converts a uri into the internal volume path starting with /Volumes/... func (fs *DatabricksVolumeFileSysClient) GetPath(uri string) (volumePath string, err error) { uri = NormalizeURI(fs, uri) - clean := uri - for _, prefix := range []string{"databricks-volume://", "databricks://", "volume://"} { - if strings.HasPrefix(clean, prefix) { - clean = strings.TrimPrefix(clean, prefix) - break - } - } + clean := stripScheme(uri) rawParts := strings.Split(strings.Trim(clean, "/"), "/") var parts []string @@ -176,7 +226,7 @@ func (fs *DatabricksVolumeFileSysClient) GetPath(uri string) (volumePath string, } } - if len(parts) > 0 && (strings.Contains(parts[0], ".databricks.com") || strings.Contains(parts[0], ".azuredatabricks.net")) { + if len(parts) > 0 && looksLikeHost(parts[0]) { parts = parts[1:] } if len(parts) > 0 && parts[0] == "Volumes" { @@ -202,18 +252,40 @@ func (fs *DatabricksVolumeFileSysClient) GetPath(uri string) (volumePath string, return "", g.Error("invalid volume URI, catalog/schema/volume must be specified: %s", uri) } - var cleanSubparts []string - for _, sp := range subparts { - if sp != "" { - cleanSubparts = append(cleanSubparts, sp) + p := fmt.Sprintf("/Volumes/%s/%s/%s", catalog, schema, volume) + if len(subparts) > 0 { + p = p + "/" + strings.Join(subparts, "/") + } + return p, nil +} + +type retryableBody struct { + reader io.Reader + retryable bool +} + +func prepareRequestBody(body io.Reader) retryableBody { + if body == nil { + return retryableBody{retryable: true} + } + if cr, ok := body.(*countingReader); ok { + if _, seekable := cr.reader.(io.Seeker); seekable { + return retryableBody{reader: cr, retryable: true} } + return retryableBody{reader: cr, retryable: false} + } + if _, ok := body.(io.Seeker); ok { + return retryableBody{reader: body, retryable: true} } - p := fmt.Sprintf("/Volumes/%s/%s/%s", catalog, schema, volume) - if len(cleanSubparts) > 0 { - p = p + "/" + strings.Join(cleanSubparts, "/") + buf, err := io.ReadAll(io.LimitReader(body, databricksVolumeRetryBuffer+1)) + if err != nil { + return retryableBody{reader: io.MultiReader(bytes.NewReader(buf), body), retryable: false} } - return p, nil + if int64(len(buf)) <= databricksVolumeRetryBuffer { + return retryableBody{reader: bytes.NewReader(buf), retryable: true} + } + return retryableBody{reader: io.MultiReader(bytes.NewReader(buf), body), retryable: false} } // doRequest executes an HTTP request against Databricks Files REST API with retries @@ -230,17 +302,20 @@ func (fs *DatabricksVolumeFileSysClient) doRequest(ctx context.Context, method, urlStr = urlStr + "?" + query.Encode() } + prepared := prepareRequestBody(body) + var lastErr error maxRetries := 3 for attempt := 0; attempt <= maxRetries; attempt++ { if attempt > 0 { - if seeker, ok := body.(io.Seeker); ok { + if !prepared.retryable { + return nil, g.Error(lastErr, "request failed for %s %s", method, urlStr) + } + if seeker, ok := prepared.reader.(io.Seeker); ok { if _, err := seeker.Seek(0, io.SeekStart); err != nil { return nil, g.Error(err, "could not rewind request body") } - } else if body != nil { - return nil, g.Error(lastErr, "request failed (body not seekable) for %s %s", method, urlStr) } sleepDur := time.Duration(1<= 500 { resp.Body.Close() lastErr = fmt.Errorf("server error %s (%d)", resp.Status, resp.StatusCode) + if !prepared.retryable { + return nil, g.Error(lastErr, "request failed for %s %s", method, urlStr) + } continue } @@ -281,11 +362,27 @@ func (fs *DatabricksVolumeFileSysClient) doRequest(ctx context.Context, method, type countingReader struct { reader io.Reader count int64 + limit int64 } func (cr *countingReader) Read(p []byte) (n int, err error) { n, err = cr.reader.Read(p) cr.count += int64(n) + if cr.limit > 0 && cr.count > cr.limit { + return n, g.Error("Databricks Volume PUT exceeds the 5 GiB single-request limit (%d bytes). Split the file (file_max_bytes) or use a smaller payload", cr.count) + } + return n, err +} + +func (cr *countingReader) Seek(offset int64, whence int) (int64, error) { + seeker, ok := cr.reader.(io.Seeker) + if !ok { + return 0, fmt.Errorf("body is not seekable") + } + n, err := seeker.Seek(offset, whence) + if err == nil && whence == io.SeekStart && offset == 0 { + cr.count = 0 + } return n, err } @@ -300,23 +397,42 @@ func (fs *DatabricksVolumeFileSysClient) Write(uri string, reader io.Reader) (bw query := url.Values{} query.Set("overwrite", "true") - // Track bytes written - cr := &countingReader{reader: reader} + cr := &countingReader{reader: reader, limit: databricksVolumeMaxPutBytes} resp, err := fs.doRequest(fs.Context().Ctx, http.MethodPut, apiPath, cr, query) if err != nil { - return cr.count, g.Error(err, "failed to PUT file to Databricks Volume: %s", volumePath) + return 0, g.Error(err, "failed to PUT file to Databricks Volume: %s", volumePath) } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { respBytes, _ := io.ReadAll(resp.Body) - return cr.count, g.Error("error writing to Databricks Volume %s (status %d): %s", volumePath, resp.StatusCode, string(respBytes)) + return 0, g.Error("error writing to Databricks Volume %s (status %d): %s", volumePath, resp.StatusCode, string(respBytes)) } return cr.count, nil } +// GetWriter pipes into Write. +func (fs *DatabricksVolumeFileSysClient) GetWriter(uri string) (writer io.Writer, err error) { + pipeR, pipeW := io.Pipe() + fs.Context().Wg.Write.Add() + go func() { + defer fs.Context().Wg.Write.Done() + _, werr := fs.Write(uri, pipeR) + pipeR.CloseWithError(werr) + }() + return pipeW, nil +} + +// Buckets returns the configured volume as the single "bucket". +func (fs *DatabricksVolumeFileSysClient) Buckets() (paths []string, err error) { + if fs.catalog != "" && fs.schema != "" && fs.volume != "" { + return []string{fmt.Sprintf("%s/%s/%s", fs.catalog, fs.schema, fs.volume)}, nil + } + return +} + // GetReader returns a reader for reading a file from Databricks volume func (fs *DatabricksVolumeFileSysClient) GetReader(uri string) (reader io.Reader, err error) { volumePath, err := fs.GetPath(uri) @@ -342,13 +458,17 @@ func (fs *DatabricksVolumeFileSysClient) GetReader(uri string) (reader io.Reader return resp.Body, nil } -// delete deletes a file or directory in a Databricks volume -func (fs *DatabricksVolumeFileSysClient) delete(uri string) (err error) { - volumePath, err := fs.GetPath(uri) +func (fs *DatabricksVolumeFileSysClient) isDirectory(volumePath string) bool { + apiPath := "/api/2.0/fs/directories" + strings.TrimRight(volumePath, "/") + resp, err := fs.doRequest(fs.Context().Ctx, http.MethodGet, apiPath, nil, nil) if err != nil { - return err + return false } + defer resp.Body.Close() + return resp.StatusCode >= 200 && resp.StatusCode < 300 +} +func (fs *DatabricksVolumeFileSysClient) deleteFile(volumePath string) error { apiPath := "/api/2.0/fs/files" + volumePath resp, err := fs.doRequest(fs.Context().Ctx, http.MethodDelete, apiPath, nil, nil) if err != nil { @@ -362,10 +482,73 @@ func (fs *DatabricksVolumeFileSysClient) delete(uri string) (err error) { respBytes, _ := io.ReadAll(resp.Body) return g.Error("error deleting Databricks Volume file %s (status %d): %s", volumePath, resp.StatusCode, string(respBytes)) } + return nil +} + +func (fs *DatabricksVolumeFileSysClient) deleteEmptyDir(volumePath string) error { + apiPath := "/api/2.0/fs/directories" + strings.TrimRight(volumePath, "/") + resp, err := fs.doRequest(fs.Context().Ctx, http.MethodDelete, apiPath, nil, nil) + if err != nil { + return g.Error(err, "failed to DELETE directory in Databricks Volume: %s", volumePath) + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil + } else if resp.StatusCode < 200 || resp.StatusCode >= 300 { + respBytes, _ := io.ReadAll(resp.Body) + return g.Error("error deleting Databricks Volume directory %s (status %d): %s", volumePath, resp.StatusCode, string(respBytes)) + } return nil } +func (fs *DatabricksVolumeFileSysClient) deleteRecursive(volumePath string) error { + nodes, err := fs.listDirRecursive(volumePath, false) + if err != nil { + return err + } + + // files first, then directories (API requires empty dirs) + for _, n := range nodes { + if n.IsDir { + continue + } + childPath, err := fs.GetPath(n.URI) + if err != nil { + return err + } + if err := fs.deleteFile(childPath); err != nil { + return err + } + } + for _, n := range nodes { + if !n.IsDir { + continue + } + childPath, err := fs.GetPath(n.URI) + if err != nil { + return err + } + if err := fs.deleteRecursive(childPath); err != nil { + return err + } + } + return fs.deleteEmptyDir(volumePath) +} + +// delete deletes a file or directory in a Databricks volume +func (fs *DatabricksVolumeFileSysClient) delete(uri string) (err error) { + volumePath, err := fs.GetPath(uri) + if err != nil { + return err + } + + if strings.HasSuffix(uri, "/") || fs.isDirectory(volumePath) { + return fs.deleteRecursive(volumePath) + } + return fs.deleteFile(volumePath) +} + // MkdirAll creates a directory in a Databricks volume func (fs *DatabricksVolumeFileSysClient) MkdirAll(uri string) (err error) { volumePath, err := fs.GetPath(uri) @@ -409,30 +592,22 @@ func (fs *DatabricksVolumeFileSysClient) doList(uri string, recursive bool) (nod return nil, g.Error(err, "error parsing glob pattern: %s", uri) } - nodes, err = fs.listDirRecursive(volumePath, recursive) + listed, err := fs.listDirRecursive(volumePath, recursive) if err != nil { return nil, err } - if pattern != nil { - filtered := FileNodes{} - for _, n := range nodes { - if (*pattern).Match(n.URI) { - filtered = append(filtered, n) - } - } - return filtered, nil - } - + ts := fs.GetRefTs().Unix() + nodes.AddWhere(pattern, ts, listed...) return nodes, nil } func (fs *DatabricksVolumeFileSysClient) listDirRecursive(volumePath string, recursive bool) (nodes FileNodes, err error) { apiPath := "/api/2.0/fs/directories" + strings.TrimRight(volumePath, "/") - query := url.Values{} pageToken := "" for { + query := url.Values{} if pageToken != "" { query.Set("page_token", pageToken) } @@ -459,12 +634,11 @@ func (fs *DatabricksVolumeFileSysClient) listDirRecursive(volumePath string, rec } for _, item := range dirResp.Contents { - nodeURI := fmt.Sprintf("databricks-volume://%s", strings.TrimPrefix(item.Path, "/")) node := FileNode{ - URI: nodeURI, + URI: volumeURIFromAPIPath(item.Path), IsDir: item.IsDirectory, Size: cast.ToUint64(item.FileSize), - Updated: item.LastModified / 1000, + Updated: item.LastModified / 1000, // Files API last_modified is epoch ms } nodes = append(nodes, node) diff --git a/core/dbio/filesys/fs_databricks_volume_test.go b/core/dbio/filesys/fs_databricks_volume_test.go index c0a133688..7460e9600 100644 --- a/core/dbio/filesys/fs_databricks_volume_test.go +++ b/core/dbio/filesys/fs_databricks_volume_test.go @@ -3,19 +3,41 @@ package filesys import ( "context" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" + "os" "strings" + "sync/atomic" "testing" + "time" + "github.com/slingdata-io/sling-cli/core/dbio" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func newVolumeTestClient(t *testing.T, props map[string]string) *DatabricksVolumeFileSysClient { + t.Helper() + client := &DatabricksVolumeFileSysClient{} + if props != nil { + client.properties = props + } else { + client.properties = map[string]string{ + "host": "adb-test.cloud.databricks.com", + "token": "dapi_test", + } + } + require.NoError(t, client.Init(context.Background())) + return client +} + func TestDatabricksVolume_URLParsing(t *testing.T) { tests := []struct { url string expectedPath string + wantErr bool }{ { url: "databricks-volume://my_cat/my_schema/my_vol/folder/data.parquet", @@ -26,24 +48,78 @@ func TestDatabricksVolume_URLParsing(t *testing.T) { expectedPath: "/Volumes/my_cat/my_schema/my_vol/folder/data.parquet", }, { - url: "volume://my_cat/my_schema/my_vol/data.parquet", - expectedPath: "/Volumes/my_cat/my_schema/my_vol/data.parquet", + url: "databricks-volume://adb-123.azuredatabricks.net/Volumes/my_cat/my_schema/my_vol/test.csv", + expectedPath: "/Volumes/my_cat/my_schema/my_vol/test.csv", }, { - url: "databricks-volume://adb-123.azuredatabricks.net/Volumes/my_cat/my_schema/my_vol/test.csv", + url: "databricks-volume://custom.dns.example/Volumes/my_cat/my_schema/my_vol/test.csv", expectedPath: "/Volumes/my_cat/my_schema/my_vol/test.csv", }, + { + url: "databricks-volume://only_cat", + wantErr: true, + }, } + client := newVolumeTestClient(t, nil) + for _, tt := range tests { - client := &DatabricksVolumeFileSysClient{} - err := client.Init(context.Background()) - assert.NoError(t, err) + t.Run(tt.url, func(t *testing.T) { + path, err := client.GetPath(tt.url) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.expectedPath, path) + }) + } +} - path, err := client.GetPath(tt.url) - assert.NoError(t, err) - assert.Equal(t, tt.expectedPath, path) +func TestDatabricksVolume_ParseURLTypeSQLGuard(t *testing.T) { + uType, _, _, err := ParseURLType("databricks://token:x@host.cloud.databricks.com/sql/1.0/warehouses/abc") + assert.NoError(t, err) + assert.NotEqual(t, dbio.TypeFileDatabricksVolume, uType) + assert.Equal(t, dbio.TypeDbDatabricks, uType) + + uType, host, _, err := ParseURLType("databricks://Volumes/my_cat/my_schema/my_vol/file.parquet") + assert.NoError(t, err) + assert.Equal(t, dbio.TypeFileDatabricksVolume, uType) + assert.Equal(t, "Volumes", host) + + uType, _, _, err = ParseURLType("databricks-volume://my_cat/my_schema/my_vol/file.parquet") + assert.NoError(t, err) + assert.Equal(t, dbio.TypeFileDatabricksVolume, uType) + + _, _, _, err = ParseURLType("volume://my_cat/my_schema/my_vol/file.parquet") + assert.Error(t, err) +} + +func TestDatabricksVolume_InitRequiresHostToken(t *testing.T) { + client := &DatabricksVolumeFileSysClient{} + client.properties = map[string]string{"catalog": "c", "schema": "s", "volume": "v"} + err := client.Init(context.Background()) + assert.Error(t, err) + assert.Contains(t, err.Error(), "host is required") + + client = &DatabricksVolumeFileSysClient{} + client.properties = map[string]string{"host": "adb.example.com"} + err = client.Init(context.Background()) + assert.Error(t, err) + assert.Contains(t, err.Error(), "token is required") +} + +func TestDatabricksVolume_InitHostFromURL(t *testing.T) { + client := &DatabricksVolumeFileSysClient{} + client.properties = map[string]string{ + "token": "tok", + "url": "databricks-volume://adb-123.azuredatabricks.net/Volumes/my_cat/my_schema/my_vol/file.csv", } + require.NoError(t, client.Init(context.Background())) + assert.Equal(t, "adb-123.azuredatabricks.net", client.host) + assert.Equal(t, "my_cat", client.catalog) + assert.Equal(t, "my_schema", client.schema) + assert.Equal(t, "my_vol", client.volume) } func TestDatabricksVolume_RESTOperations(t *testing.T) { @@ -51,11 +127,14 @@ func TestDatabricksVolume_RESTOperations(t *testing.T) { var receivedAuthHeader string var receivedMethod string var receivedURLPath string + var receivedQuery string + var listPages int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { receivedAuthHeader = r.Header.Get("Authorization") receivedMethod = r.Method receivedURLPath = r.URL.Path + receivedQuery = r.URL.RawQuery switch { case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/files/"): @@ -64,19 +143,47 @@ func TestDatabricksVolume_RESTOperations(t *testing.T) { w.WriteHeader(http.StatusOK) case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/files/"): + if strings.Contains(r.URL.Path, "missing") { + w.WriteHeader(http.StatusNotFound) + return + } w.WriteHeader(http.StatusOK) w.Write([]byte("mock volume content")) case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/files/"): + if strings.Contains(r.URL.Path, "gone") { + w.WriteHeader(http.StatusNotFound) + return + } + w.WriteHeader(http.StatusOK) + + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/directories/"): w.WriteHeader(http.StatusOK) case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/directories/"): + listPages++ + if r.URL.Query().Get("page_token") == "" { + resp := databricksDirListResp{ + Contents: []databricksFileInfo{ + { + Path: "/Volumes/my_cat/my_schema/my_vol/file1.parquet", + IsDirectory: false, + FileSize: 1024, + LastModified: 1700000000000, + }, + }, + NextPageToken: "page-2", + } + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(resp) + return + } resp := databricksDirListResp{ Contents: []databricksFileInfo{ { - Path: "/Volumes/my_cat/my_schema/my_vol/file1.parquet", + Path: "/Volumes/my_cat/my_schema/my_vol/file2.parquet", IsDirectory: false, - FileSize: 1024, + FileSize: 2048, }, }, } @@ -91,20 +198,15 @@ func TestDatabricksVolume_RESTOperations(t *testing.T) { host := strings.TrimPrefix(server.URL, "http://") - client := &DatabricksVolumeFileSysClient{} - client.properties = map[string]string{ + client := newVolumeTestClient(t, map[string]string{ "host": host, "protocol": "http", "token": "dapi_test_token_123", "catalog": "my_cat", "schema": "my_schema", "volume": "my_vol", - } - - err := client.Init(context.Background()) - assert.NoError(t, err) + }) - // 1. Test Streaming Write (PUT) testContent := "column1,column2\nval1,val2\n" bw, err := client.Write("databricks-volume://my_cat/my_schema/my_vol/data.csv", strings.NewReader(testContent)) assert.NoError(t, err) @@ -112,9 +214,9 @@ func TestDatabricksVolume_RESTOperations(t *testing.T) { assert.Equal(t, "Bearer dapi_test_token_123", receivedAuthHeader) assert.Equal(t, http.MethodPut, receivedMethod) assert.Equal(t, "/api/2.0/fs/files/Volumes/my_cat/my_schema/my_vol/data.csv", receivedURLPath) + assert.Contains(t, receivedQuery, "overwrite=true") assert.Equal(t, testContent, string(receivedPutBody)) - // 2. Test Read (GET) reader, err := client.GetReader("databricks-volume://my_cat/my_schema/my_vol/data.csv") if assert.NoError(t, err) && reader != nil { readBytes, err := io.ReadAll(reader) @@ -122,15 +224,208 @@ func TestDatabricksVolume_RESTOperations(t *testing.T) { assert.Equal(t, "mock volume content", string(readBytes)) } - // 3. Test List + _, err = client.GetReader("databricks-volume://my_cat/my_schema/my_vol/missing.csv") + assert.Error(t, err) + assert.Contains(t, err.Error(), "file not found") + nodes, err := client.List("databricks-volume://my_cat/my_schema/my_vol") assert.NoError(t, err) - assert.Len(t, nodes, 1) - assert.Equal(t, "databricks-volume://Volumes/my_cat/my_schema/my_vol/file1.parquet", nodes[0].URI) + assert.Len(t, nodes, 2) + assert.Equal(t, "databricks-volume://my_cat/my_schema/my_vol/file1.parquet", nodes[0].URI) assert.Equal(t, uint64(1024), nodes[0].Size) + assert.Equal(t, int64(1700000000), nodes[0].Updated) + assert.Equal(t, "databricks-volume://my_cat/my_schema/my_vol/file2.parquet", nodes[1].URI) + assert.Equal(t, 2, listPages) - // 4. Test Delete err = client.delete("databricks-volume://my_cat/my_schema/my_vol/data.csv") assert.NoError(t, err) assert.Equal(t, http.MethodDelete, receivedMethod) + + err = client.delete("databricks-volume://my_cat/my_schema/my_vol/gone.csv") + assert.NoError(t, err) +} + +func TestDatabricksVolume_DeleteDirectory(t *testing.T) { + var methods []string + var paths []string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + methods = append(methods, r.Method+" "+r.URL.Path) + paths = append(paths, r.URL.Path) + + switch { + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/api/2.0/fs/directories/"): + if strings.HasSuffix(r.URL.Path, "/my_vol/folder") { + resp := databricksDirListResp{ + Contents: []databricksFileInfo{ + {Path: "/Volumes/my_cat/my_schema/my_vol/folder/a.csv", IsDirectory: false, FileSize: 10}, + }, + } + json.NewEncoder(w).Encode(resp) + return + } + json.NewEncoder(w).Encode(databricksDirListResp{}) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusOK) + default: + w.WriteHeader(http.StatusOK) + } + })) + defer server.Close() + + host := strings.TrimPrefix(server.URL, "http://") + client := newVolumeTestClient(t, map[string]string{ + "host": host, + "protocol": "http", + "token": "tok", + "catalog": "my_cat", + "schema": "my_schema", + "volume": "my_vol", + }) + + err := client.delete("databricks-volume://my_cat/my_schema/my_vol/folder/") + assert.NoError(t, err) + + joined := strings.Join(methods, "\n") + assert.Contains(t, joined, "DELETE /api/2.0/fs/directories/Volumes/my_cat/my_schema/my_vol/folder") + assert.Contains(t, joined, "DELETE /api/2.0/fs/files/Volumes/my_cat/my_schema/my_vol/folder/a.csv") +} + +func TestDatabricksVolume_RetrySeekableVsNonSeekable(t *testing.T) { + t.Run("seekable retried on 500", func(t *testing.T) { + var puts int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + w.WriteHeader(http.StatusOK) + return + } + n := atomic.AddInt32(&puts, 1) + io.ReadAll(r.Body) + if n == 1 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + host := strings.TrimPrefix(server.URL, "http://") + client := newVolumeTestClient(t, map[string]string{ + "host": host, + "protocol": "http", + "token": "tok", + "catalog": "my_cat", + "schema": "my_schema", + "volume": "my_vol", + }) + + _, err := client.Write("databricks-volume://my_cat/my_schema/my_vol/ok.csv", strings.NewReader("hello")) + assert.NoError(t, err) + assert.GreaterOrEqual(t, atomic.LoadInt32(&puts), int32(2)) + }) + + t.Run("non-seekable not retried", func(t *testing.T) { + var puts int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + w.WriteHeader(http.StatusOK) + return + } + atomic.AddInt32(&puts, 1) + io.ReadAll(r.Body) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + host := strings.TrimPrefix(server.URL, "http://") + client := newVolumeTestClient(t, map[string]string{ + "host": host, + "protocol": "http", + "token": "tok", + "catalog": "my_cat", + "schema": "my_schema", + "volume": "my_vol", + }) + + pr, pw := io.Pipe() + go func() { + pw.Write([]byte("streamed-bytes")) + pw.Close() + }() + + _, err := client.Write("databricks-volume://my_cat/my_schema/my_vol/pipe.csv", pr) + assert.Error(t, err) + assert.Equal(t, int32(1), atomic.LoadInt32(&puts)) + }) +} + +func TestDatabricksVolume_Prefix(t *testing.T) { + client := newVolumeTestClient(t, map[string]string{ + "host": "adb.example.com", + "token": "tok", + "catalog": "my_cat", + "schema": "my_schema", + "volume": "my_vol", + }) + assert.Equal(t, "databricks-volume://my_cat/my_schema/my_vol/", client.Prefix()) +} + +// Live Files REST check. Not in the default suite — skip unless: +// +// DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_VOLUME (catalog.schema.volume) +func TestDatabricksVolume_Live(t *testing.T) { + host := os.Getenv("DATABRICKS_HOST") + token := os.Getenv("DATABRICKS_TOKEN") + vol := os.Getenv("DATABRICKS_VOLUME") + if host == "" || token == "" || vol == "" { + t.Skip("DATABRICKS_HOST, DATABRICKS_TOKEN, DATABRICKS_VOLUME not set") + } + + parts := strings.Split(vol, ".") + require.Len(t, parts, 3, "DATABRICKS_VOLUME must be catalog.schema.volume") + catalog, schema, volume := parts[0], parts[1], parts[2] + + client := newVolumeTestClient(t, map[string]string{ + "host": host, + "token": token, + "catalog": catalog, + "schema": schema, + "volume": volume, + }) + + ts := time.Now().UTC().Format("20060102T150405Z") + dirURI := fmt.Sprintf("databricks-volume://%s/%s/%s/sling_test/%s", catalog, schema, volume, ts) + fileURI := dirURI + "/hello.txt" + body := "sling volume live test\n" + + bw, err := client.Write(fileURI, strings.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, int64(len(body)), bw) + + t.Cleanup(func() { + _ = client.delete(dirURI + "/") + }) + + reader, err := client.GetReader(fileURI) + require.NoError(t, err) + got, err := io.ReadAll(reader) + if rc, ok := reader.(io.Closer); ok { + rc.Close() + } + require.NoError(t, err) + assert.Equal(t, body, string(got)) + + nodes, err := client.List(dirURI) + require.NoError(t, err) + require.NotEmpty(t, nodes) + assert.Equal(t, fileURI, nodes[0].URI) + + aliasURI := fmt.Sprintf("databricks://Volumes/%s/%s/%s/sling_test/%s/alias.txt", catalog, schema, volume, ts) + _, err = client.Write(aliasURI, strings.NewReader("alias\n")) + require.NoError(t, err) + + err = client.delete(fileURI) + require.NoError(t, err) + _, err = client.GetReader(fileURI) + assert.Error(t, err) } diff --git a/core/dbio/filesys/fs_file_node.go b/core/dbio/filesys/fs_file_node.go index df85c8202..3b8764f62 100644 --- a/core/dbio/filesys/fs_file_node.go +++ b/core/dbio/filesys/fs_file_node.go @@ -76,12 +76,14 @@ func (fn *FileNode) Path() string { return fn.path } - fType, _, path, err := ParseURLType(fn.URI) + fType, host, path, err := ParseURLType(fn.URI) if g.LogError(err) { return "" } switch fType { + case dbio.TypeFileDatabricksVolume: + path = stripDatabricksVolumePrefix(host, path) case dbio.TypeFileAzure: pathContainer := strings.Split(path, "/")[0] @@ -315,10 +317,14 @@ func ParseURLType(uri string) (uType dbio.Type, host string, path string, err er return dbio.TypeFileHTTP, host, path, nil } else if scheme == "gdrive" { return dbio.TypeFileGoogleDrive, host, path, nil - } else if scheme == "databricks-volume" || scheme == "volume" { - return dbio.TypeFileDatabricksVolume, host, path, nil - } else if scheme == "databricks" && strings.HasPrefix(u.U.Path, "/Volumes") { + } else if scheme == "databricks-volume" { return dbio.TypeFileDatabricksVolume, host, path, nil + } else if scheme == "databricks" && (strings.EqualFold(host, "Volumes") || strings.HasPrefix(u.U.Path, "/Volumes")) { + // Alias databricks://Volumes////... — never steal SQL URLs + // like databricks://token:@host/sql/1.0/warehouses/... + if u.Username() != "token" { + return dbio.TypeFileDatabricksVolume, host, path, nil + } } else if g.In(scheme, "http", "https") { return dbio.TypeFileHTTP, host, path, nil } diff --git a/core/dbio/templates/connections.json b/core/dbio/templates/connections.json index 2f58b8f82..3685738f2 100644 --- a/core/dbio/templates/connections.json +++ b/core/dbio/templates/connections.json @@ -1082,6 +1082,76 @@ "description": "Table format to use when creating tables", "enum": ["delta", "iceberg"], "default": "delta" + }, + "copy_method": { + "type": "string", + "description": "Bulk load method. stage uses Unity Catalog volumes + COPY INTO (default). aws uses S3. zerobus streams Arrow RecordBatches into an existing Delta table (Beta).", + "enum": ["stage", "aws", "zerobus"], + "default": "stage" + }, + "zerobus_endpoint": { + "type": "string", + "description": "Zerobus shard URL (not the workspace host), e.g. https://.zerobus..cloud.databricks.com. Required when copy_method is zerobus." + }, + "client_id": { + "type": "string", + "description": "OAuth M2M service principal client ID for Zerobus" + }, + "client_secret": { + "type": "string", + "description": "OAuth M2M service principal client secret for Zerobus" + }, + "batch_size": { + "type": "number", + "description": "Rows per Arrow RecordBatch when copy_method is zerobus", + "default": 10000 + }, + "ipc_compression": { + "type": "string", + "description": "Arrow IPC compression for Zerobus", + "enum": ["none", "lz4", "zstd"], + "default": "none" + }, + "max_inflight_batches": { + "type": "number", + "description": "Max in-flight Zerobus batches pending acknowledgment", + "default": 1000 + } + } + }, + "databricks-volume": { + "name": "Databricks Volume", + "description": "Unity Catalog Volume via the Files REST API (file target, not a table loader)", + "category": "storage", + "properties": { + "host": { + "type": "string", + "description": "Databricks workspace hostname", + "required": true, + "examples": ["adb-123.cloud.databricks.com"] + }, + "token": { + "type": "string", + "description": "Personal access token", + "required": true + }, + "catalog": { + "type": "string", + "description": "Unity Catalog name" + }, + "schema": { + "type": "string", + "description": "Schema name" + }, + "volume": { + "type": "string", + "description": "Volume name" + }, + "protocol": { + "type": "string", + "description": "HTTP protocol. Use http only for tests/mock servers.", + "enum": ["https", "http"], + "default": "https" } } }, diff --git a/core/dbio/templates/databricks.yaml b/core/dbio/templates/databricks.yaml index 6b49b1508..fd87e16c3 100644 --- a/core/dbio/templates/databricks.yaml +++ b/core/dbio/templates/databricks.yaml @@ -267,7 +267,8 @@ metadata: data_type, character_maximum_length as maximum_length, numeric_precision as precision, - numeric_scale as scale + numeric_scale as scale, + CASE WHEN is_nullable = 'YES' THEN 'true' ELSE 'false' END AS is_nullable from information_schema.columns where table_schema = lower('{schema}') and table_name = lower('{table}') diff --git a/core/env/vars.go b/core/env/vars.go index 1e8243c67..4c5d60f74 100644 --- a/core/env/vars.go +++ b/core/env/vars.go @@ -32,6 +32,8 @@ var envVars = []string{ "DIGITALOCEAN_ACCESS_TOKEN", "GITHUB_ACCESS_TOKEN", "SURVEYMONKEY_ACCESS_TOKEN", + "DATABRICKS_HOST", "DATABRICKS_TOKEN", + "SEND_ANON_USAGE", "DBIO_HOME", } diff --git a/go.mod b/go.mod index 6946aaa30..1fc02f6c9 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.0 github.com/clbanning/mxj/v2 v2.7.0 github.com/databricks/databricks-sql-go v1.9.0 + github.com/databricks/zerobus-sdk/go v1.6.0 github.com/denisbrodbeck/machineid v1.0.1 github.com/dustin/go-humanize v1.0.1 github.com/elastic/go-elasticsearch/v8 v8.17.0 @@ -84,6 +85,7 @@ require ( github.com/shirou/gopsutil/v3 v3.24.4 github.com/shopspring/decimal v1.4.0 github.com/sijms/go-ora/v2 v2.8.24 + github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/slingdata-io/godbc v0.0.9 github.com/slingdata-io/golyglot v1.0.20 github.com/slingdata-io/sling v0.0.0-20260715135102-01cd9a07a3c8 @@ -339,7 +341,6 @@ require ( github.com/shirou/gopsutil/v4 v4.25.1 // indirect github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/slingdata-io/pocketbase v0.22.136 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect diff --git a/go.sum b/go.sum index 781d35577..1b55a2b78 100644 --- a/go.sum +++ b/go.sum @@ -313,6 +313,8 @@ github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53E github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= +github.com/databricks/zerobus-sdk/go v1.6.0 h1:TTArmmZMx1tygLQA1fIo70qZl7rEFNaDHsDA/xFBYzA= +github.com/databricks/zerobus-sdk/go v1.6.0/go.mod h1:5wZnWhYl2B5cyT5cc1/eic/j50d1KH2GX+pPdapuOYg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= diff --git a/tests/replications/r.125.databricks_volume.yaml b/tests/replications/r.125.databricks_volume.yaml new file mode 100644 index 000000000..b5f1cc2a3 --- /dev/null +++ b/tests/replications/r.125.databricks_volume.yaml @@ -0,0 +1,52 @@ +# Postgres → Databricks Unity Catalog Volume (parquet). +# Requires a `databricks-volume` connection named DATABRICKS_VOLUME +# (host + token + catalog/schema/volume). Not in the default CLI suite. +source: postgres +target: DATABRICKS_VOLUME + +defaults: + mode: full-refresh + object: 'sling_test/{stream_table}.parquet' + +hooks: + start: + - type: query + connection: '{source.name}' + query: | + DROP TABLE IF EXISTS public.sling_databricks_volume_src; + CREATE TABLE public.sling_databricks_volume_src ( + id bigint, + name varchar(100) + ); + INSERT INTO public.sling_databricks_volume_src VALUES + (1, 'alice'), + (2, 'bob'), + (3, 'carol'); + + end: + - type: check + check: execution.status.error == 0 + on_failure: break + + - type: list + id: listed + location: 'DATABRICKS_VOLUME/sling_test/' + only: files + + - type: check + check: length(state.listed.result) >= 1 + failure_message: "expected at least one parquet file in the volume" + + - type: delete + location: 'DATABRICKS_VOLUME/sling_test/' + recursive: true + + - type: query + connection: '{source.name}' + query: DROP TABLE IF EXISTS public.sling_databricks_volume_src + +streams: + public.sling_databricks_volume_src: + object: 'sling_test/sling_databricks_volume_src.parquet' + target_options: + format: parquet diff --git a/tests/suite.cli.yaml b/tests/suite.cli.yaml index 29c5f8f41..9ecac06c1 100644 --- a/tests/suite.cli.yaml +++ b/tests/suite.cli.yaml @@ -2940,6 +2940,13 @@ - 'STEP 1 OK - snapshot loaded 2 rows' - 'CDC UUID test PASSED' +# Databricks Unity Catalog Volume file target. Needs DATABRICKS_VOLUME conn. +# - id: 602 +# name: 'Postgres to Databricks Volume parquet' +# run: 'sling run -d -r tests/replications/r.125.databricks_volume.yaml' +# output_contains: +# - 'execution succeeded' + # Nested CLI suites. Loader in sling_cli_test.go merges these in. - suite: suite.cli.assist.yaml - suite: suite.cli.build.yaml From 194cc0e5656fc8043ea569d1ffec4f36ed6d739a Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 19 Sep 2026 14:18:36 -0300 Subject: [PATCH 15/56] test: add Zerobus integration tests and validation functions --- .../database_databricks_zerobus_test.go | 348 ------------------ core/dbio/database/database_test.go | 339 +++++++++++++++++ 2 files changed, 339 insertions(+), 348 deletions(-) delete mode 100644 core/dbio/database/database_databricks_zerobus_test.go diff --git a/core/dbio/database/database_databricks_zerobus_test.go b/core/dbio/database/database_databricks_zerobus_test.go deleted file mode 100644 index 8958f1fdb..000000000 --- a/core/dbio/database/database_databricks_zerobus_test.go +++ /dev/null @@ -1,348 +0,0 @@ -package database - -import ( - "bytes" - "context" - "fmt" - "testing" - - "github.com/apache/arrow-go/v18/arrow" - "github.com/apache/arrow-go/v18/arrow/array" - "github.com/apache/arrow-go/v18/arrow/ipc" - "github.com/apache/arrow-go/v18/arrow/memory" - "github.com/databricks/zerobus-sdk/go" - "github.com/slingdata-io/sling-cli/core/dbio/iop" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type fakeZerobusStream struct { - batches [][]byte - ingestErr error - flushErr error - closeErr error - ingestCalled int - flushCalled int - closeCalled int -} - -func (f *fakeZerobusStream) IngestBatch(ipcBytes []byte) (int64, error) { - f.ingestCalled++ - if f.ingestErr != nil { - return -1, f.ingestErr - } - f.batches = append(f.batches, ipcBytes) - return int64(f.ingestCalled), nil -} - -func (f *fakeZerobusStream) Flush() error { - f.flushCalled++ - return f.flushErr -} - -func (f *fakeZerobusStream) Close() error { - f.closeCalled++ - return f.closeErr -} - -func (f *fakeZerobusStream) GetUnackedBatches() ([][]byte, error) { - return f.batches, nil -} - -func newTestDatabricksConn() *DatabricksConn { - conn := &DatabricksConn{} - conn.setContext(context.Background(), 1) - return conn -} - -func TestMapZerobusIPCCompression(t *testing.T) { - none, err := mapZerobusIPCCompression("none") - require.NoError(t, err) - assert.Equal(t, zerobus.IPCCompressionNone, none) - assert.NotEqual(t, zerobus.IPCCompressionDefault, none) - - empty, err := mapZerobusIPCCompression("") - require.NoError(t, err) - assert.Equal(t, zerobus.IPCCompressionNone, empty) - - lz4, err := mapZerobusIPCCompression("lz4") - require.NoError(t, err) - assert.Equal(t, zerobus.IPCCompressionLZ4Frame, lz4) - - zstd, err := mapZerobusIPCCompression("zstd") - require.NoError(t, err) - assert.Equal(t, zerobus.IPCCompressionZstd, zstd) - - _, err = mapZerobusIPCCompression("gzip") - assert.Error(t, err) -} - -func TestIsZerobusSchemaLag(t *testing.T) { - assert.False(t, isZerobusSchemaLag(nil)) - assert.False(t, isZerobusSchemaLag(fmt.Errorf("invalid_client"))) - assert.True(t, isZerobusSchemaLag(fmt.Errorf("Schema comparison failed: Client field 'json_data' does not exist in Delta schema"))) - assert.True(t, isZerobusSchemaLag(fmt.Errorf("SCHEMA_VALIDATION_FAILED"))) - assert.True(t, isZerobusSchemaLag(fmt.Errorf("FIELD_NOT_IN_TABLE"))) -} - -func TestZerobusValidateConfig(t *testing.T) { - conn := newTestDatabricksConn() - conn.CopyMethod = "zerobus" - err := conn.validateZerobusConfig() - assert.Error(t, err) - assert.Contains(t, err.Error(), "zerobus_endpoint") - - conn.ZerobusEndpoint = "https://123.zerobus.us-west-2.cloud.databricks.com" - err = conn.validateZerobusConfig() - assert.Error(t, err) - assert.Contains(t, err.Error(), "client_id") - - conn.ClientID = "id" - conn.ClientSecret = "secret" - assert.NoError(t, conn.validateZerobusConfig()) -} - -func TestColumnsToZerobusArrowSchema(t *testing.T) { - cols := iop.Columns{ - {Name: "col_bool", Type: iop.BoolType}, - {Name: "col_tiny", Type: iop.SmallIntType, DbType: "tinyint"}, - {Name: "col_short", Type: iop.SmallIntType, DbType: "smallint"}, - {Name: "col_int", Type: iop.IntegerType}, - {Name: "col_int_tiny", Type: iop.IntegerType, DbType: "tinyint"}, - {Name: "col_bigint", Type: iop.BigIntType}, - {Name: "col_float", Type: iop.FloatType, DbType: "float"}, - {Name: "col_double", Type: iop.FloatType, DbType: "double"}, - {Name: "col_str", Type: iop.StringType}, - {Name: "col_dec", Type: iop.DecimalType, DbPrecision: 18, DbScale: 4}, - {Name: "col_bin", Type: iop.BinaryType}, - {Name: "col_date", Type: iop.DateType}, - {Name: "col_tsz", Type: iop.TimestampzType}, - {Name: "col_ntz", Type: iop.TimestampType, DbType: "timestamp_ntz"}, - } - - schema, err := ColumnsToZerobusArrowSchema(cols) - require.NoError(t, err) - - assert.Equal(t, "bool", schema.Field(0).Type.Name()) - assert.Equal(t, "int8", schema.Field(1).Type.Name()) - assert.Equal(t, "int16", schema.Field(2).Type.Name()) - assert.Equal(t, "int32", schema.Field(3).Type.Name()) - assert.Equal(t, "int8", schema.Field(4).Type.Name()) - assert.Equal(t, "int64", schema.Field(5).Type.Name()) - assert.Equal(t, "float32", schema.Field(6).Type.Name()) - assert.Equal(t, "float64", schema.Field(7).Type.Name()) - assert.Equal(t, "large_utf8", schema.Field(8).Type.Name()) - assert.Equal(t, "decimal(18, 4)", schema.Field(9).Type.String()) - assert.Equal(t, "large_binary", schema.Field(10).Type.Name()) - assert.Equal(t, "date32", schema.Field(11).Type.Name()) - assert.Equal(t, "timestamp[us, tz=UTC]", schema.Field(12).Type.String()) - assert.Equal(t, "timestamp[us]", schema.Field(13).Type.String()) -} - -func TestColumnsToZerobusArrowSchema_Unsupported(t *testing.T) { - _, err := ColumnsToZerobusArrowSchema(iop.Columns{ - {Name: "arr", Type: iop.JsonType, DbType: "array"}, - }) - assert.Error(t, err) - assert.Contains(t, err.Error(), "unsupported Zerobus type") - - _, err = ColumnsToZerobusArrowSchema(iop.Columns{ - {Name: "m", Type: iop.JsonType, DbType: "map"}, - }) - assert.Error(t, err) - - _, err = ColumnsToZerobusArrowSchema(iop.Columns{ - {Name: "s", Type: iop.JsonType, DbType: "struct"}, - }) - assert.Error(t, err) - - _, err = ColumnsToZerobusArrowSchema(iop.Columns{ - {Name: "v", Type: iop.JsonType, DbType: "variant"}, - }) - assert.Error(t, err) -} - -func TestColumnsToZerobusArrowSchema_Nullability(t *testing.T) { - cols := iop.Columns{ - {Name: "id", Type: iop.BigIntType, Metadata: map[string]string{"is_nullable": "false"}}, - {Name: "name", Type: iop.StringType, Metadata: map[string]string{"is_nullable": "true"}}, - } - schema, err := ColumnsToZerobusArrowSchema(cols) - require.NoError(t, err) - assert.False(t, schema.Field(0).Nullable) - assert.True(t, schema.Field(1).Nullable) -} - -func TestSerializeRecordToIPC_RoundTrip(t *testing.T) { - cols := iop.Columns{ - {Name: "id", Type: iop.BigIntType}, - {Name: "name", Type: iop.StringType}, - } - schema, err := ColumnsToZerobusArrowSchema(cols) - require.NoError(t, err) - - schemaBytes, err := SerializeSchemaToIPC(schema) - require.NoError(t, err) - assert.NotEmpty(t, schemaBytes) - - mem := memory.NewGoAllocator() - idB := array.NewInt64Builder(mem) - nameB := array.NewLargeStringBuilder(mem) - idB.AppendValues([]int64{1, 2}, nil) - nameB.AppendValues([]string{"Alice", "Bob"}, nil) - idA := idB.NewArray() - nameA := nameB.NewArray() - defer idA.Release() - defer nameA.Release() - rec := array.NewRecord(schema, []arrow.Array{idA, nameA}, 2) - defer rec.Release() - - for _, compression := range []string{"none", "lz4", "zstd"} { - t.Run(compression, func(t *testing.T) { - b, err := SerializeRecordToIPC(schema, rec, compression) - require.NoError(t, err) - assert.NotEmpty(t, b) - - r, err := ipc.NewReader(bytes.NewReader(b)) - require.NoError(t, err) - defer r.Release() - assert.True(t, r.Next()) - got := r.Record() - assert.Equal(t, int64(2), got.NumRows()) - assert.False(t, r.Next()) - assert.NoError(t, r.Err()) - }) - } -} - -func TestAlignZerobusSource(t *testing.T) { - src := iop.Columns{ - {Name: "id", Type: iop.BigIntType}, - {Name: "name", Type: iop.StringType}, - } - tgt := iop.Columns{ - {Name: "id", Type: iop.BigIntType, Metadata: map[string]string{"nullable": "false"}}, - {Name: "name", Type: iop.StringType}, - {Name: "note", Type: iop.StringType, Metadata: map[string]string{"is_nullable": "true"}}, - } - idx, err := alignZerobusSource(src, tgt) - require.NoError(t, err) - assert.Equal(t, []int{0, 1, -1}, idx) - - _, err = alignZerobusSource(src, iop.Columns{ - {Name: "id", Type: iop.BigIntType}, - {Name: "missing", Type: iop.StringType, Metadata: map[string]string{"is_nullable": "false"}}, - }) - assert.Error(t, err) - assert.Contains(t, err.Error(), "missing non-null") - - _, err = alignZerobusSource(iop.Columns{ - {Name: "id", Type: iop.BigIntType}, - {Name: "extra", Type: iop.StringType}, - }, iop.Columns{{Name: "id", Type: iop.BigIntType}}) - assert.Error(t, err) - assert.Contains(t, err.Error(), "extra column") -} - -func TestCopyViaZerobus_FakeStream(t *testing.T) { - origOpen := openZerobusStream - origDescribe := zerobusDescribe - t.Cleanup(func() { - openZerobusStream = origOpen - zerobusDescribe = origDescribe - }) - - fake := &fakeZerobusStream{} - openZerobusStream = func(endpoint, workspaceURL, tableName string, schemaIPC []byte, clientID, clientSecret string, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) { - assert.Equal(t, "https://123.zerobus.us-west-2.cloud.databricks.com", endpoint) - assert.Equal(t, "https://dbc.cloud.databricks.com", workspaceURL) - assert.Equal(t, "main.default.users", tableName) - assert.NotEmpty(t, schemaIPC) - assert.Equal(t, zerobus.IPCCompressionNone, opts.IPCCompression) - return fake, func() {}, nil - } - zerobusDescribe = func(conn *DatabricksConn, tableFName string) (iop.Columns, error) { - return iop.Columns{ - {Name: "id", Type: iop.BigIntType}, - {Name: "name", Type: iop.StringType}, - }, nil - } - - conn := newTestDatabricksConn() - conn.CopyMethod = "zerobus" - conn.ZerobusEndpoint = "123.zerobus.us-west-2.cloud.databricks.com" - conn.ClientID = "id" - conn.ClientSecret = "secret" - conn.BatchSize = 2 - conn.IPCCompression = "none" - conn.MaxInflightBatches = 1000 - conn.Catalog = "main" - conn.Schema = "default" - conn.SetProp("host", "dbc.cloud.databricks.com") - - cols := iop.Columns{ - {Name: "id", Type: iop.BigIntType}, - {Name: "name", Type: iop.StringType}, - } - data := iop.NewDataset(cols) - data.Rows = [][]any{ - {int64(1), "Alice"}, - {int64(2), "Bob"}, - {int64(3), "Charlie"}, - {int64(4), nil}, - {int64(5), "Eve"}, - } - df, err := iop.MakeDataFlow(data.Stream()) - require.NoError(t, err) - - table := Table{Database: "main", Schema: "default", Name: "users"} - count, err := conn.CopyViaZerobus(table, df) - require.NoError(t, err) - assert.Equal(t, uint64(5), count) - assert.Equal(t, 3, fake.ingestCalled) // batch_size 2 → 2+2+1 - assert.Equal(t, 1, fake.flushCalled) - assert.Equal(t, 1, fake.closeCalled) -} - -func TestCopyViaZerobus_FlushError(t *testing.T) { - origOpen := openZerobusStream - origDescribe := zerobusDescribe - t.Cleanup(func() { - openZerobusStream = origOpen - zerobusDescribe = origDescribe - }) - - fake := &fakeZerobusStream{flushErr: assert.AnError} - openZerobusStream = func(endpoint, workspaceURL, tableName string, schemaIPC []byte, clientID, clientSecret string, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) { - return fake, func() {}, nil - } - zerobusDescribe = func(conn *DatabricksConn, tableFName string) (iop.Columns, error) { - return iop.Columns{ - {Name: "id", Type: iop.BigIntType}, - }, nil - } - - conn := newTestDatabricksConn() - conn.ZerobusEndpoint = "https://z.example" - conn.ClientID = "id" - conn.ClientSecret = "secret" - conn.BatchSize = 10 - conn.IPCCompression = "none" - conn.SetProp("host", "dbc.cloud.databricks.com") - - data := iop.NewDataset(iop.Columns{{Name: "id", Type: iop.BigIntType}}) - data.Rows = [][]any{{int64(1)}} - df, err := iop.MakeDataFlow(data.Stream()) - require.NoError(t, err) - _, err = conn.CopyViaZerobus(Table{Name: "t", Schema: "s", Database: "c"}, df) - assert.Error(t, err) - assert.Contains(t, err.Error(), "zerobus flush failed") - assert.Equal(t, 1, fake.closeCalled) -} - -func TestCopyViaZerobus_MissingEndpoint(t *testing.T) { - conn := newTestDatabricksConn() - _, err := conn.CopyViaZerobus(Table{Name: "t"}, nil) - assert.Error(t, err) - assert.Contains(t, err.Error(), "zerobus_endpoint") -} diff --git a/core/dbio/database/database_test.go b/core/dbio/database/database_test.go index e9cbe1ff2..23f6f32b2 100755 --- a/core/dbio/database/database_test.go +++ b/core/dbio/database/database_test.go @@ -1,7 +1,9 @@ package database import ( + "bytes" "context" + "fmt" "io" "log" "math" @@ -14,6 +16,11 @@ import ( "testing" "time" + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + zerobus "github.com/databricks/zerobus-sdk/go" "github.com/dustin/go-humanize" "github.com/flarco/g" "github.com/slingdata-io/sling-cli/core/dbio" @@ -21,6 +28,7 @@ import ( "github.com/slingdata-io/sling-cli/core/env" "github.com/spf13/cast" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/xo/dburl" "syreclabs.com/go/faker" ) @@ -1614,3 +1622,334 @@ func TestSoftMergeGuardIsNullSafe(t *testing.T) { } } } + +type fakeZerobusStream struct { + batches [][]byte + ingestErr error + flushErr error + closeErr error + ingestCalled int + flushCalled int + closeCalled int +} + +func (f *fakeZerobusStream) IngestBatch(ipcBytes []byte) (int64, error) { + f.ingestCalled++ + if f.ingestErr != nil { + return -1, f.ingestErr + } + f.batches = append(f.batches, ipcBytes) + return int64(f.ingestCalled), nil +} + +func (f *fakeZerobusStream) Flush() error { + f.flushCalled++ + return f.flushErr +} + +func (f *fakeZerobusStream) Close() error { + f.closeCalled++ + return f.closeErr +} + +func (f *fakeZerobusStream) GetUnackedBatches() ([][]byte, error) { + return f.batches, nil +} + +func newTestDatabricksConn() *DatabricksConn { + conn := &DatabricksConn{} + conn.setContext(context.Background(), 1) + return conn +} + +func TestMapZerobusIPCCompression(t *testing.T) { + none, err := mapZerobusIPCCompression("none") + require.NoError(t, err) + assert.Equal(t, zerobus.IPCCompressionNone, none) + assert.NotEqual(t, zerobus.IPCCompressionDefault, none) + + empty, err := mapZerobusIPCCompression("") + require.NoError(t, err) + assert.Equal(t, zerobus.IPCCompressionNone, empty) + + lz4, err := mapZerobusIPCCompression("lz4") + require.NoError(t, err) + assert.Equal(t, zerobus.IPCCompressionLZ4Frame, lz4) + + zstd, err := mapZerobusIPCCompression("zstd") + require.NoError(t, err) + assert.Equal(t, zerobus.IPCCompressionZstd, zstd) + + _, err = mapZerobusIPCCompression("gzip") + assert.Error(t, err) +} + +func TestIsZerobusSchemaLag(t *testing.T) { + assert.False(t, isZerobusSchemaLag(nil)) + assert.False(t, isZerobusSchemaLag(fmt.Errorf("invalid_client"))) + assert.True(t, isZerobusSchemaLag(fmt.Errorf("Schema comparison failed: Client field 'json_data' does not exist in Delta schema"))) + assert.True(t, isZerobusSchemaLag(fmt.Errorf("SCHEMA_VALIDATION_FAILED"))) + assert.True(t, isZerobusSchemaLag(fmt.Errorf("FIELD_NOT_IN_TABLE"))) +} + +func TestZerobusValidateConfig(t *testing.T) { + conn := newTestDatabricksConn() + conn.CopyMethod = "zerobus" + err := conn.validateZerobusConfig() + assert.Error(t, err) + assert.Contains(t, err.Error(), "zerobus_endpoint") + + conn.ZerobusEndpoint = "https://123.zerobus.us-west-2.cloud.databricks.com" + err = conn.validateZerobusConfig() + assert.Error(t, err) + assert.Contains(t, err.Error(), "client_id") + + conn.ClientID = "id" + conn.ClientSecret = "secret" + assert.NoError(t, conn.validateZerobusConfig()) +} + +func TestColumnsToZerobusArrowSchema(t *testing.T) { + cols := iop.Columns{ + {Name: "col_bool", Type: iop.BoolType}, + {Name: "col_tiny", Type: iop.SmallIntType, DbType: "tinyint"}, + {Name: "col_short", Type: iop.SmallIntType, DbType: "smallint"}, + {Name: "col_int", Type: iop.IntegerType}, + {Name: "col_int_tiny", Type: iop.IntegerType, DbType: "tinyint"}, + {Name: "col_bigint", Type: iop.BigIntType}, + {Name: "col_float", Type: iop.FloatType, DbType: "float"}, + {Name: "col_double", Type: iop.FloatType, DbType: "double"}, + {Name: "col_str", Type: iop.StringType}, + {Name: "col_dec", Type: iop.DecimalType, DbPrecision: 18, DbScale: 4}, + {Name: "col_bin", Type: iop.BinaryType}, + {Name: "col_date", Type: iop.DateType}, + {Name: "col_tsz", Type: iop.TimestampzType}, + {Name: "col_ntz", Type: iop.TimestampType, DbType: "timestamp_ntz"}, + } + + schema, err := ColumnsToZerobusArrowSchema(cols) + require.NoError(t, err) + + assert.Equal(t, "bool", schema.Field(0).Type.Name()) + assert.Equal(t, "int8", schema.Field(1).Type.Name()) + assert.Equal(t, "int16", schema.Field(2).Type.Name()) + assert.Equal(t, "int32", schema.Field(3).Type.Name()) + assert.Equal(t, "int8", schema.Field(4).Type.Name()) + assert.Equal(t, "int64", schema.Field(5).Type.Name()) + assert.Equal(t, "float32", schema.Field(6).Type.Name()) + assert.Equal(t, "float64", schema.Field(7).Type.Name()) + assert.Equal(t, "large_utf8", schema.Field(8).Type.Name()) + assert.Equal(t, "decimal(18, 4)", schema.Field(9).Type.String()) + assert.Equal(t, "large_binary", schema.Field(10).Type.Name()) + assert.Equal(t, "date32", schema.Field(11).Type.Name()) + assert.Equal(t, "timestamp[us, tz=UTC]", schema.Field(12).Type.String()) + assert.Equal(t, "timestamp[us]", schema.Field(13).Type.String()) +} + +func TestColumnsToZerobusArrowSchema_Unsupported(t *testing.T) { + _, err := ColumnsToZerobusArrowSchema(iop.Columns{ + {Name: "arr", Type: iop.JsonType, DbType: "array"}, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unsupported Zerobus type") + + _, err = ColumnsToZerobusArrowSchema(iop.Columns{ + {Name: "m", Type: iop.JsonType, DbType: "map"}, + }) + assert.Error(t, err) + + _, err = ColumnsToZerobusArrowSchema(iop.Columns{ + {Name: "s", Type: iop.JsonType, DbType: "struct"}, + }) + assert.Error(t, err) + + _, err = ColumnsToZerobusArrowSchema(iop.Columns{ + {Name: "v", Type: iop.JsonType, DbType: "variant"}, + }) + assert.Error(t, err) +} + +func TestColumnsToZerobusArrowSchema_Nullability(t *testing.T) { + cols := iop.Columns{ + {Name: "id", Type: iop.BigIntType, Metadata: map[string]string{"is_nullable": "false"}}, + {Name: "name", Type: iop.StringType, Metadata: map[string]string{"is_nullable": "true"}}, + } + schema, err := ColumnsToZerobusArrowSchema(cols) + require.NoError(t, err) + assert.False(t, schema.Field(0).Nullable) + assert.True(t, schema.Field(1).Nullable) +} + +func TestSerializeRecordToIPC_RoundTrip(t *testing.T) { + cols := iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "name", Type: iop.StringType}, + } + schema, err := ColumnsToZerobusArrowSchema(cols) + require.NoError(t, err) + + schemaBytes, err := SerializeSchemaToIPC(schema) + require.NoError(t, err) + assert.NotEmpty(t, schemaBytes) + + mem := memory.NewGoAllocator() + idB := array.NewInt64Builder(mem) + nameB := array.NewLargeStringBuilder(mem) + idB.AppendValues([]int64{1, 2}, nil) + nameB.AppendValues([]string{"Alice", "Bob"}, nil) + idA := idB.NewArray() + nameA := nameB.NewArray() + defer idA.Release() + defer nameA.Release() + rec := array.NewRecord(schema, []arrow.Array{idA, nameA}, 2) + defer rec.Release() + + for _, compression := range []string{"none", "lz4", "zstd"} { + t.Run(compression, func(t *testing.T) { + b, err := SerializeRecordToIPC(schema, rec, compression) + require.NoError(t, err) + assert.NotEmpty(t, b) + + r, err := ipc.NewReader(bytes.NewReader(b)) + require.NoError(t, err) + defer r.Release() + assert.True(t, r.Next()) + got := r.Record() + assert.Equal(t, int64(2), got.NumRows()) + assert.False(t, r.Next()) + assert.NoError(t, r.Err()) + }) + } +} + +func TestAlignZerobusSource(t *testing.T) { + src := iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "name", Type: iop.StringType}, + } + tgt := iop.Columns{ + {Name: "id", Type: iop.BigIntType, Metadata: map[string]string{"nullable": "false"}}, + {Name: "name", Type: iop.StringType}, + {Name: "note", Type: iop.StringType, Metadata: map[string]string{"is_nullable": "true"}}, + } + idx, err := alignZerobusSource(src, tgt) + require.NoError(t, err) + assert.Equal(t, []int{0, 1, -1}, idx) + + _, err = alignZerobusSource(src, iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "missing", Type: iop.StringType, Metadata: map[string]string{"is_nullable": "false"}}, + }) + assert.Error(t, err) + assert.Contains(t, err.Error(), "missing non-null") + + _, err = alignZerobusSource(iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "extra", Type: iop.StringType}, + }, iop.Columns{{Name: "id", Type: iop.BigIntType}}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "extra column") +} + +func TestCopyViaZerobus_FakeStream(t *testing.T) { + origOpen := openZerobusStream + origDescribe := zerobusDescribe + t.Cleanup(func() { + openZerobusStream = origOpen + zerobusDescribe = origDescribe + }) + + fake := &fakeZerobusStream{} + openZerobusStream = func(endpoint, workspaceURL, tableName string, schemaIPC []byte, clientID, clientSecret string, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) { + assert.Equal(t, "https://123.zerobus.us-west-2.cloud.databricks.com", endpoint) + assert.Equal(t, "https://dbc.cloud.databricks.com", workspaceURL) + assert.Equal(t, "main.default.users", tableName) + assert.NotEmpty(t, schemaIPC) + assert.Equal(t, zerobus.IPCCompressionNone, opts.IPCCompression) + return fake, func() {}, nil + } + zerobusDescribe = func(conn *DatabricksConn, tableFName string) (iop.Columns, error) { + return iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "name", Type: iop.StringType}, + }, nil + } + + conn := newTestDatabricksConn() + conn.CopyMethod = "zerobus" + conn.ZerobusEndpoint = "123.zerobus.us-west-2.cloud.databricks.com" + conn.ClientID = "id" + conn.ClientSecret = "secret" + conn.BatchSize = 2 + conn.IPCCompression = "none" + conn.MaxInflightBatches = 1000 + conn.Catalog = "main" + conn.Schema = "default" + conn.SetProp("host", "dbc.cloud.databricks.com") + + cols := iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + {Name: "name", Type: iop.StringType}, + } + data := iop.NewDataset(cols) + data.Rows = [][]any{ + {int64(1), "Alice"}, + {int64(2), "Bob"}, + {int64(3), "Charlie"}, + {int64(4), nil}, + {int64(5), "Eve"}, + } + df, err := iop.MakeDataFlow(data.Stream()) + require.NoError(t, err) + + table := Table{Database: "main", Schema: "default", Name: "users"} + count, err := conn.CopyViaZerobus(table, df) + require.NoError(t, err) + assert.Equal(t, uint64(5), count) + assert.Equal(t, 3, fake.ingestCalled) // batch_size 2 → 2+2+1 + assert.Equal(t, 1, fake.flushCalled) + assert.Equal(t, 1, fake.closeCalled) +} + +func TestCopyViaZerobus_FlushError(t *testing.T) { + origOpen := openZerobusStream + origDescribe := zerobusDescribe + t.Cleanup(func() { + openZerobusStream = origOpen + zerobusDescribe = origDescribe + }) + + fake := &fakeZerobusStream{flushErr: assert.AnError} + openZerobusStream = func(endpoint, workspaceURL, tableName string, schemaIPC []byte, clientID, clientSecret string, opts *zerobus.ArrowStreamConfigurationOptions) (zerobusStream, func(), error) { + return fake, func() {}, nil + } + zerobusDescribe = func(conn *DatabricksConn, tableFName string) (iop.Columns, error) { + return iop.Columns{ + {Name: "id", Type: iop.BigIntType}, + }, nil + } + + conn := newTestDatabricksConn() + conn.ZerobusEndpoint = "https://z.example" + conn.ClientID = "id" + conn.ClientSecret = "secret" + conn.BatchSize = 10 + conn.IPCCompression = "none" + conn.SetProp("host", "dbc.cloud.databricks.com") + + data := iop.NewDataset(iop.Columns{{Name: "id", Type: iop.BigIntType}}) + data.Rows = [][]any{{int64(1)}} + df, err := iop.MakeDataFlow(data.Stream()) + require.NoError(t, err) + _, err = conn.CopyViaZerobus(Table{Name: "t", Schema: "s", Database: "c"}, df) + assert.Error(t, err) + assert.Contains(t, err.Error(), "zerobus flush failed") + assert.Equal(t, 1, fake.closeCalled) +} + +func TestCopyViaZerobus_MissingEndpoint(t *testing.T) { + conn := newTestDatabricksConn() + _, err := conn.CopyViaZerobus(Table{Name: "t"}, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "zerobus_endpoint") +} From 958f17dcfdabe25b97377fa973c22742f9e00bc2 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 19 Sep 2026 14:24:59 -0300 Subject: [PATCH 16/56] refactor: update zerobus import alias and increase sleep duration in CopyViaZerobus function --- core/dbio/database/database_databricks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/dbio/database/database_databricks.go b/core/dbio/database/database_databricks.go index 8b92a3f14..11d734651 100644 --- a/core/dbio/database/database_databricks.go +++ b/core/dbio/database/database_databricks.go @@ -22,7 +22,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/databricks/databricks-sql-go/driverctx" dbsqllog "github.com/databricks/databricks-sql-go/logger" - "github.com/databricks/zerobus-sdk/go" + zerobus "github.com/databricks/zerobus-sdk/go" "github.com/dustin/go-humanize" "github.com/flarco/g" "github.com/flarco/g/net" @@ -1540,7 +1540,7 @@ func (conn *DatabricksConn) CopyViaZerobus(table Table, df *iop.Dataflow) (count } return count, g.Error("zerobus SQL warehouse count is %d after streaming %d rows into %s", visible, count, table.FullName()) } - time.Sleep(500 * time.Millisecond) + time.Sleep(4 * time.Second) } } From 3f5f6623defc23eac6072a8dfa5ad52ad1382de9 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 19 Sep 2026 15:32:39 -0300 Subject: [PATCH 17/56] feat: implement volume file deletion with retry logic and add corresponding tests --- core/dbio/database/database_databricks.go | 167 ++++++++++++++++++++-- core/dbio/database/database_test.go | 92 ++++++++++++ 2 files changed, 246 insertions(+), 13 deletions(-) diff --git a/core/dbio/database/database_databricks.go b/core/dbio/database/database_databricks.go index 11d734651..2f7882de2 100644 --- a/core/dbio/database/database_databricks.go +++ b/core/dbio/database/database_databricks.go @@ -7,9 +7,11 @@ import ( "encoding/json" "fmt" "io" + "math/rand/v2" "net/http" "os" "path" + "strconv" "strings" "time" @@ -873,29 +875,169 @@ func (conn *DatabricksConn) VolumeList(volumePath string) (data iop.Dataset, err return data, nil } +var ( + volumeFilesMaxRetries = 5 + volumeFilesRetryBase = 500 * time.Millisecond + volumeFilesMaxWait = 8 * time.Second +) + +func (conn *DatabricksConn) volumeFilesAPIURL(volumePath string) string { + host := conn.GetProp("host") + scheme := "https" + switch { + case strings.HasPrefix(host, "http://"): + scheme = "http" + host = strings.TrimPrefix(host, "http://") + case strings.HasPrefix(host, "https://"): + host = strings.TrimPrefix(host, "https://") + case conn.GetProp("protocol") == "http", conn.GetProp("use_ssl") == "false", conn.GetProp("ssl") == "false": + scheme = "http" + } + host = strings.TrimRight(host, "/") + if !strings.HasPrefix(volumePath, "/") { + volumePath = "/" + volumePath + } + return fmt.Sprintf("%s://%s/api/2.0/fs/files%s", scheme, host, volumePath) +} + +func volumeFilesRetryWait(attempt int, retryAfter time.Duration) time.Duration { + if retryAfter > 0 { + if retryAfter > 30*time.Second { + return 30 * time.Second + } + return retryAfter + } + if attempt < 1 { + attempt = 1 + } + if attempt > 5 { + attempt = 5 + } + base := volumeFilesRetryBase * time.Duration(1< volumeFilesMaxWait { + base = volumeFilesMaxWait + } + jitterMax := int64(base / 2) + if jitterMax < 1 { + jitterMax = 1 + } + return base/2 + time.Duration(rand.Int64N(jitterMax)) +} + +func parseRetryAfter(h http.Header) time.Duration { + v := strings.TrimSpace(h.Get("Retry-After")) + if v == "" { + return 0 + } + secs, err := strconv.Atoi(v) + if err != nil || secs <= 0 { + return 0 + } + return time.Duration(secs) * time.Second +} + +func isVolumeFilesRetryable(status int, body string) bool { + if status == http.StatusTooManyRequests || status >= 500 { + return true + } + upper := strings.ToUpper(body) + return strings.Contains(upper, "RESOURCE_EXHAUSTED") || + strings.Contains(upper, "THROTTL") || + strings.Contains(upper, "TOO MANY REQUESTS") +} + +func (conn *DatabricksConn) volumeDeleteFile(ctx context.Context, volumePath string) error { + if volumePath == "" { + return nil + } + if ctx == nil { + ctx = context.Background() + } + + url := conn.volumeFilesAPIURL(volumePath) + token := conn.GetProp("token") + client := &http.Client{Timeout: 30 * time.Second} + + var lastErr error + var retryAfter time.Duration + for attempt := 0; attempt <= volumeFilesMaxRetries; attempt++ { + if attempt > 0 { + wait := volumeFilesRetryWait(attempt, retryAfter) + retryAfter = 0 + g.Debug("volume delete throttled for %s, retrying in %s (attempt %d/%d)", volumePath, wait, attempt, volumeFilesMaxRetries) + select { + case <-ctx.Done(): + if lastErr != nil { + return g.Error(lastErr, "could not delete volume file %s: %s", volumePath, ctx.Err()) + } + return ctx.Err() + case <-time.After(wait): + } + } + + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil) + if err != nil { + return g.Error(err, "could not build volume delete request for %s", volumePath) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + + resp, err := client.Do(req) + if err != nil { + lastErr = err + if ctx.Err() != nil { + return g.Error(err, "could not delete volume file %s", volumePath) + } + continue + } + + respBytes, _ := io.ReadAll(resp.Body) + resp.Body.Close() + body := string(respBytes) + + if resp.StatusCode == http.StatusNotFound || (resp.StatusCode >= 200 && resp.StatusCode < 300) { + return nil + } + + lastErr = g.Error("unexpected response %d deleting volume file %s: %s", resp.StatusCode, volumePath, body) + if !isVolumeFilesRetryable(resp.StatusCode, body) { + return lastErr + } + retryAfter = parseRetryAfter(resp.Header) + } + + return g.Error(lastErr, "could not delete volume file %s after %d retries", volumePath, volumeFilesMaxRetries) +} + // VolumeDelete delete files in a Databricks volume path func (conn *DatabricksConn) VolumeDelete(volumePaths ...string) (err error) { + if len(volumePaths) == 0 { + return nil + } - deleteContext := g.NewContext(conn.context.Ctx) + parent := context.Background() + if conn.Context() != nil && conn.Context().Ctx != nil { + parent = conn.Context().Ctx + } + // Cap parallelism so cleanup does not stampede S3 behind Volumes. + deleteContext := g.NewContext(parent, 3) for _, volumePath := range volumePaths { + if volumePath == "" { + continue + } deleteContext.Wg.Write.Add() - go func(volumePath string) { defer deleteContext.Wg.Write.Done() - - url := g.F("https://%s/api/2.0/fs/files%s", conn.GetProp("host"), volumePath) - headers := map[string]string{"Authorization": "Bearer " + conn.GetProp("token")} - _, respBytes, err := net.ClientDo("DELETE", url, nil, headers) - if err != nil { - deleteContext.CaptureErr(g.Error(err, "could not delete volume via API with for `%s` %s\nResponse: %s", volumePath, string(respBytes))) + // Use parent ctx so one exhausted failure does not cancel sibling backoff. + if err := conn.volumeDeleteFile(parent, volumePath); err != nil { + deleteContext.CaptureErr(g.Error(err, "could not delete volume file `%s`", volumePath)) } - }(volumePath) } deleteContext.Wg.Write.Wait() - return deleteContext.Err() } @@ -1097,9 +1239,8 @@ func (conn *DatabricksConn) UnloadViaVolume(tables ...Table) (filePath string, u defer func() { if !cast.ToBool(os.Getenv("SLING_KEEP_TEMP")) { g.Debug("deleting temporary volume: %s", volumeFolderPath) - err = conn.VolumeDelete(volumeFilePaths...) - if err != nil { - g.Warn("could not delete temporary volume files (%s): %s", volumeFolderPath, err.Error()) + if delErr := conn.VolumeDelete(volumeFilePaths...); delErr != nil { + g.Warn("could not delete temporary volume files (%s): %s", volumeFolderPath, delErr.Error()) } } }() diff --git a/core/dbio/database/database_test.go b/core/dbio/database/database_test.go index 23f6f32b2..f27acd9ab 100755 --- a/core/dbio/database/database_test.go +++ b/core/dbio/database/database_test.go @@ -7,12 +7,15 @@ import ( "io" "log" "math" + "net/http" + "net/http/httptest" "net/url" "os" "os/exec" "path/filepath" "regexp" "strings" + "sync/atomic" "testing" "time" @@ -1662,6 +1665,95 @@ func newTestDatabricksConn() *DatabricksConn { return conn } +func TestVolumeDeleteRetryOn429(t *testing.T) { + origRetries, origBase := volumeFilesMaxRetries, volumeFilesRetryBase + volumeFilesMaxRetries = 4 + volumeFilesRetryBase = time.Millisecond + defer func() { + volumeFilesMaxRetries = origRetries + volumeFilesRetryBase = origBase + }() + + t.Run("retries then succeeds", func(t *testing.T) { + var deletes int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodDelete, r.Method) + n := atomic.AddInt32(&deletes, 1) + if n < 3 { + w.Header().Set("Retry-After", "0") + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte(`{"error_code":"RESOURCE_EXHAUSTED","message":"AWS S3 is throttling requests; try again later."}`)) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + conn := newTestDatabricksConn() + conn.SetProp("host", strings.TrimPrefix(server.URL, "http://")) + conn.SetProp("protocol", "http") + conn.SetProp("token", "tok") + + err := conn.VolumeDelete("/Volumes/workspace/default/sling_volume/sling_temp/file.parquet") + require.NoError(t, err) + assert.GreaterOrEqual(t, atomic.LoadInt32(&deletes), int32(3)) + }) + + t.Run("404 is success", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + conn := newTestDatabricksConn() + conn.SetProp("host", strings.TrimPrefix(server.URL, "http://")) + conn.SetProp("protocol", "http") + conn.SetProp("token", "tok") + + err := conn.VolumeDelete("/Volumes/workspace/default/sling_volume/gone.parquet") + assert.NoError(t, err) + }) + + t.Run("403 is not retried", func(t *testing.T) { + var deletes int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&deletes, 1) + w.WriteHeader(http.StatusForbidden) + w.Write([]byte(`{"error_code":"PERMISSION_DENIED"}`)) + })) + defer server.Close() + + conn := newTestDatabricksConn() + conn.SetProp("host", strings.TrimPrefix(server.URL, "http://")) + conn.SetProp("protocol", "http") + conn.SetProp("token", "tok") + + err := conn.VolumeDelete("/Volumes/workspace/default/sling_volume/denied.parquet") + assert.Error(t, err) + assert.Equal(t, int32(1), atomic.LoadInt32(&deletes)) + }) + + t.Run("exhausted 429", func(t *testing.T) { + var deletes int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&deletes, 1) + w.WriteHeader(http.StatusTooManyRequests) + w.Write([]byte(`{"error_code":"RESOURCE_EXHAUSTED"}`)) + })) + defer server.Close() + + conn := newTestDatabricksConn() + conn.SetProp("host", strings.TrimPrefix(server.URL, "http://")) + conn.SetProp("protocol", "http") + conn.SetProp("token", "tok") + + err := conn.VolumeDelete("/Volumes/workspace/default/sling_volume/busy.parquet") + assert.Error(t, err) + assert.Contains(t, err.Error(), "retries") + assert.Equal(t, int32(volumeFilesMaxRetries+1), atomic.LoadInt32(&deletes)) + }) +} + func TestMapZerobusIPCCompression(t *testing.T) { none, err := mapZerobusIPCCompression("none") require.NoError(t, err) From 866613565766259c799be6b5f8d27adc1a750b3b Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sat, 19 Sep 2026 20:55:49 -0300 Subject: [PATCH 18/56] feat(iceberg): add merge strategies and refactor AWS config handling - Enable merge strategy tests 26-29 (insert/update/update_insert/delete_insert) for Iceberg - Replace env var-based S3 credential hack with applyIcebergS3Props property mapping - Add attachIcebergAwsConfig to attach AWS session config to the catalog - Register gocloud IO for iceberg-go v0.6+ (S3/GCS/Azure blob access) - Update dependencies (gonum, google.golang.org/api, etc.) --- cmd/sling/sling_test.go | 13 +- core/dbio/database/database_iceberg.go | 1135 ++++++++++++++--- core/dbio/database/database_test.go | 213 ++++ core/dbio/templates/iceberg.yaml | 78 ++ core/sling/config.go | 22 +- core/sling/task_run_write.go | 108 ++ go.mod | 190 ++- go.sum | 621 ++++----- .../r.126.iceberg_incremental_merge.yaml | 88 ++ tests/replications/r.127.iceberg_cdc.yaml | 106 ++ tests/suite.cli.yaml | 60 + 11 files changed, 2017 insertions(+), 617 deletions(-) create mode 100644 tests/replications/r.126.iceberg_incremental_merge.yaml create mode 100644 tests/replications/r.127.iceberg_cdc.yaml diff --git a/cmd/sling/sling_test.go b/cmd/sling/sling_test.go index 560dcca27..5c8f5d58f 100755 --- a/cmd/sling/sling_test.go +++ b/cmd/sling/sling_test.go @@ -96,6 +96,9 @@ var connMap = map[dbio.Type]connTest{ dbio.TypeDbMotherDuck: {name: "motherduck", adjustCol: g.Bool(false)}, dbio.TypeDbAthena: {name: "athena", adjustCol: g.Bool(false)}, dbio.TypeDbIceberg: {name: "iceberg_r2", adjustCol: g.Bool(false)}, + dbio.Type("iceberg_glue"): {name: "iceberg_glue", adjustCol: g.Bool(false)}, + dbio.Type("iceberg_s3"): {name: "iceberg_s3", adjustCol: g.Bool(false)}, + dbio.Type("iceberg_sql"): {name: "iceberg_sql", adjustCol: g.Bool(false)}, dbio.TypeDbMySQL: {name: "mysql", schema: "mysql"}, dbio.TypeDbOracle: {name: "oracle", schema: "oracle", useBulk: g.Bool(false)}, dbio.Type("oracle_sqlldr"): {name: "oracle", schema: "oracle", useBulk: g.Bool(true), adjustCol: g.Bool(false)}, @@ -669,7 +672,7 @@ func runOneTask(t *testing.T, ctx context.Context, file g.FileItem, connType dbi viewName := table.FullName() dropViewSQL := g.R(dbConn.GetTemplateValue("core.drop_view"), "view", viewName) dropViewSQL = strings.TrimSpace(dropViewSQL) - if g.In(connType, dbio.TypeDbIceberg) { + if tgtType == dbio.TypeDbIceberg { dropViewSQL = "" // iceberg does not support views } @@ -1201,8 +1204,12 @@ func TestSuiteDatabaseAthena(t *testing.T) { func TestSuiteDatabaseIceberg(t *testing.T) { t.Parallel() - testSuite(t, dbio.TypeDbIceberg, "1-4,6-8") - // testSuite(t, dbio.TypeDbIceberg, "1-4,6-12") + // 5 = truncate (not supported). 9-12 = incremental with views / extra tables. + // 26-29 = merge strategies (insert / update / update_insert / delete_insert). + testSuite(t, dbio.TypeDbIceberg, "1-4,6-8,26-29") + testSuite(t, dbio.Type("iceberg_glue"), "1-4,6-8,26-29") + testSuite(t, dbio.Type("iceberg_s3"), "1-4,6-8,26-29") + testSuite(t, dbio.Type("iceberg_sql"), "1-4,6-8,26-29") } func TestSuiteDatabaseDB2(t *testing.T) { diff --git a/core/dbio/database/database_iceberg.go b/core/dbio/database/database_iceberg.go index 87705a3a6..148ff0805 100644 --- a/core/dbio/database/database_iceberg.go +++ b/core/dbio/database/database_iceberg.go @@ -3,9 +3,12 @@ package database import ( "context" "database/sql" - "database/sql/driver" + "iter" "maps" "os" + "path" + "reflect" + "strconv" "strings" "time" @@ -17,11 +20,13 @@ import ( "github.com/apache/iceberg-go/catalog/glue" "github.com/apache/iceberg-go/catalog/rest" sqlcat "github.com/apache/iceberg-go/catalog/sql" + _ "github.com/apache/iceberg-go/io/gocloud" // S3 / GCS / Azure blob IO for iceberg-go v0.6+ "github.com/apache/iceberg-go/table" "github.com/apache/iceberg-go/utils" awsv2 "github.com/aws/aws-sdk-go-v2/aws" awsv2config "github.com/aws/aws-sdk-go-v2/config" awsv2creds "github.com/aws/aws-sdk-go-v2/credentials" + awsv2glue "github.com/aws/aws-sdk-go-v2/service/glue" "github.com/flarco/g" "github.com/flarco/g/net" "github.com/samber/lo" @@ -156,33 +161,7 @@ func (conn *IcebergConn) connectREST() error { } } - // Pass through S3 properties for filesystem access - for key, value := range conn.properties { - if strings.HasPrefix(key, "s3") { - // Map common S3 properties to what iceberg-go expects - // FIXME: for now set through env, need to refactor iceberg-go - switch key { - case "s3_access_key_id": - os.Setenv("AWS_ACCESS_KEY_ID", value) - props["s3.access-key-id"] = value - case "s3_secret_access_key": - os.Setenv("AWS_SECRET_ACCESS_KEY", value) - props["s3.secret-access-key"] = value - case "s3_session_token": - os.Setenv("AWS_SESSION_TOKEN", value) - props["s3.session-token"] = value - case "s3_region": - os.Setenv("AWS_REGION", value) - props["s3.region"] = value - case "s3_endpoint": - os.Setenv("AWS_ENDPOINT", value) - props["s3.endpoint"] = value - case "s3_profile": - os.Setenv("AWS_PROFILE", value) - props["s3.profile"] = value - } - } - } + maps.Copy(props, conn.applyIcebergS3Props()) if len(props) > 0 { g.Debug("using additional props for iceberg REST: %s", g.Marshal(lo.Keys(props))) @@ -246,6 +225,9 @@ func (conn *IcebergConn) connectREST() error { } conn.Catalog = cat + if err := conn.attachIcebergAwsConfig(); err != nil { + return err + } return nil } @@ -253,63 +235,126 @@ func (conn *IcebergConn) isS3TablesViaREST() bool { return strings.HasPrefix(conn.Warehouse, "arn:aws:s3tables") } +// applyIcebergS3Props maps s3_* connection keys to iceberg-go properties and +// AWS env vars (gocloud IO still reads AWS_REGION / AWS_S3_ENDPOINT). +func (conn *IcebergConn) applyIcebergS3Props() iceberg.Properties { + props := iceberg.Properties{} + if v := conn.GetProp("s3_access_key_id"); v != "" { + os.Setenv("AWS_ACCESS_KEY_ID", v) + props["s3.access-key-id"] = v + } + if v := conn.GetProp("s3_secret_access_key"); v != "" { + os.Setenv("AWS_SECRET_ACCESS_KEY", v) + props["s3.secret-access-key"] = v + } + if v := conn.GetProp("s3_session_token"); v != "" { + os.Setenv("AWS_SESSION_TOKEN", v) + props["s3.session-token"] = v + } + if v := conn.GetProp("s3_region"); v != "" { + os.Setenv("AWS_REGION", v) + os.Setenv("AWS_DEFAULT_REGION", v) + props["s3.region"] = v + } + if v := conn.GetProp("s3_endpoint"); v != "" { + os.Setenv("AWS_ENDPOINT", v) + os.Setenv("AWS_S3_ENDPOINT", v) // iceberg-go gocloud IO fallback + props["s3.endpoint"] = v + } + if v := conn.GetProp("s3_profile"); v != "" { + os.Setenv("AWS_PROFILE", v) + props["s3.profile"] = v + } + return props +} + +func (conn *IcebergConn) icebergAwsConfig() (awsv2.Config, error) { + region := conn.GetProp("s3_region") + accessKey := conn.GetProp("s3_access_key_id") + secret := conn.GetProp("s3_secret_access_key") + token := conn.GetProp("s3_session_token") + profile := conn.GetProp("s3_profile") + + opts := []func(*awsv2config.LoadOptions) error{} + if region != "" { + opts = append(opts, awsv2config.WithRegion(region)) + } + switch { + case accessKey != "" && secret != "": + g.Debug("iceberg: using static credentials (Key ID: %s)", accessKey) + opts = append(opts, awsv2config.WithCredentialsProvider( + awsv2creds.NewStaticCredentialsProvider(accessKey, secret, token), + )) + case profile != "": + g.Debug("iceberg: using AWS profile=%s region=%s", profile, region) + opts = append(opts, awsv2config.WithSharedConfigProfile(profile)) + default: + g.Debug("iceberg: using default AWS credential chain") + } + + awsCfg, err := awsv2config.LoadDefaultConfig(context.Background(), opts...) + if err != nil { + return awsv2.Config{}, g.Error(err, "Failed to create AWS config for iceberg") + } + return awsCfg, nil +} + +func (conn *IcebergConn) attachIcebergAwsConfig() error { + if conn.GetProp("s3_region") == "" && conn.GetProp("s3_access_key_id") == "" && conn.GetProp("s3_profile") == "" { + return nil + } + awsCfg, err := conn.icebergAwsConfig() + if err != nil { + return err + } + conn.context.Ctx = utils.WithAwsConfig(conn.context.Ctx, &awsCfg) + return nil +} + +func icebergNonIcebergCatalogErr(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "not an EXTERNAL_TABLE") || strings.Contains(msg, "not an iceberg table") +} + +func (conn *IcebergConn) dropGlueEntry(identifier table.Identifier) error { + if len(identifier) < 2 { + return g.Error("invalid Glue table identifier: %v", identifier) + } + awsCfg, err := conn.icebergAwsConfig() + if err != nil { + return err + } + in := &awsv2glue.DeleteTableInput{ + DatabaseName: awsv2.String(identifier[len(identifier)-2]), + Name: awsv2.String(identifier[len(identifier)-1]), + } + if accountID := conn.GetProp("glue_account_id"); accountID != "" { + in.CatalogId = awsv2.String(accountID) + } + _, err = awsv2glue.NewFromConfig(awsCfg).DeleteTable(conn.Context().Ctx, in) + if err != nil { + return g.Error(err, "cannot drop Glue table %s.%s", awsv2.ToString(in.DatabaseName), awsv2.ToString(in.Name)) + } + return nil +} + func (conn *IcebergConn) connectGlue() error { - // Get AWS credentials from connection properties - // accountID := conn.GetProp("glue_account_id") warehouse := conn.GetProp("glue_warehouse") - awsAccessKeyID := conn.GetProp("s3_access_key_id") - awsSecretAccessKey := conn.GetProp("s3_secret_access_key") - awsSessionToken := conn.GetProp("s3_session_token") awsRegion := conn.GetProp("s3_region") - awsProfile := conn.GetProp("s3_profile") if awsRegion == "" { return g.Error("AWS region not specified") } props := map[string]string{"warehouse": warehouse} + maps.Copy(props, conn.applyIcebergS3Props()) - var awsCfg awsv2.Config - var err error - - // Set credentials if provided - if awsAccessKeyID != "" && awsSecretAccessKey != "" { - g.Debug("iceberg: using static credentials (Key ID: %s)", awsAccessKeyID) - - // Create AWS config with static credentials - awsCfg, err = awsv2config.LoadDefaultConfig(context.Background(), - awsv2config.WithRegion(awsRegion), - awsv2config.WithCredentialsProvider( - awsv2creds.NewStaticCredentialsProvider( - awsAccessKeyID, - awsSecretAccessKey, - awsSessionToken, - ), - ), - ) - if err != nil { - return g.Error(err, "Failed to create AWS config with static credentials") - } - } else if awsProfile != "" { - g.Debug("iceberg: using AWS profile=%s region=%s", awsProfile, awsRegion) - - // Use specified profile from AWS credentials file - awsCfg, err = awsv2config.LoadDefaultConfig(context.Background(), - awsv2config.WithRegion(awsRegion), - awsv2config.WithSharedConfigProfile(awsProfile), - ) - if err != nil { - return g.Error(err, "Failed to create AWS config with profile %s", awsProfile) - } - } else { - g.Debug("iceberg: using default AWS credential chain") - // Use default credential chain (env vars, IAM role, credential file, etc.) - awsCfg, err = awsv2config.LoadDefaultConfig(context.Background(), - awsv2config.WithRegion(awsRegion), - ) - if err != nil { - return g.Error(err, "Failed to create AWS config with default credentials") - } + awsCfg, err := conn.icebergAwsConfig() + if err != nil { + return err } if extra := conn.GetProp("glue_extra_props"); extra != "" { @@ -404,29 +449,7 @@ func (conn *IcebergConn) connectSQL() error { conn.Warehouse = warehouse } - // Pass through S3 properties for filesystem access - for key, value := range conn.properties { - if strings.HasPrefix(key, "s3") { - // Map common S3 properties to what iceberg-go expects - switch key { - case "s3_access_key_id": - props["s3.access-key-id"] = value - case "s3_secret_access_key": - props["s3.secret-access-key"] = value - case "s3_session_token": - props["s3.session-token"] = value - case "s3_region": - props["s3.region"] = value - case "s3_endpoint": - props["s3.endpoint"] = value - case "s3_profile": - props["s3.profile"] = value - default: - // Pass through any other s3_ properties as-is - props[key] = value - } - } - } + maps.Copy(props, conn.applyIcebergS3Props()) // Add any extra properties specified by the user if extra := conn.GetProp("sql_extra_props"); extra != "" { @@ -444,11 +467,49 @@ func (conn *IcebergConn) connectSQL() error { return g.Error(err, "Failed to create SQL catalog") } + // iceberg-go v0.6 added iceberg_tables.iceberg_type. Existing catalogs + // created by older clients are missing it; CREATE TABLE IF NOT EXISTS + // will not add the column. + if err = conn.migrateSQLCatalogIcebergType(); err != nil { + conn.CatalogSQLConn.Close() + return g.Error(err, "Failed to migrate Iceberg SQL catalog schema") + } + conn.Catalog = cat + if err = conn.attachIcebergAwsConfig(); err != nil { + conn.CatalogSQLConn.Close() + return err + } return nil } +func (conn *IcebergConn) migrateSQLCatalogIcebergType() error { + if conn.CatalogSQLConn == nil { + return nil + } + + q := `ALTER TABLE iceberg_tables ADD COLUMN iceberg_type VARCHAR DEFAULT 'TABLE'` + switch conn.CatalogSQLConn.GetType() { + case dbio.TypeDbPostgres: + q = `ALTER TABLE iceberg_tables ADD COLUMN IF NOT EXISTS iceberg_type VARCHAR NOT NULL DEFAULT 'TABLE'` + } + + _, err := conn.CatalogSQLConn.Exec(q) + if err == nil { + return nil + } + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "duplicate") || + strings.Contains(msg, "already exists") || + strings.Contains(msg, "does not exist") || + strings.Contains(msg, "no such table") || + strings.Contains(msg, "unknown table") { + return nil + } + return err +} + // Close closes the connection func (conn *IcebergConn) Close() error { if conn.duck != nil { @@ -512,7 +573,7 @@ func (conn *IcebergConn) GetDatabases() (data iop.Dataset, err error) { // GetTables returns tables for given schema func (conn *IcebergConn) GetTables(schema string) (data iop.Dataset, err error) { - return conn.getTablesOrViews(schema, false) + return conn.getTablesOrViews(schema) } // GetViews returns views for given schema @@ -524,10 +585,10 @@ func (conn *IcebergConn) GetViews(schema string) (data iop.Dataset, err error) { // GetTablesAndViews returns tables and views for given schema func (conn *IcebergConn) GetTablesAndViews(schema string) (data iop.Dataset, err error) { - return conn.getTablesOrViews(schema, true) + return conn.getTablesOrViews(schema) } -func (conn *IcebergConn) getTablesOrViews(schema string, includeViews bool) (data iop.Dataset, err error) { +func (conn *IcebergConn) getTablesOrViews(schema string) (data iop.Dataset, err error) { err = reconnectIfClosed(conn) if err != nil { return data, g.Error(err, "Could not reconnect") @@ -573,7 +634,7 @@ func (conn *IcebergConn) GetColumns(tableFName string, fields ...string) (column tableID := table.Identifier{t.Schema, t.Name} // Load table - tbl, err := conn.Catalog.LoadTable(conn.Context().Ctx, tableID, nil) + tbl, err := conn.Catalog.LoadTable(conn.Context().Ctx, tableID) if err != nil { if strings.Contains(err.Error(), "Table action can_get_metadata forbidden") { return columns, g.Error("%s, check table name", err.Error()) @@ -625,7 +686,7 @@ func (conn *IcebergConn) GetDataFiles(t Table) (dataFiles []iceberg.DataFile, er // Parse table identifier tableID := table.Identifier{t.Schema, t.Name} - tbl, err := conn.Catalog.LoadTable(conn.Context().Ctx, tableID, nil) + tbl, err := conn.Catalog.LoadTable(conn.Context().Ctx, tableID) if err != nil { return nil, g.Error(err, "could not load existing table: %s", t.FullName()) } @@ -677,7 +738,7 @@ func (conn *IcebergConn) GetMaxValue(t Table, colName string) (value any, maxCol // Parse table identifier tableID := table.Identifier{t.Schema, t.Name} - tbl, err := conn.Catalog.LoadTable(conn.Context().Ctx, tableID, nil) + tbl, err := conn.Catalog.LoadTable(conn.Context().Ctx, tableID) if err != nil { return 0, maxCol, g.Error(err, "could not load existing table: %s", t.FullName()) } @@ -773,7 +834,7 @@ func (conn *IcebergConn) StreamRowsContext(ctx context.Context, sql string, opti tableID := table.Identifier{tableSchema, tableName} // Load table - tbl, err := conn.Catalog.LoadTable(queryContext.Ctx, tableID, nil) + tbl, err := conn.Catalog.LoadTable(queryContext.Ctx, tableID) if err != nil { return nil, g.Error(err, "Failed to load table %s.%s", tableSchema, tableName) } @@ -817,64 +878,50 @@ func (conn *IcebergConn) StreamRowsContext(ctx context.Context, sql string, opti // Create the greater than expression using field reference fieldRef := iceberg.Reference(field.Name) - // Parse the incremental value to the appropriate type based on field type - var literal iceberg.Literal - - // Try to parse as different types based on field type var greaterThanExpr iceberg.UnboundPredicate sp := iop.NewStreamProcessor() switch field.Type { case iceberg.PrimitiveTypes.String: - literal = iceberg.NewLiteral(incrementalValue) greaterThanExpr = iceberg.GreaterThan(fieldRef, incrementalValue) case iceberg.PrimitiveTypes.Int32: if val, parseErr := cast.ToInt32E(incrementalValue); parseErr == nil { - literal = iceberg.NewLiteral(val) greaterThanExpr = iceberg.GreaterThan(fieldRef, val) } else { return nil, g.Error("cannot parse incremental value '%s' as integer: %v", incrementalValue, parseErr) } case iceberg.PrimitiveTypes.Int64: if val, parseErr := cast.ToInt64E(incrementalValue); parseErr == nil { - literal = iceberg.NewLiteral(val) greaterThanExpr = iceberg.GreaterThan(fieldRef, val) } else { return nil, g.Error("cannot parse incremental value '%s' as long: %v", incrementalValue, parseErr) } case iceberg.PrimitiveTypes.Float32: if val, parseErr := cast.ToFloat32E(incrementalValue); parseErr == nil { - literal = iceberg.NewLiteral(val) greaterThanExpr = iceberg.GreaterThan(fieldRef, val) } else { return nil, g.Error("cannot parse incremental value '%s' as float: %v", incrementalValue, parseErr) } case iceberg.PrimitiveTypes.Float64: if val, parseErr := cast.ToFloat64E(incrementalValue); parseErr == nil { - literal = iceberg.NewLiteral(val) greaterThanExpr = iceberg.GreaterThan(fieldRef, val) } else { return nil, g.Error("cannot parse incremental value '%s' as double: %v", incrementalValue, parseErr) } case iceberg.PrimitiveTypes.Date: if val, parseErr := sp.ParseTime(incrementalValue); parseErr == nil { - literal = iceberg.NewLiteral(iceberg.Date(val.Unix() / 86400)) // Convert to days since epoch greaterThanExpr = iceberg.GreaterThan(fieldRef, iceberg.Date(val.Unix()/86400)) } else { return nil, g.Error("cannot parse incremental value '%s' as date: %v", incrementalValue, parseErr) } case iceberg.PrimitiveTypes.TimestampTz: if val, parseErr := sp.ParseTime(incrementalValue); parseErr == nil { - literal = iceberg.NewLiteral(iceberg.Timestamp(val.UnixMicro())) greaterThanExpr = iceberg.GreaterThan(fieldRef, iceberg.Timestamp(val.UnixMicro())) } else { return nil, g.Error("cannot parse incremental value '%s' as timestamp: %v", incrementalValue, parseErr) } default: - // Default to string - literal = iceberg.NewLiteral(incrementalValue) greaterThanExpr = iceberg.GreaterThan(fieldRef, incrementalValue) } - _ = literal scanOpts = append(scanOpts, table.WithRowFilter(greaterThanExpr)) } @@ -966,7 +1013,6 @@ func (conn *IcebergConn) StreamRowsContext(ctx context.Context, sql string, opti type icebergResult struct { TotalRows uint64 - res driver.Result } func (r icebergResult) LastInsertId() (int64, error) { @@ -984,11 +1030,17 @@ func (conn *IcebergConn) ExecContext(ctx context.Context, sql string, args ...in schema := strings.Trim(strings.TrimPrefix(sql, "create schema "), `"`) return icebergResult{}, conn.CreateNamespaceIfNotExists(schema) case strings.HasPrefix(sql, "drop table "): - table := strings.Trim(strings.TrimPrefix(sql, "drop table "), `"`) - return icebergResult{}, conn.DropTable(table) + name := strings.TrimSpace(strings.TrimPrefix(sql, "drop table ")) + if strings.HasPrefix(strings.ToLower(name), "if exists") { + name = strings.TrimSpace(name[len("if exists"):]) + } + return icebergResult{}, conn.DropTable(name) case strings.HasPrefix(sql, "drop view "): - table := strings.Trim(strings.TrimPrefix(sql, "drop view "), `"`) - return icebergResult{}, conn.DropTable(table) + name := strings.TrimSpace(strings.TrimPrefix(sql, "drop view ")) + if strings.HasPrefix(strings.ToLower(name), "if exists") { + name = strings.TrimSpace(name[len("if exists"):]) + } + return icebergResult{}, conn.DropTable(name) case strings.Contains(sql, `"ddl_columns":`) && strings.Contains(sql, `"table":`): m, _ := g.UnmarshalMap(sql) var table Table @@ -1039,9 +1091,11 @@ func (conn *IcebergConn) CreateTable(tableName string, cols iop.Columns, tableDD // Add table properties including format-version: 2 props := iceberg.Properties{ - "format-version": "2", - "write.format.default": "parquet", - "created-by": "sling-cli", + "format-version": "2", + "write.format.default": "parquet", + "write.delete.mode": "merge-on-read", + "write.delete.isolation-level": "snapshot", + "created-by": "sling-cli", } // Add any additional properties from connection @@ -1113,6 +1167,10 @@ func (conn *IcebergConn) TableExists(t Table) (exists bool, err error) { identifier := table.Identifier{t.Schema, t.Name} exists, err = conn.Catalog.CheckTableExists(conn.context.Ctx, identifier) if err != nil { + // Glue returns this for leftover non-Iceberg tables in the same namespace. + if icebergNonIcebergCatalogErr(err) { + return true, nil + } return false, g.Error(err, "cannot check table existence: %s", t.FullName()) } @@ -1145,6 +1203,9 @@ func (conn *IcebergConn) DropTable(tableNames ...string) (err error) { } else { err = conn.Catalog.DropTable(conn.context.Ctx, identifier) } + if err != nil && icebergNonIcebergCatalogErr(err) { + err = conn.dropGlueEntry(identifier) + } if err != nil { if g.IsDebug() && strings.Contains(err.Error(), "RuntimeIOException") { g.Warn(err.Error()) @@ -1167,10 +1228,6 @@ func (conn *IcebergConn) CreateNamespaceIfNotExists(schema string) (err error) { // Some catalogs might not implement namespace checking, log and continue g.Debug("could not check if namespace exists: %w", nsErr) } else if !exists { - // Try to create the namespace - // nsProps := iceberg.Properties{ - // "created-by": "sling-cli", - // } g.Debug("creating namespace: %s", namespace) if err = conn.Catalog.CreateNamespace(conn.Context().Ctx, namespace, nil); err != nil { return g.Error(err, "could not create namespace %s", schema) @@ -1262,7 +1319,7 @@ func (conn *IcebergConn) BulkImportStream(tableFName string, ds *iop.Datastream) // Parse table identifier tableID := table.Identifier{t.Schema, t.Name} - tbl, err := conn.Catalog.LoadTable(conn.Context().Ctx, tableID, nil) + tbl, err := conn.Catalog.LoadTable(conn.Context().Ctx, tableID) if err != nil { return 0, g.Error(err, "could not load existing table: %s", tableFName) } @@ -1586,15 +1643,49 @@ func (conn *IcebergConn) generateIcebergSchema(columns iop.Columns) (*iceberg.Sc Required: false, // Default to nullable unless we have constraint info } + // Identifier fields must be required (Iceberg spec). PK columns are + // treated as identifiers for equality-delete merge. + if icebergColumnIsPK(col) { + fields[i].Required = true + } + // Check if column has NOT NULL constraint if col.Constraint != nil && strings.Contains(strings.ToLower(col.Constraint.Expression), "not null") { fields[i].Required = true } } - // Create schema with ID 0 (initial schema) - schema := iceberg.NewSchema(0, fields...) - return schema, nil + identIDs := icebergIdentifierFieldIDs(columns, fields) + if len(identIDs) == 0 { + return iceberg.NewSchema(0, fields...), nil + } + return iceberg.NewSchemaWithIdentifiers(0, identIDs, fields...), nil +} + +func icebergIdentifierFieldIDs(columns iop.Columns, fields []iceberg.NestedField) []int { + ids := make([]int, 0, len(columns)) + for i, col := range columns { + if !icebergColumnIsPK(col) { + continue + } + if i < len(fields) { + ids = append(ids, fields[i].ID) + } + } + return ids +} + +func icebergColumnIsPK(col iop.Column) bool { + if col.IsPrimaryKey() { + return true + } + if col.Metadata == nil { + return false + } + if col.Metadata[iop.ColMetaIsPrimaryKey.String()] != "" { + return true + } + return col.Metadata[iop.PrimaryKey.MetadataKey()] != "" } // icebergArrowSchema builds the arrow schema used to append into an iceberg @@ -1607,6 +1698,10 @@ func (conn *IcebergConn) icebergArrowSchema(columns iop.Columns) *arrow.Schema { fields := schema.Fields() changed := false for i, field := range fields { + if i < len(columns) && icebergColumnIsPK(columns[i]) && field.Nullable { + fields[i].Nullable = false + changed = true + } tsType, ok := field.Type.(*arrow.TimestampType) if !ok || tsType.TimeZone != "" { continue @@ -1680,21 +1775,30 @@ func (conn *IcebergConn) queryViaDuckDB(ctx context.Context, sql string, opts ma return nil, g.Error("missing qualifier \"iceberg_catalog\"") } + if err = conn.ensureDuckCatalog(); err != nil { + return nil, err + } + + return conn.duck.StreamContext(ctx, sql, opts) +} + +// ensureDuckCatalog ATTACHes the Iceberg catalog on conn.duck for reads and DuckDB merge. +func (conn *IcebergConn) ensureDuckCatalog() (err error) { if conn.duck != nil { - return conn.duck.StreamContext(ctx, sql, opts) + return nil } - // Create a DuckDB instance - conn.duck = iop.NewDuckDb(ctx, "sling_conn_id", conn.GetProp("sling_conn_id")) + if conn.CatalogType == dbio.IcebergCatalogTypeSQL { + return g.Error("unsupported catalog for DuckDB Iceberg extension.") + } - // Add iceberg extensions + conn.duck = iop.NewDuckDb(conn.Context().Ctx, "sling_conn_id", conn.GetProp("sling_conn_id")) conn.duck.AddExtension("iceberg") conn.duck.AddExtension("https") - // Open DuckDB connection - err = conn.duck.Open() - if err != nil { - return nil, g.Error(err, "could not open DuckDB connection for Iceberg query") + if err = conn.duck.Open(); err != nil { + conn.duck = nil + return g.Error(err, "could not open DuckDB connection for Iceberg query") } attachSQL := "" @@ -1706,79 +1810,776 @@ func (conn *IcebergConn) queryViaDuckDB(ctx context.Context, sql string, opts ma switch catalogType { case dbio.IcebergCatalogTypeREST: - // make secret secretProps := MakeDuckDbSecretProps(conn, iop.DuckDbSecretTypeIceberg) secret := iop.NewDuckDbSecret("iceberg_secret", iop.DuckDbSecretTypeIceberg, secretProps) conn.duck.AddSecret(secret) - // make attach SQL restEndpoint := conn.GetProp("rest_uri") if restEndpoint == "" { - return nil, g.Error("rest_uri property is required for REST catalog") + conn.duck.Close() + conn.duck = nil + return g.Error("rest_uri property is required for REST catalog") } attachSQL = g.F("ATTACH '%s' AS iceberg_catalog (TYPE ICEBERG, SECRET iceberg_secret, ENDPOINT '%s')", conn.Warehouse, restEndpoint) case dbio.IcebergCatalogTypeGlue, dbio.IcebergCatalogTypeS3Tables: - // Map secret credentials var storageSecretType iop.DuckDbSecretType for key := range conn.properties { if storageSecretType != iop.DuckDbSecretTypeUnknown { break } else if strings.HasPrefix(key, "s3_") { storageSecretType = iop.DuckDbSecretTypeS3 + } else if strings.HasPrefix(key, "gcs_") { + storageSecretType = iop.DuckDbSecretTypeGCS } } if storageSecretType == iop.DuckDbSecretTypeUnknown { - return nil, g.Error("could not make AWS duckdb Iceberg secret for Iceberg query") + conn.duck.Close() + conn.duck = nil + return g.Error("could not make AWS duckdb Iceberg secret for Iceberg query") } - // make secret secretProps := MakeDuckDbSecretProps(conn, storageSecretType) secret := iop.NewDuckDbSecret("iceberg_storage_secret", storageSecretType, secretProps) conn.duck.AddSecret(secret) - // make attach SQL if catalogType == dbio.IcebergCatalogTypeGlue { - // see https://duckdb.org/docs/stable/core_extensions/iceberg/amazon_sagemaker_lakehouse accountID := conn.GetProp("glue_account_id") namespace := conn.GetProp("glue_namespace") if accountID == "" || namespace == "" { - return nil, g.Error("glue_account_id and glue_namespace properties are required for GLUE catalog") + conn.duck.Close() + conn.duck = nil + return g.Error("glue_account_id and glue_namespace properties are required for GLUE catalog") } warehouse := g.F("%s:s3tablescatalog/%s", accountID, namespace) - attachSQL = g.F("ATTACH '%s' AS iceberg_catalog (TYPE ICEBERG, ENDPOINT_TYPE glue, SECRET iceberg_storage_secret)", warehouse) } if catalogType == dbio.IcebergCatalogTypeS3Tables { - // see https://duckdb.org/docs/stable/core_extensions/iceberg/amazon_s3_tables arn := conn.GetProp("s3tables_arn", "rest_warehouse") if !strings.HasPrefix(arn, "arn:aws:s3tables") { - return nil, g.Error("warehouse property is required for S3 Tables catalog via DuckDB") + conn.duck.Close() + conn.duck = nil + return g.Error("warehouse property is required for S3 Tables catalog via DuckDB") } attachSQL = g.F("ATTACH '%s' AS iceberg_catalog (TYPE ICEBERG, ENDPOINT_TYPE s3_tables, SECRET iceberg_storage_secret)", arn) } default: - return nil, g.Error("Unsupported catalog type for DuckDB query: %s", catalogType) + conn.duck.Close() + conn.duck = nil + return g.Error("Unsupported catalog type for DuckDB query: %s", catalogType) + } + + if _, err = conn.duck.Exec(attachSQL + env.NoDebugKey); err != nil { + conn.duck.Close() + conn.duck = nil + return g.Error(err, "could not attach Iceberg catalog") } - // Attach - _, err = conn.duck.Exec(attachSQL + env.NoDebugKey) + // Do not USE iceberg_catalog — SET schema fails (no catalog+schema named iceberg_catalog). + return nil +} + +// icebergMergeEngine is the writer used for Iceberg incremental+PK / CDC. +type icebergMergeEngine string + +const ( + icebergMergeEngineAppend icebergMergeEngine = "append" + icebergMergeEngineGo icebergMergeEngine = "go" + icebergMergeEngineDuckDB icebergMergeEngine = "duckdb" +) + +type icebergStorageKind string + +const ( + icebergStorageS3 icebergStorageKind = "s3" + icebergStorageGCS icebergStorageKind = "gcs" + icebergStorageAzure icebergStorageKind = "azure" + icebergStorageLocal icebergStorageKind = "local" + icebergStorageUnknown icebergStorageKind = "unknown" +) + +// icebergGoHasRowDelta reports whether the linked iceberg-go build exposes +// Transaction.NewRowDelta (apache iceberg-go v0.6+). +func icebergGoHasRowDelta() bool { + t := reflect.TypeOf((*table.Transaction)(nil)) + _, ok := t.MethodByName("NewRowDelta") + return ok +} + +type icebergMergeSplit struct { + Columns iop.Columns + Upserts [][]any + Deletes [][]any // PK rows to equality-delete (may include upsert PKs) + Count uint64 +} + +func icebergPKKey(row []any, pkIdx []int) string { + parts := make([]string, len(pkIdx)) + for i, idx := range pkIdx { + if idx < len(row) { + parts[i] = cast.ToString(row[idx]) + } + } + return strings.Join(parts, "\x1f") +} + +// icebergCatalogKind returns REST / glue / sql / s3tables for routing. +func (conn *IcebergConn) icebergCatalogKind() dbio.IcebergCatalogType { + if conn != nil && conn.isS3TablesViaREST() { + return dbio.IcebergCatalogTypeS3Tables + } + if conn == nil { + return "" + } + return conn.CatalogType +} + +func (conn *IcebergConn) icebergStorageKind() icebergStorageKind { + if conn == nil { + return icebergStorageUnknown + } + warehouse := conn.Warehouse + if warehouse == "" { + warehouse = conn.GetProp("rest_warehouse", "sql_warehouse", "glue_warehouse") + } + w := strings.ToLower(warehouse) + get := func(keys ...string) string { + for _, k := range keys { + if v := conn.GetProp(k); v != "" { + return v + } + } + return "" + } + + switch { + case strings.HasPrefix(w, "gs://"), strings.HasPrefix(w, "gcs://"), get("gcs_access_key_id") != "": + return icebergStorageGCS + case strings.HasPrefix(w, "abfs"), strings.HasPrefix(w, "abfss"), strings.HasPrefix(w, "wasb"), + strings.HasPrefix(w, "wasbs"), get("azure_account_name") != "", get("azure_account_key") != "": + return icebergStorageAzure + case strings.HasPrefix(w, "s3://"), strings.HasPrefix(w, "s3a://"), strings.HasPrefix(w, "arn:aws:s3tables"), + get("s3_access_key_id") != "", get("s3_region") != "", get("s3_endpoint") != "": + return icebergStorageS3 + case strings.HasPrefix(w, "file://"), strings.HasPrefix(w, "/"): + return icebergStorageLocal + default: + return icebergStorageUnknown + } +} + +func (conn *IcebergConn) mergeEngine(needMerge bool) (icebergMergeEngine, error) { + engine, err := conn.mergeEngineWith(needMerge, icebergGoHasRowDelta()) if err != nil { - return nil, g.Error(err, "could not attach Iceberg catalog") + return "", err } + g.Debug("iceberg merge engine=%s catalog=%s storage=%s", engine, conn.icebergCatalogKind(), conn.icebergStorageKind()) + return engine, nil +} - // Use the catalog - // getting => Catalog Error: SET schema: No catalog + schema named "iceberg_catalog" found. - // _, err = conn.duck.Exec("USE iceberg_catalog;" + env.NoDebugKey) - // if err != nil { - // return nil, g.Error(err, "could not use Iceberg catalog") - // } +// mergeEngineWith chooses append / go / duckdb. DuckDB is never used for +// Glue, SQL catalog, or Azure. +func (conn *IcebergConn) mergeEngineWith(needMerge bool, goHasRowDelta bool) (icebergMergeEngine, error) { + catalog := conn.icebergCatalogKind() + storage := conn.icebergStorageKind() + if !needMerge { + return icebergMergeEngineAppend, nil + } + if goHasRowDelta { + return icebergMergeEngineGo, nil + } - // Execute the actual query - return conn.duck.StreamContext(ctx, sql, opts) + duckOK := (catalog == dbio.IcebergCatalogTypeREST || catalog == dbio.IcebergCatalogTypeS3Tables) && + (storage == icebergStorageS3 || storage == icebergStorageGCS || storage == icebergStorageUnknown) + if duckOK { + return icebergMergeEngineDuckDB, nil + } + + return "", g.Error("Iceberg merge is not supported for catalog=%s storage=%s", catalog, storage) +} + +// CheckMergeSupported fails fast when Iceberg incremental+PK / CDC cannot merge +// for this catalog/storage (no silent append). +func (conn *IcebergConn) CheckMergeSupported() error { + _, err := conn.mergeEngine(true) + return err +} + +// MergeStream applies incremental upsert or CDC (insert/update/delete) to an +// Iceberg table without a temp catalog table. Buffers one batch in memory. +func (conn *IcebergConn) MergeStream(tableFName string, ds *iop.Datastream, pkFields []string, strategy *MergeStrategy) (count uint64, err error) { + err = reconnectIfClosed(conn) + if err != nil { + return 0, g.Error(err, "Could not reconnect") + } + if len(pkFields) == 0 { + return 0, g.Error("Iceberg merge requires a primary-key") + } + + data, err := ds.Collect(0) + if err != nil { + return 0, g.Error(err, "could not collect merge rows") + } + if len(data.Rows) == 0 { + return 0, nil + } + + split, err := conn.dedupMergeRows(data.Columns, data.Rows, pkFields, strategy) + if err != nil { + return 0, err + } + count = split.Count + + engine, err := conn.mergeEngine(true) + if err != nil { + return 0, err + } + + switch engine { + case icebergMergeEngineGo: + err = conn.mergeStreamGo(tableFName, split, pkFields) + case icebergMergeEngineDuckDB: + err = conn.mergeStreamDuckDB(tableFName, split, pkFields, strategy) + default: + err = g.Error("Iceberg merge is not supported for catalog=%s storage=%s", conn.icebergCatalogKind(), conn.icebergStorageKind()) + } + if err != nil { + return count, err + } + return count, nil +} + +func (conn *IcebergConn) dedupMergeRows(columns iop.Columns, rows [][]any, pkFields []string, strategy *MergeStrategy) (icebergMergeSplit, error) { + out := icebergMergeSplit{Columns: columns, Count: uint64(len(rows))} + if len(pkFields) == 0 { + return out, g.Error("Iceberg merge requires a primary-key") + } + + fieldMap := columns.FieldMap(true) + pkIdx := make([]int, len(pkFields)) + for i, pk := range pkFields { + idx, ok := fieldMap[strings.ToLower(pk)] + if !ok { + return out, g.Error("primary-key column %s not found in merge stream", pk) + } + pkIdx[i] = idx + } + + opIdx, seqIdx := -1, -1 + if i, ok := fieldMap[strings.ToLower(env.ReservedFields.SyncedOp)]; ok { + opIdx = i + } + if i, ok := fieldMap[strings.ToLower(env.ReservedFields.CDCSeq)]; ok { + seqIdx = i + } + + st := MergeStrategyNone + if strategy != nil { + st = *strategy + } + isCDC := st == MergeStrategyChangeCapture || st == MergeStrategyChangeCaptureSoft + soft := st == MergeStrategyChangeCaptureSoft + + type kept struct { + row []any + seq int64 + idx int + op string + } + best := map[string]kept{} + + for i, row := range rows { + key := icebergPKKey(row, pkIdx) + seq := int64(0) + if seqIdx >= 0 && seqIdx < len(row) { + seq = cast.ToInt64(row[seqIdx]) + } + op := "" + if opIdx >= 0 && opIdx < len(row) { + op = strings.ToUpper(strings.TrimSpace(cast.ToString(row[opIdx]))) + } + prev, ok := best[key] + if !ok || seq > prev.seq || (seq == prev.seq && i >= prev.idx) { + best[key] = kept{row: row, seq: seq, idx: i, op: op} + } + } + + for _, k := range best { + op := k.op + switch { + case isCDC && op == "D" && !soft: + out.Deletes = append(out.Deletes, k.row) + case isCDC && op == "D" && soft: + out.Deletes = append(out.Deletes, k.row) + out.Upserts = append(out.Upserts, k.row) + case isCDC && (op == "U" || op == "I" || op == "S" || op == ""): + if op == "U" { + out.Deletes = append(out.Deletes, k.row) + } + out.Upserts = append(out.Upserts, k.row) + default: + // incremental merge: equality-delete every PK, then append + out.Deletes = append(out.Deletes, k.row) + out.Upserts = append(out.Upserts, k.row) + } + } + + return out, nil +} + +func (conn *IcebergConn) mergeStreamGo(tableFName string, split icebergMergeSplit, pkFields []string) error { + t, err := ParseTableName(tableFName, conn.Type) + if err != nil { + return g.Error(err, "could not parse table name: %s", tableFName) + } + tableID := table.Identifier{t.Schema, t.Name} + tbl, err := conn.Catalog.LoadTable(conn.Context().Ctx, tableID) + if err != nil { + return g.Error(err, "could not load table: %s", tableFName) + } + + tbl, err = conn.ensureIdentifierFields(tbl, pkFields) + if err != nil { + return err + } + + pkFieldIDs, err := conn.pkFieldIDs(tbl.Schema(), pkFields) + if err != nil { + return err + } + + tx := tbl.NewTransaction() + snapshotProps := iceberg.Properties{ + "source": "sling-cli", + } + + var eqFiles []iceberg.DataFile + if len(split.Deletes) > 0 { + delCols, delRows := conn.projectPKRows(split.Columns, split.Deletes, pkFields) + delArrow, err := conn.pkArrowSchema(tbl.Schema(), pkFields) + if err != nil { + return err + } + eqRecords, err := conn.rowsToArrowRecords(delCols, delRows, delArrow) + if err != nil { + return g.Error(err, "could not build equality-delete records") + } + // WriteEqualityDeletes does not take ownership of batches. + defer releaseArrowRecords(eqRecords) + + eqFiles, err = tx.WriteEqualityDeletes(conn.Context().Ctx, pkFieldIDs, icebergRecordIter(eqRecords)) + if err != nil { + return g.Error(err, "could not write Iceberg equality deletes") + } + } + + var dataFiles []iceberg.DataFile + if len(split.Upserts) > 0 { + upCols := conn.columnsInSchema(split.Columns, tbl.Schema()) + upArrow := conn.annotateArrowFieldIDs(conn.icebergArrowSchema(upCols), tbl.Schema()) + upRecords, err := conn.rowsToArrowRecords(split.Columns, split.Upserts, upArrow) + if err != nil { + return g.Error(err, "could not build upsert records") + } + // WriteRecords releases each yielded batch; retain so our defer is safe. + defer releaseArrowRecords(upRecords) + + for df, werr := range table.WriteRecords(conn.Context().Ctx, tbl, upArrow, icebergRecordIterRetain(upRecords)) { + if werr != nil { + return g.Error(werr, "could not write Iceberg upsert data files") + } + dataFiles = append(dataFiles, df) + } + } + + if len(eqFiles) == 0 && len(dataFiles) == 0 { + return nil + } + + delta := tx.NewRowDelta(snapshotProps) + if len(eqFiles) > 0 { + delta.AddDeletes(eqFiles...) + } + if len(dataFiles) > 0 { + delta.AddRows(dataFiles...) + } + if err = delta.Commit(conn.Context().Ctx); err != nil { + return g.Error(err, "could not commit Iceberg row delta") + } + + newTable, err := tx.Commit(conn.Context().Ctx) + if err != nil { + return g.Error(err, "could not commit Iceberg merge transaction") + } + + details := g.M("location", tbl.Location(), "engine", "go", "eq_deletes", len(eqFiles), "data_files", len(dataFiles)) + if currSnapshot := newTable.CurrentSnapshot(); currSnapshot != nil { + details["snapshot_id"] = currSnapshot.SnapshotID + } + g.Debug("committed iceberg snapshot", details) + return nil +} + +func (conn *IcebergConn) ensureIdentifierFields(tbl *table.Table, pkFields []string) (*table.Table, error) { + if tbl == nil || len(pkFields) == 0 { + return tbl, nil + } + schema := tbl.Schema() + if len(schema.IdentifierFieldIDs) > 0 { + return tbl, nil + } + + paths := make([][]string, 0, len(pkFields)) + for _, pk := range pkFields { + field, ok := schema.FindFieldByNameCaseInsensitive(pk) + if !ok { + g.Warn("iceberg identifier field %s not found in schema", pk) + continue + } + paths = append(paths, []string{field.Name}) + } + if len(paths) == 0 { + return tbl, nil + } + + tx := tbl.NewTransaction() + if err := tx.UpdateSchema(false, false).SetIdentifierField(paths).Commit(); err != nil { + return nil, g.Error(err, "could not set Iceberg identifier fields %v", pkFields) + } + newTable, err := tx.Commit(conn.Context().Ctx) + if err != nil { + return nil, g.Error(err, "could not commit Iceberg identifier fields") + } + g.Debug("set iceberg identifier fields %v", pkFields) + return newTable, nil +} + +func (conn *IcebergConn) pkFieldIDs(schema *iceberg.Schema, pkFields []string) ([]int, error) { + ids := make([]int, 0, len(pkFields)) + for _, pk := range pkFields { + field, ok := schema.FindFieldByNameCaseInsensitive(pk) + if !ok { + return nil, g.Error("primary-key column %s not found in Iceberg schema", pk) + } + ids = append(ids, field.ID) + } + return ids, nil +} + +func (conn *IcebergConn) projectPKRows(columns iop.Columns, rows [][]any, pkFields []string) (iop.Columns, [][]any) { + fieldMap := columns.FieldMap(true) + pkIdx := make([]int, 0, len(pkFields)) + pkCols := make(iop.Columns, 0, len(pkFields)) + for _, pk := range pkFields { + idx, ok := fieldMap[strings.ToLower(pk)] + if !ok { + continue + } + pkIdx = append(pkIdx, idx) + pkCols = append(pkCols, columns[idx]) + } + out := make([][]any, len(rows)) + for i, row := range rows { + proj := make([]any, len(pkIdx)) + for j, idx := range pkIdx { + if idx < len(row) { + proj[j] = row[idx] + } + } + out[i] = proj + } + return pkCols, out +} + +func (conn *IcebergConn) pkArrowSchema(iceSchema *iceberg.Schema, pkFields []string) (*arrow.Schema, error) { + if iceSchema == nil { + return nil, g.Error("Iceberg schema is required for equality deletes") + } + fields := make([]iceberg.NestedField, 0, len(pkFields)) + for _, pk := range pkFields { + field, ok := iceSchema.FindFieldByNameCaseInsensitive(pk) + if !ok { + return nil, g.Error("primary-key column %s not found in Iceberg schema", pk) + } + fields = append(fields, field) + } + sc := iceberg.NewSchema(0, fields...) + arrowSc, err := table.SchemaToArrowSchema(sc, nil, true, false) + if err != nil { + return nil, g.Error(err, "could not convert equality-delete schema to Arrow") + } + return arrowSc, nil +} + +// columnsInSchema keeps source columns that exist on the Iceberg table. +// Extra source fields (e.g. json_data on an upsert CSV) are dropped so +// WriteRecords schema-compat does not fail. +func (conn *IcebergConn) columnsInSchema(columns iop.Columns, iceSchema *iceberg.Schema) iop.Columns { + if iceSchema == nil { + return columns + } + out := make(iop.Columns, 0, len(columns)) + for _, col := range columns { + if _, ok := iceSchema.FindFieldByNameCaseInsensitive(col.Name); ok { + out = append(out, col) + } + } + if len(out) == 0 { + return columns + } + return out +} + +func (conn *IcebergConn) annotateArrowFieldIDs(arrowSchema *arrow.Schema, iceSchema *iceberg.Schema) *arrow.Schema { + if arrowSchema == nil || iceSchema == nil { + return arrowSchema + } + fields := arrowSchema.Fields() + changed := false + for i, f := range fields { + iceField, ok := iceSchema.FindFieldByNameCaseInsensitive(f.Name) + if !ok { + continue + } + if _, has := f.Metadata.GetValue(table.ArrowParquetFieldIDKey); has { + continue + } + keys := append(append([]string{}, f.Metadata.Keys()...), table.ArrowParquetFieldIDKey) + vals := append(append([]string{}, f.Metadata.Values()...), strconv.Itoa(iceField.ID)) + fields[i].Metadata = arrow.NewMetadata(keys, vals) + changed = true + } + if !changed { + return arrowSchema + } + return arrow.NewSchema(fields, nil) +} + +func (conn *IcebergConn) rowsToArrowRecords(columns iop.Columns, rows [][]any, arrowSchema *arrow.Schema) ([]arrow.Record, error) { + if len(rows) == 0 { + return nil, nil + } + if arrowSchema == nil { + arrowSchema = conn.icebergArrowSchema(columns) + } + alloc := memory.NewGoAllocator() + builder := array.NewRecordBuilder(alloc, arrowSchema) + defer builder.Release() + + fieldMap := columns.FieldMap(true) + nFields := len(builder.Fields()) + + for _, row := range rows { + for i := 0; i < nFields; i++ { + name := arrowSchema.Field(i).Name + idx, ok := fieldMap[strings.ToLower(name)] + var val any + col := &iop.Column{Name: name} + if ok { + col = &columns[idx] + if idx < len(row) { + val = row[idx] + } + } else if i < len(columns) { + col = &columns[i] + if i < len(row) { + val = row[i] + } + } + iop.AppendToBuilder(builder.Field(i), col, val) + } + } + rec := builder.NewRecord() + return []arrow.Record{rec}, nil +} + +func icebergRecordIter(records []arrow.Record) iter.Seq2[arrow.RecordBatch, error] { + return icebergRecordIterMaybeRetain(records, false) +} + +func icebergRecordIterRetain(records []arrow.Record) iter.Seq2[arrow.RecordBatch, error] { + return icebergRecordIterMaybeRetain(records, true) +} + +func icebergRecordIterMaybeRetain(records []arrow.Record, retain bool) iter.Seq2[arrow.RecordBatch, error] { + return func(yield func(arrow.RecordBatch, error) bool) { + for _, rec := range records { + if rec == nil { + continue + } + if retain { + rec.Retain() + } + if !yield(rec, nil) { + return + } + } + } +} + +func releaseArrowRecords(records []arrow.Record) { + for _, rec := range records { + if rec != nil { + rec.Release() + } + } +} + +func (conn *IcebergConn) mergeStreamDuckDB(tableFName string, split icebergMergeSplit, pkFields []string, strategy *MergeStrategy) error { + if err := conn.ensureDuckCatalog(); err != nil { + return err + } + + t, err := ParseTableName(tableFName, conn.Type) + if err != nil { + return g.Error(err, "could not parse table name: %s", tableFName) + } + + tgt := conn.duckQualified(t.Schema, t.Name) + srcName := "sling_merge_src" + + fieldMap := split.Columns.FieldMap(true) + pkIdx := make([]int, 0, len(pkFields)) + for _, pk := range pkFields { + if i, ok := fieldMap[strings.ToLower(pk)]; ok { + pkIdx = append(pkIdx, i) + } + } + allRows := make([][]any, 0, len(split.Upserts)+len(split.Deletes)) + seen := map[string]struct{}{} + for _, row := range append(append([][]any{}, split.Upserts...), split.Deletes...) { + key := icebergPKKey(row, pkIdx) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + allRows = append(allRows, row) + } + + if err = conn.duckLoadTempTable(srcName, split.Columns, allRows); err != nil { + return err + } + + st := MergeStrategyNone + if strategy != nil { + st = *strategy + } + + sqls, err := conn.duckMergeSQL(tgt, srcName, split.Columns, pkFields, st) + if err != nil { + return err + } + + if _, err = conn.duck.ExecMultiContext(conn.Context().Ctx, sqls...); err != nil { + return g.Error(err, "DuckDB Iceberg merge failed") + } + + g.Debug("committed iceberg snapshot", g.M("engine", "duckdb", "target", tgt, "rows", split.Count)) + return nil +} + +func (conn *IcebergConn) duckQualified(schema, name string) string { + q := conn.Quote + if schema == "" { + return "iceberg_catalog." + q(name) + } + return "iceberg_catalog." + q(schema) + "." + q(name) +} + +func (conn *IcebergConn) duckMergeSQL(tgt, src string, columns iop.Columns, pkFields []string, strategy MergeStrategy) ([]string, error) { + if len(pkFields) == 0 { + return nil, g.Error("Iceberg merge requires a primary-key") + } + + if strategy == MergeStrategyNone { + strategy = MergeStrategy(conn.GetTemplateValue("variable.default_merge_strategy")) + } + if strategy == MergeStrategyNone { + strategy = MergeStrategyDeleteInsert + } + + tmpl := conn.GetTemplateValue(g.F("core.merge_%s", strategy)) + if tmpl == "" { + return nil, g.Error("merge strategy `%s` not supported for iceberg", strategy) + } + + pkMap := map[string]struct{}{} + pkQuoted := make([]string, len(pkFields)) + pkEqual := make([]string, len(pkFields)) + for i, pk := range pkFields { + qpk := conn.Quote(pk) + pkQuoted[i] = qpk + pkEqual[i] = g.F("src.%s = tgt.%s", qpk, qpk) + pkMap[strings.ToLower(pk)] = struct{}{} + } + + insertFields := make([]string, 0, len(columns)) + srcFields := make([]string, 0, len(columns)) + setFields := make([]string, 0, len(columns)) + setFieldsAll := make([]string, 0, len(columns)) + for _, col := range columns { + qcol := conn.Quote(col.Name) + insertFields = append(insertFields, qcol) + srcFields = append(srcFields, qcol) + setField := g.F("%s = src.%s", qcol, qcol) + setFieldsAll = append(setFieldsAll, setField) + if _, isPK := pkMap[strings.ToLower(col.Name)]; !isPK { + setFields = append(setFields, setField) + } + } + if len(setFields) == 0 { + setFields = setFieldsAll + } + + sql := g.R( + tmpl, + "src_table", src, + "tgt_table", tgt, + "src_tgt_pk_equal", strings.Join(pkEqual, " and "), + "pk_fields", strings.Join(pkQuoted, ", "), + "insert_fields", strings.Join(insertFields, ", "), + "src_fields", strings.Join(srcFields, ", "), + "set_fields", strings.Join(setFields, ", "), + ) + + sqls := ParseSQLMultiStatements(sql, conn.GetType()) + if len(sqls) == 0 { + return nil, g.Error("empty merge SQL for iceberg strategy `%s`", strategy) + } + return sqls, nil +} + +func (conn *IcebergConn) duckLoadTempTable(srcName string, columns iop.Columns, rows [][]any) error { + folder := path.Join(env.GetTempFolder(), "iceberg-merge", g.RandSuffix("src", 6)) + if err := os.MkdirAll(folder, 0755); err != nil { + return g.Error(err, "could not create temp dir for Iceberg merge") + } + defer env.RemoveAllLocalTempFile(folder) + + csvPath := path.Join(folder, "src.csv") + f, err := os.Create(csvPath) + if err != nil { + return g.Error(err, "could not create temp csv") + } + data := iop.NewDataset(columns) + data.Rows = rows + if _, err = data.WriteCsv(f); err != nil { + f.Close() + return g.Error(err, "could not write merge csv") + } + if err = f.Close(); err != nil { + return err + } + + escaped := strings.ReplaceAll(csvPath, `'`, `''`) + sql := g.F("CREATE OR REPLACE TEMP TABLE %s AS SELECT * FROM read_csv_auto('%s', header=true)", srcName, escaped) + if _, err = conn.duck.Exec(sql + env.NoDebugKey); err != nil { + return g.Error(err, "could not load DuckDB temp merge table") + } + return nil } diff --git a/core/dbio/database/database_test.go b/core/dbio/database/database_test.go index f27acd9ab..a01862098 100755 --- a/core/dbio/database/database_test.go +++ b/core/dbio/database/database_test.go @@ -2045,3 +2045,216 @@ func TestCopyViaZerobus_MissingEndpoint(t *testing.T) { assert.Error(t, err) assert.Contains(t, err.Error(), "zerobus_endpoint") } + +func icebergCdcCols() iop.Columns { + return iop.NewColumnsFromFields("id", "val", "_sling_synced_op", "_sling_cdc_seq") +} + +func icebergPkOf(split icebergMergeSplit) (upsertIDs, deleteIDs []any) { + for _, row := range split.Upserts { + upsertIDs = append(upsertIDs, row[0]) + } + for _, row := range split.Deletes { + deleteIDs = append(deleteIDs, row[0]) + } + return +} + +func newTestIcebergConn(t *testing.T) *IcebergConn { + t.Helper() + conn := &IcebergConn{} + conn.setContext(context.Background(), 1) + require.NoError(t, conn.Init()) + return conn +} + +func icebergConnFor(catalog dbio.IcebergCatalogType, storage icebergStorageKind) *IcebergConn { + conn := &IcebergConn{CatalogType: catalog} + conn.setContext(context.Background(), 1) + switch storage { + case icebergStorageS3: + conn.Warehouse = "s3://bucket/wh" + case icebergStorageGCS: + conn.Warehouse = "gs://bucket/wh" + case icebergStorageAzure: + conn.Warehouse = "abfss://c@acct.dfs.core.windows.net/wh" + } + return conn +} + +func TestIcebergDedupCDCHigherSeqWins(t *testing.T) { + st := MergeStrategyChangeCapture + cols := icebergCdcCols() + rows := [][]any{ + {1, "a", "U", int64(1)}, + {1, "b", "U", int64(3)}, + {1, "c", "U", int64(2)}, + } + split, err := newTestIcebergConn(t).dedupMergeRows(cols, rows, []string{"id"}, &st) + require.NoError(t, err) + require.Len(t, split.Upserts, 1) + assert.Equal(t, "b", split.Upserts[0][1]) + assert.Equal(t, uint64(3), split.Count) +} + +func TestIcebergDedupCDCUThenD(t *testing.T) { + st := MergeStrategyChangeCapture + cols := icebergCdcCols() + rows := [][]any{ + {1, "a", "U", int64(1)}, + {1, "a", "D", int64(2)}, + } + split, err := newTestIcebergConn(t).dedupMergeRows(cols, rows, []string{"id"}, &st) + require.NoError(t, err) + upsertIDs, deleteIDs := icebergPkOf(split) + assert.Empty(t, upsertIDs) + assert.Equal(t, []any{1}, deleteIDs) +} + +func TestIcebergDedupCDCDThenU(t *testing.T) { + st := MergeStrategyChangeCapture + cols := icebergCdcCols() + rows := [][]any{ + {1, "a", "D", int64(1)}, + {1, "b", "U", int64(2)}, + } + split, err := newTestIcebergConn(t).dedupMergeRows(cols, rows, []string{"id"}, &st) + require.NoError(t, err) + upsertIDs, deleteIDs := icebergPkOf(split) + assert.Equal(t, []any{1}, upsertIDs) + assert.Equal(t, []any{1}, deleteIDs) + assert.Equal(t, "b", split.Upserts[0][1]) +} + +func TestIcebergDedupSoftDelete(t *testing.T) { + st := MergeStrategyChangeCaptureSoft + cols := icebergCdcCols() + rows := [][]any{ + {1, "a", "D", int64(5)}, + } + split, err := newTestIcebergConn(t).dedupMergeRows(cols, rows, []string{"id"}, &st) + require.NoError(t, err) + require.Len(t, split.Upserts, 1) + require.Len(t, split.Deletes, 1) + assert.Equal(t, "D", split.Upserts[0][2]) +} + +func TestIcebergDedupIncrementalAllPKsInBothSets(t *testing.T) { + cols := iop.NewColumnsFromFields("id", "val") + rows := [][]any{ + {1, "a"}, + {2, "b"}, + {1, "a2"}, // last row wins + } + split, err := newTestIcebergConn(t).dedupMergeRows(cols, rows, []string{"id"}, nil) + require.NoError(t, err) + require.Len(t, split.Upserts, 2) + require.Len(t, split.Deletes, 2) + + got := map[any]any{} + for _, row := range split.Upserts { + got[row[0]] = row[1] + } + assert.Equal(t, "a2", got[1]) + assert.Equal(t, "b", got[2]) +} + +func TestIcebergDedupMergeWithoutPK(t *testing.T) { + _, err := newTestIcebergConn(t).dedupMergeRows(icebergCdcCols(), [][]any{{1, "a", "I", int64(1)}}, nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "primary-key") +} + +func TestIcebergMergeEngineRouter(t *testing.T) { + tests := []struct { + name string + needMerge bool + catalog dbio.IcebergCatalogType + storage icebergStorageKind + goHas bool + want icebergMergeEngine + wantErr bool + neverDuckDB bool + }{ + {"append no merge", false, dbio.IcebergCatalogTypeREST, icebergStorageS3, true, icebergMergeEngineAppend, false, false}, + {"sql catalog go", true, dbio.IcebergCatalogTypeSQL, icebergStorageS3, true, icebergMergeEngineGo, false, true}, + {"sql catalog old fork", true, dbio.IcebergCatalogTypeSQL, icebergStorageS3, false, "", true, true}, + {"azure go", true, dbio.IcebergCatalogTypeREST, icebergStorageAzure, true, icebergMergeEngineGo, false, true}, + {"azure old fork", true, dbio.IcebergCatalogTypeREST, icebergStorageAzure, false, "", true, true}, + {"rest old fork duckdb", true, dbio.IcebergCatalogTypeREST, icebergStorageS3, false, icebergMergeEngineDuckDB, false, false}, + {"s3tables old fork duckdb", true, dbio.IcebergCatalogTypeS3Tables, icebergStorageS3, false, icebergMergeEngineDuckDB, false, false}, + {"glue go never duckdb", true, dbio.IcebergCatalogTypeGlue, icebergStorageS3, true, icebergMergeEngineGo, false, true}, + {"glue old fork error", true, dbio.IcebergCatalogTypeGlue, icebergStorageS3, false, "", true, true}, + {"gcs rest old fork duckdb", true, dbio.IcebergCatalogTypeREST, icebergStorageGCS, false, icebergMergeEngineDuckDB, false, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := icebergConnFor(tt.catalog, tt.storage).mergeEngineWith(tt.needMerge, tt.goHas) + if tt.wantErr { + require.Error(t, err) + assert.Contains(t, err.Error(), "Iceberg merge is not supported") + assert.NotEqual(t, icebergMergeEngineDuckDB, got) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + if tt.neverDuckDB { + assert.NotEqual(t, icebergMergeEngineDuckDB, got) + } + }) + } +} + +func TestIcebergStorageKind(t *testing.T) { + assert.Equal(t, icebergStorageS3, icebergConnFor("", icebergStorageS3).icebergStorageKind()) + assert.Equal(t, icebergStorageGCS, icebergConnFor("", icebergStorageGCS).icebergStorageKind()) + assert.Equal(t, icebergStorageAzure, icebergConnFor("", icebergStorageAzure).icebergStorageKind()) + + azure := &IcebergConn{} + azure.setContext(context.Background(), 1) + azure.SetProp("azure_account_name", "acct") + assert.Equal(t, icebergStorageAzure, azure.icebergStorageKind()) + + s3 := &IcebergConn{Warehouse: "warehouse1"} + s3.setContext(context.Background(), 1) + s3.SetProp("s3_region", "us-east-1") + assert.Equal(t, icebergStorageS3, s3.icebergStorageKind()) +} + +func TestIcebergGoHasRowDelta(t *testing.T) { + assert.True(t, icebergGoHasRowDelta(), "apache iceberg-go v0.6+ should expose NewRowDelta") +} + +func TestIcebergDuckMergeSQLFromTemplates(t *testing.T) { + conn := newTestIcebergConn(t) + tgt := `iceberg_catalog."sling_test"."t"` + src := "sling_merge_src" + cols := iop.NewColumnsFromFields("id", "val") + + sqls, err := conn.duckMergeSQL(tgt, src, cols, []string{"id"}, MergeStrategyNone) + require.NoError(t, err) + joined := strings.Join(sqls, "\n") + assert.Contains(t, joined, `DELETE FROM iceberg_catalog."sling_test"."t"`) + assert.Contains(t, joined, `INSERT INTO iceberg_catalog."sling_test"."t"`) + assert.Contains(t, joined, `src."id" = tgt."id"`) + + sqls, err = conn.duckMergeSQL(tgt, src, cols, []string{"id"}, MergeStrategyInsert) + require.NoError(t, err) + assert.Contains(t, strings.Join(sqls, "\n"), "WHERE NOT EXISTS") + + sqls, err = conn.duckMergeSQL(tgt, src, icebergCdcCols(), []string{"id"}, MergeStrategyChangeCapture) + require.NoError(t, err) + joined = strings.Join(sqls, "\n") + assert.Contains(t, joined, "_sling_cdc_seq") + assert.Contains(t, joined, "_sling_synced_op != 'D'") + + sqls, err = conn.duckMergeSQL(tgt, src, icebergCdcCols(), []string{"id"}, MergeStrategyChangeCaptureSoft) + require.NoError(t, err) + joined = strings.Join(sqls, "\n") + assert.Contains(t, joined, "_sling_synced_op = 'D'") + assert.Contains(t, joined, "CURRENT_TIMESTAMP") + + _, err = conn.duckMergeSQL(tgt, src, cols, nil, MergeStrategyDeleteInsert) + require.Error(t, err) +} diff --git a/core/dbio/templates/iceberg.yaml b/core/dbio/templates/iceberg.yaml index baa803472..b1bc8c6b3 100644 --- a/core/dbio/templates/iceberg.yaml +++ b/core/dbio/templates/iceberg.yaml @@ -1,4 +1,6 @@ core: + drop_table: drop table {table} + incremental_select: | select {fields} from {table} where ({incremental_where_cond}){where_and} order by {update_key} asc --iceberg-json={"table_name": "{table_name}", "table_schema": "{table_schema}", "incremental_key": {update_key}, "incremental_value": "{incremental_value}", "fields_array": {fields_array}} @@ -9,8 +11,84 @@ core: backfill_where: '{update_key} >= {start_value} and {update_key} <= {end_value}' + # DuckDB-attached Iceberg catalog (merge fallback). MERGE INTO is not used. + merge_insert: | + INSERT INTO {tgt_table} ({insert_fields}) + SELECT {src_fields} FROM {src_table} src + WHERE NOT EXISTS ( + SELECT 1 FROM {tgt_table} tgt WHERE {src_tgt_pk_equal} + ) + + merge_update: | + UPDATE {tgt_table} tgt + SET {set_fields} + FROM {src_table} src + WHERE {src_tgt_pk_equal} + + # Iceberg tables have no PK constraint for INSERT OR REPLACE / MERGE INTO. + merge_update_insert: | + DELETE FROM {tgt_table} tgt + WHERE EXISTS ( + SELECT 1 FROM {src_table} src + WHERE {src_tgt_pk_equal} + ); + INSERT INTO {tgt_table} ({insert_fields}) + SELECT {src_fields} FROM {src_table} src + + merge_delete_insert: | + DELETE FROM {tgt_table} tgt + WHERE EXISTS ( + SELECT 1 FROM {src_table} src + WHERE {src_tgt_pk_equal} + ); + INSERT INTO {tgt_table} ({insert_fields}) + SELECT {src_fields} FROM {src_table} src + + merge_change_capture: | + DELETE FROM {tgt_table} tgt + WHERE EXISTS ( + SELECT 1 FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY {pk_fields} ORDER BY _sling_cdc_seq DESC) as _rn + FROM {src_table} + ) src + WHERE src._rn = 1 AND {src_tgt_pk_equal} + ); + INSERT INTO {tgt_table} ({insert_fields}) + SELECT {src_fields} FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY {pk_fields} ORDER BY _sling_cdc_seq DESC) as _rn + FROM {src_table} + WHERE _sling_synced_op != 'D' + ) src WHERE _rn = 1 + + merge_change_capture_soft: | + UPDATE {tgt_table} tgt SET _sling_synced_at = CURRENT_TIMESTAMP, _sling_synced_op = 'D' + WHERE COALESCE(_sling_synced_op, '') != 'D' + AND EXISTS ( + SELECT 1 FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY {pk_fields} ORDER BY _sling_cdc_seq DESC) as _rn + FROM {src_table} + ) src + WHERE src._rn = 1 AND src._sling_synced_op = 'D' AND {src_tgt_pk_equal} + ); + DELETE FROM {tgt_table} tgt + WHERE EXISTS ( + SELECT 1 FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY {pk_fields} ORDER BY _sling_cdc_seq DESC) as _rn + FROM {src_table} + WHERE _sling_synced_op != 'D' + ) src + WHERE src._rn = 1 AND {src_tgt_pk_equal} + ); + INSERT INTO {tgt_table} ({insert_fields}) + SELECT {src_fields} FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY {pk_fields} ORDER BY _sling_cdc_seq DESC) as _rn + FROM {src_table} + WHERE _sling_synced_op != 'D' + ) src WHERE _rn = 1 + variable: quote_char: '"' + default_merge_strategy: delete_insert native_type_map: binary: binary diff --git a/core/sling/config.go b/core/sling/config.go index 4414a5cce..66b0068ab 100644 --- a/core/sling/config.go +++ b/core/sling/config.go @@ -744,11 +744,13 @@ func (cfg *Config) Prepare() (err error) { case TruncateMode, BackfillMode: return g.Error("mode '%s' not yet supported for iceberg target.", cfg.Mode) case IncrementalMode: - if !cfg.Source.HasUpdateKey() { - return g.Error("for mode '%s' with iceberg target, must provided update-key", cfg.Mode) - } else if cfg.Source.HasPrimaryKey() { - g.Warn("for mode '%s' with iceberg target, primary-key is ineffective, incremental merge is not yet supported (only appends)", cfg.Mode) - cfg.Source.PrimaryKeyI = nil // delete PK + if !cfg.Source.HasUpdateKey() && !cfg.Source.HasPrimaryKey() { + return g.Error("for mode '%s' with iceberg target, must provide update-key and/or primary-key", cfg.Mode) + } + // primary-key is kept: incremental+PK uses Iceberg merge (row delta / DuckDB fallback) + case ChangeCaptureMode: + if !cfg.Source.HasPrimaryKey() { + return g.Error("for mode '%s' with iceberg target, must provide primary-key", cfg.Mode) } } } @@ -1573,6 +1575,16 @@ func (cfg *Config) CDCChangeFeed() string { return "" } +func (cfg *Config) icebergNeedsMerge(tgtConn database.Connection) bool { + if tgtConn.GetType() != dbio.TypeDbIceberg { + return false + } + if cfg.Mode == ChangeCaptureMode { + return true + } + return cfg.Mode == IncrementalMode && len(cfg.Source.PrimaryKey()) > 0 +} + // CDCSlotLevel returns the effective slot level after applying defaults func (cfg *Config) CDCSlotLevel() database.CDCSlotLevel { supported := cfg.SrcConn.Type.IsPostgresLike() || cfg.SrcConn.Type.IsMySQLLike() diff --git a/core/sling/task_run_write.go b/core/sling/task_run_write.go index 0644341f9..48ad24044 100644 --- a/core/sling/task_run_write.go +++ b/core/sling/task_run_write.go @@ -211,6 +211,11 @@ func (t *TaskExecution) WriteToDb(cfg *Config, df *iop.Dataflow, tgtConn databas // INSERT is upsert-by-PK for these stores upsertByInsert := g.In(tgtConn.GetType(), dbio.TypeDbScyllaDB, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable) + // Iceberg incremental+PK and CDC use MergeStream (row delta), not append and not SQL temp tables. + if cfg.icebergNeedsMerge(tgtConn) { + return t.writeToIcebergMerge(cfg, df, tgtConn) + } + // set direct insert mode directInsert := g.PtrVal(cfg.Target.Options.DirectInsert) || cast.ToBool(os.Getenv("SLING_DIRECT_INSERT")) @@ -669,6 +674,109 @@ func (t *TaskExecution) writeToDbDirectly(cfg *Config, df *iop.Dataflow, tgtConn return cnt, nil } +func (t *TaskExecution) writeToIcebergMerge(cfg *Config, df *iop.Dataflow, tgtConn database.Connection) (cnt uint64, err error) { + icebergConn, ok := tgtConn.Self().(*database.IcebergConn) + if !ok { + return 0, g.Error("iceberg merge requires IcebergConn") + } + + if len(cfg.Source.PrimaryKey()) == 0 { + return 0, g.Error("Iceberg merge requires a primary-key") + } + + if err = icebergConn.CheckMergeSupported(); err != nil { + return 0, err + } + + targetTable, err := initializeTargetTable(cfg, tgtConn) + if err != nil { + return 0, err + } + + if err := ensureSchemaExists(tgtConn, targetTable.Schema); err != nil { + return 0, err + } + + if paused := df.Pause(); !paused { + return 0, g.Error("could not pause streams to infer columns") + } + + sampleData, err := prepareDataflowForWriteDB(t, df, tgtConn) + if err != nil { + return 0, err + } + + targetTable.Columns = sampleData.Columns + if err := targetTable.SetKeys(cfg.Source.PrimaryKey(), cfg.Source.UpdateKey, cfg.Target.Options.TableKeys); err != nil { + return 0, g.Error(err, "could not set keys for "+targetTable.FullName()) + } + + if err := executeSQL(t, tgtConn, cfg.Target.Options.PreSQL, "pre"); err != nil { + return 0, err + } + + if err := createTable(t, tgtConn, targetTable, sampleData, false); err != nil { + return 0, err + } + + df.Columns = sampleData.Columns + setStage("5 - load-into-final") + + if err = t.ExecuteHooks(HookStagePreMerge); err != nil { + return 0, g.Error(err, "error executing pre-merge hooks") + } + + df.Unpause() + t.SetProgress("streaming data (iceberg merge)") + + data, err := df.Collect() + if err != nil { + return 0, g.Error(err, "could not collect Iceberg merge rows") + } + cnt = uint64(len(data.Rows)) + + ds := data.Stream() + pk := cfg.Source.PrimaryKey() + if casing := cfg.Target.Options.ColumnCasing; casing != nil { + for i, col := range pk { + pk[i] = casing.Apply(col, tgtConn.GetType()) + } + } + + merged, err := icebergConn.MergeStream(targetTable.FullName(), ds, pk, cfg.Target.Options.MergeStrategy) + if err != nil { + return 0, g.Error(err, "could not merge into "+targetTable.FullName()) + } + if merged > 0 { + cnt = merged + } + + df.SyncColumns() + df.SyncStats() + + if err = t.ExecuteHooks(HookStagePostMerge); err != nil { + return 0, g.Error(err, "error executing post-merge hooks") + } + + applySchemaAttributes(cfg, tgtConn, targetTable, df.Columns) + + if cnt == 0 { + g.Warn("no data or records found in stream. Nothing to merge.") + } + + if err := executeSQL(t, tgtConn, cfg.Target.Options.PostSQL, "post"); err != nil { + return cnt, err + } + + if err := df.Err(); err != nil { + setStage("6 - closing") + return cnt, err + } + + setStage("6 - closing") + return cnt, nil +} + // applySchemaAttributes applies foreign keys and indexes when schema migration is enabled // dfColumns should contain the source columns with FK/index metadata from the dataflow func applySchemaAttributes(cfg *Config, tgtConn database.Connection, targetTable database.Table, dfColumns iop.Columns) (err error) { diff --git a/go.mod b/go.mod index 1fc02f6c9..75995f566 100644 --- a/go.mod +++ b/go.mod @@ -3,30 +3,30 @@ module github.com/slingdata-io/sling-cli go 1.26.0 require ( - cloud.google.com/go v0.121.6 - cloud.google.com/go/bigquery v1.72.0 - cloud.google.com/go/bigtable v1.37.0 + cloud.google.com/go v0.123.0 + cloud.google.com/go/bigquery v1.74.0 + cloud.google.com/go/bigtable v1.42.0 cloud.google.com/go/cloudsqlconn v1.19.0 - cloud.google.com/go/storage v1.56.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 + cloud.google.com/go/storage v1.62.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.2 github.com/BurntSushi/toml v1.4.0 github.com/ClickHouse/clickhouse-go/v2 v2.34.0 github.com/PuerkitoBio/goquery v1.6.0 github.com/apache/arrow-adbc/go/adbc v1.9.0 - github.com/apache/arrow-go/v18 v18.5.0 - github.com/apache/iceberg-go v0.3.0 - github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.31.2 - github.com/aws/aws-sdk-go-v2/credentials v1.18.6 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 + github.com/apache/arrow-go/v18 v18.6.0 + github.com/apache/iceberg-go v0.6.0 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/config v1.32.17 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 github.com/aws/aws-sdk-go-v2/service/athena v1.51.0 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 - github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 - github.com/aws/smithy-go v1.24.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 + github.com/aws/smithy-go v1.25.1 github.com/bits-and-blooms/bloom/v3 v3.7.0 github.com/charmbracelet/bubbletea v1.3.6 github.com/charmbracelet/huh v1.0.0 @@ -59,7 +59,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/kardianos/service v1.2.4 - github.com/klauspost/compress v1.18.5 + github.com/klauspost/compress v1.18.6 github.com/kshedden/datareader v0.0.0-20210325133423-816b6ffdd011 github.com/labstack/echo/v4 v4.10.2 github.com/lib/pq v1.10.9 @@ -67,7 +67,7 @@ require ( github.com/maja42/goval v1.4.0 github.com/mark3labs/mcp-go v0.57.0 github.com/mattn/go-isatty v0.0.20 - github.com/mattn/go-sqlite3 v1.14.28 + github.com/mattn/go-sqlite3 v1.14.34 github.com/microsoft/go-mssqldb v1.9.3 github.com/nikolalohinski/gonja/v2 v2.9.0 github.com/nqd/flat v0.1.1 @@ -113,7 +113,7 @@ require ( golang.org/x/sys v0.45.0 golang.org/x/term v0.43.0 golang.org/x/text v0.37.0 - google.golang.org/api v0.258.0 + google.golang.org/api v0.278.0 gopkg.in/cheggaaa/pb.v2 v2.0.7 gopkg.in/inf.v0 v0.9.1 gopkg.in/yaml.v2 v2.4.0 @@ -129,26 +129,26 @@ require ( atomicgo.dev/keyboard v0.2.9 // indirect atomicgo.dev/schedule v0.1.0 // indirect cel.dev/expr v0.25.1 // indirect - cloud.google.com/go/auth v0.17.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect - cloud.google.com/go/iam v1.5.2 // indirect - cloud.google.com/go/longrunning v0.6.7 // indirect - cloud.google.com/go/monitoring v1.24.2 // indirect + cloud.google.com/go/iam v1.7.0 // indirect + cloud.google.com/go/longrunning v0.9.0 // indirect + cloud.google.com/go/monitoring v1.24.3 // indirect filippo.io/edwards25519 v1.1.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect github.com/AlecAivazis/survey/v2 v2.3.7 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/ClickHouse/ch-go v0.65.1 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/andybalholm/brotli v1.2.0 // indirect + github.com/andybalholm/brotli v1.2.1 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect @@ -156,23 +156,22 @@ require ( github.com/apache/thrift v0.23.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/atotto/clipboard v0.1.4 // indirect - github.com/aws/aws-sdk-go v1.55.6 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/glue v1.113.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/glue v1.141.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.22.0 // indirect + github.com/bits-and-blooms/bitset v1.24.2 // indirect github.com/boombuler/barcode v1.0.1 // indirect github.com/catppuccin/go v0.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect @@ -183,8 +182,7 @@ require ( github.com/charmbracelet/x/cellbuf v0.0.13 // indirect github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect - github.com/clipperhouse/stringish v0.1.1 // indirect - github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect github.com/cockroachdb/apd/v3 v3.2.1 // indirect github.com/coder/websocket v1.8.14 // indirect @@ -199,7 +197,6 @@ require ( github.com/disintegration/imaging v1.6.2 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dnephin/pflag v1.0.7 // indirect - github.com/docker/docker v28.5.2+incompatible // indirect github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/domodwyer/mailyak/v3 v3.6.2 // indirect @@ -212,7 +209,7 @@ require ( github.com/exasol/error-reporting-go v0.2.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/francoispqt/gojay v1.2.13 // indirect - github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect github.com/ganigeorgiev/fexpr v0.4.1 // indirect github.com/go-faster/city v1.0.1 // indirect @@ -228,8 +225,7 @@ require ( github.com/go-openapi/errors v0.21.0 // indirect github.com/go-openapi/strfmt v0.22.0 // indirect github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect + github.com/goccy/go-json v0.10.6 // indirect github.com/goccy/go-yaml v1.17.1 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect @@ -238,24 +234,24 @@ require ( github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/snappy v1.0.0 // indirect - github.com/google/flatbuffers v25.9.23+incompatible // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/jsonschema-go v0.4.2 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/wire v0.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect - github.com/googleapis/gax-go/v2 v2.15.0 // indirect - github.com/gookit/color v1.5.4 // indirect + github.com/google/wire v0.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect + github.com/gookit/color v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect - github.com/hamba/avro/v2 v2.30.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hashicorp/go-version v1.7.0 // indirect + github.com/hashicorp/go-version v1.9.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 // indirect @@ -273,7 +269,6 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -285,17 +280,13 @@ require ( github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.20 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect - github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/go-archive v0.2.0 // indirect github.com/moby/moby/api v1.55.0 // indirect github.com/moby/moby/client v0.5.1 // indirect - github.com/moby/sys/user v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.7.0 // indirect @@ -309,14 +300,14 @@ require ( github.com/nats-io/nats.go v1.36.0 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect - github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec // indirect github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a // indirect github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d // indirect @@ -325,11 +316,11 @@ require ( github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pocketbase/dbx v1.11.0 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/pquerna/otp v1.5.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/pterm/pterm v0.12.82 // indirect + github.com/pterm/pterm v0.12.83 // indirect github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/richardlehane/mscfb v1.0.4 // indirect @@ -338,29 +329,30 @@ require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.4.0 // indirect - github.com/shirou/gopsutil/v4 v4.25.1 // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/shoenig/go-m1cpu v0.2.1 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/slingdata-io/pocketbase v0.22.136 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect - github.com/stretchr/objx v0.5.2 // indirect - github.com/substrait-io/substrait v0.75.0 // indirect - github.com/substrait-io/substrait-go/v7 v7.2.0 // indirect - github.com/substrait-io/substrait-protobuf/go v0.75.0 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/substrait-io/substrait v0.87.0 // indirect + github.com/substrait-io/substrait-go/v8 v8.1.0 // indirect + github.com/substrait-io/substrait-protobuf/go v0.85.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tiendc/go-deepcopy v1.6.0 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect + github.com/twmb/avro v1.7.2 // indirect github.com/twmb/murmur3 v1.1.8 // indirect - github.com/uptrace/bun v1.2.12 // indirect - github.com/uptrace/bun/dialect/mssqldialect v1.2.12 // indirect - github.com/uptrace/bun/dialect/mysqldialect v1.2.12 // indirect - github.com/uptrace/bun/dialect/oracledialect v1.2.12 // indirect - github.com/uptrace/bun/dialect/pgdialect v1.2.12 // indirect - github.com/uptrace/bun/dialect/sqlitedialect v1.2.12 // indirect - github.com/uptrace/bun/extra/bundebug v1.2.12 // indirect + github.com/uptrace/bun v1.2.18 // indirect + github.com/uptrace/bun/dialect/mssqldialect v1.2.18 // indirect + github.com/uptrace/bun/dialect/mysqldialect v1.2.18 // indirect + github.com/uptrace/bun/dialect/oracledialect v1.2.18 // indirect + github.com/uptrace/bun/dialect/pgdialect v1.2.18 // indirect + github.com/uptrace/bun/dialect/sqlitedialect v1.2.18 // indirect + github.com/uptrace/bun/extra/bundebug v1.2.18 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect github.com/viant/xunsafe v0.8.0 // indirect @@ -374,32 +366,32 @@ require ( github.com/xuri/nfp v0.0.1 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - github.com/zeebo/xxh3 v1.0.2 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect - go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - gocloud.dev v0.41.0 // indirect + go.uber.org/zap v1.27.1 // indirect + gocloud.dev v0.45.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/image v0.38.0 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect - golang.org/x/time v0.14.0 // indirect + golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.44.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect - google.golang.org/grpc v1.79.3 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect + google.golang.org/grpc v1.80.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/VividCortex/ewma.v1 v1.1.1 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect @@ -411,19 +403,19 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gotest.tools/gotestsum v1.8.2 // indirect - modernc.org/libc v1.66.10 // indirect + modernc.org/libc v1.72.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.42.2 // indirect + modernc.org/sqlite v1.49.1 // indirect ) // replace github.com/slingdata-io/golyglot => ../golyglot replace github.com/slingdata-io/sling => ../sling -replace github.com/apache/iceberg-go => github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2 - -// replace github.com/apache/iceberg-go => ../iceberg-go +// apache iceberg-go v0.6+ provides NewRowDelta / WriteEqualityDeletes. +// Local fork work: replace github.com/apache/iceberg-go => ../iceberg-go +// replace github.com/apache/iceberg-go => github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2 // replace github.com/slingdata-io/godbc => ../godbc diff --git a/go.sum b/go.sum index 1b55a2b78..d09dee811 100644 --- a/go.sum +++ b/go.sum @@ -12,36 +12,36 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= -cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= -cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= -cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= -cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ= +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= -cloud.google.com/go/bigquery v1.72.0 h1:D/yLju+3Ens2IXx7ou1DJ62juBm+/coBInn4VVOg5Cw= -cloud.google.com/go/bigquery v1.72.0/go.mod h1:GUbRtmeCckOE85endLherHD9RsujY+gS7i++c1CqssQ= -cloud.google.com/go/bigtable v1.37.0 h1:Q+x7y04lQ0B+WXp03wc1/FLhFt4CwcQdkwWT0M4Jp3w= -cloud.google.com/go/bigtable v1.37.0/go.mod h1:HXqddP6hduwzrtiTCqZPpj9ij4hGZb4Zy1WF/dT+yaU= +cloud.google.com/go/bigquery v1.74.0 h1:Q6bAMv+eyvufOpIrfrYxhM46qq1D3ZQTdgUDQqKS+n8= +cloud.google.com/go/bigquery v1.74.0/go.mod h1:iViO7Cx3A/cRKcHNRsHB3yqGAMInFBswrE9Pxazsc90= +cloud.google.com/go/bigtable v1.42.0 h1:SREvT4jLhJQZXUjsLmFs/1SMQJ+rKEj1cJuPE9liQs8= +cloud.google.com/go/bigtable v1.42.0/go.mod h1:oZ30nofVB6/UYGg7lBwGLWSea7NZUvw/WvBBgLY07xU= cloud.google.com/go/cloudsqlconn v1.19.0 h1:zYq/RRfQ1qHP31p/YqbtGsII/T3qnLwepLf6ZNDSYGo= cloud.google.com/go/cloudsqlconn v1.19.0/go.mod h1:jAiwYjGNf8T00l8jRQWM7csHrIPuussPy+pQUT2klrc= cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= -cloud.google.com/go/datacatalog v1.26.0 h1:eFgygb3DTufTWWUB8ARk+dSuXz+aefNJXTlkWlQcWwE= -cloud.google.com/go/datacatalog v1.26.0/go.mod h1:bLN2HLBAwB3kLTFT5ZKLHVPj/weNz6bR0c7nYp0LE14= -cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= -cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= -cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= -cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= -cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= -cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= -cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= -cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= -cloud.google.com/go/storage v1.56.0 h1:iixmq2Fse2tqxMbWhLWC9HfBj1qdxqAmiK8/eqtsLxI= -cloud.google.com/go/storage v1.56.0/go.mod h1:Tpuj6t4NweCLzlNbw9Z9iwxEkrSem20AetIeH/shgVU= -cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= -cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= -dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= -dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +cloud.google.com/go/datacatalog v1.26.1 h1:bCRKA8uSQN8wGW3Tw0gwko4E9a64GRmbW1nCblhgC2k= +cloud.google.com/go/datacatalog v1.26.1/go.mod h1:2Qcq8vsHNxMDgjgadRFmFG47Y+uuIVsyEGUrlrKEdrg= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= +cloud.google.com/go/logging v1.13.2 h1:qqlHCBvieJT9Cdq4QqYx1KPadCQ2noD4FK02eNqHAjA= +cloud.google.com/go/logging v1.13.2/go.mod h1:zaybliM3yun1J8mU2dVQ1/qDzjbOqEijZCn6hSBtKak= +cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY= +cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= +cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE= +cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI= +cloud.google.com/go/storage v1.62.1 h1:Os0G3XbUbjZumkpDUf2Y0rLoXJTCF1kU2kWUujKYXD8= +cloud.google.com/go/storage v1.62.1/go.mod h1:cpYz/kRVZ+UQAF1uHeea10/9ewcRbxGoGNKsS9daSXA= +cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U= +cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= @@ -53,28 +53,26 @@ github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMb github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= github.com/99designs/keyring v1.2.2/go.mod h1:wes/FrByc8j7lFOAGLGSNEg8f/PaI3cgTBqhFkHUrPk= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.0 h1:mXlQ+2C8A4KpXTIIYYxgFYqSivjGTBQidq/b0xxZLuk= github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.4.0/go.mod h1:K//Ck7MUa+r9jpV69WLeWnnju5WJx5120AFsEzvumII= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1 h1:Wgf5rZba3YZqeTNJPtvqZoBu1sBN/L4sry+u2U3Y75w= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1/go.mod h1:xxCBG/f/4Vbmh2XQJBsOmNdxWUY5j/s27jujKPbQf14= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1 h1:bFWuoEKg+gImo7pvkiQEFAc8ocibADgXeiLAxWhWmkI= github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1/go.mod h1:Vih/3yc6yac2JzU4hzpaDupBJP0Flaia9rXXrU8xyww= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.2 h1:Uw4a4PZDGqGJoC3UTiXi7CpMSOPKUoKZJfcdD6+Tnxc= github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake v1.4.2/go.mod h1:qwMm9zmWPwY4OyGJH9/0F+2plJQe/aj28RPHpaO/Hgg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= @@ -85,8 +83,8 @@ github.com/Azure/go-autorest/autorest/to v0.4.1 h1:CxNHBqdzTr7rLtdrtb5CMjJcDut+W github.com/Azure/go-autorest/autorest/to v0.4.1/go.mod h1:EtaofgU4zmtvn1zT2ARsjRFdq9vXx0YWtmElwL+GZ9M= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= -github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= @@ -98,14 +96,14 @@ github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7Oputl github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4bOvKmDIx0WeTv9Qz+hghsgyjikFiPrseXHlKepO0= github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0/go.mod h1:vB2GH9GAYYJTO3mEn8oYwzEdhlayZIdQz6zdzgUIRvA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 h1:0s6TxfCu2KHkkZPnBfsQ2y5qia0jl3MMrmBhu3nCOYk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0/go.mod h1:Mf6O40IAyB9zR/1J8nGDDPirZQQPbYJni8Yisy7NTMc= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/MarvinJWendt/testza v0.1.0/go.mod h1:7AxNvlfeHP7Z/hDQ5JtE3OKYT3XFUeLCDE2DQninSqs= @@ -117,8 +115,6 @@ github.com/MarvinJWendt/testza v0.3.0/go.mod h1:eFcL4I0idjtIx8P9C6KkAuLgATNKpX4/ github.com/MarvinJWendt/testza v0.4.2/go.mod h1:mSdhXiKH8sg/gQehJ63bINcCKp7RtYewEjXsvsVUPbE= github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi+zB4= github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= @@ -135,73 +131,73 @@ github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZ github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= -github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= +github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= -github.com/apache/arrow-go/v18 v18.5.0 h1:rmhKjVA+MKVnQIMi/qnM0OxeY4tmHlN3/Pvu+Itmd6s= -github.com/apache/arrow-go/v18 v18.5.0/go.mod h1:F1/wPb3bUy6ZdP4kEPWC7GUZm+yDmxXFERK6uDSkhr8= +github.com/apache/arrow-go/v18 v18.6.0 h1:GX/Jyd3R7mCLiECAwY9FWbbaYblie2WXBSz4Sw8fNpM= +github.com/apache/arrow-go/v18 v18.6.0/go.mod h1:gm3MiPpY82fLYK5VKPB3WoJbsiLVDfT7flD5/vHReKw= github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/apache/arrow/go/v18 v18.0.0-20241007013041-ab95a4d25142 h1:6EtsUpu9/vLtVl6oVpFiZe9GRax7STd2bG55VNwsRdI= github.com/apache/arrow/go/v18 v18.0.0-20241007013041-ab95a4d25142/go.mod h1:GjCnS5QddrJzyqrdYqCUvwlND7SfAw4WH/722M2U2NM= +github.com/apache/iceberg-go v0.6.0 h1:tOVhC5BhBOEgPTowo5AVrPnAgsDo00qEbUPprFcLd4s= +github.com/apache/iceberg-go v0.6.0/go.mod h1:kESfDlyaW/6hK0WW6TzA4EWps+0NMhhrDYE4qajVjlo= github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s= github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= -github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= -github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= -github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.31.2 h1:NOaSZpVGEH2Np/c1toSeW0jooNl+9ALmsUTZ8YvkJR0= -github.com/aws/aws-sdk-go-v2/config v1.31.2/go.mod h1:17ft42Yb2lF6OigqSYiDAiUcX4RIkEMY6XxEMJsrAes= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6 h1:AmmvNEYrru7sYNJnp3pf57lGbiarX4T9qU/6AZ9SucU= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6/go.mod h1:/jdQkh1iVPa01xndfECInp1v1Wnp70v3K4MvtlLGVEc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 h1:lpdMwTzmuDLkgW7086jE94HweHCqG+uOJwHf3LZs7T0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4/go.mod h1:9xzb8/SV62W6gHQGC/8rrvgNXU6ZoYM3sAIJCIrXJxY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 h1:2FFgK3oFA8PTNBjprLFfcmkgg7U9YuSimBvR64RUmiA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0/go.mod h1:xdxj6nC1aU/jAO80RIlIj3fU40MOSqutEA9N2XFct04= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= +github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= github.com/aws/aws-sdk-go-v2/service/athena v1.51.0 h1:Fmh66wriOXgBJDnA/78aur8hH6DrvrWz7ZMzdoS33Yw= github.com/aws/aws-sdk-go-v2/service/athena v1.51.0/go.mod h1:xsG8Y2fMenmHTdukyknTUO1uQhEZ/entaNHvPmD1klE= -github.com/aws/aws-sdk-go-v2/service/glue v1.113.0 h1:ceM8p2ApgB7vAV90rEfCU5wyj/IOtYBE23twMegak7M= -github.com/aws/aws-sdk-go-v2/service/glue v1.113.0/go.mod h1:6FqWCqW0Py6VOvY42NQyf9e7N+sNVnDEiHFklCCCoQc= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 h1:ve9dYBB8CfJGTFqcQ3ZLAAb/KXWgYlgu/2R2TZL2Ko0= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2/go.mod h1:n9bTZFZcBa9hGGqVz3i/a6+NG0zmZgtkB9qVVFDqPA8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 h1:pd9G9HQaM6UZAZh19pYOkpKSQkyQQ9ftnl/LttQOcGI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2/go.mod h1:eknndR9rU8UpE/OmFpqU78V1EcXPKFTTm5l/buZYgvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 h1:iV1Ko4Em/lkJIsoKyGfc0nQySi+v0Udxr6Igq+y9JZc= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0/go.mod h1:bEPcjW7IbolPfK67G1nilqWyoxYMSPrDiIQ3RdIdKgo= -github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= -github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2/service/glue v1.141.0 h1:QeZdzcEB+eB76YvIHqZyxsmxH+f9f2kKDdz48UC4HCM= +github.com/aws/aws-sdk-go-v2/service/glue v1.141.0/go.mod h1:FZCvt95CcJWRkZNRcmMW1uQkpDHmyUSkZfz3awyZgqg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250407191926-092f3e54b837 h1:8eMceEa0ib+nqJuGsyowuZaVBVAr685oK6WrNIit+0g= github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250407191926-092f3e54b837/go.mod h1:9Oj/8PZn3D5Ftp/Z1QWrIEFE0daERMqfJawL9duHRfc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -213,8 +209,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= -github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0= +github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.7.0 h1:VfknkqV4xI+PsaDIsoHueyxVDZrfvMn56jeWUzvzdls= github.com/bits-and-blooms/bloom/v3 v3.7.0/go.mod h1:VKlUSvp0lFIYqxJjzdnSsZEw4iHb1kOL2tfHTgyJBHg= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -264,10 +260,8 @@ github.com/charmbracelet/x/xpty v0.1.2/go.mod h1:XK2Z0id5rtLWcpeNiMYBccNNBrP2IJn github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= @@ -275,15 +269,15 @@ github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEa github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= -github.com/compose-spec/compose-go/v2 v2.6.0 h1:/+oBD2ixSENOeN/TlJqWZmUak0xM8A7J08w/z661Wd4= -github.com/compose-spec/compose-go/v2 v2.6.0/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA= +github.com/compose-spec/compose-go/v2 v2.10.2 h1:USa1NUbDcl/cjb8T9iwnuFsnO79H+2ho2L5SjFKz3uI= +github.com/compose-spec/compose-go/v2 v2.10.2/go.mod h1:ZU6zlcweCZKyiB7BVfCizQT9XmkEIMFE+PRZydVcsZg= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/qqsc= github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= -github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= -github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= -github.com/containerd/containerd/v2 v2.0.5 h1:2vg/TjUXnaohAxiHnthQg8K06L9I4gdYEMcOLiMc8BQ= -github.com/containerd/containerd/v2 v2.0.5/go.mod h1:Qqo0UN43i2fX1FLkrSTCg6zcHNfjN7gEnx3NPRZI+N0= +github.com/containerd/containerd/api v1.10.0 h1:5n0oHYVBwN4VhoX9fFykCV9dF1/BvAXeg2F8W6UYq1o= +github.com/containerd/containerd/api v1.10.0/go.mod h1:NBm1OAk8ZL+LG8R0ceObGxT5hbUYj7CzTmR3xh0DlMM= +github.com/containerd/containerd/v2 v2.2.2 h1:mjVQdtfryzT7lOqs5EYUFZm8ioPVjOpkSoG1GJPxEMY= +github.com/containerd/containerd/v2 v2.2.2/go.mod h1:5Jhevmv6/2J+Iu/A2xXAdUIdI5Ah/hfyO7okJ4AFIdY= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -292,10 +286,10 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= -github.com/containerd/platforms v1.0.0-rc.1 h1:83KIq4yy1erSRgOVHNk1HYdPvzdJ5CnsWaRoJX4C41E= -github.com/containerd/platforms v1.0.0-rc.1/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4= -github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= -github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/platforms v1.0.0-rc.4 h1:M42JrUT4zfZTqtkUwkr0GzmUWbfyO5VO0Q5b3op97T4= +github.com/containerd/platforms v1.0.0-rc.4/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI8L2v0J2ZbYvNsbq1A= +github.com/containerd/ttrpc v1.2.8 h1:xbVu6D4qF2jihdh9rDVOKqUMiFBQk6YctTdo1zk087Y= +github.com/containerd/ttrpc v1.2.8/go.mod h1:wyZW2K79t4Hfcxl+GUvkZqRBzJlqFFvgEeeWXa42tyE= github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40= github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM= @@ -321,8 +315,6 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -331,26 +323,18 @@ github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yA github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= -github.com/docker/buildx v0.22.0 h1:pGTcGZa+kxpYUlM/6ACsp1hXhkEDulz++RNXPdE8Afk= -github.com/docker/buildx v0.22.0/go.mod h1:ThbnUe4kNiStlq6cLXruElyEdSTdPL3k/QerNUmPvHE= -github.com/docker/cli v28.0.4+incompatible h1:pBJSJeNd9QeIWPjRcV91RVJihd/TXB77q1ef64XEu4A= -github.com/docker/cli v28.0.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli-docs-tool v0.9.0 h1:CVwQbE+ZziwlPqrJ7LRyUF6GvCA+6gj7MTCsayaK9t0= -github.com/docker/cli-docs-tool v0.9.0/go.mod h1:ClrwlNW+UioiRyH9GiAOe1o3J/TsY3Tr1ipoypjAUtc= -github.com/docker/compose/v2 v2.35.0 h1:bU23OeFrbGyHYrKijMSEwkOeDg2TLhAGntU2F3hwX1o= -github.com/docker/compose/v2 v2.35.0/go.mod h1:S5ejUILn9KTYC6noX3IxznWu3/sb3FxdZqIYbq4seAk= -github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= -github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/buildx v0.33.0 h1:xuZeuQe/C/2tvLDgiIA6+Ynq3FFWSfsGNWIHM3q1hD8= +github.com/docker/buildx v0.33.0/go.mod h1:7JVma62htERKE5iy5YD1q64PKiAHUzXuhSBd4oq3I74= +github.com/docker/cli v29.4.0+incompatible h1:+IjXULMetlvWJiuSI0Nbor36lcJ5BTcVpUmB21KBoVM= +github.com/docker/cli v29.4.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/compose/v5 v5.1.2 h1:HxtfGZA0DESw/+hvrNJDjM8VKmCond7OkmgVJCHku48= +github.com/docker/compose/v5 v5.1.2/go.mod h1:JJ2H+lRSugOH50/zxCbkBwN8jU91qDtRCp7gEHWZdgo= github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= -github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= -github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0= -github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q= +github.com/docker/docker-credential-helpers v0.9.5 h1:EFNN8DHvaiK8zVqFA2DT6BjXE0GzfLOZ38ggPTKePkY= +github.com/docker/docker-credential-helpers v0.9.5/go.mod h1:v1S+hepowrQXITkEfw6o4+BMbGot02wiKpzWhGUZK6c= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= -github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= @@ -368,8 +352,6 @@ github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHo github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= github.com/elastic/go-elasticsearch/v8 v8.17.0 h1:e9cWksE/Fr7urDRmGPGp47Nsp4/mvNOrU8As1l2HQQ0= github.com/elastic/go-elasticsearch/v8 v8.17.0/go.mod h1:lGMlgKIbYoRvay3xWBeKahAiJOgmFDsjZC39nmO3H64= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -401,8 +383,6 @@ github.com/flarco/bigquery v0.0.9 h1:WfxO6XuuHZTJV+55Bq24FhdHYpmAOzgVk9xOcJpEecY github.com/flarco/bigquery v0.0.9/go.mod h1:IpRSw4quaXxHjFyDSXUo7B6v+XcNF2pSmnNfeqXa/gM= github.com/flarco/databricks-sql-go v0.0.0-20250613120556-51f7c1f3b4ad h1:z5mgsXmNXsgskClg/s6zelILFihJTyK6x7+zX1jUgyU= github.com/flarco/databricks-sql-go v0.0.0-20250613120556-51f7c1f3b4ad/go.mod h1:mnsep6/uctUEgRONJy1pQdb17s/lRtqgcZZZV/WPdkc= -github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2 h1:k01grALOonWx3nBt0LfbUqY39kjaIXaiNP+2LHNuJYY= -github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2/go.mod h1:S7rRApCtYThojNe57sOBhV/ZmH7EMfgEyou4TBEdQHU= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= @@ -412,12 +392,10 @@ github.com/fsnotify/fsevents v0.2.0 h1:BRlvlqjvNTfogHfeBOFvSC9N0Ddy+wzQCQukyoD7o github.com/fsnotify/fsevents v0.2.0/go.mod h1:B3eEk39i4hz8y1zaWS/wPrAP4O6wkIl7HQwKBr1qH/w= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQmYw= github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= github.com/ganigeorgiev/fexpr v0.4.1 h1:hpUgbUEEWIZhSDBtf4M9aUNfQQ0BZkGRaMePy7Gcx5k= @@ -452,14 +430,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY= github.com/go-openapi/errors v0.21.0/go.mod h1:jxNTMUxRCKj65yb/okJGEtahVd7uvWnuWfj53bse4ho= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/strfmt v0.22.0 h1:Ew9PnEYc246TwrEspvBdDHS4BVKXy/AOVsfqGDgAcaI= github.com/go-openapi/strfmt v0.22.0/go.mod h1:HzJ9kokGIju3/K6ap8jL+OlGAbjpSv27135Yr9OivU4= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -468,19 +440,19 @@ github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1 github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.17.1 h1:LI34wktB2xEE3ONG/2Ar54+/HJVBriAGJ55PHls4YuY= github.com/goccy/go-yaml v1.17.1/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= -github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= +github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= +github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -518,10 +490,8 @@ github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/flatbuffers v25.9.23+incompatible h1:rGZKv+wOb6QPzIdkM2KxhBZCDrA0DeN6DNmRDrqIsQU= -github.com/google/flatbuffers v25.9.23+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -543,8 +513,6 @@ github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6 github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= @@ -558,25 +526,24 @@ github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= -github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= -github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= -github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= +github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= +github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas= +github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= -github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= -github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0= +github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E= github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= -github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= -github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= +github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA= +github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= @@ -585,12 +552,10 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= -github.com/hamba/avro/v2 v2.30.0 h1:OaIdh0+dZIJ331FO/+YYBwZZRdGVyyHuRSyHsjZLJoA= -github.com/hamba/avro/v2 v2.30.0/go.mod h1:X6gDhYv6DQVAT56VqOKuW+PLnQrEQqGB9l1nhlMdAdQ= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -605,8 +570,10 @@ github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFO github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= -github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= @@ -614,8 +581,10 @@ github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4Dvx github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/in-toto/in-toto-golang v0.5.0 h1:hb8bgwr0M2hGdDsLjkJ3ZqJ8JFLL/tgYdAxF/XEFBbY= -github.com/in-toto/in-toto-golang v0.5.0/go.mod h1:/Rq0IZHLV7Ku5gielPT4wPHJfH1GdHMCq8+WPxw8/BE= +github.com/in-toto/attestation v1.1.2 h1:MBFn6lsMq6dptQZJBhalXTcWMb/aJy3V+GX3VYj/V1E= +github.com/in-toto/attestation v1.1.2/go.mod h1:gYFddHMZj3DiQ0b62ltNi1Vj5rC879bTmBbrv9CRHpM= +github.com/in-toto/in-toto-golang v0.11.0 h1:nfidMYBFx+E0lnmX5KUnN2Pdm8zdNKal1ayjJuzzRoA= +github.com/in-toto/in-toto-golang v0.11.0/go.mod h1:u3PjTnwFKjp5a1YCcw8SJg0G+tMeKfVoWsWeFMDCMtw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s= @@ -683,8 +652,6 @@ github.com/jmoiron/sqlx v1.3.3 h1:j82X0bf7oQ27XeqxicSZsTU5suPwKElg3oyxNn43iTk= github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -704,8 +671,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -746,8 +713,6 @@ github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maja42/goval v1.4.0 h1:tlX0X+GvjKzWW2Q6qzWwL4Av2KV1bLtzxwzgxiiwEPc= github.com/maja42/goval v1.4.0/go.mod h1:LDMwF8ocOwIsMZdwoyHC/3UpV8ABDwEzalxkVV2z/rI= github.com/mark3labs/mcp-go v0.57.0 h1:jzWKyCzdWnwnZt05cvcQQ+ngiUl2RnixXJa7Kj4qP1E= @@ -772,14 +737,14 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ= +github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= -github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= +github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= @@ -787,8 +752,6 @@ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyex github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microsoft/go-mssqldb v1.9.3 h1:hy4p+LDC8LIGvI3JATnLVmBOLMJbmn5X400mr5j0lPs= github.com/microsoft/go-mssqldb v1.9.3/go.mod h1:GBbW9ASTiDC+mpgWDGKdm3FnFLTUsLYN3iFL90lQ+PA= -github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= -github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= @@ -797,8 +760,8 @@ github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4 github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/moby/buildkit v0.20.1 h1:sT0ZXhhNo5rVbMcYfgttma3TdUHfO5JjFA0UAL8p9fY= -github.com/moby/buildkit v0.20.1/go.mod h1:Rq9nB/fJImdk6QeM0niKtOHJqwKeYMrK847hTTDVuA4= +github.com/moby/buildkit v0.29.0 h1:wxLEFbCOJntEDjSNNN2YWd8zxltZxT5muDQ0LzpbtpU= +github.com/moby/buildkit v0.29.0/go.mod h1:Dmv2FeDe34t75QuzeU87rBoZpAAkcpT5zeu4hXzmASc= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= @@ -809,16 +772,12 @@ github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw= github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= -github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk= github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I= -github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= -github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0= @@ -840,8 +799,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.7.0 h1:r3y12KyNxj/Sb/iOE46ws+3mS1+MZca1wlHQFPsY/JU= github.com/montanaflynn/stats v0.7.0/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ= +github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= @@ -854,8 +813,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nalgeon/redka v0.5.2 h1:CX71v88kYj55EwJ10zq7U2eJdH0xcLAIjvKFvUMoM0o= github.com/nalgeon/redka v0.5.2/go.mod h1:vLxjY3XS9IwBID2YEFWeeMiN4Ar/DtKd4JW62JTAxuU= github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU= @@ -864,8 +821,8 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -897,12 +854,12 @@ github.com/parquet-go/parquet-go v0.23.0/go.mod h1:MnwbUcFHU6uBYMymKAlPPAw9yh3kE github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= -github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec h1:3EiGmeJWoNixU+EwllIn26x6s4njiWRXewdx2zlYa84= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= @@ -924,8 +881,9 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pocketbase/dbx v1.11.0 h1:LpZezioMfT3K4tLrqA55wWFw1EtH1pM4tzSVa7kgszU= github.com/pocketbase/dbx v1.11.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -952,14 +910,10 @@ github.com/pterm/pterm v0.12.31/go.mod h1:32ZAWZVXD7ZfG0s8qqHXePte42kdz8ECtRyEej github.com/pterm/pterm v0.12.33/go.mod h1:x+h2uL+n7CP/rel9+bImHD5lF3nM9vJj80k9ybiiTTE= github.com/pterm/pterm v0.12.36/go.mod h1:NjiL09hFhT/vWjQHSj1athJpx6H8cjpHXNAK5bUw8T8= github.com/pterm/pterm v0.12.40/go.mod h1:ffwPLwlbXxP+rxT0GsgDTzS3y3rmpAO1NMjUkGTYf8s= -github.com/pterm/pterm v0.12.82 h1:+D9wYhCaeaK0FIQoZtqbNQuNpe2lB2tajKKsTd5paVQ= -github.com/pterm/pterm v0.12.82/go.mod h1:TyuyrPjnxfwP+ccJdBTeWHtd/e0ybQHkOS/TakajZCw= +github.com/pterm/pterm v0.12.83 h1:ie+YmGmA727VuhxBlyGr74Ks+7McV6kT99IB8EU80aA= +github.com/pterm/pterm v0.12.83/go.mod h1:xlgc6bFWyJIMtmLJvGim+L7jhSReilOlOnodeIYe4Tk= github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg= github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= -github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2O7ZhWYVGYSR2iVcjzQuPV+o= -github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc/go.mod h1:S8xSOnV3CgpNrWd0GQ/OoQfMtlg2uPRSuTzcSGrzwK8= -github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI= -github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= @@ -984,8 +938,8 @@ github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEV github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/scylladb/gocql v1.18.0 h1:bmaMHNOUJyu0GHnTsVv0RRUAquQoTtYjCL/9jTxrg9Q= github.com/scylladb/gocql v1.18.0/go.mod h1:PZU+XJQ3fDymccIlTacmTdO+aTGGDSgXF1hw+yULUMk= -github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE= -github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs= +github.com/secure-systems-lab/go-securesystemslib v0.10.0 h1:l+H5ErcW0PAehBNrBxoGv1jjNpGYdZ9RcheFkB2WI14= +github.com/secure-systems-lab/go-securesystemslib v0.10.0/go.mod h1:MRKONWmRoFzPNQ9USRF9i1mc7MvAVvF1LlW8X5VWDvk= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOVmy8= @@ -996,14 +950,12 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU= -github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= -github.com/shirou/gopsutil/v4 v4.25.1 h1:QSWkTc+fu9LTAWfkZwZ6j8MSUk4A2LV7rbH0ZqmLjXs= -github.com/shirou/gopsutil/v4 v4.25.1/go.mod h1:RoUCUpndaJFtT+2zsZzzmhvbfGoDCJ7nFXKJf8GqJbI= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= @@ -1034,10 +986,14 @@ github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1l github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= +github.com/sigstore/sigstore v1.10.4 h1:ytOmxMgLdcUed3w1SbbZOgcxqwMG61lh1TmZLN+WeZE= +github.com/sigstore/sigstore v1.10.4/go.mod h1:tDiyrdOref3q6qJxm2G+JHghqfmvifB7hw+EReAfnbI= +github.com/sigstore/sigstore-go v1.1.4 h1:wTTsgCHOfqiEzVyBYA6mDczGtBkN7cM8mPpjJj5QvMg= +github.com/sigstore/sigstore-go v1.1.4/go.mod h1:2U/mQOT9cjjxrtIUeKDVhL+sHBKsnWddn8URlswdBsg= github.com/sijms/go-ora/v2 v2.8.24 h1:TODRWjWGwJ1VlBOhbTLat+diTYe8HXq2soJeB+HMjnw= github.com/sijms/go-ora/v2 v2.8.24/go.mod h1:QgFInVi3ZWyqAiJwzBQA+nbKYKH77tdp1PYoCqhR2dU= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260806214312-6da3e7189c98 h1:HT0E/6EiVUe8wew4uIgYEpUYPHcswmGkxpbbqFges3g= @@ -1054,18 +1010,19 @@ github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:Udh github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1079,19 +1036,17 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/substrait-io/substrait v0.75.0 h1:26l61irh5kQBLeDUcgDLnm+p4s2dJloHU6PPK0RAiB0= -github.com/substrait-io/substrait v0.75.0/go.mod h1:MPFNw6sToJgpD5Z2rj0rQrdP/Oq8HG7Z2t3CAEHtkHw= -github.com/substrait-io/substrait-go/v7 v7.2.0 h1:49QFXeydO8OZyedzWOwHG0fG4BcXXlOwsbMalPwGWpA= -github.com/substrait-io/substrait-go/v7 v7.2.0/go.mod h1:4GZ6c+UaojOGEG4ynyHrDFFmWGCVtbKdfzp6LXWdHmc= -github.com/substrait-io/substrait-protobuf/go v0.75.0 h1:6SjuEESDB8oOhdQMPMGb4uy0tyZOYt1EBwS/Wcr61fs= -github.com/substrait-io/substrait-protobuf/go v0.75.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ= +github.com/substrait-io/substrait v0.87.0 h1:40rP4LejyK6SNQlWz7NX6kQELf8cmScWMBGruWhN4io= +github.com/substrait-io/substrait v0.87.0/go.mod h1:MPFNw6sToJgpD5Z2rj0rQrdP/Oq8HG7Z2t3CAEHtkHw= +github.com/substrait-io/substrait-go/v8 v8.1.0 h1:58nSM6Qsctke4DIHU2y4aR8FN5YC7syo1mk4zFKAo4U= +github.com/substrait-io/substrait-go/v8 v8.1.0/go.mod h1:6GLz9k21udB64g4lLKq8632TKfQCRAVfhuU3NSXtZWY= +github.com/substrait-io/substrait-protobuf/go v0.85.0 h1:zk6MtNWLtDSl8a7qCZRFH0+EIIXVrrd/hsgYK/SQTgM= +github.com/substrait-io/substrait-protobuf/go v0.85.0/go.mod h1:hn+Szm1NmZZc91FwWK9EXD/lmuGBSRTJ5IvHhlG1YnQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/testcontainers/testcontainers-go v0.37.0 h1:L2Qc0vkTw2EHWQ08djon0D2uw7Z/PtHS/QzZZ5Ra/hg= -github.com/testcontainers/testcontainers-go v0.37.0/go.mod h1:QPzbxZhQ6Bclip9igjLFj6z0hs01bU8lrl2dHQmgFGM= -github.com/testcontainers/testcontainers-go/modules/compose v0.37.0 h1:AE6XYnyUMkiyuo8GZ3B36d0i4L/HMSjaQ6QtAffkD4k= -github.com/testcontainers/testcontainers-go/modules/compose v0.37.0/go.mod h1:fgzGeGw5iVyzS6qWOAYDbvv3iWp/wCtqWNSH4Aev8hs= -github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c= -github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw= +github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= +github.com/testcontainers/testcontainers-go/modules/compose v0.42.0 h1:+t1ZN31TD36cwxmeLqGwe7wIdvblBm0Z+vlj4SX8Mv0= +github.com/testcontainers/testcontainers-go/modules/compose v0.42.0/go.mod h1:CfMpouDHqNTCHC8CijEURU2ZotTV3QhH6pXd48s6ofk= github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/jsonc v0.3.3 h1:RVQqL3xFfDkKKXIDsrBiVQiEpBtxoKbmMXONb2H/y2w= @@ -1109,45 +1064,49 @@ github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375 h1:QB54BJwA6x8 github.com/tilt-dev/fsnotify v1.4.8-0.20220602155310-fff9c274a375/go.mod h1:xRroudyp5iVtxKqZCrA6n2TLFRBf8bmnjr1UD4x+z7g= github.com/timeplus-io/proton-go-driver/v2 v2.0.19 h1:hpoS4GLCUs80shiutsAI6VDcFewzlf1bImlRQn3gZf0= github.com/timeplus-io/proton-go-driver/v2 v2.0.19/go.mod h1:rUs4zvXvKsmuyFpzdJnnid6p8IvRJTa/n/jNQ2B6Dfw= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= -github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205 h1:eUk79E1w8yMtXeHSzjKorxuC8qJOnyXQnLaJehxpJaI= -github.com/tonistiigi/dchapes-mode v0.0.0-20241001053921-ca0759fec205/go.mod h1:3Iuxbr0P7D3zUzBMAZB+ois3h/et0shEz0qApgHYGpY= -github.com/tonistiigi/fsutil v0.0.0-20250113203817-b14e27f4135a h1:EfGw4G0x/8qXWgtcZ6KVaPS+wpWOQMaypczzP8ojkMY= -github.com/tonistiigi/fsutil v0.0.0-20250113203817-b14e27f4135a/go.mod h1:Dl/9oEjK7IqnjAm21Okx/XIxUCFJzvh+XdVHUlBwXTw= -github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 h1:7I5c2Ig/5FgqkYOh/N87NzoyI9U15qUPXhDD8uCupv8= -github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= +github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323 h1:r0p7fK56l8WPequOaR3i9LBqfPtEdXIQbUTzT55iqT4= +github.com/tonistiigi/dchapes-mode v0.0.0-20250318174251-73d941a28323/go.mod h1:3Iuxbr0P7D3zUzBMAZB+ois3h/et0shEz0qApgHYGpY= +github.com/tonistiigi/fsutil v0.0.0-20251211185533-a2aa163d723f h1:Z4NEQ86qFl1mHuCu9gwcE+EYCwDKfXAYXZbdIXyxmEA= +github.com/tonistiigi/fsutil v0.0.0-20251211185533-a2aa163d723f/go.mod h1:BKdcez7BiVtBvIcef90ZPc6ebqIWr4JWD7+EvLm6J98= +github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 h1:2f304B10LaZdB8kkVEaoXvAMVan2tl9AiK4G0odjQtE= +github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0= github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab h1:H6aJ0yKQ0gF49Qb2z5hI1UHxSQt4JMyxebFR15KnApw= github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= github.com/trinodb/trino-go-client v0.328.0 h1:X6hrGGysA3nvyVcz8kJbBS98srLNTNsnNYwRkMC1atA= github.com/trinodb/trino-go-client v0.328.0/go.mod h1:e/nck9W6hy+9bbyZEpXKFlNsufn3lQGpUgDL1d5f1FI= +github.com/twmb/avro v1.7.2 h1:cmrEBRSbELRqsg/dRkQvVWuOaR2EfGifHIt/2iJ9lfI= +github.com/twmb/avro v1.7.2/go.mod h1:X0fT1dY2xcbV4YuCE4mYro+qljHl4kUF5uA/2z1rgSk= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/twpayne/go-geom v1.6.1 h1:iLE+Opv0Ihm/ABIcvQFGIiFBXd76oBIar9drAwHFhR4= github.com/twpayne/go-geom v1.6.1/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028= -github.com/uptrace/bun v1.2.12 h1:XRvGko5tIZ5A3T0GkLsbFFnSR2Yd6knuML8eJVGTLM0= -github.com/uptrace/bun v1.2.12/go.mod h1:ZS4nPaEv2Du3OFqAD/irk3WVP6xTB3/9TWqjJbgKYBU= -github.com/uptrace/bun/dialect/mssqldialect v1.2.12 h1:dkApi4IKagaGgo7udjIQkULoMr4aRj0PCCUqr7rGiig= -github.com/uptrace/bun/dialect/mssqldialect v1.2.12/go.mod h1:jHDQ8nO34nZCCnn/hkpVnfgaU5QB0e5ozPI0wn8MAuo= -github.com/uptrace/bun/dialect/mysqldialect v1.2.12 h1:3+pGxF70Yuve0IT9mqz21UiZXR8OAhZIKFSY7tUYqVc= -github.com/uptrace/bun/dialect/mysqldialect v1.2.12/go.mod h1:/X32gQ392MafIB/2+Ig/WVz63lV8zGr41Oalr+c8CDs= -github.com/uptrace/bun/dialect/oracledialect v1.2.12 h1:Ne7PSnz3f2iFt6hV17avdgWxOgdtC4zTLtCaIggxTv8= -github.com/uptrace/bun/dialect/oracledialect v1.2.12/go.mod h1:MyqGYD/oBi2lPcTF072DA/FHStUFleX+W5Mjlu3HuYY= -github.com/uptrace/bun/dialect/pgdialect v1.2.12 h1:UxbxJXqQPeSBDnPMWAi9kDOeWX9HooFf1xkCZR1gsRA= -github.com/uptrace/bun/dialect/pgdialect v1.2.12/go.mod h1:Q7xQWbFs2Msg87BxkJtKkBSWSLmmXN+gB5mEi7212PI= -github.com/uptrace/bun/dialect/sqlitedialect v1.2.12 h1:AYyIK20jLXuhaKhME0vstXXIoNVNoG0OXBsbZOrBmSw= -github.com/uptrace/bun/dialect/sqlitedialect v1.2.12/go.mod h1:vAhs4+/Aiq4KN/z6xA+HQKJCufIHFRmHeamF1oGZz9E= -github.com/uptrace/bun/driver/sqliteshim v1.2.12 h1:KClnHj9+Z6NvCRp22doGKAp1T2ZBghoQpH/Q+DZFdls= -github.com/uptrace/bun/driver/sqliteshim v1.2.12/go.mod h1:rd3iBIaNZ7VpPKLGoIix72MJquhypu2OO62lfUx+UWo= -github.com/uptrace/bun/extra/bundebug v1.2.12 h1:hiPesTgVZAGfIgC2hi1SYuP9EztzhupL4uC5yd2d3Hw= -github.com/uptrace/bun/extra/bundebug v1.2.12/go.mod h1:QOncBc89nWhNWRwMibGyFDcIDhbFD3OzwDyiJZxgSlY= +github.com/uptrace/bun v1.2.18 h1:3HnRcMfS6OBPMG1eSOzlbFJ/X/AyMEJb7rMxE6VQvDU= +github.com/uptrace/bun v1.2.18/go.mod h1:wNltaKJk4JtOt4SG5I5zmA7v0/Mzjh1+/S906Rayd3Y= +github.com/uptrace/bun/dialect/mssqldialect v1.2.18 h1:nYzHoyJKJlIyl5i95Exi8ZTK8ooKWG+o3z3f404d/yQ= +github.com/uptrace/bun/dialect/mssqldialect v1.2.18/go.mod h1:Su45Je7z66sfeZ3d1ZsnOQEK8xfzGgaMzBvtoE8yFhk= +github.com/uptrace/bun/dialect/mysqldialect v1.2.18 h1:w+3iuWa4cVmsXXt8w28A0+Ikve77AU0tiBWG6UvGvM8= +github.com/uptrace/bun/dialect/mysqldialect v1.2.18/go.mod h1:FhJEK620SM9HJ9fx0/IHT7k1cpn2+6MmtKvNptWezPY= +github.com/uptrace/bun/dialect/oracledialect v1.2.18 h1:L51zA1S02Um3ukYwrqyqlaYv9NAw8M5EmcUc7cPEHpQ= +github.com/uptrace/bun/dialect/oracledialect v1.2.18/go.mod h1:+wFOA+RxcjTWa1S2OASNj7U0cC2Mp4vQClY/OkThoPU= +github.com/uptrace/bun/dialect/pgdialect v1.2.18 h1:IZ6nM2+OYrL8lkEAy7UkSEZvoa3vluTAUlZfPtlRB2k= +github.com/uptrace/bun/dialect/pgdialect v1.2.18/go.mod h1:Tqdf4QP1okrGYpXfodXvCOK6Ob1OOTwSaoAzCgBB3IU= +github.com/uptrace/bun/dialect/sqlitedialect v1.2.18 h1:Z33SY/U++XK9uGWqS4h8OZVxfCXguIG+sU9cYq2PGFQ= +github.com/uptrace/bun/dialect/sqlitedialect v1.2.18/go.mod h1:1MVOS/Ncy4FZbkJcgUFH6OqYoQinYNjkEwsmNQEXz2A= +github.com/uptrace/bun/driver/sqliteshim v1.2.18 h1:fDCXp4L46A23OuUikDbL14SRmm3y+7XO4fkFe1bs2A4= +github.com/uptrace/bun/driver/sqliteshim v1.2.18/go.mod h1:MqvqMCAAKNn6M0HF9YK/Z6xrnCP6sih5OZ37AxdAlHw= +github.com/uptrace/bun/extra/bundebug v1.2.18 h1:5cgkqdvhpSHIEONazSytm4RWYFneNtcznaWLt6r8m4M= +github.com/uptrace/bun/extra/bundebug v1.2.18/go.mod h1:M+U9YJVJcmk0RrszCb2Q1oskJiJ0LuC44FxDhZLP1ws= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= @@ -1161,8 +1120,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= @@ -1205,12 +1162,10 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zclconf/go-cty v1.16.0 h1:xPKEhst+BW5D0wxebMZkxgapvOE/dw7bFTlgSc9nD6w= -github.com/zclconf/go-cty v1.16.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= @@ -1221,28 +1176,28 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE= go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0 h1:4BZHA+B1wXEQoGNHxW8mURaLhcdGwvRnmhGbm+odRbc= -go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.63.0 h1:2pn7OzMewmYRiNtv1doZnLo3gONcnMHlFnmOR8Vgt+8= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.63.0/go.mod h1:rjbQTDEPQymPE0YnRQp9/NuPwwtL0sesz/fnqRW/v84= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0 h1:FZ6ei8GFW7kyPYdxJaV2rgI6M+4tvZzhYsQ2wgyVC08= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0/go.mod h1:MdEu/mC6j3D+tTEfvI15b5Ci2Fn7NneJ71YMoiS3tpI= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0/go.mod h1:hg1zaDMpyZJuUzjFxFsRYBoccE86tM9Uf4IqNMUxvrY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 h1:in9O8ESIOlwJAEGTkkf34DesGRAc/Pn8qJ7k3r/42LM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0/go.mod h1:Rp0EXBm5tfnv0WL+ARyO/PHBEaEAT8UUHQ6AGJcSq6c= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0/go.mod h1:HBy4BjzgVE8139ieRI75oXm3EcDN+6GhD88JT1Kjvxg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= @@ -1257,8 +1212,8 @@ go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfC go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= -go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -1269,20 +1224,22 @@ go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwE go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= -go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U= +go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -gocloud.dev v0.41.0 h1:qBKd9jZkBKEghYbP/uThpomhedK5s2Gy6Lz7h/zYYrM= -gocloud.dev v0.41.0/go.mod h1:IetpBcWLUwroOOxKr90lhsZ8vWxeSkuszBnW62sbcf0= +gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY= +gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1294,7 +1251,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= @@ -1316,7 +1272,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= @@ -1347,7 +1302,6 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= @@ -1412,7 +1366,6 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -1430,7 +1383,6 @@ golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= @@ -1454,8 +1406,8 @@ golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1473,7 +1425,6 @@ golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= @@ -1483,13 +1434,13 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= -google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc= -google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww= +google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E= +google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1502,12 +1453,12 @@ google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= -google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= +google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1516,8 +1467,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1535,8 +1486,6 @@ gopkg.in/VividCortex/ewma.v1 v1.1.1 h1:tWHEKkKq802K/JT9RiqGCBU5fW3raAPnJGTE9ostZ gopkg.in/VividCortex/ewma.v1 v1.1.1/go.mod h1:TekXuFipeiHWiAlO1+wSS23vTcyFau5u3rxXUSXj710= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= -gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= -gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1590,30 +1539,20 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0= -k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk= -k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw= -k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc= -k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= -modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= -modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= -modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= -modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U= +modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8= +modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU= +modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= -modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= +modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= +modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -1622,8 +1561,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74= -modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8= +modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U= +modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= @@ -1632,15 +1571,11 @@ pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= syreclabs.com/go/faker v1.2.2 h1:D6ImaMO9Ht3RYmGbrCHiXJIDmqdkPMqK21BR3Es7sYY= syreclabs.com/go/faker v1.2.2/go.mod h1:NAXInmkPsC2xuO5MKZFe80PUXX5LU8cFdJIHGs+nSBE= -tags.cncf.io/container-device-interface v1.0.1 h1:KqQDr4vIlxwfYh0Ed/uJGVgX+CHAkahrgabg6Q8GYxc= -tags.cncf.io/container-device-interface v1.0.1/go.mod h1:JojJIOeW3hNbcnOH2q0NrWNha/JuHoDZcmYxAZwb2i0= +tags.cncf.io/container-device-interface v1.1.0 h1:RnxNhxF1JOu6CJUVpetTYvrXHdxw9j9jFYgZpI+anSY= +tags.cncf.io/container-device-interface v1.1.0/go.mod h1:76Oj0Yqp9FwTx/pySDc8Bxjpg+VqXfDb50cKAXVJ34Q= diff --git a/tests/replications/r.126.iceberg_incremental_merge.yaml b/tests/replications/r.126.iceberg_incremental_merge.yaml new file mode 100644 index 000000000..961925ae0 --- /dev/null +++ b/tests/replications/r.126.iceberg_incremental_merge.yaml @@ -0,0 +1,88 @@ +source: postgres +target: '{target}' + +hooks: + start: + - type: query + connection: '{source.name}' + query: | + DROP TABLE IF EXISTS public.iceberg_inc_merge_src; + CREATE TABLE public.iceberg_inc_merge_src ( + id int primary key, + val text, + updated_at timestamp + ); + INSERT INTO public.iceberg_inc_merge_src (id, val, updated_at) VALUES + (1, 'a', '2020-01-01 00:00:00'), + (2, 'b', '2020-01-01 00:00:00'), + (3, 'c', '2020-01-01 00:00:00'); + + end: + - type: check + check: execution.status.error == 0 + on_failure: break + + - type: check + id: target_check + check: lower(target.name) != "iceberg_gcp" && lower(target.name) != "iceberg_sql" && lower(target.name) != "iceberg_glue" + failure_message: skipping duckdb verify for {target.name} + on_failure: break + + - type: query + connection: '{target}' + query: | + select count(*) as cnt, count(distinct id) as did from iceberg_catalog.public.iceberg_inc_merge + into: counts + + - type: log + message: 'iceberg merge counts => { store.counts[0] }' + + - type: check + check: int_parse(store.counts[0].cnt) == 4 + message: 'expected 4 rows (3 seed + 1 insert, 2 updates in place), got {store.counts[0].cnt}' + + - type: check + check: int_parse(store.counts[0].did) == 4 + message: 'duplicate primary keys in Iceberg merge result: distinct={store.counts[0].did} count={store.counts[0].cnt}' + + - type: query + connection: '{target}' + query: select id, val from iceberg_catalog.public.iceberg_inc_merge order by id + into: rows + + - type: check + check: store.rows[0].val == "a2" + message: "id=1 should be updated to a2, got '{store.rows[0].val}'" + + - type: check + check: store.rows[1].val == "b2" + message: "id=2 should be updated to b2, got '{store.rows[1].val}'" + + - type: check + check: store.rows[3].val == "d" + message: "id=4 should be inserted, got '{store.rows[3].val}'" + +streams: + seed: + sql: select * from public.iceberg_inc_merge_src + object: public.iceberg_inc_merge + mode: full-refresh + primary_key: [id] + + merge: + sql: select * from public.iceberg_inc_merge_src where {incremental_where_cond} + object: public.iceberg_inc_merge + mode: incremental + primary_key: [id] + update_key: updated_at + hooks: + pre: + - type: query + connection: '{source.name}' + query: | + UPDATE public.iceberg_inc_merge_src SET val = 'a2', updated_at = now() WHERE id = 1; + UPDATE public.iceberg_inc_merge_src SET val = 'b2', updated_at = now() WHERE id = 2; + INSERT INTO public.iceberg_inc_merge_src (id, val, updated_at) VALUES (4, 'd', now()); + +env: + target: ${TARGET} diff --git a/tests/replications/r.127.iceberg_cdc.yaml b/tests/replications/r.127.iceberg_cdc.yaml new file mode 100644 index 000000000..5a8128f17 --- /dev/null +++ b/tests/replications/r.127.iceberg_cdc.yaml @@ -0,0 +1,106 @@ +# Synthetic CDC against Iceberg: events in _sling_synced_op / _sling_cdc_seq +# applied via MergeStream (change_capture), not a live binlog. +source: postgres +target: '{target}' + +hooks: + start: + - type: query + connection: '{source.name}' + query: | + DROP TABLE IF EXISTS public.iceberg_cdc_merge_src; + CREATE TABLE public.iceberg_cdc_merge_src ( + id int primary key, + name text, + amount int, + updated_at timestamp + ); + INSERT INTO public.iceberg_cdc_merge_src (id, name, amount, updated_at) VALUES + (1, 'Alice', 100, '2020-01-01 00:00:00'), + (2, 'Bob', 200, '2020-01-01 00:00:00'), + (3, 'Charlie', 300, '2020-01-01 00:00:00'); + + end: + - type: check + check: execution.status.error == 0 + on_failure: break + + - type: check + id: target_check + check: lower(target.name) != "iceberg_gcp" && lower(target.name) != "iceberg_sql" && lower(target.name) != "iceberg_glue" + failure_message: skipping duckdb verify for {target.name} + on_failure: break + + - type: query + connection: '{target}' + query: | + select count(*) as cnt, count(distinct id) as did from iceberg_catalog.public.iceberg_cdc_merge + into: counts + + - type: log + message: 'iceberg cdc counts => { store.counts[0] }' + + - type: check + check: int_parse(store.counts[0].cnt) == 3 + message: 'expected 3 rows (1 updated, 1 deleted, 1 inserted, 1 unchanged), got {store.counts[0].cnt}' + + - type: check + check: int_parse(store.counts[0].did) == 3 + message: 'duplicate PKs after CDC merge' + + - type: query + connection: '{target}' + query: select name, amount from iceberg_catalog.public.iceberg_cdc_merge where id = 1 + into: row_1 + + - type: check + check: store.row_1[0].name == "Alice Final" + message: "id=1 should be Alice Final, got '{store.row_1[0].name}'" + + - type: query + connection: '{target}' + query: select count(*) as cnt from iceberg_catalog.public.iceberg_cdc_merge where id = 2 + into: row_2 + + - type: check + check: int_parse(store.row_2[0].cnt) == 0 + message: 'id=2 should be hard-deleted' + + - type: query + connection: '{target}' + query: select name from iceberg_catalog.public.iceberg_cdc_merge where id = 4 + into: row_4 + + - type: check + check: store.row_4[0].name == "Diana" + message: "id=4 should be inserted as Diana" + +streams: + seed: + sql: | + select id, name, amount, updated_at, + 'S' as _sling_synced_op, + 0::bigint as _sling_cdc_seq + from public.iceberg_cdc_merge_src + object: public.iceberg_cdc_merge + mode: full-refresh + primary_key: [id] + + cdc_events: + sql: | + SELECT * FROM ( + SELECT 1 as id, 'Alice Updated' as name, 150 as amount, now() as updated_at, 'U' as _sling_synced_op, 1::bigint as _sling_cdc_seq + UNION ALL SELECT 2, 'Bob', 200, now(), 'D', 2 + UNION ALL SELECT 4, 'Diana', 400, now(), 'I', 3 + UNION ALL SELECT 1, 'Alice Final', 175, now(), 'U', 4 + ) e + where {incremental_where_cond} + object: public.iceberg_cdc_merge + mode: incremental + primary_key: [id] + update_key: updated_at + target_options: + merge_strategy: change_capture + +env: + target: ${TARGET} diff --git a/tests/suite.cli.yaml b/tests/suite.cli.yaml index b9bcbdaa0..662569c73 100644 --- a/tests/suite.cli.yaml +++ b/tests/suite.cli.yaml @@ -2976,6 +2976,66 @@ - 'OK bug 1 fixed: incremental checkpoint matched 445 documents' - 'SUCCESS: both issue 802 MongoDB date filter bugs are fixed' +- id: 604 + name: Run sling iceberg_r2 incremental merge + run: | + sling run -d -r tests/replications/r.126.iceberg_incremental_merge.yaml + env: + TARGET: iceberg_r2 + output_contains: + - iceberg merge engine=go + - committed iceberg snapshot + - iceberg merge counts + - execution succeeded + +- id: 605 + name: Run sling iceberg_s3 incremental merge + run: | + sling run -d -r tests/replications/r.126.iceberg_incremental_merge.yaml + env: + TARGET: iceberg_s3 + after: [604] + output_contains: + - committed iceberg snapshot + - execution succeeded + +- id: 606 + name: Run sling iceberg_sql incremental merge + run: | + sling run -d -r tests/replications/r.126.iceberg_incremental_merge.yaml + env: + TARGET: iceberg_sql + after: [605] + output_contains: + - iceberg merge engine=go + - committed iceberg snapshot + - execution succeeded + +- id: 607 + name: Run sling iceberg_glue incremental merge + run: | + sling run -d -r tests/replications/r.126.iceberg_incremental_merge.yaml + env: + TARGET: iceberg_glue + after: [606] + output_contains: + - iceberg merge engine=go + - committed iceberg snapshot + - execution succeeded + +- id: 608 + name: Run sling iceberg_r2 synthetic CDC merge + run: | + sling run -d -r tests/replications/r.127.iceberg_cdc.yaml + env: + TARGET: iceberg_r2 + after: [604] + output_contains: + - iceberg merge engine=go + - committed iceberg snapshot + - iceberg cdc counts + - execution succeeded + # Nested CLI suites. Loader in sling_cli_test.go merges these in. - suite: suite.cli.assist.yaml - suite: suite.cli.build.yaml From a10a9464f92579d2d75103e80e127edcee16c1d8 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 20 Sep 2026 08:14:44 -0300 Subject: [PATCH 19/56] build: promote aws-sdk-go-v2 glue service to direct dependency --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 75995f566..0278c5632 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 github.com/aws/aws-sdk-go-v2/service/athena v1.51.0 + github.com/aws/aws-sdk-go-v2/service/glue v1.141.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 github.com/aws/smithy-go v1.25.1 @@ -161,7 +162,6 @@ require ( github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect - github.com/aws/aws-sdk-go-v2/service/glue v1.141.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect From 7be5b7a383474836188c38e1fdd3c2e2e96c36ac Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 20 Sep 2026 09:43:35 -0300 Subject: [PATCH 20/56] feat: add support for Cloudflare R2 in icebergStorageKind function --- core/dbio/database/database_iceberg.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/dbio/database/database_iceberg.go b/core/dbio/database/database_iceberg.go index 148ff0805..d857f7d94 100644 --- a/core/dbio/database/database_iceberg.go +++ b/core/dbio/database/database_iceberg.go @@ -1957,6 +1957,13 @@ func (conn *IcebergConn) icebergStorageKind() icebergStorageKind { return "" } + // Cloudflare R2 vends S3-compatible credentials through the REST catalog, so + // the connection carries no s3_* props and the warehouse is an opaque id. + // Detect it from the catalog endpoint instead, before the generic checks. + if strings.Contains(strings.ToLower(conn.GetProp("rest_uri")), "cloudflarestorage.com") { + return icebergStorageS3 + } + switch { case strings.HasPrefix(w, "gs://"), strings.HasPrefix(w, "gcs://"), get("gcs_access_key_id") != "": return icebergStorageGCS From 7001308df560611fea6fcde74b941bee70d162c2 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 20 Sep 2026 10:00:45 -0300 Subject: [PATCH 21/56] refactor(tests): simplify test cases for DuckDb and Iceberg databases --- cmd/sling/sling_test.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/cmd/sling/sling_test.go b/cmd/sling/sling_test.go index 5c8f5d58f..c2f8d2edc 100755 --- a/cmd/sling/sling_test.go +++ b/cmd/sling/sling_test.go @@ -1174,11 +1174,10 @@ func TestSuiteDatabaseDuckDb(t *testing.T) { testSuite(t, dbio.TypeDbMotherDuck) // DUCKLAKE - tests := "1-17,19+" // soft-delete is not supported - testSuite(t, dbio.TypeDbDuckLake, tests) - // testSuite(t, dbio.Type("ducklake_az"), tests) - testSuite(t, dbio.Type("ducklake_r2"), tests) - testSuite(t, dbio.Type("ducklake_s3"), tests) + testSuite(t, dbio.TypeDbDuckLake) + testSuite(t, dbio.Type("ducklake_az")) + testSuite(t, dbio.Type("ducklake_r2")) + testSuite(t, dbio.Type("ducklake_s3")) } func TestSuiteDatabaseExasol(t *testing.T) { @@ -1205,11 +1204,16 @@ func TestSuiteDatabaseAthena(t *testing.T) { func TestSuiteDatabaseIceberg(t *testing.T) { t.Parallel() // 5 = truncate (not supported). 9-12 = incremental with views / extra tables. - // 26-29 = merge strategies (insert / update / update_insert / delete_insert). - testSuite(t, dbio.TypeDbIceberg, "1-4,6-8,26-29") - testSuite(t, dbio.Type("iceberg_glue"), "1-4,6-8,26-29") - testSuite(t, dbio.Type("iceberg_s3"), "1-4,6-8,26-29") - testSuite(t, dbio.Type("iceberg_sql"), "1-4,6-8,26-29") + // 18 = delete_missing, needs a SQL UPDATE; IcebergConn.NewTransaction is nil. + // 26-29 = merge strategies (insert / update / update_insert / delete_insert), + // which now route through MergeStream (equality deletes + row delta). + // Incremental-merge and change-capture coverage lives in the CLI suite + // (suite.cli.yaml 604-608, r.126 / r.127), not in the numbered db template. + tests := "1-4,6-8,26-29" + testSuite(t, dbio.TypeDbIceberg, tests) + testSuite(t, dbio.Type("iceberg_glue"), tests) + testSuite(t, dbio.Type("iceberg_s3"), tests) + testSuite(t, dbio.Type("iceberg_sql"), tests) } func TestSuiteDatabaseDB2(t *testing.T) { From b31b93c8d2b2a11bd5a33b69a7d09d47e9f5306e Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Sun, 20 Sep 2026 22:28:56 -0300 Subject: [PATCH 22/56] Add --json output to sling build run `sling build run` previously had no machine-readable output, so callers (CI, editors, wrappers) had to parse the human summary to learn per-node results. The pipeline `type: build` step already returns node results in its state, so this reuses that shape instead of inventing a second one. - Add buildOutputFlags to the `run` subcommand and PrintRunJSON, which prints RunResultsPayload, or SubProjectsPayload when the directory holds independent sub-projects (node names are not unique across them). - Share RunResultsPayload between the CLI and hook_runner so `sling build run --json` and hooks report one contract. - Emit the JSON payload before returning on failure: a failed run still hands back per-node errors alongside the non-zero exit code. - Collect failed sub-builds and aggregate their row/byte counts, sorting by project dir since goroutines finish in arbitrary order. - Route error diagnostics and the "No models selected"/summary blank line to stderr when --json is set, keeping stdout parseable. - Add fixture project plus tests 613-616 covering success, per-model errors, `--help`, and `-R` sub-project grouping. --- cmd/sling/sling_build.go | 22 +++--- core/sling/build/build.go | 37 +++++++++- core/sling/build/executor.go | 15 ++-- core/sling/build/hook_runner.go | 69 ++++++++++++++----- .../json_run_project/marts/fct_orders.sql | 1 + tests/build/json_run_project/sling_build.yml | 5 ++ .../json_run_project/staging/stg_bad.sql | 3 + .../json_run_project/staging/stg_orders.sql | 3 + tests/suite.cli.build.yaml | 59 +++++++++++++++- 9 files changed, 180 insertions(+), 34 deletions(-) create mode 100644 tests/build/json_run_project/marts/fct_orders.sql create mode 100644 tests/build/json_run_project/sling_build.yml create mode 100644 tests/build/json_run_project/staging/stg_bad.sql create mode 100644 tests/build/json_run_project/staging/stg_orders.sql diff --git a/cmd/sling/sling_build.go b/cmd/sling/sling_build.go index ed1e029ce..5f2ef80f1 100644 --- a/cmd/sling/sling_build.go +++ b/cmd/sling/sling_build.go @@ -152,7 +152,7 @@ var cliBuild = &g.CliSC{ Name: "run", Description: "Materialize models, then run each model's declarative tests", PosFlags: []g.Flag{buildPathFlag}, - Flags: concatFlags(buildCommonFlags, buildRunFlags), + Flags: concatFlags(buildCommonFlags, buildRunFlags, buildOutputFlags), }, { Name: "list", @@ -335,15 +335,21 @@ func processBuild(c *g.CliSC) (ok bool, err error) { return ok, nil } - // Execute the build - if err := b.Execute(); err != nil { - if opts.Test && opts.JSON { - b.PrintTestJSON() - } - return ok, g.Error(err, "build execution failed") + // Execute the build. Machine-readable output is emitted before returning, + // so a failed run still hands the caller the per-node results along with + // the non-zero exit code. + runErr := b.Execute() + switch { + case opts.Test && opts.JSON: + b.PrintTestJSON() + case opts.JSON: + b.PrintRunJSON(projectPath) + } + + if runErr != nil { + return ok, g.Error(runErr, "build execution failed") } if opts.Test && opts.JSON { - b.PrintTestJSON() return ok, nil } if err := testOutput(int64(b.ExecRows), b.ExecBytes, 0); err != nil { diff --git a/core/sling/build/build.go b/core/sling/build/build.go index 51eeb0d94..a59ac23b2 100644 --- a/core/sling/build/build.go +++ b/core/sling/build/build.go @@ -280,6 +280,8 @@ func (b *Build) Execute() error { } var wg sync.WaitGroup + var mu sync.Mutex + subBuilds := make([]*Build, 0, len(b.Project.SubProjects)) sem := make(chan struct{}, threads) errCh := make(chan error, len(b.Project.SubProjects)) @@ -298,8 +300,16 @@ func (b *Build) Execute() error { errCh <- g.Error(err, "could not compile sub-project %s", sp.Dir) return } - if err := subBuild.Execute(); err != nil { - errCh <- g.Error(err, "could not execute sub-project %s", sp.Dir) + execErr := subBuild.Execute() + + // Collect regardless of the error: a failed sub-project still + // has per-node results, which callers report. + mu.Lock() + subBuilds = append(subBuilds, subBuild) + mu.Unlock() + + if execErr != nil { + errCh <- g.Error(execErr, "could not execute sub-project %s", sp.Dir) } }(subProject) } @@ -307,6 +317,16 @@ func (b *Build) Execute() error { wg.Wait() close(errCh) + // Deterministic order (goroutines finish in any order) + sort.Slice(subBuilds, func(i, j int) bool { + return subBuilds[i].Project.Dir < subBuilds[j].Project.Dir + }) + b.SubBuilds = subBuilds + for _, subBuild := range subBuilds { + b.ExecRows += subBuild.ExecRows + b.ExecBytes += subBuild.ExecBytes + } + var errs []string for err := range errCh { errs = append(errs, err.Error()) @@ -452,6 +472,19 @@ func (b *Build) PrintTestJSON() { fmt.Println(g.Marshal(items)) } +// PrintRunJSON prints the run payload as JSON (per-node results, counts, and +// row/byte totals). Uses the same shape as the pipeline `type: build` step +// state, so `sling build run --json` and hooks share one contract. +// The payload is printed even when the run failed, so callers get the +// per-node errors alongside the non-zero exit code. +func (b *Build) PrintRunJSON(path string) { + if len(b.Project.SubProjects) > 0 { + fmt.Println(g.Marshal(SubProjectsPayload(path, b.SubBuilds))) + return + } + fmt.Println(g.Marshal(RunResultsPayload(path, b.GetTarget(), b.Results))) +} + // PrintCompileOutput prints the compile output in YAML format for each selected node. func (b *Build) PrintCompileOutput() { if len(b.SubBuilds) > 0 { diff --git a/core/sling/build/executor.go b/core/sling/build/executor.go index a9e798ddf..ed53b04dc 100644 --- a/core/sling/build/executor.go +++ b/core/sling/build/executor.go @@ -277,7 +277,9 @@ func (e *Executor) Execute() error { total := len(e.Build.Selected) if total == 0 { - fmt.Println("No models selected.") + if !e.Build.Options.JSON { + fmt.Println("No models selected.") + } if !sling.IsPipelineRunMode() { e.syncBuildStatus(sling.ExecStatusSuccess, nil) } @@ -437,7 +439,9 @@ func (e *Executor) Execute() error { e.syncModelStatus(name, result, sling.ExecStatusSkipped) } - fmt.Println() + if !e.Build.Options.JSON { + fmt.Println() + } e.printSummary() var errs []error @@ -1515,12 +1519,13 @@ func (e *Executor) printSummary() { g.Info("Build Completed in %s | %s | %s%s\n", g.DurationString(totalDuration), successStr, failureStr, skippedStr) - // Print errors section + // Print errors section. Diagnostics go to stderr so stdout stays reserved + // for machine-readable output (--json). if len(failedResults) > 0 { - fmt.Println(env.RedString("Errors:")) + fmt.Fprintln(os.Stderr, env.RedString("Errors:")) for _, r := range failedResults { errMsg := strings.ReplaceAll(strings.TrimSpace(g.ErrMsgSimple(r.Err)), "\n", "\n ") - fmt.Printf(" - %s:\n %s\n", r.Name, env.RedString(errMsg)) + fmt.Fprintf(os.Stderr, " - %s:\n %s\n", r.Name, env.RedString(errMsg)) } } } diff --git a/core/sling/build/hook_runner.go b/core/sling/build/hook_runner.go index e1ae526df..ddf0da0bb 100644 --- a/core/sling/build/hook_runner.go +++ b/core/sling/build/hook_runner.go @@ -55,25 +55,13 @@ func RunForHook(path string, opts sling.HookBuildRunOptions) (map[string]any, er return compileToState(path, b), nil } - // Multi-target / recursive sub-projects use Build.Execute (no per-node results). + // Independent sub-projects (no root sling_build.yml): per-node results are + // grouped per sub-project rather than merged. if len(b.Project.SubProjects) > 0 { if err := b.Execute(); err != nil { - return g.M( - "path", path, - "target", b.GetTarget(), - "sub_projects", len(b.Project.SubProjects), - ), g.Error(err, "build failed") + return SubProjectsPayload(path, b.SubBuilds), g.Error(err, "build failed") } - return g.M( - "path", path, - "target", b.GetTarget(), - "sub_projects", len(b.Project.SubProjects), - "results", []map[string]any{}, - "total", 0, - "ok", 0, - "failed", 0, - "skipped", 0, - ), nil + return SubProjectsPayload(path, b.SubBuilds), nil } executor, err := NewExecutor(b) @@ -82,14 +70,59 @@ func RunForHook(path string, opts sling.HookBuildRunOptions) (map[string]any, er } runErr := executor.Execute() - data := resultsToState(path, b.GetTarget(), executor.Results) + data := RunResultsPayload(path, b.GetTarget(), executor.Results) if runErr != nil { return data, g.Error(runErr, "build failed") } return data, nil } -func resultsToState(path, target string, results []ExecutionResult) map[string]any { +// SubProjectsPayload is the run payload for a project directory that holds +// independent builds (no root sling_build.yml, only subdirectory ones). Node +// names are not unique across independent projects, so each is reported under +// its own entry in `sub_projects`; the top-level counts aggregate them. +// Kept in the same shape as RunResultsPayload so callers can read one contract. +func SubProjectsPayload(path string, subs []*Build) map[string]any { + all := make([]map[string]any, 0, len(subs)) + total, ok, failed, skipped := 0, 0, 0, 0 + var totalRows, totalBytes uint64 + + for _, sub := range subs { + all = append(all, RunResultsPayload(sub.Project.Dir, sub.GetTarget(), sub.Results)) + for _, r := range sub.Results { + total++ + switch { + case r.Skipped: + skipped++ + case r.Err != nil: + failed++ + default: + ok++ + } + totalRows += r.Rows + totalBytes += r.Bytes + } + } + + return g.M( + "path", path, + "target", "", + "sub_projects", all, + "results", []map[string]any{}, + "total", total, + "ok", ok, + "failed", failed, + "skipped", skipped, + "rows", totalRows, + "bytes", totalBytes, + "ok_names", "", + ) +} + +// RunResultsPayload is the machine-readable run payload: per-node results, +// counts, and row/byte totals. Shared by `sling build run --json` and pipeline +// `type: build` steps (state..results). +func RunResultsPayload(path, target string, results []ExecutionResult) map[string]any { rows := make([]map[string]any, 0, len(results)) ok, failed, skipped := 0, 0, 0 var totalRows, totalBytes uint64 diff --git a/tests/build/json_run_project/marts/fct_orders.sql b/tests/build/json_run_project/marts/fct_orders.sql new file mode 100644 index 000000000..57bbf9f43 --- /dev/null +++ b/tests/build/json_run_project/marts/fct_orders.sql @@ -0,0 +1 @@ +SELECT * FROM {{ ref("stg_orders") }} diff --git a/tests/build/json_run_project/sling_build.yml b/tests/build/json_run_project/sling_build.yml new file mode 100644 index 000000000..301ca0bbc --- /dev/null +++ b/tests/build/json_run_project/sling_build.yml @@ -0,0 +1,5 @@ +# Fixture for `sling build run --json`. No target: the tests pass --target to a +# DuckDB connection defined in the suite's env, so no database is required. + +defaults: + mode: full-refresh diff --git a/tests/build/json_run_project/staging/stg_bad.sql b/tests/build/json_run_project/staging/stg_bad.sql new file mode 100644 index 000000000..3c076c28b --- /dev/null +++ b/tests/build/json_run_project/staging/stg_bad.sql @@ -0,0 +1,3 @@ +-- Intentionally invalid: the `run --json` test selects this model to assert the +-- error payload and the non-zero exit code. Never select it in a passing run. +SELECT * FROM table_that_does_not_exist diff --git a/tests/build/json_run_project/staging/stg_orders.sql b/tests/build/json_run_project/staging/stg_orders.sql new file mode 100644 index 000000000..c50167f00 --- /dev/null +++ b/tests/build/json_run_project/staging/stg_orders.sql @@ -0,0 +1,3 @@ +SELECT 1 AS id, 'a' AS name +UNION ALL +SELECT 2 AS id, 'b' AS name diff --git a/tests/suite.cli.build.yaml b/tests/suite.cli.build.yaml index 2153adfe5..3210bee57 100644 --- a/tests/suite.cli.build.yaml +++ b/tests/suite.cli.build.yaml @@ -593,7 +593,7 @@ - 'OK' - 'Build Completed' -- id: 462 +- id: 609 name: 'sling build clickhouse incremental counts temp table before merge' group: build after: [440] @@ -756,3 +756,60 @@ - 'fct_orders' output_does_not_contain: - 'revenue' + +# Machine-readable run output (`--json`) + +- id: 613 + name: 'sling build run --json emits per-node results' + env: + JSON_RUN_DUCK: '{type: duckdb, instance: "temp/build_json_run/test.duckdb"}' + run: | + mkdir -p temp/build_json_run + rm -f temp/build_json_run/test.duckdb + sling build run tests/build/json_run_project --target JSON_RUN_DUCK --json -s stg_orders,fct_orders + group: build + output_contains: + - '"path":"tests/build/json_run_project"' + - '"name":"staging.stg_orders"' + - '"name":"marts.fct_orders"' + - '"status":"success"' + - '"total":2' + - '"ok":2' + - '"failed":0' + +- id: 614 + name: 'sling build run --json reports per-model errors and exits non-zero' + env: + JSON_RUN_DUCK: '{type: duckdb, instance: "temp/build_json_run/test.duckdb"}' + run: | + mkdir -p temp/build_json_run + sling build run tests/build/json_run_project --target JSON_RUN_DUCK --json -s stg_bad + err: true + group: build + after: [613] + output_contains: + - '"name":"staging.stg_bad"' + - '"status":"error"' + - '"total":1' + - '"ok":0' + - '"failed":1' + +- id: 615 + name: 'sling build run --help lists --json' + run: 'sling build run --help' + output_contains: + - '--json' + - '--full-refresh' + - '--select' + +- id: 616 + name: 'sling build run -R --json groups results per sub-project' + run: 'sling build run tests/build/multi_target_project -R --json' + group: build + after: [614] + output_contains: + - '"sub_projects":[{' + - '"name":"staging.stg_orders"' + - '"name":"staging.stg_events"' + - '"ok":2' + - '"failed":0' From b2d9ad3850daf69f1082fd37d012df235b4e1d33 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Mon, 21 Sep 2026 12:10:20 -0300 Subject: [PATCH 23/56] Add LanceDB connection type backed by DuckDB lance extension LanceDB is reachable through DuckDB's lance extension, so it reuses the DuckDB driver, dialect and temp-table handling. Register the type, map `lancedb://{path}` URLs (path is the namespace root and may itself be an `s3://`, `gs://` or `az://` URI), and let the connection round-trip `path`. - Exclude LanceDB from CREATE OR REPLACE TABLE: the lance extension serves a stale projection after a schema-changing replace, so drop first. - Mark it index-less like DuckLake and adjust the shared DB suite for JSON-as-varchar; skip the five cases that depend on views, since the lance extension keeps views in-session only. - Add a local, path-backed CLI suite covering full refresh, merge and CDC. --- README.md | 2 +- cmd/sling/resource/llm_CONNECTION.md | 8 +++-- cmd/sling/sling_test.go | 20 ++++++++++++ core/dbio/connection/connection.go | 24 +++++++++++++- core/dbio/connection/connection_test.go | 31 +++++++++++++++++++ core/dbio/database/database.go | 4 ++- core/dbio/database/schemata_keys.go | 1 + core/dbio/dbio_types.go | 10 ++++-- core/dbio/templates/_properties.yaml | 17 ++++++++++ core/dbio/templates/connections.json | 12 +++++++ core/sling/assist/skills/sling/CONNECTIONS.md | 2 +- core/sling/build/build.go | 2 +- core/sling/build/ddl.go | 4 ++- core/sling/config.go | 4 +-- core/sling/task.go | 2 +- tests/suite.cli.yaml | 20 ++++++++++++ 16 files changed, 148 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 3512c82b4..a795dd894 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Example [Replication](https://docs.slingdata.io/sling-cli/run/configuration/repl Available Connectors: - **Databases**: [`adbc`](https://docs.slingdata.io/connections/database-connections/adbc) [`azuredwh`](https://docs.slingdata.io/connections/database-connections/azuredwh) [`azuresql`](https://docs.slingdata.io/connections/database-connections/azuresql) [`azuretable`](https://docs.slingdata.io/connections/database-connections/azuretable) [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`d1`](https://docs.slingdata.io/connections/database-connections/d1) [`databricks`](https://docs.slingdata.io/connections/database-connections/databricks) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`elasticsearch`](https://docs.slingdata.io/connections/database-connections/elasticsearch) [`exasol`](https://docs.slingdata.io/connections/database-connections/exasol) [`fabric`](https://docs.slingdata.io/connections/database-connections/fabric) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`mongodb`](https://docs.slingdata.io/connections/database-connections/mongodb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`odbc`](https://docs.slingdata.io/connections/database-connections/odbc) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`scylladb`](https://docs.slingdata.io/connections/database-connections/scylladb) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`trino`](https://docs.slingdata.io/connections/database-connections/trino) -- **Data Lakes**:[`athena`](https://docs.slingdata.io/connections/datalake-connections/athena) [`Ducklake`](https://docs.slingdata.io/connections/datalake-connections/ducklake) [`iceberg`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`S3 Tables`](https://docs.slingdata.io/connections/datalake-connections/iceberg) +- **Data Lakes**:[`athena`](https://docs.slingdata.io/connections/datalake-connections/athena) [`Ducklake`](https://docs.slingdata.io/connections/datalake-connections/ducklake) [`iceberg`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`S3 Tables`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`lancedb`](https://docs.slingdata.io/connections/datalake-connections/lancedb) - **File Systems**: [`azure`](https://docs.slingdata.io/connections/file-connections/azure) [`b2`](https://docs.slingdata.io/connections/file-connections/b2) [`dospaces`](https://docs.slingdata.io/connections/file-connections/dospaces) [`gs`](https://docs.slingdata.io/connections/file-connections/gs) [`local`](https://docs.slingdata.io/connections/file-connections/local) [`minio`](https://docs.slingdata.io/connections/file-connections/minio) [`r2`](https://docs.slingdata.io/connections/file-connections/r2) [`s3`](https://docs.slingdata.io/connections/file-connections/s3) [`sftp`](https://docs.slingdata.io/connections/file-connections/sftp) [`wasabi`](https://docs.slingdata.io/connections/file-connections/wasabi) - **File Formats**: `csv`, `parquet`, `xlsx`, `json`, `geojson`, `avro`, `xml`, `sas7bday` diff --git a/cmd/sling/resource/llm_CONNECTION.md b/cmd/sling/resource/llm_CONNECTION.md index 89ac04abc..8727708b7 100644 --- a/cmd/sling/resource/llm_CONNECTION.md +++ b/cmd/sling/resource/llm_CONNECTION.md @@ -106,7 +106,7 @@ For database operations (queries, schema exploration) and file system operations ## 3. Connection Types Overview -Sling supports 41+ different connection types across four categories: +Sling supports 42+ different connection types across four categories: ### Database Connections (24 types) - **Relational**: PostgreSQL, MySQL, MariaDB, SQLServer, Oracle, SQLite @@ -122,9 +122,9 @@ Sling supports 41+ different connection types across four categories: - **Cloud Drives**: Google Drive - **Local**: Local file system -### Datalake Connections (4 types) +### Datalake Connections (5 types) - **Query Engines**: Athena, DuckLake -- **Table Formats**: Iceberg +- **Table Formats**: Iceberg, LanceDB ### API Connections - **Custom APIs**: User-defined API specifications in YAML format @@ -223,6 +223,8 @@ Navigate or fetch the content of the connector from the below respective URL to **DuckLake (`ducklake`)** -> https://docs.slingdata.io/connections/datalake-connections/ducklake +**LanceDB (`lancedb`)** -> https://docs.slingdata.io/connections/datalake-connections/lancedb + --- ## 7. API Connections diff --git a/cmd/sling/sling_test.go b/cmd/sling/sling_test.go index c2f8d2edc..999ef12ec 100755 --- a/cmd/sling/sling_test.go +++ b/cmd/sling/sling_test.go @@ -96,6 +96,7 @@ var connMap = map[dbio.Type]connTest{ dbio.TypeDbMotherDuck: {name: "motherduck", adjustCol: g.Bool(false)}, dbio.TypeDbAthena: {name: "athena", adjustCol: g.Bool(false)}, dbio.TypeDbIceberg: {name: "iceberg_r2", adjustCol: g.Bool(false)}, + dbio.TypeDbLanceDB: {name: "lancedb", schema: "main", adjustCol: g.Bool(false)}, dbio.Type("iceberg_glue"): {name: "iceberg_glue", adjustCol: g.Bool(false)}, dbio.Type("iceberg_s3"): {name: "iceberg_s3", adjustCol: g.Bool(false)}, dbio.Type("iceberg_sql"): {name: "iceberg_sql", adjustCol: g.Bool(false)}, @@ -945,6 +946,14 @@ func runOneTask(t *testing.T, ctx context.Context, file g.FileItem, connType dbi if correctType == iop.JsonType { correctType = iop.TextType // sqlserver uses varchar(max) for json } + case tgtType == dbio.TypeDbLanceDB: + if correctType == iop.JsonType { + correctType = iop.TextType // lance stores json as varchar + } + case srcType == dbio.TypeDbLanceDB && tgtType == dbio.TypeDbPostgres: + if correctType == iop.JsonType { + correctType = iop.TextType // lance stores json as varchar + } case tgtType == dbio.TypeDbRedshift: if correctType == iop.JsonType { correctType = iop.TextType // redshift uses text for json @@ -1216,6 +1225,17 @@ func TestSuiteDatabaseIceberg(t *testing.T) { testSuite(t, dbio.Type("iceberg_sql"), tests) } +// TestSuiteDatabaseLanceDb runs the shared DB suite against a LanceDB +// namespace. Every excluded case depends on the `[table]_vw` view that test 9 +// creates: 10 and 11 discover it, 13 reads it, 19 reads the postgres copy of it +// (`[table]_pg_vw`), and 22 drops `[table]_vw_pg`. The lance extension keeps +// views in the session only (CREATE VIEW succeeds but the view is not written +// into the namespace), so those five cannot pass for any LanceDB target. +func TestSuiteDatabaseLanceDb(t *testing.T) { + t.Parallel() + testSuite(t, dbio.TypeDbLanceDB, "1-9,12,14-18,20-21,23-29") +} + func TestSuiteDatabaseDB2(t *testing.T) { t.Parallel() testSuite(t, dbio.Type("db2"), "1-5,7+") diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index 0e454c26c..13ce13cac 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -599,7 +599,7 @@ func (c *Connection) setURL() (err error) { pathValue := strings.ReplaceAll(U.Path(), "/", "") setIfMissing("schema", U.PopParam("schema")) - if !g.In(c.Type, dbio.TypeDbMotherDuck, dbio.TypeDbDuckDb, dbio.TypeDbDuckLake, dbio.TypeDbSQLite, dbio.TypeDbD1, dbio.TypeDbBigQuery) { + if !g.In(c.Type, dbio.TypeDbMotherDuck, dbio.TypeDbDuckDb, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB, dbio.TypeDbSQLite, dbio.TypeDbD1, dbio.TypeDbBigQuery) { setIfMissing("host", U.Hostname()) setIfMissing("user", U.Username()) setIfMissing("username", U.Username()) @@ -624,6 +624,9 @@ func (c *Connection) setURL() (err error) { case dbio.TypeDbSQLite, dbio.TypeDbDuckDb: setIfMissing("instance", U.Path()) setIfMissing("schema", "main") + case dbio.TypeDbLanceDB: + setIfMissing("path", lanceDBPathFromURL(c.URL())) + setIfMissing("schema", "main") case dbio.TypeDbMotherDuck: setIfMissing("schema", "main") case dbio.TypeDbD1: @@ -956,6 +959,13 @@ func (c *Connection) setURL() (err error) { // Build the ducklake URL based on catalog configuration // Default to simple ducklake:// if no specific catalog URL is provided template = "ducklake://" + case dbio.TypeDbLanceDB: + // the namespace root is a directory path or an object store URI + if val, ok := c.Data["path"]; ok { + c.Data["path"] = strings.ReplaceAll(cast.ToString(val), `\`, `/`) // windows path fix + } + setIfMissing("schema", "main") + template = "lancedb://{path}" case dbio.TypeDbMotherDuck: setIfMissing("schema", "main") setIfMissing("interactive", true) @@ -1431,6 +1441,18 @@ func (i *Info) IsURL() bool { return strings.Contains(i.Name, "://") } +// lanceDBPathFromURL extracts the namespace root from a `lancedb://` URL. +// The root is everything after the scheme, so that it can itself be an object +// store URI (`lancedb://s3://bucket/prefix`) as well as a local directory +// (`lancedb:///data/lancedb`). Query params are not part of the path. +func lanceDBPathFromURL(connURL string) string { + connPath := strings.TrimPrefix(connURL, "lancedb://") + if i := strings.Index(connPath, "?"); i >= 0 { + connPath = connPath[:i] + } + return connPath +} + // SchemeType returns the correct scheme of the url func SchemeType(url string) dbio.Type { if t, _, _, err := filesys.ParseURLType(url); err == nil { diff --git a/core/dbio/connection/connection_test.go b/core/dbio/connection/connection_test.go index f5aa43267..a7aa625e1 100644 --- a/core/dbio/connection/connection_test.go +++ b/core/dbio/connection/connection_test.go @@ -487,3 +487,34 @@ func TestSQLServerNamedInstance(t *testing.T) { }) } } + +// A LanceDB namespace root can itself be an object store URI, so everything +// after the `lancedb://` scheme is the path. +func TestLanceDBConnectionURL(t *testing.T) { + cases := []struct{ url, path string }{ + {"lancedb:///data/lancedb", "/data/lancedb"}, + {"lancedb://./data/lancedb", "./data/lancedb"}, + {"lancedb://relative/dir", "relative/dir"}, + {"lancedb://s3://my-bucket/prefix", "s3://my-bucket/prefix"}, + {"lancedb:///data/lancedb?schema=main", "/data/lancedb"}, + } + + for _, tc := range cases { + t.Run(tc.url, func(t *testing.T) { + c, err := NewConnectionFromURL("LANCEDB_TEST", tc.url) + require.NoError(t, err) + assert.Equal(t, dbio.TypeDbLanceDB, c.Type) + assert.Equal(t, tc.path, c.DataS(true)["path"]) + }) + } + + // the `path` property round-trips through the connection URL + c, err := NewConnection("LANCEDB_TEST", dbio.TypeDbLanceDB, map[string]any{"path": "s3://my-bucket/prefix"}) + require.NoError(t, err) + assert.Equal(t, "lancedb://s3://my-bucket/prefix", c.URL()) + + reparsed, err := NewConnectionFromURL("LANCEDB_TEST", c.URL()) + require.NoError(t, err) + assert.Equal(t, dbio.TypeDbLanceDB, reparsed.Type) + assert.Equal(t, "s3://my-bucket/prefix", reparsed.DataS(true)["path"]) +} diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index ab534cd8b..5156a5456 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -320,6 +320,8 @@ func NewConnContext(ctx context.Context, URL string, props ...string) (Connectio conn = &DuckLakeConn{DuckDbConn: DuckDbConn{URL: URL}} } else if strings.HasPrefix(URL, "iceberg:") { conn = &IcebergConn{URL: URL} + } else if strings.HasPrefix(URL, "lancedb:") { + conn = &LanceDBConn{DuckDbConn: DuckDbConn{URL: URL}} } else if strings.HasPrefix(URL, "azuretable:") { conn = &AzureTableConn{URL: URL} } else if strings.HasPrefix(URL, "adbc:") || strings.HasPrefix(URL, "flightsql:") { @@ -382,7 +384,7 @@ func getDriverName(conn Connection) (driverName string) { driverName = "databricks" case dbio.TypeDbSQLite: driverName = "sqlite3" - case dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake: + case dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB: driverName = "duckdb" case dbio.TypeDbSQLServer: driverName = "sqlserver" diff --git a/core/dbio/database/schemata_keys.go b/core/dbio/database/schemata_keys.go index 090a067ba..28b05ea2b 100644 --- a/core/dbio/database/schemata_keys.go +++ b/core/dbio/database/schemata_keys.go @@ -616,6 +616,7 @@ var indexCapabilities = map[dbio.Type]indexCapability{ dbio.TypeDbDuckDb: {supportsUnique: true}, dbio.TypeDbMotherDuck: {supportsUnique: true}, dbio.TypeDbDuckLake: {noIndexes: true}, // DuckLake does not support indexes + dbio.TypeDbLanceDB: {noIndexes: true}, // Lance datasets have no secondary indexes dbio.TypeDbOracle: {supportsUnique: true, supportsType: true, typeClosedSet: []string{"bitmap"}}, dbio.TypeDbSQLite: {supportsWhere: true, supportsUnique: true}, } diff --git a/core/dbio/dbio_types.go b/core/dbio/dbio_types.go index b10fa2460..03d464055 100644 --- a/core/dbio/dbio_types.go +++ b/core/dbio/dbio_types.go @@ -86,6 +86,7 @@ const ( TypeDbProton Type = "proton" TypeDbAthena Type = "athena" TypeDbIceberg Type = "iceberg" + TypeDbLanceDB Type = "lancedb" TypeDbAzureTable Type = "azuretable" TypeDbExasol Type = "exasol" TypeDbArrowDBC Type = "adbc" @@ -132,6 +133,7 @@ var AllType = []struct { {TypeDbTrino, "TypeDbTrino"}, {TypeDbAthena, "TypeDbAthena"}, {TypeDbIceberg, "TypeDbIceberg"}, + {TypeDbLanceDB, "TypeDbLanceDB"}, {TypeDbClickhouse, "TypeDbClickhouse"}, {TypeDbElasticsearch, "TypeDbElasticsearch"}, {TypeDbMongoDB, "TypeDbMongoDB"}, @@ -163,7 +165,7 @@ func ValidateType(tStr string) (Type, bool) { case TypeApi, TypeFileLocal, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileDatabricksVolume, - TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: + TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: return t, true } @@ -229,7 +231,7 @@ func (t Type) DBNameUpperCase() bool { func (t Type) Kind() Kind { switch t { case TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbBigTable, - TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: + TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: return KindDatabase case TypeFileLocal, TypeFileHDFS, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileHTTP, TypeFileDatabricksVolume, Type("https"): return KindFile @@ -291,7 +293,7 @@ func (t Type) IsColumnStore() bool { TypeDbSnowflake, TypeDbBigQuery, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbRedshift, TypeDbDatabricks, TypeDbStarRocks, - TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbExasol, + TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbExasol, TypeDbAzureDWH, TypeDbFabric, ) } @@ -340,6 +342,7 @@ func (t Type) NameLong() string { TypeDbTrino: "DB - Trino", TypeDbAthena: "DB - Athena", TypeDbIceberg: "DB - Iceberg", + TypeDbLanceDB: "DB - LanceDB", TypeDbClickhouse: "DB - Clickhouse", TypeDbPrometheus: "DB - Prometheus", TypeDbElasticsearch: "DB - Elasticsearch", @@ -391,6 +394,7 @@ func (t Type) Name() string { TypeDbTrino: "Trino", TypeDbAthena: "Athena", TypeDbIceberg: "Iceberg", + TypeDbLanceDB: "LanceDB", TypeDbClickhouse: "Clickhouse", TypeDbPrometheus: "Prometheus", TypeDbElasticsearch: "Elasticsearch", diff --git a/core/dbio/templates/_properties.yaml b/core/dbio/templates/_properties.yaml index 99b67c719..005397c0f 100644 --- a/core/dbio/templates/_properties.yaml +++ b/core/dbio/templates/_properties.yaml @@ -1183,6 +1183,23 @@ ducklake: description: 'Data format' examples: ['parquet', 'delta', 'iceberg'] +lancedb: + title: 'LanceDB' + kind: database + required: ["path"] + url_template: 'lancedb://{path}' + docs: https://docs.slingdata.io/connections/datalake-connections/lancedb + properties: + name: + type: text + title: Connection Name + description: 'The name of the connection. No spaces are allowed, will be all caps. Example: LANCEDB_MAIN' + examples: ['LANCEDB_MAIN', 'LANCEDB_LOCAL', 'LANCE_ANALYTICS'] + path: + type: text + description: 'LanceDB namespace root: the directory holding the .lance datasets (a local path, or an `s3://`, `gs://` or `az://` URI)' + examples: ['./data/lancedb', '/data/lancedb', 's3://my-bucket/lancedb', 'gs://my-bucket/lancedb'] + iceberg: title: 'Apache Iceberg' kind: database diff --git a/core/dbio/templates/connections.json b/core/dbio/templates/connections.json index 3685738f2..299aa3105 100644 --- a/core/dbio/templates/connections.json +++ b/core/dbio/templates/connections.json @@ -1486,6 +1486,18 @@ } } }, + "lancedb": { + "name": "LanceDB", + "description": "LanceDB vector / lakehouse format, served through DuckDB's lance extension", + "category": "datalake", + "properties": { + "path": { + "type": "string", + "description": "LanceDB namespace root: directory holding the .lance datasets (local path, or an s3://, gs:// or az:// URI)", + "required": true + } + } + }, "iceberg": { "name": "Apache Iceberg", "description": "Apache Iceberg table format", diff --git a/core/sling/assist/skills/sling/CONNECTIONS.md b/core/sling/assist/skills/sling/CONNECTIONS.md index 8d53ac554..712b8a5f8 100644 --- a/core/sling/assist/skills/sling/CONNECTIONS.md +++ b/core/sling/assist/skills/sling/CONNECTIONS.md @@ -17,7 +17,7 @@ Resolve each row before you write YAML, in this order: the user's request, exist | Type | Examples | Kind | |------|----------|------| | Database | postgres, mysql, snowflake, bigquery | `database` | -| Datalake | iceberg, ducklake, athena, s3 tables | `database` | +| Datalake | iceberg, ducklake, lancedb, athena, s3 tables | `database` | | File System | s3, gcs, azure, sftp, local | `file` | | API | Custom REST APIs via specs | `api` | diff --git a/core/sling/build/build.go b/core/sling/build/build.go index a59ac23b2..b3ddfd218 100644 --- a/core/sling/build/build.go +++ b/core/sling/build/build.go @@ -779,7 +779,7 @@ func mapDialect(dbType dbio.Type) string { return "bigquery" case dbio.TypeDbSnowflake: return "snowflake" - case dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake: + case dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB: return "duckdb" case dbio.TypeDbDatabricks: return "databricks" diff --git a/core/sling/build/ddl.go b/core/sling/build/ddl.go index f2e4792d8..2b48002b2 100644 --- a/core/sling/build/ddl.go +++ b/core/sling/build/ddl.go @@ -21,12 +21,14 @@ func (e *Executor) quoteFullTableName(fullName string) (string, error) { func supportsDropCascade(t dbio.Type) bool { return g.In(t, dbio.TypeDbPostgres, dbio.TypeDbRedshift, - dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake, + dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB, dbio.TypeDbSnowflake, // accepted (no-op-ish) but harmless ) } // supportsCreateOrReplaceTable reports dialects with atomic CREATE OR REPLACE TABLE. +// LanceDB is excluded: the lance extension serves a stale projection after a +// replace that changes the schema, so the table must be dropped first. func supportsCreateOrReplaceTable(t dbio.Type) bool { return g.In(t, dbio.TypeDbSnowflake, diff --git a/core/sling/config.go b/core/sling/config.go index 66b0068ab..0e688b946 100644 --- a/core/sling/config.go +++ b/core/sling/config.go @@ -1027,8 +1027,8 @@ func (cfg *Config) FormatTargetObjectName() (err error) { tableTmp.Name = strings.ToUpper(tableTmp.Name) } tgtOpts.TableTmp = tableTmp.FullName() - } else if g.In(dbType, dbio.TypeDbDuckDb, dbio.TypeDbDuckLake) { - // for duckdb and ducklake, we'll use a temp table, which uses the 'main' schema + } else if g.In(dbType, dbio.TypeDbDuckDb, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB) { + // for duckdb, ducklake and lancedb, we'll use a temp table, which uses the 'main' schema tableTmp := makeTempTableName(dbType, table, "_sling_duckdb_tmp") tableTmp.Schema = "main" tgtOpts.TableTmp = tableTmp.FullName() diff --git a/core/sling/task.go b/core/sling/task.go index e5d352a37..30d200de6 100644 --- a/core/sling/task.go +++ b/core/sling/task.go @@ -632,7 +632,7 @@ func ErrorHelper(err error, connTypes ...dbio.Type) (helpString string) { // which accepts the `copy_method` property usesDuckDb := false for _, connType := range connTypes { - if g.In(connType, dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake) { + if g.In(connType, dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB) { usesDuckDb = true } } diff --git a/tests/suite.cli.yaml b/tests/suite.cli.yaml index 662569c73..2af6e4db3 100644 --- a/tests/suite.cli.yaml +++ b/tests/suite.cli.yaml @@ -3036,6 +3036,26 @@ - iceberg cdc counts - execution succeeded +# LanceDB: fully local (path-backed namespace), so the test sets up its own +# connection in a temp home dir and needs no external credentials. +- id: 617 + name: 'Run sling lancedb full refresh, merge and CDC' + run: | + HOME_DIR=$(mktemp -d) + NAMESPACE_DIR=$(mktemp -d)/nested/lancedb + sling conns set --home-dir "$HOME_DIR" LANCEDB_TEST --type lancedb path="$NAMESPACE_DIR" + sling run --home-dir "$HOME_DIR" -r tests/replications/r.128.lancedb_full_refresh.yaml + sling run --home-dir "$HOME_DIR" -r tests/replications/r.129.lancedb_merge.yaml + sling run --home-dir "$HOME_DIR" -r tests/replications/r.130.lancedb_cdc_seed.yaml + sling run --home-dir "$HOME_DIR" -r tests/replications/r.131.lancedb_cdc.yaml + sling run --home-dir "$HOME_DIR" --src-conn LANCEDB_TEST --src-stream "select 'cdc_counts' as marker, count(*) as cnt, count(distinct id) as did from main.lancedb_cdc_test" --stdout + sling run --home-dir "$HOME_DIR" --src-conn LANCEDB_TEST --src-stream "select id, name from main.lancedb_cdc_test order by id" --stdout + output_contains: + - 'cdc_counts,3,3' + - '1,Alice Final' + - '5,Eve' + - execution succeeded + # Nested CLI suites. Loader in sling_cli_test.go merges these in. - suite: suite.cli.assist.yaml - suite: suite.cli.build.yaml From c349c9de5dc38f7f67708ea14fd043fd214203d4 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Mon, 21 Sep 2026 12:14:37 -0300 Subject: [PATCH 24/56] add LanceDB database connection support via DuckDB lance extension - LanceDBConn attaches the namespace root as an in-memory DuckDB catalog (TYPE lance), so `.lance` datasets are plain tables and DuckDB's dialect, type mapping and merge strategies apply unchanged. - Resolve the namespace root from `path` (alias `instance`) and clear `instance`, so it never becomes DuckDB's own database file; reject `read_only`, which neither the CLI nor the lance attach accepts. - Register scoped lance secrets for object-store roots (s3/gs/az families, with credential-chain fallback), pass unknown schemes through so the extension reports them, and skip secrets for stores that self-resolve. - Add the lancedb template: metadata queries pinned to `current_database()` so the temp-table catalog is excluded, and every merge strategy written as a single MERGE INTO since the extension cannot plan DELETE/UPDATE that read a second table. Matched rows are updated in place for delete_insert (same end state), and views stay session-only as the namespace has no view operations. - Cover namespace-root handling, attach quoting, scheme/scope mapping and secret construction in database_test.go, plus full-refresh, merge and synthetic CDC replications. --- core/dbio/database/database_lancedb.go | 209 +++++++++ core/dbio/database/database_test.go | 160 +++++++ core/dbio/templates/lancedb.yaml | 395 ++++++++++++++++++ .../r.128.lancedb_full_refresh.yaml | 12 + tests/replications/r.129.lancedb_merge.yaml | 16 + .../replications/r.130.lancedb_cdc_seed.yaml | 16 + tests/replications/r.131.lancedb_cdc.yaml | 20 + 7 files changed, 828 insertions(+) create mode 100644 core/dbio/database/database_lancedb.go create mode 100755 core/dbio/templates/lancedb.yaml create mode 100644 tests/replications/r.128.lancedb_full_refresh.yaml create mode 100644 tests/replications/r.129.lancedb_merge.yaml create mode 100644 tests/replications/r.130.lancedb_cdc_seed.yaml create mode 100644 tests/replications/r.131.lancedb_cdc.yaml diff --git a/core/dbio/database/database_lancedb.go b/core/dbio/database/database_lancedb.go new file mode 100644 index 000000000..d444dc828 --- /dev/null +++ b/core/dbio/database/database_lancedb.go @@ -0,0 +1,209 @@ +package database + +import ( + "strings" + "time" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/spf13/cast" +) + +// LanceDB exposes Lance datasets as a SQL database by way of DuckDB's `lance` +// extension: the namespace root is attached as a DuckDB catalog, so every +// `
.lance` dataset under the root is a table in its `main` schema. +// The DuckDB dialect, type mapping and merge strategies therefore apply. +// See https://duckdb.org/docs/current/core_extensions/lance.html + +// lanceDBNamespace is the catalog name the namespace root is attached as. +const lanceDBNamespace = "lancedb" + +// LanceDBConn is a LanceDB connection +type LanceDBConn struct { + DuckDbConn + + Path string // namespace root: local directory or object store URI +} + +// Init initiates the object +func (conn *LanceDBConn) Init() error { + conn.Path = conn.GetProp("path") + if conn.Path == "" { + conn.Path = conn.GetProp("instance") + } + if strings.TrimSpace(conn.Path) == "" { + return g.Error("did not provide 'path' for LanceDB connection (the namespace root)") + } + conn.SetProp("path", conn.Path) + + // the DuckDB process runs in memory and the namespace is attached to it, + // so `instance` must not become the process's database file + conn.SetProp("instance", "") + + if cast.ToBool(conn.GetProp("read_only")) { + // the DuckDB CLI cannot launch its in-memory database read-only, and + // the lance extension rejects a READ_ONLY attach + return g.Error("the `read_only` property is not supported for LanceDB connections") + } + + conn.BaseConn.URL = conn.URL + conn.BaseConn.Type = dbio.TypeDbLanceDB + + // initialize DuckDB instance (inheriting from DuckDbConn) + conn.duck = iop.NewDuckDb(conn.Context().Ctx, g.MapToKVArr(conn.properties)...) + + instance := Connection(conn) + conn.BaseConn.instance = &instance + + return conn.BaseConn.Init() +} + +// Connect establishes the LanceDB connection +func (conn *LanceDBConn) Connect(timeOut ...int) (err error) { + // the namespace is attached to an in-memory DuckDB instance + conn.DuckDbConn.URL = "duckdb::memory:" + + if err = conn.DuckDbConn.Connect(timeOut...); err != nil { + return g.Error(err, "could not connect to DuckDB for LanceDB") + } + + conn.SetProp("connected", "true") + conn.SetProp("connect_time", cast.ToString(time.Now())) + + // the lance extension provides Lance dataset read/write + conn.duck.AddExtension("lance") + + if conn.isObjectStore() { + // the lance extension requires httpfs to reach object stores + conn.duck.AddExtension("httpfs") + if secret, ok := conn.makeSecret(); ok { + conn.duck.AddSecret(secret) + } + } + // a local namespace root is created by the extension on first write + + if _, err = conn.Exec(conn.buildAttachSQL() + noDebugKey); err != nil { + return g.Error(err, "could not attach LanceDB namespace: %s", conn.Path) + } + + // make the namespace the default catalog, so unqualified tables resolve to it + if _, err = conn.Exec("USE " + lanceDBNamespace + ";" + noDebugKey); err != nil { + return g.Error(err, "could not use LanceDB namespace") + } + + return nil +} + +// buildAttachSQL creates the ATTACH statement for the namespace root +func (conn *LanceDBConn) buildAttachSQL() string { + path := strings.ReplaceAll(conn.Path, "'", "''") + return g.F("ATTACH IF NOT EXISTS '%s' AS %s (TYPE lance)", path, lanceDBNamespace) +} + +// GetURL returns the processed URL +func (conn *LanceDBConn) GetURL(newURL ...string) string { + connURL := conn.BaseConn.URL + if len(newURL) > 0 { + connURL = newURL[0] + } + return connURL +} + +// isObjectStore returns whether the namespace root lives in an object store +func (conn *LanceDBConn) isObjectStore() bool { + switch conn.objectStoreScheme() { + case "", "file": + return false + } + return true +} + +// objectStoreScheme returns the object store family of the namespace root. Only +// the schemes the lance extension knows are mapped: it accepts `s3` (plus the +// `s3a` / `s3n` aliases), `gs` and `az` (plus the `abfss` alias). Any other +// scheme is returned as-is, so that an unsupported one (e.g. `r2://`) surfaces +// the extension's own error instead of being silently rewritten. +func (conn *LanceDBConn) objectStoreScheme() string { + scheme, _, ok := strings.Cut(strings.ToLower(conn.Path), "://") + if !ok { + return "" + } + switch scheme { + case "s3", "s3a", "s3n": + return "s3" + case "gs": + return "gs" + case "az", "abfss": + return "az" + default: + return scheme + } +} + +// lanceScope returns the secret scope of the namespace root: the URI up to the +// end of its bucket / container (e.g. `s3://my-bucket/`). Secrets are matched +// by URI prefix, and datasets live below the namespace root. +func (conn *LanceDBConn) lanceScope() string { + path := strings.ReplaceAll(conn.Path, "\\", "/") + scheme, rest, ok := strings.Cut(path, "://") + if !ok { + return path + } + bucket, _, _ := strings.Cut(rest, "/") + return g.F("%s://%s/", scheme, bucket) +} + +// makeSecret builds the scoped Lance secret used to reach an object store. +// Credentials are taken from the connection properties when provided, and fall +// back to the upstream SDK credential chain otherwise. It returns false for +// object stores whose credentials the extension resolves on its own (e.g. OSS, +// Hugging Face Hub), where a secret would only get in the way. +func (conn *LanceDBConn) makeSecret() (iop.DuckDbSecret, bool) { + props := map[string]string{"scope": conn.lanceScope()} + + switch conn.objectStoreScheme() { + case "s3": + accessKey := conn.GetProp("s3_access_key_id") + secretKey := conn.GetProp("s3_secret_access_key") + if accessKey == "" || secretKey == "" { + props["provider"] = "credential_chain" + } else { + props["provider"] = "config" + props["access_key_id"] = accessKey + props["secret_access_key"] = secretKey + if val := conn.GetProp("s3_session_token"); val != "" { + props["session_token"] = val + } + } + if val := conn.GetProp("s3_region"); val != "" { + props["region"] = val + } + if val := conn.GetProp("s3_endpoint"); val != "" { + props["endpoint"] = val + props["allow_http"] = cast.ToString(strings.HasPrefix(val, "http://")) + } + case "gs": + props["provider"] = "credential_chain" + if val := conn.GetProp("gcs_key_file"); val != "" { + props["provider"] = "config" + props["google_application_credentials"] = val + } + case "az": + props["provider"] = "config" + if val := conn.GetProp("azure_account_name"); val != "" { + props["account_name"] = val + } + if val := conn.GetProp("azure_account_key"); val != "" { + props["account_key"] = val + } else if val := conn.GetProp("azure_sas_token"); val != "" { + props["sas_token"] = strings.TrimPrefix(val, "?") + } else { + props["provider"] = "credential_chain" + } + default: + return iop.DuckDbSecret{}, false + } + + return iop.NewDuckDbSecret("lance_secret", iop.DuckDbSecretType("lance"), props), true +} diff --git a/core/dbio/database/database_test.go b/core/dbio/database/database_test.go index a01862098..964cb180f 100755 --- a/core/dbio/database/database_test.go +++ b/core/dbio/database/database_test.go @@ -2258,3 +2258,163 @@ func TestIcebergDuckMergeSQLFromTemplates(t *testing.T) { _, err = conn.duckMergeSQL(tgt, src, cols, nil, MergeStrategyDeleteInsert) require.Error(t, err) } + +func newTestLanceDBConn(t *testing.T, props map[string]string) *LanceDBConn { + t.Helper() + conn := &LanceDBConn{} + conn.setContext(context.Background(), 1) + for k, v := range props { + conn.SetProp(k, v) + } + return conn +} + +// The namespace root comes from `path`, with `instance` as an alias. It must +// never reach DuckDB as the process's database file. +func TestLanceDBConnNamespaceRoot(t *testing.T) { + err := newTestLanceDBConn(t, nil).Init() + require.Error(t, err) + assert.Contains(t, err.Error(), "'path'") + + conn := newTestLanceDBConn(t, map[string]string{"instance": "/data/lancedb"}) + require.NoError(t, conn.Init()) + assert.Equal(t, "/data/lancedb", conn.Path) + assert.Empty(t, conn.GetProp("instance")) + assert.Equal(t, "ATTACH IF NOT EXISTS '/data/lancedb' AS lancedb (TYPE lance)", conn.buildAttachSQL()) +} + +// The DuckDB CLI cannot open an in-memory database read-only, and the lance +// extension rejects a READ_ONLY attach, so the property is refused up front. +func TestLanceDBConnReadOnlyRefused(t *testing.T) { + conn := newTestLanceDBConn(t, map[string]string{"path": "/data/lancedb", "read_only": "true"}) + err := conn.Init() + require.Error(t, err) + assert.Contains(t, err.Error(), "read_only") +} + +func TestLanceDBConnAttachSQLQuoting(t *testing.T) { + conn := newTestLanceDBConn(t, map[string]string{"path": "/data/my'db"}) + require.NoError(t, conn.Init()) + assert.Equal(t, "ATTACH IF NOT EXISTS '/data/my''db' AS lancedb (TYPE lance)", conn.buildAttachSQL()) +} + +// Only the schemes the lance extension serves are mapped to a secret family; +// everything else is passed through so the extension reports it. +func TestLanceDBConnObjectStore(t *testing.T) { + tests := []struct { + path string + scheme string + scope string + objectStore bool + }{ + {path: "/data/lancedb", scheme: "", scope: "/data/lancedb"}, + {path: "./data/lancedb", scheme: "", scope: "./data/lancedb"}, + {path: "file:///data/lancedb", scheme: "file", scope: "file:///"}, + {path: "s3://my-bucket/lancedb", scheme: "s3", scope: "s3://my-bucket/", objectStore: true}, + {path: "s3a://my-bucket/nested/lancedb", scheme: "s3", scope: "s3a://my-bucket/", objectStore: true}, + {path: "gs://my-bucket/lancedb", scheme: "gs", scope: "gs://my-bucket/", objectStore: true}, + { + path: "abfss://container@acct.dfs.core.windows.net/lancedb", + scheme: "az", + scope: "abfss://container@acct.dfs.core.windows.net/", + objectStore: true, + }, + {path: "r2://my-bucket/lancedb", scheme: "r2", scope: "r2://my-bucket/", objectStore: true}, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + conn := newTestLanceDBConn(t, map[string]string{"path": tt.path}) + require.NoError(t, conn.Init()) + assert.Equal(t, tt.scheme, conn.objectStoreScheme()) + assert.Equal(t, tt.objectStore, conn.isObjectStore()) + assert.Equal(t, tt.scope, conn.lanceScope()) + }) + } +} + +// A secret is built for the object stores whose credentials sling can supply. +// The keys are the ones the lance extension's secret provider accepts, and the +// scope covers the bucket so that every dataset below it matches. +func TestLanceDBConnSecrets(t *testing.T) { + conn := newTestLanceDBConn(t, map[string]string{ + "path": "s3://my-bucket/lancedb", + "s3_access_key_id": "AKIAEXAMPLE", + "s3_secret_access_key": "secret", + "s3_session_token": "token", + "s3_region": "us-east-1", + "s3_endpoint": "http://localhost:9000", + }) + require.NoError(t, conn.Init()) + + secret, ok := conn.makeSecret() + require.True(t, ok) + assert.Equal(t, "lance_secret", secret.Name) + assert.Equal(t, iop.DuckDbSecretType("lance"), secret.Type) + assert.Equal(t, "s3://my-bucket/", secret.Props["scope"]) + assert.Equal(t, "config", secret.Props["provider"]) + assert.Equal(t, "AKIAEXAMPLE", secret.Props["access_key_id"]) + assert.Equal(t, "secret", secret.Props["secret_access_key"]) + assert.Equal(t, "token", secret.Props["session_token"]) + assert.Equal(t, "us-east-1", secret.Props["region"]) + assert.Equal(t, "http://localhost:9000", secret.Props["endpoint"]) + assert.Equal(t, "true", secret.Props["allow_http"]) + + // without explicit keys, the upstream credential chain is used + fallback := newTestLanceDBConn(t, map[string]string{"path": "s3://my-bucket/lancedb"}) + require.NoError(t, fallback.Init()) + secret, ok = fallback.makeSecret() + require.True(t, ok) + assert.Equal(t, "credential_chain", secret.Props["provider"]) + assert.NotContains(t, secret.Props, "access_key_id") + + azure := newTestLanceDBConn(t, map[string]string{ + "path": "az://my-container/lancedb", + "azure_account_name": "acct", + "azure_account_key": "a2V5", + "azure_sas_token": "?sv=2024", + "azure_tenant_id": "tenant", + "azure_client_id": "client", + "azure_client_secret": "client-secret", + }) + require.NoError(t, azure.Init()) + secret, ok = azure.makeSecret() + require.True(t, ok) + assert.Equal(t, "az://my-container/", secret.Props["scope"]) + assert.Equal(t, "acct", secret.Props["account_name"]) + assert.Equal(t, "a2V5", secret.Props["account_key"]) + + // SAS is only used when there is no account key + sas := newTestLanceDBConn(t, map[string]string{ + "path": "abfss://container@acct.dfs.core.windows.net/lancedb", + "azure_account_name": "acct", + "azure_sas_token": "?sv=2024", + }) + require.NoError(t, sas.Init()) + secret, ok = sas.makeSecret() + require.True(t, ok) + assert.Equal(t, "abfss://container@acct.dfs.core.windows.net/", secret.Props["scope"]) + assert.Equal(t, "sv=2024", secret.Props["sas_token"]) + assert.NotContains(t, secret.Props, "account_key") + + gcs := newTestLanceDBConn(t, map[string]string{ + "path": "gs://my-bucket/lancedb", + "gcs_key_file": "/tmp/gcs.json", + }) + require.NoError(t, gcs.Init()) + secret, ok = gcs.makeSecret() + require.True(t, ok) + assert.Equal(t, "config", secret.Props["provider"]) + assert.Equal(t, "/tmp/gcs.json", secret.Props["google_application_credentials"]) +} + +// Object stores outside the s3 / gs / az families resolve their own +// credentials, so no secret is registered for them. +func TestLanceDBConnSecretSkippedForOtherStores(t *testing.T) { + for _, path := range []string{"oss://my-bucket/lancedb", "hf://datasets/org/repo", "r2://my-bucket/lancedb"} { + conn := newTestLanceDBConn(t, map[string]string{"path": path}) + require.NoError(t, conn.Init()) + _, ok := conn.makeSecret() + assert.False(t, ok, path) + } +} diff --git a/core/dbio/templates/lancedb.yaml b/core/dbio/templates/lancedb.yaml new file mode 100755 index 000000000..b096967d2 --- /dev/null +++ b/core/dbio/templates/lancedb.yaml @@ -0,0 +1,395 @@ +# LanceDB template. LanceDB datasets are served by the DuckDB `lance` +# extension: the namespace root is ATTACHed as a DuckDB catalog, so the SQL +# dialect, type mapping and merge strategies are DuckDB's. Metadata queries are +# scoped to `current_database()` (the attached namespace) so that the in-memory +# catalog holding sling's temp tables is not reported as part of the database. +# Note: the extension keeps `create view` in the session only (nothing is written +# into the namespace, and the Lance namespace spec has no view operations), so a +# view does not survive the connection that created it. +core: + drop_table: drop table if exists {table} + drop_view: drop view if exists {view} + drop_index: drop index if exists {index} + create_index: create index {index} on {table} ({cols}) + create_unique_index: create unique index {index} on {table} ({cols}) + create_table: create table if not exists {table} ({col_types}) + create_temporary_table: create temp table if not exists {table} ({col_types}) + replace: replace into {table} ({names}) values({values}) + truncate_table: delete from {table} + insert_option: "" + modify_column: 'alter {column} type {type}' + select_stream_scanner: select {fields} from {stream_scanner} {where} + export_to_local: | + COPY ( + select {select_expr} + from {table} + ) TO '{local_path}' + ( + format '{format}', overwrite true, {file_size_bytes_expr} {file_extension_expr} + compression '{compression}' + ) + export_to_local_partitions: | + COPY ( + select + {select_expr}, + {partition_expressions} + from {table} + ) TO '{local_path}' + ( + format '{format}', {file_size_bytes_expr} {file_extension_expr} + compression '{compression}', + overwrite true, + write_partition_columns {write_partition_columns}, + partition_by ( {partition_columns} ) + ) + + # The lance extension cannot plan DELETE or UPDATE statements that read a + # second table (`unsupported DELETE plan: expected 1 child, got 2`), so every + # strategy is expressed as a single MERGE INTO, which it does support. + merge_insert: | + MERGE INTO {tgt_table} AS tgt + USING {src_table} AS src + ON {src_tgt_pk_equal} + WHEN NOT MATCHED THEN INSERT ({insert_fields}) VALUES ({src_insert_fields}) + + merge_update: | + MERGE INTO {tgt_table} AS tgt + USING {src_table} AS src + ON {src_tgt_pk_equal} + WHEN MATCHED THEN UPDATE SET {set_fields} + + merge_update_insert: | + MERGE INTO {tgt_table} AS tgt + USING {src_table} AS src + ON {src_tgt_pk_equal} + WHEN MATCHED THEN UPDATE SET {set_fields} + WHEN NOT MATCHED THEN INSERT ({insert_fields}) VALUES ({src_insert_fields}) + + # matched rows are updated in place rather than deleted and re-inserted: the + # DELETE ... WHERE EXISTS form cannot be planned by the lance extension, and + # the end state is the same (every non-PK column is overwritten from source). + merge_delete_insert: | + MERGE INTO {tgt_table} AS tgt + USING {src_table} AS src + ON {src_tgt_pk_equal} + WHEN MATCHED THEN UPDATE SET {set_fields} + WHEN NOT MATCHED THEN INSERT ({insert_fields}) VALUES ({src_insert_fields}) + + merge_change_capture: | + MERGE INTO {tgt_table} AS tgt + USING ( + SELECT * FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY {pk_fields} ORDER BY _sling_cdc_seq DESC) as _rn + FROM {src_table} + ) WHERE _rn = 1 + ) AS src + ON {src_tgt_pk_equal} + WHEN MATCHED AND src._sling_synced_op = 'D' THEN DELETE + WHEN MATCHED THEN UPDATE SET {set_fields} + WHEN NOT MATCHED THEN INSERT ({insert_fields}) VALUES ({src_insert_fields}) + + merge_change_capture_soft: | + MERGE INTO {tgt_table} AS tgt + USING ( + SELECT * FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY {pk_fields} ORDER BY _sling_cdc_seq DESC) as _rn + FROM {src_table} + ) WHERE _rn = 1 + ) AS src + ON {src_tgt_pk_equal} + WHEN MATCHED AND src._sling_synced_op = 'D' AND COALESCE(tgt._sling_synced_op, '') != 'D' + THEN UPDATE SET _sling_synced_at = CURRENT_TIMESTAMP, _sling_synced_op = 'D' + WHEN MATCHED AND src._sling_synced_op != 'D' THEN UPDATE SET {set_fields} + WHEN NOT MATCHED AND src._sling_synced_op != 'D' THEN INSERT ({insert_fields}) VALUES ({src_insert_fields}) + + # the lance extension does not support ALTER TABLE ADD CONSTRAINT + add_foreign_key: null + + # Column comment/description template + add_column_comment: COMMENT ON COLUMN {table}."{column}" IS {comment} + + # Table comment/description template + add_table_comment: COMMENT ON TABLE {table} IS {comment} + +metadata: + databases: PRAGMA database_list + + current_database: PRAGMA database_list + + schemas: | + select distinct schema_name + from information_schema.schemata + order by schema_name + + tables: | + select table_schema as schema_name, table_name, 'false' as is_view + from information_schema.tables + where table_type = 'BASE TABLE' + and table_catalog = current_database() + {{if .schema -}} and table_schema = '{schema}' {{- end}} + order by table_schema, table_name + + + views: | + select table_schema as schema_name, table_name, 'true' as is_view + from information_schema.tables + where table_type in ('VIEW') + and table_catalog = current_database() + {{if .schema -}} and table_schema = '{schema}' {{- end}} + order by table_schema, table_name + + columns: | + select column_name, data_type, coalesce(numeric_precision, character_maximum_length) as precision, numeric_scale as scale + from information_schema.columns + where table_schema = '{schema}' + and table_name = '{table}' + order by ordinal_position + + primary_keys: | + select '{table}.key' as pk_name, + constraint_index as position, + replace(replace(constraint_text, 'PRIMARY KEY(', ''), ')', '') as column_name + from duckdb_constraints() + where schema_name = '{schema}' + and table_name = '{table}' + and constraint_type = 'PRIMARY KEY' + + # Use expressions column cast to array and unnest to extract column names + indexes: | + SELECT + index_name, + unnest(expressions::VARCHAR[]) AS column_name, + CASE WHEN is_unique THEN 'true' ELSE 'false' END AS is_unique + FROM duckdb_indexes() + WHERE schema_name = '{schema}' + AND table_name = '{table}' + + columns_full: | + with tables_cte as ( + select + table_catalog, + table_schema, + table_name, + case table_type + when 'VIEW' then true + when 'FOREIGN' then true + else false + end as is_view + from information_schema.tables + where table_schema = '{schema}' + and table_name = '{table}' + ) + select + cols.table_schema as schema_name, + cols.table_name as table_name, + cols.column_name as column_name, + cols.data_type as data_type, + cols.ordinal_position as position + from information_schema.columns cols + join tables_cte + on tables_cte.table_schema = cols.table_schema + and tables_cte.table_name = cols.table_name + order by cols.table_catalog, cols.table_schema, cols.table_name, cols.ordinal_position + + schemata: | + with tables_cte as ( + select + table_catalog, + table_schema, + table_name, + case table_type + when 'VIEW' then true + else false + end as is_view + from information_schema.tables + where table_catalog = current_database() + {{if .schema -}} and table_schema = '{schema}' {{- end}} + {{if .tables -}} and table_name in ({tables}) {{- end}} + ) + select + cols.table_schema as schema_name, + cols.table_name as table_name, + tables_cte.is_view as is_view, + cols.column_name as column_name, + cols.data_type as data_type, + cols.ordinal_position as position + from information_schema.columns cols + join tables_cte + on tables_cte.table_schema = cols.table_schema + and tables_cte.table_name = cols.table_name + order by cols.table_catalog, cols.table_schema, cols.table_name, cols.ordinal_position + + ddl_table: | + PRAGMA table_info('{schema}.{table}') + + ddl_view: | + PRAGMA table_info('{schema}.{table}') + + # Extended column attributes for schema migration + columns_extended: | + SELECT + c.column_name, + CASE WHEN c.is_nullable = 'YES' THEN 'true' ELSE 'false' END AS is_nullable, + c.column_default AS default_value, + 'false' AS is_auto_increment, + '1' AS identity_seed, + '1' AS identity_increment, + CASE WHEN pk.column_name IS NOT NULL THEN 'true' ELSE 'false' END AS is_primary_key, + CAST(NULL AS VARCHAR) AS description + FROM information_schema.columns c + LEFT JOIN ( + SELECT replace(replace(constraint_text, 'PRIMARY KEY(', ''), ')', '') AS column_name + FROM duckdb_constraints() + WHERE schema_name = '{schema}' + AND table_name = '{table}' + AND constraint_type = 'PRIMARY KEY' + ) pk ON lower(pk.column_name) = lower(c.column_name) + WHERE c.table_schema = '{schema}' AND c.table_name = '{table}' + ORDER BY c.ordinal_position + + # Foreign key relationships (DuckDB enforces these!) + # Uses native constraint columns instead of regex parsing for reliability + foreign_keys: | + SELECT + dc.constraint_name, + unnest(dc.constraint_column_names) AS column_name, + split_part(dc.constraint_text, 'REFERENCES ', 2)::VARCHAR AS ref_part, + COALESCE(regexp_extract(dc.constraint_text, 'ON DELETE ([A-Z ]+)', 1), 'NO ACTION') AS on_delete, + COALESCE(regexp_extract(dc.constraint_text, 'ON UPDATE ([A-Z ]+)', 1), 'NO ACTION') AS on_update, + unnest(dc.constraint_column_names) AS fk_column, + split_part(split_part(dc.constraint_text, 'REFERENCES ', 2), '.', 1) AS referenced_schema, + split_part(split_part(split_part(dc.constraint_text, 'REFERENCES ', 2), '.', 2), '(', 1) AS referenced_table, + unnest(regexp_extract_all(dc.constraint_text, 'REFERENCES [^(]+\(([^)]+)\)')) AS referenced_column + FROM duckdb_constraints() dc + WHERE dc.schema_name = '{schema}' + AND dc.table_name = '{table}' + AND dc.constraint_type = 'FOREIGN KEY' + + # Extended table attributes for schema migration + # Note: DuckDB comments are retrieved from duckdb_tables() + table_extended: | + SELECT CAST(NULL AS VARCHAR) AS description + +analysis: + chars: | + select + '{schema}' as schema_nm, + '{table}' as table_nm, + '{field}' as field, sum(case when {field}::text ~ '\n' then 1 else 0 end) as cnt_nline, + sum(case when {field}::text ~ '\t' then 1 else 0 end) as cnt_tab, + sum(case when {field}::text ~ ',' then 1 else 0 end) as cnt_comma, + sum(case when {field}::text ~ '"' then 1 else 0 end) as cnt_dquote, + min(length({field}::text)) as f_min_len, + max(length({field}::text)) as f_max_len + from "{schema}"."{table}" + + fields: | + fields_deep: | + fields_distro: | + fields_distro_group: | + fields_date_distro: | + fields_date_distro_wide: | + fields_group: | + +function: + sleep: select sqlite3_sleep({seconds}*1000) + checksum_datetime: CAST((epoch({field}) || substr(strftime({field}, '%f'),4) ) as bigint) + checksum_decimal: 'abs(cast({field} as bigint))' + checksum_boolean: 'length({field}::string)' + cast_to_text: 'cast({field} as text)' + + iceberg_scanner: iceberg_scan('{uri}', allow_moved_paths = true) + delta_scanner: delta_scan('{uri}') + parquet_scanner: read_parquet([{uris}]{filename_expr}) + # csv_scanner: read_csv('{uri}', delim='{delimiter}', header={header}, columns={columns}, max_line_size=2000000, parallel=true, quote='{quote}', escape='{escape}', nullstr='{null_if}') + csv_scanner: read_csv([{uris}], delim='{delimiter}', header={header}, max_line_size=2000000, parallel=true, quote='{quote}', escape='{escape}', nullstr='{null_if}'{filename_expr}) + +variable: + # DuckDB default is delete_insert as it's more reliable + default_merge_strategy: delete_insert + bool_as: integer + bind_string: ${c} + batch_rows: 50 + batch_values: 1000 + timestamp_layout: '2006-01-02 15:04:05.000000' + timestampz_layout: '2006-01-02 15:04:05.000000-07:00' + max_string_type: text + max_string_length: 2147483647 + +native_type_map: + bigint: bigint + binary: binary + blob: binary + boolean: bool + char: string + date: date + datetime: datetime + decimal: decimal + double: float + enum: string + float: float + geometry: geometry + hugeint: bigint + integer: integer + interval: string + json: json + list: text + map: json + smallint: smallint + struct: json + text: text + time: time + timestamp: timestamp + "timestamp with time zone": timestampz + tinyblob: text + tinyint: smallint + ubigint: bigint + uinteger: bigint + usmallint: integer + utinyint: integer + uuid: uuid + varchar: text + +general_type_map: + bigint: bigint + binary: binary + bool: bool + date: date + datetime: datetime + decimal: "decimal(,)" + float: double + geometry: geometry + integer: integer + json: json + smallint: smallint + string: "varchar()" + text: text + time: time + timestamp: timestamp + timestampz: timestamptz + timez: time + uuid: uuid + +# Schema migration: default value translation between native and generalized forms +default_value_map: + to_general: + "current_timestamp": "current_timestamp" + "CURRENT_TIMESTAMP": "current_timestamp" + "now()": "current_timestamp" + "current_date": "current_date" + "CURRENT_DATE": "current_date" + "current_time": "current_time" + "CURRENT_TIME": "current_time" + "gen_random_uuid()": "uuid()" + "uuid()": "uuid()" + "true": "true" + "false": "false" + from_general: + "current_timestamp": "current_timestamp" + "current_timestamp_utc": "current_timestamp" + "current_date": "current_date" + "current_time": "current_time" + "uuid()": "gen_random_uuid()" + "true": "true" + "false": "false" + "null": "NULL" diff --git a/tests/replications/r.128.lancedb_full_refresh.yaml b/tests/replications/r.128.lancedb_full_refresh.yaml new file mode 100644 index 000000000..8081c6aeb --- /dev/null +++ b/tests/replications/r.128.lancedb_full_refresh.yaml @@ -0,0 +1,12 @@ +# LanceDB full refresh from a local CSV. CLI test 617 creates the LANCEDB_TEST +# connection (a temp namespace directory) before running this file. +source: LOCAL +target: LANCEDB_TEST + +defaults: + mode: full-refresh + +streams: + file://tests/files/test1.csv: + object: main.lancedb_cli_test + primary_key: [id] diff --git a/tests/replications/r.129.lancedb_merge.yaml b/tests/replications/r.129.lancedb_merge.yaml new file mode 100644 index 000000000..094200ed9 --- /dev/null +++ b/tests/replications/r.129.lancedb_merge.yaml @@ -0,0 +1,16 @@ +# LanceDB incremental merge (upsert) from a local CSV, using the default +# delete_insert strategy, which the lance extension serves as MERGE INTO. +source: LOCAL +target: LANCEDB_TEST + +defaults: + mode: incremental + primary_key: id + update_key: create_dt + target_options: + add_new_columns: true + use_bulk: true + +streams: + file://tests/files/test1.upsert.csv: + object: main.lancedb_cli_test diff --git a/tests/replications/r.130.lancedb_cdc_seed.yaml b/tests/replications/r.130.lancedb_cdc_seed.yaml new file mode 100644 index 000000000..8d2e4e2c3 --- /dev/null +++ b/tests/replications/r.130.lancedb_cdc_seed.yaml @@ -0,0 +1,16 @@ +# LanceDB seed for the synthetic CDC test: a SQL stream from the LanceDB +# connection itself, so the test needs no other connection. +source: LANCEDB_TEST +target: LANCEDB_TEST + +defaults: + mode: full-refresh + +streams: + seed: + sql: | + SELECT 1 as id, 'Alice' as name, 100 as amount, '2020-01-01 00:00:00'::timestamp as updated_at, 'S' as _sling_synced_op, 0::bigint as _sling_cdc_seq + UNION ALL SELECT 2, 'Bob', 200, '2020-01-01 00:00:00'::timestamp, 'S', 0 + UNION ALL SELECT 3, 'Charlie', 300, '2020-01-01 00:00:00'::timestamp, 'S', 0 + object: main.lancedb_cdc_test + primary_key: [id] diff --git a/tests/replications/r.131.lancedb_cdc.yaml b/tests/replications/r.131.lancedb_cdc.yaml new file mode 100644 index 000000000..7a8a8da35 --- /dev/null +++ b/tests/replications/r.131.lancedb_cdc.yaml @@ -0,0 +1,20 @@ +# LanceDB synthetic CDC: events in _sling_synced_op / _sling_cdc_seq applied +# with the change_capture merge strategy (MERGE INTO). Expected result: +# id=1 updated (Alice Final), id=2 deleted, id=3 unchanged, id=5 inserted. +source: LANCEDB_TEST +target: LANCEDB_TEST + +defaults: + mode: incremental + primary_key: id + +streams: + cdc_events: + sql: | + SELECT 1 as id, 'Alice Final' as name, 175 as amount, '2022-01-01 00:00:00'::timestamp as updated_at, 'U' as _sling_synced_op, 4::bigint as _sling_cdc_seq + UNION ALL SELECT 1, 'Alice Updated', 150, '2022-01-01 00:00:00'::timestamp, 'U', 1 + UNION ALL SELECT 2, 'Bob', 200, '2022-01-01 00:00:00'::timestamp, 'D', 2 + UNION ALL SELECT 5, 'Eve', 500, '2022-01-01 00:00:00'::timestamp, 'I', 3 + object: main.lancedb_cdc_test + target_options: + merge_strategy: change_capture From 4e5a28cacd719964dfa5994f751b815c04dc8e97 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Mon, 21 Sep 2026 13:13:11 -0300 Subject: [PATCH 25/56] fix(gcs): pass ADC via HTTP client to avoid credential clash storage.NewClient internally appends option.WithAuthCredentials for the credentials it resolves itself. With google.golang.org/api v0.258.0 any additional credential option sling passed (WithCredentials, WithCredentialsFile, WithTokenSource) was counted as a second one, failing GCS/BigQuery gc_bucket connections with "dialing: multiple credential options provided". Pass an oauth2-backed HTTP client instead: an authenticated HTTP client is not treated as a credential option, and Application Default Credentials still flow through unchanged. Add TestFileSysGoogleADC covering a GCS client built with no explicit credential props, which is the path BigQuery with gc_bucket takes. --- core/dbio/filesys/fs_google.go | 11 +++++++---- core/dbio/filesys/fs_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/core/dbio/filesys/fs_google.go b/core/dbio/filesys/fs_google.go index 768a02f34..8db38d6e2 100644 --- a/core/dbio/filesys/fs_google.go +++ b/core/dbio/filesys/fs_google.go @@ -11,6 +11,7 @@ import ( "github.com/samber/lo" "github.com/slingdata-io/sling-cli/core/dbio/iop" "github.com/spf13/cast" + "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/iterator" "google.golang.org/api/option" @@ -100,10 +101,12 @@ func (fs *GoogleFileSysClient) Connect() (err error) { if err != nil { return g.Error(err, "No Google credentials provided or could not find Application Default Credentials.") } - // Do NOT use option.WithCredentials — storage.NewClient appends - // WithAuthCredentials internally (google.golang.org/api >= v0.258.0) and - // collides with "multiple credential options provided". - authOption = option.WithTokenSource(creds.TokenSource) + // Do NOT pass any credential option (WithCredentials / WithCredentialsFile / + // WithTokenSource): storage.NewClient calls AuthCreds internally and appends + // WithAuthCredentials, which google.golang.org/api v0.258.0 counts as a second + // credential option, yielding "multiple credential options provided". + // An authenticated HTTP client is not counted as a credential option. + authOption = option.WithHTTPClient(oauth2.NewClient(fs.Context().Ctx, creds.TokenSource)) } fs.bucket = fs.GetProp("BUCKET") diff --git a/core/dbio/filesys/fs_test.go b/core/dbio/filesys/fs_test.go index 005aff5a4..fd2adaa46 100755 --- a/core/dbio/filesys/fs_test.go +++ b/core/dbio/filesys/fs_test.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "testing" "time" @@ -1002,6 +1003,31 @@ func TestFileSysAzure(t *testing.T) { // Delete(fs, writeFolderPath) } +// TestFileSysGoogleADC reproduces https://github.com/slingdata-io/sling-cli/issues/808 +// A BigQuery connection with `gc_bucket` set creates a GCS client without any explicit +// credential props, so it must fall back to Application Default Credentials. +// Regression for "dialing: multiple credential options provided": storage.NewClient +// appends WithAuthCredentials internally, which google.golang.org/api v0.258.0 counts +// as a second credential option when sling also passes one. +func TestFileSysGoogleADC(t *testing.T) { + // fake Application Default Credentials (no token is ever fetched) + adc := `{"type":"authorized_user","client_id":"fake.apps.googleusercontent.com","client_secret":"fake","refresh_token":"fake"}` + adcDir := t.TempDir() + adcFile := filepath.Join(adcDir, "application_default_credentials.json") + if !assert.NoError(t, os.WriteFile(adcFile, []byte(adc), 0600)) { + return + } + t.Setenv("CLOUDSDK_CONFIG", adcDir) + t.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "") + + fs, err := NewFileSysClient(dbio.TypeFileGoogle, "BUCKET=some_bucket") + if !assert.NoError(t, err) { + return + } + defer fs.Close() + assert.NotNil(t, fs.Client()) +} + func TestFileSysGoogle(t *testing.T) { t.Parallel() From e8c34ce3023f3b0bb7cad24bd1c2378f1edcb4a5 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Mon, 21 Sep 2026 13:15:53 -0300 Subject: [PATCH 26/56] Guard dev build trigger to version branches only - The workflow-dispatch step pushed builds for any branch, so non-version branches were pushed to slingdata-io/sling. - Add a check that only proceeds for `v...` branch names, matching the repo's version-branch push convention. --- .github/workflows/build-dev.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-dev.yml b/.github/workflows/build-dev.yml index 278e0f056..6b7e0dad8 100644 --- a/.github/workflows/build-dev.yml +++ b/.github/workflows/build-dev.yml @@ -12,8 +12,20 @@ jobs: trigger: runs-on: ubuntu-latest steps: - + + - name: Check Version Branch + id: check_version_branch + run: | + # Only version branches (v...) are pushed to slingdata-io/sling + if [[ "${GITHUB_REF_NAME}" =~ ^v[0-9] ]]; then + echo "eligible=true" >> "$GITHUB_OUTPUT" + else + echo "eligible=false" >> "$GITHUB_OUTPUT" + echo "Skipping trigger: '${GITHUB_REF_NAME}' is not a version branch" + fi + - name: Trigger Dev Build on Specific Branch + if: steps.check_version_branch.outputs.eligible == 'true' uses: benc-uk/workflow-dispatch@v1 with: token: ${{ secrets.REPO_ACCESS_TOKEN }} From f7afc4200e5d64fd93e3cf6d75501511ba4b8d79 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Mon, 21 Sep 2026 16:04:11 -0300 Subject: [PATCH 27/56] Add Firebolt database connector - Implement FireboltConn with a native database/sql driver that speaks SQL over HTTP, streaming JSONLines_Compact results, tracking session/transaction params from response headers, and encoding bind vars as query_parameters - Register TypeDbFirebolt across dbio types, connection URL templating, index capabilities, and SQL templates (firebolt.yaml) - Add connection properties, README entry, and test suite wiring; Firebolt is column-store with no plain secondary indexes - Remove obsolete overlaySpecConn test --- README.md | 2 +- cmd/sling/sling_assist_test.go | 52 -- cmd/sling/sling_test.go | 6 + core/dbio/connection/connection.go | 10 + core/dbio/database/database.go | 2 + core/dbio/database/database_firebolt.go | 800 ++++++++++++++++++++++++ core/dbio/database/schemata_keys.go | 5 +- core/dbio/dbio_types.go | 11 +- core/dbio/templates/_properties.yaml | 48 ++ core/dbio/templates/firebolt.yaml | 226 +++++++ 10 files changed, 1105 insertions(+), 57 deletions(-) create mode 100644 core/dbio/database/database_firebolt.go create mode 100644 core/dbio/templates/firebolt.yaml diff --git a/README.md b/README.md index a795dd894..67c6bb845 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Example [Replication](https://docs.slingdata.io/sling-cli/run/configuration/repl --- Available Connectors: -- **Databases**: [`adbc`](https://docs.slingdata.io/connections/database-connections/adbc) [`azuredwh`](https://docs.slingdata.io/connections/database-connections/azuredwh) [`azuresql`](https://docs.slingdata.io/connections/database-connections/azuresql) [`azuretable`](https://docs.slingdata.io/connections/database-connections/azuretable) [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`d1`](https://docs.slingdata.io/connections/database-connections/d1) [`databricks`](https://docs.slingdata.io/connections/database-connections/databricks) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`elasticsearch`](https://docs.slingdata.io/connections/database-connections/elasticsearch) [`exasol`](https://docs.slingdata.io/connections/database-connections/exasol) [`fabric`](https://docs.slingdata.io/connections/database-connections/fabric) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`mongodb`](https://docs.slingdata.io/connections/database-connections/mongodb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`odbc`](https://docs.slingdata.io/connections/database-connections/odbc) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`scylladb`](https://docs.slingdata.io/connections/database-connections/scylladb) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`trino`](https://docs.slingdata.io/connections/database-connections/trino) +- **Databases**: [`adbc`](https://docs.slingdata.io/connections/database-connections/adbc) [`azuredwh`](https://docs.slingdata.io/connections/database-connections/azuredwh) [`azuresql`](https://docs.slingdata.io/connections/database-connections/azuresql) [`azuretable`](https://docs.slingdata.io/connections/database-connections/azuretable) [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`d1`](https://docs.slingdata.io/connections/database-connections/d1) [`databricks`](https://docs.slingdata.io/connections/database-connections/databricks) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`elasticsearch`](https://docs.slingdata.io/connections/database-connections/elasticsearch) [`exasol`](https://docs.slingdata.io/connections/database-connections/exasol) [`fabric`](https://docs.slingdata.io/connections/database-connections/fabric) [`firebolt`](https://docs.slingdata.io/connections/database-connections/firebolt) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`mongodb`](https://docs.slingdata.io/connections/database-connections/mongodb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`odbc`](https://docs.slingdata.io/connections/database-connections/odbc) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`scylladb`](https://docs.slingdata.io/connections/database-connections/scylladb) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`trino`](https://docs.slingdata.io/connections/database-connections/trino) - **Data Lakes**:[`athena`](https://docs.slingdata.io/connections/datalake-connections/athena) [`Ducklake`](https://docs.slingdata.io/connections/datalake-connections/ducklake) [`iceberg`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`S3 Tables`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`lancedb`](https://docs.slingdata.io/connections/datalake-connections/lancedb) diff --git a/cmd/sling/sling_assist_test.go b/cmd/sling/sling_assist_test.go index 5542fa1f3..82b166dd8 100644 --- a/cmd/sling/sling_assist_test.go +++ b/cmd/sling/sling_assist_test.go @@ -5,10 +5,6 @@ import ( "path/filepath" "strings" "testing" - - "github.com/flarco/g" - "github.com/slingdata-io/sling-cli/core/dbio" - "github.com/slingdata-io/sling-cli/core/dbio/connection" ) func TestAssistBrowseFlagRemoved(t *testing.T) { @@ -193,51 +189,3 @@ func TestParseKVListUnquotedStillSplits(t *testing.T) { t.Fatalf("%v", got) } } - -func TestOverlaySpecConn(t *testing.T) { - dir := t.TempDir() - specPath := filepath.Join(dir, "draft.yaml") - if err := os.WriteFile(specPath, []byte("name: draft\n"), 0o644); err != nil { - t.Fatal(err) - } - - apiConn, err := connection.NewConnection("MY_API", dbio.TypeApi, g.M("type", "api", "spec", "baseline")) - if err != nil { - t.Fatal(err) - } - otherConn, err := connection.NewConnection("LOCAL", dbio.TypeFileLocal, g.M("type", "file")) - if err != nil { - t.Fatal(err) - } - entries := connection.ConnEntries{ - {Name: "MY_API", Connection: apiConn}, - {Name: "LOCAL", Connection: otherConn}, - } - - out, err := overlaySpecConn(entries, "MY_API", "draft.yaml", dir) - if err != nil { - t.Fatal(err) - } - - got := out.Get("MY_API").Connection.Data["spec"] - want := "file://" + specPath - if got != want { - t.Fatalf("overlay spec=%q want %q", got, want) - } - // original entries stay untouched - if entries.Get("MY_API").Connection.Data["spec"] != "baseline" { - t.Fatalf("original entry was mutated") - } - if out.Get("LOCAL").Connection.Data["spec"] != nil { - t.Fatalf("unrelated entry changed") - } - - // missing file errors - if _, err := overlaySpecConn(entries, "MY_API", "nope.yaml", dir); err == nil { - t.Fatal("expected error for missing spec file") - } - // unknown connection errors - if _, err := overlaySpecConn(entries, "NOPE", specPath, dir); err == nil { - t.Fatal("expected error for unknown connection") - } -} diff --git a/cmd/sling/sling_test.go b/cmd/sling/sling_test.go index 999ef12ec..7b3d09010 100755 --- a/cmd/sling/sling_test.go +++ b/cmd/sling/sling_test.go @@ -122,6 +122,7 @@ var connMap = map[dbio.Type]connTest{ dbio.TypeDbPrometheus: {name: "prometheus", schema: "prometheus"}, dbio.TypeDbProton: {name: "proton", schema: "default", useBulk: g.Bool(true)}, dbio.TypeDbScyllaDB: {name: "scylladb", schema: "sling"}, + dbio.TypeDbFirebolt: {name: "firebolt", schema: "sling_test"}, dbio.TypeFileLocal: {name: "local"}, dbio.TypeFileSftp: {name: "sftp"}, @@ -1312,6 +1313,11 @@ func TestSuiteDatabaseScylladb(t *testing.T) { testSuite(t, dbio.TypeDbScyllaDB, "1,3-9,17,20,23-26") } +func TestSuiteDatabaseFirebolt(t *testing.T) { + t.Parallel() + testSuite(t, dbio.TypeDbFirebolt) +} + // rewriteScyllaDropSQL: add IF EXISTS and quote identifiers func rewriteScyllaDropSQL(sql string) string { parts := strings.Split(sql, ";") diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index 13ce13cac..ac575d6a8 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -1099,6 +1099,16 @@ func (c *Connection) setURL() (err error) { setIfMissing("port", c.Type.DefPort()) setIfMissing("keyspace", "") template = "scylladb://{username}:{password}@{host}:{port}/{keyspace}" + case dbio.TypeDbFirebolt: + // Firebolt Core has no authentication; username/password stay optional + setIfMissing("username", c.Data["user"]) + setIfMissing("password", "") + setIfMissing("port", c.Type.DefPort()) + setIfMissing("database", "firebolt") + setIfMissing("schema", "public") + setIfMissing("secure", "false") + setIfMissing("skip_verify", "false") + template = "firebolt://{username}:{password}@{host}:{port}/{database}?secure={secure}&skip_verify={skip_verify}" case dbio.TypeFileSftp, dbio.TypeFileFtp: setIfMissing("password", "") setIfMissing("port", c.Type.DefPort()) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index 5156a5456..828b34e8b 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -330,6 +330,8 @@ func NewConnContext(ctx context.Context, URL string, props ...string) (Connectio conn = &ODBCConn{URL: URL} } else if strings.HasPrefix(URL, "scylladb:") { conn = &ScyllaDBConn{URL: URL} + } else if strings.HasPrefix(URL, "firebolt:") { + conn = &FireboltConn{URL: URL} } else { conn = &BaseConn{URL: URL} } diff --git a/core/dbio/database/database_firebolt.go b/core/dbio/database/database_firebolt.go new file mode 100644 index 000000000..f14e8efb1 --- /dev/null +++ b/core/dbio/database/database_firebolt.go @@ -0,0 +1,800 @@ +package database + +import ( + "bufio" + "context" + "crypto/tls" + "database/sql" + "database/sql/driver" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/spf13/cast" +) + +const fireboltDriverName = "firebolt" + +// Firebolt protocol constants. Each engine accepts SQL over HTTP: the request +// body is a single SQL statement, results come back as JSONLines_Compact so +// they can be streamed without buffering the whole result set. +const ( + fireboltOutputFormat = "JSONLines_Compact" + fireboltNullSuffix = " null" + fireboltUpdateParams = "Firebolt-Update-Parameters" + fireboltRemoveParams = "Firebolt-Remove-Parameters" + fireboltQueryParamsKV = "query_parameters" +) + +func init() { + sql.Register(fireboltDriverName, &fireboltDriver{}) +} + +// FireboltConn is a Firebolt connection +type FireboltConn struct { + BaseConn + URL string +} + +// Init initiates the connection +func (conn *FireboltConn) Init() error { + conn.BaseConn.URL = conn.URL + conn.BaseConn.Type = dbio.TypeDbFirebolt + conn.BaseConn.defaultPort = 3473 + + instance := Connection(conn) + conn.BaseConn.instance = &instance + + return conn.BaseConn.Init() +} + +// ---------------------------------------------------------------- driver + +type fireboltDriver struct{} + +func (d *fireboltDriver) Open(dsn string) (driver.Conn, error) { + return newFireboltConn(dsn) +} + +type fireboltConn struct { + client *http.Client + baseURL string // scheme://host:port + params map[string]string + mux sync.Mutex +} + +func newFireboltConn(dsn string) (*fireboltConn, error) { + u, err := url.Parse(dsn) + if err != nil { + return nil, g.Error(err, "could not parse Firebolt connection URL") + } + + host := u.Hostname() + if host == "" { + return nil, g.Error("Firebolt host is required") + } + port := u.Port() + if port == "" { + port = "3473" + } + + values := u.Query() + secure := cast.ToBool(values.Get("secure")) + scheme := "http" + if secure { + scheme = "https" + } + + conn := &fireboltConn{ + client: fireboltHTTPClient(secure, secure && cast.ToBool(values.Get("skip_verify"))), + baseURL: fmt.Sprintf("%s://%s", scheme, net.JoinHostPort(host, port)), + params: map[string]string{}, + } + + if database := strings.Trim(u.Path, "/"); database != "" { + conn.params["database"] = database + } + + return conn, nil +} + +// fireboltClients caches one HTTP client per TLS setting. The pool holds several +// physical connections per sling connection (a suite run peaks at ~10 concurrent +// sockets to the engine), and http.Client is safe for concurrent use, so a single +// client and its TCP pool serve them all rather than one Transport per session. +var ( + fireboltClientsMu sync.Mutex + fireboltClients = map[string]*http.Client{} +) + +func fireboltHTTPClient(secure, skipVerify bool) *http.Client { + key := fmt.Sprintf("%t|%t", secure, skipVerify) + + fireboltClientsMu.Lock() + defer fireboltClientsMu.Unlock() + + if client, ok := fireboltClients[key]; ok { + return client + } + + transport := &http.Transport{ + DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + MaxIdleConns: 16, + MaxIdleConnsPerHost: 16, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + } + if skipVerify { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + client := &http.Client{Transport: transport} + fireboltClients[key] = client + + return client +} + +func (c *fireboltConn) Prepare(query string) (driver.Stmt, error) { + return &fireboltStmt{conn: c, query: query}, nil +} + +func (c *fireboltConn) PrepareContext(_ context.Context, query string) (driver.Stmt, error) { + return c.Prepare(query) +} + +func (c *fireboltConn) Close() error { + // The HTTP client is shared process-wide (see fireboltHTTPClient), so a + // driver connection must not tear down its idle sockets: that would close + // the sockets of every other session in the pool. Idle connections expire + // on their own via the transport's IdleConnTimeout. + return nil +} + +func (c *fireboltConn) Begin() (driver.Tx, error) { + return c.BeginTx(context.Background(), driver.TxOptions{}) +} + +func (c *fireboltConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if opts.ReadOnly { + return nil, g.Error("Firebolt does not support read-only transactions") + } + // BEGIN TRANSACTION replies with the transaction id to pass on every + // following request of the same transaction. + if _, err := c.execContext(ctx, "BEGIN TRANSACTION", nil); err != nil { + return nil, g.Error(err, "could not begin Firebolt transaction") + } + c.mux.Lock() + txID := c.params["transaction_id"] + c.mux.Unlock() + if txID == "" { + return nil, g.Error("Firebolt did not return a transaction_id") + } + return &fireboltTx{conn: c}, nil +} + +func (c *fireboltConn) Ping(ctx context.Context) error { + _, err := c.execContext(ctx, "SELECT 1", nil) + return err +} + +// CheckNamedValue converts any value sling hands over into a Firebolt parameter +// value. Maps and slices (JSON columns) are marshalled to JSON text. +func (c *fireboltConn) CheckNamedValue(nv *driver.NamedValue) error { + switch nv.Value.(type) { + case nil, string, []byte, bool, time.Time, int64, float64: + return nil + case json.RawMessage: + nv.Value = string(nv.Value.(json.RawMessage)) + return nil + } + + val, err := driver.DefaultParameterConverter.ConvertValue(nv.Value) + if err != nil { + // fall back to JSON text for structured values + bytes, jErr := json.Marshal(nv.Value) + if jErr != nil { + return err + } + nv.Value = string(bytes) + return nil + } + nv.Value = val + return nil +} + +func (c *fireboltConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + return c.execContext(ctx, query, args) +} + +func (c *fireboltConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + return c.queryContext(ctx, query, args) +} + +func (c *fireboltConn) execContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + // Firebolt only accepts one statement per request + for _, sql := range ParseSQLMultiStatements(query, dbio.TypeDbFirebolt) { + if strings.TrimSpace(sql) == "" { + continue + } + resp, err := c.do(ctx, sql, args) + if err != nil { + return nil, err + } + _, err = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if err != nil { + return nil, g.Error(err, "error reading Firebolt response") + } + } + return fireboltResult{}, nil +} + +func (c *fireboltConn) queryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + resp, err := c.do(ctx, query, args) + if err != nil { + return nil, err + } + return newFireboltRows(c, resp) +} + +// do sends one statement and returns the streaming response. +func (c *fireboltConn) do(ctx context.Context, query string, args []driver.NamedValue) (*http.Response, error) { + c.mux.Lock() + values := url.Values{"output_format": {fireboltOutputFormat}} + for k, v := range c.params { + values.Set(k, v) + } + c.mux.Unlock() + + if len(args) > 0 { + paramsJSON, err := fireboltQueryParameters(args) + if err != nil { + return nil, err + } + values.Set(fireboltQueryParamsKV, paramsJSON) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/?"+values.Encode(), strings.NewReader(query)) + if err != nil { + return nil, g.Error(err, "could not build Firebolt request") + } + req.Header.Set("Content-Type", "text/plain; charset=utf-8") + req.Header.Set("User-Agent", "sling") + + resp, err := c.client.Do(req) + if err != nil { + return nil, g.Error(err, "could not reach Firebolt engine") + } + + c.applySessionHeaders(resp.Header) + + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return nil, g.Error("Firebolt SQL Error: %s", fireboltErrorFrom(body)) + } + + return resp, nil +} + +// applySessionHeaders keeps track of the connection parameters the engine asks +// the client to send on the following requests (database, transaction_id, +// transaction_sequence_id, ...). Both headers repeat once per parameter, so +// every value must be read: `header.Get` would only return the first one and +// leave a stale transaction_sequence_id behind. +func (c *fireboltConn) applySessionHeaders(header http.Header) { + updates := header.Values(fireboltUpdateParams) + removes := header.Values(fireboltRemoveParams) + if len(updates) == 0 && len(removes) == 0 { + return + } + + c.mux.Lock() + defer c.mux.Unlock() + for _, raw := range updates { + for _, pair := range strings.Split(raw, ",") { + kv := strings.SplitN(strings.TrimSpace(pair), "=", 2) + if len(kv) == 2 && kv[0] != "" { + c.params[kv[0]] = kv[1] + } + } + } + for _, raw := range removes { + for _, key := range strings.Split(raw, ",") { + delete(c.params, strings.TrimSpace(key)) + } + } +} + +// fireboltTx runs statements inside an explicit Firebolt transaction. +type fireboltTx struct { + conn *fireboltConn + done bool +} + +func (tx *fireboltTx) Commit() error { return tx.finish("COMMIT TRANSACTION") } +func (tx *fireboltTx) Rollback() error { return tx.finish("ROLLBACK TRANSACTION") } + +func (tx *fireboltTx) finish(statement string) error { + if tx.done { + return nil + } + tx.done = true + _, err := tx.conn.execContext(context.Background(), statement, nil) + return err +} + +type fireboltStmt struct { + conn *fireboltConn + query string +} + +func (s *fireboltStmt) Close() error { return nil } +func (s *fireboltStmt) NumInput() int { return -1 } // placeholders are server-side + +func (s *fireboltStmt) Exec(args []driver.Value) (driver.Result, error) { + return s.ExecContext(context.Background(), fireboltNamedValues(args)) +} + +func (s *fireboltStmt) Query(args []driver.Value) (driver.Rows, error) { + return s.QueryContext(context.Background(), fireboltNamedValues(args)) +} + +func (s *fireboltStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + return s.conn.execContext(ctx, s.query, args) +} + +func (s *fireboltStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + return s.conn.queryContext(ctx, s.query, args) +} + +func fireboltNamedValues(args []driver.Value) []driver.NamedValue { + named := make([]driver.NamedValue, len(args)) + for i, arg := range args { + named[i] = driver.NamedValue{Ordinal: i + 1, Value: arg} + } + return named +} + +type fireboltResult struct{} + +func (r fireboltResult) LastInsertId() (int64, error) { return 0, nil } +func (r fireboltResult) RowsAffected() (int64, error) { return 0, nil } + +// fireboltQueryParameters renders `$1, $2, ...` bind variables as the +// query_parameters JSON array the engine expects. +func fireboltQueryParameters(args []driver.NamedValue) (string, error) { + type param struct { + Name string `json:"name"` + Value any `json:"value"` + } + + params := make([]param, len(args)) + for i, arg := range args { + ordinal := arg.Ordinal + if ordinal == 0 { + ordinal = i + 1 + } + value, err := fireboltParamValue(arg.Value) + if err != nil { + return "", err + } + params[i] = param{Name: g.F("$%d", ordinal), Value: value} + } + + bytes, err := json.Marshal(params) + if err != nil { + return "", g.Error(err, "could not encode Firebolt query parameters") + } + return string(bytes), nil +} + +// fireboltParamValue converts a Go value into the string form the engine +// accepts as a query parameter (null is sent as JSON null). +func fireboltParamValue(value any) (any, error) { + switch v := value.(type) { + case nil: + return nil, nil + case string: + return v, nil + case []byte: + return "\\x" + hex.EncodeToString(v), nil + case bool: + if v { + return "true", nil + } + return "false", nil + case int64: + return strconv.FormatInt(v, 10), nil + case float64: + return strconv.FormatFloat(v, 'f', -1, 64), nil + case time.Time: + // date-only values (midnight UTC) are sent as dates, so a date column + // does not receive a time part + if v.Truncate(24 * time.Hour).Equal(v) { + return v.Format("2006-01-02"), nil + } + if _, offset := v.Zone(); offset != 0 { + return v.Format("2006-01-02 15:04:05.000000-07:00"), nil + } + return v.Format("2006-01-02 15:04:05.000000"), nil + case json.RawMessage: + return string(v), nil + case fmt.Stringer: + return v.String(), nil + } + + // structured values (maps, slices) are sent as JSON text + if bytes, err := json.Marshal(value); err == nil { + return string(bytes), nil + } + return nil, g.Error("unsupported Firebolt parameter type: %T", value) +} + +// ---------------------------------------------------------------- rows + +// fireboltRows streams the JSONLines_Compact response of one statement. +type fireboltRows struct { + conn *fireboltConn + resp *http.Response + reader *bufio.Reader + + columns []string + dbTypes []string + nullable []bool + lengths []int64 + precs []int64 + scales []int64 + + chunk [][]any + pos int + done bool + err error +} + +// fireboltMessages are the JSONLines_Compact message types +type fireboltMessage struct { + MessageType string `json:"message_type"` + ResultColumns []fireboltCol `json:"result_columns"` + Data [][]any `json:"data"` + Errors []fireboltErr `json:"errors"` + QueryID string `json:"query_id"` + QueryLabel *string `json:"query_label"` + RequestID string `json:"request_id"` + Statistics json.RawMessage `json:"statistics"` + UpdateEndpoint string `json:"update_endpoint"` +} + +type fireboltCol struct { + Name string `json:"name"` + Type string `json:"type"` +} + +type fireboltErr struct { + Code string `json:"code"` + Description string `json:"description"` +} + +func newFireboltRows(conn *fireboltConn, resp *http.Response) (*fireboltRows, error) { + rows := &fireboltRows{ + conn: conn, + resp: resp, + reader: bufio.NewReaderSize(resp.Body, 1024*1024), + } + + // the START message carries the column list and must be read eagerly so the + // driver can report columns before iterating + for { + msg, err := rows.readMessage() + if err != nil { + resp.Body.Close() + return nil, err + } else if msg == nil { + resp.Body.Close() + return nil, g.Error("Firebolt returned no result columns") + } + + switch msg.MessageType { + case "START": + rows.setColumns(msg.ResultColumns) + return rows, nil + case "FINISH_WITH_ERRORS", "ERROR": + resp.Body.Close() + return nil, g.Error("Firebolt SQL Error: %s", fireboltErrors(msg.Errors)) + case "FINISH_SUCCESSFULLY": + resp.Body.Close() + return rows, nil + } + } +} + +func (r *fireboltRows) setColumns(cols []fireboltCol) { + r.columns = make([]string, len(cols)) + r.dbTypes = make([]string, len(cols)) + r.nullable = make([]bool, len(cols)) + r.lengths = make([]int64, len(cols)) + r.precs = make([]int64, len(cols)) + r.scales = make([]int64, len(cols)) + + for i, col := range cols { + dbType := col.Type + if strings.HasSuffix(dbType, fireboltNullSuffix) { + dbType = strings.TrimSuffix(dbType, fireboltNullSuffix) + r.nullable[i] = true + } + + r.columns[i] = col.Name + r.dbTypes[i] = dbType + + if precision, scale, ok := fireboltDecimalSize(dbType); ok { + r.precs[i], r.scales[i] = precision, scale + } + } +} + +func (r *fireboltRows) Columns() []string { return r.columns } + +func (r *fireboltRows) Close() error { + if !r.done { + r.done = true + r.resp.Body.Close() + } + return nil +} + +func (r *fireboltRows) Next(dest []driver.Value) error { + for { + if r.err != nil { + return r.err + } else if r.pos < len(r.chunk) { + row := r.chunk[r.pos] + r.pos++ + for i := range dest { + if i >= len(row) { + dest[i] = nil + continue + } + value, err := fireboltRowValue(row[i], r.dbTypes[i]) + if err != nil { + r.err = err + return err + } + dest[i] = value + } + return nil + } else if r.done { + return io.EOF + } + + msg, err := r.readMessage() + if err != nil { + r.err = err + return err + } else if msg == nil { + r.done = true + return io.EOF + } + + switch msg.MessageType { + case "DATA": + r.chunk = msg.Data + r.pos = 0 + case "FINISH_WITH_ERRORS", "ERROR": + r.err = g.Error("Firebolt SQL Error: %s", fireboltErrors(msg.Errors)) + r.done = true + return r.err + case "FINISH_SUCCESSFULLY": + r.done = true + } + } +} + +func (r *fireboltRows) readMessage() (*fireboltMessage, error) { + for { + line, err := r.reader.ReadBytes('\n') + if len(line) > 0 { + line = []byte(strings.TrimSpace(string(line))) + if len(line) == 0 { + continue + } + msg := &fireboltMessage{} + if err := json.Unmarshal(line, msg); err != nil { + return nil, g.Error(err, "could not parse Firebolt response: %s", g.F("%.200s", line)) + } + return msg, nil + } + if err != nil { + if err == io.EOF { + return nil, nil + } + return nil, g.Error(err, "error reading Firebolt response") + } + } +} + +func (r *fireboltRows) ColumnTypeDatabaseTypeName(index int) string { + return r.dbTypes[index] +} + +func (r *fireboltRows) ColumnTypeNullable(index int) (nullable, ok bool) { + return r.nullable[index], true +} + +func (r *fireboltRows) ColumnTypeLength(index int) (length int64, ok bool) { + if r.lengths[index] > 0 { + return r.lengths[index], true + } + return 0, false +} + +func (r *fireboltRows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) { + precision, scale = r.precs[index], r.scales[index] + return precision, scale, precision > 0 +} + +// ---------------------------------------------------------------- values + +// fireboltRowValue converts a JSON value from the result stream into the Go +// value sling expects for the given Firebolt type. +func fireboltRowValue(raw any, dbType string) (driver.Value, error) { + if raw == nil { + return nil, nil + } + + switch fireboltBaseType(dbType) { + case "bigint", "long", "hugeint", "integer", "int", "smallint", "tinyint": + switch v := raw.(type) { + case string: + return strconv.ParseInt(strings.TrimSpace(v), 10, 64) + case float64: + return int64(v), nil + case bool: + return cast.ToInt64(v), nil + } + case "double", "double precision", "real", "float": + switch v := raw.(type) { + case string: + return strconv.ParseFloat(strings.TrimSpace(v), 64) + case float64: + return v, nil + case bool: + return cast.ToFloat64(v), nil + } + case "numeric", "decimal": + // decimals arrive as strings to preserve precision + return fireboltToString(raw), nil + case "boolean", "bool": + switch v := raw.(type) { + case bool: + return v, nil + case string: + return cast.ToBool(v), nil + case float64: + return v != 0, nil + } + case "date": + return fireboltParseTime(fireboltToString(raw), "2006-01-02") + case "timestamp", "timestampntz": + return fireboltParseTime(fireboltToString(raw), fireboltTimestampLayouts...) + case "timestamptz": + return fireboltParseTime(fireboltToString(raw), fireboltTimestampzLayouts...) + case "bytea": + return fireboltParseBytes(fireboltToString(raw)) + case "json", "array", "struct", "geography": + return fireboltJSONText(raw), nil + } + + switch v := raw.(type) { + case string, bool, float64, nil: + return v, nil + } + return fireboltToString(raw), nil +} + +var fireboltTimestampLayouts = []string{ + "2006-01-02 15:04:05.999999999", + "2006-01-02 15:04:05", + "2006-01-02T15:04:05.999999999", +} + +var fireboltTimestampzLayouts = []string{ + "2006-01-02 15:04:05.999999999-07:00", + "2006-01-02 15:04:05.999999999-07", + "2006-01-02 15:04:05-07:00", + "2006-01-02 15:04:05-07", + "2006-01-02T15:04:05.999999999Z07:00", +} + +func fireboltParseTime(value string, layouts ...string) (driver.Value, error) { + value = strings.TrimSpace(value) + for _, layout := range layouts { + if t, err := time.Parse(layout, value); err == nil { + return t, nil + } + } + // leave unparseable values untouched so nothing is silently lost + return value, nil +} + +func fireboltParseBytes(value string) (driver.Value, error) { + value = strings.TrimSpace(value) + if hexValue, ok := strings.CutPrefix(value, "\\x"); ok { + bytes, err := hex.DecodeString(hexValue) + if err != nil { + return nil, g.Error(err, "could not decode Firebolt bytea value") + } + return bytes, nil + } + return []byte(value), nil +} + +func fireboltJSONText(raw any) string { + if text, ok := raw.(string); ok { + return text + } else if bytes, err := json.Marshal(raw); err == nil { + return string(bytes) + } + return fmt.Sprint(raw) +} + +func fireboltToString(raw any) string { + if text, ok := raw.(string); ok { + return text + } + return fmt.Sprint(raw) +} + +// fireboltBaseType returns the type name without its parameters, so +// `numeric(38, 9)` and `array(integer null)` resolve to `numeric` / `array`. +func fireboltBaseType(dbType string) string { + base, _, _ := strings.Cut(strings.ToLower(strings.TrimSpace(dbType)), "(") + return strings.TrimSpace(base) +} + +func fireboltDecimalSize(dbType string) (precision, scale int64, ok bool) { + lowered := strings.ToLower(strings.TrimSpace(dbType)) + if !strings.HasPrefix(lowered, "numeric(") && !strings.HasPrefix(lowered, "decimal(") { + return 0, 0, false + } + inner, _, found := strings.Cut(strings.TrimPrefix(strings.TrimPrefix(lowered, "numeric("), "decimal("), ")") + if !found { + return 0, 0, false + } + parts := strings.Split(inner, ",") + if len(parts) != 2 { + return 0, 0, false + } + precision, errP := strconv.ParseInt(strings.TrimSpace(parts[0]), 10, 64) + scale, errS := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64) + if errP != nil || errS != nil { + return 0, 0, false + } + return precision, scale, true +} + +// fireboltErrorFrom extracts the error descriptions from a non-200 response. +func fireboltErrorFrom(body []byte) string { + msg := fireboltMessage{} + if err := json.Unmarshal(body, &msg); err == nil && len(msg.Errors) > 0 { + return fireboltErrors(msg.Errors) + } + return strings.TrimSpace(string(body)) +} + +func fireboltErrors(errs []fireboltErr) string { + descriptions := make([]string, len(errs)) + for i, e := range errs { + descriptions[i] = e.Description + } + return strings.Join(descriptions, "; ") +} diff --git a/core/dbio/database/schemata_keys.go b/core/dbio/database/schemata_keys.go index 28b05ea2b..11d4bf139 100644 --- a/core/dbio/database/schemata_keys.go +++ b/core/dbio/database/schemata_keys.go @@ -612,13 +612,16 @@ var indexCapabilities = map[dbio.Type]indexCapability{ dbio.TypeDbBigQuery: {noIndexes: true}, // StarRocks indexes (BITMAP/inverted) only apply to specific column/table // models and don't fit the generic CREATE INDEX form; no-op for now. - dbio.TypeDbStarRocks: {noIndexes: true}, + dbio.TypeDbStarRocks: {noIndexes: true}, dbio.TypeDbDuckDb: {supportsUnique: true}, dbio.TypeDbMotherDuck: {supportsUnique: true}, dbio.TypeDbDuckLake: {noIndexes: true}, // DuckLake does not support indexes dbio.TypeDbLanceDB: {noIndexes: true}, // Lance datasets have no secondary indexes dbio.TypeDbOracle: {supportsUnique: true, supportsType: true, typeClosedSet: []string{"bitmap"}}, dbio.TypeDbSQLite: {supportsWhere: true, supportsUnique: true}, + // Firebolt has no plain secondary index: only specialized FULL_TEXT / + // INVERTED_INDEX / HNSW / SKIP_INDEX, which don't fit the generic form. + dbio.TypeDbFirebolt: {noIndexes: true}, } var indexCapDefault = indexCapability{supportsUnique: true} diff --git a/core/dbio/dbio_types.go b/core/dbio/dbio_types.go index 03d464055..87a7f3260 100644 --- a/core/dbio/dbio_types.go +++ b/core/dbio/dbio_types.go @@ -92,6 +92,7 @@ const ( TypeDbArrowDBC Type = "adbc" TypeDbODBC Type = "odbc" TypeDbScyllaDB Type = "scylladb" + TypeDbFirebolt Type = "firebolt" ) var AllType = []struct { @@ -144,6 +145,7 @@ var AllType = []struct { {TypeDbArrowDBC, "TypeDbArrowDBC"}, {TypeDbODBC, "TypeDbODBC"}, {TypeDbScyllaDB, "TypeDbScyllaDB"}, + {TypeDbFirebolt, "TypeDbFirebolt"}, } // ValidateType returns true is type is valid @@ -165,7 +167,7 @@ func ValidateType(tStr string) (Type, bool) { case TypeApi, TypeFileLocal, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileDatabricksVolume, - TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: + TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbFirebolt: return t, true } @@ -202,6 +204,7 @@ func (t Type) DefPort() int { TypeFileFtp: 21, TypeFileSftp: 22, TypeDbScyllaDB: 9042, + TypeDbFirebolt: 3473, } return connTypesDefPort[t] } @@ -231,7 +234,7 @@ func (t Type) DBNameUpperCase() bool { func (t Type) Kind() Kind { switch t { case TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbBigTable, - TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: + TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbFirebolt: return KindDatabase case TypeFileLocal, TypeFileHDFS, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileHTTP, TypeFileDatabricksVolume, Type("https"): return KindFile @@ -293,7 +296,7 @@ func (t Type) IsColumnStore() bool { TypeDbSnowflake, TypeDbBigQuery, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbRedshift, TypeDbDatabricks, TypeDbStarRocks, - TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbExasol, + TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbExasol, TypeDbFirebolt, TypeDbAzureDWH, TypeDbFabric, ) } @@ -352,6 +355,7 @@ func (t Type) NameLong() string { TypeDbArrowDBC: "DB - Arrow DBC", TypeDbODBC: "DB - ODBC", TypeDbScyllaDB: "DB - ScyllaDB", + TypeDbFirebolt: "DB - Firebolt", } return mapping[t] @@ -405,6 +409,7 @@ func (t Type) Name() string { TypeDbAzureTable: "Azure Table", TypeDbArrowDBC: "Arrow DBC", TypeDbODBC: "ODBC", + TypeDbFirebolt: "Firebolt", } return mapping[t] diff --git a/core/dbio/templates/_properties.yaml b/core/dbio/templates/_properties.yaml index 005397c0f..541a69341 100644 --- a/core/dbio/templates/_properties.yaml +++ b/core/dbio/templates/_properties.yaml @@ -940,6 +940,54 @@ azuretable: description: 'Specific table name for table-specific SAS tokens' examples: ['MyTable', 'LogData', 'UserData'] +firebolt: + title: 'Firebolt' + kind: database + required: ["host", "port", "database"] + url_template: 'firebolt://{username}:{password}@{host}:{port}/{database}?secure={secure}&skip_verify={skip_verify}' + docs: https://docs.slingdata.io/connections/database-connections/firebolt + properties: + name: + type: text + title: Connection Name + description: 'The name of the connection. No spaces are allowed, will be all caps. Example: FIREBOLT_MAIN' + examples: ['FIREBOLT_MAIN', 'FIREBOLT_DEV', 'FB_ANALYTICS'] + database: + type: text + description: 'The database name to run queries against' + examples: ['firebolt', 'mydb', 'analytics'] + default: firebolt + host: + type: text + description: 'The hostname / ip of the engine, e.g.: my.engine.firebolt.io or 127.0.0.1 for a local Firebolt Core' + examples: ['127.0.0.1', 'my.engine.firebolt.io', 'firebolt.example.com'] + username: + type: text + description: 'The username to access the instance. Not needed for Firebolt Core (no authentication)' + examples: ['myuser', 'admin'] + password: + type: text + description: 'The password to access the instance. Not needed for Firebolt Core (no authentication)' + examples: ['mypassword123'] + secret: true + port: + type: integer + description: 'The port of the instance' + examples: ['3473'] + default: 3473 + secure: + type: dropdown + options: ['true', 'false'] + title: Use HTTPS / TLS + examples: ['true', 'false'] + default: 'false' + skip_verify: + type: dropdown + options: ['true', 'false'] + title: Skip TLS certificate verification + examples: ['true', 'false'] + default: 'false' + ftp: title: 'FTP' kind: file diff --git a/core/dbio/templates/firebolt.yaml b/core/dbio/templates/firebolt.yaml new file mode 100644 index 000000000..71b7ad2ee --- /dev/null +++ b/core/dbio/templates/firebolt.yaml @@ -0,0 +1,226 @@ +core: + drop_table: drop table if exists {table} + drop_view: drop view if exists {view} + drop_schema: drop schema if exists {schema} cascade + create_schema: create schema if not exists {schema} + create_table: create table if not exists {table} ({col_types}) + truncate_table: truncate table {table} + rename_table: alter table {table} rename to {new_table} + rename_column: alter table {table} rename column {column} to {new_column} + add_column: alter table {table} add column if not exists {column} {type} + drop_column: alter table {table} drop column if exists {column} + insert: insert into {table} ({fields}) values ({values}) + update: update {table} set {set_fields} where {pk_fields_equal} + delete: delete from {table} where {where} + # Firebolt has no plain CREATE INDEX. Secondary indexes are specialized + # (FULL_TEXT / INVERTED_INDEX / HNSW / SKIP_INDEX) and are declared separately. + create_index: "select 'create_index not implemented'" + create_unique_index: "select 'create_unique_index not implemented'" + drop_index: "select 'drop_index not implemented'" + # Firebolt supports LIMIT/OFFSET on derived tables + limit_sql: | + select * from ( + {sql} + ) as t limit {limit} offset {offset} + + merge_insert: | + INSERT INTO {tgt_table} ({insert_fields}) + SELECT {src_fields} FROM {src_table} src + WHERE NOT EXISTS ( + SELECT 1 FROM {tgt_table} tgt WHERE {src_tgt_pk_equal} + ) + + merge_update: | + UPDATE {tgt_table} tgt + SET {set_fields} + FROM {src_table} src + WHERE {src_tgt_pk_equal} + + merge_update_insert: | + MERGE INTO {tgt_table} tgt + USING (SELECT {src_fields} FROM {src_table}) src + ON ({src_tgt_pk_equal}) + WHEN MATCHED THEN UPDATE SET {set_fields} + WHEN NOT MATCHED THEN INSERT ({insert_fields}) VALUES ({src_insert_fields}) + + merge_delete_insert: | + DELETE FROM {tgt_table} tgt + WHERE EXISTS ( + SELECT 1 FROM {src_table} src + WHERE {src_tgt_pk_equal} + ); + INSERT INTO {tgt_table} ({insert_fields}) + SELECT {src_fields} FROM {src_table} src + +metadata: + current_database: | + select current_database() + + current_schema: | + select current_schema() + + databases: | + select catalog_name as name + from information_schema.catalogs + order by catalog_name + + schemas: | + select schema_name + from information_schema.schemata + where schema_name not in ('information_schema', 'pg_catalog') + {{if .schema -}} and schema_name = '{schema}' {{- end}} + order by schema_name + + schemata: | + select + c.table_schema as schema_name, + c.table_name as table_name, + 'table' as table_type, + c.column_name, + c.data_type, + c.is_nullable, + c.ordinal_position as "position" + from information_schema.columns c + where 1=1 + {{if .schema -}} and c.table_schema = '{schema}' {{- end}} + {{if .tables -}} and c.table_name in ({tables}) {{- end}} + order by c.table_schema, c.table_name, c.ordinal_position + + tables: | + select + table_schema as schema_name, + table_name, + 'false' as is_view + from information_schema.tables + where table_type = 'BASE TABLE' + {{if .schema -}} and table_schema = '{schema}' {{- end}} + {{if .table -}} and table_name = '{table}' {{- end}} + order by table_schema, table_name + + views: | + select + table_schema as schema_name, + table_name, + 'true' as is_view + from information_schema.views + where 1=1 + {{if .schema -}} and table_schema = '{schema}' {{- end}} + {{if .view -}} and table_name = '{view}' {{- end}} + order by table_schema, table_name + + columns: | + select + table_schema as schema_name, + table_name, + column_name, + data_type, + is_nullable, + column_default, + character_maximum_length, + numeric_precision, + numeric_scale, + ordinal_position + from information_schema.columns + where 1=1 + {{if .schema -}} and table_schema = '{schema}' {{- end}} + {{if .table -}} and table_name = '{table}' {{- end}} + order by table_schema, table_name, ordinal_position + + primary_keys: | + select + tc.constraint_name as pk_name, + kcu.ordinal_position as position, + kcu.column_name + from information_schema.table_constraints tc + join information_schema.key_column_usage kcu + on kcu.constraint_name = tc.constraint_name + and kcu.constraint_schema = tc.constraint_schema + and kcu.table_schema = tc.table_schema + and kcu.table_name = tc.table_name + where tc.constraint_type = 'PRIMARY KEY' + {{if .schema -}} and tc.table_schema = '{schema}' {{- end}} + {{if .table -}} and tc.table_name = '{table}' {{- end}} + order by tc.table_name, kcu.ordinal_position + + # Firebolt has no secondary indexes (only specialized FULL_TEXT / INVERTED / + # HNSW / SKIP indexes), so report an empty set + indexes: | + select + '{schema}' as schema_name, + '{table}' as table_name, + cast(null as text) as index_name, + cast(null as text) as column_name + where 1 = 0 + + ddl: | + select ddl + from information_schema.tables + where table_schema = '{schema}' + and table_name = '{table}' + +function: + string_type: varchar + # Firebolt's length() only accepts text/array/bytea, so cast first + checksum_boolean: 'length(cast({field} as varchar))' + checksum_json: "length(replace(cast({field} as varchar), ' ', ''))" + cast_to_string: cast({field} as varchar) + cast_to_text: cast({field} as varchar) + date_trunc_key: date_trunc('{unit}', {field}) + date_format: to_char({field}, {format}) + date_parse_format: to_timestamp({string}, {format}) + hash: md5({fields}) + random: random() + uuid: uuid() + +variable: + bind_string: ${c} + column_upper: false + max_column_length: 128 + max_string_type: text + max_string_length: 65535 + timestamp_layout: '2006-01-02 15:04:05.000000' + timestampz_layout: '2006-01-02 15:04:05.000000-07' + default_merge_strategy: update_insert + +native_type_map: + bigint: bigint + long: bigint + integer: integer + int: integer + "double precision": float + double: float + real: float + float: float + text: text + varchar: text + boolean: bool + bool: bool + date: date + timestamp: timestamp + timestamptz: timestampz + timestampntz: timestamp + numeric: decimal + decimal: decimal + bytea: binary + json: json + array: json + struct: json + geography: text + +general_type_map: + bigint: bigint + binary: bytea + bool: boolean + date: date + datetime: timestamp + decimal: numeric + float: "double precision" + integer: integer + json: json + smallint: integer + string: text + text: text + time: text + timestamp: timestamp + timestampz: timestamptz + uuid: text From ecb996437e5780b99c2483c960e98e6ed50db50c Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Tue, 22 Sep 2026 08:29:07 -0300 Subject: [PATCH 28/56] Add OpenSearch database connector support - Register the `opensearch` type end to end: URL parsing, default port 9200, NoSQL kind, connection templates, docs and README entry. - Implement OpenSearchConn with index/mapping discovery, scroll-based reads, plus limit, incremental and backfill range queries. - Fix the Elasticsearch-family scroll decoder to consume the initial search response before scrolling; previously only the first page of hits was read, so multi-page reads returned a single doc per page. - Treat OpenSearch as read-only for now (writes rejected at config validation), matching Elasticsearch. - Add connector capability tests for both OpenSearch and Elasticsearch, and add the opensearch-go/v2 dependency. --- README.md | 2 +- cmd/sling/resource/llm_CONNECTION.md | 2 + cmd/sling/sling_conns.go | 2 +- cmd/sling/sling_test.go | 87 ++- core/dbio/connection/connection.go | 16 + core/dbio/database/database.go | 2 + core/dbio/database/database_elasticsearch.go | 22 +- core/dbio/database/database_opensearch.go | 630 +++++++++++++++++++ core/dbio/database/schemata.go | 2 +- core/dbio/dbio_types.go | 11 +- core/dbio/dbio_types_test.go | 1 + core/dbio/templates/_properties.yaml | 30 + core/dbio/templates/connections.json | 24 + core/dbio/templates/opensearch.yaml | 20 + core/sling/config.go | 2 +- core/sling/task_run_write.go | 2 +- go.mod | 122 +--- go.sum | 254 ++------ 18 files changed, 935 insertions(+), 296 deletions(-) create mode 100644 core/dbio/database/database_opensearch.go create mode 100644 core/dbio/templates/opensearch.yaml diff --git a/README.md b/README.md index 67c6bb845..40a0ca120 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Example [Replication](https://docs.slingdata.io/sling-cli/run/configuration/repl --- Available Connectors: -- **Databases**: [`adbc`](https://docs.slingdata.io/connections/database-connections/adbc) [`azuredwh`](https://docs.slingdata.io/connections/database-connections/azuredwh) [`azuresql`](https://docs.slingdata.io/connections/database-connections/azuresql) [`azuretable`](https://docs.slingdata.io/connections/database-connections/azuretable) [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`d1`](https://docs.slingdata.io/connections/database-connections/d1) [`databricks`](https://docs.slingdata.io/connections/database-connections/databricks) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`elasticsearch`](https://docs.slingdata.io/connections/database-connections/elasticsearch) [`exasol`](https://docs.slingdata.io/connections/database-connections/exasol) [`fabric`](https://docs.slingdata.io/connections/database-connections/fabric) [`firebolt`](https://docs.slingdata.io/connections/database-connections/firebolt) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`mongodb`](https://docs.slingdata.io/connections/database-connections/mongodb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`odbc`](https://docs.slingdata.io/connections/database-connections/odbc) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`scylladb`](https://docs.slingdata.io/connections/database-connections/scylladb) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`trino`](https://docs.slingdata.io/connections/database-connections/trino) +- **Databases**: [`adbc`](https://docs.slingdata.io/connections/database-connections/adbc) [`azuredwh`](https://docs.slingdata.io/connections/database-connections/azuredwh) [`azuresql`](https://docs.slingdata.io/connections/database-connections/azuresql) [`azuretable`](https://docs.slingdata.io/connections/database-connections/azuretable) [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`d1`](https://docs.slingdata.io/connections/database-connections/d1) [`databricks`](https://docs.slingdata.io/connections/database-connections/databricks) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`elasticsearch`](https://docs.slingdata.io/connections/database-connections/elasticsearch) [`opensearch`](https://docs.slingdata.io/connections/database-connections/opensearch) [`exasol`](https://docs.slingdata.io/connections/database-connections/exasol) [`fabric`](https://docs.slingdata.io/connections/database-connections/fabric) [`firebolt`](https://docs.slingdata.io/connections/database-connections/firebolt) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`mongodb`](https://docs.slingdata.io/connections/database-connections/mongodb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`odbc`](https://docs.slingdata.io/connections/database-connections/odbc) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`scylladb`](https://docs.slingdata.io/connections/database-connections/scylladb) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`trino`](https://docs.slingdata.io/connections/database-connections/trino) - **Data Lakes**:[`athena`](https://docs.slingdata.io/connections/datalake-connections/athena) [`Ducklake`](https://docs.slingdata.io/connections/datalake-connections/ducklake) [`iceberg`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`S3 Tables`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`lancedb`](https://docs.slingdata.io/connections/datalake-connections/lancedb) diff --git a/cmd/sling/resource/llm_CONNECTION.md b/cmd/sling/resource/llm_CONNECTION.md index 8727708b7..ddca3a9d6 100644 --- a/cmd/sling/resource/llm_CONNECTION.md +++ b/cmd/sling/resource/llm_CONNECTION.md @@ -165,6 +165,8 @@ Navigate or fetch the content of the connector from the below respective URL to **ElasticSearch (`elasticsearch`)** -> https://docs.slingdata.io/connections/database-connections/elasticsearch +**OpenSearch (`opensearch`)** -> https://docs.slingdata.io/connections/database-connections/opensearch + **Prometheus (`prometheus`)** -> https://docs.slingdata.io/connections/database-connections/prometheus **StarRocks (`starrocks`)** -> https://docs.slingdata.io/connections/database-connections/starrocks diff --git a/cmd/sling/sling_conns.go b/cmd/sling/sling_conns.go index ce580eacb..2a182ae71 100644 --- a/cmd/sling/sling_conns.go +++ b/cmd/sling/sling_conns.go @@ -198,7 +198,7 @@ func processConns(c *g.CliSC) (ok bool, err error) { return ok, g.Error(err, "cannot parse query") } - if len(database.ParseSQLMultiStatements(query)) == 1 && (!sQuery.IsQuery() || (strings.Contains(strings.ToLower(query), "select") && !strings.Contains(strings.ToLower(query), "insert")) || g.In(conn.Connection.Type, dbio.TypeDbPrometheus, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch)) { + if len(database.ParseSQLMultiStatements(query)) == 1 && (!sQuery.IsQuery() || (strings.Contains(strings.ToLower(query), "select") && !strings.Contains(strings.ToLower(query), "insert")) || g.In(conn.Connection.Type, dbio.TypeDbPrometheus, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbOpenSearch)) { // Limit handling: // - limit > 0: wrap the SQL with the dialect's limit_sql template via diff --git a/cmd/sling/sling_test.go b/cmd/sling/sling_test.go index 7b3d09010..b92e8e606 100755 --- a/cmd/sling/sling_test.go +++ b/cmd/sling/sling_test.go @@ -119,6 +119,7 @@ var connMap = map[dbio.Type]connTest{ dbio.TypeDbMongoDB: {name: "mongo", schema: "default"}, dbio.TypeDbAzureTable: {name: "azure_table", schema: "default"}, dbio.TypeDbElasticsearch: {name: "elasticsearch", schema: "default"}, + dbio.TypeDbOpenSearch: {name: "opensearch", schema: "default"}, dbio.TypeDbPrometheus: {name: "prometheus", schema: "prometheus"}, dbio.TypeDbProton: {name: "proton", schema: "default", useBulk: g.Bool(true)}, dbio.TypeDbScyllaDB: {name: "scylladb", schema: "sling"}, @@ -879,7 +880,7 @@ func runOneTask(t *testing.T, ctx context.Context, file g.FileItem, connType dbi for colName, correctType := range correctTypeMap { // skip those - if g.In(srcType, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable, dbio.TypeDbScyllaDB) || + if g.In(srcType, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable, dbio.TypeDbScyllaDB, dbio.TypeDbElasticsearch, dbio.TypeDbOpenSearch) || g.In(tgtType, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable) || taskCfg.TgtConn.IsADBC() || taskCfg.SrcConn.IsADBC() || taskCfg.TgtConn.Type == dbio.TypeDbODBC || @@ -1297,6 +1298,90 @@ func TestSuiteDatabaseMongo(t *testing.T) { testSuite(t, dbio.TypeDbMongoDB, "table_full_refresh_into_postgres,discover_schemas") } +func TestSuiteDatabaseOpenSearch(t *testing.T) { + t.Parallel() + testSuite(t, dbio.TypeDbOpenSearch, "table_full_refresh_into_postgres,discover_schemas") +} + +func TestSuiteDatabaseElasticsearch(t *testing.T) { + t.Parallel() + testSuite(t, dbio.TypeDbElasticsearch, "table_full_refresh_into_postgres,discover_schemas") +} + +// testSearchConnectorCaps exercises the read capabilities of the ES-family +// (Elasticsearch / OpenSearch) connectors directly against a live instance: +// full scroll read (regression guard for multi-page scrolling), limit, +// incremental (update_key gt), backfill (update_key gte/lte range), and +// schema/column discovery via the index mapping. The index is expected to hold +// 1000 docs with a numeric `id` field 1..1000 (seeded from tests/files/test1.csv). +func testSearchConnectorCaps(t *testing.T, connType dbio.Type, connName, index string) { + c := connection.GetLocalConns().Get(connName) + if c.Name == "" { + t.Skipf("no connection found for %s", connName) + return + } + + conn, err := c.Connection.AsDatabase() + if !g.AssertNoError(t, err) { + return + } + if err = conn.Connect(); !g.AssertNoError(t, err) { + return + } + defer conn.Close() + + streamCount := func(opts map[string]interface{}) int { + ds, err := conn.StreamRows(index, opts) + if !g.AssertNoError(t, err) { + return -1 + } + data, err := ds.Collect(0) + if !g.AssertNoError(t, err) { + return -1 + } + return len(data.Rows) + } + + // full read: must return every doc across all scroll pages (guards the + // scroll-pagination bug where only ~1 doc per page was yielded) + assert.Equal(t, 1000, streamCount(map[string]interface{}{}), "full scroll read (%s)", connType) + + // limit: caps the number of returned rows + assert.Equal(t, 50, streamCount(map[string]interface{}{"limit": 50}), "limited read (%s)", connType) + + // incremental: update_key range gt -> ids 501..1000 + assert.Equal(t, 500, streamCount(map[string]interface{}{"update_key": "id", "value": "500"}), "incremental read (%s)", connType) + + // backfill: update_key range gte/lte -> ids 200..400 inclusive + assert.Equal(t, 201, streamCount(map[string]interface{}{"update_key": "id", "start_value": "200", "end_value": "400"}), "backfill read (%s)", connType) + + // schema discovery: the index shows up as a schema/table + schemas, err := conn.GetSchemas() + if g.AssertNoError(t, err) { + assert.Contains(t, schemas.ColValuesStr(0), index, "GetSchemas should contain index (%s)", connType) + } + + // column discovery: the index mapping is parsed into typed columns + schemata, err := conn.GetSchemata(database.SchemataLevelColumn, index) + if g.AssertNoError(t, err) { + cols := iop.Columns(lo.Values(schemata.Columns())) + assert.Greater(t, len(cols), 5, "column discovery from mapping (%s)", connType) + names := strings.Join(cols.Names(), ",") + assert.Contains(t, strings.ToLower(names), "id", "columns should include id (%s)", connType) + assert.Contains(t, strings.ToLower(names), "email", "columns should include email (%s)", connType) + } +} + +func TestOpenSearchConnectorCaps(t *testing.T) { + t.Parallel() + testSearchConnectorCaps(t, dbio.TypeDbOpenSearch, "opensearch", "test1k_opensearch") +} + +func TestElasticsearchConnectorCaps(t *testing.T) { + t.Parallel() + testSearchConnectorCaps(t, dbio.TypeDbElasticsearch, "elasticsearch", "test1k_elasticsearch") +} + func TestSuiteDatabaseAzureTable(t *testing.T) { t.Parallel() testSuite(t, dbio.TypeDbAzureTable, "table_full_refresh_into_postgres,discover_schemas") diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index ac575d6a8..3b9b01cfa 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -817,6 +817,22 @@ func (c *Connection) setURL() (err error) { } else { template = "elasticsearch://{username}:{password}@{host}:{port}" } + case dbio.TypeDbOpenSearch: + setIfMissing("username", c.Data["user"]) + setIfMissing("password", "") + setIfMissing("port", c.Type.DefPort()) + + // parse http url + if httpUrlStr, ok := c.Data["http_url"]; ok { + u, err := url.Parse(cast.ToString(httpUrlStr)) + if err != nil { + g.Warn("invalid http_url: %s", err.Error()) + } else { + setIfMissing("host", u.Hostname()) + } + } + + template = "opensearch://{username}:{password}@{host}:{port}" case dbio.TypeDbPrometheus: setIfMissing("api_key", "") setIfMissing("port", c.Type.DefPort()) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index 828b34e8b..0b29900b9 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -289,6 +289,8 @@ func NewConnContext(ctx context.Context, URL string, props ...string) (Connectio conn = &MongoDBConn{URL: URL} } else if strings.HasPrefix(URL, "elasticsearch") { conn = &ElasticsearchConn{URL: URL} + } else if strings.HasPrefix(URL, "opensearch") { + conn = &OpenSearchConn{URL: URL} } else if strings.HasPrefix(URL, "prometheus") { conn = &PrometheusConn{URL: URL} } else if strings.HasPrefix(URL, "mariadb:") { diff --git a/core/dbio/database/database_elasticsearch.go b/core/dbio/database/database_elasticsearch.go index 10b30cc2b..90ecd48d0 100644 --- a/core/dbio/database/database_elasticsearch.go +++ b/core/dbio/database/database_elasticsearch.go @@ -436,9 +436,14 @@ func (d *elasticDecoder) Decode(obj interface{}) error { return io.EOF } - // Get next batch if needed - if d.searchResponse == nil || (d.hits == nil || d.currentHit >= len(d.hits)) { - if d.searchResponse == nil { + // Fetch a new page only when the current batch is exhausted. + // The initial search response is page 1; subsequent pages come from scroll. + if d.hits == nil || d.currentHit >= len(d.hits) { + var resp map[string]any + if d.searchResponse != nil { + resp = d.searchResponse + d.searchResponse = nil + } else { // Get the next batch of results using the scroll ID res, err := d.conn.Client.Scroll( d.conn.Client.Scroll.WithContext(d.ctx), @@ -449,22 +454,20 @@ func (d *elasticDecoder) Decode(obj interface{}) error { return g.Error(err, "error scrolling results") } - if err := json.NewDecoder(res.Body).Decode(&d.searchResponse); err != nil { + if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { res.Body.Close() return g.Error(err, "error decoding scroll response") } res.Body.Close() // Update scroll ID for next batch - newScrollID, ok := d.searchResponse["_scroll_id"].(string) - if !ok { - return io.EOF + if newScrollID, ok := resp["_scroll_id"].(string); ok { + d.scrollID = newScrollID } - d.scrollID = newScrollID } // Get hits from response - hits, ok := d.searchResponse["hits"].(map[string]any) + hits, ok := resp["hits"].(map[string]any) if !ok { return g.Error("hits not found in response") } @@ -480,7 +483,6 @@ func (d *elasticDecoder) Decode(obj interface{}) error { d.hits = hitsArray d.currentHit = 0 - d.searchResponse = nil } // Get next hit diff --git a/core/dbio/database/database_opensearch.go b/core/dbio/database/database_opensearch.go new file mode 100644 index 000000000..e86ecefc7 --- /dev/null +++ b/core/dbio/database/database_opensearch.go @@ -0,0 +1,630 @@ +package database + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "io" + "net/http" + "strings" + "time" + + "github.com/flarco/g" + opensearch "github.com/opensearch-project/opensearch-go/v2" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/spf13/cast" +) + +// OpenSearchConn is an opensearch connection +type OpenSearchConn struct { + BaseConn + URL string + Client *opensearch.Client +} + +// Init initiates the object +func (conn *OpenSearchConn) Init() error { + conn.BaseConn.URL = conn.URL + conn.BaseConn.Type = dbio.TypeDbOpenSearch + + instance := Connection(conn) + conn.BaseConn.instance = &instance + return conn.BaseConn.Init() +} + +// getNewClient creates a new opensearch client +func (conn *OpenSearchConn) getNewClient(timeOut ...int) (client *opensearch.Client, err error) { + cfg := opensearch.Config{} + + // Handle HTTP URLs or default localhost + if httpURLs := conn.GetProp("http_url"); httpURLs != "" { + cfg.Addresses = strings.Split(httpURLs, ",") + } else { + host := conn.GetProp("host") + port := conn.GetProp("port") + cfg.Addresses = []string{g.F("http://%s:%s", host, port)} + } + + // Handle Basic Auth + if user := conn.GetProp("user", "username"); user != "" { + cfg.Username = user + cfg.Password = conn.GetProp("password") + } + + // Handle TLS config + tlsConfig, err := conn.makeTlsConfig() + if err != nil { + return nil, g.Error(err) + } + if tlsConfig != nil { + cfg.Transport = &http.Transport{TLSClientConfig: tlsConfig} + // Update URLs to use HTTPS if using TLS + for i, addr := range cfg.Addresses { + if !strings.HasPrefix(addr, "https://") { + cfg.Addresses[i] = strings.Replace(addr, "http://", "https://", 1) + } + } + } + + client, err = opensearch.NewClient(cfg) + if err != nil { + return nil, g.Error(err, "could not connect to OpenSearch server") + } + + // Test connection with timeout + to := 15 + if len(timeOut) > 0 { + to = timeOut[0] + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(to)*time.Second) + defer cancel() + + info, err := client.Info( + client.Info.WithContext(ctx), + ) + if err != nil { + return nil, g.Error(err, "could not connect to OpenSearch server") + } + defer info.Body.Close() + + return client, nil +} + +// Connect connects to the database +func (conn *OpenSearchConn) Connect(timeOut ...int) error { + var err error + conn.Client, err = conn.getNewClient(timeOut...) + if err != nil { + return g.Error(err, "Failed to connect to client") + } + + if !cast.ToBool(conn.GetProp("silent")) { + g.Debug(`opened "%s" connection (%s)`, conn.Type, conn.GetProp("sling_conn_id")) + } + + conn.SetProp("connected", "true") + conn.SetProp("connect_time", cast.ToString(time.Now())) + + return nil +} + +func (conn *OpenSearchConn) Close() error { + g.Debug(`closed "%s" connection (%s)`, conn.Type, conn.GetProp("sling_conn_id")) + return nil +} + +// NewTransaction creates a new transaction +func (conn *OpenSearchConn) NewTransaction(ctx context.Context, options ...*sql.TxOptions) (tx Transaction, err error) { + // OpenSearch does not support transactions + return nil, g.Error("transactions not supported in OpenSearch") +} + +// GetTableColumns returns columns for a table +func (conn *OpenSearchConn) GetTableColumns(table *Table, fields ...string) (columns iop.Columns, err error) { + // Get mapping for the index + mapping, err := conn.Client.Indices.GetMapping( + conn.Client.Indices.GetMapping.WithIndex(table.Name), + ) + if err != nil { + return columns, g.Error(err, "could not get mapping for index %s", table.Name) + } + defer mapping.Body.Close() + + var mappingResponse map[string]any + if err := json.NewDecoder(mapping.Body).Decode(&mappingResponse); err != nil { + return columns, g.Error(err, "could not decode mapping response") + } + + // Navigate to properties + indexMapping, ok := mappingResponse[table.Name].(map[string]any) + if !ok { + return columns, g.Error("unexpected mapping structure for index %s: %s", table.Name, g.Marshal(mappingResponse[table.Name])) + } + + mappings, ok := indexMapping["mappings"].(map[string]any) + if !ok { + return columns, g.Error("no mappings found for index %s", table.Name) + } + + properties, ok := mappings["properties"].(map[string]any) + if !ok { + // Try structure where type is implicit + if props, ok := mappings["_doc"].(map[string]any); ok { + properties = props["properties"].(map[string]any) + } else { + // If no properties found, return a single JSON column + columns = append(columns, iop.Column{ + Name: "data", + Type: iop.ColumnType("json"), + Table: table.Name, + Schema: table.Schema, + Database: table.Database, + Position: 1, + DbType: "json", + }) + return columns, nil + } + } + + position := 1 + var processProperties func(prefix string, props map[string]any) + processProperties = func(prefix string, props map[string]any) { + for fieldName, fieldDef := range props { + fieldDefMap, ok := fieldDef.(map[string]any) + if !ok { + continue + } + + fieldType, _ := fieldDefMap["type"].(string) + + // Handle nested objects + if fieldType == "object" { + if nestedProps, ok := fieldDefMap["properties"].(map[string]any); ok { + newPrefix := prefix + if prefix != "" { + newPrefix = prefix + "." + } + processProperties(newPrefix+fieldName, nestedProps) + } + continue + } + + // Map OpenSearch types to general types + var colType iop.ColumnType + switch fieldType { + case "text", "keyword", "string": + colType = iop.TextType + case "long", "integer": + colType = iop.BigIntType + case "float", "double": + colType = iop.FloatType + case "date": + colType = iop.TimestampType + case "boolean": + colType = iop.BoolType + case "binary": + colType = iop.BinaryType + default: + colType = iop.TextType // default to text for unknown types + } + + columnName := fieldName + if prefix != "" { + columnName = prefix + "." + fieldName + } + + columns = append(columns, iop.Column{ + Name: columnName, + Type: colType, + Table: table.Name, + Schema: table.Schema, + Database: table.Database, + Position: position, + DbType: fieldType, + }) + position++ + } + } + + processProperties("", properties) + + // If no columns were found, return a single JSON column + if len(columns) == 0 { + columns = append(columns, iop.Column{ + Name: "data", + Type: iop.ColumnType("json"), + Table: table.Name, + Schema: table.Schema, + Database: table.Database, + Position: 1, + DbType: "json", + }) + } + + return columns, nil +} + +func (conn *OpenSearchConn) ExecContext(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error) { + return nil, g.Error("ExecContext not implemented on OpenSearch") +} + +func (conn *OpenSearchConn) BulkExportFlow(table Table) (df *iop.Dataflow, err error) { + options, _ := g.UnmarshalMap(table.SQL) + + // add columns if present + if len(table.Columns) > 0 { + options["columns"] = table.Columns + } + + ds, err := conn.StreamRowsContext(conn.Context().Ctx, table.Name, options) + if err != nil { + return df, g.Error(err, "could start datastream") + } + + df, err = iop.MakeDataFlow(ds) + if err != nil { + return df, g.Error(err, "could start dataflow") + } + + return df, nil +} + +func (conn *OpenSearchConn) StreamRowsContext(ctx context.Context, tableName string, Opts ...map[string]any) (ds *iop.Datastream, err error) { + opts := getQueryOptions(Opts) + Limit := int64(0) // infinite + if val := cast.ToInt64(opts["limit"]); val > 0 { + Limit = val + } + + // Handle incremental and backfill options + var searchBody map[string]any + if updateKey := cast.ToString(opts["update_key"]); updateKey != "" { + if incrementalValue := cast.ToString(opts["value"]); incrementalValue != "" { + // Incremental mode + searchBody = map[string]any{ + "query": map[string]any{ + "range": map[string]any{ + updateKey: map[string]any{ + "gt": incrementalValue, + }, + }, + }, + } + } else if startValue := cast.ToString(opts["start_value"]); startValue != "" { + if endValue := cast.ToString(opts["end_value"]); endValue != "" { + // Backfill mode + searchBody = map[string]any{ + "query": map[string]any{ + "range": map[string]any{ + updateKey: map[string]any{ + "gte": startValue, + "lte": endValue, + }, + }, + }, + } + } + } + } + + // If no specific query is provided, use match_all + if searchBody == nil { + searchBody = map[string]any{ + "query": map[string]any{ + "match_all": map[string]any{}, + }, + } + } + + // Add size if limit is specified + if Limit > 0 { + searchBody["size"] = Limit + } + + // Create search request + searchBytes, err := json.Marshal(searchBody) + if err != nil { + return nil, g.Error(err, "could not marshal search body") + } + + // Create the search request + table, _ := ParseTableName(tableName, conn.Type) + indexName := table.Name + res, err := conn.Client.Search( + conn.Client.Search.WithContext(ctx), + conn.Client.Search.WithIndex(indexName), + conn.Client.Search.WithBody(bytes.NewReader(searchBytes)), + conn.Client.Search.WithScroll(time.Minute), + ) + if err != nil { + return nil, g.Error(err, "could not execute search") + } else if res.StatusCode >= 400 { + bytes, _ := io.ReadAll(res.Body) + return nil, g.Error("could not execute search (status %d) => %s", res.StatusCode, string(bytes)) + } + + var searchResponse map[string]any + if err := json.NewDecoder(res.Body).Decode(&searchResponse); err != nil { + return nil, g.Error(err, "could not decode search response") + } + res.Body.Close() + + scrollID, ok := searchResponse["_scroll_id"].(string) + if !ok { + // no result, return empty datastream + data := iop.NewDataset(iop.NewColumnsFromFields("data")) + return data.Stream(), nil + } + + // Create base datastream + ds = iop.NewDatastreamContext(ctx, iop.Columns{}) + + // Handle flattening option + flatten := 0 + if val := conn.GetProp("flatten"); val != "" { + flatten = cast.ToInt(val) + } + + // Create a custom decoder for OpenSearch scrolling + decoder := &openSearchDecoder{ + conn: conn, + ctx: ctx, + scrollID: scrollID, + searchResponse: searchResponse, + limit: cast.ToUint64(Limit), + counter: 0, + } + + // Create JSON stream iterator + js := iop.NewJSONStream(ds, decoder, flatten, conn.GetProp("jmespath"), conn.GetProp("jq")) + js.HasMapPayload = true + + // Set up the iterator + it := ds.NewIterator(ds.Columns, js.NextFunc) + ds.SetIterator(it) + ds.SetMetadata(conn.GetProp("METADATA")) + ds.SetConfig(conn.Props()) + + err = ds.Start() + if err != nil { + return ds, g.Error(err, "could not start datastream") + } + + // unmarshal columns if none detected + // otherwise this may error when creating a temp table with no columns + if len(ds.Columns) == 0 { + g.JSONConvert(opts["columns"], &ds.Columns) + } + + return ds, nil +} + +// openSearchDecoder implements the decoderLike interface for OpenSearch scrolling +type openSearchDecoder struct { + conn *OpenSearchConn + ctx context.Context + scrollID string + searchResponse map[string]any + hits []interface{} + currentHit int + limit uint64 + counter uint64 +} + +// Decode implements the decoderLike interface +func (d *openSearchDecoder) Decode(obj interface{}) error { + // Check context and limits + if d.ctx.Err() != nil { + return d.ctx.Err() + } + if d.limit > 0 && d.counter >= d.limit { + return io.EOF + } + + // Get next batch if needed + // Fetch a new page only when the current batch is exhausted. + // The initial search response is page 1; subsequent pages come from scroll. + if d.hits == nil || d.currentHit >= len(d.hits) { + var resp map[string]any + if d.searchResponse != nil { + resp = d.searchResponse + d.searchResponse = nil + } else { + // Get the next batch of results using the scroll ID + res, err := d.conn.Client.Scroll( + d.conn.Client.Scroll.WithContext(d.ctx), + d.conn.Client.Scroll.WithScrollID(d.scrollID), + d.conn.Client.Scroll.WithScroll(time.Minute), + ) + if err != nil { + return g.Error(err, "error scrolling results") + } + + if err := json.NewDecoder(res.Body).Decode(&resp); err != nil { + res.Body.Close() + return g.Error(err, "error decoding scroll response") + } + res.Body.Close() + + // Update scroll ID for next batch + if newScrollID, ok := resp["_scroll_id"].(string); ok { + d.scrollID = newScrollID + } + } + + // Get hits from response + hits, ok := resp["hits"].(map[string]any) + if !ok { + return g.Error("hits not found in response") + } + + hitsArray, ok := hits["hits"].([]interface{}) + if !ok { + return g.Error("hits array not found in response") + } + + if len(hitsArray) == 0 { + return io.EOF + } + + d.hits = hitsArray + d.currentHit = 0 + } + + // Get next hit + hit, ok := d.hits[d.currentHit].(map[string]any) + if !ok { + d.currentHit++ + return d.Decode(obj) // skip invalid hit + } + + source, ok := hit["_source"].(map[string]any) + if !ok { + d.currentHit++ + return d.Decode(obj) // skip invalid source + } + + // Set the source as the object to decode + objMap, ok := obj.(*map[string]any) + if !ok { + return g.Error("invalid object type for decoding") + } + *objMap = source + + d.currentHit++ + d.counter++ + return nil +} + +// GetSchemas returns schemas +func (conn *OpenSearchConn) GetSchemas() (data iop.Dataset, err error) { + // In OpenSearch, indices are similar to schemas/databases + // We'll list all indices + indices, err := conn.Client.Cat.Indices( + conn.Client.Cat.Indices.WithFormat("json"), + ) + if err != nil { + return data, g.Error(err, "could not list indices") + } + defer indices.Body.Close() + + var indicesResponse []map[string]any + if err := json.NewDecoder(indices.Body).Decode(&indicesResponse); err != nil { + return data, g.Error(err, "could not decode indices response") + } + + data = iop.NewDataset(iop.NewColumnsFromFields("schema_name")) + for _, index := range indicesResponse { + if indexName, ok := index["index"].(string); ok { + data.Append([]interface{}{indexName}) + } + } + + return data, nil +} + +// GetTables returns tables +func (conn *OpenSearchConn) GetTables(schema string) (data iop.Dataset, err error) { + // In OpenSearch, we can consider mappings as tables + // For a given index (schema), get its mapping + mapping, err := conn.Client.Indices.GetMapping( + conn.Client.Indices.GetMapping.WithIndex(schema), + ) + if err != nil { + return data, g.Error(err, "could not get mapping for index %s", schema) + } + defer mapping.Body.Close() + + var mappingResponse map[string]any + if err := json.NewDecoder(mapping.Body).Decode(&mappingResponse); err != nil { + return data, g.Error(err, "could not decode mapping response") + } + + data = iop.NewDataset(iop.NewColumnsFromFields("table_name")) + // Each index has one mapping type + data.Append([]interface{}{"_doc"}) + + return data, nil +} + +// GetSchemata returns the database schemata +func (conn *OpenSearchConn) GetSchemata(level SchemataLevel, schema string, tables ...string) (schemata Schemata, err error) { + schemata = Schemata{ + Databases: map[string]Database{}, + conn: conn, + } + + // Get list of indices (schemas) + indices, err := conn.Client.Cat.Indices( + conn.Client.Cat.Indices.WithFormat("json"), + ) + if err != nil { + return schemata, g.Error(err, "could not get indices") + } + defer indices.Body.Close() + + type indexInfo struct { + Index string `json:"index"` + } + var indicesResponse []indexInfo + if err := json.NewDecoder(indices.Body).Decode(&indicesResponse); err != nil { + return schemata, g.Error(err, "could not decode indices response") + } + + // Filter indices if schema is specified + if schema != "" { + filteredIndices := []indexInfo{} + for _, idx := range indicesResponse { + if idx.Index == schema { + filteredIndices = append(filteredIndices, idx) + } + } + indicesResponse = filteredIndices + } + + // Create database entry + database := Database{ + Name: "default", + Schemas: map[string]Schema{}, + } + + // Process each index + for _, idx := range indicesResponse { + // Skip system indices + if strings.HasPrefix(idx.Index, ".") { + continue + } + + // Create schema entry + schema := Schema{ + Name: idx.Index, + Database: database.Name, + Tables: map[string]Table{}, + } + + // Create table entry (in OpenSearch, index is both schema and table) + table := Table{ + Name: idx.Index, + Schema: idx.Index, + Database: database.Name, + Dialect: dbio.TypeDbOpenSearch, + } + + // Get columns if requested + if level == SchemataLevelColumn { + table.Columns, err = conn.GetTableColumns(&table) + if err != nil { + g.Warn("could not get columns for index %s: %s", idx.Index, err) + continue + } + } + + schema.Tables[strings.ToLower(table.Name)] = table + database.Schemas[strings.ToLower(schema.Name)] = schema + } + + schemata.Databases[strings.ToLower(database.Name)] = database + return schemata, nil +} diff --git a/core/dbio/database/schemata.go b/core/dbio/database/schemata.go index 021dd1766..9433c3fc8 100644 --- a/core/dbio/database/schemata.go +++ b/core/dbio/database/schemata.go @@ -286,7 +286,7 @@ func (t *Table) Select(Opts ...SelectOptions) (sql string) { t.SQL = g.F("%s\n--iceberg-json=%s", t.SQL, g.Marshal(m)) } return t.SQL - case dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbAzureTable: + case dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbOpenSearch, dbio.TypeDbAzureTable: m, _ := g.UnmarshalMap(t.SQL) if m == nil { m = g.M() diff --git a/core/dbio/dbio_types.go b/core/dbio/dbio_types.go index 87a7f3260..48c57a15c 100644 --- a/core/dbio/dbio_types.go +++ b/core/dbio/dbio_types.go @@ -82,6 +82,7 @@ const ( TypeDbClickhouse Type = "clickhouse" TypeDbMongoDB Type = "mongodb" TypeDbElasticsearch Type = "elasticsearch" + TypeDbOpenSearch Type = "opensearch" TypeDbPrometheus Type = "prometheus" TypeDbProton Type = "proton" TypeDbAthena Type = "athena" @@ -137,6 +138,7 @@ var AllType = []struct { {TypeDbLanceDB, "TypeDbLanceDB"}, {TypeDbClickhouse, "TypeDbClickhouse"}, {TypeDbElasticsearch, "TypeDbElasticsearch"}, + {TypeDbOpenSearch, "TypeDbOpenSearch"}, {TypeDbMongoDB, "TypeDbMongoDB"}, {TypeDbPrometheus, "TypeDbPrometheus"}, {TypeDbProton, "TypeDbProton"}, @@ -167,7 +169,7 @@ func ValidateType(tStr string) (Type, bool) { case TypeApi, TypeFileLocal, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileDatabricksVolume, - TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbFirebolt: + TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbMongoDB, TypeDbElasticsearch, TypeDbOpenSearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbFirebolt: return t, true } @@ -197,6 +199,7 @@ func (t Type) DefPort() int { TypeDbClickhouse: 9000, TypeDbMongoDB: 27017, TypeDbElasticsearch: 9200, + TypeDbOpenSearch: 9200, TypeDbPrometheus: 9090, TypeDbProton: 8463, TypeDbDatabricks: 443, @@ -234,7 +237,7 @@ func (t Type) DBNameUpperCase() bool { func (t Type) Kind() Kind { switch t { case TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbBigTable, - TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbFirebolt: + TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbOpenSearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbFirebolt: return KindDatabase case TypeFileLocal, TypeFileHDFS, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileHTTP, TypeFileDatabricksVolume, Type("https"): return KindFile @@ -252,7 +255,7 @@ func (t Type) IsDb() bool { // IsDb returns true if database connection func (t Type) IsNoSQL() bool { switch t { - case TypeDbBigTable, TypeDbAzureTable, TypeDbMongoDB, TypeDbElasticsearch, TypeDbScyllaDB: + case TypeDbBigTable, TypeDbAzureTable, TypeDbMongoDB, TypeDbElasticsearch, TypeDbOpenSearch, TypeDbScyllaDB: return true } return false @@ -349,6 +352,7 @@ func (t Type) NameLong() string { TypeDbClickhouse: "DB - Clickhouse", TypeDbPrometheus: "DB - Prometheus", TypeDbElasticsearch: "DB - Elasticsearch", + TypeDbOpenSearch: "DB - OpenSearch", TypeDbMongoDB: "DB - MongoDB", TypeDbProton: "DB - Proton", TypeDbAzureTable: "DB - Azure Table", @@ -402,6 +406,7 @@ func (t Type) Name() string { TypeDbClickhouse: "Clickhouse", TypeDbPrometheus: "Prometheus", TypeDbElasticsearch: "Elasticsearch", + TypeDbOpenSearch: "OpenSearch", TypeDbMongoDB: "MongoDB", TypeDbFabric: "Fabric", TypeDbAzure: "Azure", diff --git a/core/dbio/dbio_types_test.go b/core/dbio/dbio_types_test.go index 230cf8148..a59a427cd 100644 --- a/core/dbio/dbio_types_test.go +++ b/core/dbio/dbio_types_test.go @@ -98,6 +98,7 @@ func TestExplainTemplatePresentForSQLDatabases(t *testing.T) { noExplain := map[Type]bool{ TypeDbMongoDB: true, TypeDbElasticsearch: true, + TypeDbOpenSearch: true, TypeDbAzureTable: true, TypeDbBigTable: true, TypeDbPrometheus: true, diff --git a/core/dbio/templates/_properties.yaml b/core/dbio/templates/_properties.yaml index 541a69341..ac87e730e 100644 --- a/core/dbio/templates/_properties.yaml +++ b/core/dbio/templates/_properties.yaml @@ -883,6 +883,36 @@ elasticsearch: description: 'Default index name' examples: ['logs', 'documents', 'analytics'] +opensearch: + title: 'OpenSearch' + kind: database + required: ["url"] + url_template: 'opensearch://{user}:{password}@{url}' + docs: https://docs.slingdata.io/connections/database-connections/opensearch + properties: + name: + type: text + title: Connection Name + description: 'The name of the connection. No spaces are allowed, will be all caps. Example: OS_MAIN' + examples: ['OS_MAIN', 'OPENSEARCH_PROD', 'OS_LOGS'] + url: + type: text + description: 'OpenSearch URL' + examples: ['http://localhost:9200', 'https://opensearch.example.com:9200', 'https://my-domain.us-east-1.es.amazonaws.com'] + user: + type: text + description: 'Username for authentication' + examples: ['admin', 'opensearch', 'os_user'] + password: + type: text + description: 'Password for authentication' + examples: ['mypassword123'] + secret: true + index: + type: text + description: 'Default index name' + examples: ['logs', 'documents', 'analytics'] + d1: title: 'Cloudflare D1' kind: database diff --git a/core/dbio/templates/connections.json b/core/dbio/templates/connections.json index 299aa3105..1448fedba 100644 --- a/core/dbio/templates/connections.json +++ b/core/dbio/templates/connections.json @@ -1210,6 +1210,30 @@ } } }, + "opensearch": { + "name": "OpenSearch", + "description": "OpenSearch search and analytics engine", + "category": "database", + "properties": { + "url": { + "type": "string", + "description": "OpenSearch URL", + "required": true + }, + "user": { + "type": "string", + "description": "Username for authentication" + }, + "password": { + "type": "string", + "description": "Password for authentication" + }, + "index": { + "type": "string", + "description": "Default index name" + } + } + }, "prometheus": { "name": "Prometheus", "description": "Prometheus monitoring system", diff --git a/core/dbio/templates/opensearch.yaml b/core/dbio/templates/opensearch.yaml new file mode 100644 index 000000000..6e7fbb482 --- /dev/null +++ b/core/dbio/templates/opensearch.yaml @@ -0,0 +1,20 @@ +core: + explain: "" + incremental_select: '{incremental_where_cond}' + incremental_where: '{ "update_key": "{update_key}", "value": "{value}" }' + backfill_where: '{ "update_key": "{update_key}", "start_value": "{start_value}", "end_value": "{end_value}" }' + +variable: + tmp_folder: /tmp + timestamp_layout_str: '"{value}"' + timestamp_layout: '2006-01-02T15:04:05.000Z' + timestampz_layout_str: '"{value}"' + timestampz_layout: '2006-01-02T15:04:05.000Z' + date_layout_str: '"{value}"' + date_layout: '2006-01-02' + error_filter_table_exists: already + error_ignore_drop_table: NotFound + quote_char: '' + +native_type_map: + json: json diff --git a/core/sling/config.go b/core/sling/config.go index 0e688b946..64b98ecbb 100644 --- a/core/sling/config.go +++ b/core/sling/config.go @@ -737,7 +737,7 @@ func (cfg *Config) Prepare() (err error) { // validate capability to write switch cfg.Target.Type { - case dbio.TypeDbPrometheus, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbBigTable, dbio.TypeDbAzureTable: + case dbio.TypeDbPrometheus, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbOpenSearch, dbio.TypeDbBigTable, dbio.TypeDbAzureTable: return g.Error("sling cannot currently write to %s", cfg.Target.Type) case dbio.TypeDbIceberg: switch cfg.Mode { diff --git a/core/sling/task_run_write.go b/core/sling/task_run_write.go index 48ad24044..b35885afa 100644 --- a/core/sling/task_run_write.go +++ b/core/sling/task_run_write.go @@ -207,7 +207,7 @@ func (t *TaskExecution) WriteToDb(cfg *Config, df *iop.Dataflow, tgtConn databas } // write directly for iceberg / NoSQL (no SQL temp-table merge) - writeDirectly := g.In(tgtConn.GetType(), dbio.TypeDbIceberg, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbAzureTable, dbio.TypeDbScyllaDB) + writeDirectly := g.In(tgtConn.GetType(), dbio.TypeDbIceberg, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbOpenSearch, dbio.TypeDbAzureTable, dbio.TypeDbScyllaDB) // INSERT is upsert-by-PK for these stores upsertByInsert := g.In(tgtConn.GetType(), dbio.TypeDbScyllaDB, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable) diff --git a/go.mod b/go.mod index 0278c5632..9f35f9102 100644 --- a/go.mod +++ b/go.mod @@ -19,15 +19,15 @@ require ( github.com/apache/arrow-adbc/go/adbc v1.9.0 github.com/apache/arrow-go/v18 v18.6.0 github.com/apache/iceberg-go v0.6.0 - github.com/aws/aws-sdk-go-v2 v1.41.7 - github.com/aws/aws-sdk-go-v2/config v1.32.17 - github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2 v1.43.0 + github.com/aws/aws-sdk-go-v2/config v1.32.31 + github.com/aws/aws-sdk-go-v2/credentials v1.19.30 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 github.com/aws/aws-sdk-go-v2/service/athena v1.51.0 github.com/aws/aws-sdk-go-v2/service/glue v1.141.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 - github.com/aws/smithy-go v1.25.1 + github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 + github.com/aws/smithy-go v1.27.3 github.com/bits-and-blooms/bloom/v3 v3.7.0 github.com/charmbracelet/bubbletea v1.3.6 github.com/charmbracelet/huh v1.0.0 @@ -49,7 +49,6 @@ require ( github.com/google/uuid v1.6.0 github.com/integrii/flaggy v1.5.2 github.com/itchyny/gojq v0.12.18 - github.com/itchyny/timefmt-go v0.1.7 github.com/jackc/pgx/v5 v5.9.2 github.com/jaswdr/faker v1.19.1 github.com/jedib0t/go-pretty v4.3.0+incompatible @@ -59,19 +58,18 @@ require ( github.com/jmoiron/sqlx v1.3.3 github.com/json-iterator/go v1.1.12 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 - github.com/kardianos/service v1.2.4 github.com/klauspost/compress v1.18.6 github.com/kshedden/datareader v0.0.0-20210325133423-816b6ffdd011 github.com/labstack/echo/v4 v4.10.2 github.com/lib/pq v1.10.9 github.com/linkedin/goavro/v2 v2.12.0 github.com/maja42/goval v1.4.0 - github.com/mark3labs/mcp-go v0.57.0 github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-sqlite3 v1.14.34 github.com/microsoft/go-mssqldb v1.9.3 github.com/nikolalohinski/gonja/v2 v2.9.0 github.com/nqd/flat v0.1.1 + github.com/opensearch-project/opensearch-go/v2 v2.3.0 github.com/orcaman/concurrent-map/v2 v2.0.1 github.com/parquet-go/parquet-go v0.23.0 github.com/pkg/sftp v1.13.9 @@ -86,14 +84,11 @@ require ( github.com/shirou/gopsutil/v3 v3.24.4 github.com/shopspring/decimal v1.4.0 github.com/sijms/go-ora/v2 v2.8.24 - github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 - github.com/slingdata-io/godbc v0.0.9 github.com/slingdata-io/golyglot v1.0.20 - github.com/slingdata-io/sling v0.0.0-20260715135102-01cd9a07a3c8 github.com/snowflakedb/gosnowflake v1.17.1 github.com/spf13/cast v1.7.1 github.com/stretchr/testify v1.11.1 - github.com/tidwall/gjson v1.14.2 + github.com/tidwall/gjson v1.18.0 github.com/tidwall/jsonc v0.3.3 github.com/tidwall/sjson v1.2.5 github.com/timeplus-io/proton-go-driver/v2 v2.0.19 @@ -103,17 +98,12 @@ require ( github.com/xuri/excelize/v2 v2.9.1 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a go.mongodb.org/mongo-driver v1.14.0 - go.opentelemetry.io/otel v1.43.0 - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 - go.opentelemetry.io/otel/log v0.14.0 - go.opentelemetry.io/otel/sdk v1.43.0 - go.opentelemetry.io/otel/sdk/log v0.14.0 - golang.org/x/crypto v0.52.0 - golang.org/x/net v0.55.0 + golang.org/x/crypto v0.53.0 + golang.org/x/net v0.56.0 golang.org/x/oauth2 v0.36.0 - golang.org/x/sys v0.45.0 - golang.org/x/term v0.43.0 - golang.org/x/text v0.37.0 + golang.org/x/sys v0.46.0 + golang.org/x/term v0.44.0 + golang.org/x/text v0.38.0 google.golang.org/api v0.278.0 gopkg.in/cheggaaa/pb.v2 v2.0.7 gopkg.in/inf.v0 v0.9.1 @@ -139,7 +129,6 @@ require ( filippo.io/edwards25519 v1.1.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect - github.com/AlecAivazis/survey/v2 v2.3.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect @@ -148,7 +137,6 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andybalholm/brotli v1.2.1 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect @@ -158,23 +146,21 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.2 // indirect - github.com/boombuler/barcode v1.0.1 // indirect github.com/catppuccin/go v0.3.0 // indirect - github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect @@ -185,21 +171,12 @@ require ( github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect github.com/cockroachdb/apd/v3 v3.2.1 // indirect - github.com/coder/websocket v1.8.14 // indirect github.com/containerd/console v1.0.5 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/coreos/go-oidc/v3 v3.21.0 // indirect github.com/creasty/defaults v1.8.0 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/disintegration/imaging v1.6.2 // indirect - github.com/distribution/reference v0.6.0 // indirect github.com/dnephin/pflag v1.0.7 // indirect - github.com/docker/go-connections v0.7.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/domodwyer/mailyak/v3 v3.6.2 // indirect github.com/dvsekhvalnov/jose2go v1.7.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect @@ -211,31 +188,23 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect - github.com/ganigeorgiev/fexpr v0.4.1 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect - github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.5.0 // indirect - github.com/go-git/go-git/v5 v5.11.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-mysql-org/go-mysql v1.13.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/errors v0.21.0 // indirect github.com/go-openapi/strfmt v0.22.0 // indirect - github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/goccy/go-yaml v1.17.1 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect - github.com/google/jsonschema-go v0.4.2 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.7.0 // indirect @@ -243,22 +212,18 @@ require ( github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/gookit/color v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hashicorp/go-version v1.9.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/imdario/mergo v0.3.16 // indirect - github.com/jackc/pgio v1.0.0 // indirect - github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 // indirect + github.com/itchyny/timefmt-go v0.1.7 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -268,24 +233,18 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 // indirect github.com/labstack/gommon v0.4.0 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.20 // indirect - github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/moby/api v1.55.0 // indirect github.com/moby/moby/client v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -296,50 +255,38 @@ require ( github.com/muesli/termenv v0.16.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect - github.com/nalgeon/redka v0.5.2 // indirect - github.com/nats-io/nats.go v1.36.0 // indirect - github.com/nats-io/nkeys v0.4.7 // indirect - github.com/nats-io/nuid v1.0.1 // indirect - github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec // indirect - github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a // indirect - github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/pocketbase/dbx v1.11.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/pquerna/otp v1.5.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/pterm/pterm v0.12.83 // indirect github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.4 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.4.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/slingdata-io/pocketbase v0.22.136 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/substrait-io/substrait v0.87.0 // indirect github.com/substrait-io/substrait-go/v8 v8.1.0 // indirect github.com/substrait-io/substrait-protobuf/go v0.85.0 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tiendc/go-deepcopy v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect @@ -364,7 +311,6 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xuri/efp v0.0.1 // indirect github.com/xuri/nfp v0.0.1 // indirect - github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect @@ -372,21 +318,19 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect - go.uber.org/atomic v1.11.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.1 // indirect gocloud.dev v0.45.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/image v0.38.0 // indirect - golang.org/x/mod v0.35.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect + golang.org/x/mod v0.38.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.44.0 // indirect + golang.org/x/tools v0.47.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect @@ -400,13 +344,7 @@ require ( gopkg.in/mattn/go-colorable.v0 v0.1.0 // indirect gopkg.in/mattn/go-isatty.v0 v0.0.4 // indirect gopkg.in/mattn/go-runewidth.v0 v0.0.4 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect gotest.tools/gotestsum v1.8.2 // indirect - modernc.org/libc v1.72.0 // indirect - modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.49.1 // indirect ) // replace github.com/slingdata-io/golyglot => ../golyglot diff --git a/go.sum b/go.sum index d09dee811..3ede75f7a 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,6 @@ github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMb github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= github.com/99designs/keyring v1.2.2/go.mod h1:wes/FrByc8j7lFOAGLGSNEg8f/PaI3cgTBqhFkHUrPk= -github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= -github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= @@ -117,8 +115,6 @@ github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= -github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= @@ -148,63 +144,74 @@ github.com/apache/iceberg-go v0.6.0 h1:tOVhC5BhBOEgPTowo5AVrPnAgsDo00qEbUPprFcLd github.com/apache/iceberg-go v0.6.0/go.mod h1:kESfDlyaW/6hK0WW6TzA4EWps+0NMhhrDYE4qajVjlo= github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s= github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= -github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aws/aws-sdk-go v1.44.263/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= -github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.43.0 h1:fharf/WhbRAVZ1du0QL7roNFxZ6T/sWr+4Ni617bwSI= +github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= -github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= -github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= -github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= -github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/config v1.18.25/go.mod h1:dZnYpD5wTW/dQF0rRNLVypB396zWCcPiBIvdvSWHEg4= +github.com/aws/aws-sdk-go-v2/config v1.32.31 h1:n4nY9O3QKoHIkL85EX+V8RcMFtOhlpTFhGArg915PXk= +github.com/aws/aws-sdk-go-v2/config v1.32.31/go.mod h1:PN0NYDCCoOpGGsZ2+elDUidmHfQBPyYzN2GCgl8HEBs= +github.com/aws/aws-sdk-go-v2/credentials v1.13.24/go.mod h1:jYPYi99wUOPIFi0rhiOvXeSEReVOzBqFNOX5bXYoG2o= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30 h1:TTCvvzFU6gXa4iJecNG/0F/B0oYTiazoRECr2XyLHrY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30/go.mod h1:jKxAp2AEncnliinzpgOSZDFv6+VjvWhjw/AtbfsWT9U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3/go.mod h1:4Q0UFP0YJf0NrsEuEYHpM9fTSEVnD16Z3uyEF7J9JGM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 h1:kfVL5wAunCJycL6MOQ6aNh6PlAYEymflcjuKmrWUA0o= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31/go.mod h1:nWfRNDAppujCQgOUd43lKT4yeLv9z3nJ3bw1G3BgQKo= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 h1:Z8F3hfCY33IGpJjFAnv0wvtv1FIKj1GHmRDEYqy64tw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 h1:hyOxUyXdh3AyjE93gBgsfziJag9ACwcs+ZpDBLzi8mw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34/go.mod h1:Etz2dj6UHYuw+Xw830KfzCfWGMzqvUTCjUj5b76GVDc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 h1:0MrUL35H/Y4kdFfItoR5jCgtDQ4Z/8LudAoIHRfA4hE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc= github.com/aws/aws-sdk-go-v2/service/athena v1.51.0 h1:Fmh66wriOXgBJDnA/78aur8hH6DrvrWz7ZMzdoS33Yw= github.com/aws/aws-sdk-go-v2/service/athena v1.51.0/go.mod h1:xsG8Y2fMenmHTdukyknTUO1uQhEZ/entaNHvPmD1klE= github.com/aws/aws-sdk-go-v2/service/glue v1.141.0 h1:QeZdzcEB+eB76YvIHqZyxsmxH+f9f2kKDdz48UC4HCM= github.com/aws/aws-sdk-go-v2/service/glue v1.141.0/go.mod h1:FZCvt95CcJWRkZNRcmMW1uQkpDHmyUSkZfz3awyZgqg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.27/go.mod h1:EOwBD4J4S5qYszS5/3DpkejfuK+Z5/1uzICfPaZLtqw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 h1:w2SIhW92DZPFrSL4ksVCr8IYff5OZwIcxg8+95tzvAI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= -github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= -github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= -github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= -github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 h1:OHH5iTQvVGmfHjX/5Q+vFuA/Rf2x6/95aJ/75QCQSm4= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0/go.mod h1:mCF3AK9PpL49oOrhniUXWAfhVBVQ/XbytoE5eccZUIs= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.10/go.mod h1:ouy2P4z6sJN70fR3ka3wD3Ro3KezSxU6eKGQI2+2fjI= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 h1:CaJyYhxBE0M/HJX/YvSaSmQlsI91VHB0lKU8LtLxL3A= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0/go.mod h1:+e6BMRMPjBQoCw/WovYR9GLy2IU0z4Q77smOB1DraSg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10/go.mod h1:AFvkxc8xfBe8XA+5St5XIHHrQQtkxqrRincx4hmMHOk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 h1:tC323YV77QdafeBr6LUhLDTsboyuyHLNRwAyCP44kGU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0/go.mod h1:SfLK1sgviHmbI+MozR9iDwDjL4cdCVZtahsjoR+z7wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.19.0/go.mod h1:BgQOMsg8av8jset59jelyPW7NoZcZXLVpDsXunGDrk8= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 h1:Pd6PNlp4t8PTXxqzstICl52Wsy78vpjFZ7PRUj44mJc= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0/go.mod h1:rmQ0TnHzuLPmabgjPcsywhsSOmaBDgzR4zvDxSPsGdg= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250407191926-092f3e54b837 h1:8eMceEa0ib+nqJuGsyowuZaVBVAr685oK6WrNIit+0g= github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250407191926-092f3e54b837/go.mod h1:9Oj/8PZn3D5Ftp/Z1QWrIEFE0daERMqfJawL9duHRfc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -213,9 +220,6 @@ github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.7.0 h1:VfknkqV4xI+PsaDIsoHueyxVDZrfvMn56jeWUzvzdls= github.com/bits-and-blooms/bloom/v3 v3.7.0/go.mod h1:VKlUSvp0lFIYqxJjzdnSsZEw4iHb1kOL2tfHTgyJBHg= -github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs= -github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= @@ -267,8 +271,6 @@ github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/T github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= -github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= -github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/compose-spec/compose-go/v2 v2.10.2 h1:USa1NUbDcl/cjb8T9iwnuFsnO79H+2ho2L5SjFKz3uI= github.com/compose-spec/compose-go/v2 v2.10.2/go.mod h1:ZU6zlcweCZKyiB7BVfCizQT9XmkEIMFE+PRZydVcsZg= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -298,13 +300,10 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= github.com/databricks/zerobus-sdk/go v1.6.0 h1:TTArmmZMx1tygLQA1fIo70qZl7rEFNaDHsDA/xFBYzA= @@ -315,12 +314,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= -github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= -github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= -github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/docker/buildx v0.33.0 h1:xuZeuQe/C/2tvLDgiIA6+Ynq3FFWSfsGNWIHM3q1hD8= @@ -337,8 +332,6 @@ github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= -github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -377,8 +370,6 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fergusstrange/embedded-postgres v1.31.0 h1:JmRxw2BcPRcU141nOEuGXbIU6jsh437cBB40rmftZSk= -github.com/fergusstrange/embedded-postgres v1.31.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk= github.com/flarco/bigquery v0.0.9 h1:WfxO6XuuHZTJV+55Bq24FhdHYpmAOzgVk9xOcJpEecY= github.com/flarco/bigquery v0.0.9/go.mod h1:IpRSw4quaXxHjFyDSXUo7B6v+XcNF2pSmnNfeqXa/gM= github.com/flarco/databricks-sql-go v0.0.0-20250613120556-51f7c1f3b4ad h1:z5mgsXmNXsgskClg/s6zelILFihJTyK6x7+zX1jUgyU= @@ -398,8 +389,6 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= -github.com/ganigeorgiev/fexpr v0.4.1 h1:hpUgbUEEWIZhSDBtf4M9aUNfQQ0BZkGRaMePy7Gcx5k= -github.com/ganigeorgiev/fexpr v0.4.1/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -411,12 +400,6 @@ github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= -github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= -github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= -github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -424,17 +407,12 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-mysql-org/go-mysql v1.13.0 h1:Hlsa5x1bX/wBFtMbdIOmb6YzyaVNBWnwrb8gSIEPMDc= -github.com/go-mysql-org/go-mysql v1.13.0/go.mod h1:FQxw17uRbFvMZFK+dPtIPufbU46nBdrGaxOw0ac9MFs= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY= github.com/go-openapi/errors v0.21.0/go.mod h1:jxNTMUxRCKj65yb/okJGEtahVd7uvWnuWfj53bse4ho= github.com/go-openapi/strfmt v0.22.0 h1:Ew9PnEYc246TwrEspvBdDHS4BVKXy/AOVsfqGDgAcaI= github.com/go-openapi/strfmt v0.22.0/go.mod h1:HzJ9kokGIju3/K6ap8jL+OlGAbjpSv27135Yr9OivU4= -github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= -github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= @@ -456,8 +434,6 @@ github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= -github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= @@ -513,8 +489,6 @@ github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6 github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= -github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= @@ -551,6 +525,7 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0 h1:WcmKMm43DR7RdtlkEXQJyo5ws8iTp98CyhCCbOHMvNI= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= @@ -576,8 +551,6 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= -github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -601,8 +574,6 @@ github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 h1:+PJCokZ2BhyDKlncScmiNzBwqOx+yH1i8xRlWN/wn6A= -github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510/go.mod h1:UzTJ5Jjuf4O9hYWW+HYVwVldYz9J7CaePW0iuNJkrPQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= @@ -619,8 +590,6 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaswdr/faker v1.19.1 h1:xBoz8/O6r0QAR8eEvKJZMdofxiRH+F0M/7MU9eNKhsM= github.com/jaswdr/faker v1.19.1/go.mod h1:x7ZlyB1AZqwqKZgyQlnqEG8FDptmHlncA5u2zY/yi6w= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -660,10 +629,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= -github.com/kardianos/service v1.2.4 h1:XNlGtZOYNx2u91urOdg/Kfmc+gfmuIo1Dd3rEi2OgBk= -github.com/kardianos/service v1.2.4/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -694,8 +659,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= -github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 h1:FwuzbVh87iLiUQj1+uQUsuw9x5t9m5n5g7rG7o4svW4= -github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61/go.mod h1:paQfF1YtHe+GrGg5fOgjsjoCX/UKDr9bc1DoWpZfns8= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -715,18 +678,12 @@ github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/maja42/goval v1.4.0 h1:tlX0X+GvjKzWW2Q6qzWwL4Av2KV1bLtzxwzgxiiwEPc= github.com/maja42/goval v1.4.0/go.mod h1:LDMwF8ocOwIsMZdwoyHC/3UpV8ABDwEzalxkVV2z/rI= -github.com/mark3labs/mcp-go v0.57.0 h1:jzWKyCzdWnwnZt05cvcQQ+ngiUl2RnixXJa7Kj4qP1E= -github.com/mark3labs/mcp-go v0.57.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= -github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= -github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -746,9 +703,6 @@ github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= -github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microsoft/go-mssqldb v1.9.3 h1:hy4p+LDC8LIGvI3JATnLVmBOLMJbmn5X400mr5j0lPs= github.com/microsoft/go-mssqldb v1.9.3/go.mod h1:GBbW9ASTiDC+mpgWDGKdm3FnFLTUsLYN3iFL90lQ+PA= @@ -813,14 +767,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nalgeon/redka v0.5.2 h1:CX71v88kYj55EwJ10zq7U2eJdH0xcLAIjvKFvUMoM0o= -github.com/nalgeon/redka v0.5.2/go.mod h1:vLxjY3XS9IwBID2YEFWeeMiN4Ar/DtKd4JW62JTAxuU= -github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU= -github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= -github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= -github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= @@ -844,6 +790,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runc v1.1.13 h1:98S2srgG9vw0zWcDpFMn5TRrh8kLxa/5OFUstuUhmRs= github.com/opencontainers/runc v1.1.13/go.mod h1:R016aXacfp/gwQBYw2FDGa9m+n6atbLWrYY8hNMT/sA= +github.com/opensearch-project/opensearch-go/v2 v2.3.0 h1:nQIEMr+A92CkhHrZgUhcfsrZjibvB3APXf2a1VwCmMQ= +github.com/opensearch-project/opensearch-go/v2 v2.3.0/go.mod h1:8LDr9FCgUTVoT+5ESjc2+iaZuldqE+23Iq0r1XeNue8= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= @@ -860,13 +808,8 @@ github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9F github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= -github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec h1:3EiGmeJWoNixU+EwllIn26x6s4njiWRXewdx2zlYa84= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a h1:WIhmJBlNGmnCWH6TLMdZfNEDaiU8cFpZe3iaqDbQ0M8= -github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= -github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d h1:3Ej6eTuLZp25p3aH/EXdReRHY12hjZYs3RrGp7iLdag= -github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d/go.mod h1:+8feuexTKcXHZF/dkDfvCwEyBAmgb4paFc3/WeYV2eE= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -879,13 +822,9 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pocketbase/dbx v1.11.0 h1:LpZezioMfT3K4tLrqA55wWFw1EtH1pM4tzSVa7kgszU= -github.com/pocketbase/dbx v1.11.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= -github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= @@ -998,12 +937,8 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EE github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260806214312-6da3e7189c98 h1:HT0E/6EiVUe8wew4uIgYEpUYPHcswmGkxpbbqFges3g= github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260806214312-6da3e7189c98/go.mod h1:ikEb6zQgczMp7Alx/RYGaBEn/Mwwz1kfXbirnkbgWUo= -github.com/slingdata-io/godbc v0.0.9 h1:wh5MSI6l+eyTQ8pSbQcO4j195HwBXrVmNYRnHaGJKK8= -github.com/slingdata-io/godbc v0.0.9/go.mod h1:oBLg0zDZSK1BhLpcNhZjsvCLdfvB4OiTReAreXCrb3M= github.com/slingdata-io/golyglot v1.0.20 h1:qZry5N4xKEOcJWdRStuH7W9zH/Xu5f6YH/rIEBsBttU= github.com/slingdata-io/golyglot v1.0.20/go.mod h1:Gw0MF6BgUt9XUWAxYahbTsfWzaTdQQoSccUXbOxlGUs= -github.com/slingdata-io/pocketbase v0.22.136 h1:RtAvPvYdK0qm9EB1r8GzNeEfSiqDK+tV8jyxwbpKKBA= -github.com/slingdata-io/pocketbase v0.22.136/go.mod h1:RYAdoMZtW+3OIgKqg+YhgWGIiwjtcBHGxRcVF2+1klA= github.com/snowflakedb/gosnowflake v1.17.1 h1:sBYExPDRv6hHF7fCqeXMT745L326Byw/cROxvCiEJzo= github.com/snowflakedb/gosnowflake v1.17.1/go.mod h1:TaHvQGh9MA2lopZZMm1AvvENDfwcnKtuskIr1e6Fpic= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= @@ -1032,6 +967,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -1047,15 +983,17 @@ github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44Xt github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= github.com/testcontainers/testcontainers-go/modules/compose v0.42.0 h1:+t1ZN31TD36cwxmeLqGwe7wIdvblBm0Z+vlj4SX8Mv0= github.com/testcontainers/testcontainers-go/modules/compose v0.42.0/go.mod h1:CfMpouDHqNTCHC8CijEURU2ZotTV3QhH6pXd48s6ofk= -github.com/tidwall/gjson v1.14.2 h1:6BBkirS0rAHjumnjHF6qgy5d2YAJ1TLIaFE2lzfOLqo= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/jsonc v0.3.3 h1:RVQqL3xFfDkKKXIDsrBiVQiEpBtxoKbmMXONb2H/y2w= github.com/tidwall/jsonc v0.3.3/go.mod h1:dw+3CIxqHi+t8eFSpzzMlcVYxKp08UP5CD8/uSFCyJE= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo= @@ -1136,8 +1074,6 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/dburl v0.3.0 h1:KGkeJB/oQhY/DeeJoYl/1+pNE/JnF6ouAuA8nzpQEQ8= github.com/xo/dburl v0.3.0/go.mod h1:TM8VMBT+LWqC3MBOulZjb8FAthcvZq0t/qvDLwS6skU= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= @@ -1151,8 +1087,6 @@ github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= -github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= @@ -1184,8 +1118,6 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:Oyrsyzu go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY= @@ -1198,16 +1130,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJK go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= -go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= -go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= -go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= @@ -1215,22 +1141,10 @@ go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLh go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= -go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -1254,12 +1168,11 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= -golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1274,8 +1187,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1287,7 +1200,6 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1298,14 +1210,15 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1328,13 +1241,12 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1370,27 +1282,27 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4= -golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -1402,8 +1314,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -1416,7 +1328,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1426,8 +1337,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1445,7 +1356,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1508,10 +1418,6 @@ gopkg.in/mattn/go-isatty.v0 v0.0.4 h1:NtS1rQGQr4IaFWBGz4Cz4BhB///gyys4gDVtKA7hIs gopkg.in/mattn/go-isatty.v0 v0.0.4/go.mod h1:wt691ab7g0X4ilKZNmMII3egK0bTxl37fEn/Fwbd8gc= gopkg.in/mattn/go-runewidth.v0 v0.0.4 h1:r0P71TnzQDlNIcizCqvPSSANoFa3WVGtcNJf3TWurcY= gopkg.in/mattn/go-runewidth.v0 v0.0.4/go.mod h1:BmXejnxvhwdaATwiJbB1vZ2dtXkQKZGu9yLFCZb4msQ= -gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1539,36 +1445,14 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U= -modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8= -modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU= -modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0= -modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= -modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= -modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= -modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= -modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= -modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= -modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= -modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= -modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= -modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U= modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= -modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= -modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= -pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= From ddd3771813df15ff3d0bd296e4f60ce0b7e1d861 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Tue, 22 Sep 2026 08:32:18 -0300 Subject: [PATCH 29/56] Refactor code structure for improved readability and maintainability --- go.mod | 134 +++++++++++++++++++-------- go.sum | 281 ++++++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 308 insertions(+), 107 deletions(-) diff --git a/go.mod b/go.mod index 9f35f9102..58696d340 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/google/uuid v1.6.0 github.com/integrii/flaggy v1.5.2 github.com/itchyny/gojq v0.12.18 + github.com/itchyny/timefmt-go v0.1.7 github.com/jackc/pgx/v5 v5.9.2 github.com/jaswdr/faker v1.19.1 github.com/jedib0t/go-pretty v4.3.0+incompatible @@ -58,12 +59,14 @@ require ( github.com/jmoiron/sqlx v1.3.3 github.com/json-iterator/go v1.1.12 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 + github.com/kardianos/service v1.3.0 github.com/klauspost/compress v1.18.6 github.com/kshedden/datareader v0.0.0-20210325133423-816b6ffdd011 github.com/labstack/echo/v4 v4.10.2 github.com/lib/pq v1.10.9 github.com/linkedin/goavro/v2 v2.12.0 github.com/maja42/goval v1.4.0 + github.com/mark3labs/mcp-go v1.1.0 github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-sqlite3 v1.14.34 github.com/microsoft/go-mssqldb v1.9.3 @@ -84,10 +87,13 @@ require ( github.com/shirou/gopsutil/v3 v3.24.4 github.com/shopspring/decimal v1.4.0 github.com/sijms/go-ora/v2 v2.8.24 + github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 + github.com/slingdata-io/godbc v0.0.9 github.com/slingdata-io/golyglot v1.0.20 + github.com/slingdata-io/sling v0.0.0-00010101000000-000000000000 github.com/snowflakedb/gosnowflake v1.17.1 github.com/spf13/cast v1.7.1 - github.com/stretchr/testify v1.11.1 + github.com/stretchr/testify v1.12.1 github.com/tidwall/gjson v1.18.0 github.com/tidwall/jsonc v0.3.3 github.com/tidwall/sjson v1.2.5 @@ -98,12 +104,17 @@ require ( github.com/xuri/excelize/v2 v2.9.1 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a go.mongodb.org/mongo-driver v1.14.0 - golang.org/x/crypto v0.53.0 - golang.org/x/net v0.56.0 + go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 + go.opentelemetry.io/otel/log v0.22.0 + go.opentelemetry.io/otel/sdk v1.46.0 + go.opentelemetry.io/otel/sdk/log v0.22.0 + golang.org/x/crypto v0.55.0 + golang.org/x/net v0.58.0 golang.org/x/oauth2 v0.36.0 - golang.org/x/sys v0.46.0 - golang.org/x/term v0.44.0 - golang.org/x/text v0.38.0 + golang.org/x/sys v0.47.0 + golang.org/x/term v0.45.0 + golang.org/x/text v0.41.0 google.golang.org/api v0.278.0 gopkg.in/cheggaaa/pb.v2 v2.0.7 gopkg.in/inf.v0 v0.9.1 @@ -119,7 +130,7 @@ require ( atomicgo.dev/cursor v0.2.0 // indirect atomicgo.dev/keyboard v0.2.9 // indirect atomicgo.dev/schedule v0.1.0 // indirect - cel.dev/expr v0.25.1 // indirect + cel.dev/expr v0.25.2 // indirect cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect @@ -129,14 +140,16 @@ require ( filippo.io/edwards25519 v1.1.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect + github.com/AlecAivazis/survey/v2 v2.3.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/ClickHouse/ch-go v0.65.1 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andybalholm/brotli v1.2.1 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect @@ -160,7 +173,9 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.24.2 // indirect + github.com/boombuler/barcode v1.0.1 // indirect github.com/catppuccin/go v0.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect @@ -169,42 +184,59 @@ require ( github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect - github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/cockroachdb/apd/v3 v3.2.1 // indirect + github.com/coder/websocket v1.8.14 // indirect github.com/containerd/console v1.0.5 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/coreos/go-oidc/v3 v3.21.0 // indirect github.com/creasty/defaults v1.8.0 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/danieljoos/wincred v1.2.2 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/disintegration/imaging v1.6.2 // indirect + github.com/distribution/reference v0.6.0 // indirect github.com/dnephin/pflag v1.0.7 // indirect + github.com/docker/go-connections v0.7.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/domodwyer/mailyak/v3 v3.6.2 // indirect github.com/dvsekhvalnov/jose2go v1.7.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect - github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect - github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/exasol/error-reporting-go v0.2.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect + github.com/ganigeorgiev/fexpr v0.4.1 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-git/v5 v5.11.0 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-mysql-org/go-mysql v1.13.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/errors v0.21.0 // indirect - github.com/go-openapi/strfmt v0.22.0 // indirect + github.com/go-openapi/errors v0.22.8 // indirect + github.com/go-openapi/strfmt v0.27.0 // indirect + github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/goccy/go-yaml v1.17.1 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/jsonschema-go v0.4.2 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.7.0 // indirect @@ -212,18 +244,22 @@ require ( github.com/googleapis/gax-go/v2 v2.22.0 // indirect github.com/gookit/color v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/go-version v1.9.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/imdario/mergo v0.3.16 // indirect - github.com/itchyny/timefmt-go v0.1.7 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -233,18 +269,23 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 // indirect github.com/labstack/gommon v0.4.0 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.20 // indirect + github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/moby/api v1.55.0 // indirect github.com/moby/moby/client v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -255,32 +296,43 @@ require ( github.com/muesli/termenv v0.16.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect - github.com/oklog/ulid v1.3.1 // indirect + github.com/nalgeon/redka v0.5.2 // indirect + github.com/nats-io/nats.go v1.36.0 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/oklog/ulid/v2 v2.1.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec // indirect + github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a // indirect + github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/pocketbase/dbx v1.11.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/pquerna/otp v1.5.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/pterm/pterm v0.12.83 // indirect github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.4 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.4.0 // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/slingdata-io/pocketbase v0.22.136 // indirect + github.com/spiffe/go-spiffe/v2 v2.7.0 // indirect github.com/stretchr/objx v0.5.3 // indirect github.com/substrait-io/substrait v0.87.0 // indirect github.com/substrait-io/substrait-go/v8 v8.1.0 // indirect @@ -311,32 +363,36 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xuri/efp v0.0.1 // indirect github.com/xuri/nfp v0.0.1 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect - go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect gocloud.dev v0.45.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/image v0.38.0 // indirect golang.org/x/mod v0.38.0 // indirect golang.org/x/sync v0.22.0 // indirect - golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect + golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 // indirect golang.org/x/time v0.15.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/tools v0.48.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect - google.golang.org/grpc v1.80.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/grpc v1.83.1 // indirect + google.golang.org/protobuf v1.36.12 // indirect gopkg.in/VividCortex/ewma.v1 v1.1.1 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/fatih/color.v1 v1.7.0 // indirect @@ -344,7 +400,13 @@ require ( gopkg.in/mattn/go-colorable.v0 v0.1.0 // indirect gopkg.in/mattn/go-isatty.v0 v0.0.4 // indirect gopkg.in/mattn/go-runewidth.v0 v0.0.4 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gotest.tools/gotestsum v1.8.2 // indirect + modernc.org/libc v1.72.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.49.1 // indirect ) // replace github.com/slingdata-io/golyglot => ../golyglot diff --git a/go.sum b/go.sum index 3ede75f7a..66eb6dac4 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8= atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ= atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs= atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU= -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -53,6 +53,8 @@ github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMb github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= github.com/99designs/keyring v1.2.2/go.mod h1:wes/FrByc8j7lFOAGLGSNEg8f/PaI3cgTBqhFkHUrPk= +github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ= +github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4= @@ -94,8 +96,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7Oputl github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e h1:rd4bOvKmDIx0WeTv9Qz+hghsgyjikFiPrseXHlKepO0= github.com/DefangLabs/secret-detector v0.0.0-20250403165618-22662109213e/go.mod h1:blbwPQh4DTlCZEfk1BLU4oMIhLda2U+A840Uag9DsZw= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 h1:l7+6kwRMJNwdCvYdDl7Eax+wzEYHSnNY7zrrfbhDdTA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0/go.mod h1:IA1C1U7jO/ENqm/vhi7V9YYpBsp+IMyqNrEN94N7tVc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.55.0 h1:7t/qx5Ost0s0wbA/VDrByOooURhp+ikYwv20i9Y07TQ= @@ -115,6 +117,8 @@ github.com/MarvinJWendt/testza v0.5.2 h1:53KDo64C1z/h/d/stCYCPY69bt/OSwjq5KpFNwi github.com/MarvinJWendt/testza v0.5.2/go.mod h1:xu53QFE5sCdjtMCKk8YMQ2MnymimEctc4n3EjyIYvEY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= @@ -144,6 +148,7 @@ github.com/apache/iceberg-go v0.6.0 h1:tOVhC5BhBOEgPTowo5AVrPnAgsDo00qEbUPprFcLd github.com/apache/iceberg-go v0.6.0/go.mod h1:kESfDlyaW/6hK0WW6TzA4EWps+0NMhhrDYE4qajVjlo= github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s= github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= @@ -212,6 +217,7 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY= github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -220,6 +226,9 @@ github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.7.0 h1:VfknkqV4xI+PsaDIsoHueyxVDZrfvMn56jeWUzvzdls= github.com/bits-and-blooms/bloom/v3 v3.7.0/go.mod h1:VKlUSvp0lFIYqxJjzdnSsZEw4iHb1kOL2tfHTgyJBHg= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= @@ -267,10 +276,12 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= -github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= +github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= +github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/compose-spec/compose-go/v2 v2.10.2 h1:USa1NUbDcl/cjb8T9iwnuFsnO79H+2ho2L5SjFKz3uI= github.com/compose-spec/compose-go/v2 v2.10.2/go.mod h1:ZU6zlcweCZKyiB7BVfCizQT9XmkEIMFE+PRZydVcsZg= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -300,10 +311,13 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= github.com/databricks/zerobus-sdk/go v1.6.0 h1:TTArmmZMx1tygLQA1fIo70qZl7rEFNaDHsDA/xFBYzA= @@ -314,8 +328,12 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/docker/buildx v0.33.0 h1:xuZeuQe/C/2tvLDgiIA6+Ynq3FFWSfsGNWIHM3q1hD8= @@ -332,6 +350,8 @@ github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= +github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -350,13 +370,13 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= -github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= -github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/exasol/error-reporting-go v0.2.0 h1:nKIe4zYiTHbYrKJRlSNJcmGjTJCZredDh5akVHfIbRs= @@ -368,8 +388,10 @@ github.com/exasol/exasol-test-setup-abstraction-server/go-client v0.3.11/go.mod github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/fergusstrange/embedded-postgres v1.31.0 h1:JmRxw2BcPRcU141nOEuGXbIU6jsh437cBB40rmftZSk= +github.com/fergusstrange/embedded-postgres v1.31.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk= github.com/flarco/bigquery v0.0.9 h1:WfxO6XuuHZTJV+55Bq24FhdHYpmAOzgVk9xOcJpEecY= github.com/flarco/bigquery v0.0.9/go.mod h1:IpRSw4quaXxHjFyDSXUo7B6v+XcNF2pSmnNfeqXa/gM= github.com/flarco/databricks-sql-go v0.0.0-20250613120556-51f7c1f3b4ad h1:z5mgsXmNXsgskClg/s6zelILFihJTyK6x7+zX1jUgyU= @@ -389,6 +411,8 @@ github.com/fvbommel/sortorder v1.1.0 h1:fUmoe+HLsBTctBDoaBwpQo5N+nrCp8g/BjKb/6ZQ github.com/fvbommel/sortorder v1.1.0/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= +github.com/ganigeorgiev/fexpr v0.4.1 h1:hpUgbUEEWIZhSDBtf4M9aUNfQQ0BZkGRaMePy7Gcx5k= +github.com/ganigeorgiev/fexpr v0.4.1/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -400,19 +424,32 @@ github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= +github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-mysql-org/go-mysql v1.13.0 h1:Hlsa5x1bX/wBFtMbdIOmb6YzyaVNBWnwrb8gSIEPMDc= +github.com/go-mysql-org/go-mysql v1.13.0/go.mod h1:FQxw17uRbFvMZFK+dPtIPufbU46nBdrGaxOw0ac9MFs= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY= -github.com/go-openapi/errors v0.21.0/go.mod h1:jxNTMUxRCKj65yb/okJGEtahVd7uvWnuWfj53bse4ho= -github.com/go-openapi/strfmt v0.22.0 h1:Ew9PnEYc246TwrEspvBdDHS4BVKXy/AOVsfqGDgAcaI= -github.com/go-openapi/strfmt v0.22.0/go.mod h1:HzJ9kokGIju3/K6ap8jL+OlGAbjpSv27135Yr9OivU4= +github.com/go-openapi/errors v0.22.8 h1:oP7sW7TWc3wFFjrzzj0nI83H2qMBkNjNfSd+XRejk/I= +github.com/go-openapi/errors v0.22.8/go.mod h1:BuUoHcYrU6E7V9gfj1I5wLQqgtIHnup/alXZ8KdgQ0w= +github.com/go-openapi/strfmt v0.27.0 h1:kbcTeaD9TXuXD0hhMXzuYa1sdTo6+dWGvwjW93E80IM= +github.com/go-openapi/strfmt v0.27.0/go.mod h1:s/qhDqfY72irigXUGJmtgid2Rm+3tnz3k8hZaRmvWYc= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= @@ -434,6 +471,8 @@ github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= @@ -489,6 +528,8 @@ github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6 github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk= github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= @@ -525,10 +566,9 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway v1.5.0 h1:WcmKMm43DR7RdtlkEXQJyo5ws8iTp98CyhCCbOHMvNI= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -551,6 +591,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -574,6 +616,8 @@ github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 h1:+PJCokZ2BhyDKlncScmiNzBwqOx+yH1i8xRlWN/wn6A= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510/go.mod h1:UzTJ5Jjuf4O9hYWW+HYVwVldYz9J7CaePW0iuNJkrPQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= @@ -590,6 +634,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaswdr/faker v1.19.1 h1:xBoz8/O6r0QAR8eEvKJZMdofxiRH+F0M/7MU9eNKhsM= github.com/jaswdr/faker v1.19.1/go.mod h1:x7ZlyB1AZqwqKZgyQlnqEG8FDptmHlncA5u2zY/yi6w= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -629,6 +675,10 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFdTI= +github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -659,6 +709,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= +github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 h1:FwuzbVh87iLiUQj1+uQUsuw9x5t9m5n5g7rG7o4svW4= +github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61/go.mod h1:paQfF1YtHe+GrGg5fOgjsjoCX/UKDr9bc1DoWpZfns8= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -678,12 +730,18 @@ github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/maja42/goval v1.4.0 h1:tlX0X+GvjKzWW2Q6qzWwL4Av2KV1bLtzxwzgxiiwEPc= github.com/maja42/goval v1.4.0/go.mod h1:LDMwF8ocOwIsMZdwoyHC/3UpV8ABDwEzalxkVV2z/rI= +github.com/mark3labs/mcp-go v1.1.0 h1:9kZwJreq58QIaM+5h+k4JXbG0f0y0j9+DJCV5rGBkJA= +github.com/mark3labs/mcp-go v1.1.0/go.mod h1:r2fW4o3wsoJ7IMsx1Wuq5xeP8PRGXPDfNveoGAYbb/s= +github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= +github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -703,6 +761,9 @@ github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/microsoft/go-mssqldb v1.9.3 h1:hy4p+LDC8LIGvI3JATnLVmBOLMJbmn5X400mr5j0lPs= github.com/microsoft/go-mssqldb v1.9.3/go.mod h1:GBbW9ASTiDC+mpgWDGKdm3FnFLTUsLYN3iFL90lQ+PA= @@ -712,8 +773,6 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/buildkit v0.29.0 h1:wxLEFbCOJntEDjSNNN2YWd8zxltZxT5muDQ0LzpbtpU= github.com/moby/buildkit v0.29.0/go.mod h1:Dmv2FeDe34t75QuzeU87rBoZpAAkcpT5zeu4hXzmASc= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= @@ -767,6 +826,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nalgeon/redka v0.5.2 h1:CX71v88kYj55EwJ10zq7U2eJdH0xcLAIjvKFvUMoM0o= +github.com/nalgeon/redka v0.5.2/go.mod h1:vLxjY3XS9IwBID2YEFWeeMiN4Ar/DtKd4JW62JTAxuU= +github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU= +github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= @@ -776,8 +843,8 @@ github.com/nikolalohinski/gonja/v2 v2.9.0 h1:QICtNWj0siM3PF5xUjVod/l+C9XnGfI+wrf github.com/nikolalohinski/gonja/v2 v2.9.0/go.mod h1:UIzXPVuOsr5h7dZ5DUbqk3/Z7oFA/NLGQGMjqT4L2aU= github.com/nqd/flat v0.1.1 h1:sKa3CZipbb7WYD9tORSJD6Ylm/00f6D9Wse7+UkSa+4= github.com/nqd/flat v0.1.1/go.mod h1:FOuslZmNY082wVfVUUb7qAGWKl8z8Nor9FMg+Xj2Nss= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/oklog/ulid/v2 v2.1.1 h1:suPZ4ARWLOJLegGFiZZ1dFAkqzhMjL3J1TzI+5wHz8s= +github.com/oklog/ulid/v2 v2.1.1/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo/v2 v2.23.4 h1:ktYTpKJAVZnDT4VjxSbiBenUjmlL/5QkBEocaWXiQus= @@ -802,14 +869,20 @@ github.com/parquet-go/parquet-go v0.23.0/go.mod h1:MnwbUcFHU6uBYMymKAlPPAw9yh3kE github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec h1:3EiGmeJWoNixU+EwllIn26x6s4njiWRXewdx2zlYa84= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a h1:WIhmJBlNGmnCWH6TLMdZfNEDaiU8cFpZe3iaqDbQ0M8= +github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= +github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d h1:3Ej6eTuLZp25p3aH/EXdReRHY12hjZYs3RrGp7iLdag= +github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d/go.mod h1:+8feuexTKcXHZF/dkDfvCwEyBAmgb4paFc3/WeYV2eE= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -822,9 +895,13 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pocketbase/dbx v1.11.0 h1:LpZezioMfT3K4tLrqA55wWFw1EtH1pM4tzSVa7kgszU= +github.com/pocketbase/dbx v1.11.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs= +github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= @@ -937,8 +1014,12 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EE github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260806214312-6da3e7189c98 h1:HT0E/6EiVUe8wew4uIgYEpUYPHcswmGkxpbbqFges3g= github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260806214312-6da3e7189c98/go.mod h1:ikEb6zQgczMp7Alx/RYGaBEn/Mwwz1kfXbirnkbgWUo= +github.com/slingdata-io/godbc v0.0.9 h1:wh5MSI6l+eyTQ8pSbQcO4j195HwBXrVmNYRnHaGJKK8= +github.com/slingdata-io/godbc v0.0.9/go.mod h1:oBLg0zDZSK1BhLpcNhZjsvCLdfvB4OiTReAreXCrb3M= github.com/slingdata-io/golyglot v1.0.20 h1:qZry5N4xKEOcJWdRStuH7W9zH/Xu5f6YH/rIEBsBttU= github.com/slingdata-io/golyglot v1.0.20/go.mod h1:Gw0MF6BgUt9XUWAxYahbTsfWzaTdQQoSccUXbOxlGUs= +github.com/slingdata-io/pocketbase v0.22.136 h1:RtAvPvYdK0qm9EB1r8GzNeEfSiqDK+tV8jyxwbpKKBA= +github.com/slingdata-io/pocketbase v0.22.136/go.mod h1:RYAdoMZtW+3OIgKqg+YhgWGIiwjtcBHGxRcVF2+1klA= github.com/snowflakedb/gosnowflake v1.17.1 h1:sBYExPDRv6hHF7fCqeXMT745L326Byw/cROxvCiEJzo= github.com/snowflakedb/gosnowflake v1.17.1/go.mod h1:TaHvQGh9MA2lopZZMm1AvvENDfwcnKtuskIr1e6Fpic= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= @@ -950,8 +1031,8 @@ github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiT github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= -github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= +github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -970,8 +1051,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/substrait-io/substrait v0.87.0 h1:40rP4LejyK6SNQlWz7NX6kQELf8cmScWMBGruWhN4io= github.com/substrait-io/substrait v0.87.0/go.mod h1:MPFNw6sToJgpD5Z2rj0rQrdP/Oq8HG7Z2t3CAEHtkHw= github.com/substrait-io/substrait-go/v8 v8.1.0 h1:58nSM6Qsctke4DIHU2y4aR8FN5YC7syo1mk4zFKAo4U= @@ -1074,6 +1155,8 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/dburl v0.3.0 h1:KGkeJB/oQhY/DeeJoYl/1+pNE/JnF6ouAuA8nzpQEQ8= github.com/xo/dburl v0.3.0/go.mod h1:TM8VMBT+LWqC3MBOulZjb8FAthcvZq0t/qvDLwS6skU= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= @@ -1087,6 +1170,8 @@ github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= @@ -1108,16 +1193,18 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/detectors/gcp v1.39.0 h1:kWRNZMsfBHZ+uHjiH4y7Etn2FK26LAGkNFw7RHv1DhE= -go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0 h1:oECp5f+hN7nkwjU/8BxQ/q23bGPb8FIrD839owX222E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.70.0/go.mod h1:DqEFwLumhzMBDQv9PcWbyoDxHI/4lAk6CM4nJBH39sc= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.63.0 h1:2pn7OzMewmYRiNtv1doZnLo3gONcnMHlFnmOR8Vgt+8= go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.63.0/go.mod h1:rjbQTDEPQymPE0YnRQp9/NuPwwtL0sesz/fnqRW/v84= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= -go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= -go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0 h1:LMuyCAyfalSjDyjdC65nK6N0zoTT63+E/u95X0JovZI= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.70.0/go.mod h1:085m8qbm4hgc8rZWGDEa4vmyyo2c3nPxUslYUKUIU04= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.43.0 h1:w1K+pCJoPpQifuVpsKamUdn9U0zM3xUziVOqsGksUrY= @@ -1130,25 +1217,45 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJK go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0 h1:TC+BewnDpeiAmcscXbGMfxkO+mwYUwE/VySwvw88PfA= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.43.0/go.mod h1:J/ZyF4vfPwsSr9xJSPyQ4LqtcTPULFR64KwTikGLe+A= -go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= -go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= -go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= -go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= -go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= -go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= -go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= +go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0= +go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/metric/x v0.68.0 h1:TA/cBT23D3MnxYPwHL7YFOdYGdx0A0v+s7Mzotpd1dU= +go.opentelemetry.io/otel/metric/x v0.68.0/go.mod h1:agudOmvWhwUTjgibWDzxD2PoWYnpw5Ht5jISYOD2Hd4= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk= +go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g= +go.opentelemetry.io/otel/sdk/log/logtest v0.22.0 h1:infPnfNrhCNgOUZRs3gWUg8vhoBUHihq02gwK05gzlg= +go.opentelemetry.io/otel/sdk/log/logtest v0.22.0/go.mod h1:gkQZA3z15Bv3KU9vigBTi8dFechSozRP7v94X4VZv+s= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U= go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= @@ -1168,11 +1275,12 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1200,6 +1308,7 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1217,8 +1326,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1247,6 +1356,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1282,11 +1392,11 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= -golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= +golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959 h1:RJhm5l6Fo4rmEIcndxDllNhhf/fAx8qIm4t6A7vpm2A= +golang.org/x/telemetry v0.0.0-20260708182218-49f421fb7959/go.mod h1:LV7u5Oco+Z/g6XI7PqN+EUUUGGkEcmB1uj2ceI0fOVg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1299,10 +1409,11 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -1314,8 +1425,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -1328,6 +1439,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1337,8 +1449,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1356,6 +1468,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1365,10 +1478,10 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= -google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1377,8 +1490,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1390,8 +1503,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/VividCortex/ewma.v1 v1.1.1 h1:tWHEKkKq802K/JT9RiqGCBU5fW3raAPnJGTE9ostZvg= gopkg.in/VividCortex/ewma.v1 v1.1.1/go.mod h1:TekXuFipeiHWiAlO1+wSS23vTcyFau5u3rxXUSXj710= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= @@ -1418,6 +1531,10 @@ gopkg.in/mattn/go-isatty.v0 v0.0.4 h1:NtS1rQGQr4IaFWBGz4Cz4BhB///gyys4gDVtKA7hIs gopkg.in/mattn/go-isatty.v0 v0.0.4/go.mod h1:wt691ab7g0X4ilKZNmMII3egK0bTxl37fEn/Fwbd8gc= gopkg.in/mattn/go-runewidth.v0 v0.0.4 h1:r0P71TnzQDlNIcizCqvPSSANoFa3WVGtcNJf3TWurcY= gopkg.in/mattn/go-runewidth.v0 v0.0.4/go.mod h1:BmXejnxvhwdaATwiJbB1vZ2dtXkQKZGu9yLFCZb4msQ= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1445,14 +1562,36 @@ honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +modernc.org/cc/v4 v4.27.3 h1:uNCgn37E5U09mTv1XgskEVUJ8ADKpmFMPxzGJ0TSo+U= +modernc.org/cc/v4 v4.27.3/go.mod h1:3YjcbCqhoTTHPycJDRl2WZKKFj0nwcOIPBfEZK0Hdk8= +modernc.org/ccgo/v4 v4.32.4 h1:L5OB8rpEX4ZsXEQwGozRfJyJSFHbbNVOoQ59DU9/KuU= +modernc.org/ccgo/v4 v4.32.4/go.mod h1:lY7f+fiTDHfcv6YlRgSkxYfhs+UvOEEzj49jAn2TOx0= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U= modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= From fe8561372e5e3bd9f39223c7d88a4b87f30ea5cb Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Tue, 22 Sep 2026 08:36:27 -0300 Subject: [PATCH 30/56] Add DynamoDB connector with scan, batch write and delete_missing - DynamoDB has no SQL engine, so reads become paginated Scans driven by a JSON scan descriptor (filter/fields/limit) and writes become BatchWriteItem puts that upsert by primary key. - Tables cannot exist without a key: `primary_key` and unique keys are honored, and a `_sling_id` column is added when a stream declares none, so repeated data values cannot collapse rows on write. - delete_missing soft/hard are supported by parsing the rendered update/delete statements and resolving the `not exists` join against the stream's temp keys, keeping rows the stream still holds. - Credentials come from the AWS credential chain, region from the URL host or env, and a custom `endpoint` supports DynamoDB Local. - Registered in type, template, URL and direct-write plumbing, with unit tests, a CLI suite entry and an end-to-end pipeline. --- .gitignore | 2 + README.md | 2 +- cmd/sling/sling_conns.go | 2 +- cmd/sling/sling_test.go | 25 +- core/dbio/connection/connection.go | 28 +- core/dbio/connection/connection_test.go | 38 + core/dbio/database/database.go | 2 + core/dbio/database/database_dynamodb.go | 2661 +++++++++++++++++++++++ core/dbio/database/database_test.go | 533 +++++ core/dbio/database/schemata.go | 47 + core/dbio/database/schemata_keys.go | 3 +- core/dbio/dbio_types.go | 10 +- core/dbio/dbio_types_test.go | 1 + core/dbio/templates/_properties.yaml | 39 + core/dbio/templates/connections.json | 32 + core/dbio/templates/dynamodb.yaml | 59 + core/sling/task_run_write.go | 6 +- go.mod | 2 + go.sum | 4 + tests/pipelines/p.54.dynamodb.yaml | 354 +++ tests/suite.cli.yaml | 18 + 21 files changed, 3855 insertions(+), 13 deletions(-) create mode 100644 core/dbio/database/database_dynamodb.go create mode 100644 core/dbio/templates/dynamodb.yaml create mode 100644 tests/pipelines/p.54.dynamodb.yaml diff --git a/.gitignore b/.gitignore index 36bc38862..484111e7c 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ docs/ tests/evals/results/*.jsonl tests/evals/results/*.summary.json tests/evals/eval.duckdb + +cmd/sling/sling diff --git a/README.md b/README.md index a795dd894..fdf2ea54c 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Example [Replication](https://docs.slingdata.io/sling-cli/run/configuration/repl --- Available Connectors: -- **Databases**: [`adbc`](https://docs.slingdata.io/connections/database-connections/adbc) [`azuredwh`](https://docs.slingdata.io/connections/database-connections/azuredwh) [`azuresql`](https://docs.slingdata.io/connections/database-connections/azuresql) [`azuretable`](https://docs.slingdata.io/connections/database-connections/azuretable) [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`d1`](https://docs.slingdata.io/connections/database-connections/d1) [`databricks`](https://docs.slingdata.io/connections/database-connections/databricks) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`elasticsearch`](https://docs.slingdata.io/connections/database-connections/elasticsearch) [`exasol`](https://docs.slingdata.io/connections/database-connections/exasol) [`fabric`](https://docs.slingdata.io/connections/database-connections/fabric) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`mongodb`](https://docs.slingdata.io/connections/database-connections/mongodb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`odbc`](https://docs.slingdata.io/connections/database-connections/odbc) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`scylladb`](https://docs.slingdata.io/connections/database-connections/scylladb) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`trino`](https://docs.slingdata.io/connections/database-connections/trino) +- **Databases**: [`adbc`](https://docs.slingdata.io/connections/database-connections/adbc) [`azuredwh`](https://docs.slingdata.io/connections/database-connections/azuredwh) [`azuresql`](https://docs.slingdata.io/connections/database-connections/azuresql) [`azuretable`](https://docs.slingdata.io/connections/database-connections/azuretable) [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`d1`](https://docs.slingdata.io/connections/database-connections/d1) [`databricks`](https://docs.slingdata.io/connections/database-connections/databricks) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`dynamodb`](https://docs.slingdata.io/connections/database-connections/dynamodb) [`elasticsearch`](https://docs.slingdata.io/connections/database-connections/elasticsearch) [`exasol`](https://docs.slingdata.io/connections/database-connections/exasol) [`fabric`](https://docs.slingdata.io/connections/database-connections/fabric) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`mongodb`](https://docs.slingdata.io/connections/database-connections/mongodb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`odbc`](https://docs.slingdata.io/connections/database-connections/odbc) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`scylladb`](https://docs.slingdata.io/connections/database-connections/scylladb) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`trino`](https://docs.slingdata.io/connections/database-connections/trino) - **Data Lakes**:[`athena`](https://docs.slingdata.io/connections/datalake-connections/athena) [`Ducklake`](https://docs.slingdata.io/connections/datalake-connections/ducklake) [`iceberg`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`S3 Tables`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`lancedb`](https://docs.slingdata.io/connections/datalake-connections/lancedb) diff --git a/cmd/sling/sling_conns.go b/cmd/sling/sling_conns.go index ce580eacb..a55b288ed 100644 --- a/cmd/sling/sling_conns.go +++ b/cmd/sling/sling_conns.go @@ -198,7 +198,7 @@ func processConns(c *g.CliSC) (ok bool, err error) { return ok, g.Error(err, "cannot parse query") } - if len(database.ParseSQLMultiStatements(query)) == 1 && (!sQuery.IsQuery() || (strings.Contains(strings.ToLower(query), "select") && !strings.Contains(strings.ToLower(query), "insert")) || g.In(conn.Connection.Type, dbio.TypeDbPrometheus, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch)) { + if len(database.ParseSQLMultiStatements(query)) == 1 && (!sQuery.IsQuery() || (strings.Contains(strings.ToLower(query), "select") && !strings.Contains(strings.ToLower(query), "insert")) || g.In(conn.Connection.Type, dbio.TypeDbPrometheus, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbDynamoDB)) { // Limit handling: // - limit > 0: wrap the SQL with the dialect's limit_sql template via diff --git a/cmd/sling/sling_test.go b/cmd/sling/sling_test.go index 999ef12ec..5fdb3ead1 100755 --- a/cmd/sling/sling_test.go +++ b/cmd/sling/sling_test.go @@ -117,6 +117,7 @@ var connMap = map[dbio.Type]connTest{ dbio.TypeDbStarRocks: {name: "starrocks"}, dbio.TypeDbTrino: {name: "trino", adjustCol: g.Bool(false)}, dbio.TypeDbMongoDB: {name: "mongo", schema: "default"}, + dbio.TypeDbDynamoDB: {name: "dynamodb", schema: "default"}, dbio.TypeDbAzureTable: {name: "azure_table", schema: "default"}, dbio.TypeDbElasticsearch: {name: "elasticsearch", schema: "default"}, dbio.TypeDbPrometheus: {name: "prometheus", schema: "prometheus"}, @@ -877,9 +878,11 @@ func runOneTask(t *testing.T, ctx context.Context, file g.FileItem, connType dbi failed := false for colName, correctType := range correctTypeMap { - // skip those - if g.In(srcType, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable, dbio.TypeDbScyllaDB) || - g.In(tgtType, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable) || + // skip those: schemaless stores infer column types from sampled + // values, so logical types (decimal/bigint, date/timestamp, tz) are + // not preserved end to end + if g.In(srcType, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable, dbio.TypeDbScyllaDB, dbio.TypeDbDynamoDB) || + g.In(tgtType, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable, dbio.TypeDbDynamoDB) || taskCfg.TgtConn.IsADBC() || taskCfg.SrcConn.IsADBC() || taskCfg.TgtConn.Type == dbio.TypeDbODBC || taskCfg.SrcConn.Type == dbio.TypeDbODBC { @@ -1312,6 +1315,22 @@ func TestSuiteDatabaseScylladb(t *testing.T) { testSuite(t, dbio.TypeDbScyllaDB, "1,3-9,17,20,23-26") } +// TestSuiteDatabaseDynamoDB runs the shared DB suite against DynamoDB +// (DynamoDB Local works: `docker run -p 8000:8000 amazon/dynamodb-local`). +// +// DynamoDB has no SQL engine and no views, so the cases that create or read the +// `[table]_vw` view are out: 9 creates it, 10 and 11 discover it, 13 reads it +// into postgres and 19 reads the postgres copy of it. Tests 12, 14, 15, 18 and +// 21 validate against test1.result.csv, which only holds after test 9's upsert, +// so they depend on that view chain too (18 and 21 also need the rows that test +// 12 then writes into postgres). Test 22 backfills the range 2020-01-01 to +// 2021-01-01 while the suite's rows hold `create_dt` values from 2019, so it can +// never read a row. +func TestSuiteDatabaseDynamoDB(t *testing.T) { + t.Parallel() + testSuite(t, dbio.TypeDbDynamoDB, "1-8,16-17,20,23-29") +} + // rewriteScyllaDropSQL: add IF EXISTS and quote identifiers func rewriteScyllaDropSQL(sql string) string { parts := strings.Split(sql, ";") diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index 13ce13cac..fbc5707a8 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -599,7 +599,7 @@ func (c *Connection) setURL() (err error) { pathValue := strings.ReplaceAll(U.Path(), "/", "") setIfMissing("schema", U.PopParam("schema")) - if !g.In(c.Type, dbio.TypeDbMotherDuck, dbio.TypeDbDuckDb, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB, dbio.TypeDbSQLite, dbio.TypeDbD1, dbio.TypeDbBigQuery) { + if !g.In(c.Type, dbio.TypeDbMotherDuck, dbio.TypeDbDuckDb, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB, dbio.TypeDbSQLite, dbio.TypeDbD1, dbio.TypeDbBigQuery, dbio.TypeDbDynamoDB) { setIfMissing("host", U.Hostname()) setIfMissing("user", U.Username()) setIfMissing("username", U.Username()) @@ -627,6 +627,9 @@ func (c *Connection) setURL() (err error) { case dbio.TypeDbLanceDB: setIfMissing("path", lanceDBPathFromURL(c.URL())) setIfMissing("schema", "main") + case dbio.TypeDbDynamoDB: + // `dynamodb://us-east-1` carries the region in the host + setIfMissing("aws_region", U.Hostname()) case dbio.TypeDbMotherDuck: setIfMissing("schema", "main") case dbio.TypeDbD1: @@ -1099,6 +1102,29 @@ func (c *Connection) setURL() (err error) { setIfMissing("port", c.Type.DefPort()) setIfMissing("keyspace", "") template = "scylladb://{username}:{password}@{host}:{port}/{keyspace}" + case dbio.TypeDbDynamoDB: + // AWS SDK based: credentials come from the AWS credential chain when absent + region := cast.ToString(c.Data["region"]) + if region == "" { + region = os.Getenv("AWS_REGION") + } + if region == "" { + region = os.Getenv("AWS_DEFAULT_REGION") + } + if region == "" { + region = "us-east-1" + } + setIfMissing("aws_region", region) + + setIfMissing("aws_access_key_id", cast.ToString(c.Data["user"])) + setIfMissing("aws_access_key_id", cast.ToString(c.Data["access_key_id"])) + setIfMissing("aws_secret_access_key", cast.ToString(c.Data["password"])) + setIfMissing("aws_secret_access_key", cast.ToString(c.Data["secret_access_key"])) + setIfMissing("aws_session_token", cast.ToString(c.Data["session_token"])) + setIfMissing("aws_profile", cast.ToString(c.Data["profile"])) + // DynamoDB tables have no schema: `default` keeps object names simple + setIfMissing("schema", "default") + template = "dynamodb://{aws_region}" case dbio.TypeFileSftp, dbio.TypeFileFtp: setIfMissing("password", "") setIfMissing("port", c.Type.DefPort()) diff --git a/core/dbio/connection/connection_test.go b/core/dbio/connection/connection_test.go index a7aa625e1..a7a0037b0 100644 --- a/core/dbio/connection/connection_test.go +++ b/core/dbio/connection/connection_test.go @@ -518,3 +518,41 @@ func TestLanceDBConnectionURL(t *testing.T) { assert.Equal(t, dbio.TypeDbLanceDB, reparsed.Type) assert.Equal(t, "s3://my-bucket/prefix", reparsed.DataS(true)["path"]) } + +// DynamoDB connections carry the region in the URL host (`dynamodb://us-east-1`) +// and take credentials from the AWS credential chain when they are not set. +func TestDynamoDBConnectionURL(t *testing.T) { + c, err := NewConnectionFromURL("DDB", "dynamodb://eu-west-1") + require.NoError(t, err) + assert.Equal(t, dbio.TypeDbDynamoDB, c.Type) + assert.Equal(t, "eu-west-1", c.DataS(true)["aws_region"]) + assert.Equal(t, "default", c.DataS(true)["schema"]) // DynamoDB has no schemas + + // properties map onto the AWS SDK options and round-trip through the URL + c, err = NewConnection("DDB", dbio.TypeDbDynamoDB, map[string]any{ + "region": "us-east-2", + "access_key_id": "AKIA_TEST", + "secret_access_key": "SECRET_TEST", + "session_token": "TOKEN_TEST", + "endpoint": "http://localhost:8000", + }) + require.NoError(t, err) + assert.Equal(t, "us-east-2", c.DataS(true)["aws_region"]) + assert.Equal(t, "AKIA_TEST", c.DataS(true)["aws_access_key_id"]) + assert.Equal(t, "SECRET_TEST", c.DataS(true)["aws_secret_access_key"]) + assert.Equal(t, "TOKEN_TEST", c.DataS(true)["aws_session_token"]) + assert.Equal(t, "http://localhost:8000", c.DataS(true)["endpoint"]) + assert.Equal(t, "dynamodb://us-east-2", c.URL()) + + reparsed, err := NewConnectionFromURL("DDB", c.URL()) + require.NoError(t, err) + assert.Equal(t, dbio.TypeDbDynamoDB, reparsed.Type) + assert.Equal(t, "us-east-2", reparsed.DataS(true)["aws_region"]) + + // a region from the environment is used when the URL and props have none + t.Setenv("AWS_REGION", "ap-southeast-1") + t.Setenv("AWS_DEFAULT_REGION", "") + c, err = NewConnection("DDB", dbio.TypeDbDynamoDB, map[string]any{}) + require.NoError(t, err) + assert.Equal(t, "ap-southeast-1", c.DataS(true)["aws_region"]) +} diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index 5156a5456..54105917e 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -330,6 +330,8 @@ func NewConnContext(ctx context.Context, URL string, props ...string) (Connectio conn = &ODBCConn{URL: URL} } else if strings.HasPrefix(URL, "scylladb:") { conn = &ScyllaDBConn{URL: URL} + } else if strings.HasPrefix(URL, "dynamodb:") { + conn = &DynamoDBConn{URL: URL} } else { conn = &BaseConn{URL: URL} } diff --git a/core/dbio/database/database_dynamodb.go b/core/dbio/database/database_dynamodb.go new file mode 100644 index 000000000..9d99ae5fe --- /dev/null +++ b/core/dbio/database/database_dynamodb.go @@ -0,0 +1,2661 @@ +package database + +import ( + "context" + "database/sql" + "encoding/json" + "path" + "regexp" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/flarco/g" + "github.com/samber/lo" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/spf13/cast" +) + +// DynamoDB has no SQL engine: this connector maps sling's operations onto the +// DynamoDB API. +// +// - reading is a `Scan` (paginated with ExclusiveStartKey). The rendered +// `select` is a JSON descriptor: {"filter": {...}, "fields": [...], "limit": n}. +// Filters accept {"col": value} for equality and {"col": {"$gt": value, ...}} +// for comparisons ($eq, $ne, $lt, $lte, $gt, $gte, $between, $in, +// $begins_with, $contains, $exists). +// - writing is `BatchWriteItem` with `PutRequest`, which upserts by primary +// key and is idempotent per key. +// - tables are key-value collections: a partition key is required, an +// optional sort key may follow, and every other attribute is schemaless. +// The primary key is read off the generated `create table` DDL, so set +// `primary_key` (or let sling pick a column named `id`). +// +// Only key attributes are typed in DynamoDB (S, N or B). Values are stored as: +// text as S, numbers as N, booleans as BOOL, binary as B, JSON as M/L, and +// timestamps/dates as ISO-8601 S strings (DynamoDB has no date type). +type DynamoDBConn struct { + BaseConn + URL string + Client *dynamodb.Client + Region string + Endpoint string + + syntheticKeySeq uint64 // sequence for values of an added key column +} + +const dynamoDBBatchSize = 25 // BatchWriteItem allows at most 25 requests +const dynamoDBSampleSize = 25 + +// dynamoDBSyntheticKeyPrefix names the key column that is added when a stream +// declares no primary key: DynamoDB has no key-less tables, and keying on a +// data column would collapse rows whenever its values repeat (DynamoDB upserts +// on the key). This mirrors the other key-value connectors (MongoDB's `_id`, +// Azure Tables' RowKey). +const dynamoDBSyntheticKeyPrefix = "_sling_id" + +// isDynamoDBSyntheticKey reports whether the key column was added by sling +func isDynamoDBSyntheticKey(name string) bool { + return strings.HasPrefix(name, dynamoDBSyntheticKeyPrefix) +} + +// dynamoDBSyntheticKeyName returns a key column name that the data does not use +func dynamoDBSyntheticKeyName(columns iop.Columns) (name string) { + name = dynamoDBSyntheticKeyPrefix + for i := 2; g.In(name, columns.Names()...); i++ { + name = g.F("%s%d", dynamoDBSyntheticKeyPrefix, i) + } + return name +} + +// dynamoDBSelectRegex matches the only SQL shape DynamoDB can honor: a field +// list and a table (the connector has no SQL engine) +var dynamoDBSelectRegex = regexp.MustCompile(`(?is)^select\s+(.+?)\s+from\s+([^\s,;()]+)`) + +// dynamoDBSelectLimitRegex matches a trailing `limit n` +var dynamoDBSelectLimitRegex = regexp.MustCompile(`(?is)\blimit\s+(\d+)\s*;?\s*$`) + +// dynamoDBSetRegex and dynamoDBWhereRegex locate the clauses of the statements +// sling renders for table lifecycle and deletes +var dynamoDBSetRegex = regexp.MustCompile(`(?is)\sset\s`) +var dynamoDBWhereRegex = regexp.MustCompile(`(?is)\swhere\s`) + +// dynamoDBClauseRegex matches SQL clauses DynamoDB cannot apply +var dynamoDBClauseRegex = regexp.MustCompile(`(?is)\b(where|group\s+by|order\s+by|having|join|offset|union)\b`) + +// dynamoDBTableNameRegex matches a DynamoDB table name (3-255 chars of a-z, A-Z, +// 0-9, `_`, `-` and `.`), optionally quoted +var dynamoDBTableNameRegex = regexp.MustCompile("^[`\"']?[A-Za-z0-9_.\\-]+[`\"']?$") + +// dynamoDBIsTableName reports whether the text can be a table name (as opposed +// to a rendered condition or a statement) +func dynamoDBIsTableName(text string) bool { + return dynamoDBTableNameRegex.MatchString(strings.TrimSpace(text)) +} + +// dynamoDBUnquote strips the identifier quoting a statement may carry +func dynamoDBUnquote(name string) string { + name = strings.TrimSpace(name) + if len(name) >= 2 { + if (name[0] == '"' && name[len(name)-1] == '"') || + (name[0] == '`' && name[len(name)-1] == '`') || + (name[0] == '[' && name[len(name)-1] == ']') || + (name[0] == '\'' && name[len(name)-1] == '\'') { + return name[1 : len(name)-1] + } + } + return name +} + +// dynamoDBScanRef resolves the table reference handed to the connector. sling +// renders a `select` into a JSON scan descriptor ({"table": ..., "filter": ...}), +// so accept that, a `select ... from
[limit n]` statement, or a plain +// table name. +func dynamoDBScanRef(ref string) (tableName string, descriptor map[string]any, err error) { + ref = strings.TrimSpace(ref) + + if ref == "" { + return "", nil, g.Error("no table specified for DynamoDB") + } + + if strings.HasPrefix(ref, "{") { + // sling appends markers to the rendered select (e.g. `/* nD */`), so read + // the leading JSON value instead of requiring the whole text to be JSON + if err = json.NewDecoder(strings.NewReader(ref)).Decode(&descriptor); err != nil { + return "", nil, g.Error(err, "could not parse scan descriptor: %s", ref) + } + tableName = cast.ToString(descriptor["table"]) + if tableName == "" { + return "", nil, g.Error("scan descriptor is missing the table: %s", ref) + } + return tableName, descriptor, nil + } + + if !dynamoDBSelectRegex.MatchString(ref) { + return ref, nil, nil + } + + // anything else would be silently dropped, which would return wrong rows + if clause := dynamoDBClauseRegex.FindString(ref); clause != "" { + return "", nil, g.Error( + "DynamoDB has no SQL engine: `%s` cannot be applied here. Use the `where` stream option with a filter expression", + strings.ToUpper(strings.Join(strings.Fields(clause), " ")), + ) + } + + matches := dynamoDBSelectRegex.FindStringSubmatch(ref) + tableName = dynamoDBUnquote(matches[2]) + descriptor = map[string]any{"table": tableName} + + if fields := strings.TrimSpace(matches[1]); fields != "*" { + descriptor["fields"] = lo.Map(strings.Split(fields, ","), func(field string, _ int) string { + return dynamoDBUnquote(field) + }) + } + + if limit := dynamoDBSelectLimitRegex.FindStringSubmatch(ref); limit != nil { + descriptor["limit"] = cast.ToInt(limit[1]) + } + + return tableName, descriptor, nil +} + +// mergeDynamoDBScanOptions merges a scan descriptor over the caller's options: +// the descriptor is the rendered statement, so it wins (a `limit` inside the +// statement is more specific than a default), while caller-only options such as +// `columns` are kept +func mergeDynamoDBScanOptions(opts map[string]any, descriptor map[string]any) map[string]any { + merged := map[string]any{} + for key, val := range opts { + merged[key] = val + } + for key, val := range descriptor { + merged[key] = val + } + delete(merged, "table") + return merged +} + +// dynamoDBResult implements sql.Result (DynamoDB has no row counts) +type dynamoDBResult struct { + rowsAffected int64 +} + +func (r *dynamoDBResult) LastInsertId() (int64, error) { + return 0, nil +} + +func (r *dynamoDBResult) RowsAffected() (int64, error) { + return r.rowsAffected, nil +} + +// Init initiates the object +func (conn *DynamoDBConn) Init() error { + conn.BaseConn.URL = conn.URL + conn.BaseConn.Type = dbio.TypeDbDynamoDB + + // rows are written straight to the table (no SQL temp table) + conn.BaseConn.SetProp("use_bulk", "true") + + instance := Connection(conn) + conn.BaseConn.instance = &instance + return conn.BaseConn.Init() +} + +// Connect connects to the database +func (conn *DynamoDBConn) Connect(timeOut ...int) (err error) { + ctx := conn.Context().Ctx + if len(timeOut) > 0 && timeOut[0] > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(timeOut[0])*time.Second) + defer cancel() + } + + conn.Region = conn.GetProp("aws_region", "region") + conn.Endpoint = conn.GetProp("aws_endpoint", "endpoint") + + props := conn.Props() + if conn.Endpoint != "" && conn.GetProp("aws_access_key_id", "access_key_id") == "" { + // local / non-AWS endpoints (DynamoDB Local) ignore credentials, but the + // SDK still needs a signer, so fall back to placeholder values + props["aws_access_key_id"] = "local" + props["aws_secret_access_key"] = "local" + } + + cfg, err := iop.MakeAwsConfig(ctx, props) + if err != nil { + return g.Error(err, "could not create AWS config") + } + conn.Client = dynamodb.NewFromConfig(cfg) + + if _, err = conn.ListTables(ctx); err != nil { + return g.Error(err, "could not list DynamoDB tables") + } + + if !cast.ToBool(conn.GetProp("silent")) { + g.Debug(`opened "%s" connection (%s)`, conn.Type, conn.GetProp("sling_conn_id")) + } + + conn.SetProp("connected", "true") + conn.SetProp("connect_time", cast.ToString(time.Now())) + + return nil +} + +// Close closes the connection +// ensureClient connects on demand. sling hands out fresh connection objects for +// metadata calls (counts, discovery, checksums) that are not connected yet, and +// reuses a connection object after Close(), so every entry point that talks to +// the API must be able to (re)build the client. +func (conn *DynamoDBConn) ensureClient() (err error) { + if conn.Client != nil { + return nil + } + return conn.Connect() +} + +func (conn *DynamoDBConn) Close() error { + conn.Client = nil + g.Debug(`closed "%s" connection (%s)`, conn.Type, conn.GetProp("sling_conn_id")) + return nil +} + +// NewTransaction creates a new transaction (unsupported in DynamoDB) +func (conn *DynamoDBConn) NewTransaction(ctx context.Context, options ...*sql.TxOptions) (tx Transaction, err error) { + return nil, g.Error("transactions not supported in DynamoDB") +} + +// ListTables returns the table names +func (conn *DynamoDBConn) ListTables(ctx context.Context) (names []string, err error) { + if err = conn.ensureClient(); err != nil { + return nil, err + } + var start *string + for { + out, err := conn.Client.ListTables(ctx, &dynamodb.ListTablesInput{ExclusiveStartTableName: start}) + if err != nil { + return nil, g.Error(err, "could not list DynamoDB tables") + } + names = append(names, out.TableNames...) + if out.LastEvaluatedTableName == nil || *out.LastEvaluatedTableName == "" { + break + } + start = out.LastEvaluatedTableName + } + return names, nil +} + +// describeTable returns the table description, or found=false when it does not exist +func (conn *DynamoDBConn) describeTable(ctx context.Context, tableName string) (table *ddbtypes.TableDescription, found bool, err error) { + if err = conn.ensureClient(); err != nil { + return nil, false, err + } + if strings.TrimSpace(tableName) == "" { + return nil, false, g.Error("did not provide a table name") + } + + out, err := conn.Client.DescribeTable(ctx, &dynamodb.DescribeTableInput{TableName: aws.String(tableName)}) + if err != nil { + if isDynamoDBNotFound(err) { + return nil, false, nil + } + return nil, false, g.Error(err, "could not describe table %s", tableName) + } else if out.Table == nil { + return nil, false, g.Error("could not describe table %s", tableName) + } + return out.Table, true, nil +} + +// isDynamoDBNotFound returns true when the error is a ResourceNotFoundException +func isDynamoDBNotFound(err error) bool { + return err != nil && strings.Contains(err.Error(), "ResourceNotFoundException") +} + +// isDynamoDBAlreadyExists returns true when the table already exists (or is being deleted) +func isDynamoDBAlreadyExists(err error) bool { + return err != nil && (strings.Contains(err.Error(), "ResourceInUseException") || + strings.Contains(err.Error(), "TableAlreadyExistsException")) +} + +// TableExists checks if a table exists +func (conn *DynamoDBConn) TableExists(table Table) (exists bool, err error) { + desc, found, err := conn.describeTable(conn.Context().Ctx, table.Name) + if err != nil || !found { + return false, err + } + + // a table being deleted cannot be re-created until DynamoDB is done with it, + // so wait it out and report it as gone + if desc.TableStatus == ddbtypes.TableStatusDeleting { + err = conn.waitForTableAbsent(conn.Context().Ctx, table.Name) + return false, err + } + + return true, nil +} + +// waitForTableActive waits until the table is ready to serve reads and writes +func (conn *DynamoDBConn) waitForTableActive(ctx context.Context, tableName string) (err error) { + timeOut := cast.ToInt(conn.GetProp("table_timeout")) + if timeOut == 0 { + timeOut = 60 + } + + start := time.Now() + for { + desc, found, err := conn.describeTable(ctx, tableName) + if err != nil { + return err + } else if !found { + return g.Error("table %s does not exist anymore", tableName) + } else if desc.TableStatus == ddbtypes.TableStatusActive { + return nil + } + + if time.Since(start) > time.Duration(timeOut)*time.Second { + return g.Error("table %s is not active after %d seconds (status: %s)", tableName, timeOut, desc.TableStatus) + } else if ctx.Err() != nil { + return ctx.Err() + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(250 * time.Millisecond): + } + } +} + +// waitForTableAbsent waits until the table is fully deleted +func (conn *DynamoDBConn) waitForTableAbsent(ctx context.Context, tableName string) (err error) { + timeOut := cast.ToInt(conn.GetProp("table_timeout")) + if timeOut == 0 { + timeOut = 60 + } + + start := time.Now() + for { + _, found, err := conn.describeTable(ctx, tableName) + if err != nil { + return err + } else if !found { + return nil + } + + if time.Since(start) > time.Duration(timeOut)*time.Second { + return g.Error("table %s is still not deleted after %d seconds", tableName, timeOut) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(250 * time.Millisecond): + } + } +} + +// dropTable deletes a table and waits for the deletion to complete +func (conn *DynamoDBConn) dropTable(ctx context.Context, tableName string) (err error) { + if err = conn.ensureClient(); err != nil { + return err + } + _, err = conn.Client.DeleteTable(ctx, &dynamodb.DeleteTableInput{TableName: aws.String(tableName)}) + if err != nil { + if isDynamoDBNotFound(err) { + return nil // already gone + } + return g.Error(err, "could not delete table %s", tableName) + } + return conn.waitForTableAbsent(ctx, tableName) +} + +// GenerateDDL generates the DDL that ExecContext parses to create the table. +// DynamoDB requires a primary key: the target's `table_keys`, sling's +// `primary_key`, or a suitable column supplies it (see dynamoDBKeyColumns). +func (conn *DynamoDBConn) GenerateDDL(table Table, data iop.Dataset, temporary bool) (ddl string, err error) { + if pkCols := conn.dynamoDBKeyColumns(table, data.Columns); len(pkCols) > 0 { + if err = data.Columns.SetKeys(iop.PrimaryKey, pkCols...); err != nil { + return "", g.Error(err) + } + } else { + // no key declared: add one so every row survives, instead of keying on a + // data column that may repeat values + keyCol := iop.Column{ + Name: dynamoDBSyntheticKeyName(data.Columns), + Type: iop.StringType, + Position: len(data.Columns) + 1, + } + keyCol.SetMetadata(string(iop.PrimaryKey.MetadataKey()), "true") + data.Columns = append(data.Columns, keyCol) + g.Warn("no primary key specified for DynamoDB; adding column %s as the key", keyCol.Name) + } + + ddl, err = conn.BaseConn.GenerateDDL(table, data, temporary) + if err != nil { + return ddl, g.Error(err) + } + + ddl, err = table.AddPrimaryKeyToDDL(ddl, data.Columns) + if err != nil { + return ddl, g.Error(err) + } + + return strings.TrimSpace(ddl), nil +} + +// dynamoDBKeyColumns resolves the table's key columns: explicit target keys +// first, then the source primary key. sling records `primary_key` as column +// metadata instead of setting the key type on target columns, but DynamoDB has +// no key-less tables and the key is the upsert identity, so it is honored here. +func (conn *DynamoDBConn) dynamoDBKeyColumns(table Table, columns iop.Columns) (pkCols []string) { + if keys := table.Keys[iop.PrimaryKey]; len(keys) > 0 { + return keys + } else if keys := table.Keys[iop.UniqueKey]; len(keys) > 0 && len(keys) <= 2 { + // a declared unique key is the natural upsert identity in a key-value + // store (DynamoDB keys serve as its unique constraints) + return keys + } else if keys := columns.GetKeys(iop.PrimaryKey); len(keys) > 0 { + return keys.Names() + } + + for _, col := range columns { + if strings.EqualFold(col.Metadata[iop.PrimaryKey.MetadataKey()], "source") { + pkCols = append(pkCols, col.Name) + } + } + return pkCols +} + +// ExecContext executes a DDL statement against DynamoDB. Only the statements +// sling emits for table lifecycle are supported; there is no SQL engine. +func (conn *DynamoDBConn) ExecContext(ctx context.Context, sqlText string, args ...any) (result sql.Result, err error) { + text := strings.TrimSpace(sqlText) + if !isDynamoDBStatement(text) { + return nil, g.Error("SQL operation not supported on DynamoDB: %s", sqlText) + } + + lower := strings.ToLower(text) + switch { + case createTableRegex.MatchString(lower): + err = conn.createTableFromDDL(ctx, text) + case strings.HasPrefix(lower, "drop table"): + err = conn.dropTable(ctx, parseDynamoDBTableToken(text, "drop table")) + case strings.HasPrefix(lower, "truncate table"): + err = conn.truncateTable(ctx, parseDynamoDBTableToken(text, "truncate table")) + case strings.HasPrefix(lower, "update "): + // sling soft-deletes missing rows with a plain update + err = conn.updateItems(ctx, text) + default: // `delete from` + err = conn.deleteItems(ctx, text) + } + + if err != nil { + return nil, err + } + return &dynamoDBResult{}, nil +} + +// isDynamoDBStatement reports whether the text is one of the lifecycle +// statements sling emits, as opposed to a table name or a scan descriptor +func isDynamoDBStatement(text string) bool { + lower := strings.ToLower(strings.TrimSpace(text)) + return createTableRegex.MatchString(lower) || + strings.HasPrefix(lower, "drop table") || + strings.HasPrefix(lower, "truncate table") || + strings.HasPrefix(lower, "update ") || + strings.HasPrefix(lower, "delete from") +} + +// parseDynamoDBTableToken returns the table name following the given keyword, +// dropping any schema qualifier and quotes. +func parseDynamoDBTableToken(text string, keyword string) (name string) { + fields := strings.Fields(strings.TrimSpace(text[len(keyword):])) + if len(fields) == 0 { + return "" + } + + if strings.EqualFold(fields[0], "if") && len(fields) > 2 && strings.EqualFold(fields[1], "exists") { + fields = fields[2:] // `drop table if exists
` + } + + name = strings.TrimRight(fields[0], ";") + if parts := strings.Split(name, "."); len(parts) > 1 { + name = parts[len(parts)-1] // drop the schema qualifier + } + + return strings.Trim(name, "`\"") +} + +// dynamoDBTableDef carries the information DynamoDB needs to create a table +type dynamoDBTableDef struct { + Name string + KeyColumns []string + KeyTypes map[string]ddbtypes.ScalarAttributeType +} + +// parseDynamoDBDDL reads the create statement generated by GenerateDDL +func parseDynamoDBDDL(ddl string) (def dynamoDBTableDef, err error) { + def = dynamoDBTableDef{KeyTypes: map[string]ddbtypes.ScalarAttributeType{}} + + loc := createTableRegex.FindStringIndex(ddl) + if loc == nil { + return def, g.Error("could not find CREATE TABLE in DDL") + } + + rest := ddl[loc[1]:] + openParen := strings.Index(rest, "(") + if openParen == -1 { + return def, g.Error("could not find column list in DDL") + } + def.Name = parseDynamoDBTableToken("x "+strings.TrimSpace(rest[:openParen]), "x") + + // balanced parenthesis scan for the column list + depth, closeParen := 0, -1 + for i := openParen; i < len(rest); i++ { + switch rest[i] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + closeParen = i + } + } + if closeParen != -1 { + break + } + } + if closeParen == -1 { + return def, g.Error("could not find closing parenthesis in DDL") + } + + columnTypes := map[string]ddbtypes.ScalarAttributeType{} + for _, item := range splitDDLTopLevel(rest[openParen+1 : closeParen]) { + item = strings.TrimSpace(item) + if item == "" { + continue + } + + if strings.HasPrefix(strings.ToLower(item), "primary key") { + start, end := strings.Index(item, "("), strings.LastIndex(item, ")") + if start == -1 || end <= start { + return def, g.Error("could not parse PRIMARY KEY clause: %s", item) + } + for _, name := range strings.Split(item[start+1:end], ",") { + name = strings.Trim(strings.TrimSpace(name), `"`) + if name != "" { + def.KeyColumns = append(def.KeyColumns, name) + } + } + continue + } + + fields := strings.Fields(item) + if len(fields) == 0 { + continue + } + colName := strings.Trim(fields[0], `"`) + colType := "" + if len(fields) > 1 { + colType = fields[1] + } + columnTypes[colName] = dynamoDBKeyAttributeType(colType) + } + + if len(def.KeyColumns) == 0 { + return def, g.Error("DynamoDB requires a primary key: set `primary_key` for the stream") + } else if len(def.KeyColumns) > 2 { + return def, g.Error("DynamoDB supports at most 2 key columns (partition key + sort key), got %d: %s", + len(def.KeyColumns), strings.Join(def.KeyColumns, ", ")) + } + + for _, name := range def.KeyColumns { + keyType, ok := columnTypes[name] + if !ok { + return def, g.Error("key column %s is not part of the table definition", name) + } + def.KeyTypes[name] = keyType + } + + return def, nil +} + +// splitDDLTopLevel splits a column list on commas that are not inside parenthesis +func splitDDLTopLevel(text string) (items []string) { + depth := 0 + start := 0 + for i, r := range text { + switch r { + case '(': + depth++ + case ')': + depth-- + case ',': + if depth == 0 { + items = append(items, text[start:i]) + start = i + 1 + } + } + } + return append(items, text[start:]) +} + +// dynamoDBKeyAttributeType maps a DDL type to the DynamoDB key attribute type +func dynamoDBKeyAttributeType(ddlType string) ddbtypes.ScalarAttributeType { + ddlType = strings.ToLower(strings.Split(strings.TrimSpace(ddlType), "(")[0]) + switch { + case strings.Contains(ddlType, "int"), strings.Contains(ddlType, "number"), + strings.Contains(ddlType, "decimal"), strings.Contains(ddlType, "numeric"), + strings.Contains(ddlType, "float"), strings.Contains(ddlType, "double"), + strings.Contains(ddlType, "real"): + return ddbtypes.ScalarAttributeTypeN + case strings.Contains(ddlType, "binary"), strings.Contains(ddlType, "blob"), strings.Contains(ddlType, "bytea"): + return ddbtypes.ScalarAttributeTypeB + } + return ddbtypes.ScalarAttributeTypeS +} + +// createTableFromDDL creates a table from the DDL generated by GenerateDDL +func (conn *DynamoDBConn) createTableFromDDL(ctx context.Context, ddl string) (err error) { + if err = conn.ensureClient(); err != nil { + return err + } + def, err := parseDynamoDBDDL(ddl) + if err != nil { + return err + } + + keySchema := []ddbtypes.KeySchemaElement{{ + AttributeName: aws.String(def.KeyColumns[0]), + KeyType: ddbtypes.KeyTypeHash, + }} + if len(def.KeyColumns) == 2 { + keySchema = append(keySchema, ddbtypes.KeySchemaElement{ + AttributeName: aws.String(def.KeyColumns[1]), + KeyType: ddbtypes.KeyTypeRange, + }) + } + + attrDefs := []ddbtypes.AttributeDefinition{} + for _, name := range def.KeyColumns { + attrDefs = append(attrDefs, ddbtypes.AttributeDefinition{ + AttributeName: aws.String(name), + AttributeType: def.KeyTypes[name], + }) + } + + conn.LogSQL(g.F("create table %s (%s)", def.Name, strings.Join(def.KeyColumns, ", "))) + _, err = conn.Client.CreateTable(ctx, &dynamodb.CreateTableInput{ + TableName: aws.String(def.Name), + AttributeDefinitions: attrDefs, + KeySchema: keySchema, + BillingMode: ddbtypes.BillingModePayPerRequest, + }) + if err != nil { + if isDynamoDBAlreadyExists(err) { + return g.Error(err, "table %s already exists", def.Name) + } + return g.Error(err, "could not create table %s", def.Name) + } + + return conn.waitForTableActive(ctx, def.Name) +} + +// truncateTable deletes every item, keeping the table and its key schema +func (conn *DynamoDBConn) truncateTable(ctx context.Context, tableName string) (err error) { + desc, found, err := conn.describeTable(ctx, tableName) + if err != nil { + return err + } else if !found { + return nil + } + + keyNames := []string{} + projection := []string{} + names := map[string]string{} + for i, ks := range desc.KeySchema { + keyNames = append(keyNames, *ks.AttributeName) + alias := g.F("#k%d", i) + names[alias] = *ks.AttributeName + projection = append(projection, alias) + } + + var lastKey map[string]ddbtypes.AttributeValue + for { + out, err := conn.Client.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(tableName), + ProjectionExpression: aws.String(strings.Join(projection, ", ")), + ExpressionAttributeNames: names, + ExclusiveStartKey: lastKey, + }) + if err != nil { + return g.Error(err, "could not scan table %s for truncate", tableName) + } + + keys := []map[string]ddbtypes.AttributeValue{} + for _, item := range out.Items { + key := map[string]ddbtypes.AttributeValue{} + for _, name := range keyNames { + if val, ok := item[name]; ok { + key[name] = val + } + } + if len(key) == len(keyNames) { + keys = append(keys, key) + } + } + + if err = conn.batchWriteRequests(ctx, tableName, deleteRequests(keys)); err != nil { + return g.Error(err, "could not delete items from table %s", tableName) + } + + lastKey = out.LastEvaluatedKey + if len(lastKey) == 0 { + break + } + } + + return nil +} + +// GetSchemas returns schemas (DynamoDB has no schema, `default` is used) +func (conn *DynamoDBConn) GetSchemas() (data iop.Dataset, err error) { + data = iop.NewDataset(iop.NewColumnsFromFields("schema_name")) + data.Append([]any{"default"}) + return data, nil +} + +// GetTables returns the list of tables +func (conn *DynamoDBConn) GetTables(schema string) (data iop.Dataset, err error) { + data = iop.NewDataset(iop.NewColumnsFromFields("table_name")) + + names, err := conn.ListTables(conn.Context().Ctx) + if err != nil { + return data, err + } + + for _, name := range names { + data.Append([]any{name}) + } + + return data, nil +} + +// dynamoDBMatchTableNames reports whether a table matches a discover pattern: +// an exact name, or a name carrying the `*`/`?` wildcards sling passes through +func dynamoDBMatchTableNames(tableName string, patterns []string) bool { + for _, pattern := range patterns { + if g.In(tableName, pattern) { + return true + } + if strings.ContainsAny(pattern, "*?[") { + if matched, err := path.Match(pattern, tableName); err == nil && matched { + return true + } + } + } + return false +} + +// GetSchemata obtain full schemata info for a schema and/or table +func (conn *DynamoDBConn) GetSchemata(level SchemataLevel, schemaName string, tableNames ...string) (Schemata, error) { + schemata := Schemata{ + Databases: map[string]Database{}, + conn: conn, + } + + database := Database{ + Name: "dynamodb", + Schemas: map[string]Schema{}, + } + + schema := Schema{ + Name: "default", + Database: database.Name, + Tables: map[string]Table{}, + } + + tablesData, err := conn.GetTables(schema.Name) + if err != nil { + return schemata, g.Error(err, "Could not get tables") + } + + // discover passes an empty table name when the pattern only names a schema + tableNames = lo.Filter(tableNames, func(name string, _ int) bool { + return strings.TrimSpace(name) != "" + }) + + for _, tableRow := range tablesData.Rows { + tableName := cast.ToString(tableRow[0]) + if len(tableNames) > 0 && !dynamoDBMatchTableNames(tableName, tableNames) { + continue + } + + table := Table{ + Name: tableName, + Schema: schema.Name, + Database: database.Name, + IsView: false, + Dialect: conn.GetType(), + } + + if g.In(level, SchemataLevelTable, SchemataLevelColumn) { + if level == SchemataLevelColumn { + columns, err := conn.GetTableColumns(&table) + if err != nil { + g.Warn("could not get columns for table %s: %s", tableName, err) + } else { + table.Columns = columns + } + } + schema.Tables[strings.ToLower(tableName)] = table + } + } + + database.Schemas[strings.ToLower(schema.Name)] = schema + schemata.Databases[strings.ToLower(database.Name)] = database + + return schemata, nil +} + +// GetTableColumns returns the columns of a table: the key attributes (the only +// typed columns) followed by the attributes found in a sample of items. +// A table that does not exist yields no columns and no error, like an empty +// SQL result set. +func (conn *DynamoDBConn) GetTableColumns(table *Table, fields ...string) (columns iop.Columns, err error) { + ctx := conn.Context().Ctx + desc, found, err := conn.describeTable(ctx, table.Name) + if err != nil { + return nil, err + } else if !found { + return iop.Columns{}, nil + } + + attrTypes := map[string]ddbtypes.ScalarAttributeType{} + for _, attrDef := range desc.AttributeDefinitions { + attrTypes[*attrDef.AttributeName] = attrDef.AttributeType + } + + seen := map[string]bool{} + position := 1 + for _, keySchema := range desc.KeySchema { + name := *keySchema.AttributeName + col := dynamoDBColumn(name, string(attrTypes[name]), nil) + col.Table = table.Name + col.Schema = table.Schema + col.Position = position + columns = append(columns, col) + seen[name] = true + position++ + } + + items, err := conn.sampleItems(ctx, table.Name, dynamoDBSampleSize) + if err != nil { + return columns, err + } + + for _, item := range items { + for name, av := range item { + if seen[name] { + continue + } + col := dynamoDBColumn(name, dynamoDBAttributeType(av), av) + col.Table = table.Name + col.Schema = table.Schema + col.Position = position + columns = append(columns, col) + seen[name] = true + position++ + } + } + + return columns, nil +} + +// sampleItems returns up to count items, so column names and types can be inferred +func (conn *DynamoDBConn) sampleItems(ctx context.Context, tableName string, count int) (items []map[string]ddbtypes.AttributeValue, err error) { + if err = conn.ensureClient(); err != nil { + return nil, err + } + out, err := conn.Client.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(tableName), + Limit: aws.Int32(int32(count)), + }) + if err != nil { + if isDynamoDBNotFound(err) { + return nil, nil + } + return nil, g.Error(err, "could not sample table %s", tableName) + } + + if len(out.Items) > count { + return out.Items[:count], nil + } + return out.Items, nil +} + +// dynamoDBColumn builds a column from a DynamoDB attribute (value is optional) +func dynamoDBColumn(name string, attrType string, av ddbtypes.AttributeValue) iop.Column { + colType, dbType := dynamoDBTypeToIop(attrType) + col := iop.Column{Name: name, Type: colType, DbType: dbType} + + switch attrType { + case string(ddbtypes.ScalarAttributeTypeN): + col.Type = dynamoDBNumberType(av) + col.DbType = "number" + case string(ddbtypes.ScalarAttributeTypeS): + if av != nil { + if member, ok := av.(*ddbtypes.AttributeValueMemberS); ok && isISODateString(member.Value) { + col.Type = iop.TimestampType + col.DbType = "timestamp" + } + } + } + + return col +} + +// dynamoDBTypeToIop maps a DynamoDB attribute type to a general type +func dynamoDBTypeToIop(attrType string) (colType iop.ColumnType, dbType string) { + switch attrType { + case string(ddbtypes.ScalarAttributeTypeS): + return iop.TextType, "string" + case string(ddbtypes.ScalarAttributeTypeN): + return iop.DecimalType, "number" + case string(ddbtypes.ScalarAttributeTypeB): + return iop.BinaryType, "binary" + case "BOOL": + return iop.BoolType, "bool" + case "NULL": + return iop.TextType, "null" + case "L", "M", "SS", "NS", "BS": + return iop.JsonType, "json" + } + return iop.TextType, "string" +} + +// dynamoDBNumberType returns the narrowest general numeric type for a number value +func dynamoDBNumberType(av ddbtypes.AttributeValue) iop.ColumnType { + if av == nil { + return iop.DecimalType + } + member, ok := av.(*ddbtypes.AttributeValueMemberN) + if !ok { + return iop.DecimalType + } + if _, err := strconv.ParseInt(member.Value, 10, 64); err == nil { + return iop.BigIntType + } + return iop.DecimalType +} + +// dynamoDBAttributeType names a DynamoDB attribute value's type +func dynamoDBAttributeType(av ddbtypes.AttributeValue) string { + switch av.(type) { + case *ddbtypes.AttributeValueMemberS: + return "S" + case *ddbtypes.AttributeValueMemberN: + return "N" + case *ddbtypes.AttributeValueMemberB: + return "B" + case *ddbtypes.AttributeValueMemberBOOL: + return "BOOL" + case *ddbtypes.AttributeValueMemberNULL: + return "NULL" + case *ddbtypes.AttributeValueMemberL: + return "L" + case *ddbtypes.AttributeValueMemberM: + return "M" + case *ddbtypes.AttributeValueMemberSS: + return "SS" + case *ddbtypes.AttributeValueMemberNS: + return "NS" + case *ddbtypes.AttributeValueMemberBS: + return "BS" + } + return "NULL" +} + +// isISODateString returns true for RFC3339 timestamps and ISO dates +func isISODateString(text string) bool { + if len(text) < 10 { + return false + } + if _, err := time.Parse(time.RFC3339Nano, text); err == nil { + return true + } + _, err := time.Parse("2006-01-02", text) + return err == nil +} + +// GetCount returns the number of items in a table +func (conn *DynamoDBConn) GetCount(tableFName string) (int64, error) { + if err := conn.ensureClient(); err != nil { + return 0, err + } + table, err := ParseTableName(tableFName, conn.Type) + if err != nil { + return 0, g.Error(err, "could not parse table name: %s", tableFName) + } + + ctx := conn.Context().Ctx + var count int64 + var lastKey map[string]ddbtypes.AttributeValue + for { + out, err := conn.Client.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(table.Name), + Select: ddbtypes.SelectCount, + ExclusiveStartKey: lastKey, + }) + if err != nil { + return 0, g.Error(err, "could not count table %s", table.Name) + } + count += cast.ToInt64(out.Count) + lastKey = out.LastEvaluatedKey + if len(lastKey) == 0 { + break + } + } + + return count, nil +} + +// GetMaxValue returns the maximum value of a column. DynamoDB has no +// aggregates, so the attribute is scanned (projected) and reduced client side. +func (conn *DynamoDBConn) GetMaxValue(table Table, colName string) (value any, maxCol iop.Column, err error) { + if err = conn.ensureClient(); err != nil { + return nil, iop.Column{}, err + } + ctx := conn.Context().Ctx + maxCol = iop.Column{Name: colName, Table: table.Name, Schema: table.Schema} + + _, found, err := conn.describeTable(ctx, table.Name) + if err != nil { + return nil, maxCol, err + } else if !found { + return nil, maxCol, nil + } + + var maxAttr ddbtypes.AttributeValue + var lastKey map[string]ddbtypes.AttributeValue + for { + out, err := conn.Client.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(table.Name), + ProjectionExpression: aws.String("#max_col"), + ExpressionAttributeNames: map[string]string{"#max_col": colName}, + ExclusiveStartKey: lastKey, + }) + if err != nil { + return nil, maxCol, g.Error(err, "could not get max value of %s", colName) + } + + for _, item := range out.Items { + av, ok := item[colName] + if !ok { + continue + } + if maxAttr == nil || dynamoDBAttributeLess(maxAttr, av) { + maxAttr = av + } + } + + lastKey = out.LastEvaluatedKey + if len(lastKey) == 0 { + break + } + } + + if maxAttr == nil { + return nil, maxCol, nil + } + + attrType := dynamoDBAttributeType(maxAttr) + sampled := dynamoDBColumn(colName, attrType, maxAttr) + maxCol.Type = sampled.Type + maxCol.DbType = sampled.DbType + + // return the value as stored, so the incremental filter matches it exactly + raw, _ := dynamoDBAttributeString(maxAttr) + return raw, maxCol, nil +} + +// dynamoDBAttributeLess compares two attribute values of the same attribute +func dynamoDBAttributeLess(a, b ddbtypes.AttributeValue) bool { + switch first := a.(type) { + case *ddbtypes.AttributeValueMemberN: + second, ok := b.(*ddbtypes.AttributeValueMemberN) + return ok && cast.ToFloat64(first.Value) < cast.ToFloat64(second.Value) + case *ddbtypes.AttributeValueMemberB: + second, ok := b.(*ddbtypes.AttributeValueMemberB) + return ok && string(first.Value) < string(second.Value) + } + + firstStr, _ := dynamoDBAttributeString(a) + secondStr, _ := dynamoDBAttributeString(b) + return firstStr < secondStr +} + +// dynamoDBAttributeString renders an attribute value as text +func dynamoDBAttributeString(av ddbtypes.AttributeValue) (text string, ok bool) { + switch val := av.(type) { + case *ddbtypes.AttributeValueMemberS: + return val.Value, true + case *ddbtypes.AttributeValueMemberN: + return val.Value, true + case *ddbtypes.AttributeValueMemberB: + return string(val.Value), true + case *ddbtypes.AttributeValueMemberBOOL: + return strconv.FormatBool(val.Value), true + case *ddbtypes.AttributeValueMemberNULL: + return "", false + } + + if text, err := marshalDynamoJSON(av); err == nil { + return text, true + } + return "", false +} + +// BulkExportFlow returns a dataflow for the table +func (conn *DynamoDBConn) BulkExportFlow(table Table) (df *iop.Dataflow, err error) { + options, _ := g.UnmarshalMap(table.SQL) + + // add columns if present + if len(table.Columns) > 0 { + options["columns"] = table.Columns + } + + ds, err := conn.StreamRowsContext(conn.Context().Ctx, conn.scanRef(table), options) + if err != nil { + return df, g.Error(err, "could start datastream") + } + + df, err = iop.MakeDataFlow(ds) + if err != nil { + return df, g.Error(err, "could start dataflow") + } + + return +} + +// BulkExportStream returns a datastream for the table +func (conn *DynamoDBConn) BulkExportStream(table Table) (ds *iop.Datastream, err error) { + options, _ := g.UnmarshalMap(table.SQL) + return conn.StreamRowsContext(conn.Context().Ctx, conn.scanRef(table), options) +} + +// scanRef returns the reference to hand to StreamRowsContext: the table name +// when it is known, else the rendered select (a JSON scan descriptor, or a +// `select ... from
` statement) +func (conn *DynamoDBConn) scanRef(table Table) string { + if name := table.FullName(); name != "" { + return name + } + return table.SQL +} + +// StreamRowsContext scans a table, applying the rendered filter/fields/limit +func (conn *DynamoDBConn) StreamRowsContext(ctx context.Context, tableName string, Opts ...map[string]any) (ds *iop.Datastream, err error) { + if err = conn.ensureClient(); err != nil { + return nil, err + } + + // a lifecycle statement (`drop table if exists x`) reaches the read path + // when a query hook runs it, since it cannot tell a statement from a select + // for a connector without a SQL engine: run it and return no rows + if isDynamoDBStatement(tableName) { + if _, err = conn.ExecContext(ctx, tableName); err != nil { + return nil, err + } + + ds = iop.NewDatastreamIt(ctx, iop.Columns{}, func(it *iop.Iterator) bool { return false }) + if err = ds.Start(); err != nil { + return ds, g.Error(err, "could start datastream") + } + return ds, nil + } + + opts := getQueryOptions(Opts) + + // the reference is a table name, a JSON scan descriptor (as rendered by the + // template) or a `select ... from
` statement + tableRef, descriptor, err := dynamoDBScanRef(tableName) + if err != nil { + return nil, err + } else if len(descriptor) > 0 { + opts = mergeDynamoDBScanOptions(opts, descriptor) + } + + table, err := ParseTableName(tableRef, conn.Type) + if err != nil { + return nil, g.Error(err, "could not parse table name: %s", tableRef) + } + + columns := iop.Columns{} + if val, ok := opts["columns"]; ok { + g.JSONConvert(val, &columns) + } + if len(columns) == 0 { + if columns, err = conn.GetTableColumns(&table); err != nil { + return nil, g.Error(err, "could not get columns for table %s", table.Name) + } else if len(columns) == 0 { + // an existing table always has at least its key attributes + return nil, g.Error("table %s does not exist", table.Name) + } + } + allColumns := columns + + // select fields + fields := cast.ToStringSlice(opts["fields"]) + projection := []string{} + if len(fields) > 0 && fields[0] != "*" { + selected := iop.Columns{} + for _, field := range fields { + for _, col := range columns { + if strings.EqualFold(col.Name, field) { + projection = append(projection, col.Name) + selected = append(selected, col) + break + } + } + } + columns = selected + } + + // filter -> FilterExpression. Filters resolve values against every column + // (not just the projected ones) so that a filtered attribute keeps its + // stored type. + filter := newDynamoDBFilter() + if val, ok := opts["filter"]; ok && val != nil { + if err = filter.add(val, allColumns); err != nil { + return nil, g.Error(err, "could not build filter for table %s", table.Name) + } + } + + // incremental and backfill conditions, rendered into options by the template + updateKey := cast.ToString(opts["update_key"]) + incrementalValue := cast.ToString(opts["value"]) + startValue := cast.ToString(opts["start_value"]) + endValue := cast.ToString(opts["end_value"]) + if updateKey != "" { + switch { + case incrementalValue != "": + err = filter.addCondition(updateKey, map[string]any{"$gt": incrementalValue}, allColumns) + case startValue != "" && endValue != "": + err = filter.addCondition(updateKey, map[string]any{"$gte": startValue, "$lte": endValue}, allColumns) + } + if err != nil { + return nil, g.Error(err, "could not build incremental filter for table %s", table.Name) + } + } + + limit := cast.ToInt64(opts["limit"]) + + input := &dynamodb.ScanInput{TableName: aws.String(table.Name)} + if fexpr := filter.expression(); fexpr != "" { + input.FilterExpression = aws.String(fexpr) + } + if len(filter.names) > 0 { + input.ExpressionAttributeNames = filter.names + } + if len(filter.values) > 0 { + input.ExpressionAttributeValues = filter.values + } + if len(projection) > 0 { + input.ProjectionExpression = aws.String(strings.Join(projection, ", ")) + } + + conn.LogSQL(g.F("table=%s options=%s", table.Name, g.Marshal(opts))) + + ds = iop.NewDatastreamContext(ctx, nil) + ds.Columns = columns + + counter := uint64(0) + var pageItems []map[string]ddbtypes.AttributeValue + var lastKey map[string]ddbtypes.AttributeValue + itemIndex := 0 + done := false + + nextFunc := func(it *iop.Iterator) bool { + if limit > 0 && counter >= uint64(limit) { + return false + } else if it.Context.Err() != nil { + return false + } + + for { + if itemIndex < len(pageItems) { + item := pageItems[itemIndex] + itemIndex++ + + row, err := conn.itemToRow(item, ds.Columns) + if err != nil { + it.Context.CaptureErr(err) + return false + } + + it.Row = row + counter++ + return true + } + + if done { + return false + } + + input.ExclusiveStartKey = lastKey + out, err := conn.Client.Scan(ctx, input) + if err != nil { + it.Context.CaptureErr(g.Error(err, "could not scan table %s", table.Name)) + return false + } + + pageItems = out.Items + itemIndex = 0 + lastKey = out.LastEvaluatedKey + done = len(lastKey) == 0 + } + } + + ds.SetIterator(ds.NewIterator(ds.Columns, nextFunc)) + ds.NoDebug = strings.Contains(tableName, noDebugKey) + ds.SetMetadata(conn.GetProp("METADATA")) + ds.SetConfig(conn.Props()) + + if err = ds.Start(); err != nil { + return ds, g.Error(err, "could start datastream") + } + + // unmarshal columns if none detected, + // otherwise this may error when creating a temp table with no columns + if len(ds.Columns) == 0 { + g.JSONConvert(opts["columns"], &ds.Columns) + } + + return ds, nil +} + +// itemToRow converts an item into a row matching the given columns +func (conn *DynamoDBConn) itemToRow(item map[string]ddbtypes.AttributeValue, columns iop.Columns) (row []any, err error) { + row = make([]any, len(columns)) + + for i, col := range columns { + av, ok := item[col.Name] + if !ok { + row[i] = nil + continue + } + if row[i], err = dynamoDBValueToGo(av, col.Type); err != nil { + return nil, g.Error(err, "could not convert column %s", col.Name) + } + } + + return row, nil +} + +// dynamoDBValueToGo converts an attribute value to a Go value, guided by the column type +func dynamoDBValueToGo(av ddbtypes.AttributeValue, colType iop.ColumnType) (any, error) { + switch val := av.(type) { + case *ddbtypes.AttributeValueMemberS: + if colType.IsDate() || colType.IsDatetime() { + if parsed, err := parseISOTime(val.Value); err == nil { + return parsed, nil + } + } + return val.Value, nil + case *ddbtypes.AttributeValueMemberN: + if colType == iop.BigIntType { + if parsed, err := strconv.ParseInt(val.Value, 10, 64); err == nil { + return parsed, nil + } + } + if colType == iop.FloatType { + return cast.ToFloat64(val.Value), nil + } + if parsed, err := strconv.ParseInt(val.Value, 10, 64); err == nil { + return parsed, nil + } else if _, err := strconv.ParseFloat(val.Value, 64); err == nil { + return val.Value, nil // keep full precision as a string + } + return val.Value, nil + case *ddbtypes.AttributeValueMemberB: + return val.Value, nil + case *ddbtypes.AttributeValueMemberBOOL: + return val.Value, nil + case *ddbtypes.AttributeValueMemberNULL: + return nil, nil + case *ddbtypes.AttributeValueMemberL, *ddbtypes.AttributeValueMemberM, + *ddbtypes.AttributeValueMemberSS, *ddbtypes.AttributeValueMemberNS, + *ddbtypes.AttributeValueMemberBS: + native, err := dynamoDBValueToGoNative(av, colType) + if err != nil { + return nil, err + } + bytes, err := json.Marshal(native) + return string(bytes), err + } + return nil, nil +} + +// dynamoDBValueToGoNative converts an attribute value into a Go value, keeping +// nested structures native (a list stays a slice, a map stays a map) so that a +// document attribute is not stringified field by field +func dynamoDBValueToGoNative(av ddbtypes.AttributeValue, colType iop.ColumnType) (any, error) { + switch val := av.(type) { + case *ddbtypes.AttributeValueMemberL: + arr := make([]any, len(val.Value)) + for i, item := range val.Value { + v, err := dynamoDBValueToGoNative(item, "") + if err != nil { + return nil, err + } + arr[i] = v + } + return arr, nil + case *ddbtypes.AttributeValueMemberM: + m := map[string]any{} + for key, item := range val.Value { + v, err := dynamoDBValueToGoNative(item, "") + if err != nil { + return nil, err + } + m[key] = v + } + return m, nil + case *ddbtypes.AttributeValueMemberSS: + return val.Value, nil + case *ddbtypes.AttributeValueMemberNS: + return val.Value, nil + case *ddbtypes.AttributeValueMemberBS: + arr := make([]string, len(val.Value)) + for i, item := range val.Value { + arr[i] = string(item) + } + return arr, nil + } + return dynamoDBValueToGo(av, colType) +} + +// parseISOTime parses RFC3339 timestamps and ISO dates +func parseISOTime(text string) (time.Time, error) { + if parsed, err := time.Parse(time.RFC3339Nano, text); err == nil { + return parsed, nil + } + return time.Parse("2006-01-02", text) +} + +// marshalDynamoJSON renders a complex attribute value as JSON text +func marshalDynamoJSON(av any) (string, error) { + switch val := av.(type) { + case []ddbtypes.AttributeValue: + arr := make([]any, len(val)) + for i, item := range val { + v, err := dynamoDBValueToGoNative(item, "") + if err != nil { + return "", err + } + arr[i] = v + } + bytes, err := json.Marshal(arr) + return string(bytes), err + case map[string]ddbtypes.AttributeValue: + m := map[string]any{} + for k, item := range val { + v, err := dynamoDBValueToGoNative(item, "") + if err != nil { + return "", err + } + m[k] = v + } + bytes, err := json.Marshal(m) + return string(bytes), err + case []string: + bytes, err := json.Marshal(val) + return string(bytes), err + case [][]byte: + arr := make([]string, len(val)) + for i, item := range val { + arr[i] = string(item) + } + bytes, err := json.Marshal(arr) + return string(bytes), err + } + bytes, err := json.Marshal(av) + return string(bytes), err +} + +// BulkImportFlow imports data into DynamoDB +func (conn *DynamoDBConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error) { + defer df.CleanUp() + + df.Context.SetConcurrencyLimit(conn.Context().Wg.Limit) + + doImport := func(tableFName string, ds *iop.Datastream) { + defer df.Context.Wg.Write.Done() + + cnt, err := conn.BulkImportStream(tableFName, ds) + count += cnt + if err != nil { + df.Context.CaptureErr(g.Error(err, "could not bulk import into %s", tableFName)) + } else if err = ds.Err(); err != nil { + df.Context.CaptureErr(g.Error(err, "could not bulk import into %s", tableFName)) + } + } + + for ds := range df.StreamCh { + df.Context.Wg.Write.Add() + doImport(tableFName, ds) + } + + df.Context.Wg.Write.Wait() + + return count, df.Err() +} + +// BulkImportStream writes a datastream with BatchWriteItem +func (conn *DynamoDBConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error) { + table, err := ParseTableName(tableFName, conn.Type) + if err != nil { + return 0, g.Error(err, "could not parse table name: %s", tableFName) + } + + desc, found, err := conn.describeTable(conn.Context().Ctx, table.Name) + if err != nil { + return 0, err + } else if !found { + return 0, g.Error("table %s does not exist", table.Name) + } + + keyNames := []string{} + for _, keySchema := range desc.KeySchema { + keyNames = append(keyNames, *keySchema.AttributeName) + } + + ctx := conn.Context().Ctx + items := []map[string]ddbtypes.AttributeValue{} + + flush := func() (err error) { + if len(items) == 0 { + return nil + } + + // a batch cannot carry the same key twice, so repeated keys collapse to + // their last value (DynamoDB upserts on write) + unique := dedupeByKey(items, keyNames) + if collapsed := len(items) - len(unique); collapsed > 0 { + g.Warn("collapsed %d row(s) on duplicate key(s) %s (DynamoDB writes are keyed upserts)", + collapsed, strings.Join(keyNames, ", ")) + } + + if err = conn.batchWriteRequests(ctx, table.Name, putRequests(unique)); err != nil { + return g.Error(err, "could not write to table %s", table.Name) + } + count += uint64(len(unique)) + items = items[:0] + return nil + } + + for row := range ds.Rows() { + item, err := conn.rowToItem(ds.Columns, row, keyNames) + if err != nil { + return count, err + } + items = append(items, item) + + if len(items) >= dynamoDBBatchSize { + if err = flush(); err != nil { + return count, err + } + } + } + + if err = ds.Err(); err != nil { + return count, g.Error(err, "stream error") + } + + if err = flush(); err != nil { + return count, err + } + + return count, nil +} + +// InsertStream writes a datastream (DynamoDB upserts on write) +func (conn *DynamoDBConn) InsertStream(tableFName string, ds *iop.Datastream) (count uint64, err error) { + return conn.BulkImportStream(tableFName, ds) +} + +// InsertBatchStream writes a datastream (DynamoDB upserts on write) +func (conn *DynamoDBConn) InsertBatchStream(tableFName string, ds *iop.Datastream) (count uint64, err error) { + return conn.BulkImportStream(tableFName, ds) +} + +// dynamoDBNowExpressions are the SQL functions that mean "now" +var dynamoDBNowExpressions = []string{ + "current_timestamp", "current_timestamp()", "now()", "getdate()", "sysdate()", "sysdate", "localtimestamp", +} + +// updateItems applies `update
set = where ` by +// rewriting the matching items. sling renders this shape to soft-delete the rows +// that are missing from a stream (`delete_missing: soft`), which is the only +// update DynamoDB needs to support. +func (conn *DynamoDBConn) updateItems(ctx context.Context, text string) (err error) { + setLoc := dynamoDBSetRegex.FindStringIndex(text) + if setLoc == nil { + return g.Error("could not parse update statement: %s", text) + } + + tableName := parseDynamoDBTableToken(text, "update") + setEnd := len(text) + if whereLoc := dynamoDBWhereRegex.FindStringIndex(text[setLoc[1]:]); whereLoc != nil { + setEnd = setLoc[1] + whereLoc[0] + } + + assignments, err := parseDynamoDBAssignments(text[setLoc[1]:setEnd]) + if err != nil { + return err + } + + filter := map[string]any{} + join := dynamoDBNotExistsJoin{} + if setEnd < len(text) { + whereText := text[setEnd:] + join, err = parseDynamoDBNotExistsJoin(whereText) + if err != nil { + return err + } else if join.Table != "" { + whereText = stripDynamoDBNotExistsJoin(whereText) + } + + if filter, err = parseDynamoDBWhere(whereText); err != nil { + return err + } + } + + excluded := map[string]bool{} + if join.Table != "" { + if excluded, err = conn.loadDynamoDBKeySet(ctx, join); err != nil { + return err + } + } + + pending := []map[string]ddbtypes.AttributeValue{} + flush := func() (err error) { + if len(pending) == 0 { + return nil + } + if err = conn.batchWriteRequests(ctx, tableName, putRequests(pending)); err != nil { + return g.Error(err, "could not update table %s", tableName) + } + pending = nil + return nil + } + + err = conn.scanDynamoDBItems(ctx, tableName, filter, func(item map[string]ddbtypes.AttributeValue) error { + if len(join.Columns) > 0 && excluded[dynamoDBItemKey(item, join.Columns)] { + return nil // the stream still holds this row, so keep it untouched + } + + for col, assignment := range assignments { + if assignment.remove { + delete(item, col) + } else { + item[col] = assignment.value + } + } + + pending = append(pending, item) + if len(pending) >= dynamoDBBatchSize { + return flush() + } + return nil + }) + if err != nil { + return err + } + + return flush() +} + +// deleteItems applies `delete from
[where ]`. sling renders this +// shape to drop the rows that are missing from a stream +// (`delete_missing: hard`). +func (conn *DynamoDBConn) deleteItems(ctx context.Context, text string) (err error) { + tableName := parseDynamoDBTableToken(text, "delete from") + + filter := map[string]any{} + join := dynamoDBNotExistsJoin{} + if whereLoc := dynamoDBWhereRegex.FindStringIndex(text); whereLoc != nil { + whereText := text[whereLoc[0]:] + join, err = parseDynamoDBNotExistsJoin(whereText) + if err != nil { + return err + } else if join.Table != "" { + whereText = stripDynamoDBNotExistsJoin(whereText) + } + + if filter, err = parseDynamoDBWhere(whereText); err != nil { + return err + } + } + + excluded := map[string]bool{} + if join.Table != "" { + if excluded, err = conn.loadDynamoDBKeySet(ctx, join); err != nil { + return err + } + } + + desc, found, err := conn.describeTable(ctx, tableName) + if err != nil { + return err + } else if !found { + return nil + } + + keyNames := []string{} + for _, keySchema := range desc.KeySchema { + keyNames = append(keyNames, *keySchema.AttributeName) + } + + keys := []map[string]ddbtypes.AttributeValue{} + flush := func() (err error) { + if len(keys) == 0 { + return nil + } + if err = conn.batchWriteRequests(ctx, tableName, deleteRequests(keys)); err != nil { + return g.Error(err, "could not delete from table %s", tableName) + } + keys = nil + return nil + } + + err = conn.scanDynamoDBItems(ctx, tableName, filter, func(item map[string]ddbtypes.AttributeValue) error { + if len(join.Columns) > 0 && excluded[dynamoDBItemKey(item, join.Columns)] { + return nil // the stream still holds this row, so do not delete it + } + + key := map[string]ddbtypes.AttributeValue{} + for _, name := range keyNames { + key[name] = item[name] + } + + keys = append(keys, key) + if len(keys) >= dynamoDBBatchSize { + return flush() + } + return nil + }) + if err != nil { + return err + } + + return flush() +} + +// dynamoDBNotExistsJoin describes the clause sling appends to its soft and hard +// delete statements, which keeps the rows that the stream still holds: +// +// and not exists (select 1 from where .= ....) +type dynamoDBNotExistsJoin struct { + Table string // table holding the keys of the current stream + Columns []string // columns compared between that table and the target +} + +// parseDynamoDBNotExistsJoin reads the `not exists (...)` clause, if present +func parseDynamoDBNotExistsJoin(text string) (join dynamoDBNotExistsJoin, err error) { + lower := strings.ToLower(text) + idx := strings.Index(lower, "not exists") + if idx == -1 { + return join, nil + } + + openIdx := strings.Index(text[idx:], "(") + if openIdx == -1 { + return join, g.Error("could not parse `not exists` clause: %s", text) + } else if closeIdx := matchDynamoDBParen(text[idx+openIdx:]); closeIdx == -1 { + return join, g.Error("unbalanced parenthesis in `not exists` clause: %s", text) + } else { + clause := text[idx+openIdx+1 : idx+openIdx+closeIdx] + + fromIdx := indexDynamoDBKeyword(clause, "from") + if fromIdx == -1 { + return join, g.Error("could not find `from` in `not exists` clause: %s", clause) + } + + rest := strings.TrimSpace(clause[fromIdx+len("from"):]) + if whereIdx := indexDynamoDBKeyword(rest, "where"); whereIdx != -1 { + join.Table = unquoteDynamoDBIdentifier(rest[:whereIdx]) + rest = rest[whereIdx+len("where"):] + } else { + join.Table = unquoteDynamoDBIdentifier(rest) + rest = "" + } + + for _, condition := range strings.Split(rest, " and ") { + sides := strings.Split(condition, "=") + if len(sides) != 2 { + continue + } + + left := unquoteDynamoDBIdentifier(sides[0]) + right := unquoteDynamoDBIdentifier(sides[1]) + if left == "" || right == "" { + continue + } else if left == right { + join.Columns = append(join.Columns, left) + continue + } + + // the clause compares `.= .`, so the column + // name is the same on both sides of a well-formed join + return join, g.Error("unsupported `not exists` join condition: %s", strings.TrimSpace(condition)) + } + } + + return join, nil +} + +// indexDynamoDBKeyword returns the offset of a standalone keyword, which sling +// renders on its own line (with indentation) inside the delete statements. +func indexDynamoDBKeyword(text, keyword string) int { + lower := strings.ToLower(text) + offset := 0 + + for { + idx := strings.Index(lower[offset:], keyword) + if idx == -1 { + return -1 + } + idx += offset + + end := idx + len(keyword) + isBoundary := func(position int) bool { + if position < 0 || position >= len(lower) { + return true + } + switch lower[position] { + case ' ', '\t', '\n', '\r', '(', ')', ',': + return true + } + return false + } + + if isBoundary(idx-1) && isBoundary(end) { + return idx + } + offset = idx + 1 + } +} + +// isDynamoDBIdentifier reports whether a name is a plain column name: DynamoDB +// has no functions or expressions to fold into a filter, but attribute names may +// carry dashes and dots +func isDynamoDBIdentifier(name string) bool { + if name == "" { + return false + } + + for i, char := range name { + switch { + case char >= 'a' && char <= 'z', char >= 'A' && char <= 'Z', char == '_': + case char >= '0' && char <= '9' && i > 0: + case char == '-' || char == '.' || char == '$': + default: + return false + } + } + + return true +} + +// stripDynamoDBNotExistsJoin removes the `not exists (...)` clause, so the +// remaining conditions can be parsed +func stripDynamoDBNotExistsJoin(text string) string { + lower := strings.ToLower(text) + idx := strings.Index(lower, "not exists") + if idx == -1 { + return text + } + + openIdx := strings.Index(text[idx:], "(") + if openIdx == -1 { + return text + } + closeIdx := matchDynamoDBParen(text[idx+openIdx:]) + if closeIdx == -1 { + return text + } + + rest := text[:idx] + text[idx+openIdx+closeIdx+1:] + rest = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(rest), "and")) + return strings.TrimSpace(rest) +} + +// matchDynamoDBParen returns the index of the parenthesis closing the one at +// position 0 of text +func matchDynamoDBParen(text string) int { + depth, quote := 0, byte(0) + for i := 0; i < len(text); i++ { + switch char := text[i]; { + case quote != 0: + if char == quote { + quote = 0 + } + case char == '\'' || char == '"': + quote = char + case char == '(': + depth++ + case char == ')': + depth-- + if depth == 0 { + return i + } + } + } + return -1 +} + +// dynamoDBItemKey renders the joined columns of an item as a signature +func dynamoDBItemKey(item map[string]ddbtypes.AttributeValue, columns []string) string { + signature := strings.Builder{} + for _, col := range columns { + signature.WriteString(g.F("%s\x00%s\x00", col, dynamoDBKeyValue(item[col]))) + } + return signature.String() +} + +// loadDynamoDBKeySet reads the key signatures of the stream's temp table +func (conn *DynamoDBConn) loadDynamoDBKeySet(ctx context.Context, join dynamoDBNotExistsJoin) (keys map[string]bool, err error) { + keys = map[string]bool{} + if join.Table == "" { + return keys, nil + } + + err = conn.scanDynamoDBItems(ctx, join.Table, nil, func(item map[string]ddbtypes.AttributeValue) error { + keys[dynamoDBItemKey(item, join.Columns)] = true + return nil + }) + if err != nil { + return nil, g.Error(err, "could not read %s", join.Table) + } + + return keys, nil +} + +// scanDynamoDBItems pages through the items that match a filter +func (conn *DynamoDBConn) scanDynamoDBItems(ctx context.Context, tableName string, filter map[string]any, fn func(item map[string]ddbtypes.AttributeValue) error) (err error) { + if err = conn.ensureClient(); err != nil { + return err + } + + f := newDynamoDBFilter() + if len(filter) > 0 { + if err = f.add(filter, iop.Columns{}); err != nil { + return err + } + } + + var lastKey map[string]ddbtypes.AttributeValue + for { + input := &dynamodb.ScanInput{ + TableName: aws.String(tableName), + ExclusiveStartKey: lastKey, + } + if expression := f.expression(); expression != "" { + input.FilterExpression = aws.String(expression) + input.ExpressionAttributeNames = f.names + if len(f.values) > 0 { + input.ExpressionAttributeValues = f.values + } + } + + out, err := conn.Client.Scan(ctx, input) + if err != nil { + return g.Error(err, "could not scan table %s", tableName) + } + + for _, item := range out.Items { + if err = fn(item); err != nil { + return err + } + } + + lastKey = out.LastEvaluatedKey + if len(lastKey) == 0 { + return nil + } + } +} + +// dynamoDBAssignment is a column assignment of an update statement +type dynamoDBAssignment struct { + value ddbtypes.AttributeValue + remove bool // `set col = null` drops the attribute +} + +// parseDynamoDBAssignments reads the `col = value` list of an update statement +func parseDynamoDBAssignments(text string) (assignments map[string]dynamoDBAssignment, err error) { + assignments = map[string]dynamoDBAssignment{} + + for _, item := range splitDynamoDBTopLevel(text, ",") { + parts := strings.SplitN(item, "=", 2) + if len(parts) != 2 { + return nil, g.Error("could not parse assignment %q", strings.TrimSpace(item)) + } + + col := strings.Trim(strings.TrimSpace(parts[0]), "`\"") + expr := strings.TrimSpace(parts[1]) + + if g.In(strings.ToLower(expr), dynamoDBNowExpressions...) { + assignments[col] = dynamoDBAssignment{ + value: &ddbtypes.AttributeValueMemberS{Value: time.Now().UTC().Format(time.RFC3339Nano)}, + } + continue + } else if strings.EqualFold(expr, "null") { + assignments[col] = dynamoDBAssignment{remove: true} + continue + } + + literal, ok := parseDynamoDBLiteral(expr) + if !ok { + return nil, g.Error("unsupported value %q in update statement", expr) + } + + value, err := dynamoDBFilterValue(literal, nil) + if err != nil { + return nil, err + } + assignments[col] = dynamoDBAssignment{value: value} + } + + return assignments, nil +} + +// parseDynamoDBWhere converts the conditions sling renders for missing records +// (`col is null`, `col is not null`, and `col literal`, joined by AND) into +// the filter object the scan builder understands. +func parseDynamoDBWhere(text string) (filter map[string]any, err error) { + filter = map[string]any{} + + conditions := strings.TrimSpace(text) + if strings.HasPrefix(strings.ToLower(conditions), "where ") { + conditions = strings.TrimSpace(conditions[len("where "):]) + } + + for _, condition := range splitDynamoDBTopLevel(conditions, " and ") { + condition = strings.Trim(strings.TrimSpace(condition), "()") + if condition == "" { + continue + } + + // an unfiltered update/delete renders `where 1=1`, which asks for nothing + if strings.ReplaceAll(condition, " ", "") == "1=1" { + continue + } + + lower := strings.ToLower(condition) + switch { + case strings.HasSuffix(lower, " is null"): + col := unquoteDynamoDBIdentifier(condition[:len(condition)-len(" is null")]) + if !isDynamoDBIdentifier(col) { + return nil, g.Error("could not parse condition %q", condition) + } + filter[col] = map[string]any{"$exists": false} + case strings.HasSuffix(lower, " is not null"): + col := unquoteDynamoDBIdentifier(condition[:len(condition)-len(" is not null")]) + if !isDynamoDBIdentifier(col) { + return nil, g.Error("could not parse condition %q", condition) + } + filter[col] = map[string]any{"$exists": true} + default: + col, op, literal, ok := parseDynamoDBComparison(condition) + if !ok || !isDynamoDBIdentifier(col) { + // a condition the connector cannot evaluate must not be dropped: + // deleting rows the user excluded is worse than failing loudly + return nil, g.Error("could not parse condition %q", condition) + } + filter[col] = map[string]any{op: literal} + } + } + + return filter, nil +} + +// parseDynamoDBComparison reads a `col literal` condition +func parseDynamoDBComparison(condition string) (col string, op string, literal any, ok bool) { + operators := []string{"<=", ">=", "<>", "!=", "=", "<", ">"} + for _, operator := range operators { + idx := strings.Index(condition, operator) + if idx == -1 { + continue + } + + col = unquoteDynamoDBIdentifier(condition[:idx]) + literalText := strings.TrimSpace(condition[idx+len(operator):]) + value, isLiteral := parseDynamoDBLiteral(literalText) + if !isLiteral { + return "", "", nil, false + } + + switch operator { + case "=": + op = "$eq" + case "<>", "!=": + op = "$ne" + case "<": + op = "$lt" + case "<=": + op = "$lte" + case ">": + op = "$gt" + case ">=": + op = "$gte" + } + + return col, op, value, true + } + + return "", "", nil, false +} + +// parseDynamoDBLiteral reads a SQL literal: a quoted string, a number, or a bool +func parseDynamoDBLiteral(expr string) (value any, ok bool) { + text := strings.TrimSpace(expr) + if len(text) >= 2 { + quote := text[0] + if (quote == '\'' || quote == '"') && text[len(text)-1] == quote { + inner := text[1 : len(text)-1] + return strings.ReplaceAll(inner, string([]byte{quote, quote}), string(quote)), true + } + } + + switch strings.ToLower(text) { + case "true": + return true, true + case "false": + return false, true + } + + if number, err := strconv.ParseFloat(text, 64); err == nil { + return number, true + } + + return nil, false +} + +// unquoteDynamoDBIdentifier strips the quoting and qualifier of a column name +func unquoteDynamoDBIdentifier(text string) string { + name := strings.Trim(strings.TrimSpace(text), "`\"'") + name = strings.TrimPrefix(name, "()") + name = strings.Trim(name, "()") + if parts := strings.Split(name, "."); len(parts) > 1 { + name = parts[len(parts)-1] + } + return strings.Trim(name, "`\"'") +} + +// splitDynamoDBTopLevel splits text on a separator that is not inside quotes or +// parenthesis +func splitDynamoDBTopLevel(text string, separator string) (items []string) { + depth := 0 + quote := byte(0) + start := 0 + + for i := 0; i < len(text); i++ { + switch char := text[i]; { + case quote != 0: + if char == quote { + quote = 0 + } + case char == '\'' || char == '"': + quote = char + case char == '(': + depth++ + case char == ')': + depth-- + case depth == 0 && strings.EqualFold(text[i:min(i+len(separator), len(text))], separator): + items = append(items, text[start:i]) + i += len(separator) - 1 + start = i + 1 + } + } + + return append(items, text[start:]) +} + +// batchWriteRequests sends write requests in batches of 25, retrying unprocessed items +func (conn *DynamoDBConn) batchWriteRequests(ctx context.Context, tableName string, requests []ddbtypes.WriteRequest) (err error) { + if err = conn.ensureClient(); err != nil { + return err + } + for len(requests) > 0 { + batch := requests + if len(batch) > dynamoDBBatchSize { + batch = batch[:dynamoDBBatchSize] + } + requests = requests[len(batch):] + + pending := map[string][]ddbtypes.WriteRequest{tableName: batch} + for attempt := 1; ; attempt++ { + out, err := conn.Client.BatchWriteItem(ctx, &dynamodb.BatchWriteItemInput{RequestItems: pending}) + if err != nil { + return g.Error(err, "batch write failed") + } + + unprocessed := out.UnprocessedItems[tableName] + if len(unprocessed) == 0 { + break + } else if attempt >= 10 { + return g.Error("could not write %d item(s) after %d attempts (throttled)", len(unprocessed), attempt) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(attempt*50) * time.Millisecond): + } + pending = map[string][]ddbtypes.WriteRequest{tableName: unprocessed} + } + } + + return nil +} + +// dedupeByKey keeps the last item seen for each key, preserving the order in +// which keys were first written. DynamoDB rejects a batch that carries the same +// key twice, and a repeated key means the later value wins anyway. +func dedupeByKey(items []map[string]ddbtypes.AttributeValue, keyNames []string) (unique []map[string]ddbtypes.AttributeValue) { + if len(keyNames) == 0 { + return items + } + + positions := map[string]int{} + for _, item := range items { + sig := "" + for _, name := range keyNames { + sig += g.F("%s\x00%s\x00", name, dynamoDBKeyValue(item[name])) + } + + if idx, ok := positions[sig]; ok { + unique[idx] = item // keep the position, take the latest value + continue + } + positions[sig] = len(unique) + unique = append(unique, item) + } + return unique +} + +// dynamoDBKeyValue renders a key attribute as text for comparison +func dynamoDBKeyValue(av ddbtypes.AttributeValue) string { + switch v := av.(type) { + case *ddbtypes.AttributeValueMemberS: + return v.Value + case *ddbtypes.AttributeValueMemberN: + return v.Value + case *ddbtypes.AttributeValueMemberB: + return string(v.Value) + } + return g.Marshal(av) +} + +// putRequests builds PutRequests (upsert by primary key) +func putRequests(items []map[string]ddbtypes.AttributeValue) (requests []ddbtypes.WriteRequest) { + for _, item := range items { + requests = append(requests, ddbtypes.WriteRequest{ + PutRequest: &ddbtypes.PutRequest{Item: item}, + }) + } + return requests +} + +// deleteRequests builds DeleteRequests from item keys +func deleteRequests(keys []map[string]ddbtypes.AttributeValue) (requests []ddbtypes.WriteRequest) { + for _, key := range keys { + requests = append(requests, ddbtypes.WriteRequest{ + DeleteRequest: &ddbtypes.DeleteRequest{Key: key}, + }) + } + return requests +} + +// dynamoDBSyntheticKeyValue returns a unique value for an added key column. A +// timestamp is not enough on its own: bulk writes can emit many rows in the +// same millisecond, so a sequence is appended. +func (conn *DynamoDBConn) dynamoDBSyntheticKeyValue() string { + return g.F("%s%08d", g.NewTsID(""), atomic.AddUint64(&conn.syntheticKeySeq, 1)) +} + +// rowToItem converts a row into a DynamoDB item +func (conn *DynamoDBConn) rowToItem(columns iop.Columns, row []any, keyNames []string) (item map[string]ddbtypes.AttributeValue, err error) { + item = map[string]ddbtypes.AttributeValue{} + + for i, col := range columns { + if i >= len(row) || row[i] == nil { + continue + } + + av, err := goToDynamoDBValue(row[i], col.Type) + if err != nil { + return nil, g.Error(err, "could not convert column %s", col.Name) + } else if av == nil { + continue + } + item[col.Name] = av + } + + for _, name := range keyNames { + if _, ok := item[name]; ok { + continue + } + + if isDynamoDBSyntheticKey(name) { + // the key column sling added: every row needs its own value + item[name] = &ddbtypes.AttributeValueMemberS{Value: conn.dynamoDBSyntheticKeyValue()} + continue + } + + return nil, g.Error("key attribute %s is missing (DynamoDB requires it on every item)", name) + } + + return item, nil +} + +// goToDynamoDBValue converts a Go value into a DynamoDB attribute value +func goToDynamoDBValue(val any, colType iop.ColumnType) (ddbtypes.AttributeValue, error) { + switch { + case val == nil: + return nil, nil + case colType.IsBool(): + return &ddbtypes.AttributeValueMemberBOOL{Value: cast.ToBool(val)}, nil + case colType.IsInteger(): + return &ddbtypes.AttributeValueMemberN{Value: strconv.FormatInt(cast.ToInt64(val), 10)}, nil + case colType.IsNumber(): + return &ddbtypes.AttributeValueMemberN{Value: numberString(cast.ToFloat64(val))}, nil + case colType.IsBinary(): + switch v := val.(type) { + case []byte: + return &ddbtypes.AttributeValueMemberB{Value: v}, nil + default: + return &ddbtypes.AttributeValueMemberB{Value: []byte(cast.ToString(val))}, nil + } + case colType == iop.JsonType: + if parsed, ok := unmarshalDynamoJSON(val); ok { + return parsed, nil + } + return &ddbtypes.AttributeValueMemberS{Value: cast.ToString(val)}, nil + case colType.IsDate() || colType.IsDatetime(): + if err := trySetTime(val); err != nil { + return nil, err + } + return &ddbtypes.AttributeValueMemberS{Value: cast.ToTime(val).Format(time.RFC3339Nano)}, nil + } + + return &ddbtypes.AttributeValueMemberS{Value: cast.ToString(val)}, nil +} + +// trySetTime validates that a value can be represented as a timestamp +func trySetTime(val any) error { + if cast.ToTime(val).IsZero() { + return g.Error("could not convert value %#v to a timestamp", val) + } + return nil +} + +// numberString renders a float without an exponent, so numbers round-trip +func numberString(val float64) string { + return strconv.FormatFloat(val, 'f', -1, 64) +} + +// unmarshalDynamoJSON converts JSON text into native DynamoDB attribute values +func unmarshalDynamoJSON(val any) (av ddbtypes.AttributeValue, ok bool) { + text, isText := val.(string) + if !isText { + return nil, false + } + + parsed := any(nil) + if err := json.Unmarshal([]byte(text), &parsed); err != nil { + return nil, false + } + return goValueToDynamoJSON(parsed), true +} + +// goValueToDynamoJSON converts a decoded JSON value into an attribute value +func goValueToDynamoJSON(val any) ddbtypes.AttributeValue { + switch v := val.(type) { + case nil: + return &ddbtypes.AttributeValueMemberNULL{Value: true} + case bool: + return &ddbtypes.AttributeValueMemberBOOL{Value: v} + case float64: + if v == float64(int64(v)) { + return &ddbtypes.AttributeValueMemberN{Value: strconv.FormatInt(int64(v), 10)} + } + return &ddbtypes.AttributeValueMemberN{Value: numberString(v)} + case string: + return &ddbtypes.AttributeValueMemberS{Value: v} + case []any: + items := make([]ddbtypes.AttributeValue, len(v)) + for i, item := range v { + items[i] = goValueToDynamoJSON(item) + } + return &ddbtypes.AttributeValueMemberL{Value: items} + case map[string]any: + m := map[string]ddbtypes.AttributeValue{} + for k, item := range v { + m[k] = goValueToDynamoJSON(item) + } + return &ddbtypes.AttributeValueMemberM{Value: m} + } + return &ddbtypes.AttributeValueMemberS{Value: cast.ToString(val)} +} + +// AddMissingColumns is a no-op: DynamoDB attributes other than the key are schemaless +func (conn *DynamoDBConn) AddMissingColumns(table Table, newCols iop.Columns) (ok bool, err error) { + return false, nil +} + +// CompareChecksums is not supported by DynamoDB +func (conn *DynamoDBConn) CompareChecksums(tableName string, columns iop.Columns) (err error) { + return nil +} + +// dynamoDBFilter builds a DynamoDB filter expression from the JSON filter that +// sling renders for a stream. Supported shapes: +// +// { "id": 5 } -> #n0 = :v0 +// { "id": { "$gt": 5 } } -> #n0 > :v0 +// { "id": { "$between": [1, 5] } } -> #n0 BETWEEN :v0 AND :v1 +// { "id": { "$in": [1, 2] } } -> #n0 IN (:v0, :v1) +// { "name": { "$begins_with": "a" }} -> begins_with(#n0, :v0) +// { "name": { "$contains": "a" } } -> contains(#n0, :v0) +// { "col": { "$exists": false } } -> attribute_not_exists(#n0) +// +// Multiple columns and operators are joined with AND. +type dynamoDBFilter struct { + parts []string + names map[string]string // alias -> column + aliases map[string]string // column -> alias + values map[string]ddbtypes.AttributeValue +} + +func newDynamoDBFilter() *dynamoDBFilter { + return &dynamoDBFilter{ + names: map[string]string{}, + aliases: map[string]string{}, + values: map[string]ddbtypes.AttributeValue{}, + } +} + +func (f *dynamoDBFilter) expression() string { + return strings.Join(f.parts, " AND ") +} + +// name registers an attribute name and returns its expression placeholder +func (f *dynamoDBFilter) name(col string) string { + if alias, ok := f.aliases[col]; ok { + return alias // one alias per column, reused across its conditions + } + + alias := g.F("#n%d", len(f.names)) + f.names[alias] = col + f.aliases[col] = alias + return alias +} + +// value registers an attribute value and returns its expression placeholder +func (f *dynamoDBFilter) value(col string, raw any, columns iop.Columns) (placeholder string, err error) { + av, err := dynamoDBFilterValue(raw, columns.GetColumn(col)) + if err != nil { + return "", g.Error(err, "invalid filter value for %s", col) + } else if av == nil { + return "", g.Error("filter value for %s is null", col) + } + + placeholder = g.F(":v%d", len(f.values)) + f.values[placeholder] = av + return placeholder, nil +} + +// add adds every condition of a filter object +func (f *dynamoDBFilter) add(filter any, columns iop.Columns) (err error) { + m, ok := filter.(map[string]any) + if !ok { + return g.Error("filter must be a JSON object, got %T", filter) + } + + for col, spec := range m { + if err = f.addCondition(col, spec, columns); err != nil { + return err + } + } + return nil +} + +// addCondition adds one column condition +func (f *dynamoDBFilter) addCondition(col string, spec any, columns iop.Columns) (err error) { + operators, isOperatorMap := spec.(map[string]any) + if !isOperatorMap { + // a plain value is an equality check + placeholder, err := f.value(col, spec, columns) + if err != nil { + return err + } + f.parts = append(f.parts, g.F("%s = %s", f.name(col), placeholder)) + return nil + } + + comparisons := map[string]string{"$eq": "=", "$ne": "<>", "$lt": "<", "$lte": "<=", "$gt": ">", "$gte": ">="} + for op, val := range operators { + switch op { + case "$eq", "$ne", "$lt", "$lte", "$gt", "$gte": + placeholder, err := f.value(col, val, columns) + if err != nil { + return err + } + f.parts = append(f.parts, g.F("%s %s %s", f.name(col), comparisons[op], placeholder)) + case "$begins_with", "$contains": + placeholder, err := f.value(col, val, columns) + if err != nil { + return err + } + function := strings.TrimPrefix(op, "$") + f.parts = append(f.parts, g.F("%s(%s, %s)", function, f.name(col), placeholder)) + case "$in": + items, ok := val.([]any) + if !ok { + return g.Error("$in expects an array for %s", col) + } + placeholders := []string{} + for _, item := range items { + placeholder, err := f.value(col, item, columns) + if err != nil { + return err + } + placeholders = append(placeholders, placeholder) + } + f.parts = append(f.parts, g.F("%s IN (%s)", f.name(col), strings.Join(placeholders, ", "))) + case "$between": + items, ok := val.([]any) + if !ok || len(items) != 2 { + return g.Error("$between expects [start, end] for %s", col) + } + start, err := f.value(col, items[0], columns) + if err != nil { + return err + } + end, err := f.value(col, items[1], columns) + if err != nil { + return err + } + f.parts = append(f.parts, g.F("%s BETWEEN %s AND %s", f.name(col), start, end)) + case "$exists": + if cast.ToBool(val) { + f.parts = append(f.parts, g.F("attribute_exists(%s)", f.name(col))) + } else { + f.parts = append(f.parts, g.F("attribute_not_exists(%s)", f.name(col))) + } + default: + return g.Error("unsupported filter operator %s for %s", op, col) + } + } + + return nil +} + +// dynamoDBFilterValue converts a filter value into an attribute value of the +// type stored in the table, so that comparisons match the stored attributes. +func dynamoDBFilterValue(raw any, col *iop.Column) (ddbtypes.AttributeValue, error) { + switch val := raw.(type) { + case nil: + return nil, nil + case bool: + return &ddbtypes.AttributeValueMemberBOOL{Value: val}, nil + case float64: + if val == float64(int64(val)) { + return &ddbtypes.AttributeValueMemberN{Value: strconv.FormatInt(int64(val), 10)}, nil + } + return &ddbtypes.AttributeValueMemberN{Value: numberString(val)}, nil + case map[string]any, []any: + return goValueToDynamoJSON(raw), nil + case string: + // values rendered from another connection (incremental max values) are + // quoted as SQL literals, and possibly carrying doubled quotes + val = normalizeDynamoDBFilterString(val) + + if col != nil { + if col.Type.IsBool() { + return &ddbtypes.AttributeValueMemberBOOL{Value: cast.ToBool(val)}, nil + } + if col.Type.IsNumber() { + if col.Type.IsInteger() { + if parsed, err := strconv.ParseInt(val, 10, 64); err == nil { + return &ddbtypes.AttributeValueMemberN{Value: strconv.FormatInt(parsed, 10)}, nil + } + } else if parsed, err := strconv.ParseFloat(val, 64); err == nil { + return &ddbtypes.AttributeValueMemberN{Value: numberString(parsed)}, nil + } + return nil, g.Error("value %q is not a number for column %s (%s)", val, col.Name, col.Type) + } + } + return &ddbtypes.AttributeValueMemberS{Value: val}, nil + } + + return &ddbtypes.AttributeValueMemberS{Value: cast.ToString(raw)}, nil +} + +// normalizeDynamoDBFilterString removes the quoting that FormatValue applies to +// string values rendered for another connection. +func normalizeDynamoDBFilterString(val string) string { + if len(val) >= 2 && strings.HasPrefix(val, "'") && strings.HasSuffix(val, "'") { + return strings.ReplaceAll(val[1:len(val)-1], "''", "'") + } + return val +} + +var _ Connection = (*DynamoDBConn)(nil) diff --git a/core/dbio/database/database_test.go b/core/dbio/database/database_test.go index 964cb180f..ed7767e47 100755 --- a/core/dbio/database/database_test.go +++ b/core/dbio/database/database_test.go @@ -23,6 +23,7 @@ import ( "github.com/apache/arrow-go/v18/arrow/array" "github.com/apache/arrow-go/v18/arrow/ipc" "github.com/apache/arrow-go/v18/arrow/memory" + ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" zerobus "github.com/databricks/zerobus-sdk/go" "github.com/dustin/go-humanize" "github.com/flarco/g" @@ -2418,3 +2419,535 @@ func TestLanceDBConnSecretSkippedForOtherStores(t *testing.T) { assert.False(t, ok, path) } } + +// --------------------------------------------------------------------------- +// DynamoDB +// --------------------------------------------------------------------------- + +// sling renders a select into a JSON scan descriptor, but a table name or a +// `select ... from
` can reach the connector as well. +func TestDynamoDBScanRef(t *testing.T) { + cases := []struct { + name string + ref string + table string + descr map[string]any + errMatch string + }{ + { + name: "table name", + ref: "my_table", + table: "my_table", + }, + { + name: "schema qualified table name", + ref: "default.my_table", + table: "default.my_table", + }, + { + name: "scan descriptor", + ref: `{"table": "my_table", "filter": {"code": {"$gt": 5}}, "fields": ["id"], "limit": 10}`, + table: "my_table", + descr: map[string]any{ + "table": "my_table", + "filter": map[string]any{"code": map[string]any{"$gt": float64(5)}}, + "fields": []any{"id"}, + "limit": float64(10), + }, + }, + { + name: "scan descriptor with sql markers", + ref: `{"table": "my_table", "limit": 1} /* GetSQLColumns */ /* nD */`, + table: "my_table", + descr: map[string]any{"table": "my_table", "limit": float64(1)}, + }, + { + name: "scan descriptor without table", + ref: `{"limit": 10}`, + errMatch: "missing the table", + }, + { + name: "select with fields and limit", + ref: "select id, name from my_table limit 5", + table: "my_table", + descr: map[string]any{ + "table": "my_table", + "fields": []string{"id", "name"}, + "limit": 5, + }, + }, + { + name: "select star", + ref: "SELECT * FROM `my_table`;", + table: "my_table", + descr: map[string]any{"table": "my_table"}, + }, + { + name: "select with where", + ref: "select id from my_table where rating > 5", + errMatch: "`WHERE` cannot be applied", + }, + { + name: "select with order by", + ref: "select id from my_table order by id", + errMatch: "`ORDER BY` cannot be applied", + }, + { + name: "empty reference", + ref: " ", + errMatch: "no table specified", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + table, descr, err := dynamoDBScanRef(tc.ref) + if tc.errMatch != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errMatch) + return + } + + require.NoError(t, err) + assert.Equal(t, tc.table, table) + assert.Equal(t, tc.descr, descr) + }) + } +} + +func TestDynamoDBIsTableName(t *testing.T) { + for _, name := range []string{"my_table", "My.Table-1", `"my_table"`} { + assert.True(t, dynamoDBIsTableName(name), name) + } + for _, name := range []string{"", "1=1", "select 1", "my table", "{}"} { + assert.False(t, dynamoDBIsTableName(name), name) + } +} + +// The DDL sling generates is the only carrier of the key definition, so parsing +// it back out must handle single and composite keys. +func TestDynamoDBParseDDL(t *testing.T) { + ddl := "create table \"default\".\"my_table\" (\n \"id\" numeric,\n \"code\" bigint,\n \"name\" text,\n primary key (\"id\", \"code\")\n)" + + def, err := parseDynamoDBDDL(ddl) + require.NoError(t, err) + assert.Equal(t, "my_table", def.Name) + assert.Equal(t, []string{"id", "code"}, def.KeyColumns) + assert.Equal(t, ddbtypes.ScalarAttributeTypeN, def.KeyTypes["id"]) + assert.Equal(t, ddbtypes.ScalarAttributeTypeN, def.KeyTypes["code"]) + + // single key, and the key type follows the column type + def, err = parseDynamoDBDDL("create table my_table (id text, primary key (id))") + require.NoError(t, err) + assert.Equal(t, []string{"id"}, def.KeyColumns) + assert.Equal(t, ddbtypes.ScalarAttributeTypeS, def.KeyTypes["id"]) + + // no primary key at all + _, err = parseDynamoDBDDL("create table my_table (id numeric)") + require.Error(t, err) + assert.Contains(t, err.Error(), "requires a primary key") + + // more than two key columns cannot be a DynamoDB key + _, err = parseDynamoDBDDL("create table my_table (a text, b text, c text, primary key (a, b, c))") + require.Error(t, err) + assert.Contains(t, err.Error(), "at most 2 key columns") + + // a key column that is not part of the column list + _, err = parseDynamoDBDDL("create table my_table (a text, primary key (b))") + require.Error(t, err) + assert.Contains(t, err.Error(), "not part of the table definition") +} + +// `primary_key` reaches the connector as column metadata (sling does not set the +// key type on target columns), and DynamoDB tables cannot exist without a key. +func TestDynamoDBKeyColumns(t *testing.T) { + columns := iop.Columns{ + {Name: "id", Position: 1, Type: iop.BigIntType}, + {Name: "code", Position: 2, Type: iop.BigIntType}, + {Name: "email", Position: 3, Type: iop.StringType}, + } + + conn := &DynamoDBConn{} + + // explicit target keys win + table := Table{Name: "t", Dialect: dbio.TypeDbDynamoDB, Keys: TableKeys{iop.PrimaryKey: []string{"email"}}} + assert.Equal(t, []string{"email"}, conn.dynamoDBKeyColumns(table, columns)) + + // then the source primary key, as recorded by sling + sourced := columns.Clone() + require.NoError(t, sourced.SetMetadata(iop.PrimaryKey.MetadataKey(), "source", "id", "code")) + assert.Equal(t, []string{"id", "code"}, conn.dynamoDBKeyColumns(Table{Name: "t", Dialect: dbio.TypeDbDynamoDB}, sourced)) + + // then a key type set on the columns themselves + keyed := columns.Clone() + require.NoError(t, keyed.SetKeys(iop.PrimaryKey, "email")) + assert.Equal(t, []string{"email"}, conn.dynamoDBKeyColumns(Table{Name: "t", Dialect: dbio.TypeDbDynamoDB}, keyed)) + + // and nothing when no key is declared + assert.Empty(t, conn.dynamoDBKeyColumns(Table{Name: "t", Dialect: dbio.TypeDbDynamoDB}, columns)) +} + +// Values are stored with the type of their column: numbers as N (never as S), +// booleans as BOOL, and timestamps as ISO strings. +func TestDynamoDBFilterValueTypes(t *testing.T) { + cases := []struct { + name string + raw any + col *iop.Column + want ddbtypes.AttributeValue + errText string + }{ + {name: "json number", raw: float64(5), col: &iop.Column{Name: "code", Type: iop.BigIntType}, + want: &ddbtypes.AttributeValueMemberN{Value: "5"}}, + {name: "integer column", raw: "42", col: &iop.Column{Name: "code", Type: iop.BigIntType}, + want: &ddbtypes.AttributeValueMemberN{Value: "42"}}, + {name: "float column", raw: "89.983", col: &iop.Column{Name: "rating", Type: iop.FloatType}, + want: &ddbtypes.AttributeValueMemberN{Value: "89.983"}}, + {name: "quoted sql literal", raw: "'abc'", col: &iop.Column{Name: "name", Type: iop.StringType}, + want: &ddbtypes.AttributeValueMemberS{Value: "abc"}}, + {name: "boolean column", raw: "true", col: &iop.Column{Name: "target", Type: iop.BoolType}, + want: &ddbtypes.AttributeValueMemberBOOL{Value: true}}, + {name: "timestamp column", raw: "2019-08-19T17:02:09.000Z", col: &iop.Column{Name: "create_dt", Type: iop.TimestampType}, + want: &ddbtypes.AttributeValueMemberS{Value: "2019-08-19T17:02:09.000Z"}}, + {name: "not a number", raw: "abc", col: &iop.Column{Name: "code", Type: iop.BigIntType}, + errText: "is not a number"}, + {name: "null value", raw: nil, col: &iop.Column{Name: "code", Type: iop.BigIntType}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + av, err := dynamoDBFilterValue(tc.raw, tc.col) + if tc.errText != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errText) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, av) + }) + } +} + +func TestDynamoDBFilterExpression(t *testing.T) { + columns := iop.Columns{ + {Name: "id", Position: 1, Type: iop.BigIntType}, + {Name: "name", Position: 2, Type: iop.StringType}, + {Name: "code", Position: 3, Type: iop.BigIntType}, + } + + // one condition per column: expressions are built in map order, so each + // assertion stays on a single column + filter := newDynamoDBFilter() + require.NoError(t, filter.add(map[string]any{"code": map[string]any{"$gt": float64(5)}}, columns)) + assert.Equal(t, "#n0 > :v0", filter.expression()) + assert.Equal(t, "5", filter.values[":v0"].(*ddbtypes.AttributeValueMemberN).Value) + + // conditions on the same column share one attribute name alias + filter = newDynamoDBFilter() + require.NoError(t, filter.add(map[string]any{"code": map[string]any{"$gt": float64(5), "$lte": float64(10)}}, columns)) + assert.Equal(t, map[string]string{"#n0": "code"}, filter.names) + assert.Len(t, filter.values, 2) + assert.Contains(t, filter.expression(), "#n0 > ") + assert.Contains(t, filter.expression(), "#n0 <= ") + + filter = newDynamoDBFilter() + require.NoError(t, filter.add(map[string]any{"name": map[string]any{"$begins_with": "ab"}}, columns)) + assert.Equal(t, "begins_with(#n0, :v0)", filter.expression()) + assert.Equal(t, "ab", filter.values[":v0"].(*ddbtypes.AttributeValueMemberS).Value) + + filter = newDynamoDBFilter() + require.NoError(t, filter.add(map[string]any{"id": float64(3)}, columns)) + assert.Equal(t, "#n0 = :v0", filter.expression()) + + filter = newDynamoDBFilter() + require.NoError(t, filter.add(map[string]any{"code": map[string]any{"$in": []any{float64(1), float64(2)}}}, columns)) + assert.Equal(t, "#n0 IN (:v0, :v1)", filter.expression()) + + filter = newDynamoDBFilter() + require.NoError(t, filter.add(map[string]any{"code": map[string]any{"$between": []any{float64(1), float64(9)}}}, columns)) + assert.Equal(t, "#n0 BETWEEN :v0 AND :v1", filter.expression()) + + filter = newDynamoDBFilter() + require.NoError(t, filter.add(map[string]any{"code": map[string]any{"$exists": true}}, columns)) + assert.Equal(t, "attribute_exists(#n0)", filter.expression()) + + // multiple columns are ANDed + filter = newDynamoDBFilter() + require.NoError(t, filter.add(map[string]any{ + "id": float64(3), + "code": map[string]any{"$gt": float64(1)}, + }, columns)) + assert.Len(t, filter.names, 2) + assert.Contains(t, filter.expression(), " AND ") + + // unsupported operators are rejected instead of silently ignored + filter = newDynamoDBFilter() + err := filter.add(map[string]any{"code": map[string]any{"gt": float64(1)}}, columns) + require.Error(t, err) + assert.Contains(t, err.Error(), "unsupported filter operator gt") +} + +// Writing then reading an item must preserve the value and its type, since +// DynamoDB stores only the attribute types. +func TestDynamoDBRowRoundTrip(t *testing.T) { + columns := iop.Columns{ + {Name: "id", Position: 1, Type: iop.BigIntType}, + {Name: "rating", Position: 2, Type: iop.FloatType}, + {Name: "email", Position: 3, Type: iop.StringType}, + {Name: "target", Position: 4, Type: iop.BoolType}, + {Name: "create_dt", Position: 5, Type: iop.TimestampType}, + {Name: "tags", Position: 6, Type: iop.JsonType}, + {Name: "missing", Position: 7, Type: iop.StringType}, + } + + createDt := time.Date(2019, 8, 19, 17, 2, 9, 0, time.UTC) + row := []any{int64(2), 89.983, "tmee1@example.com", true, createDt, `{"a": [1, 2], "b": "c"}`, nil} + + conn := &DynamoDBConn{} + item, err := conn.rowToItem(columns, row, []string{"id"}) + require.NoError(t, err) + + assert.Equal(t, &ddbtypes.AttributeValueMemberN{Value: "2"}, item["id"]) + assert.Equal(t, &ddbtypes.AttributeValueMemberN{Value: "89.983"}, item["rating"]) + assert.Equal(t, &ddbtypes.AttributeValueMemberS{Value: "tmee1@example.com"}, item["email"]) + assert.Equal(t, &ddbtypes.AttributeValueMemberBOOL{Value: true}, item["target"]) + assert.Equal(t, &ddbtypes.AttributeValueMemberS{Value: "2019-08-19T17:02:09Z"}, item["create_dt"]) + // nulls are omitted, not written as NULL attributes + _, ok := item["missing"] + assert.False(t, ok) + // json is stored natively, not as a string + _, ok = item["tags"].(*ddbtypes.AttributeValueMemberM) + assert.True(t, ok, "expected a map attribute for json, got %T", item["tags"]) + + back, err := conn.itemToRow(item, columns) + require.NoError(t, err) + assert.Equal(t, int64(2), back[0]) + assert.Equal(t, 89.983, back[1]) + assert.Equal(t, "tmee1@example.com", back[2]) + assert.Equal(t, true, back[3]) + assert.Equal(t, createDt, back[4]) + assert.JSONEq(t, `{"a": [1, 2], "b": "c"}`, back[5].(string)) + assert.Nil(t, back[6]) + + // the key must be present on every item + _, err = conn.rowToItem(columns, []any{nil, 1.5, "x", false, createDt, nil, nil}, []string{"id"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "key attribute id is missing") +} + +// A table cannot exist without a key, and keying on a data column would collapse +// rows whose values repeat (writes are upserts), so sling adds one when the +// stream declares none. +func TestDynamoDBSyntheticKey(t *testing.T) { + conn, err := NewConn("dynamodb://us-east-1") + require.NoError(t, err) + dynamo := conn.(*DynamoDBConn) + + columns := iop.Columns{ + {Name: "id", Position: 1, Type: iop.BigIntType}, + {Name: "name", Position: 2, Type: iop.StringType}, + } + + // no key declared: a key column is added and becomes the primary key + data := columns.Dataset() + ddl, err := dynamo.GenerateDDL(Table{Name: "t", Dialect: dbio.TypeDbDynamoDB}, data, false) + require.NoError(t, err) + assert.Contains(t, ddl, `primary key (_sling_id)`) + assert.Contains(t, ddl, "_sling_id string") + + def, err := parseDynamoDBDDL(ddl) + require.NoError(t, err) + assert.Equal(t, []string{"_sling_id"}, def.KeyColumns) + + // a declared primary key is used as-is + declared := columns.Dataset() + require.NoError(t, declared.Columns.SetKeys(iop.PrimaryKey, "id")) + ddl, err = dynamo.GenerateDDL(Table{Name: "t", Dialect: dbio.TypeDbDynamoDB}, declared, false) + require.NoError(t, err) + assert.Contains(t, ddl, `primary key (id)`) + assert.NotContains(t, ddl, "_sling_id") + + // a declared unique key is the upsert identity of a key-value store + unique := columns.Dataset() + table := Table{Name: "t", Dialect: dbio.TypeDbDynamoDB, Keys: TableKeys{iop.UniqueKey: []string{"id"}}} + ddl, err = dynamo.GenerateDDL(table, unique, false) + require.NoError(t, err) + assert.Contains(t, ddl, `primary key (id)`) + + // the added column avoids the data's own column names + assert.Equal(t, "_sling_id", dynamoDBSyntheticKeyName(columns)) + assert.Equal(t, "_sling_id2", dynamoDBSyntheticKeyName(iop.Columns{{Name: "_sling_id"}})) + + // every row of a table keyed by the added column gets its own key value + row := []any{int64(1), "a"} + item, err := dynamo.rowToItem(columns, row, []string{"_sling_id"}) + require.NoError(t, err) + other, err := dynamo.rowToItem(columns, row, []string{"_sling_id"}) + require.NoError(t, err) + assert.NotEmpty(t, item["_sling_id"]) + assert.NotEqual(t, item["_sling_id"], other["_sling_id"]) +} + +// sling soft-deletes the rows missing from a stream with an update carrying a +// `not exists` subquery over the stream's keys. +func TestDynamoDBParseNotExistsJoin(t *testing.T) { + // as rendered by the `core.delete_where_not_exist` / `update_where_not_exist` + // templates: the `not exists` clause sits on its own indented lines + text := `where _sling_deleted_at is null + and not exists ( + select 1 from "default"."test1k_dynamodb_pg_temp_ids" + where "default"."test1k_dynamodb_pg".id = "default"."test1k_dynamodb_pg_temp_ids".id and "default"."test1k_dynamodb_pg".email = "default"."test1k_dynamodb_pg_temp_ids".email + )` + + join, err := parseDynamoDBNotExistsJoin(text) + require.NoError(t, err) + assert.Equal(t, "test1k_dynamodb_pg_temp_ids", join.Table) + assert.Equal(t, []string{"id", "email"}, join.Columns) + + keyJoin := join + + // a statement without the clause carries no join + join, err = parseDynamoDBNotExistsJoin("where _sling_deleted_at is null") + require.NoError(t, err) + assert.Empty(t, join.Table) + + // the remaining conditions stay parseable + filter, err := parseDynamoDBWhere(stripDynamoDBNotExistsJoin(text)) + require.NoError(t, err) + assert.Equal(t, map[string]any{"_sling_deleted_at": map[string]any{"$exists": false}}, filter) + + // the key signature of an item compares the joined columns only + item := map[string]ddbtypes.AttributeValue{ + "id": &ddbtypes.AttributeValueMemberN{Value: "1"}, + "email": &ddbtypes.AttributeValueMemberS{Value: "a@example.com"}, + "other": &ddbtypes.AttributeValueMemberS{Value: "ignored"}, + } + sameKey := map[string]ddbtypes.AttributeValue{ + "id": &ddbtypes.AttributeValueMemberN{Value: "1"}, + "email": &ddbtypes.AttributeValueMemberS{Value: "a@example.com"}, + } + other := map[string]ddbtypes.AttributeValue{ + "id": &ddbtypes.AttributeValueMemberN{Value: "2"}, + "email": &ddbtypes.AttributeValueMemberS{Value: "a@example.com"}, + } + assert.Equal(t, dynamoDBItemKey(item, keyJoin.Columns), dynamoDBItemKey(sameKey, keyJoin.Columns)) + assert.NotEqual(t, dynamoDBItemKey(item, keyJoin.Columns), dynamoDBItemKey(other, keyJoin.Columns)) +} + +func TestDynamoDBParseWhere(t *testing.T) { + cases := []struct { + name string + where string + want map[string]any + errText string + }{ + {name: "is null", where: `where "flag" is null`, + want: map[string]any{"flag": map[string]any{"$exists": false}}}, + {name: "is not null", where: "where flag is not null", + want: map[string]any{"flag": map[string]any{"$exists": true}}}, + {name: "equality", where: "where id = 5", want: map[string]any{"id": map[string]any{"$eq": float64(5)}}}, + {name: "quoted string", where: `where name = 'a b'`, want: map[string]any{"name": map[string]any{"$eq": "a b"}}}, + {name: "comparison", where: "where code >= 2.5", want: map[string]any{"code": map[string]any{"$gte": 2.5}}}, + {name: "not equal", where: `where op <> 'D'`, want: map[string]any{"op": map[string]any{"$ne": "D"}}}, + {name: "qualified column and parenthesis", where: `where ("t"."id" > 1)`, want: map[string]any{"id": map[string]any{"$gt": float64(1)}}}, + {name: "joined conditions", where: "where id > 1 and name = 'x'", + want: map[string]any{"id": map[string]any{"$gt": float64(1)}, "name": map[string]any{"$eq": "x"}}}, + {name: "empty", where: "", want: map[string]any{}}, + {name: "attribute name with dash", where: `where "first-name" is null`, + want: map[string]any{"first-name": map[string]any{"$exists": false}}}, + {name: "negative literal", where: "where code > -5", want: map[string]any{"code": map[string]any{"$gt": float64(-5)}}}, + {name: "tautology", where: "where 1=1", want: map[string]any{}}, + {name: "tautology with spaces", where: "where 1 = 1 and code > 2", + want: map[string]any{"code": map[string]any{"$gt": float64(2)}}}, + {name: "unsupported", where: "where lower(name) = 'x'", errText: "could not parse condition"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + filter, err := parseDynamoDBWhere(tc.where) + if tc.errText != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errText) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, filter) + }) + } +} + +func TestDynamoDBParseAssignments(t *testing.T) { + cases := []struct { + name string + set string + col string + want ddbtypes.AttributeValue + remove bool + isTime bool + errText string + }{ + {name: "now", set: "set _sling_deleted_at = current_timestamp", col: "_sling_deleted_at", isTime: true}, + {name: "quoted literal", set: "set op = 'D'", col: "op", want: &ddbtypes.AttributeValueMemberS{Value: "D"}}, + {name: "number literal", set: "set code = 5", col: "code", want: &ddbtypes.AttributeValueMemberN{Value: "5"}}, + {name: "bool literal", set: "set target = false", col: "target", want: &ddbtypes.AttributeValueMemberBOOL{Value: false}}, + {name: "null removes the attribute", set: "set deleted_at = null", col: "deleted_at", remove: true}, + {name: "unsupported expression", set: "set code = code + 1", errText: "unsupported value"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assignments, err := parseDynamoDBAssignments(tc.set[len("set "):]) + if tc.errText != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errText) + return + } + require.NoError(t, err) + require.Contains(t, assignments, tc.col) + + assignment := assignments[tc.col] + assert.Equal(t, tc.remove, assignment.remove) + if tc.remove { + return + } + if tc.isTime { + member, ok := assignment.value.(*ddbtypes.AttributeValueMemberS) + require.True(t, ok) + _, err := time.Parse(time.RFC3339Nano, member.Value) + require.NoError(t, err) + return + } + assert.Equal(t, tc.want, assignment.value) + }) + } +} + +// discover passes table patterns, not just names: `default.*` arrives as a +// wildcard while a bare schema arrives as an empty name. +func TestDynamoDBMatchTableNames(t *testing.T) { + assert.True(t, dynamoDBMatchTableNames("test1k_dynamodb", []string{"test1k_dynamodb"})) + assert.False(t, dynamoDBMatchTableNames("test1k_dynamodb", []string{"test1k_dynamodb_wide"})) + assert.True(t, dynamoDBMatchTableNames("test1k_dynamodb", []string{"*"})) + assert.True(t, dynamoDBMatchTableNames("test1k_dynamodb_wide", []string{"test1k_dynamodb_*"})) + assert.False(t, dynamoDBMatchTableNames("test1k_dynamodb_wide", []string{"test1k_dynamodb_v*"})) + assert.True(t, dynamoDBMatchTableNames("t1", []string{"other", "t?"})) +} + +// sling runs lifecycle statements through the read path too (a query hook +// cannot tell a statement from a select without a SQL engine), so they must be +// told apart from table names and scan descriptors. +func TestDynamoDBIsStatement(t *testing.T) { + assert.True(t, isDynamoDBStatement("drop table if exists my_table")) + assert.True(t, isDynamoDBStatement(" DROP TABLE my_table ")) + assert.True(t, isDynamoDBStatement("truncate table my_table")) + assert.True(t, isDynamoDBStatement(`create table "t" (id number, primary key (id))`)) + assert.True(t, isDynamoDBStatement("delete from t where 1=1")) + assert.True(t, isDynamoDBStatement("update t set a = 1")) + + assert.False(t, isDynamoDBStatement("my_table")) + assert.False(t, isDynamoDBStatement("default.my_table")) + assert.False(t, isDynamoDBStatement(`{"table": "my_table", "limit": 3}`)) + assert.False(t, isDynamoDBStatement("select * from my_table")) + assert.False(t, isDynamoDBStatement("select id, code from my_table limit 3")) + assert.False(t, isDynamoDBStatement("")) +} diff --git a/core/dbio/database/schemata.go b/core/dbio/database/schemata.go index 021dd1766..c2b490a06 100644 --- a/core/dbio/database/schemata.go +++ b/core/dbio/database/schemata.go @@ -286,6 +286,53 @@ func (t *Table) Select(Opts ...SelectOptions) (sql string) { t.SQL = g.F("%s\n--iceberg-json=%s", t.SQL, g.Marshal(m)) } return t.SQL + case dbio.TypeDbDynamoDB: + // DynamoDB has no SQL engine: the select is a JSON scan descriptor the + // connector executes. It always carries the table, so a bare table name + // and a `select ... from
` both resolve to a scan. + m, _ := g.UnmarshalMap(t.SQL) + if m == nil { + m = g.M() + } + + if t.Name != "" { + m["table"] = t.Name + } else if refName, refOpts, refErr := dynamoDBScanRef(t.SQL); refErr == nil && dynamoDBIsTableName(refName) { + m["table"] = refName + for key, val := range refOpts { + if _, ok := m[key]; !ok { + m[key] = val + } + } + } else if refErr != nil { + // hand the statement through untouched so the connector reports why + // it cannot be applied + return t.SQL + } + + if opts.Where != "" { + var where any + g.Unmarshal(opts.Where, &where) + m["filter"] = where // json object + } + + if len(fields) > 0 && fields[0] != "*" { + m["fields"] = lo.Map(fields, func(v string, i int) string { + return strings.TrimSpace(v) + }) + } + + // an explicit limit wins unless the statement carries its own + if opts.Limit != nil { + if _, ok := m["limit"]; !ok { + m["limit"] = opts.Limit + } + } + + if len(m) > 0 { + return g.Marshal(m) + } + return t.SQL case dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbAzureTable: m, _ := g.UnmarshalMap(t.SQL) if m == nil { diff --git a/core/dbio/database/schemata_keys.go b/core/dbio/database/schemata_keys.go index 28b05ea2b..0affccd1c 100644 --- a/core/dbio/database/schemata_keys.go +++ b/core/dbio/database/schemata_keys.go @@ -612,11 +612,12 @@ var indexCapabilities = map[dbio.Type]indexCapability{ dbio.TypeDbBigQuery: {noIndexes: true}, // StarRocks indexes (BITMAP/inverted) only apply to specific column/table // models and don't fit the generic CREATE INDEX form; no-op for now. - dbio.TypeDbStarRocks: {noIndexes: true}, + dbio.TypeDbStarRocks: {noIndexes: true}, dbio.TypeDbDuckDb: {supportsUnique: true}, dbio.TypeDbMotherDuck: {supportsUnique: true}, dbio.TypeDbDuckLake: {noIndexes: true}, // DuckLake does not support indexes dbio.TypeDbLanceDB: {noIndexes: true}, // Lance datasets have no secondary indexes + dbio.TypeDbDynamoDB: {noIndexes: true}, // DynamoDB keys are declared at table creation dbio.TypeDbOracle: {supportsUnique: true, supportsType: true, typeClosedSet: []string{"bitmap"}}, dbio.TypeDbSQLite: {supportsWhere: true, supportsUnique: true}, } diff --git a/core/dbio/dbio_types.go b/core/dbio/dbio_types.go index 03d464055..76e898737 100644 --- a/core/dbio/dbio_types.go +++ b/core/dbio/dbio_types.go @@ -92,6 +92,7 @@ const ( TypeDbArrowDBC Type = "adbc" TypeDbODBC Type = "odbc" TypeDbScyllaDB Type = "scylladb" + TypeDbDynamoDB Type = "dynamodb" ) var AllType = []struct { @@ -144,6 +145,7 @@ var AllType = []struct { {TypeDbArrowDBC, "TypeDbArrowDBC"}, {TypeDbODBC, "TypeDbODBC"}, {TypeDbScyllaDB, "TypeDbScyllaDB"}, + {TypeDbDynamoDB, "TypeDbDynamoDB"}, } // ValidateType returns true is type is valid @@ -165,7 +167,7 @@ func ValidateType(tStr string) (Type, bool) { case TypeApi, TypeFileLocal, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileDatabricksVolume, - TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: + TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbSnowflake, TypeDbDatabricks, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbAzureDWH, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbAzureTable, TypeDbFabric, TypeDbExasol, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbDynamoDB: return t, true } @@ -231,7 +233,7 @@ func (t Type) DBNameUpperCase() bool { func (t Type) Kind() Kind { switch t { case TypeDbPostgres, TypeDbRedshift, TypeDbStarRocks, TypeDbMySQL, TypeDbMariaDB, TypeDbOracle, TypeDbBigQuery, TypeDbBigTable, - TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB: + TypeDbSnowflake, TypeDbDatabricks, TypeDbExasol, TypeDbSQLite, TypeDbD1, TypeDbSQLServer, TypeDbAzure, TypeDbClickhouse, TypeDbTrino, TypeDbAthena, TypeDbIceberg, TypeDbLanceDB, TypeDbDuckDb, TypeDbDuckLake, TypeDbMotherDuck, TypeDbMongoDB, TypeDbElasticsearch, TypeDbPrometheus, TypeDbProton, TypeDbAzureTable, TypeDbFabric, TypeDbArrowDBC, TypeDbODBC, TypeDbScyllaDB, TypeDbDynamoDB: return KindDatabase case TypeFileLocal, TypeFileHDFS, TypeFileS3, TypeFileAzure, TypeFileAzureABFS, TypeFileGoogle, TypeFileGoogleDrive, TypeFileSftp, TypeFileFtp, TypeFileHTTP, TypeFileDatabricksVolume, Type("https"): return KindFile @@ -249,7 +251,7 @@ func (t Type) IsDb() bool { // IsDb returns true if database connection func (t Type) IsNoSQL() bool { switch t { - case TypeDbBigTable, TypeDbAzureTable, TypeDbMongoDB, TypeDbElasticsearch, TypeDbScyllaDB: + case TypeDbBigTable, TypeDbAzureTable, TypeDbMongoDB, TypeDbElasticsearch, TypeDbScyllaDB, TypeDbDynamoDB: return true } return false @@ -352,6 +354,7 @@ func (t Type) NameLong() string { TypeDbArrowDBC: "DB - Arrow DBC", TypeDbODBC: "DB - ODBC", TypeDbScyllaDB: "DB - ScyllaDB", + TypeDbDynamoDB: "DB - DynamoDB", } return mapping[t] @@ -405,6 +408,7 @@ func (t Type) Name() string { TypeDbAzureTable: "Azure Table", TypeDbArrowDBC: "Arrow DBC", TypeDbODBC: "ODBC", + TypeDbDynamoDB: "DynamoDB", } return mapping[t] diff --git a/core/dbio/dbio_types_test.go b/core/dbio/dbio_types_test.go index 230cf8148..eb6da4078 100644 --- a/core/dbio/dbio_types_test.go +++ b/core/dbio/dbio_types_test.go @@ -101,6 +101,7 @@ func TestExplainTemplatePresentForSQLDatabases(t *testing.T) { TypeDbAzureTable: true, TypeDbBigTable: true, TypeDbPrometheus: true, + TypeDbDynamoDB: true, } for _, td := range AllType { if !td.Value.IsDb() { diff --git a/core/dbio/templates/_properties.yaml b/core/dbio/templates/_properties.yaml index 005397c0f..a8384d470 100644 --- a/core/dbio/templates/_properties.yaml +++ b/core/dbio/templates/_properties.yaml @@ -940,6 +940,45 @@ azuretable: description: 'Specific table name for table-specific SAS tokens' examples: ['MyTable', 'LogData', 'UserData'] +dynamodb: + title: 'DynamoDB' + kind: database + required: ["region"] + url_template: 'dynamodb://{region}' + docs: https://docs.slingdata.io/connections/database-connections/dynamodb + properties: + name: + type: text + title: Connection Name + description: 'The name of the connection. No spaces are allowed, will be all caps. Example: DYNAMODB_MAIN' + examples: ['DYNAMODB_MAIN', 'DDB_PROD', 'DYNAMO_ANALYTICS'] + region: + type: text + description: 'AWS region of the tables' + examples: ['us-east-1', 'us-west-2', 'eu-west-1'] + access_key_id: + type: text + description: 'AWS access key ID (uses the AWS credential chain when omitted)' + examples: ['AKIAIOSFODNN7EXAMPLE'] + secret_access_key: + type: text + description: 'AWS secret access key' + examples: ['wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'] + secret: true + session_token: + type: text + description: 'AWS session token (for temporary credentials)' + examples: ['FwoGZXIvYXdzEBEaEXAMPLE'] + secret: true + profile: + type: text + description: 'AWS profile name from ~/.aws/credentials' + examples: ['default', 'prod'] + endpoint: + type: text + description: 'Custom endpoint, for example DynamoDB Local (`http://localhost:8000`)' + examples: ['http://localhost:8000'] + ftp: title: 'FTP' kind: file diff --git a/core/dbio/templates/connections.json b/core/dbio/templates/connections.json index 299aa3105..7fac90f7d 100644 --- a/core/dbio/templates/connections.json +++ b/core/dbio/templates/connections.json @@ -903,6 +903,38 @@ } } }, + "dynamodb": { + "name": "DynamoDB", + "description": "AWS DynamoDB key-value / document database", + "category": "database", + "properties": { + "region": { + "type": "string", + "description": "AWS region of the tables", + "required": true + }, + "access_key_id": { + "type": "string", + "description": "AWS access key ID (uses the AWS credential chain when omitted)" + }, + "secret_access_key": { + "type": "string", + "description": "AWS secret access key" + }, + "session_token": { + "type": "string", + "description": "AWS session token (for temporary credentials)" + }, + "profile": { + "type": "string", + "description": "AWS profile name from ~/.aws/credentials" + }, + "endpoint": { + "type": "string", + "description": "Custom endpoint, for example DynamoDB Local (http://localhost:8000)" + } + } + }, "azuresql": { "name": "Azure SQL Database", "description": "Azure SQL Database connection", diff --git a/core/dbio/templates/dynamodb.yaml b/core/dbio/templates/dynamodb.yaml new file mode 100644 index 000000000..960d33595 --- /dev/null +++ b/core/dbio/templates/dynamodb.yaml @@ -0,0 +1,59 @@ +# DynamoDB template. DynamoDB is a key-value store with no SQL engine, so this +# template carries only what sling renders around a run: the incremental and +# backfill conditions (which the connector turns into the scan's filter +# expression), the quoting and error-matching values, and the type maps. The +# `select` is rendered into a JSON scan descriptor that the connector executes +# through the AWS SDK. +core: + explain: "" + # DynamoDB has no views: sling renders `drop view` around target runs, so the + # template must render it away instead of emitting unsupported SQL + drop_view: "" + incremental_select: '{incremental_where_cond}' + incremental_where: '{ "update_key": "{update_key}", "value": "{value}" }' + backfill_where: '{ "update_key": "{update_key}", "start_value": "{start_value}", "end_value": "{end_value}" }' + +variable: + quote_char: '' + error_filter_table_exists: already + error_ignore_drop_table: not found + date_layout_str: '{value}' + date_layout: '2006-01-02' + timestamp_layout_str: '{value}' + timestamp_layout: '2006-01-02T15:04:05.000Z' + timestampz_layout_str: '{value}' + timestampz_layout: '2006-01-02T15:04:05.000Z' + +# DynamoDB attribute types (as reported by the connector) to general types +native_type_map: + BOOL: bool + B: binary + N: decimal + S: text + NULL: text + L: json + M: json + BS: json + NS: json + SS: json + +# general types to the DDL types the connector parses back out of a CREATE +# statement (only key columns carry real typing in DynamoDB) +general_type_map: + bigint: number + binary: binary + bool: bool + date: string + datetime: string + decimal: number + float: number + integer: number + json: json + smallint: number + string: string + text: string + time: string + timestamp: string + timestampz: string + timez: string + uuid: string diff --git a/core/sling/task_run_write.go b/core/sling/task_run_write.go index 48ad24044..019ab6b7e 100644 --- a/core/sling/task_run_write.go +++ b/core/sling/task_run_write.go @@ -207,9 +207,9 @@ func (t *TaskExecution) WriteToDb(cfg *Config, df *iop.Dataflow, tgtConn databas } // write directly for iceberg / NoSQL (no SQL temp-table merge) - writeDirectly := g.In(tgtConn.GetType(), dbio.TypeDbIceberg, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbAzureTable, dbio.TypeDbScyllaDB) + writeDirectly := g.In(tgtConn.GetType(), dbio.TypeDbIceberg, dbio.TypeDbMongoDB, dbio.TypeDbElasticsearch, dbio.TypeDbAzureTable, dbio.TypeDbScyllaDB, dbio.TypeDbDynamoDB) // INSERT is upsert-by-PK for these stores - upsertByInsert := g.In(tgtConn.GetType(), dbio.TypeDbScyllaDB, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable) + upsertByInsert := g.In(tgtConn.GetType(), dbio.TypeDbScyllaDB, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable, dbio.TypeDbDynamoDB) // Iceberg incremental+PK and CDC use MergeStream (row delta), not append and not SQL temp tables. if cfg.icebergNeedsMerge(tgtConn) { @@ -504,7 +504,7 @@ func (t *TaskExecution) WriteToDb(cfg *Config, df *iop.Dataflow, tgtConn databas func (t *TaskExecution) writeToDbDirectly(cfg *Config, df *iop.Dataflow, tgtConn database.Connection) (cnt uint64, err error) { // incremental+PK needs merge unless INSERT is upsert-by-PK - upsertByInsert := g.In(tgtConn.GetType(), dbio.TypeDbScyllaDB, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable) + upsertByInsert := g.In(tgtConn.GetType(), dbio.TypeDbScyllaDB, dbio.TypeDbMongoDB, dbio.TypeDbAzureTable, dbio.TypeDbDynamoDB) if g.In(cfg.Mode, IncrementalMode, BackfillMode) && len(cfg.Source.PrimaryKey()) > 0 && !upsertByInsert { return 0, g.Error("mode '%s' with a primary-key is not supported for direct write.", cfg.Mode) } diff --git a/go.mod b/go.mod index 0278c5632..6036f58fb 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 github.com/aws/aws-sdk-go-v2/service/athena v1.51.0 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.53.2 github.com/aws/aws-sdk-go-v2/service/glue v1.141.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 @@ -164,6 +165,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.14 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect diff --git a/go.sum b/go.sum index d09dee811..08bfa6e82 100644 --- a/go.sum +++ b/go.sum @@ -176,12 +176,16 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= github.com/aws/aws-sdk-go-v2/service/athena v1.51.0 h1:Fmh66wriOXgBJDnA/78aur8hH6DrvrWz7ZMzdoS33Yw= github.com/aws/aws-sdk-go-v2/service/athena v1.51.0/go.mod h1:xsG8Y2fMenmHTdukyknTUO1uQhEZ/entaNHvPmD1klE= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.53.2 h1:+/HEQj1fQGr17AQ0fAKpefDHw2hxQ3f0q96hY39J8Ao= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.53.2/go.mod h1:bz4cZH7uK5fLxQbj7hL4MFDL+pjReC9en/nM2Wfwxsk= github.com/aws/aws-sdk-go-v2/service/glue v1.141.0 h1:QeZdzcEB+eB76YvIHqZyxsmxH+f9f2kKDdz48UC4HCM= github.com/aws/aws-sdk-go-v2/service/glue v1.141.0/go.mod h1:FZCvt95CcJWRkZNRcmMW1uQkpDHmyUSkZfz3awyZgqg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.14 h1:3exo28cClRTVnxdj/LULxkESZSSv74RUIjZ7tfHXfWQ= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.11.14/go.mod h1:yLon9pByjyB6JZq5IAmwnjE3ObIhD0QibfRWH7tUhLU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= diff --git a/tests/pipelines/p.54.dynamodb.yaml b/tests/pipelines/p.54.dynamodb.yaml new file mode 100644 index 000000000..fccb4298c --- /dev/null +++ b/tests/pipelines/p.54.dynamodb.yaml @@ -0,0 +1,354 @@ +# DynamoDB end to end, self-contained: it seeds every source it needs. +# +# 1. local CSV -> DynamoDB full refresh, then the same file as an upsert +# 2. composite primary key, read back with a limit +# 3. DynamoDB -> postgres incremental on `code` (only the appended rows load) +# 4. delete_missing soft: the row missing from the stream stays, flagged +# 5. delete_missing hard: the row missing from the stream is deleted +# +# Needs a `DYNAMODB` connection (DynamoDB Local works: endpoint + region, no +# credentials) and a `POSTGRES` connection. DynamoDB tables are dropped at the +# end, so re-runs start clean. + +steps: + - log: "DynamoDB pipeline: full refresh, upsert, composite key, incremental, delete_missing" + + # drop leftovers from an interrupted run (the same tables the pipeline creates) + - type: group + loop: [dynamodb_pipe_test, dynamodb_pipe_composite, dynamodb_pipe_inc, dynamodb_pipe_delmiss_soft, dynamodb_pipe_delmiss_hard] + steps: + - connection: DYNAMODB + on_failure: warn + query: drop table if exists {loop.value} + + - type: group + loop: [public.dynamodb_pipe_test_pg, public.dynamodb_pipe_test_pg_tmp, public.dynamodb_pipe_composite_pg, public.dynamodb_pipe_composite_pg_tmp, public.dynamodb_pipe_inc_pg, public.dynamodb_pipe_inc_pg_tmp, public.dynamodb_pipe_delmiss_soft_pg, public.dynamodb_pipe_delmiss_soft_pg_tmp, public.dynamodb_pipe_delmiss_hard_pg, public.dynamodb_pipe_delmiss_hard_pg_tmp, public.dynamodb_pipe_delmiss_soft_src, public.dynamodb_pipe_delmiss_hard_src] + steps: + - connection: POSTGRES + on_failure: warn + query: drop table if exists {loop.value} + + # ---------- 1. full refresh, then upsert of the same file ---------- + - id: ddb_full_refresh + replication: + source: LOCAL + target: DYNAMODB + defaults: + mode: full-refresh + streams: + file://tests/files/test1.1.csv: + object: dynamodb_pipe_test + primary_key: [id] + on_failure: abort + + # INSERT is upsert-by-key for DynamoDB, so the same 18 rows must not duplicate + - replication: + source: LOCAL + target: DYNAMODB + defaults: + mode: incremental + streams: + file://tests/files/test1.1.csv: + object: dynamodb_pipe_test + primary_key: [id] + on_failure: abort + + - replication: + source: DYNAMODB + target: POSTGRES + defaults: + mode: full-refresh + streams: + default.dynamodb_pipe_test: + object: public.dynamodb_pipe_test_pg + on_failure: abort + + - connection: POSTGRES + query: select count(*) as cnt, count(distinct id) as ids from public.dynamodb_pipe_test_pg + into: upsert_counts + + - log: "upsert: {store.upsert_counts[0].cnt} rows, {store.upsert_counts[0].ids} distinct ids (expect 18, 18)" + + - check: 'int_parse(store.upsert_counts[0].cnt) == 18 && int_parse(store.upsert_counts[0].ids) == 18' + failure_message: "Expected 18 rows with 18 distinct ids after the upsert, got {store.upsert_counts[0].cnt}/{store.upsert_counts[0].ids}" + on_failure: abort + + # ---------- 2. composite primary key, read back with a limit ---------- + - replication: + source: LOCAL + target: DYNAMODB + defaults: + mode: full-refresh + streams: + file://tests/files/test1.1.csv: + object: dynamodb_pipe_composite + primary_key: [id, code] + on_failure: abort + + - replication: + source: DYNAMODB + target: POSTGRES + defaults: + mode: full-refresh + streams: + "select id, code from dynamodb_pipe_composite limit 3": + object: public.dynamodb_pipe_composite_pg + on_failure: abort + + - connection: POSTGRES + query: select count(*) as cnt from public.dynamodb_pipe_composite_pg + into: composite_counts + + - log: "composite key slice: {store.composite_counts[0].cnt} rows (expect 3)" + + - check: int_parse(store.composite_counts[0].cnt) == 3 + failure_message: "Expected 3 rows from the limited composite read, got {store.composite_counts[0].cnt}" + on_failure: abort + + # ---------- 3. incremental read, update_key on a numeric column ---------- + - replication: + source: LOCAL + target: DYNAMODB + defaults: + mode: full-refresh + primary_key: [id] + streams: + file://tests/files/test1.1.csv: + object: dynamodb_pipe_inc + on_failure: abort + + - id: inc_full + replication: + source: DYNAMODB + target: POSTGRES + defaults: + mode: full-refresh + primary_key: [id] + update_key: code + streams: + default.dynamodb_pipe_inc: + object: public.dynamodb_pipe_inc_pg + select: [id, code, email, rating] + on_failure: abort + + # two rows with code > 18 (the max of test1.1.csv) + - type: write + to: local//tmp/dynamodb_pipe_new_rows.csv + content: | + id,code,email,rating + 19,19,nineteen@example.com,50.5 + 20,20,twenty@example.com,60.5 + + - replication: + source: LOCAL + target: DYNAMODB + defaults: + mode: incremental + primary_key: [id] + streams: + file:///tmp/dynamodb_pipe_new_rows.csv: + object: dynamodb_pipe_inc + on_failure: abort + + - id: inc_reload + replication: + source: DYNAMODB + target: POSTGRES + defaults: + mode: incremental + primary_key: [id] + update_key: code + streams: + default.dynamodb_pipe_inc: + object: public.dynamodb_pipe_inc_pg + select: [id, code, email, rating] + on_failure: abort + + - connection: POSTGRES + query: | + select count(*) as cnt, + count(distinct id) as ids, + max(code) as max_code + from public.dynamodb_pipe_inc_pg + into: inc_counts + + - log: "incremental: {store.inc_counts[0].cnt} rows, {store.inc_counts[0].ids} distinct ids, max code {store.inc_counts[0].max_code} (expect 20, 20, 20)" + + - check: 'int_parse(store.inc_counts[0].cnt) == 20 && int_parse(store.inc_counts[0].ids) == 20' + failure_message: "Expected 20 rows with 20 distinct ids after the incremental load, got {store.inc_counts[0].cnt}/{store.inc_counts[0].ids}" + on_failure: abort + + # the reload must read only the appended rows, not re-pull all 18 + - check: int_parse(state.inc_reload.rows) == 2 + failure_message: "The incremental reload must read only the 2 appended rows, got {state.inc_reload.rows}" + on_failure: abort + + # ---------- 4. delete_missing soft ---------- + - connection: POSTGRES + on_failure: warn + query: drop table if exists public.dynamodb_pipe_delmiss_soft_src + + - connection: POSTGRES + query: | + create table public.dynamodb_pipe_delmiss_soft_src ( + id bigint, create_dt timestamp, name text + ); + insert into public.dynamodb_pipe_delmiss_soft_src values + (1, '2019-01-01 00:00:00', 'alice'), + (2, '2019-01-01 00:00:00', 'bob'), + (3, '2019-01-01 00:00:00', 'carol'); + + - replication: + source: POSTGRES + target: DYNAMODB + defaults: + mode: full-refresh + primary_key: [id] + update_key: create_dt + streams: + public.dynamodb_pipe_delmiss_soft_src: + object: dynamodb_pipe_delmiss_soft + on_failure: abort + + # the source loses row 3; the survivors move to a newer create_dt + - connection: POSTGRES + query: | + truncate public.dynamodb_pipe_delmiss_soft_src; + insert into public.dynamodb_pipe_delmiss_soft_src values + (1, '2020-01-01 00:00:00', 'alice'), + (2, '2020-01-01 00:00:00', 'bob'); + + - replication: + source: POSTGRES + target: DYNAMODB + defaults: + mode: incremental + primary_key: [id] + update_key: create_dt + target_options: + delete_missing: soft + streams: + public.dynamodb_pipe_delmiss_soft_src: + object: dynamodb_pipe_delmiss_soft + on_failure: abort + + - replication: + source: DYNAMODB + target: POSTGRES + defaults: + mode: full-refresh + streams: + default.dynamodb_pipe_delmiss_soft: + object: public.dynamodb_pipe_delmiss_soft_pg + on_failure: abort + + - connection: POSTGRES + query: | + select count(*) as total, + count(_sling_deleted_at) as flagged, + count(case when id = 3 and _sling_deleted_at is not null then 1 end) as flagged_row, + count(case when id = 1 and _sling_deleted_at is null then 1 end) as kept_row + from public.dynamodb_pipe_delmiss_soft_pg + into: soft_counts + + - log: "soft delete: {store.soft_counts[0].total} rows, {store.soft_counts[0].flagged} flagged, row 3 flagged {store.soft_counts[0].flagged_row}" + + - check: int_parse(store.soft_counts[0].total) == 3 + failure_message: "Soft delete must keep all 3 rows, got {store.soft_counts[0].total}" + on_failure: abort + + - check: 'int_parse(store.soft_counts[0].flagged) == 1 && int_parse(store.soft_counts[0].flagged_row) == 1' + failure_message: "Only row 3 (missing from the stream) must be flagged, got {store.soft_counts[0].flagged} flagged rows" + on_failure: abort + + - check: int_parse(store.soft_counts[0].kept_row) == 1 + failure_message: "Row 1 must stay unflagged after a soft delete" + on_failure: abort + + # ---------- 5. delete_missing hard ---------- + - connection: POSTGRES + on_failure: warn + query: drop table if exists public.dynamodb_pipe_delmiss_hard_src + + - connection: POSTGRES + query: | + create table public.dynamodb_pipe_delmiss_hard_src ( + id bigint, create_dt timestamp, name text + ); + insert into public.dynamodb_pipe_delmiss_hard_src values + (1, '2019-01-01 00:00:00', 'alice'), + (2, '2019-01-01 00:00:00', 'bob'), + (3, '2019-01-01 00:00:00', 'carol'); + + - replication: + source: POSTGRES + target: DYNAMODB + defaults: + mode: full-refresh + primary_key: [id] + update_key: create_dt + streams: + public.dynamodb_pipe_delmiss_hard_src: + object: dynamodb_pipe_delmiss_hard + on_failure: abort + + - connection: POSTGRES + query: | + truncate public.dynamodb_pipe_delmiss_hard_src; + insert into public.dynamodb_pipe_delmiss_hard_src values + (1, '2020-01-01 00:00:00', 'alice'), + (2, '2020-01-01 00:00:00', 'bob'); + + - replication: + source: POSTGRES + target: DYNAMODB + defaults: + mode: incremental + primary_key: [id] + update_key: create_dt + target_options: + delete_missing: hard + streams: + public.dynamodb_pipe_delmiss_hard_src: + object: dynamodb_pipe_delmiss_hard + on_failure: abort + + - replication: + source: DYNAMODB + target: POSTGRES + defaults: + mode: full-refresh + streams: + default.dynamodb_pipe_delmiss_hard: + object: public.dynamodb_pipe_delmiss_hard_pg + on_failure: abort + + - connection: POSTGRES + query: | + select count(*) as total, + count(case when id = 3 then 1 end) as stale + from public.dynamodb_pipe_delmiss_hard_pg + into: hard_counts + + - log: "hard delete: {store.hard_counts[0].total} rows, row 3 present {store.hard_counts[0].stale}" + + - check: 'int_parse(store.hard_counts[0].total) == 2 && int_parse(store.hard_counts[0].stale) == 0' + failure_message: "Hard delete must leave 2 rows without row 3, got {store.hard_counts[0].total} rows with {store.hard_counts[0].stale} stale" + on_failure: abort + + - log: "SUCCESS: DynamoDB full refresh, upsert, composite key, incremental and delete_missing all verified" + + # ---------- cleanup ---------- + - type: group + loop: [dynamodb_pipe_test, dynamodb_pipe_composite, dynamodb_pipe_inc, dynamodb_pipe_delmiss_soft, dynamodb_pipe_delmiss_hard] + steps: + - connection: DYNAMODB + on_failure: warn + query: drop table if exists {loop.value} + + - type: group + loop: [public.dynamodb_pipe_test_pg, public.dynamodb_pipe_test_pg_tmp, public.dynamodb_pipe_composite_pg, public.dynamodb_pipe_composite_pg_tmp, public.dynamodb_pipe_inc_pg, public.dynamodb_pipe_inc_pg_tmp, public.dynamodb_pipe_delmiss_soft_pg, public.dynamodb_pipe_delmiss_soft_pg_tmp, public.dynamodb_pipe_delmiss_hard_pg, public.dynamodb_pipe_delmiss_hard_pg_tmp, public.dynamodb_pipe_delmiss_soft_src, public.dynamodb_pipe_delmiss_hard_src] + steps: + - connection: POSTGRES + on_failure: warn + query: drop table if exists {loop.value} diff --git a/tests/suite.cli.yaml b/tests/suite.cli.yaml index 2af6e4db3..a9a60f5d6 100644 --- a/tests/suite.cli.yaml +++ b/tests/suite.cli.yaml @@ -3056,6 +3056,24 @@ - '5,Eve' - execution succeeded +# DynamoDB: needs a DynamoDB Local endpoint. The CI step starts one and sets the +# DYNAMODB connection (type dynamodb, endpoint, region and dummy credentials). +# The pipeline seeds its own sources and asserts every scenario with checks: +# full refresh, upsert, composite key, limited read, incremental, and +# delete_missing soft/hard. +- id: 618 + name: 'Run sling dynamodb pipeline' + group: dynamodb + run: sling run -p tests/pipelines/p.54.dynamodb.yaml + output_contains: + - 'upsert: 18 rows, 18 distinct ids (expect 18, 18)' + - 'composite key slice: 3 rows (expect 3)' + - 'incremental: 20 rows, 20 distinct ids, max code 20 (expect 20, 20, 20)' + - 'soft delete: 3 rows, 1 flagged, row 3 flagged 1' + - 'hard delete: 2 rows, row 3 present 0' + - 'SUCCESS: DynamoDB full refresh, upsert, composite key, incremental and delete_missing all verified' + - execution succeeded + # Nested CLI suites. Loader in sling_cli_test.go merges these in. - suite: suite.cli.assist.yaml - suite: suite.cli.build.yaml From 31b326e3d8e74ec687828b5bd9d0c24332cf6070 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Tue, 22 Sep 2026 09:41:33 -0300 Subject: [PATCH 31/56] Add read-only dBase (.dbf) database connector - New `dbase` connection type (aliases `dbf`, `foxpro`) that reads a single `.dbf` file or a directory of them as tables of the `main` schema, with a native Go reader instead of SQL. - dBase has no query engine, so only the select statements sling generates for a table read (field list, limit/offset) are accepted; filters and custom SQL are rejected with an explicit error. - Values come from go-dbase, with the raw record inspected to report blank numbers/dates/logicals as NULL and to resolve the variable length varchar/varbinary markers the library misreads. - Add fixtures and tests for dBase III/IV and FoxPro files, plus a CLI suite case and connector docs. --- README.md | 2 +- cmd/sling/resource/llm_CONNECTION.md | 6 +- cmd/sling/resource/llm_CONNECTION_DATABASE.md | 2 +- core/dbio/connection/connection.go | 13 +- core/dbio/database/database.go | 2 + core/dbio/database/database_dbase.go | 1248 +++++++++++++++++ core/dbio/database/database_test.go | 355 +++++ core/dbio/database/test/dbf/README.md | 13 + core/dbio/database/test/dbf/TEST.DBF | Bin 0 -> 1936 bytes core/dbio/database/test/dbf/TEST.FPT | Bin 0 -> 603 bytes core/dbio/database/test/dbf/dbase_03.dbf | Bin 0 -> 9286 bytes core/dbio/database/test/dbf/dbase_8b.dbf | Bin 0 -> 1826 bytes core/dbio/database/test/dbf/dbase_8b.dbt | Bin 0 -> 5120 bytes .../database/test/dbf/expense categories.dbf | Bin 0 -> 688 bytes core/dbio/database/test/dbf/nullable.dbf | Bin 0 -> 710 bytes core/dbio/database/test/dbf/test1k_dbase.dbf | Bin 0 -> 170616 bytes core/dbio/dbio_types.go | 10 +- core/dbio/dbio_types_test.go | 5 + core/dbio/templates/dbase.yaml | 54 + go.mod | 19 +- go.sum | 20 + tests/suite.cli.yaml | 19 + 22 files changed, 1751 insertions(+), 17 deletions(-) create mode 100644 core/dbio/database/database_dbase.go create mode 100644 core/dbio/database/test/dbf/README.md create mode 100644 core/dbio/database/test/dbf/TEST.DBF create mode 100644 core/dbio/database/test/dbf/TEST.FPT create mode 100644 core/dbio/database/test/dbf/dbase_03.dbf create mode 100644 core/dbio/database/test/dbf/dbase_8b.dbf create mode 100644 core/dbio/database/test/dbf/dbase_8b.dbt create mode 100644 core/dbio/database/test/dbf/expense categories.dbf create mode 100644 core/dbio/database/test/dbf/nullable.dbf create mode 100644 core/dbio/database/test/dbf/test1k_dbase.dbf create mode 100644 core/dbio/templates/dbase.yaml diff --git a/README.md b/README.md index a795dd894..7a31c6487 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Example [Replication](https://docs.slingdata.io/sling-cli/run/configuration/repl --- Available Connectors: -- **Databases**: [`adbc`](https://docs.slingdata.io/connections/database-connections/adbc) [`azuredwh`](https://docs.slingdata.io/connections/database-connections/azuredwh) [`azuresql`](https://docs.slingdata.io/connections/database-connections/azuresql) [`azuretable`](https://docs.slingdata.io/connections/database-connections/azuretable) [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`d1`](https://docs.slingdata.io/connections/database-connections/d1) [`databricks`](https://docs.slingdata.io/connections/database-connections/databricks) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`elasticsearch`](https://docs.slingdata.io/connections/database-connections/elasticsearch) [`exasol`](https://docs.slingdata.io/connections/database-connections/exasol) [`fabric`](https://docs.slingdata.io/connections/database-connections/fabric) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`mongodb`](https://docs.slingdata.io/connections/database-connections/mongodb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`odbc`](https://docs.slingdata.io/connections/database-connections/odbc) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`scylladb`](https://docs.slingdata.io/connections/database-connections/scylladb) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`trino`](https://docs.slingdata.io/connections/database-connections/trino) +- **Databases**: [`adbc`](https://docs.slingdata.io/connections/database-connections/adbc) [`azuredwh`](https://docs.slingdata.io/connections/database-connections/azuredwh) [`azuresql`](https://docs.slingdata.io/connections/database-connections/azuresql) [`azuretable`](https://docs.slingdata.io/connections/database-connections/azuretable) [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`d1`](https://docs.slingdata.io/connections/database-connections/d1) [`databricks`](https://docs.slingdata.io/connections/database-connections/databricks) [`dbase`](https://docs.slingdata.io/connections/database-connections/dbase) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`elasticsearch`](https://docs.slingdata.io/connections/database-connections/elasticsearch) [`exasol`](https://docs.slingdata.io/connections/database-connections/exasol) [`fabric`](https://docs.slingdata.io/connections/database-connections/fabric) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`mongodb`](https://docs.slingdata.io/connections/database-connections/mongodb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`odbc`](https://docs.slingdata.io/connections/database-connections/odbc) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`scylladb`](https://docs.slingdata.io/connections/database-connections/scylladb) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`trino`](https://docs.slingdata.io/connections/database-connections/trino) - **Data Lakes**:[`athena`](https://docs.slingdata.io/connections/datalake-connections/athena) [`Ducklake`](https://docs.slingdata.io/connections/datalake-connections/ducklake) [`iceberg`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`S3 Tables`](https://docs.slingdata.io/connections/datalake-connections/iceberg) [`lancedb`](https://docs.slingdata.io/connections/datalake-connections/lancedb) diff --git a/cmd/sling/resource/llm_CONNECTION.md b/cmd/sling/resource/llm_CONNECTION.md index 8727708b7..44c7367e1 100644 --- a/cmd/sling/resource/llm_CONNECTION.md +++ b/cmd/sling/resource/llm_CONNECTION.md @@ -108,8 +108,8 @@ For database operations (queries, schema exploration) and file system operations Sling supports 42+ different connection types across four categories: -### Database Connections (24 types) -- **Relational**: PostgreSQL, MySQL, MariaDB, SQLServer, Oracle, SQLite +### Database Connections (25 types) +- **Relational**: PostgreSQL, MySQL, MariaDB, SQLServer, Oracle, SQLite, dBase - **Cloud Warehouses**: Snowflake, BigQuery, Redshift, Databricks - **Analytics**: ClickHouse, DuckDB, MotherDuck, StarRocks, Trino, Proton - **NoSQL**: MongoDB, ElasticSearch, Prometheus @@ -161,6 +161,8 @@ Navigate or fetch the content of the connector from the below respective URL to **SQLite (`sqlite`)** -> https://docs.slingdata.io/connections/database-connections/sqlite +**dBase (`dbase`)** -> https://docs.slingdata.io/connections/database-connections/dbase + **MotherDuck (`motherduck`)** -> https://docs.slingdata.io/connections/database-connections/motherduck **ElasticSearch (`elasticsearch`)** -> https://docs.slingdata.io/connections/database-connections/elasticsearch diff --git a/cmd/sling/resource/llm_CONNECTION_DATABASE.md b/cmd/sling/resource/llm_CONNECTION_DATABASE.md index e363c038a..ef57b35e8 100644 --- a/cmd/sling/resource/llm_CONNECTION_DATABASE.md +++ b/cmd/sling/resource/llm_CONNECTION_DATABASE.md @@ -30,7 +30,7 @@ Database operations in Sling provide comprehensive functionality for interacting ### Supported Database Types -- **Relational**: PostgreSQL, MySQL, MariaDB, SQL Server, Oracle, SQLite +- **Relational**: PostgreSQL, MySQL, MariaDB, SQL Server, Oracle, SQLite, dBase - **Cloud Warehouses**: Snowflake, BigQuery, Redshift, Databricks - **Analytics**: ClickHouse, DuckDB, MotherDuck, StarRocks, Trino, Proton - **NoSQL**: MongoDB, ElasticSearch, Prometheus diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index 13ce13cac..75011e6f3 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -314,7 +314,7 @@ func (c *Connection) URL() string { } switch c.Type { - case dbio.TypeDbDuckDb: + case dbio.TypeDbDuckDb, dbio.TypeDbDBase: // fix windows path url = strings.ReplaceAll(url, `\`, `/`) } @@ -599,7 +599,7 @@ func (c *Connection) setURL() (err error) { pathValue := strings.ReplaceAll(U.Path(), "/", "") setIfMissing("schema", U.PopParam("schema")) - if !g.In(c.Type, dbio.TypeDbMotherDuck, dbio.TypeDbDuckDb, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB, dbio.TypeDbSQLite, dbio.TypeDbD1, dbio.TypeDbBigQuery) { + if !g.In(c.Type, dbio.TypeDbMotherDuck, dbio.TypeDbDuckDb, dbio.TypeDbDuckLake, dbio.TypeDbLanceDB, dbio.TypeDbSQLite, dbio.TypeDbDBase, dbio.TypeDbD1, dbio.TypeDbBigQuery) { setIfMissing("host", U.Hostname()) setIfMissing("user", U.Username()) setIfMissing("username", U.Username()) @@ -624,6 +624,9 @@ func (c *Connection) setURL() (err error) { case dbio.TypeDbSQLite, dbio.TypeDbDuckDb: setIfMissing("instance", U.Path()) setIfMissing("schema", "main") + case dbio.TypeDbDBase: + setIfMissing("path", database.DbasePathFromURL(c.URL())) + setIfMissing("schema", "main") case dbio.TypeDbLanceDB: setIfMissing("path", lanceDBPathFromURL(c.URL())) setIfMissing("schema", "main") @@ -903,6 +906,12 @@ func (c *Connection) setURL() (err error) { } } template = "sqlite://{instance}?cache=shared&mode=rwc&_journal_mode=WAL&_synchronous=NORMAL" + case dbio.TypeDbDBase: + if val, ok := c.Data["path"]; ok { + c.Data["path"] = strings.ReplaceAll(cast.ToString(val), `\`, `/`) // windows path fix + } + setIfMissing("schema", "main") + template = "dbase://{path}" case dbio.TypeDbDuckDb: if val, ok := c.Data["instance"]; ok { dbURL, err := net.NewURL(cast.ToString(val)) diff --git a/core/dbio/database/database.go b/core/dbio/database/database.go index 5156a5456..925e9cbc6 100755 --- a/core/dbio/database/database.go +++ b/core/dbio/database/database.go @@ -314,6 +314,8 @@ func NewConnContext(ctx context.Context, URL string, props ...string) (Connectio conn = &D1Conn{URL: URL} } else if strings.HasPrefix(URL, "sqlite:") { conn = &SQLiteConn{URL: URL} + } else if strings.HasPrefix(URL, "dbase:") || strings.HasPrefix(URL, "dbf:") { + conn = &DbaseConn{URL: URL} } else if strings.HasPrefix(URL, "duckdb:") || strings.HasPrefix(URL, "motherduck:") { conn = &DuckDbConn{URL: URL} } else if strings.HasPrefix(URL, "ducklake:") { diff --git a/core/dbio/database/database_dbase.go b/core/dbio/database/database_dbase.go new file mode 100644 index 000000000..b46db1402 --- /dev/null +++ b/core/dbio/database/database_dbase.go @@ -0,0 +1,1248 @@ +package database + +import ( + "bytes" + "context" + "database/sql" + "encoding/binary" + "io" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/flarco/g" + "github.com/samber/lo" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/spf13/cast" + "github.com/valentin-kaiser/go-dbase/dbase" +) + +// DbaseConn reads dBase / FoxPro tables (`.dbf` files). +// +// The connection root is a `.dbf` file (a single table) or a directory holding +// `.dbf` files, where every file is a table of the `main` schema. dBase has no +// query engine, so the connector is read-only and answers the select statements +// sling generates for a table read: a field list, plus `limit` / `offset`. +// Filters and custom SQL are rejected with an explicit error. +// +// Values are read through the reader library, with the raw record inspected +// where the library cannot report the value as stored: +// +// - a blank number, date or logical is stored as spaces (a blank datetime as +// NULs), which the library reports as a zero value; such a field is NULL +// - a variable length varchar / varbinary field stores the length of its +// value in the last byte of the field, marked in the record's null flag +// field; the library resolves those markers from the first varchar field +// of the table for every field, so they are resolved here instead +type DbaseConn struct { + BaseConn + + URL string + Path string // root: a .dbf file, or a directory of .dbf files +} + +const dbaseDefaultSchema = "main" + +// dbfTextNullTypes are the dBase types whose blank value is stored as text +// (spaces) or NULs, and which the reader library turns into a zero value. +var dbfTextNullTypes = []dbase.DataType{ + dbase.Numeric, dbase.Float, dbase.Date, dbase.DateTime, dbase.Logical, +} + +// Init initiates the object +func (conn *DbaseConn) Init() error { + conn.Path = strings.TrimSpace(conn.GetProp("path")) + if conn.Path == "" { + conn.Path = strings.TrimSpace(conn.GetProp("instance")) + } + if conn.Path == "" { + conn.Path = DbasePathFromURL(conn.URL) + } + if conn.Path == "" { + return g.Error("did not provide 'path' for dBase connection (a .dbf file or a directory of .dbf files)") + } + conn.SetProp("path", conn.Path) + + if conn.GetProp("schema") == "" { + conn.SetProp("schema", dbaseDefaultSchema) + } + + conn.BaseConn.URL = conn.URL + conn.BaseConn.Type = dbio.TypeDbDBase + + instance := Connection(conn) + conn.BaseConn.instance = &instance + + return conn.BaseConn.Init() +} + +// DbasePathFromURL returns the path of a dBase connection URL. The whole part +// after the scheme is the path, so that both `dbase:///data/tables` and +// `dbase://./tables` resolve as written. +func DbasePathFromURL(connURL string) string { + path := strings.TrimSpace(connURL) + for _, scheme := range []string{"dbase://", "dbf://"} { + if strings.HasPrefix(strings.ToLower(path), scheme) { + path = path[len(scheme):] + if unescaped, err := url.PathUnescape(path); err == nil { + path = unescaped + } + return path + } + } + return path +} + +// Connect validates the connection root +func (conn *DbaseConn) Connect(timeOut ...int) (err error) { + if _, err := os.Stat(conn.Path); err != nil { + return g.Error(err, "could not access dBase path: %s", conn.Path) + } + + conn.SetProp("connected", "true") + conn.SetProp("connect_time", cast.ToString(time.Now())) + return nil +} + +// GetURL returns the processed URL +func (conn *DbaseConn) GetURL(newURL ...string) string { + if len(newURL) > 0 { + return newURL[0] + } + return conn.BaseConn.URL +} + +// ExecContext is not supported: a dBase file cannot be written by sling +func (conn *DbaseConn) ExecContext(ctx context.Context, query string, args ...interface{}) (result sql.Result, err error) { + return nil, g.Error("dBase connections are read-only, cannot execute: %s", strings.TrimSpace(query)) +} + +// tableFiles maps every table of the connection to its file path, keyed by the +// table name (the file name without its extension) +func (conn *DbaseConn) tableFiles() (files map[string]string, err error) { + files = map[string]string{} + + info, err := os.Stat(conn.Path) + if err != nil { + return nil, g.Error(err, "could not access dBase path: %s", conn.Path) + } + + if !info.IsDir() { + name := strings.TrimSuffix(filepath.Base(conn.Path), filepath.Ext(conn.Path)) + files[name] = conn.Path + return files, nil + } + + entries, err := os.ReadDir(conn.Path) + if err != nil { + return nil, g.Error(err, "could not read dBase directory: %s", conn.Path) + } + + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".dbf") { + continue + } + name := strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())) + files[name] = filepath.Join(conn.Path, entry.Name()) + } + + return files, nil +} + +// tableFile resolves a (possibly qualified) table name to its file path. Table +// names are matched without case, as the file system of the data source may be +// case sensitive. +func (conn *DbaseConn) tableFile(tableName string) (filePath string, err error) { + name := dbaseTableName(tableName) + + files, err := conn.tableFiles() + if err != nil { + return "", err + } + + if filePath, ok := files[name]; ok { + return filePath, nil + } + + for fileName, path := range files { + if strings.EqualFold(fileName, name) { + return path, nil + } + } + + available := sortedTableNames(files) + if len(available) == 0 { + return "", g.Error("dbf table not found: %s (no .dbf files under %s)", tableName, conn.Path) + } + return "", g.Error("dbf table not found: %s (available: %s)", tableName, strings.Join(available, ", ")) +} + +// dbaseTableName returns the table name of a table reference, which may be +// quoted and qualified (`"main"."customers"`, `main.customers` or `customers`) +func dbaseTableName(tableName string) string { + parts := strings.Split(strings.TrimSpace(tableName), ".") + return strings.Trim(strings.TrimSpace(parts[len(parts)-1]), `"`) +} + +// openTable opens the table file for reading +func (conn *DbaseConn) openTable(tableName string) (file *dbase.File, filePath string, err error) { + filePath, err = conn.tableFile(tableName) + if err != nil { + return nil, "", err + } + + cfg := &dbase.Config{ + Filename: filePath, + ReadOnly: true, + // dBase III/IV and FoxBase files are common and are not on the reader + // library's tested list; the header and column definitions are still + // validated when the table is opened. + Untested: true, + TrimSpaces: conn.trimSpaces(), + } + + if codePage := strings.TrimSpace(conn.GetProp("code_page")); codePage != "" { + mark, err := strconv.ParseUint(strings.TrimPrefix(strings.ToLower(codePage), "0x"), lo.Ternary(strings.HasPrefix(strings.ToLower(codePage), "0x"), 16, 10), 8) + if err != nil { + return nil, "", g.Error(err, "invalid code_page: %s (expected a code page mark, e.g. 0x03)", codePage) + } + cfg.Converter = dbase.ConverterFromCodePage(byte(mark)) + } + + file, err = dbase.OpenTable(cfg) + if err != nil { + return nil, "", g.Error(err, "could not open dbf file: %s", filePath) + } + return file, filePath, nil +} + +// checkDbfMemo returns an error when a table has a memo field whose memo file +// cannot be read. The reader only resolves a FoxPro `.fpt` file, so a dBase +// III/IV table with a `.dbt` memo would otherwise fail row by row with an +// opaque error. +func checkDbfMemo(file *dbase.File, filePath string) error { + hasMemo := false + for _, column := range file.Columns() { + if dbase.DataType(column.DataType) == dbase.Memo { + hasMemo = true + break + } + } + + if !hasMemo { + return nil + } + + if _, related := file.GetHandle(); related != nil { + return nil + } + + memoFile := strings.TrimSuffix(filePath, filepath.Ext(filePath)) + ".dbt" + if _, err := os.Stat(memoFile); err != nil { + memoFile = strings.TrimSuffix(filePath, filepath.Ext(filePath)) + ".DBT" + } + if _, err := os.Stat(memoFile); err == nil { + return g.Error("dbf table %s has a memo field, but its memo file is not a FoxPro `.fpt` file: %s is not supported by the reader", filepath.Base(filePath), memoFile) + } + + return g.Error("dbf table %s has a memo field, but no readable memo file (.fpt) was found for it", filepath.Base(filePath)) +} + +// trimSpaces returns whether string values should be trimmed (default true) +func (conn *DbaseConn) trimSpaces() bool { + if val := conn.GetProp("trim_spaces"); val != "" { + return cast.ToBool(val) + } + return true +} + +// makeColumns converts the table's column definitions into sling columns. The +// general type comes from the connection template's `native_type_map`, so it +// can be overridden like any other connection's type mapping. +func (conn *DbaseConn) makeColumns(file *dbase.File, tableName string) iop.Columns { + columns := make(iop.Columns, 0, len(file.Columns())) + names := map[string]bool{} + for i, column := range file.Columns() { + dataType := dbase.DataType(column.DataType) + + col := iop.Column{ + Position: i + 1, + Name: uniqueDbfColumnName(names, column.Name()), + Type: iop.NativeTypeToGeneral(column.Name(), dbfTypeName(dataType), conn.GetType()), + DbType: dbfNativeType(column), + Sourced: true, + Table: tableName, + Schema: conn.GetProp("schema"), + Database: conn.GetProp("schema"), + } + + // a dBase numeric is an exact decimal, and the template maps the bare + // `numeric` name to a whole number + if dataType == dbase.Numeric { + col.DbPrecision = int(column.Length) + col.DbScale = int(column.Decimals) + if column.Decimals > 0 { + col.Type = iop.DecimalType + } + } + + columns = append(columns, col) + } + return columns +} + +// uniqueDbfColumnName returns a name that is not yet used, appending a number to +// repeated ones (`Point_ID`, `Point_ID1`, ...). A dBase table can hold columns +// with the same name, which sling cannot tell apart, so they are made unique as +// the file readers do with repeated headers. +func uniqueDbfColumnName(used map[string]bool, name string) string { + for i := 0; ; i++ { + candidate := lo.Ternary(i == 0, name, g.F("%s%d", name, i)) + if !used[strings.ToLower(candidate)] { + used[strings.ToLower(candidate)] = true + return candidate + } + } +} + +// dbfTypeName returns the dBase type name of a column +func dbfTypeName(dataType dbase.DataType) string { + switch dataType { + case dbase.Character: + return "character" + case dbase.Varchar: + return "varchar" + case dbase.Memo: + return "memo" + case dbase.Numeric: + return "numeric" + case dbase.Float: + return "float" + case dbase.Currency: + return "currency" + case dbase.Double: + return "double" + case dbase.Integer: + return "integer" + case dbase.Date: + return "date" + case dbase.DateTime: + return "datetime" + case dbase.Logical: + return "logical" + case dbase.Blob: + return "blob" + case dbase.General: + return "general" + case dbase.Picture: + return "picture" + case dbase.Varbinary: + return "varbinary" + } + return "unknown" +} + +// dbfNativeType returns the dBase type of a column with its length, as stored +// in the table definition (e.g. `character(20)`, `numeric(16,6)`) +func dbfNativeType(column *dbase.Column) string { + switch dbase.DataType(column.DataType) { + case dbase.Numeric, dbase.Float: + if column.Decimals > 0 { + return g.F("%s(%d,%d)", dbfTypeName(dbase.DataType(column.DataType)), column.Length, column.Decimals) + } + case dbase.Character, dbase.Varchar, dbase.Memo, dbase.Blob, dbase.General, dbase.Picture, dbase.Varbinary: + return g.F("%s(%d)", dbfTypeName(dbase.DataType(column.DataType)), column.Length) + } + return dbfTypeName(dbase.DataType(column.DataType)) +} + +// GetTableColumns returns the columns of a table, read from the table header +func (conn *DbaseConn) GetTableColumns(table *Table, fields ...string) (columns iop.Columns, err error) { + file, _, err := conn.openTable(table.Name) + if err != nil { + return columns, err + } + defer file.Close() + + allColumns := conn.makeColumns(file, table.Name) + if len(fields) == 0 { + return allColumns, nil + } + + columns = make(iop.Columns, 0, len(fields)) + for _, field := range fields { + col := allColumns.GetColumn(field) + if col == nil { + return nil, g.Error("provided field '%s' not found in table %s", field, table.FullName()) + } + col.Position = len(columns) + 1 + columns = append(columns, *col) + } + + if len(columns) == 0 { + return columns, g.Error("did not find any columns for %s", table.FullName()) + } + return columns, nil +} + +// GetSQLColumns returns the columns of a statement, resolved from the table +// definition rather than by executing the statement +func (conn *DbaseConn) GetSQLColumns(table Table) (columns iop.Columns, err error) { + if !table.IsQuery() { + return conn.GetTableColumns(&table) + } + + sel, err := parseDbaseSelect(table.SQL) + if err != nil { + return columns, err + } + + file, _, err := conn.openTable(sel.table) + if err != nil { + return columns, err + } + defer file.Close() + + columns, _, err = selectDbaseColumns(conn.makeColumns(file, dbaseTableName(sel.table)), sel.fields) + return columns, err +} + +// GetCount returns the number of records of a table, excluding deleted ones +func (conn *DbaseConn) GetCount(tableFName string) (int64, error) { + file, _, err := conn.openTable(tableFName) + if err != nil { + return 0, err + } + defer file.Close() + + // the record count in the header includes deleted records, which are not + // part of the table + var count int64 + for !file.EOF() { + deleted, err := file.Deleted() + if err != nil { + return count, g.Error(err, "could not read deleted flag of record %d", file.Pointer()) + } else if !deleted { + count++ + } + file.Skip(1) + } + + return count, nil +} + +// TableExists returns whether the table file exists +func (conn *DbaseConn) TableExists(table Table) (exists bool, err error) { + _, err = conn.tableFile(table.Name) + if err != nil { + if strings.Contains(err.Error(), "table not found") { + return false, nil + } + return false, err + } + return true, nil +} + +// GetSchemas returns the schemas of the connection +func (conn *DbaseConn) GetSchemas() (data iop.Dataset, err error) { + data = iop.NewDataset(iop.NewColumnsFromFields("schema_name")) + data.Append([]interface{}{conn.GetProp("schema")}) + return data, nil +} + +// GetTables returns the tables of a schema +func (conn *DbaseConn) GetTables(schema string) (data iop.Dataset, err error) { + data = iop.NewDataset(iop.NewColumnsFromFields("schema_name", "table_name", "is_view")) + + files, err := conn.tableFiles() + if err != nil { + return data, err + } + + schema = lo.Ternary(schema == "", conn.GetProp("schema"), schema) + for _, name := range sortedTableNames(files) { + data.Append([]interface{}{schema, name, false}) + } + return data, nil +} + +// GetViews returns no views: dBase tables have no views +func (conn *DbaseConn) GetViews(schema string) (data iop.Dataset, err error) { + data = iop.NewDataset(iop.NewColumnsFromFields("schema_name", "table_name", "is_view")) + return data, nil +} + +// GetPrimaryKeys returns no keys: dBase tables expose no key metadata +func (conn *DbaseConn) GetPrimaryKeys(tableFName string) (data iop.Dataset, err error) { + data = iop.NewDataset(iop.NewColumnsFromFields("pk_name", "position", "column_name")) + return data, nil +} + +// GetIndexes returns no indexes: index files (.cdx/.idx) are not readable +func (conn *DbaseConn) GetIndexes(tableFName string) (data iop.Dataset, err error) { + data = iop.NewDataset(iop.NewColumnsFromFields("table_name", "column_name")) + return data, nil +} + +// CurrentDatabase returns the database name +func (conn *DbaseConn) CurrentDatabase() (dbName string, err error) { + return conn.GetProp("schema"), nil +} + +// CurrentSchema returns the schema name +func (conn *DbaseConn) CurrentSchema() (schemaName string, err error) { + return conn.GetProp("schema"), nil +} + +// GetDatabases returns the databases of the connection +func (conn *DbaseConn) GetDatabases() (data iop.Dataset, err error) { + data = iop.NewDataset(iop.NewColumnsFromFields("name")) + data.Append([]interface{}{conn.GetProp("schema")}) + return data, nil +} + +// GetSchemata obtains the schemata of the connection +func (conn *DbaseConn) GetSchemata(level SchemataLevel, schemaName string, tableNames ...string) (Schemata, error) { + schemata := Schemata{ + Databases: map[string]Database{}, + conn: conn, + } + + files, err := conn.tableFiles() + if err != nil { + return schemata, err + } + + schemaName = lo.Ternary(schemaName == "", conn.GetProp("schema"), schemaName) + schema := Schema{ + Name: schemaName, + Tables: map[string]Table{}, + } + + filters := lo.Filter(tableNames, func(name string, _ int) bool { return strings.TrimSpace(name) != "" }) + + for _, tableName := range sortedTableNames(files) { + if len(filters) > 0 && !g.IsMatched(filters, tableName) { + continue + } + + table := Table{ + Name: tableName, + Schema: schemaName, + Database: schemaName, + Dialect: conn.GetType(), + } + + if level == SchemataLevelColumn { + columns, err := conn.GetTableColumns(&table) + if err != nil { + return schemata, err + } + table.Columns = columns + } + + schema.Tables[strings.ToLower(tableName)] = table + } + + schemata.Databases[strings.ToLower(schemaName)] = Database{ + Name: schemaName, + Schemas: map[string]Schema{strings.ToLower(schemaName): schema}, + } + + return schemata, nil +} + +// StreamRowsContext streams the rows of a statement. Only the select form sling +// generates for a table read is supported: a field list, plus limit / offset. +func (conn *DbaseConn) StreamRowsContext(ctx context.Context, query string, options ...map[string]interface{}) (ds *iop.Datastream, err error) { + opts := getQueryOptions(options) + + sel, err := parseDbaseSelect(query) + if err != nil { + return ds, err + } + + if limit := cast.ToInt(opts["limit"]); limit > 0 && (sel.limit == 0 || limit < sel.limit) { + sel.limit = limit + } + + file, filePath, err := conn.openTable(sel.table) + if err != nil { + return ds, err + } + + if err := checkDbfMemo(file, filePath); err != nil { + file.Close() + return ds, err + } + + columns, colIndexes, err := selectDbaseColumns(conn.makeColumns(file, dbaseTableName(sel.table)), sel.fields) + if err != nil { + file.Close() + return ds, err + } + + queryContext := g.NewContext(ctx) + nextFunc, closeFunc := conn.newRowIterator(file, filePath, len(columns), colIndexes, sel) + + ds = iop.NewDatastreamIt(queryContext.Ctx, columns, nextFunc) + ds.Defer(closeFunc) + ds.NoDebug = strings.Contains(query, noDebugKey) + ds.Inferred = !InferDBStream && ds.Columns.Sourced() + ds.Metadata.StreamURL.Value = filePath + conn.LogSQL(query) + if !ds.NoDebug { + // don't set metadata for internal queries + ds.SetMetadata(conn.GetProp("METADATA")) + ds.SetConfig(conn.Props()) + } + + err = ds.Start() + if err != nil { + queryContext.Cancel() + return ds, g.Error(err, "could start datastream") + } + return ds, nil +} + +// newRowIterator returns the row provider for a table read, along with the +// cleanup function that releases the table file +func (conn *DbaseConn) newRowIterator(file *dbase.File, filePath string, colCount int, colIndexes []int, sel dbaseSelect) (nextFunc func(it *iop.Iterator) bool, closeFunc func()) { + fields := file.Columns() + offsets := dbfFieldOffsets(fields) + nullFlags := newDbfNullFlags(filePath, fields, conn.trimSpaces()) + header := file.Header() + + // a blank number / date / logical can only be told from a real zero value + // by looking at the raw record, as can the variable length of a varchar. + // The raw record is read through the same file handle the reader library + // uses: os.File.ReadAt does not move that handle's offset. + var rawFile *os.File + if handle, _ := file.GetHandle(); handle != nil { + rawFile, _ = handle.(*os.File) + } + + rawBuf := make([]byte, header.RowLength) + limit := sel.limit + offset := sel.offset + noRows := sel.noRows + + nextFunc = func(it *iop.Iterator) bool { + for !file.EOF() { + record, err := file.Next() + if err != nil { + it.Context.CaptureErr(g.Error(err, "could not read row %d of %s", file.Pointer(), file.TableName())) + return false + } else if record.Deleted { + continue // deleted records are not part of the table + } else if offset > 0 { + offset-- + continue + } else if noRows || (limit > 0 && it.Counter >= uint64(limit)) { + return false + } + + raw := readDbfRecord(rawFile, rawBuf, header, record.Position) + recordFields := record.Fields() + + row := make([]any, colCount) + for i, colIndex := range colIndexes { + if colIndex >= len(recordFields) { + continue + } + field := recordFields[colIndex] + if value, ok := nullFlags.variableValue(raw, field, offsets[colIndex], colIndex); ok { + row[i] = value + } else { + row[i] = dbfValue(field, raw, offsets[colIndex]) + } + } + it.Row = row + return true + } + return false + } + + closeFunc = func() { file.Close() } + + return nextFunc, closeFunc +} + +// readDbfRecord returns the raw bytes of a record, or nil when they are not +// available. The buffer is reused across rows. +func readDbfRecord(rawFile *os.File, buf []byte, header *dbase.Header, position uint32) []byte { + if rawFile == nil || len(buf) == 0 { + return nil + } + + offset := int64(header.FirstRow) + int64(position)*int64(header.RowLength) + n, err := rawFile.ReadAt(buf, offset) + if err != nil && err != io.EOF { + return nil + } else if n != len(buf) { + return nil + } + return buf +} + +// dbfFieldBits are the `_NullFlags` bit positions of a varchar / varbinary +// field: the variable length marker and the null marker +type dbfFieldBits struct { + varlen int // -1 when the field holds no variable length marker + null int // -1 when the field holds no null marker +} + +// dbfNullFlags resolves the `_NullFlags` bits of a table's records. The reader +// resolves those bits from the first varchar / varbinary field of the table for +// every field, so a table holding more than one of them reads wrong +// variable-length markers; the bits are resolved here instead, from the flag +// bytes of the record. +type dbfNullFlags struct { + offset int // byte offset of the `_NullFlags` field within a record + length int // length of the `_NullFlags` field in bytes + trimSpaces bool // whether string values are trimmed + bits []dbfFieldBits // indexed by field position +} + +// newDbfNullFlags returns the `_NullFlags` layout of a table, or nil when the +// table holds no flag field +func newDbfNullFlags(filePath string, columns []*dbase.Column, trimSpaces bool) *dbfNullFlags { + offset, length, ok := dbfNullFlagField(filePath) + if !ok { + return nil + } + + flags := &dbfNullFlags{ + offset: offset, + length: length, + trimSpaces: trimSpaces, + bits: make([]dbfFieldBits, len(columns)), + } + + // a variable length marker per varchar / varbinary field, plus a null marker + // for the nullable ones, in the order of the table definition + bit := 0 + for i, column := range columns { + flags.bits[i] = dbfFieldBits{varlen: -1, null: -1} + if !g.In(dbase.DataType(column.DataType), dbase.Varchar, dbase.Varbinary) { + continue + } + flags.bits[i].varlen = bit + bit++ + if column.Flag.Has(byte(dbase.NullableFlag)) { + flags.bits[i].null = bit + bit++ + } + } + + return flags +} + +// dbfNullFlagField returns the byte offset and length of the `_NullFlags` field +// of a table, as recorded in its column definitions +func dbfNullFlagField(filePath string) (offset int, length int, ok bool) { + rawFile, err := os.Open(filePath) + if err != nil { + return 0, 0, false + } + defer rawFile.Close() + + header := make([]byte, 32) + if _, err := rawFile.ReadAt(header, 0); err != nil { + return 0, 0, false + } + + firstRow := int(binary.LittleEndian.Uint16(header[8:10])) + descriptor := make([]byte, 32) + for pos := 32; pos+len(descriptor) <= firstRow; pos += len(descriptor) { + if _, err := rawFile.ReadAt(descriptor, int64(pos)); err != nil { + return 0, 0, false + } else if descriptor[0] == byte(dbase.ColumnEnd) { + break + } + + name := string(bytes.TrimRight(descriptor[:11], "\x00")) + if !strings.EqualFold(name, "_NullFlags") { + continue + } + return int(binary.LittleEndian.Uint32(descriptor[12:16])), int(descriptor[16]), true + } + + return 0, 0, false +} + +// bit returns the value of a `_NullFlags` bit of a record +func (flags *dbfNullFlags) bit(raw []byte, index int) (value bool, ok bool) { + if flags == nil || index < 0 || index >= flags.length*8 { + return false, false + } else if flags.offset+flags.length > len(raw) { + return false, false + } + + return raw[flags.offset+index/8]&(1<<(index%8)) != 0, true +} + +// variableValue returns the value of a variable length varchar / varbinary +// field, as recorded by the `_NullFlags` bits of the record. The second return +// value reports whether the field is a flagged one, and so whether the value of +// the reader library is to be replaced. +func (flags *dbfNullFlags) variableValue(raw []byte, field *dbase.Field, offset int, colIndex int) (value any, ok bool) { + if flags == nil || raw == nil { + return nil, false + } + + column := field.Column() + if !g.In(dbase.DataType(column.DataType), dbase.Varchar, dbase.Varbinary) { + return nil, false + } + + length := int(column.Length) + if offset < 0 || offset+length > len(raw) { + return nil, false + } + + if colIndex < 0 || colIndex >= len(flags.bits) { + return nil, false + } + + bits := flags.bits[colIndex] + if bits.varlen < 0 { + return nil, false + } + + if null, _ := flags.bit(raw, bits.null); null { + return nil, true + } + + if varlen, _ := flags.bit(raw, bits.varlen); !varlen { + return nil, false // a fixed length value is read correctly + } + + // the last byte of the field holds the length of the value + if size := int(raw[offset+length-1]); size <= length { + raw = raw[offset : offset+size] + } else { + raw = raw[offset : offset+length] + } + + if !flags.trimSpaces { + raw = bytes.ReplaceAll(raw, []byte{0x00}, []byte{}) + } else { + raw = bytes.TrimSpace(bytes.ReplaceAll(raw, []byte{0x00}, []byte{})) + } + + if dbase.DataType(column.DataType) == dbase.Varbinary { + return raw, true + } + return string(raw), true +} + +// dbfValue returns the value of one column of a record, mapping a blank field +// to NULL +func dbfValue(field *dbase.Field, raw []byte, offset int) any { + if field == nil { + return nil + } + + value := field.GetValue() + + if g.In(field.Type(), dbfTextNullTypes...) { + // a blank number / date / logical is stored as spaces (datetime as + // NULs), which the reader library reports as a zero value + length := int(field.Column().Length) + if raw != nil && offset >= 0 && offset+length <= len(raw) && dbfBlank(raw[offset:offset+length], field.Type()) { + return nil + } + return value + } + + // an empty memo / varchar / varbinary / blob is reported as an empty slice + if bytes, ok := value.([]byte); ok && len(bytes) == 0 { + return nil + } + + return value +} + +// dbfFieldOffsets returns the byte offset of every column within a record. The +// reader library parses the fields sequentially after the delete flag and does +// not fill in the column displacement, so it is computed here. +func dbfFieldOffsets(columns []*dbase.Column) (offsets []int) { + offsets = make([]int, len(columns)) + offset := 1 // the delete flag + for i, column := range columns { + offsets[i] = offset + offset += int(column.Length) + } + return +} + +// dbfBlank returns whether the raw bytes of a field hold no value +func dbfBlank(raw []byte, dataType dbase.DataType) bool { + for _, b := range raw { + if b == ' ' || b == 0x00 { + continue + } else if dataType == dbase.Logical && b == '?' { + continue // '?' marks an undetermined logical + } + return false + } + return true +} + +// dbaseSelect is the parsed form of a supported statement +type dbaseSelect struct { + fields []dbaseField + table string + limit int + offset int + noRows bool // `where 1=0`: columns only, no rows +} + +// dbaseField is a selected column, with an optional alias +type dbaseField struct { + name string + alias string +} + +// dbaseClauseKeywords are the keywords that can follow the table reference +var dbaseClauseKeywords = []string{"where", "order by", "group by", "having", "limit", "offset", "union", "join"} + +// parseDbaseSelect parses the statements the connector accepts: a select of +// plain columns from one table, with limit / offset. dBase has no query engine, +// so anything else is rejected instead of being silently ignored. +func parseDbaseSelect(sql string) (sel dbaseSelect, err error) { + statement := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(sql), ";")) + + if !hasKeywordPrefix(statement, "select") { + return sel, g.Error("dBase connections only support `select from
`, got: %s", strings.TrimSpace(sql)) + } + + remainder := strings.TrimSpace(statement[len("select"):]) + + fromIndex := findDbaseKeyword(remainder, "from") + if fromIndex < 0 { + return sel, g.Error("dBase connections only support `select from
`, got: %s", strings.TrimSpace(sql)) + } + + sel.fields, err = parseDbaseFields(remainder[:fromIndex]) + if err != nil { + return sel, err + } + + remainder = strings.TrimSpace(remainder[fromIndex+len("from"):]) + + // the table reference ends at the first clause keyword + tableEnd := len(remainder) + for _, keyword := range dbaseClauseKeywords { + if index := findDbaseKeyword(remainder, keyword); index >= 0 && index < tableEnd { + tableEnd = index + } + } + + tableRef := strings.TrimSpace(remainder[:tableEnd]) + clauses := strings.TrimSpace(remainder[tableEnd:]) + + if err = parseDbaseClauses(clauses, &sel, statement); err != nil { + return sel, err + } + + if !strings.HasPrefix(tableRef, "(") { + sel.table = strings.Trim(tableRef, `"`) + if sel.table == "" || strings.ContainsAny(sel.table, "(),") { + return sel, g.Error("dBase connections only support reading one table, got: %s", strings.TrimSpace(sql)) + } + return sel, nil + } + + // a derived table over a single table read: `select * from (