diff --git a/.github/postgres/Dockerfile b/.github/postgres/Dockerfile new file mode 100644 index 0000000..e822923 --- /dev/null +++ b/.github/postgres/Dockerfile @@ -0,0 +1,5 @@ +# PostgreSQL test image with the PostGIS and pgvector extensions (used to test the extension data types). +FROM postgres:17 +RUN apt-get update \ + && apt-get install -y --no-install-recommends postgresql-17-postgis-3 postgresql-17-pgvector \ + && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 31220eb..1356d40 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -12,10 +12,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v7 - name: Setup .NET - uses: actions/setup-dotnet@v1 + uses: actions/setup-dotnet@v6 with: dotnet-version: | 8.0.x @@ -28,26 +28,8 @@ jobs: - name: Build run: dotnet build --no-restore - - name: Pull SqlServer - run: docker pull mcr.microsoft.com/mssql/server - - - name: Start SqlServer - run: docker run -e "ACCEPT_EULA=Y" -e "SA_PASSWORD=yourStrong(!)Password" --name sqlserver -p 1433:1433 -d mcr.microsoft.com/mssql/server:2019-latest - - - name: Pull MySQL - run: docker pull mysql - - - name: Start MySQL - run: docker run --name db-mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=yourStrong#!Password -d mysql - - - name: Pull Postgres - run: docker pull postgres - - - name: Start Postgres - run: docker run --name db-postgres -p 5432:5432 -e POSTGRES_PASSWORD=yourStrong#!Password -d postgres - - - name: Sleep (allow databases to complete startup) - run: sleep 10 + - name: Start databases (SQL Server, MySQL, Postgres) + run: docker compose up -d --build --wait --wait-timeout 180 - name: Set EnvVar for Test run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index f52bdd1..3a034f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ Represents the **NuGet** versions. +## v3.3.0 +- *Enhancement:* Non-primitive database types are now mapped to their correct .NET types (unqualified; a `global using` is expected) instead of throwing `Database data type 'x' does not have corresponding .NET type mapping defined`, which previously failed the `Data`, `Reset` and `Inspect` commands for the whole database. Originated from PR #72 by Wesley Teixeira. + - SQL Server: `geography`/`geometry` (`Geometry`), `hierarchyid` (`HierarchyId`), `vector` (`SqlVector`), `xml`/`json` (`string`) and `sql_variant` (`object`). + - Postgres: network, text-search, geometric, bit-string, range/multirange, array, `hstore`, `ltree`, `citext`, enum, PostGIS and pgvector types. + - MySql: `mediumint`, `year`, `bit(n)`, `vector` and spatial types (a spatial value with an `SRID=n;` prefix is interpreted as longitude then latitude, consistent with Postgres and SQL Server). +- *Enhancement:* Added `DbColumnSchema.DataParserType` (the primitive type the data parser uses to parse YAML/JSON values; `DotNetType` remains the true EF type), `IsDotNetTypeAClass` (generated EF models now emit `= default!` for non-nullable class types), `IsDataComparable`, `UdtName` and `NativeSqlType`. The provider `DatabaseSchemaConfig` has corresponding virtual methods. +- *Fixed:* SQL Server `BINARY(n)` and `VECTOR(n)` columns are now formatted with their length/dimensions. +- *Fixed:* SQL Server data merge no longer compares non-comparable columns (e.g. `xml`, `geography`) using `EXCEPT`. +- *Note:* Test databases are now SQL Server 2025, MySQL 9 and Postgres 17 with PostGIS and pgvector (see `.github/postgres/Dockerfile`). + ## v3.2.0 - *Enhancement:* The `MigrationCommand.Execute` has been enhanced to support the execution of YAML/JSON data files. Raw SQL must _now_ be prefixed by `>` to explicitly differentiate from a file path. - *Enhancement:* The `MigrationCommand.Inspect` has been enhanced to also report whether each column has been identified as a JSON column (i.e. `DbColumnSchema.IsJson`). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e3102b5..6ee03d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,6 +51,8 @@ We use [`NUnit`](https://github.com/nunit/nunit) for all unit testing. We understand there is more work to be performed in generating a higher level of code coverage; this technical debt is on the backlog. +The tests are executed against live SQL Server, MySQL and Postgres (with PostGIS and pgvector) databases. These can be started using the [`docker-compose.yml`](./docker-compose.yml) (`docker compose up -d --build`); the `DbEx_ConnectionStrings__*` environment variables, as set within the [CI](./.github/workflows/CI.yml) workflow, must then be configured to reference them. +
## Code reviews and checkins diff --git a/Common.targets b/Common.targets index edd22f3..2e9deb9 100644 --- a/Common.targets +++ b/Common.targets @@ -1,6 +1,6 @@ - 3.2.0 + 3.3.0 preview Avanade Avanade diff --git a/README.md b/README.md index 3fde8fb..2e887b9 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,31 @@ Within a code-generation, or other context, the database schema may need to be i The [`Database`](./src/DbEx/DatabaseExtensions.cs) class provides a `SelectSchemaAsync` method to return a [`DbTableSchema`](./src/DbEx/DbSchema/DbTableSchema.cs) list, including the respective columns for each table (see [`DbColumnSchema`](./src/DbEx/DbSchema/DbColumnSchema.cs)). +### Data types + +Each [`DbColumnSchema`](./src/DbEx/DbSchema/DbColumnSchema.cs) exposes the inferred .NET type, and how data is parsed: + +Property | Description +-|- +`DotNetType` | The .NET type a column maps to, e.g. `string`, `int`, `Geometry` or `NpgsqlRange`. Types are _not_ namespace-qualified; the consuming project is expected to have the required `global using` statements in place for the types used (e.g. NetTopologySuite, Npgsql and Pgvector types). +`IsDotNetTypeAClass` | Indicates whether `DotNetType` is a reference type (e.g. a non-nullable property that needs initializing, such as `= default!`). Value types, including structs such as `NpgsqlCidr`, return `false`. +`DataParserType` | The primitive .NET type used to parse a YAML/JSON data value for the column. For a type that has no primitive representation, e.g. spatial, vector and network types, this is `string` and the value is specified using its text form (see below); the provider then converts to the database type. +`NativeSqlType` | The full native database type (including precision, length and dimensions) where available, e.g. `vector(3)`. +`UdtName` | The underlying user-defined/extension type name where applicable (Postgres). +`IsDataComparable` | Indicates whether the database can compare values (e.g. SQL Server `xml` and `geography` cannot). Where not, the SQL Server `$` merge updates matched rows without the changed-data check. + +Non-primitive types are supported by each provider, including (non-exhaustive): + +Provider | Types +-|- +SQL Server | `geography` and `geometry` (`Geometry`), `hierarchyid` (`HierarchyId`), `vector` (`SqlVector`), `xml` and `json` (`string`), `sql_variant` (`object`). +Postgres | network (`inet`, `cidr`, `macaddr`), text search, geometric, bit-string, range and multirange, array, `hstore`, `ltree`, `citext`, enum, plus PostGIS (`geometry`, `geography`) and pgvector (`vector`, `halfvec`, `sparsevec`) extensions. +MySQL | `mediumint`, `year`, `bit(n)`, `vector`, and the spatial types (`geometry`, `point`, `linestring`, `polygon`, etc.). + +For the _data_ YAML/JSON, spatial values are specified as [well-known text](https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry) (WKT), with an optional [extended](https://postgis.net/docs/using_postgis_dbmanagement.html#EWKB_EWKT) `SRID=n;` prefix, e.g. `SRID=4326;POINT(-122.349 47.651)`. The coordinate order is _always_ longitude then latitude (x y); MySQL is explicitly instructed to use this order as it otherwise defaults to latitude then longitude for geographic coordinate systems. A vector is specified as `[1, 2, 3]`, and a multi-bit value as a bit string, e.g. `10101010`. + +> Note: where a database extension is required (e.g. PostGIS, pgvector) it must be enabled by a migration script before the data is loaded. See the [`DbEx.Test.PostgresConsole`](./tests/DbEx.Test.PostgresConsole/Migrations/006-create-test-extratypes-table.pgsql) test for an example. +
## Other considerations diff --git a/docker-compose.yml b/docker-compose.yml index 6974ad3..e331380 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,23 +2,43 @@ name: DbEx services: db-sql-server: - image: mcr.microsoft.com/mssql/server:2019-latest + image: mcr.microsoft.com/mssql/server:2025-latest environment: SA_PASSWORD: "yourStrong(!)Password" ACCEPT_EULA: "Y" ports: - "1433:1433" + healthcheck: + test: ["CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P \"$$SA_PASSWORD\" -Q \"SELECT 1\" -b -o /dev/null"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 10s db-mysql: - image: mysql + image: mysql:9 environment: MYSQL_ROOT_PASSWORD: "yourStrong#!Password" ports: - "3306:3306" + healthcheck: + # TCP (not the socket) as the temporary initialization server only listens on the socket. + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -p\"$$MYSQL_ROOT_PASSWORD\" --silent"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 10s db-postgres: - image: postgres + build: ./.github/postgres + image: dbex-postgres-test environment: POSTGRES_PASSWORD: "yourStrong#!Password" ports: - - "5432:5432" \ No newline at end of file + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U postgres"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 10s \ No newline at end of file diff --git a/src/DbEx.MySql/MySqlSchemaConfig.cs b/src/DbEx.MySql/MySqlSchemaConfig.cs index 646173b..b76a52e 100644 --- a/src/DbEx.MySql/MySqlSchemaConfig.cs +++ b/src/DbEx.MySql/MySqlSchemaConfig.cs @@ -81,10 +81,11 @@ public override DbColumnSchema CreateColumnFromInformationSchema(DbTableSchema t Scale = dr.GetValue("NUMERIC_SCALE"), DefaultValue = dr.GetValue("COLUMN_DEFAULT"), IsDotNetDateOnly = RemovePrecisionFromDataType(dt).Equals("DATE", StringComparison.OrdinalIgnoreCase), - IsDotNetTimeOnly = RemovePrecisionFromDataType(dt).Equals("TIME", StringComparison.OrdinalIgnoreCase) + IsDotNetTimeOnly = RemovePrecisionFromDataType(dt).Equals("TIME", StringComparison.OrdinalIgnoreCase), + NativeSqlType = dr.GetValue("COLUMN_TYPE") }; - c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.DotNetType == "string" && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal)); + c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal) && c.DotNetType == "string"); if (c.IsJsonContent && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal)) c.DotNetCleanedName = DbTableSchema.CreateDotNetName(c.Name[..^JsonColumnNameSuffix.Length]); @@ -160,26 +161,43 @@ public override string ToDotNetTypeName(DbColumnSchema schema) else if (Migration.Args.EmitDotNetTimeOnly && schema.IsDotNetTimeOnly) return "TimeOnly"; + // Source of truth: https://mysqlconnector.net/overview/mysql-to-dotnet-types/ and the Pomelo.EntityFrameworkCore.MySql type mappings (spatial types use NetTopologySuite). return dbType.ToUpperInvariant() switch { "CHAR" or "VARCHAR" or "TINYTEXT" or "TEXT" or "MEDIUMTEXT" or "LONGTEXT" or "SET" or "ENUM" or "NCHAR" or "NVARCHAR" or "JSON" => "string", - "DECIMAL" => "decimal", + "DECIMAL" or "NUMERIC" => "decimal", "DATETIME" or "TIMESTAMP" => "DateTime", "DATE" => "DateOnly", "TIME" => "TimeOnly", - "BINARY" or "VARBINARY" or "TINYBLOB" or "BLOB" or "MEDIUMBLOB" or "LONGBLOB" => "byte[]", - "BIT" or "BOOL" or "BOOLEAN" => "bool", + "BINARY" or "VARBINARY" or "TINYBLOB" or "BLOB" or "MEDIUMBLOB" or "LONGBLOB" or "VECTOR" => "byte[]", + "BOOL" or "BOOLEAN" => "bool", + "BIT" => schema.Precision is null or 1 ? "bool" : "ulong", "DOUBLE" => "double", - "INT" => "int", + "INT" or "INTEGER" or "MEDIUMINT" => "int", "BIGINT" => "long", - "SMALLINT" => "short", + "SMALLINT" or "YEAR" => "short", "TINYINT" => "byte", "FLOAT" => "float", - "UNIQUEIDENTIFIER" => "Guid", + "GEOMETRY" => "Geometry", + "POINT" => "Point", + "LINESTRING" => "LineString", + "POLYGON" => "Polygon", + "MULTIPOINT" => "MultiPoint", + "MULTILINESTRING" => "MultiLineString", + "MULTIPOLYGON" => "MultiPolygon", + "GEOMETRYCOLLECTION" or "GEOMCOLLECTION" => "GeometryCollection", _ => throw new InvalidOperationException($"Database data type '{dbType}' does not have corresponding .NET type mapping defined."), }; } + /// + /// A VECTOR is a byte[] in .NET, but is parsed from (and written as) its text (string) representation (e.g. [1,2,3]); likewise a multi-bit BIT is a ulong in .NET (as per the MySql.Data and Pomelo EF providers), but is parsed from its bit-string (e.g. 10101010). + public override string ToDataParserTypeName(DbColumnSchema schema) + { + var type = schema.ThrowIfNull(nameof(schema)).Type.ToUpperInvariant(); + return (type is "VECTOR" && schema.DotNetType == "byte[]") || (type is "BIT" && schema.DotNetType == "ulong") ? "string" : base.ToDataParserTypeName(schema); + } + /// public override string ToFormattedSqlType(DbColumnSchema schema, bool includeNullability = true) { @@ -190,7 +208,9 @@ public override string ToFormattedSqlType(DbColumnSchema schema, bool includeNul "CHAR" or "VARCHAR" or "NCHAR" or "NVARCHAR" => schema.Length.HasValue && schema.Length.Value > 0 ? $"({schema.Length.Value})" : "(MAX)", "DECIMAL" => $"({schema.Precision}, {schema.Scale})", "NUMERIC" => $"({schema.Precision}, {schema.Scale})", + "BIT" => schema.Precision.HasValue ? $"({schema.Precision.Value})" : string.Empty, "TIME" or "DATETIME" or "TIMESTAMP" => schema.Scale.HasValue && schema.Scale.Value > 0 ? $"({schema.Scale})" : string.Empty, + "VECTOR" => ExtractTypeArguments(schema.NativeSqlType), _ => string.Empty }); @@ -200,19 +220,56 @@ public override string ToFormattedSqlType(DbColumnSchema schema, bool includeNul return sb.ToString(); } + /// + /// Extracts the parenthesized type arguments (e.g. (3)) from a native SQL type (e.g. vector(3)). + /// + private static string ExtractTypeArguments(string? nativeSqlType) + { + var start = nativeSqlType?.IndexOf('(') ?? -1; + var end = nativeSqlType?.IndexOf(')') ?? -1; + return nativeSqlType is not null && start >= 0 && end > start ? nativeSqlType[start..(end + 1)] : string.Empty; + } + /// - public override string ToFormattedSqlStatementValue(DbColumnSchema dbColumnSchema, object? value) => value switch + public override string ToFormattedSqlStatementValue(DbColumnSchema dbColumnSchema, object? value) { - null => "NULL", - string str => $"'{str.Replace("'", "''", StringComparison.Ordinal)}'", - bool b => b ? "true" : "false", - Guid => $"'{value}'", - DateTime dt => $"'{dt.ToString(Migration.Args.DataParserArgs.DateTimeFormat, System.Globalization.CultureInfo.InvariantCulture)}'", - DateTimeOffset dto => $"'{dto.ToString(Migration.Args.DataParserArgs.DateTimeOffsetFormat, System.Globalization.CultureInfo.InvariantCulture)}'", + if (value is string text) + { + var type = dbColumnSchema.Type.ToUpperInvariant(); + if (type is "VECTOR") + return $"STRING_TO_VECTOR('{text.Replace("'", "''", StringComparison.Ordinal)}')"; + + if (type is "BIT") + return $"b'{text.Replace("'", "''", StringComparison.Ordinal)}'"; + + if (type is "GEOMETRY" or "POINT" or "LINESTRING" or "POLYGON" or "MULTIPOINT" or "MULTILINESTRING" or "MULTIPOLYGON" or "GEOMETRYCOLLECTION" or "GEOMCOLLECTION") + { + // Supports the extended well-known text (EWKT) form of 'SRID=n;WKT' to specify the spatial reference system; the long-lat axis order is explicit as MySQL otherwise defaults to lat-long for geographic SRIDs (differs to PostGIS and SQL Server). + var srid = string.Empty; + if (text.StartsWith("SRID=", StringComparison.OrdinalIgnoreCase) && text.IndexOf(';') is var i and > 0 && uint.TryParse(text[5..i], out var s)) + { + srid = $", {s}, 'axis-order=long-lat'"; + text = text[(i + 1)..]; + } + + return $"ST_GeomFromText('{text.Replace("'", "''", StringComparison.Ordinal)}'{srid})"; + } + } + + return value switch + { + null => "NULL", + string str => $"'{str.Replace("'", "''", StringComparison.Ordinal)}'", + bool b when dbColumnSchema.Type.Equals("BIT", StringComparison.OrdinalIgnoreCase) => b ? "b'1'" : "b'0'", + bool b => b ? "true" : "false", + Guid => $"'{value}'", + DateTime dt => $"'{dt.ToString(Migration.Args.DataParserArgs.DateTimeFormat, System.Globalization.CultureInfo.InvariantCulture)}'", + DateTimeOffset dto => $"'{dto.ToString(Migration.Args.DataParserArgs.DateTimeOffsetFormat, System.Globalization.CultureInfo.InvariantCulture)}'", #if NET7_0_OR_GREATER - DateOnly d => $"'{d.ToString(Migration.Args.DataParserArgs.DateOnlyFormat, System.Globalization.CultureInfo.InvariantCulture)}'", - TimeOnly t => $"'{t.ToString(Migration.Args.DataParserArgs.TimeOnlyFormat, System.Globalization.CultureInfo.InvariantCulture)}'", + DateOnly d => $"'{d.ToString(Migration.Args.DataParserArgs.DateOnlyFormat, System.Globalization.CultureInfo.InvariantCulture)}'", + TimeOnly t => $"'{t.ToString(Migration.Args.DataParserArgs.TimeOnlyFormat, System.Globalization.CultureInfo.InvariantCulture)}'", #endif - _ => value.ToString()! - }; + _ => value.ToString()! + }; + } } \ No newline at end of file diff --git a/src/DbEx.Postgres/PostgresSchemaConfig.cs b/src/DbEx.Postgres/PostgresSchemaConfig.cs index 6e4101e..667768c 100644 --- a/src/DbEx.Postgres/PostgresSchemaConfig.cs +++ b/src/DbEx.Postgres/PostgresSchemaConfig.cs @@ -6,6 +6,8 @@ /// The owning . public class PostgresSchemaConfig(PostgresMigration migration) : DatabaseSchemaConfig(migration, true, "public", "pgsql") { + private readonly HashSet _enumTypeNames = new(StringComparer.Ordinal); + /// /// Value is '_id'. public override string IdColumnNameSuffix => "_id"; @@ -80,10 +82,11 @@ public override DbColumnSchema CreateColumnFromInformationSchema(DbTableSchema t IsComputed = dr.GetValue("IS_GENERATED") != "NEVER", IsIdentity = dr.GetValue("COLUMN_DEFAULT")?.StartsWith("nextval(", StringComparison.OrdinalIgnoreCase) ?? false, IsDotNetDateOnly = RemovePrecisionFromDataType(dr.GetValue("DATA_TYPE")!).Equals("DATE", StringComparison.OrdinalIgnoreCase), - IsDotNetTimeOnly = RemovePrecisionFromDataType(dr.GetValue("DATA_TYPE")!).Equals("TIME WITHOUT TIME ZONE", StringComparison.OrdinalIgnoreCase) + IsDotNetTimeOnly = RemovePrecisionFromDataType(dr.GetValue("DATA_TYPE")!).Equals("TIME WITHOUT TIME ZONE", StringComparison.OrdinalIgnoreCase), + UdtName = dr.GetValue("udt_name") }; - c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.DotNetType == "string" && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal)); + c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal) && c.DotNetType == "string"); if (c.IsJsonContent && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal)) c.DotNetCleanedName = DbTableSchema.CreateDotNetName(c.Name[..^JsonColumnNameSuffix.Length]); @@ -95,6 +98,29 @@ public override DbColumnSchema CreateColumnFromInformationSchema(DbTableSchema t /// private static string RemovePrecisionFromDataType(string type) => type.Contains('(') ? type[..type.IndexOf('(')] : type; + /// + public override async Task LoadNativeTypesSchema(IDatabase database, List tables, CancellationToken cancellationToken) + { + // Select the full native types (including type modifiers) and enum types; these are not available via INFORMATION_SCHEMA for the likes of array, enum and extension types (e.g. PostGIS, pgvector). + _enumTypeNames.Clear(); + using var sr = DatabaseMigrationBase.GetRequiredResourcesStreamReader($"SelectTableNativeTypes.{ScriptSuffix}", [typeof(PostgresSchemaConfig).Assembly]); + await database.SqlStatement(await sr.ReadToEndAsync(cancellationToken).ConfigureAwait(false)).SelectQueryAsync(dr => + { + if (dr.GetValue("type_kind") == "e") + _enumTypeNames.Add(dr.GetValue("udt_name")!); + + if (dr.GetValue("element_type_kind") == "e") + _enumTypeNames.Add(dr.GetValue("element_udt_name")!); + + var t = tables.SingleOrDefault(x => x.Schema == dr.GetValue("table_schema") && x.Name == dr.GetValue("table_name")); + var c = t?.Columns.SingleOrDefault(x => x.Name == dr.GetValue("column_name")); + if (c is not null) + c.NativeSqlType = dr.GetValue("native_type"); + + return 0; + }, cancellationToken).ConfigureAwait(false); + } + /// public override async Task LoadAdditionalInformationSchema(IDatabase database, List tables, CancellationToken cancellationToken) { @@ -165,7 +191,7 @@ public override string ToDotNetTypeName(DbColumnSchema schema) "TIME WITHOUT TIME ZONE" => "TimeOnly", "DATE" => "DateOnly", "BYTEA" => "byte[]", - "BOOLEAN" or "BIT(1)" => "bool", + "BOOLEAN" => "bool", "DOUBLE PRECISION" => "double", "INTEGER" => "int", "BIGINT" => "long", @@ -173,22 +199,155 @@ public override string ToDotNetTypeName(DbColumnSchema schema) "REAL" => "float", "UUID" => "Guid", "XID" => "uint", - _ => throw new InvalidOperationException($"Database data type '{dbType}' does not have corresponding .NET type mapping defined."), + "BIT" => schema.Length == 1 ? "bool" : "BitArray", + "ARRAY" => ToDotNetArrayElementTypeName(schema), + _ => ToDotNetTypeNameFromUdtName(schema.UdtName, schema) ?? throw new InvalidOperationException($"Database data type '{dbType}'{(string.IsNullOrEmpty(schema.UdtName) ? string.Empty : $" ('{schema.UdtName}')")} does not have corresponding .NET type mapping defined."), }; } + /// + /// Gets the .NET array type name for an ARRAY where the element type is derived from the (internal names are prefixed with an underscore). + /// + private string ToDotNetArrayElementTypeName(DbColumnSchema schema) + { + // Only the single array prefix underscore is removed; the element type name may itself start with an underscore. + var udt = schema.UdtName is { Length: > 0 } u && u[0] == '_' ? u[1..] : schema.UdtName; + var element = string.IsNullOrEmpty(udt) ? null : ToDotNetTypeNameFromUdtName(udt, null); + return element is null + ? throw new InvalidOperationException($"Database data type 'ARRAY' ('{schema.UdtName}') does not have corresponding .NET type mapping defined.") + : $"{element}[]"; + } + + /// + /// Gets the .NET type name for the PostgreSQL internal (udt_name) type name; null where not mapped. + /// + /// This covers the built-in types not already mapped by , enums, and the more well-known extensions (PostGIS, pgvector, hstore, ltree and citext) using the Npgsql and EF Core provider conventions. + private string? ToDotNetTypeNameFromUdtName(string? udtName, DbColumnSchema? schema) + { + if (string.IsNullOrEmpty(udtName)) + return null; + + if (_enumTypeNames.Contains(udtName)) + return "string"; + + return udtName.ToLowerInvariant() switch + { + "text" or "varchar" or "bpchar" or "citext" or "json" or "jsonb" or "jsonpath" or "xml" or "name" or "ltree" or "lquery" or "ltxtquery" => "string", + "numeric" or "money" => "decimal", + "timestamp" => "DateTime", + "timetz" or "timestamptz" => "DateTimeOffset", + "interval" => "TimeSpan", + "time" => "TimeOnly", + "date" => "DateOnly", + "bytea" => "byte[]", + "bool" => "bool", + "float8" => "double", + "float4" => "float", + "int8" => "long", + "int4" => "int", + "int2" => "short", + "uuid" => "Guid", + "oid" or "xid" or "cid" => "uint", + "xid8" => "ulong", + "char" => "char", + "pg_lsn" => "NpgsqlLogSequenceNumber", + "bit" => schema?.Length == 1 ? "bool" : "BitArray", + "varbit" => "BitArray", + "inet" => "IPAddress", + "cidr" => "NpgsqlCidr", + "macaddr" or "macaddr8" => "PhysicalAddress", + "tsvector" => "NpgsqlTsVector", + "tsquery" => "NpgsqlTsQuery", + "point" => "NpgsqlPoint", + "line" => "NpgsqlLine", + "lseg" => "NpgsqlLSeg", + "box" => "NpgsqlBox", + "path" => "NpgsqlPath", + "polygon" => "NpgsqlPolygon", + "circle" => "NpgsqlCircle", + "hstore" => "Dictionary", + "geometry" or "geography" => ToDotNetSpatialTypeName(schema?.NativeSqlType), + "vector" => "Vector", + "halfvec" => "HalfVector", + "sparsevec" => "SparseVector", + "int4range" => "NpgsqlRange", + "int8range" => "NpgsqlRange", + "numrange" => "NpgsqlRange", + "daterange" => "NpgsqlRange", + "tsrange" or "tstzrange" => "NpgsqlRange", + "int4multirange" => "NpgsqlRange[]", + "int8multirange" => "NpgsqlRange[]", + "nummultirange" => "NpgsqlRange[]", + "datemultirange" => "NpgsqlRange[]", + "tsmultirange" or "tstzmultirange" => "NpgsqlRange[]", + _ => null + }; + } + + /// + /// Gets the NetTopologySuite type name from the PostGIS native type (e.g. geometry(Point,4326)); defaults to Geometry where not specific. + /// + private static string ToDotNetSpatialTypeName(string? nativeSqlType) + { + var start = nativeSqlType?.IndexOf('(') ?? -1; + if (nativeSqlType is null || start < 0) + return "Geometry"; + + var subtype = nativeSqlType[(start + 1)..].Split(',', ')')[0].Trim(); + if (subtype.EndsWith("ZM", StringComparison.OrdinalIgnoreCase)) + subtype = subtype[..^2]; + else if (subtype.EndsWith('Z') || subtype.EndsWith('z') || subtype.EndsWith('M') || subtype.EndsWith('m')) + subtype = subtype[..^1]; + + return subtype.ToUpperInvariant() switch + { + "POINT" => "Point", + "LINESTRING" => "LineString", + "POLYGON" => "Polygon", + "MULTIPOINT" => "MultiPoint", + "MULTILINESTRING" => "MultiLineString", + "MULTIPOLYGON" => "MultiPolygon", + "GEOMETRYCOLLECTION" => "GeometryCollection", + _ => "Geometry" + }; + } + + private static readonly HashSet _structTypeNames = new(StringComparer.Ordinal) + { + "NpgsqlLogSequenceNumber", "NpgsqlCidr", "NpgsqlPoint", "NpgsqlLine", "NpgsqlLSeg", "NpgsqlBox", "NpgsqlPath", "NpgsqlPolygon", "NpgsqlCircle" + }; + + /// + public override bool IsDotNetTypeAClass(DbColumnSchema schema) + { + var type = schema.ThrowIfNull(nameof(schema)).DotNetType; + if (_structTypeNames.Contains(type) || (type.StartsWith("NpgsqlRange<", StringComparison.Ordinal) && !type.EndsWith("[]", StringComparison.Ordinal))) + return false; + + return base.IsDotNetTypeAClass(schema); + } + /// public override string ToFormattedSqlType(DbColumnSchema schema, bool includeNullability = true) { - var sb = new StringBuilder(schema.Type!.ToUpperInvariant()); - - sb.Append(schema.Type.ToUpperInvariant() switch + var sb = new StringBuilder(); + if (schema.Type.Equals("ARRAY", StringComparison.OrdinalIgnoreCase) || schema.Type.Equals("USER-DEFINED", StringComparison.OrdinalIgnoreCase)) { - "CHARACTER VARYING" or "CHARACTER" => schema.Length.HasValue && schema.Length.Value > 0 ? $"({schema.Length.Value})" : "(MAX)", - "NUMERIC" => $"({schema.Precision}, {schema.Scale})", - "TIMESTAMP WITHOUT TIME ZONE" or "TIMESTAMP WITH TIME ZONE" or "TIME WITH TIME ZONE" or "TIME WITHOUT TIME ZONE" => schema.Scale.HasValue && schema.Scale.Value > 0 ? $"({schema.Scale})" : string.Empty, - _ => string.Empty - }); + // Use the native type as-is (it is already correctly quoted/qualified where required); upper-casing is not safe. + sb.Append(schema.NativeSqlType ?? (schema.Type.Equals("ARRAY", StringComparison.OrdinalIgnoreCase) ? $"{schema.UdtName?.TrimStart('_')}[]" : schema.UdtName)); + } + else + { + sb.Append(schema.Type!.ToUpperInvariant()); + sb.Append(schema.Type.ToUpperInvariant() switch + { + "CHARACTER VARYING" or "CHARACTER" => schema.Length.HasValue && schema.Length.Value > 0 ? $"({schema.Length.Value})" : "(MAX)", + "BIT" or "BIT VARYING" => schema.Length.HasValue && schema.Length.Value > 0 ? $"({schema.Length.Value})" : string.Empty, + "NUMERIC" => $"({schema.Precision}, {schema.Scale})", + "TIMESTAMP WITHOUT TIME ZONE" or "TIMESTAMP WITH TIME ZONE" or "TIME WITH TIME ZONE" or "TIME WITHOUT TIME ZONE" => schema.Scale.HasValue && schema.Scale.Value > 0 ? $"({schema.Scale})" : string.Empty, + _ => string.Empty + }); + } if (includeNullability && schema.IsNullable) sb.Append(" NULL"); @@ -201,6 +360,7 @@ public override string ToFormattedSqlType(DbColumnSchema schema, bool includeNul { null => "NULL", string str => $"'{str.Replace("'", "''", StringComparison.Ordinal)}'", + bool b when dbColumnSchema.Type.StartsWith("BIT", StringComparison.OrdinalIgnoreCase) => b ? "B'1'" : "B'0'", bool b => b ? "true" : "false", Guid => $"uuid('{value}')", DateTime dt => $"'{dt.ToString(Migration.Args.DataParserArgs.DateTimeFormat, System.Globalization.CultureInfo.InvariantCulture)}'", diff --git a/src/DbEx.Postgres/Resources/SelectTableNativeTypes.pgsql b/src/DbEx.Postgres/Resources/SelectTableNativeTypes.pgsql new file mode 100644 index 0000000..85f9dc0 --- /dev/null +++ b/src/DbEx.Postgres/Resources/SelectTableNativeTypes.pgsql @@ -0,0 +1,12 @@ +SELECT n.nspname AS table_schema, c.relname AS table_name, a.attname AS column_name, + format_type(a.atttypid, a.atttypmod) AS native_type, + t.typname::text AS udt_name, t.typtype::text AS type_kind, + CASE WHEN t.typcategory = 'A' THEN et.typname::text END AS element_udt_name, + CASE WHEN t.typcategory = 'A' THEN et.typtype::text END AS element_type_kind + FROM pg_catalog.pg_attribute AS a + INNER JOIN pg_catalog.pg_class AS c ON c.oid = a.attrelid + INNER JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace + INNER JOIN pg_catalog.pg_type AS t ON t.oid = a.atttypid + LEFT OUTER JOIN pg_catalog.pg_type AS et ON et.oid = t.typelem + WHERE a.attnum > 0 AND NOT a.attisdropped AND c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND n.nspname NOT IN ('information_schema', 'pg_catalog') diff --git a/src/DbEx.SqlServer/Resources/DatabaseData_sql.hbs b/src/DbEx.SqlServer/Resources/DatabaseData_sql.hbs index d3d62e8..ddb9a00 100644 --- a/src/DbEx.SqlServer/Resources/DatabaseData_sql.hbs +++ b/src/DbEx.SqlServer/Resources/DatabaseData_sql.hbs @@ -16,10 +16,10 @@ INSERT INTO #temp ({{#each Columns}}[{{Name}}]{{#unless @last}}, {{/unless}}{{/e MERGE INTO [{{Schema}}].[{{Name}}] WITH (HOLDLOCK) as [t] USING (SELECT {{#each Columns}}[{{Name}}]{{#unless @last}}, {{/unless}}{{/each}} FROM #temp) AS [s] ON ({{#if IsRefData}}[s].[{{DbTable.RefDataCodeColumn.Name}}] = [t].[{{DbTable.RefDataCodeColumn.Name}}]{{else}}{{#each PrimaryKeyColumns}}{{#unless @first}} ON {{/unless}}[s].[{{Name}}] = [t].[{{Name}}]{{/each}}{{/if}}{{#ifval DbTable.TenantIdColumn}} AND [s].[{{DbTable.TenantIdColumn.Name}}] = [t].[{{DbTable.TenantIdColumn.Name}}]{{/ifval}}) - WHEN MATCHED AND EXISTS ( + WHEN MATCHED{{#unless HasNonComparableMergeMatchColumns}} AND EXISTS ( SELECT {{#each MergeMatchColumns}}[s].[{{Name}}]{{#unless @last}}, {{/unless}}{{/each}} EXCEPT - SELECT {{#each MergeMatchColumns}}[t].[{{Name}}]{{#unless @last}}, {{/unless}}{{/each}}) + SELECT {{#each MergeMatchColumns}}[t].[{{Name}}]{{#unless @last}}, {{/unless}}{{/each}}){{/unless}} THEN UPDATE SET {{#each MergeUpdateColumns}}[t].[{{Name}}] = [s].[{{Name}}]{{#unless @last}}, {{/unless}}{{/each}} WHEN NOT MATCHED BY TARGET THEN INSERT ({{#each MergeInsertColumns}}[{{Name}}]{{#unless @last}}, {{/unless}}{{/each}}) diff --git a/src/DbEx.SqlServer/Resources/SelectTableVectorColumns.sql b/src/DbEx.SqlServer/Resources/SelectTableVectorColumns.sql new file mode 100644 index 0000000..d6648a0 --- /dev/null +++ b/src/DbEx.SqlServer/Resources/SelectTableVectorColumns.sql @@ -0,0 +1,4 @@ +SELECT SCHEMA_NAME(o.schema_id) AS TABLE_SCHEMA, o.name AS TABLE_NAME, c.name AS COLUMN_NAME, c.vector_dimensions AS VECTOR_DIMENSIONS + FROM sys.columns AS c + INNER JOIN sys.objects AS o ON o.object_id = c.object_id + WHERE c.vector_dimensions IS NOT NULL diff --git a/src/DbEx.SqlServer/SqlServerSchemaConfig.cs b/src/DbEx.SqlServer/SqlServerSchemaConfig.cs index 6028264..71fdeb3 100644 --- a/src/DbEx.SqlServer/SqlServerSchemaConfig.cs +++ b/src/DbEx.SqlServer/SqlServerSchemaConfig.cs @@ -80,7 +80,7 @@ public override DbColumnSchema CreateColumnFromInformationSchema(DbTableSchema t IsDotNetTimeOnly = RemovePrecisionFromDataType(dr.GetValue("DATA_TYPE")!).Equals("TIME", StringComparison.OrdinalIgnoreCase), }; - c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.DotNetType == "string" && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal)); + c.IsJsonContent = c.Type.Equals("JSON", StringComparison.OrdinalIgnoreCase) || (c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal) && c.DotNetType == "string"); if (c.IsJsonContent && c.Name.EndsWith(JsonColumnNameSuffix, StringComparison.Ordinal)) c.DotNetCleanedName = DbTableSchema.CreateDotNetName(c.Name[..^JsonColumnNameSuffix.Length]); @@ -181,6 +181,25 @@ await database.SqlStatement(await sr6.ReadToEndAsync().ConfigureAwait(false)).Se c.IsComputed = true; return 0; }, cancellationToken).ConfigureAwait(false); + + // Select the vector dimensions (INFORMATION_SCHEMA does not expose them); only queried where a vector column exists as the catalog column requires SQL Server 2025+. + if (tables.Any(t => t.Columns.Any(c => c.Type.Equals("VECTOR", StringComparison.OrdinalIgnoreCase)))) + { + using var sr7 = DatabaseMigrationBase.GetRequiredResourcesStreamReader("SelectTableVectorColumns.sql", [typeof(SqlServerSchemaConfig).Assembly]); +#if NET7_0_OR_GREATER + await database.SqlStatement(await sr7.ReadToEndAsync(cancellationToken).ConfigureAwait(false)).SelectQueryAsync(dr => +#else + await database.SqlStatement(await sr7.ReadToEndAsync().ConfigureAwait(false)).SelectQueryAsync(dr => +#endif + { + var t = tables.SingleOrDefault(x => x.Schema == dr.GetValue("TABLE_SCHEMA") && x.Name == dr.GetValue("TABLE_NAME")); + var c = t?.Columns.SingleOrDefault(x => x.Name == dr.GetValue("COLUMN_NAME")); + if (c is not null) + c.Length = (ulong?)dr.GetValue("VECTOR_DIMENSIONS"); + + return 0; + }, cancellationToken).ConfigureAwait(false); + } } /// @@ -212,10 +231,21 @@ public override string ToDotNetTypeName(DbColumnSchema schema) "TINYINT" => "byte", "REAL" => "float", "UNIQUEIDENTIFIER" => "Guid", + "XML" => "string", + "GEOGRAPHY" or "GEOMETRY" => "Geometry", + "HIERARCHYID" => "HierarchyId", + "VECTOR" => "SqlVector", + "SQL_VARIANT" => "object", _ => throw new InvalidOperationException($"Database data type '{dbType}' does not have corresponding .NET type mapping defined."), }; } + /// + public override bool IsDotNetTypeAClass(DbColumnSchema schema) => schema.ThrowIfNull(nameof(schema)).DotNetType != "SqlVector" && base.IsDotNetTypeAClass(schema); + + /// + public override bool IsDataComparable(DbColumnSchema schema) => RemovePrecisionFromDataType(schema.ThrowIfNull(nameof(schema)).Type).ToUpperInvariant() is not ("XML" or "GEOGRAPHY" or "GEOMETRY" or "VECTOR" or "JSON" or "TEXT" or "NTEXT" or "IMAGE"); + /// public override string ToFormattedSqlType(DbColumnSchema schema, bool includeNullability = true) { @@ -223,11 +253,11 @@ public override string ToFormattedSqlType(DbColumnSchema schema, bool includeNul sb.Append(schema.Type.ToUpperInvariant() switch { - "CHAR" or "VARCHAR" or "NCHAR" or "NVARCHAR" => schema.Length.HasValue && schema.Length.Value > 0 ? $"({schema.Length.Value})" : "(MAX)", + "CHAR" or "VARCHAR" or "NCHAR" or "NVARCHAR" or "BINARY" or "VARBINARY" => schema.Length.HasValue && schema.Length.Value > 0 ? $"({schema.Length.Value})" : "(MAX)", "DECIMAL" => $"({schema.Precision}, {schema.Scale})", "NUMERIC" => $"({schema.Precision}, {schema.Scale})", "TIME" => schema.Scale.HasValue && schema.Scale.Value > 0 ? $"({schema.Scale})" : string.Empty, - "BINARY" or "VARBINARY" => $"(schema.Precision)", + "VECTOR" => schema.Length.HasValue && schema.Length.Value > 0 ? $"({schema.Length.Value})" : string.Empty, _ => string.Empty }); @@ -238,18 +268,63 @@ public override string ToFormattedSqlType(DbColumnSchema schema, bool includeNul } /// - public override string ToFormattedSqlStatementValue(DbColumnSchema dbColumnSchema, object? value) => value switch + public override string ToFormattedSqlStatementValue(DbColumnSchema dbColumnSchema, object? value) { - null => "NULL", - string str => $"N'{str.Replace("'", "''", StringComparison.Ordinal)}'", - bool b => b ? "1" : "0", - Guid => $"CONVERT(UNIQUEIDENTIFIER, '{value}')", - DateTime dt => $"'{dt.ToString(Migration.Args.DataParserArgs.DateTimeFormat, System.Globalization.CultureInfo.InvariantCulture)}'", - DateTimeOffset dto => $"'{dto.ToString(Migration.Args.DataParserArgs.DateTimeOffsetFormat, System.Globalization.CultureInfo.InvariantCulture)}'", + // The data parser converts these types as text; the database must be explicitly told how to convert (implicit conversion is not supported for all of them). + if (value is string text) + { + var literal = $"N'{text.Replace("'", "''", StringComparison.Ordinal)}'"; + switch (RemovePrecisionFromDataType(dbColumnSchema.ThrowIfNull(nameof(dbColumnSchema)).Type).ToUpperInvariant()) + { + case "GEOGRAPHY": + return ToFormattedSpatialValue("geography", text); + + case "GEOMETRY": + return ToFormattedSpatialValue("geometry", text); + + case "HIERARCHYID": + return $"hierarchyid::Parse({literal})"; + + case "XML": + return $"CAST({literal} AS XML)"; + + case "VECTOR": + return $"CAST({literal} AS {dbColumnSchema.SqlType2})"; + + case "SQL_VARIANT": + return $"CAST({literal} AS SQL_VARIANT)"; + } + } + + return value switch + { + null => "NULL", + string str => $"N'{str.Replace("'", "''", StringComparison.Ordinal)}'", + bool b => b ? "1" : "0", + Guid => $"CONVERT(UNIQUEIDENTIFIER, '{value}')", + DateTime dt => $"'{dt.ToString(Migration.Args.DataParserArgs.DateTimeFormat, System.Globalization.CultureInfo.InvariantCulture)}'", + DateTimeOffset dto => $"'{dto.ToString(Migration.Args.DataParserArgs.DateTimeOffsetFormat, System.Globalization.CultureInfo.InvariantCulture)}'", #if NET7_0_OR_GREATER - DateOnly d => $"'{d.ToString(Migration.Args.DataParserArgs.DateOnlyFormat, System.Globalization.CultureInfo.InvariantCulture)}'", - TimeOnly t => $"'{t.ToString(Migration.Args.DataParserArgs.TimeOnlyFormat, System.Globalization.CultureInfo.InvariantCulture)}'", + DateOnly d => $"'{d.ToString(Migration.Args.DataParserArgs.DateOnlyFormat, System.Globalization.CultureInfo.InvariantCulture)}'", + TimeOnly t => $"'{t.ToString(Migration.Args.DataParserArgs.TimeOnlyFormat, System.Globalization.CultureInfo.InvariantCulture)}'", #endif - _ => value.ToString()! - }; + _ => value.ToString()! + }; + } + + /// + /// Formats the spatial (well-known text) value; an optional leading 'SRID=n;' (EWKT) is supported to specify the spatial reference identifier. + /// + private static string ToFormattedSpatialValue(string type, string wkt) + { + var text = wkt.Trim(); + if (text.StartsWith("SRID=", StringComparison.OrdinalIgnoreCase)) + { + var i = text.IndexOf(';', StringComparison.Ordinal); + if (i > 5 && int.TryParse(text[5..i], System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var srid)) + return $"{type}::STGeomFromText(N'{text[(i + 1)..].Replace("'", "''", StringComparison.Ordinal)}', {srid})"; + } + + return $"{type}::Parse(N'{text.Replace("'", "''", StringComparison.Ordinal)}')"; + } } \ No newline at end of file diff --git a/src/DbEx/DatabaseSchemaConfig.cs b/src/DbEx/DatabaseSchemaConfig.cs index 3b58989..b86797f 100644 --- a/src/DbEx/DatabaseSchemaConfig.cs +++ b/src/DbEx/DatabaseSchemaConfig.cs @@ -129,6 +129,16 @@ public virtual void PrepareMigrationArgs() /// The . public abstract DbColumnSchema CreateColumnFromInformationSchema(DbTableSchema table, DatabaseRecord dr); + /// + /// Opportunity to load additional native column type information that is specific to the database (for example, ). + /// + /// The . + /// The list to load additional data into. + /// The . + /// This is invoked immediately after the tables and columns are inferred, and before any other schema inference that may require the (for example, reference data detection); as the + /// is cached on first access, anything required to determine the type must be loaded here and not within . + public virtual Task LoadNativeTypesSchema(IDatabase database, List tables, CancellationToken cancellationToken) => Task.CompletedTask; + /// /// Opportunity to load additional `InformationSchema` related data that is specific to the database. /// @@ -152,6 +162,44 @@ public virtual void PrepareMigrationArgs() /// The .NET name. public abstract string ToDotNetTypeName(DbColumnSchema schema); + /// + /// Gets the .NET name that the data parser (see ) uses to convert the textual (JSON/YAML) value for the specified . + /// + /// The . + /// The data parser .NET name. + /// Defaults to where the data parser supports the type; otherwise, string. A database type that has no sensible textual .NET representation (for example, spatial types) is therefore parsed as text and + /// the database is then responsible for converting the text (see ). Override to change this behavior. + public virtual string ToDataParserTypeName(DbColumnSchema schema) + { + var type = ToDotNetTypeName(schema); + return DataParserTypeNames.Contains(type) ? type : "string"; + } + + /// + /// Indicates whether the .NET (see ) for the specified is a reference type (class); otherwise, a value type (struct). + /// + /// The . + /// true where a class; otherwise, false. + /// Defaults to false for the known .NET primitive and other value types (e.g. int, DateTime, Guid); otherwise, true (the safe default as initializing a struct using default! is harmless). + public virtual bool IsDotNetTypeAClass(DbColumnSchema schema) => !DotNetValueTypeNames.Contains(schema.ThrowIfNull(nameof(schema)).DotNetType); + + /// + /// Indicates whether the column value can be compared for equality within the database (for example, using EXCEPT) when determining whether data has changed. + /// + /// The . + /// true where comparable (default); otherwise, false. + public virtual bool IsDataComparable(DbColumnSchema schema) => true; + + /// + /// Gets the known .NET primitive and other value type names (unqualified and without nullability). + /// + protected static HashSet DotNetValueTypeNames { get; } = ["bool", "byte", "sbyte", "short", "ushort", "int", "uint", "long", "ulong", "float", "double", "decimal", "char", "DateTime", "DateTimeOffset", "DateOnly", "TimeOnly", "TimeSpan", "Guid"]; + + /// + /// Gets the .NET type names that are natively supported by the data parser (see ). + /// + protected static HashSet DataParserTypeNames { get; } = ["string", "bool", "byte", "short", "ushort", "int", "uint", "long", "ulong", "float", "double", "decimal", "DateTime", "DateTimeOffset", "DateOnly", "TimeOnly", "TimeSpan", "Guid", "byte[]"]; + /// /// Gets the long-form formatted SQL type; includes size, precision, etc. /// diff --git a/src/DbEx/DbSchema/DbColumnSchema.cs b/src/DbEx/DbSchema/DbColumnSchema.cs index d3f160a..f6ea24e 100644 --- a/src/DbEx/DbSchema/DbColumnSchema.cs +++ b/src/DbEx/DbSchema/DbColumnSchema.cs @@ -11,6 +11,8 @@ public class DbColumnSchema(DbTableSchema dbTable, string name, string type, string? dotNetNameOverride = null) { private string? _dotNetType; + private string? _dataParserType; + private bool? _isDotNetTypeAClass; private string? _dotNetName = dotNetNameOverride; private string? _dotNetCleanedName; private string? _sqlType; @@ -31,6 +33,16 @@ public class DbColumnSchema(DbTableSchema dbTable, string name, string type, str /// public string Type { get; } = type.ThrowIfNull(nameof(type)); + /// + /// Gets or sets the provider-specific underlying type name where the is generic (for example, the Postgres udt_name for USER-DEFINED and ARRAY types). + /// + public string? UdtName { get; set; } + + /// + /// Gets or sets the full database-native formatted type (for example, vector(3), geometry(Point,4326), integer[]) where it is able to be determined and is richer than the , , and . + /// + public string? NativeSqlType { get; set; } + /// /// Indicates whether the column is nullable. /// @@ -186,10 +198,27 @@ public class DbColumnSchema(DbTableSchema dbTable, string name, string type, str /// public string DotNetType => _dotNetType ??= DbTable?.Migration.SchemaConfig.ToDotNetTypeName(this) ?? throw new InvalidOperationException($"The {nameof(DbTable)} must be set before the {nameof(DotNetType)} property can be accessed."); + /// + /// Gets the .NET name (excludes nullability) that the data parser (see ) uses to convert the textual (JSON/YAML) value. + /// + /// Generally the same as the ; however, a database type that has no sensible textual .NET representation (for example, spatial types) is parsed as a string and converted by the database. + public string DataParserType => _dataParserType ??= DbTable?.Migration.SchemaConfig.ToDataParserTypeName(this) ?? throw new InvalidOperationException($"The {nameof(DbTable)} must be set before the {nameof(DataParserType)} property can be accessed."); + + /// + /// Indicates whether the is a reference type (class); otherwise, a value type (struct). + /// + /// Used to determine whether a non-nullable property requires initialization (e.g. = default!) to avoid compiler nullable warnings. + public bool IsDotNetTypeAClass => _isDotNetTypeAClass ??= DbTable?.Migration.SchemaConfig.IsDotNetTypeAClass(this) ?? throw new InvalidOperationException($"The {nameof(DbTable)} must be set before the {nameof(IsDotNetTypeAClass)} property can be accessed."); + + /// + /// Indicates whether the column value can be compared for equality within the database (for example, using EXCEPT) when determining whether data has changed. + /// + public bool IsDataComparable => DbTable?.Migration.SchemaConfig.IsDataComparable(this) ?? throw new InvalidOperationException($"The {nameof(DbTable)} must be set before the {nameof(IsDataComparable)} property can be accessed."); + /// /// Gets the corresponding .NET name (including nullability). /// - /// A type is always considered nullable; otherwise, the nullability is determined by the property. + /// The nullability is determined by the property. public string DotNetTypeWithNullability => IsNullable ? $"{DotNetType}?" : DotNetType; /// @@ -251,9 +280,13 @@ public DbColumnSchema Clone() public void CopyFrom(DbColumnSchema column) { _dotNetType = column.ThrowIfNull(nameof(column))._dotNetType; + _dataParserType = column._dataParserType; + _isDotNetTypeAClass = column._isDotNetTypeAClass; _dotNetName = column._dotNetName; _dotNetCleanedName = column._dotNetCleanedName; _sqlType = column._sqlType; + UdtName = column.UdtName; + NativeSqlType = column.NativeSqlType; IsNullable = column.IsNullable; Length = column.Length; Precision = column.Precision; diff --git a/src/DbEx/Migration/Data/DataRow.cs b/src/DbEx/Migration/Data/DataRow.cs index 70df979..10a91d5 100644 --- a/src/DbEx/Migration/Data/DataRow.cs +++ b/src/DbEx/Migration/Data/DataRow.cs @@ -78,7 +78,7 @@ public void AddColumn(DataColumn column) { str = column.Table.DbTable.Migration.SchemaConfig.ToFormattedDataParserValue(column.Table.ParserArgs, column.Value); - switch (col.DotNetType) + switch (col.DataParserType) { case "string": column.Value = str; break; case "bool": column.Value = str switch { "1" or "Y" => true, "0" or "N" or "" => false, _ => bool.Parse(str) }; break; diff --git a/src/DbEx/Migration/Data/DataTable.cs b/src/DbEx/Migration/Data/DataTable.cs index c4efbad..dd2eb3c 100644 --- a/src/DbEx/Migration/Data/DataTable.cs +++ b/src/DbEx/Migration/Data/DataTable.cs @@ -115,6 +115,11 @@ internal DataTable(DataParser parser, string schema, string name) /// public List MergeMatchColumns => [.. Columns.Where(x => !x.IsCreatedAudit && !x.IsUpdatedAudit && !(UseIdentifierGenerator && x.IsPrimaryKey))]; + /// + /// Indicates whether any of the are not ; therefore, the data change detection cannot be performed by the database. + /// + public bool HasNonComparableMergeMatchColumns => MergeMatchColumns.Any(x => !x.IsDataComparable); + /// /// Gets the merge insert columns. /// diff --git a/src/DbEx/Migration/Database.cs b/src/DbEx/Migration/Database.cs index 00fa13d..11b8492 100644 --- a/src/DbEx/Migration/Database.cs +++ b/src/DbEx/Migration/Database.cs @@ -100,6 +100,9 @@ await SqlStatement(await ReadSqlAsync(migration, sr, cancellationToken).Configur if (tables.Count == 0) return tables; + // Load the native type information specific to the database provider; must occur before anything that may access the .NET type (as it is cached). + await migration.SchemaConfig.LoadNativeTypesSchema(this, tables, cancellationToken).ConfigureAwait(false); + // Configure all the single column primary and unique constraints. using var sr2 = DatabaseMigrationBase.GetRequiredResourcesStreamReader($"SelectTablePrimaryKey.{migration.SchemaConfig.ScriptSuffix}", probeAssemblies); var pks = await SqlStatement(await ReadSqlAsync(migration, sr2, cancellationToken).ConfigureAwait(false)).SelectQueryAsync(dr => new diff --git a/src/DbEx/Templates/EfModel_cs.hbs b/src/DbEx/Templates/EfModel_cs.hbs index 686a980..68b191f 100644 --- a/src/DbEx/Templates/EfModel_cs.hbs +++ b/src/DbEx/Templates/EfModel_cs.hbs @@ -19,7 +19,7 @@ public partial class {{EfModelName}} {{#if DbColumn.IsPrimaryKey}} /// This is {{#ifeq Parent.DbTable.PrimaryKeyColumns.Count 1}}the primary key{{else}}part of the primary key{{/ifeq}}. {{/if}} - public {{Type}} {{Property}} { get; set; }{{#ifval Default}} = {{Default}};{{else}}{{#unless DbColumn.IsNullable}}{{#ifeq DbColumn.DotNetType "string"}} = default!;{{/ifeq}}{{/unless}}{{/ifval}} + public {{Type}} {{Property}} { get; set; }{{#ifval Default}} = {{Default}};{{else}}{{#unless DbColumn.IsNullable}}{{#if DbColumn.IsDotNetTypeAClass}} = default!;{{/if}}{{/unless}}{{/ifval}} {{/each}} } diff --git a/tests/DbEx.Test.Console/Data/Data.yaml b/tests/DbEx.Test.Console/Data/Data.yaml index 77d6a2f..1638859 100644 --- a/tests/DbEx.Test.Console/Data/Data.yaml +++ b/tests/DbEx.Test.Console/Data/Data.yaml @@ -15,4 +15,8 @@ - { Code: A, Text: Active, TenantId: ^88 } - { Code: I, Text: Inactive, TenantId: ^88 } - { Code: A, Text: Active, TenantId: ^99 } - - { Code: I, Text: Inactive, TenantId: ^99 } \ No newline at end of file + - { Code: I, Text: Inactive, TenantId: ^99 } +- ExtraTypes: + - { ExtraTypesId: 1, Location: "POINT(-122.349 47.651)", Shape: "POLYGON((0 0, 4 0, 4 4, 0 4, 0 0))", Path: "/1/2/", Document: "1", Embedding: "[1.5, 2.5, 3.5]", Variant: "variant", Payload: '{"a":1}' } + - { ExtraTypesId: 2, Location: "SRID=4326;POINT(-122.349 47.651)" } + - { ExtraTypesId: 3 } \ No newline at end of file diff --git a/tests/DbEx.Test.Console/Data/Data2.yaml b/tests/DbEx.Test.Console/Data/Data2.yaml index 993bb39..e3142e4 100644 --- a/tests/DbEx.Test.Console/Data/Data2.yaml +++ b/tests/DbEx.Test.Console/Data/Data2.yaml @@ -16,4 +16,8 @@ TenantId: ^99 - Code: I Text: Inactive - TenantId: ^99 \ No newline at end of file + TenantId: ^99 +# Specified again (and one new) via merge; the non-comparable columns must neither error nor be duplicated. +- $ExtraTypes: + - { ExtraTypesId: 1, Location: "POINT(-122.349 47.651)", Shape: "POLYGON((0 0, 4 0, 4 4, 0 4, 0 0))", Path: "/1/2/", Document: "1", Embedding: "[1.5, 2.5, 3.5]", Variant: "variant", Payload: '{"a":1}' } + - { ExtraTypesId: 4, Location: "POINT(1 2)", Path: "/3/" } \ No newline at end of file diff --git a/tests/DbEx.Test.Console/Migrations/009-create-test-extratypes-table.sql b/tests/DbEx.Test.Console/Migrations/009-create-test-extratypes-table.sql new file mode 100644 index 0000000..f1f061a --- /dev/null +++ b/tests/DbEx.Test.Console/Migrations/009-create-test-extratypes-table.sql @@ -0,0 +1,16 @@ +-- Create table: [Test].[ExtraTypes] - the "extra" (non-primitive) types; the same shape is used for all providers. + +BEGIN TRANSACTION + +CREATE TABLE [Test].[ExtraTypes] ( + [ExtraTypesId] INT NOT NULL PRIMARY KEY, + [Location] GEOGRAPHY NULL, + [Shape] GEOMETRY NULL, + [Path] HIERARCHYID NULL, + [Document] XML NULL, + [Embedding] VECTOR(3) NULL, + [Variant] SQL_VARIANT NULL, + [Payload] JSON NULL +) + +COMMIT TRANSACTION diff --git a/tests/DbEx.Test.Console/dbex.yaml b/tests/DbEx.Test.Console/dbex.yaml index d40c52a..2acc484 100644 --- a/tests/DbEx.Test.Console/dbex.yaml +++ b/tests/DbEx.Test.Console/dbex.yaml @@ -4,6 +4,7 @@ schema: Test outbox: true tables: - name: Contact +- name: ExtraTypes - name: Person columns: - name: AddressJson diff --git a/tests/DbEx.Test.Empty/DbEx.Test.Empty.csproj b/tests/DbEx.Test.Empty/DbEx.Test.Empty.csproj index 1ea5b6c..3f8878f 100644 --- a/tests/DbEx.Test.Empty/DbEx.Test.Empty.csproj +++ b/tests/DbEx.Test.Empty/DbEx.Test.Empty.csproj @@ -9,18 +9,26 @@ 10.0.5 + + 9.0.14 + + + 8.0.25 + + + diff --git a/tests/DbEx.Test.Empty/GlobalUsing.cs b/tests/DbEx.Test.Empty/GlobalUsing.cs index c32e607..288559f 100644 --- a/tests/DbEx.Test.Empty/GlobalUsing.cs +++ b/tests/DbEx.Test.Empty/GlobalUsing.cs @@ -1,2 +1,4 @@ global using System; -global using Microsoft.EntityFrameworkCore; \ No newline at end of file +global using Microsoft.Data.SqlTypes; +global using Microsoft.EntityFrameworkCore; +global using NetTopologySuite.Geometries; \ No newline at end of file diff --git a/tests/DbEx.Test.Empty/Persistence/ExtraTypes.g.cs b/tests/DbEx.Test.Empty/Persistence/ExtraTypes.g.cs new file mode 100644 index 0000000..f33f93a --- /dev/null +++ b/tests/DbEx.Test.Empty/Persistence/ExtraTypes.g.cs @@ -0,0 +1,40 @@ +// + +/* + * This file is automatically generated by 'DbEx.Test.Console'; any changes will be lost. + */ + +#nullable enable + +namespace DbEx.Test.Empty.Persistence; + +/// Persistence model representing the '[Test].[ExtraTypes]' database table. +public partial class ExtraTypes +{ + /// Gets or sets the value of the 'ExtraTypesId' column (INT). + /// This is the primary key. + public int ExtraTypesId { get; set; } + + /// Gets or sets the value of the 'Location' column (GEOGRAPHY NULL). + public Geometry? Location { get; set; } + + /// Gets or sets the value of the 'Shape' column (GEOMETRY NULL). + public Geometry? Shape { get; set; } + + /// Gets or sets the value of the 'Path' column (HIERARCHYID NULL). + public HierarchyId? Path { get; set; } + + /// Gets or sets the value of the 'Document' column (XML NULL). + public string? Document { get; set; } + + /// Gets or sets the value of the 'Embedding' column (VECTOR(3) NULL). + public SqlVector? Embedding { get; set; } + + /// Gets or sets the value of the 'Variant' column (SQL_VARIANT NULL). + public object? Variant { get; set; } + + /// Gets or sets the value of the 'Payload' column (JSON NULL). + public string? Payload { get; set; } +} + +#nullable restore \ No newline at end of file diff --git a/tests/DbEx.Test.Empty/Repositories/TestDbContext.g.cs b/tests/DbEx.Test.Empty/Repositories/TestDbContext.g.cs index 5ea3128..a3e1c25 100644 --- a/tests/DbEx.Test.Empty/Repositories/TestDbContext.g.cs +++ b/tests/DbEx.Test.Empty/Repositories/TestDbContext.g.cs @@ -30,6 +30,21 @@ public void AddGeneratedModels(Microsoft.EntityFrameworkCore.ModelBuilder modelB e.Property(p => p.ContactTypeCode).HasColumnName("ContactTypeCode").HasColumnType("NVARCHAR(50)"); }); + // Add the entity/model configuration for the [Test].[ExtraTypes] database table. + modelBuilder.Entity(e => + { + e.ToTable("ExtraTypes", "Test"); + e.HasKey(p => p.ExtraTypesId); + e.Property(p => p.ExtraTypesId).HasColumnName("ExtraTypesId").HasColumnType("INT"); + e.Property(p => p.Location).HasColumnName("Location").HasColumnType("GEOGRAPHY"); + e.Property(p => p.Shape).HasColumnName("Shape").HasColumnType("GEOMETRY"); + e.Property(p => p.Path).HasColumnName("Path").HasColumnType("HIERARCHYID"); + e.Property(p => p.Document).HasColumnName("Document").HasColumnType("XML"); + e.Property(p => p.Embedding).HasColumnName("Embedding").HasColumnType("VECTOR(3)"); + e.Property(p => p.Variant).HasColumnName("Variant").HasColumnType("SQL_VARIANT"); + e.Property(p => p.Payload).HasColumnName("Payload").HasColumnType("JSON"); + }); + // Add the entity/model configuration for the [Test].[Person] database table. modelBuilder.Entity(e => { diff --git a/tests/DbEx.Test.MySqlConsole/Data/Data.yaml b/tests/DbEx.Test.MySqlConsole/Data/Data.yaml index 4d76bd4..9612f08 100644 --- a/tests/DbEx.Test.MySqlConsole/Data/Data.yaml +++ b/tests/DbEx.Test.MySqlConsole/Data/Data.yaml @@ -8,4 +8,7 @@ - O: Other - contact: - { contact_id: 1, contact_type: E, gender: M, name: Bob, date_of_birth: 2001-10-22 } - - { contact_id: 2, contact_type: I, name: Jane, phone: 1234 } \ No newline at end of file + - { contact_id: 2, contact_type: I, name: Jane, phone: 1234 } +- extra_types: + - { extra_types_id: 1, medium_value: 100000, year_value: 2024, bit_flag: true, bit_mask: "10101010", enum_value: a, set_value: "x,z", geom: "POINT(1 2)", pt: "POINT(1 2)", ln: "LINESTRING(0 0, 1 1)", poly: "POLYGON((0 0, 4 0, 4 4, 0 4, 0 0))", multi_pt: "MULTIPOINT((0 0), (1 1))", multi_ln: "MULTILINESTRING((0 0, 1 1), (2 2, 3 3))", multi_poly: "MULTIPOLYGON(((0 0, 4 0, 4 4, 0 4, 0 0)))", geom_coll: "GEOMETRYCOLLECTION(POINT(1 2))", geog: "SRID=4326;POINT(10 20)", embedding: "[1, 2, 3]" } + - { extra_types_id: 2 } \ No newline at end of file diff --git a/tests/DbEx.Test.MySqlConsole/DbEx.Test.MySqlConsole.csproj b/tests/DbEx.Test.MySqlConsole/DbEx.Test.MySqlConsole.csproj index 6e67d1c..8574867 100644 --- a/tests/DbEx.Test.MySqlConsole/DbEx.Test.MySqlConsole.csproj +++ b/tests/DbEx.Test.MySqlConsole/DbEx.Test.MySqlConsole.csproj @@ -17,6 +17,7 @@ + diff --git a/tests/DbEx.Test.MySqlConsole/Migrations/006-create-test-extratypes-table.mysql b/tests/DbEx.Test.MySqlConsole/Migrations/006-create-test-extratypes-table.mysql new file mode 100644 index 0000000..93b1228 --- /dev/null +++ b/tests/DbEx.Test.MySqlConsole/Migrations/006-create-test-extratypes-table.mysql @@ -0,0 +1,19 @@ +CREATE TABLE `extra_types` ( + `extra_types_id` INT NOT NULL PRIMARY KEY, + `medium_value` MEDIUMINT NULL, + `year_value` YEAR NULL, + `bit_flag` BIT(1) NULL, + `bit_mask` BIT(8) NULL, + `enum_value` ENUM('a', 'b') NULL, + `set_value` SET('x', 'y', 'z') NULL, + `geom` GEOMETRY NULL, + `pt` POINT NULL, + `ln` LINESTRING NULL, + `poly` POLYGON NULL, + `multi_pt` MULTIPOINT NULL, + `multi_ln` MULTILINESTRING NULL, + `multi_poly` MULTIPOLYGON NULL, + `geom_coll` GEOMETRYCOLLECTION NULL, + `geog` POINT SRID 4326 NULL, + `embedding` VECTOR(3) NULL +) diff --git a/tests/DbEx.Test.PostgresConsole/Data/Data.yaml b/tests/DbEx.Test.PostgresConsole/Data/Data.yaml index 6f958c0..9341815 100644 --- a/tests/DbEx.Test.PostgresConsole/Data/Data.yaml +++ b/tests/DbEx.Test.PostgresConsole/Data/Data.yaml @@ -8,4 +8,7 @@ - O: Other - contact: - { contact_id: 1, contact_type: E, gender: M, name: Bob, date_of_birth: 2001-10-22 } - - { contact_id: 2, contact_type: I, name: Jane, phone: 1234 } \ No newline at end of file + - { contact_id: 2, contact_type: I, name: Jane, phone: 1234 } +- extra_types: + - { extra_types_id: 1, ip_address: "192.168.1.10", network: "10.1.0.0/16", mac_address: "08:00:2b:01:02:03", mac_address8: "08:00:2b:01:02:03:04:05", search_vector: "'fat':2 'cat':3", search_query: "'fat' & 'rat'", pt: "(1,2)", ln: "{1,2,3}", seg: "[(0,0),(1,1)]", bx: "(1,1),(0,0)", pth: "[(0,0),(1,1),(2,0)]", poly: "((0,0),(4,0),(4,4))", circ: "<(0,0),5>", bit_flag: true, bit_mask: "10101010", bit_varying: "101", tags: "{a,b,c}", numbers: "{1,2,3}", current_mood: happy, moods: "{sad,ok}", int_range: "[1,10)", ts_range: "[2024-01-01 00:00:00+00,2024-02-01 00:00:00+00)", date_range: "[2024-01-01,2024-02-01)", int_multirange: "{[1,3),[5,8)}", attributes: "a=>1, b=>2", label: "Top.Science.Astronomy", ci_text: "Hello", geom: "SRID=4326;POINT(1 2)", geom_any: "POLYGON((0 0, 4 0, 4 4, 0 0))", geog: "POINT(-122.349 47.651)", embedding: "[1,2,3]", half_embedding: "[1,2,3]", sparse_embedding: "{1:1,3:2}/3" } + - { extra_types_id: 2 } \ No newline at end of file diff --git a/tests/DbEx.Test.PostgresConsole/DbEx.Test.PostgresConsole.csproj b/tests/DbEx.Test.PostgresConsole/DbEx.Test.PostgresConsole.csproj index 091815c..a527fc5 100644 --- a/tests/DbEx.Test.PostgresConsole/DbEx.Test.PostgresConsole.csproj +++ b/tests/DbEx.Test.PostgresConsole/DbEx.Test.PostgresConsole.csproj @@ -16,6 +16,7 @@ + diff --git a/tests/DbEx.Test.PostgresConsole/Migrations/006-create-test-extratypes-table.pgsql b/tests/DbEx.Test.PostgresConsole/Migrations/006-create-test-extratypes-table.pgsql new file mode 100644 index 0000000..379ab8f --- /dev/null +++ b/tests/DbEx.Test.PostgresConsole/Migrations/006-create-test-extratypes-table.pgsql @@ -0,0 +1,44 @@ +CREATE EXTENSION IF NOT EXISTS postgis; +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS hstore; +CREATE EXTENSION IF NOT EXISTS ltree; +CREATE EXTENSION IF NOT EXISTS citext; + +CREATE TYPE "public"."mood" AS ENUM ('sad', 'ok', 'happy'); + +CREATE TABLE "public"."extra_types" ( + "extra_types_id" INT NOT NULL PRIMARY KEY, + "ip_address" INET NULL, + "network" CIDR NULL, + "mac_address" MACADDR NULL, + "mac_address8" MACADDR8 NULL, + "search_vector" TSVECTOR NULL, + "search_query" TSQUERY NULL, + "pt" POINT NULL, + "ln" LINE NULL, + "seg" LSEG NULL, + "bx" BOX NULL, + "pth" PATH NULL, + "poly" POLYGON NULL, + "circ" CIRCLE NULL, + "bit_flag" BIT(1) NULL, + "bit_mask" BIT(8) NULL, + "bit_varying" VARBIT(16) NULL, + "tags" TEXT[] NULL, + "numbers" INT[] NULL, + "current_mood" "public"."mood" NULL, + "moods" "public"."mood"[] NULL, + "int_range" INT4RANGE NULL, + "ts_range" TSTZRANGE NULL, + "date_range" DATERANGE NULL, + "int_multirange" INT4MULTIRANGE NULL, + "attributes" HSTORE NULL, + "label" LTREE NULL, + "ci_text" CITEXT NULL, + "geom" GEOMETRY(Point, 4326) NULL, + "geom_any" GEOMETRY NULL, + "geog" GEOGRAPHY NULL, + "embedding" VECTOR(3) NULL, + "half_embedding" HALFVEC(3) NULL, + "sparse_embedding" SPARSEVEC(3) NULL +) diff --git a/tests/DbEx.Test/DatabaseSchemaTest.cs b/tests/DbEx.Test/DatabaseSchemaTest.cs index de1c70d..fba98b7 100644 --- a/tests/DbEx.Test/DatabaseSchemaTest.cs +++ b/tests/DbEx.Test/DatabaseSchemaTest.cs @@ -1,4 +1,5 @@ using DbEx.Migration; +using DbEx.DbSchema; using DbEx.MySql.Migration; using DbEx.Postgres.Migration; using DbEx.SqlServer.Migration; @@ -7,6 +8,8 @@ using Microsoft.Extensions.Configuration; using MySql.Data.MySqlClient; using NUnit.Framework; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Assert = NUnit.Framework.Legacy.ClassicAssert; @@ -354,6 +357,20 @@ public async Task SqlServerSelectSchema() Assert.IsNull(col.ForeignTable); Assert.IsNull(col.ForeignColumn); Assert.IsNotNull(col.DefaultValue); + + // [Test].[ExtraTypes] - the .NET (EF) type is the "true" type; text is used for data parsing. + AssertExtraTypes(tables, "ExtraTypes", + [ + ("Location", "geography", "Geometry", true, "string"), + ("Shape", "geometry", "Geometry", true, "string"), + ("Path", "hierarchyid", "HierarchyId", true, "string"), + ("Document", "xml", "string", true, "string"), + ("Embedding", "vector", "SqlVector", false, "string"), + ("Variant", "sql_variant", "object", true, "string"), + ("Payload", "json", "string", true, "string"), + ]); + + Assert.AreEqual("VECTOR(3) NULL", tables.Single(x => x.Name == "ExtraTypes").Columns.Single(x => x.Name == "Embedding").SqlType); } [Test] @@ -656,6 +673,26 @@ public async Task MySqlSelectSchema() Assert.IsNull(col.ForeignTable); Assert.IsNull(col.ForeignColumn); Assert.IsNull(col.DefaultValue); + + AssertExtraTypes(tables, "extra_types", + [ + ("medium_value", "mediumint", "int", false, "int"), + ("year_value", "year", "short", false, "short"), + ("bit_flag", "bit", "bool", false, "bool"), + ("bit_mask", "bit", "ulong", false, "string"), + ("enum_value", "enum", "string", true, "string"), + ("set_value", "set", "string", true, "string"), + ("geom", "geometry", "Geometry", true, "string"), + ("pt", "point", "Point", true, "string"), + ("ln", "linestring", "LineString", true, "string"), + ("poly", "polygon", "Polygon", true, "string"), + ("multi_pt", "multipoint", "MultiPoint", true, "string"), + ("multi_ln", "multilinestring", "MultiLineString", true, "string"), + ("multi_poly", "multipolygon", "MultiPolygon", true, "string"), + ("geom_coll", "geomcollection", "GeometryCollection", true, "string"), + ("geog", "point", "Point", true, "string"), + ("embedding", "vector", "byte[]", true, "string"), + ]); } [Test] @@ -982,6 +1019,71 @@ public async Task PostgresSelectSchema() Assert.IsNull(col.ForeignTable); Assert.IsNull(col.ForeignColumn); Assert.IsNull(col.DefaultValue); + + AssertExtraTypes(tables, "extra_types", + [ + ("ip_address", "inet", "IPAddress", true, "string"), + ("network", "cidr", "NpgsqlCidr", false, "string"), + ("mac_address", "macaddr", "PhysicalAddress", true, "string"), + ("mac_address8", "macaddr8", "PhysicalAddress", true, "string"), + ("search_vector", "tsvector", "NpgsqlTsVector", true, "string"), + ("search_query", "tsquery", "NpgsqlTsQuery", true, "string"), + ("pt", "point", "NpgsqlPoint", false, "string"), + ("ln", "line", "NpgsqlLine", false, "string"), + ("seg", "lseg", "NpgsqlLSeg", false, "string"), + ("bx", "box", "NpgsqlBox", false, "string"), + ("pth", "path", "NpgsqlPath", false, "string"), + ("poly", "polygon", "NpgsqlPolygon", false, "string"), + ("circ", "circle", "NpgsqlCircle", false, "string"), + ("bit_flag", "bit", "bool", false, "bool"), + ("bit_mask", "bit", "BitArray", true, "string"), + ("bit_varying", "bit varying", "BitArray", true, "string"), + ("tags", "ARRAY", "string[]", true, "string"), + ("numbers", "ARRAY", "int[]", true, "string"), + ("current_mood", "USER-DEFINED", "string", true, "string"), + ("moods", "ARRAY", "string[]", true, "string"), + ("int_range", "int4range", "NpgsqlRange", false, "string"), + ("ts_range", "tstzrange", "NpgsqlRange", false, "string"), + ("date_range", "daterange", "NpgsqlRange", false, "string"), + ("int_multirange", "int4multirange", "NpgsqlRange[]", true, "string"), + ("attributes", "USER-DEFINED", "Dictionary", true, "string"), + ("label", "USER-DEFINED", "string", true, "string"), + ("ci_text", "USER-DEFINED", "string", true, "string"), + ("geom", "USER-DEFINED", "Point", true, "string"), + ("geom_any", "USER-DEFINED", "Geometry", true, "string"), + ("geog", "USER-DEFINED", "Geometry", true, "string"), + ("embedding", "USER-DEFINED", "Vector", true, "string"), + ("half_embedding", "USER-DEFINED", "HalfVector", true, "string"), + ("sparse_embedding", "USER-DEFINED", "SparseVector", true, "string"), + ]); + } + + /// + /// Asserts the extra (non-primitive) types for the specified table; all mismatches are reported together. + /// + private static void AssertExtraTypes(List tables, string tableName, (string Column, string Type, string DotNetType, bool IsClass, string DataParserType)[] expected) + { + var tab = tables.SingleOrDefault(x => x.Name == tableName); + Assert.IsNotNull(tab, $"Table '{tableName}' not found."); + + var errors = new List(); + foreach (var e in expected) + { + var col = tab.Columns.SingleOrDefault(x => x.Name == e.Column); + if (col is null) + { + errors.Add($"{e.Column}: column not found."); + continue; + } + + var actual = $"{col.Type}|{col.DotNetType}|{col.IsDotNetTypeAClass}|{col.DataParserType}|{col.IsNullable}"; + var expect = $"{e.Type}|{e.DotNetType}|{e.IsClass}|{e.DataParserType}|True"; + if (!string.Equals(actual, expect, StringComparison.OrdinalIgnoreCase)) + errors.Add($"{e.Column}: expected '{expect}' but was '{actual}' (SqlType: '{col.SqlType}', Native: '{col.NativeSqlType}')."); + } + + Assert.AreEqual(expected.Length + 1, tab.Columns.Count(x => !x.IsRowVersion), "Column count (including the primary key) is unexpected."); + Assert.IsEmpty(errors, string.Join(Environment.NewLine, errors)); } } } \ No newline at end of file diff --git a/tests/DbEx.Test/ExtraTypesReader.cs b/tests/DbEx.Test/ExtraTypesReader.cs new file mode 100644 index 0000000..6b8954b --- /dev/null +++ b/tests/DbEx.Test/ExtraTypesReader.cs @@ -0,0 +1,56 @@ +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Globalization; +using System.Linq; +using System.Threading.Tasks; + +namespace DbEx.Test +{ + /// + /// Reads back (as text) the extra types rows loaded by the migration to verify the data load round trip. + /// + internal static class ExtraTypesReader + { + public static async Task>> ReadAsync(DbConnection connection, string sql) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = sql; + + using var dr = await cmd.ExecuteReaderAsync().ConfigureAwait(false); + var rows = new List>(); + while (await dr.ReadAsync().ConfigureAwait(false)) + { + var row = new Dictionary(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < dr.FieldCount; i++) + row[dr.GetName(i)] = dr.IsDBNull(i) ? null : Convert.ToString(dr.GetValue(i), CultureInfo.InvariantCulture); + + rows.Add(row); + } + + return rows; + } + + /// + /// Asserts the first row matches exactly, and that the second row only has the identifier value (all other columns null). + /// + public static void AssertRows(List> rows, string idColumn, Dictionary expected) + { + Assert.That(rows, Has.Count.EqualTo(2)); + + var errors = new List(); + foreach (var kv in expected.Where(x => rows[0].GetValueOrDefault(x.Key) != x.Value || !rows[0].ContainsKey(x.Key))) + errors.Add($"row 1 '{kv.Key}': expected '{kv.Value}' but was '{(rows[0].TryGetValue(kv.Key, out var v) ? v : "")}'"); + + foreach (var kv in rows[0].Where(x => !expected.ContainsKey(x.Key))) + errors.Add($"row 1 '{kv.Key}': unexpected column (add to expected)."); + + Assert.That(rows[1][idColumn], Is.EqualTo("2")); + foreach (var kv in rows[1].Where(x => x.Value is not null && !string.Equals(x.Key, idColumn, StringComparison.OrdinalIgnoreCase))) + errors.Add($"row 2 '{kv.Key}': expected null but was '{kv.Value}'"); + + Assert.That(errors, Is.Empty, string.Join(Environment.NewLine, errors)); + } + } +} diff --git a/tests/DbEx.Test/MarkdownAssert.cs b/tests/DbEx.Test/MarkdownAssert.cs new file mode 100644 index 0000000..a6fc57c --- /dev/null +++ b/tests/DbEx.Test/MarkdownAssert.cs @@ -0,0 +1,14 @@ +using NUnit.Framework; + +namespace DbEx.Test +{ + /// + /// Compares markdown text independent of the platform line endings (CRLF versus LF) and leading/trailing whitespace. + /// + internal static class MarkdownAssert + { + public static void AreEqual(string expected, string actual) => Assert.That(Normalize(actual), Is.EqualTo(Normalize(expected))); + + private static string Normalize(string text) => text.Replace("\r\n", "\n").Replace('\r', '\n').Trim(); + } +} diff --git a/tests/DbEx.Test/MySqlMigrationTest.cs b/tests/DbEx.Test/MySqlMigrationTest.cs index 5e5c33b..d6aa50e 100644 --- a/tests/DbEx.Test/MySqlMigrationTest.cs +++ b/tests/DbEx.Test/MySqlMigrationTest.cs @@ -2,7 +2,9 @@ using DbEx.MySql.Migration; using DbEx.Postgres.Migration; using Microsoft.Extensions.Configuration; +using MySql.Data.MySqlClient; using NUnit.Framework; +using System.Collections.Generic; using System.Threading.Tasks; using Assert = NUnit.Framework.Legacy.ClassicAssert; @@ -23,6 +25,47 @@ public async Task A120_MigrateAll() Assert.IsTrue(r); } + [Test] + public async Task A125_MigrateAll_ExtraTypes_RoundTrip() + { + await A120_MigrateAll(); + + // Read everything back as text; the bit, spatial and vector types are read via functions as their default text form is binary. + const string sql = "SELECT CAST(extra_types_id AS CHAR) AS extra_types_id, CAST(medium_value AS CHAR) AS medium_value, CAST(year_value AS CHAR) AS year_value, " + + "CAST(bit_flag + 0 AS CHAR) AS bit_flag, BIN(bit_mask) AS bit_mask, enum_value, set_value, " + + "ST_AsText(geom) AS geom, ST_AsText(pt) AS pt, ST_AsText(ln) AS ln, ST_AsText(poly) AS poly, ST_AsText(multi_pt) AS multi_pt, ST_AsText(multi_ln) AS multi_ln, " + + "ST_AsText(multi_poly) AS multi_poly, ST_AsText(geom_coll) AS geom_coll, ST_AsText(geog, 'axis-order=long-lat') AS geog, CAST(ST_SRID(geog) AS CHAR) AS geog_srid, " + + "VECTOR_TO_STRING(embedding) AS embedding FROM extra_types ORDER BY extra_types_id"; + + var cs = UnitTest.GetConfig("DbEx_").GetConnectionString("MySqlDb"); + using var conn = new MySqlConnection(cs); + await conn.OpenAsync(); + + var rows = await ExtraTypesReader.ReadAsync(conn, sql); + + ExtraTypesReader.AssertRows(rows, "extra_types_id", new Dictionary + { + ["extra_types_id"] = "1", + ["medium_value"] = "100000", + ["year_value"] = "2024", + ["bit_flag"] = "1", + ["bit_mask"] = "10101010", + ["enum_value"] = "a", + ["set_value"] = "x,z", + ["geom"] = "POINT(1 2)", + ["pt"] = "POINT(1 2)", + ["ln"] = "LINESTRING(0 0,1 1)", + ["poly"] = "POLYGON((0 0,4 0,4 4,0 4,0 0))", + ["multi_pt"] = "MULTIPOINT((0 0),(1 1))", + ["multi_ln"] = "MULTILINESTRING((0 0,1 1),(2 2,3 3))", + ["multi_poly"] = "MULTIPOLYGON(((0 0,4 0,4 4,0 4,0 0)))", + ["geom_coll"] = "GEOMETRYCOLLECTION(POINT(1 2))", + ["geog"] = "POINT(10 20)", + ["geog_srid"] = "4326", + ["embedding"] = "[1.00000e+00,2.00000e+00,3.00000e+00]" + }); + } + [Test] public async Task A120_MigrateReset() { @@ -51,6 +94,7 @@ public async Task MySqlInspect() a.Parameters.Add("Param0", "unknown"); a.Parameters.Add("Param1", "gender"); a.Parameters.Add("Param2", "CONTACT"); + a.Parameters.Add("Param3", "extra_types"); using var m = new MySqlMigration(a); var (Success, Output) = await m.MigrateAndLogAsync().ConfigureAwait(false); @@ -59,7 +103,7 @@ public async Task MySqlInspect() Assert.IsTrue(Output.Length > 0); using var sr = MySqlMigration.GetRequiredResourcesStreamReader("MySqlInspect.md", [typeof(MySqlMigrationTest).Assembly]); - Assert.AreEqual(sr.ReadToEnd(), Output); + MarkdownAssert.AreEqual(sr.ReadToEnd(), Output); } } } \ No newline at end of file diff --git a/tests/DbEx.Test/PostgresMigrationTest.cs b/tests/DbEx.Test/PostgresMigrationTest.cs index c024599..7f81062 100644 --- a/tests/DbEx.Test/PostgresMigrationTest.cs +++ b/tests/DbEx.Test/PostgresMigrationTest.cs @@ -8,6 +8,8 @@ using NUnit.Framework; using System; using System.Data.Common; +using System.Linq; +using System.Collections.Generic; using System.Threading.Tasks; using Assert = NUnit.Framework.Legacy.ClassicAssert; @@ -28,6 +30,74 @@ public async Task A120_MigrateAll() Assert.IsTrue(r); } + [Test] + public async Task A125_MigrateAll_ExtraTypes_RoundTrip() + { + await A120_MigrateAll(); + + // Read everything back as text; the spatial types are read as WKT (plus SRID) as their default text form is the binary hex. + var overrides = new Dictionary + { + ["geom"] = "ST_AsText(\"geom\")", + ["geom_any"] = "ST_AsText(\"geom_any\")", + ["geog"] = "ST_AsText(\"geog\")" + }; + + var columns = new[] + { + "extra_types_id", "ip_address", "network", "mac_address", "mac_address8", "search_vector", "search_query", "pt", "ln", "seg", "bx", "pth", "poly", "circ", + "bit_flag", "bit_mask", "bit_varying", "tags", "numbers", "current_mood", "moods", "int_range", "ts_range", "date_range", "int_multirange", + "attributes", "label", "ci_text", "geom", "geom_any", "geog", "embedding", "half_embedding", "sparse_embedding" + }; + + var sql = $"SELECT {string.Join(", ", columns.Select(c => $"{(overrides.TryGetValue(c, out var e) ? e : $"\"{c}\"::text")} AS \"{c}\""))}, ST_SRID(\"geom\") AS \"geom_srid\" FROM \"public\".\"extra_types\" ORDER BY \"extra_types_id\""; + + var cs = UnitTest.GetConfig("DbEx_").GetConnectionString("PostgresDb"); + using var conn = new NpgsqlConnection(cs); + await conn.OpenAsync(); + + var rows = await ExtraTypesReader.ReadAsync(conn, sql); + + ExtraTypesReader.AssertRows(rows, "extra_types_id", new Dictionary + { + ["extra_types_id"] = "1", + ["ip_address"] = "192.168.1.10/32", + ["network"] = "10.1.0.0/16", + ["mac_address"] = "08:00:2b:01:02:03", + ["mac_address8"] = "08:00:2b:01:02:03:04:05", + ["search_vector"] = "'cat':3 'fat':2", + ["search_query"] = "'fat' & 'rat'", + ["pt"] = "(1,2)", + ["ln"] = "{1,2,3}", + ["seg"] = "[(0,0),(1,1)]", + ["bx"] = "(1,1),(0,0)", + ["pth"] = "[(0,0),(1,1),(2,0)]", + ["poly"] = "((0,0),(4,0),(4,4))", + ["circ"] = "<(0,0),5>", + ["bit_flag"] = "1", + ["bit_mask"] = "10101010", + ["bit_varying"] = "101", + ["tags"] = "{a,b,c}", + ["numbers"] = "{1,2,3}", + ["current_mood"] = "happy", + ["moods"] = "{sad,ok}", + ["int_range"] = "[1,10)", + ["ts_range"] = "[\"2024-01-01 00:00:00+00\",\"2024-02-01 00:00:00+00\")", + ["date_range"] = "[2024-01-01,2024-02-01)", + ["int_multirange"] = "{[1,3),[5,8)}", + ["attributes"] = "\"a\"=>\"1\", \"b\"=>\"2\"", + ["label"] = "Top.Science.Astronomy", + ["ci_text"] = "Hello", + ["geom"] = "POINT(1 2)", + ["geom_any"] = "POLYGON((0 0,4 0,4 4,0 0))", + ["geog"] = "POINT(-122.349 47.651)", + ["embedding"] = "[1,2,3]", + ["half_embedding"] = "[1,2,3]", + ["sparse_embedding"] = "{1:1,3:2}/3", + ["geom_srid"] = "4326" + }); + } + [Test] public async Task A120_MigrateReset() { @@ -109,6 +179,7 @@ public async Task PostgresInspect() a.Parameters.Add("Param1", "unknown"); a.Parameters.Add("Param2", "gender"); a.Parameters.Add("Param3", "CONTACT"); + a.Parameters.Add("Param4", "extra_types"); using var m = new PostgresMigration(a); var (Success, Output) = await m.MigrateAndLogAsync().ConfigureAwait(false); @@ -117,7 +188,7 @@ public async Task PostgresInspect() Assert.IsTrue(Output.Length > 0); using var sr = PostgresMigration.GetRequiredResourcesStreamReader("PostgresInspect.md", [typeof(PostgresMigrationTest).Assembly]); - Assert.AreEqual(sr.ReadToEnd(), Output); + MarkdownAssert.AreEqual(sr.ReadToEnd(), Output); } } } diff --git a/tests/DbEx.Test/Resources/MySqlInspect.md b/tests/DbEx.Test/Resources/MySqlInspect.md index 19f00e3..6449924 100644 --- a/tests/DbEx.Test/Resources/MySqlInspect.md +++ b/tests/DbEx.Test/Resources/MySqlInspect.md @@ -51,3 +51,33 @@ This command is intended to be used as a quick-and-easy way to inspect the infer | updated_on | DATETIME | Yes | | No | No | No | No | No | | contact_type_code | VARCHAR(50) | Yes | | No | No | No | No | No | +## EXTRA_TYPES - Exists: Yes + +- Schema: +- Name: extra_types +- Qualified Name: `extra_types` +- Table or View: Table +- Reference Data: No + +### Columns + +| Column | Type | Null | Default | PK | Identity | Computed | Unique | JSON | +|----------------|-----------------|------|---------|-----|----------|----------|--------|------| +| extra_types_id | INT | No | | Yes | No | No | No | No | +| medium_value | MEDIUMINT | Yes | | No | No | No | No | No | +| year_value | YEAR | Yes | | No | No | No | No | No | +| bit_flag | BIT(1) | Yes | | No | No | No | No | No | +| bit_mask | BIT(8) | Yes | | No | No | No | No | No | +| enum_value | ENUM | Yes | | No | No | No | No | No | +| set_value | SET | Yes | | No | No | No | No | No | +| geom | GEOMETRY | Yes | | No | No | No | No | No | +| pt | POINT | Yes | | No | No | No | No | No | +| ln | LINESTRING | Yes | | No | No | No | No | No | +| poly | POLYGON | Yes | | No | No | No | No | No | +| multi_pt | MULTIPOINT | Yes | | No | No | No | No | No | +| multi_ln | MULTILINESTRING | Yes | | No | No | No | No | No | +| multi_poly | MULTIPOLYGON | Yes | | No | No | No | No | No | +| geom_coll | GEOMCOLLECTION | Yes | | No | No | No | No | No | +| geog | POINT | Yes | | No | No | No | No | No | +| embedding | VECTOR(3) | Yes | | No | No | No | No | No | + diff --git a/tests/DbEx.Test/Resources/PostgresInspect.md b/tests/DbEx.Test/Resources/PostgresInspect.md index 3a49d18..91fb7fe 100644 --- a/tests/DbEx.Test/Resources/PostgresInspect.md +++ b/tests/DbEx.Test/Resources/PostgresInspect.md @@ -53,3 +53,51 @@ This command is intended to be used as a quick-and-easy way to inspect the infer | contact_type_code | CHARACTER VARYING(50) | Yes | | No | No | No | No | No | | xmin | XID | No | | No | No | Yes | No | No | +## PUBLIC.EXTRA_TYPES - Exists: Yes + +- Schema: public +- Name: extra_types +- Qualified Name: "public"."extra_types" +- Table or View: Table +- Reference Data: No + +### Columns + +| Column | Type | Null | Default | PK | Identity | Computed | Unique | JSON | +|------------------|----------------------|------|---------|-----|----------|----------|--------|------| +| extra_types_id | INTEGER | No | | Yes | No | No | No | No | +| ip_address | INET | Yes | | No | No | No | No | No | +| network | CIDR | Yes | | No | No | No | No | No | +| mac_address | MACADDR | Yes | | No | No | No | No | No | +| mac_address8 | MACADDR8 | Yes | | No | No | No | No | No | +| search_vector | TSVECTOR | Yes | | No | No | No | No | No | +| search_query | TSQUERY | Yes | | No | No | No | No | No | +| pt | POINT | Yes | | No | No | No | No | No | +| ln | LINE | Yes | | No | No | No | No | No | +| seg | LSEG | Yes | | No | No | No | No | No | +| bx | BOX | Yes | | No | No | No | No | No | +| pth | PATH | Yes | | No | No | No | No | No | +| poly | POLYGON | Yes | | No | No | No | No | No | +| circ | CIRCLE | Yes | | No | No | No | No | No | +| bit_flag | BIT(1) | Yes | | No | No | No | No | No | +| bit_mask | BIT(8) | Yes | | No | No | No | No | No | +| bit_varying | BIT VARYING(16) | Yes | | No | No | No | No | No | +| tags | text[] | Yes | | No | No | No | No | No | +| numbers | integer[] | Yes | | No | No | No | No | No | +| current_mood | mood | Yes | | No | No | No | No | No | +| moods | mood[] | Yes | | No | No | No | No | No | +| int_range | INT4RANGE | Yes | | No | No | No | No | No | +| ts_range | TSTZRANGE | Yes | | No | No | No | No | No | +| date_range | DATERANGE | Yes | | No | No | No | No | No | +| int_multirange | INT4MULTIRANGE | Yes | | No | No | No | No | No | +| attributes | hstore | Yes | | No | No | No | No | No | +| label | ltree | Yes | | No | No | No | No | No | +| ci_text | citext | Yes | | No | No | No | No | No | +| geom | geometry(Point,4326) | Yes | | No | No | No | No | No | +| geom_any | geometry | Yes | | No | No | No | No | No | +| geog | geography | Yes | | No | No | No | No | No | +| embedding | vector(3) | Yes | | No | No | No | No | No | +| half_embedding | halfvec(3) | Yes | | No | No | No | No | No | +| sparse_embedding | sparsevec(3) | Yes | | No | No | No | No | No | +| xmin | XID | No | | No | No | Yes | No | No | + diff --git a/tests/DbEx.Test/Resources/SqlServerInspect.md b/tests/DbEx.Test/Resources/SqlServerInspect.md index 330c53f..bc44487 100644 --- a/tests/DbEx.Test/Resources/SqlServerInspect.md +++ b/tests/DbEx.Test/Resources/SqlServerInspect.md @@ -48,3 +48,24 @@ This command is intended to be used as a quick-and-easy way to inspect the infer | Notes | NVARCHAR(MAX) | Yes | | No | No | No | No | No | | ContactTypeCode | NVARCHAR(50) | Yes | | No | No | No | No | No | +## TEST.EXTRATYPES - Exists: Yes + +- Schema: Test +- Name: ExtraTypes +- Qualified Name: [Test].[ExtraTypes] +- Table or View: Table +- Reference Data: No + +### Columns + +| Column | Type | Null | Default | PK | Identity | Computed | Unique | JSON | +|--------------|-------------|------|---------|-----|----------|----------|--------|------| +| ExtraTypesId | INT | No | | Yes | No | No | No | No | +| Location | GEOGRAPHY | Yes | | No | No | No | No | No | +| Shape | GEOMETRY | Yes | | No | No | No | No | No | +| Path | HIERARCHYID | Yes | | No | No | No | No | No | +| Document | XML | Yes | | No | No | No | No | No | +| Embedding | VECTOR(3) | Yes | | No | No | No | No | No | +| Variant | SQL_VARIANT | Yes | | No | No | No | No | No | +| Payload | JSON | Yes | | No | No | No | No | Yes | + diff --git a/tests/DbEx.Test/SqlServerMigrationTest.cs b/tests/DbEx.Test/SqlServerMigrationTest.cs index 86e4b89..ac100de 100644 --- a/tests/DbEx.Test/SqlServerMigrationTest.cs +++ b/tests/DbEx.Test/SqlServerMigrationTest.cs @@ -165,6 +165,56 @@ public async Task A130_MigrateAll_Console() Assert.IsNull(row.GenderId); } + [Test] + public async Task A135_MigrateAll_Console_ExtraTypes_RoundTrip() + { + var (cs, _, _) = await CreateConsoleDb().ConfigureAwait(false); + using var db = new SqlServerDatabase(() => new SqlConnection(cs)); + + async Task AssertRowsAsync() + { + var rows = (await db.SqlStatement( + "SELECT [ExtraTypesId], [Location].STAsText() AS [Location], [Location].STSrid AS [LocationSrid], [Shape].STAsText() AS [Shape], [Path].ToString() AS [Path], " + + "CAST([Document] AS NVARCHAR(MAX)) AS [Document], CAST([Embedding] AS NVARCHAR(MAX)) AS [Embedding], CAST([Variant] AS VARCHAR(50)) AS [Variant], CAST([Payload] AS NVARCHAR(MAX)) AS [Payload] " + + "FROM [Test].[ExtraTypes] ORDER BY [ExtraTypesId]").SelectQueryAsync(dr => new + { + Id = dr.GetValue("ExtraTypesId"), + Location = dr.GetValue("Location"), + LocationSrid = dr.GetValue("LocationSrid"), + Shape = dr.GetValue("Shape"), + Path = dr.GetValue("Path"), + Document = dr.GetValue("Document"), + Embedding = dr.GetValue("Embedding"), + Variant = dr.GetValue("Variant"), + Payload = dr.GetValue("Payload") + }).ConfigureAwait(false)).ToList(); + + Assert.AreEqual(4, rows.Count); + + Assert.AreEqual("POINT (-122.349 47.651)", rows[0].Location); + Assert.AreEqual("POLYGON ((0 0, 4 0, 4 4, 0 4, 0 0))", rows[0].Shape); + Assert.AreEqual("/1/2/", rows[0].Path); + Assert.AreEqual("1", rows[0].Document); + Assert.That(rows[0].Embedding, Does.Contain("1.5")); + Assert.AreEqual("variant", rows[0].Variant); + Assert.That(rows[0].Payload, Does.Contain("\"a\"")); + + Assert.AreEqual(4326, rows[1].LocationSrid); + Assert.IsNull(rows[1].Shape); + Assert.IsNull(rows[1].Document); + + Assert.AreEqual(3, rows[2].Id); + Assert.IsNull(rows[2].Location); + Assert.IsNull(rows[2].Path); + Assert.IsNull(rows[2].Embedding); + Assert.IsNull(rows[2].Variant); + Assert.IsNull(rows[2].Payload); + } + + // Data2.yaml merges ExtraTypes (existing row 1 plus new row 4) during the migration, so the merge path with non-comparable columns is exercised by these assertions. + await AssertRowsAsync().ConfigureAwait(false); + } + private static async Task<(string cs, ILogger l, SqlServerMigration m)> CreateConsoleDb() { var cs = UnitTest.GetConfig("DbEx_").GetConnectionString("ConsoleDb"); @@ -416,6 +466,7 @@ public async Task SqlServerInspect() a.Parameters.Add("Param1", "unknown"); a.Parameters.Add("Param2", "gender"); a.Parameters.Add("Param3", "CONTACT"); + a.Parameters.Add("Param4", "ExtraTypes"); using var m = new SqlServerMigration(a); var (Success, Output) = await m.MigrateAndLogAsync().ConfigureAwait(false); @@ -424,7 +475,7 @@ public async Task SqlServerInspect() Assert.IsTrue(Output.Length > 0); using var sr = SqlServerMigration.GetRequiredResourcesStreamReader("SqlServerInspect.md", [typeof(SqlServerMigrationTest).Assembly]); - Assert.AreEqual(sr.ReadToEnd(), Output); + MarkdownAssert.AreEqual(sr.ReadToEnd(), Output); } } } \ No newline at end of file