From cbf52936f6b79a0959445f59c5c71d2a1251e4a6 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Sat, 14 Nov 2026 16:12:34 +0800 Subject: [PATCH] Doc: add extension modules reference --- docs/modules/auto-explain.md | 55 ++ docs/modules/btree-gin.md | 21 + docs/modules/citext.md | 21 + docs/modules/dblink.md | 135 +++++ docs/modules/diskquota.md | 499 ++++++++++++++++++ docs/modules/fuzzystrmatch.md | 21 + docs/modules/gp-distribution-policy.md | 77 +++ docs/modules/gp-exttable-fdw.md | 72 +++ docs/modules/gp-internal-tools.md | 100 ++++ docs/modules/gp-legacy-string-agg.md | 83 +++ docs/modules/gp-relsizes-stats.md | 192 +++++++ docs/modules/gp-sparse-vector.md | 296 +++++++++++ docs/modules/gp-stats-collector.md | 284 ++++++++++ docs/modules/hstore.md | 21 + docs/modules/index.md | 162 ++++++ docs/modules/isn.md | 29 + docs/modules/ltree.md | 33 ++ docs/modules/orafce.md | 119 +++++ docs/modules/pageinspect.md | 94 ++++ docs/modules/pg-buffercache.md | 104 ++++ docs/modules/pg-trgm.md | 21 + docs/modules/pgcrypto.md | 83 +++ docs/modules/postgres-fdw.md | 119 +++++ docs/modules/reject-partition-fullscan.md | 128 +++++ docs/modules/sslinfo.md | 51 ++ docs/modules/tablefunc.md | 21 + docs/modules/tsm-system-rows.md | 47 ++ docs/modules/tsm-system-time.md | 47 ++ docs/modules/uuid-ossp.md | 45 ++ docs/modules/yezzey.md | 275 ++++++++++ sidebars.ts | 40 ++ .../version-2.x/modules/auto-explain.md | 55 ++ .../version-2.x/modules/btree-gin.md | 21 + versioned_docs/version-2.x/modules/citext.md | 21 + versioned_docs/version-2.x/modules/dblink.md | 135 +++++ .../version-2.x/modules/diskquota.md | 499 ++++++++++++++++++ .../version-2.x/modules/fuzzystrmatch.md | 21 + .../modules/gp-distribution-policy.md | 77 +++ .../version-2.x/modules/gp-exttable-fdw.md | 72 +++ .../version-2.x/modules/gp-internal-tools.md | 100 ++++ .../modules/gp-legacy-string-agg.md | 83 +++ .../version-2.x/modules/gp-relsizes-stats.md | 192 +++++++ .../version-2.x/modules/gp-sparse-vector.md | 296 +++++++++++ .../version-2.x/modules/gp-stats-collector.md | 284 ++++++++++ versioned_docs/version-2.x/modules/hstore.md | 21 + versioned_docs/version-2.x/modules/index.md | 162 ++++++ versioned_docs/version-2.x/modules/isn.md | 29 + versioned_docs/version-2.x/modules/ltree.md | 33 ++ versioned_docs/version-2.x/modules/orafce.md | 119 +++++ .../version-2.x/modules/pageinspect.md | 94 ++++ .../version-2.x/modules/pg-buffercache.md | 104 ++++ versioned_docs/version-2.x/modules/pg-trgm.md | 21 + .../version-2.x/modules/pgcrypto.md | 83 +++ .../version-2.x/modules/postgres-fdw.md | 119 +++++ .../modules/reject-partition-fullscan.md | 128 +++++ versioned_docs/version-2.x/modules/sslinfo.md | 51 ++ .../version-2.x/modules/tablefunc.md | 21 + .../version-2.x/modules/tsm-system-rows.md | 47 ++ .../version-2.x/modules/tsm-system-time.md | 47 ++ .../version-2.x/modules/uuid-ossp.md | 45 ++ versioned_docs/version-2.x/modules/yezzey.md | 275 ++++++++++ versioned_sidebars/version-2.x-sidebars.json | 39 ++ 62 files changed, 6589 insertions(+) create mode 100644 docs/modules/auto-explain.md create mode 100644 docs/modules/btree-gin.md create mode 100644 docs/modules/citext.md create mode 100644 docs/modules/dblink.md create mode 100644 docs/modules/diskquota.md create mode 100644 docs/modules/fuzzystrmatch.md create mode 100644 docs/modules/gp-distribution-policy.md create mode 100644 docs/modules/gp-exttable-fdw.md create mode 100644 docs/modules/gp-internal-tools.md create mode 100644 docs/modules/gp-legacy-string-agg.md create mode 100644 docs/modules/gp-relsizes-stats.md create mode 100644 docs/modules/gp-sparse-vector.md create mode 100644 docs/modules/gp-stats-collector.md create mode 100644 docs/modules/hstore.md create mode 100644 docs/modules/index.md create mode 100644 docs/modules/isn.md create mode 100644 docs/modules/ltree.md create mode 100644 docs/modules/orafce.md create mode 100644 docs/modules/pageinspect.md create mode 100644 docs/modules/pg-buffercache.md create mode 100644 docs/modules/pg-trgm.md create mode 100644 docs/modules/pgcrypto.md create mode 100644 docs/modules/postgres-fdw.md create mode 100644 docs/modules/reject-partition-fullscan.md create mode 100644 docs/modules/sslinfo.md create mode 100644 docs/modules/tablefunc.md create mode 100644 docs/modules/tsm-system-rows.md create mode 100644 docs/modules/tsm-system-time.md create mode 100644 docs/modules/uuid-ossp.md create mode 100644 docs/modules/yezzey.md create mode 100644 versioned_docs/version-2.x/modules/auto-explain.md create mode 100644 versioned_docs/version-2.x/modules/btree-gin.md create mode 100644 versioned_docs/version-2.x/modules/citext.md create mode 100644 versioned_docs/version-2.x/modules/dblink.md create mode 100644 versioned_docs/version-2.x/modules/diskquota.md create mode 100644 versioned_docs/version-2.x/modules/fuzzystrmatch.md create mode 100644 versioned_docs/version-2.x/modules/gp-distribution-policy.md create mode 100644 versioned_docs/version-2.x/modules/gp-exttable-fdw.md create mode 100644 versioned_docs/version-2.x/modules/gp-internal-tools.md create mode 100644 versioned_docs/version-2.x/modules/gp-legacy-string-agg.md create mode 100644 versioned_docs/version-2.x/modules/gp-relsizes-stats.md create mode 100644 versioned_docs/version-2.x/modules/gp-sparse-vector.md create mode 100644 versioned_docs/version-2.x/modules/gp-stats-collector.md create mode 100644 versioned_docs/version-2.x/modules/hstore.md create mode 100644 versioned_docs/version-2.x/modules/index.md create mode 100644 versioned_docs/version-2.x/modules/isn.md create mode 100644 versioned_docs/version-2.x/modules/ltree.md create mode 100644 versioned_docs/version-2.x/modules/orafce.md create mode 100644 versioned_docs/version-2.x/modules/pageinspect.md create mode 100644 versioned_docs/version-2.x/modules/pg-buffercache.md create mode 100644 versioned_docs/version-2.x/modules/pg-trgm.md create mode 100644 versioned_docs/version-2.x/modules/pgcrypto.md create mode 100644 versioned_docs/version-2.x/modules/postgres-fdw.md create mode 100644 versioned_docs/version-2.x/modules/reject-partition-fullscan.md create mode 100644 versioned_docs/version-2.x/modules/sslinfo.md create mode 100644 versioned_docs/version-2.x/modules/tablefunc.md create mode 100644 versioned_docs/version-2.x/modules/tsm-system-rows.md create mode 100644 versioned_docs/version-2.x/modules/tsm-system-time.md create mode 100644 versioned_docs/version-2.x/modules/uuid-ossp.md create mode 100644 versioned_docs/version-2.x/modules/yezzey.md diff --git a/docs/modules/auto-explain.md b/docs/modules/auto-explain.md new file mode 100644 index 0000000000..bd266e5130 --- /dev/null +++ b/docs/modules/auto-explain.md @@ -0,0 +1,55 @@ +--- +title: auto_explain +--- + +# auto_explain + +The `auto_explain` module logs the execution plans of slow statements automatically, without having to run `EXPLAIN` by hand. + +In Apache Cloudberry, `auto_explain` takes effect only in regular sessions on the coordinator. When the library is loaded in a segment process or in a utility-mode connection, it does nothing and does not define its configuration parameters. Otherwise, it works like the PostgreSQL `auto_explain` module. With `auto_explain.log_analyze` enabled, the logged plan also includes the per-slice execution statistics collected from the segments. + +## Load the module + +`auto_explain` is built and installed by a default Apache Cloudberry build. It is a library, not an extension: it provides no SQL objects, and you do not run `CREATE EXTENSION` for it. + +To load the module into a single session, run the following command as a superuser: + +```sql +LOAD 'auto_explain'; +``` + +To load it into all sessions, add `auto_explain` to `session_preload_libraries` or `shared_preload_libraries`. A change to `shared_preload_libraries` requires a restart of the cluster. Preloading lets you track slow queries whenever they happen, at the cost of some overhead for all queries. + +## Configuration parameters + +The module does nothing until `auto_explain.log_min_duration` is set to a value other than `-1`. Only superusers can change these parameters. + +| Parameter | Description | +|-----------|-------------| +| `auto_explain.log_min_duration` | Minimum statement execution time, in milliseconds, above which the plan is logged. `0` logs all plans. `-1` (the default) turns logging off. | +| `auto_explain.log_analyze` | Logs `EXPLAIN ANALYZE` output instead of `EXPLAIN` output. | +| `auto_explain.log_buffers` | Logs buffer usage. Takes effect only when `log_analyze` is on. | +| `auto_explain.log_wal` | Logs WAL usage. Takes effect only when `log_analyze` is on. | +| `auto_explain.log_timing` | Collects per-node timing. Takes effect only when `log_analyze` is on. | +| `auto_explain.log_triggers` | Includes trigger statistics. Takes effect only when `log_analyze` is on. | +| `auto_explain.log_verbose` | Logs `EXPLAIN VERBOSE` output. | +| `auto_explain.log_settings` | Logs the modified configuration parameters that affect query planning. | +| `auto_explain.log_format` | Output format: `text`, `xml`, `json`, or `yaml`. | +| `auto_explain.log_level` | Log level for the plan. | +| `auto_explain.log_nested_statements` | Logs statements that are run inside functions. | +| `auto_explain.sample_rate` | Fraction of statements to process in each session. | + +## Example + +```sql +LOAD 'auto_explain'; +SET auto_explain.log_min_duration = 0; +SET auto_explain.log_analyze = true; +SELECT count(*) FROM pg_class; +``` + +The plan of the `SELECT` statement is written to the coordinator log. + +## Module documentation + +See [auto_explain](https://www.postgresql.org/docs/14/auto-explain.html) in the PostgreSQL documentation for details about each configuration parameter. diff --git a/docs/modules/btree-gin.md b/docs/modules/btree-gin.md new file mode 100644 index 0000000000..e0f3c6082b --- /dev/null +++ b/docs/modules/btree-gin.md @@ -0,0 +1,21 @@ +--- +title: btree_gin +--- + +# btree_gin + +The `btree_gin` module provides sample generalized inverted index (GIN) operator classes that implement B-tree equivalent behavior for a number of data types. + +The Apache Cloudberry `btree_gin` module is equivalent to the PostgreSQL `btree_gin` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`btree_gin` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION btree_gin; +``` + +## Module documentation + +See [btree_gin](https://www.postgresql.org/docs/14/btree-gin.html) in the PostgreSQL documentation for the list of supported data types and usage examples. diff --git a/docs/modules/citext.md b/docs/modules/citext.md new file mode 100644 index 0000000000..51631c4681 --- /dev/null +++ b/docs/modules/citext.md @@ -0,0 +1,21 @@ +--- +title: citext +--- + +# citext + +The `citext` module provides a case-insensitive character string data type, `citext`. Essentially, it internally calls the `lower()` function when comparing values. Otherwise, it behaves almost exactly like the `text` data type. + +The Apache Cloudberry `citext` module is equivalent to the PostgreSQL `citext` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`citext` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION citext; +``` + +## Module documentation + +See [citext](https://www.postgresql.org/docs/14/citext.html) in the PostgreSQL documentation for details about the data type, operators, and functions that this module defines. diff --git a/docs/modules/dblink.md b/docs/modules/dblink.md new file mode 100644 index 0000000000..277213883e --- /dev/null +++ b/docs/modules/dblink.md @@ -0,0 +1,135 @@ +--- +title: dblink +--- + +# dblink + +The `dblink` module supports connections to other databases from within a database session. The other database can be in the same Apache Cloudberry cluster, in another Apache Cloudberry cluster, or in a PostgreSQL server. + +:::note +`dblink` is intended for short ad hoc queries in other databases. It is not a replacement for external tables, foreign tables, or data migration tools. +::: + +The Apache Cloudberry `dblink` module is a modified version of the PostgreSQL `dblink` module. See [MPP considerations](#mpp-considerations) for the restrictions that apply. + +## Install and register the module + +`dblink` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION dblink; +``` + +## MPP considerations + +- A named or implicit (unnamed) `dblink` connection exists only in the coordinator process that serves your session. A statement that modifies table data, such as `INSERT INTO ... SELECT ... FROM dblink(...)`, runs the `dblink()` call in a different process, which cannot see that connection. In such statements, pass the connection string directly to `dblink()`: + + ```sql + CREATE TABLE testdbllocal (a int, b text) DISTRIBUTED BY (a); + INSERT INTO testdbllocal + SELECT * FROM dblink('dbname=postgres', 'SELECT * FROM testdblink') AS dbltab(id int, product text); + ``` + + If you use a connection name instead, the statement fails with the error `could not establish connection`, and the detail `missing "=" after "" in connection info string`. + +- Asynchronous queries are not supported. The extension does not create the `dblink_send_query()`, `dblink_is_busy()`, and `dblink_get_result()` functions. + +- A non-superuser must specify both `host` and `password` in the connection string of `dblink_connect()` and `dblink()`, even to connect to a local database. Otherwise, the call fails with `host is required` or `password is required`. If a non-superuser does not specify `user`, the name of the current database user is used. + +## Use dblink + +The following steps show the basic use of `dblink`. The examples use `dblink_connect()` to create a connection to a database and `dblink()` to run an SQL query. + +1. Create a sample table in the `postgres` database: + + ```sql + CREATE TABLE testdblink (a int, b text) DISTRIBUTED BY (a); + INSERT INTO testdblink VALUES (1, 'Cheese'), (2, 'Fish'); + ``` + +2. Log in to a different database, for example `testdb`, as a superuser, and register the extension if you have not done so: + + ```sql + CREATE EXTENSION dblink; + ``` + +3. Use `dblink_connect()` to create either an implicit or a named connection to another database. The connection string is a `libpq`-style keyword/value string. This example creates a connection named `mylocalconn` to the `postgres` database on the local cluster: + + ```sql + SELECT dblink_connect('mylocalconn', 'dbname=postgres user=gpadmin'); + ``` + + ```text + dblink_connect + ---------------- + OK + (1 row) + ``` + +4. Use `dblink()` to run a query over the connection. The function returns `record`, so you must specify the names and types of the returned columns: + + ```sql + SELECT * FROM dblink('mylocalconn', 'SELECT * FROM testdblink') AS dbltab(id int, product text); + ``` + + ```text + id | product + ----+--------- + 1 | Cheese + 2 | Fish + (2 rows) + ``` + +5. Close the connection when you no longer need it: + + ```sql + SELECT dblink_disconnect('mylocalconn'); + ``` + +To connect to a remote system, include the host and password information in the connection string. For example, to create an implicit connection: + +```sql +SELECT dblink_connect('host=remotehost port=5432 dbname=postgres user=gpadmin password=secret'); +``` + +### Use dblink as a non-superuser + +A non-superuser must include the host and the password in the connection string of `dblink_connect()`. For example, the user `test_user` can create a connection to the local coordinator host `cdw` with this command: + +```sql +SELECT dblink_connect('host=cdw port=5432 dbname=postgres user=test_user password=secret'); +``` + +In addition, the remote server must actually request a password for the connection. If it does not (for example, with `trust` authentication), the call fails with `password is required`. + +To let non-superusers create connections that do not require a password, use the `dblink_connect_u()` function. It is identical to `dblink_connect()`, except that it skips these checks. `dblink_connect_u()` is installed with all privileges revoked from `PUBLIC`, so only superusers can call it until you grant the `EXECUTE` privilege on it. + +:::caution +Grant `EXECUTE` on `dblink_connect_u()` only to users that you trust. If the target system has users with an authentication method that does not involve a password, a non-superuser who can call `dblink_connect_u()` can impersonate such a user and escalate privileges. Also, a password can be supplied from the server environment, such as a `~/.pgpass` file that belongs to the operating system user that runs the server. +::: + +1. As a superuser, grant the `EXECUTE` privilege on the `dblink_connect_u()` functions: + + ```sql + GRANT EXECUTE ON FUNCTION dblink_connect_u(text) TO test_user; + GRANT EXECUTE ON FUNCTION dblink_connect_u(text, text) TO test_user; + ``` + +2. As `test_user`, create a connection without a password, and use it: + + ```sql + SELECT dblink_connect_u('testconn', 'dbname=postgres user=test_user'); + SELECT * FROM dblink('testconn', 'SELECT * FROM testdblink') AS dbltab(id int, product text); + ``` + +### Use dblink with SSL-encrypted connections + +To connect over an encrypted connection, specify the `sslmode` keyword in the connection string. Set `sslmode` to at least `require` to disallow unencrypted connections. For example: + +```sql +SELECT dblink_connect('con_sales', 'dbname=sales host=cdw user=gpadmin sslmode=require'); +``` + +## Module documentation + +See [dblink](https://www.postgresql.org/docs/14/dblink.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/docs/modules/diskquota.md b/docs/modules/diskquota.md new file mode 100644 index 0000000000..a7c89da165 --- /dev/null +++ b/docs/modules/diskquota.md @@ -0,0 +1,499 @@ +--- +title: diskquota +--- + +# diskquota + +The `diskquota` module allows Apache Cloudberry administrators to limit the amount of disk space that schemas, roles, or tablespaces use in a database. + +Apache Cloudberry includes `diskquota` 2.3 in the `gpcontrib/diskquota` directory of the source code. + +## Installation + +### Build the module + +`diskquota` is not built by default. To build and install the module with Apache Cloudberry, specify the `--with-diskquota` option when you run `configure`: + +```shell +./configure --with-diskquota [other options...] +make +make install +``` + +The module is built with CMake, so `cmake` must be installed on the build host. + +### Enable the module + +Before you can use the module, perform these steps: + +1. Create the `diskquota` database. The module uses this database to store the list of the databases in which the module is enabled. + + ```shell + createdb diskquota + ``` + +2. Add the `diskquota-2.3` shared library to the `shared_preload_libraries` server configuration parameter and restart Apache Cloudberry. Be sure to retain the previous setting of the configuration parameter. For example: + + ```shell + gpconfig -s shared_preload_libraries + gpconfig -c shared_preload_libraries -v 'diskquota-2.3' + gpstop -ar + ``` + + If `shared_preload_libraries` already contains other libraries, specify a comma-separated list, for example `'auto_explain,diskquota-2.3'`. + +3. Create the `diskquota` extension in each database in which you want to enforce disk usage quotas. The `diskquota.max_monitored_databases` parameter limits the number of databases in which you can create the extension (`50` by default). + + ```sql + CREATE EXTENSION diskquota; + ``` + +4. If the database is not empty when you create the extension, `diskquota` raises the following warning. Run the `diskquota.init_table_size_table()` function to size the existing tables. In a database with many files, this can take some time. + + ``` + WARNING: [diskquota] diskquota is not ready because current database is not empty + HINT: please run 'SELECT diskquota.init_table_size_table();' to initialize diskquota + ``` + + ```sql + SELECT diskquota.init_table_size_table(); + ``` + +:::note +The library must be loaded through `shared_preload_libraries`. If it is not, the functions of the module fail with an error that reports that `diskquota-2.3` is not in `shared_preload_libraries`. +::: + +For more information about the `gpconfig` and `gpstop` utilities, see [gpconfig](../sys-utilities/gpconfig.md) and [gpstop](../sys-utilities/gpstop.md). + +## About the diskquota module + +The disk usage of a table includes the table data, indexes, toast tables, and free space map. For append-optimized tables, the calculation also includes the auxiliary tables, such as the visibility map and the block directory table. + +You can set the following quotas with the `diskquota` module: + +- A schema disk quota sets a limit on the disk space that all the tables in a specific schema of a database can use. The disk usage of a schema is the total of the disk usage on all segments for all tables in the schema. +- A role disk quota sets a limit on the disk space that all the tables that a specific role owns in a database can use. The disk usage of a role is the total of the disk usage on all segments for all tables that the role owns. Although a role is a cluster-level object, the quota limit and the disk usage of a role are specific to each database. +- A schema tablespace disk quota sets a limit on the disk space that all the tables in a specific schema and tablespace of a database can use. +- A role tablespace disk quota sets a limit on the disk space that all the tables that a specific role owns in a specific tablespace of a database can use. +- A per-segment tablespace disk quota sets a limit on the disk space that a single segment can use when a tablespace quota is set for a schema or role. + +:::note +You cannot set a role disk quota for the system owner, that is, the bootstrap superuser that initialized the cluster. +::: + +Each database stores and manages its own quota configuration in the `diskquota.quota_config` table. + +## How diskquota monitors disk usage + +`diskquota` is based on the background worker framework. A single `diskquota` launcher process runs on the coordinator. The launcher starts `diskquota` worker processes on the coordinator for the databases in which the extension is created. No launcher or worker processes run on the segments. + +A worker process monitors the disk usage of the schemas, roles, and tablespaces in its database. On each segment, hooks in the storage manager record the tables whose files are created, extended, or truncated. These tables are called active tables. Periodically, the worker process queries the segments for the sizes of the active tables, updates the disk usage of the corresponding schemas and roles, and compares the usage with the quota limits. When a quota is exceeded, the worker puts the schema, role, or tablespace into the reject map, which is stored in shared memory. + +`diskquota` can enforce both soft limits and hard limits for disk usage: + +- `diskquota` always enforces soft limits. It checks the reject map before a query runs. If the target schema, role, or tablespace of a query that adds data is in the reject map, the query is cancelled before it starts, with an error message that reports that the quota is exceeded. If the quota is not exceeded when a query starts, the query is allowed to run, even if it eventually causes the quota to be exceeded. +- When hard limit enforcement is enabled, `diskquota` also checks disk usage during query execution. If a query exceeds a disk quota while it is running, `diskquota` terminates the query. Hard limit enforcement is deactivated by default. See [diskquota.hard_limit](#diskquotahard_limit). + +There is some delay after a quota is reached before the schema or role is added to the reject map, because the worker processes recalculate the table sizes periodically (every two seconds by default). Other queries can add more data during the delay. The same delay occurs when the disk usage falls beneath a quota because of operations that remove data, such as `DROP`, `TRUNCATE`, or `VACUUM FULL`. See [diskquota.naptime](#diskquotanaptime). + +Deleting rows or running `VACUUM` on a table does not release disk space, so these operations alone cannot remove a schema or role from the reject map. To reduce the disk space that a table uses, run `VACUUM FULL` or `TRUNCATE TABLE`. + +If the number of databases in which the extension is created is larger than `diskquota.max_workers`, the launcher starts and stops worker processes dynamically, so that the databases share the available workers. + +## Functions and views + +The functions and views of the module are in the `diskquota` schema. + +Only superusers can run the functions that set quotas, and the `pause()` and `resume()` functions. + +| Function | Description | +|---|---| +| `void init_table_size_table()` | Sizes the existing tables in the current database. | +| `void set_schema_quota(schema_name text, quota text)` | Sets a disk quota for a schema in the current database. | +| `void set_role_quota(role_name text, quota text)` | Sets a disk quota for a role in the current database. | +| `void set_schema_tablespace_quota(schema_name text, tablespace_name text, quota text)` | Sets a disk quota for a schema and tablespace combination in the current database. | +| `void set_role_tablespace_quota(role_name text, tablespace_name text, quota text)` | Sets a disk quota for a role and tablespace combination in the current database. | +| `void set_per_segment_quota(tablespace_name text, ratio float4)` | Sets a per-segment disk quota ratio for a tablespace in the current database. | +| `void pause()` | Continues to count the disk usage of the current database, but stops raising errors when a limit is exceeded. | +| `void resume()` | Resumes raising errors when a disk usage limit is exceeded in the current database. | +| `status()` | Returns a table with the `name` and `status` columns that shows the state of soft and hard limit enforcement in the current database, and the binary and schema versions of `diskquota`. | +| `bigint relation_size(relation regclass)` | Returns the size of a relation on all segments as `diskquota` calculates it. | +| `boolean wait_for_worker_new_epoch()` | Waits until the worker process of the current database finishes a new round of disk usage refresh. | + +| View | Description | +|---|---| +| `show_fast_database_size_view` | Displays the disk space usage of the current database. | +| `show_fast_schema_quota_view` | Lists the quotas and the disk usage for schemas in the current database. | +| `show_fast_role_quota_view` | Lists the quotas and the disk usage for roles in the current database. | +| `show_fast_schema_tablespace_quota_view` | Lists the quotas and the disk usage for schemas per tablespace in the current database. | +| `show_fast_role_tablespace_quota_view` | Lists the quotas and the disk usage for roles per tablespace in the current database. | +| `show_segment_ratio_quota_view` | Displays the per-segment quota ratio of the tablespaces for which a ratio is set in the current database. | +| `rejectmap` | Lists the entries in the reject map. | + +## Configuration parameters + +The module defines the following server configuration parameters. Use the `gpconfig` utility to set them. A parameter that is marked "restart" takes effect after you restart the cluster (`gpstop -ar`). A parameter that is marked "reload" takes effect after you reload the configuration (`gpstop -u`). + +| Parameter | Default | Takes effect | Description | +|---|---|---|---| +| `diskquota.naptime` | `2` | reload | The interval, in seconds, at which `diskquota` recalculates the table sizes. | +| `diskquota.hard_limit` | `off` | reload | Activates or deactivates hard limit enforcement of disk usage. | +| `diskquota.worker_timeout` | `60` | reload | The time, in seconds, after which `diskquota.wait_for_worker_new_epoch()` raises a notice that it timed out while waiting for the worker process. | +| `diskquota.max_workers` | `10` | restart | The maximum number of `diskquota` worker processes that can run at the same time. The maximum value is `20`. | +| `diskquota.max_active_tables` | `307200` (`300 * 1024`) | restart | The maximum number of active relations (tables, indexes, and so on) that `diskquota` can track at the same time. | +| `diskquota.max_table_segments` | `10485760` (`10 * 1024 * 1024`) | restart | The maximum number of table segments in the cluster. | +| `diskquota.max_monitored_databases` | `50` | restart | The maximum number of databases that `diskquota` can monitor. The maximum value is `1024`. | +| `diskquota.max_quota_probes` | `1048576` (`1024 * 1024`) | restart | The maximum number of quota probes in the cluster. | + +### diskquota.naptime + +The smaller the `diskquota.naptime` value, the less delay in detecting changes in disk usage. This example sets the value to ten seconds: + +```shell +gpconfig -c diskquota.naptime -v 10 +gpstop -u +``` + +### diskquota.hard_limit + +When hard limit enforcement is enabled, `diskquota` checks the quotas during query execution. If a running query exceeds a quota limit, `diskquota` terminates the query. + +To activate hard limit enforcement for all databases, set the parameter to `on` and reload the configuration: + +```shell +gpconfig -c diskquota.hard_limit -v 'on' +gpstop -u +``` + +Run the following query to view the hard limit enforcement setting: + +```sql +SELECT * FROM diskquota.status(); +``` + +### diskquota.max_workers + +The parameter specifies the maximum number of worker processes that can run at the same time, not including the launcher process. The `diskquota` workers are taken from the pool of worker processes that the `max_worker_processes` server configuration parameter establishes, so `diskquota.max_workers` should be less than `max_worker_processes`. + +### diskquota.max_active_tables + +Active tables are the tables whose sizes might have changed since `diskquota` last recalculated the table sizes. The active table list is stored in shared memory that is allocated when the server starts. If the active table shared memory is full, `diskquota` might fail to detect the corresponding disk usage changes in time. + +### diskquota.max_table_segments + +A table (including each child table of a partitioned table) is distributed to all segments. `diskquota` counts the part of a table on each segment as a table segment. The parameter therefore limits the number of tables that `diskquota` can monitor. + +When you expand the cluster, each table consumes more table segments, which reduces the number of tables that `diskquota` can support. If the following warning is reported, increase the `diskquota.max_table_segments` value, and then restart Apache Cloudberry: + +``` +WARNING: [diskquota] the number of tables exceeds the limit, please increase the GUC value for diskquota.max_table_segments. Current diskquota.max_table_segments value: 10485760 +``` + +## Use the diskquota module + +### View the diskquota status + +To view the state of soft and hard limit enforcement in the current database and the `diskquota` version numbers, call the `status()` function: + +```sql +SELECT * FROM diskquota.status(); +``` + +``` + name | status +------------------------+-------- + soft limits | on + hard limits | off + current binary version | 2.3.0 + current schema version | 2.3 +(4 rows) +``` + +When `diskquota` is paused in the database, the status of `soft limits` is `paused`. The status of `hard limits` is also `paused` if hard limit enforcement is enabled. + +### Pause and resume quota enforcement + +To stop raising quota exceeded errors in the current database for a period of time, pause and then resume `diskquota`: + +```sql +SELECT diskquota.pause(); +-- Perform the operations for which quotas should not be enforced. +SELECT diskquota.resume(); +``` + +`diskquota` continues to count disk usage while it is paused. + +### Set a schema or role disk quota + +Use the `diskquota.set_schema_quota()` and `diskquota.set_role_quota()` functions to set, update, or delete the disk quota limits for schemas and roles in the current database. The functions take two arguments: the schema or role name, and the quota to set. Specify the quota in units of MB, GB, TB, or PB, for example `'2TB'`. A number without a unit is in MB. A quota of `0` is not allowed. + +The functions convert the schema or role name to lower case, unless you enclose the name in double quotes inside the string, for example `'"MyRole"'`. + +The following example sets a 250GB quota for the `acct` schema: + +```sql +SELECT diskquota.set_schema_quota('acct', '250GB'); +``` + +This example sets a 500MB quota for the `nickd` role: + +```sql +SELECT diskquota.set_role_quota('nickd', '500MB'); +``` + +To change a quota, call the function again with the new quota value. + +To remove a schema or role quota, set the quota value to `'-1'`. + +### Set a tablespace disk quota for a schema or role + +Use the `diskquota.set_schema_tablespace_quota()` and `diskquota.set_role_tablespace_quota()` functions to set, update, or delete per-tablespace disk quota limits for schemas and roles in the current database. The functions take three arguments: the schema or role name, the tablespace name, and the quota to set. + +The following example sets a 250GB disk quota for the `acct` schema in the `tspaced1` tablespace: + +```sql +SELECT diskquota.set_schema_tablespace_quota('acct', 'tspaced1', '250GB'); +``` + +This example sets a 500MB disk quota for the `nickd` role in the `tspaced2` tablespace: + +```sql +SELECT diskquota.set_role_tablespace_quota('nickd', 'tspaced2', '500MB'); +``` + +To change a quota, call the function again with the new quota value. + +To remove a schema or role tablespace quota, set the quota value to `'-1'`. + +### Set a per-segment tablespace disk quota + +When you set a tablespace quota for a schema or a role, you can also define a per-segment quota for the tablespace. A per-segment quota limits the amount of disk space that the tablespace can consume on a single segment, which helps to prevent the disk of a segment from filling because of data skew. + +Use the `diskquota.set_per_segment_quota()` function to set, update, or delete a per-segment tablespace quota. The function takes two arguments: the tablespace name and a ratio. The ratio identifies how much more of the disk quota a single segment can use than the average segment quota. A ratio of `0` is not allowed. + +The average segment quota is calculated as follows: + +``` +avg_seg_quota = tablespace_quota / number_of_segments +``` + +For example, if the cluster has 8 segments and you set the following schema tablespace quota, the average segment quota for the `tspaced1` tablespace is `800GB / 8 = 100GB`: + +```sql +SELECT diskquota.set_schema_tablespace_quota('accts', 'tspaced1', '800GB'); +``` + +If you then set the following per-segment ratio, the maximum disk usage that is allowed per segment is `100GB * 2.0 = 200GB`: + +```sql +SELECT diskquota.set_per_segment_quota('tspaced1', '2.0'); +``` + +To change the ratio, call the function again with the new value. To remove a per-segment tablespace quota, set the ratio to `'-1'`. + +To view the per-segment quota ratio that is set for a tablespace, query the `show_segment_ratio_quota_view` view: + +```sql +SELECT tablespace_name, per_seg_quota_ratio + FROM diskquota.show_segment_ratio_quota_view WHERE tablespace_name IN ('tspaced1'); +``` + +``` + tablespace_name | per_seg_quota_ratio +-----------------+--------------------- + tspaced1 | 2 +(1 row) +``` + +### Display disk quotas and disk usage + +The `diskquota.show_fast_schema_quota_view` view lists the quotas for schemas in the current database. The `nspsize_in_bytes` column contains the calculated size of all the tables that belong to the schema. + +```sql +SELECT * FROM diskquota.show_fast_schema_quota_view; +``` + +``` + schema_name | schema_oid | quota_in_mb | nspsize_in_bytes +-------------+------------+-------------+------------------ + acct | 16561 | 256000 | 131072 + public | 2200 | 250 | 3014656 +(2 rows) +``` + +The `diskquota.show_fast_role_quota_view` view lists the quotas for roles in the current database. The `rolsize_in_bytes` column contains the calculated size of all the tables that the role owns. + +```sql +SELECT * FROM diskquota.show_fast_role_quota_view; +``` + +``` + role_name | role_oid | quota_in_mb | rolsize_in_bytes +-----------+----------+-------------+------------------ + mdach | 16558 | 500 | 131072 + nickd | 16577 | 500 | 144670720 +(2 rows) +``` + +To view the per-tablespace disk quotas for schemas and roles, query the `diskquota.show_fast_schema_tablespace_quota_view` and `diskquota.show_fast_role_tablespace_quota_view` views. For example: + +```sql +SELECT schema_name, tablespace_name, quota_in_mb, nspsize_tablespace_in_bytes + FROM diskquota.show_fast_schema_tablespace_quota_view + WHERE schema_name = 'acct' AND tablespace_name = 'tspaced1'; +``` + +### Identify the diskquota-monitored databases + +Run the following commands to obtain the list of the databases that `diskquota` monitors in the cluster: + +```sql +\c diskquota +SELECT d.datname FROM diskquota_namespace.database_list q, pg_database d + WHERE q.dbid = d.oid ORDER BY d.datname; +``` + +## Notes and limitations + +- Drop the `diskquota` extension from a database before you drop the database. A `diskquota` worker process is connected to each database in which the extension is created. +- `diskquota` also counts the disk usage of temporary tables. The size of a temporary table is added to the usage of the role that owns the table. It is not added to the usage of the current schema, because a temporary table is located in a temporary schema. +- When the reject map shared memory is full, data might be loaded into some schemas or roles after they have reached their quota limits. + +## Examples + +### Set a schema quota + +This example configures a schema quota and shows the soft limit behavior as data is added to the schema. The example assumes that the `diskquota` library is preloaded and that the `diskquota` database exists. + +1. Create a database named `testdb` and connect to it. + + ```shell + createdb testdb + psql -d testdb + ``` + +2. Create the `diskquota` extension in the database. + + ```sql + CREATE EXTENSION diskquota; + ``` + +3. Create a schema named `s1`. + + ```sql + CREATE SCHEMA s1; + ``` + +4. Set a 1MB disk quota for the `s1` schema. + + ```sql + SELECT diskquota.set_schema_quota('s1', '1 MB'); + ``` + +5. Create a table in the `s1` schema and insert a small amount of data into it. The schema has no data yet, so it is not in the reject map. + + ```sql + SET search_path TO s1; + CREATE TABLE a(i int) DISTRIBUTED BY (i); + INSERT INTO a SELECT generate_series(1,100); + ``` + +6. Insert a large amount of data, enough to exceed the 1MB quota of the schema. Before the `INSERT` command, the `s1` schema is not in the reject map, so with only soft limit enforcement in effect the command is allowed to run, even though it exceeds the limit of the schema. + + ```sql + INSERT INTO a SELECT generate_series(1,10000000); + ``` + +7. Try to insert a small amount of data. Because the previous command exceeded the disk quota of the schema, the schema is in the reject map and any command that loads data is cancelled. + + ```sql + INSERT INTO a SELECT generate_series(1,100); + ``` + + ``` + ERROR: schema's disk space quota exceeded with name: s1 + ``` + +8. Remove the quota from the `s1` schema by setting it to `-1`, and insert a small amount of data again. The 5-second sleep before the `INSERT` command ensures that the reject map is updated before the command runs. + + ```sql + SELECT diskquota.set_schema_quota('s1', '-1'); + SELECT pg_sleep(5); + INSERT INTO a SELECT generate_series(1,100); + ``` + +### Enable hard limit enforcement + +This example enables hard limit enforcement of disk usage, and runs the commands of the previous example again. Before you start, empty the table from the previous example so that the schema is under its quota again: + +```sql +TRUNCATE s1.a; +SELECT diskquota.wait_for_worker_new_epoch(); +``` + +1. Enable hard limit enforcement. + + ```shell + gpconfig -c diskquota.hard_limit -v 'on' + gpstop -u + ``` + +2. View the hard limit enforcement setting. + + ```sql + SELECT * FROM diskquota.status(); + ``` + +3. Set a 1MB disk quota for the `s1` schema again. + + ```sql + SELECT diskquota.set_schema_quota('s1', '1 MB'); + ``` + +4. Insert a large amount of data, enough to exceed the 1MB quota of the schema. Before the `INSERT` command, the `s1` schema is not in the reject map, so the command is allowed to start. When the operation exceeds the schema quota, `diskquota` terminates the query with an error like `schema's disk space quota exceeded with name: 17952`. With hard limit enforcement, the error reports the OID of the schema instead of its name. + + ```sql + INSERT INTO a SELECT generate_series(1,10000000); + ``` + +5. Remove the quota from the `s1` schema, and wait until the reject map is updated: + + ```sql + SELECT diskquota.set_schema_quota('s1', '-1'); + SELECT diskquota.wait_for_worker_new_epoch(); + ``` + +### Set a per-segment tablespace quota + +This example configures a tablespace quota and a per-segment tablespace quota. It uses the `testdb` database and the `s1` schema of the previous examples, and assumes that a tablespace named `tbsp1` exists in the cluster. + +1. Set a disk quota of 10MB for the `s1` schema in the `tbsp1` tablespace. + + ```sql + SELECT diskquota.set_schema_tablespace_quota('s1', 'tbsp1', '10 MB'); + ``` + +2. Create a table named `b` in the `tbsp1` tablespace and insert some data. The data does not exceed the 10MB quota. + + ```sql + SET search_path TO s1; + CREATE TABLE b(i int) TABLESPACE tbsp1 DISTRIBUTED BY (i); + INSERT INTO b SELECT generate_series(1,100000); + ``` + +3. Set a per-segment ratio of `0.1` for the `tbsp1` tablespace. With this setting, a single segment can use at most the average segment quota multiplied by `0.1`, which is less than the space that the table already uses on a segment. + + ```sql + SELECT diskquota.set_per_segment_quota('tbsp1', 0.1); + ``` + +4. Wait for `diskquota` to refresh the disk usage, and then try to insert more data. Because the per-segment tablespace quota is exceeded, the command is cancelled. + + ```sql + SELECT diskquota.wait_for_worker_new_epoch(); + INSERT INTO b SELECT generate_series(1,100); + ``` + + ``` + ERROR: tablespace: tbsp1, schema: s1 diskquota exceeded per segment quota + ``` diff --git a/docs/modules/fuzzystrmatch.md b/docs/modules/fuzzystrmatch.md new file mode 100644 index 0000000000..930a166c48 --- /dev/null +++ b/docs/modules/fuzzystrmatch.md @@ -0,0 +1,21 @@ +--- +title: fuzzystrmatch +--- + +# fuzzystrmatch + +The `fuzzystrmatch` module provides functions that determine similarities and distance between strings, based on the Soundex, Levenshtein, Metaphone, and Double Metaphone algorithms. + +The Apache Cloudberry `fuzzystrmatch` module is equivalent to the PostgreSQL `fuzzystrmatch` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`fuzzystrmatch` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION fuzzystrmatch; +``` + +## Module documentation + +See [fuzzystrmatch](https://www.postgresql.org/docs/14/fuzzystrmatch.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/docs/modules/gp-distribution-policy.md b/docs/modules/gp-distribution-policy.md new file mode 100644 index 0000000000..4c6ecaa326 --- /dev/null +++ b/docs/modules/gp-distribution-policy.md @@ -0,0 +1,77 @@ +--- +title: gp_distribution_policy +--- + +# gp_distribution_policy + +The `gp_distribution_policy` module checks whether the rows of a table are stored on the segments that the table's distribution policy assigns them to. On each segment, it scans the table, computes the hash of the distribution key columns of every row, and verifies that the row belongs to that segment. + +Use it to detect rows that ended up on a wrong segment. + +## Build and install + +`gp_distribution_policy` is built and installed by default with Apache Cloudberry. No `configure` option is required. + +## Enable the module + +Create the extension in each database where you want to use it. Creating the extension requires superuser privileges. It does not need `shared_preload_libraries`. + +```sql +CREATE EXTENSION gp_distribution_policy; +``` + +The extension installs its functions in the `public` schema. + +## Functions + +| Function | Returns | Description | +|---|---|---| +| `gp_table_distribution_check(regclass)` | `TABLE (gp_segment_id int, is_distribution_correct boolean)` | Runs the check on all segments and returns one row per segment. `is_distribution_correct` is `true` if all rows stored on that segment belong to it, and `false` if at least one row is on the wrong segment. | +| `gp_distribution_policy_table_check(relid regclass)` | `boolean` | Checks the rows of the table on the instance where the function is executed. It is the per-segment function behind `gp_table_distribution_check()`. Call `gp_table_distribution_check()` instead of calling this function directly. | + +## Example + +```sql +CREATE EXTENSION gp_distribution_policy; + +CREATE TABLE some_other_table(id int, some_other_data int) DISTRIBUTED BY (id); +INSERT INTO some_other_table SELECT i, i FROM generate_series(1,12) i; + +SELECT gp_table_distribution_check('some_other_table'); +``` + +On a cluster with three segments where all rows are placed correctly, the output is: + +```text + gp_table_distribution_check +----------------------------- + (0,t) + (1,t) + (2,t) +(3 rows) +``` + +If a segment holds rows that hash to a different segment, that segment is reported with `f`. For example, after the same row has been inserted directly on every segment, only one segment is correct: + +```text + gp_table_distribution_check +----------------------------- + (0,t) + (1,f) + (2,f) +(3 rows) +``` + +To get the result as separate columns: + +```sql +SELECT * FROM gp_table_distribution_check('some_other_table'); +``` + +## Usage notes + +- The input relation must be a table, a materialized view, or a directory table. For views, foreign tables, and partitioned tables, the function reports the error `input relation is not a table`. To check a partitioned table, run the check on each leaf partition. +- Heap tables and append-optimized tables are supported. +- The check is meaningful only for hash-distributed tables. For randomly distributed and replicated tables, the function reports `false` for some segments even though the data is placed correctly, so do not use it for such tables. +- The check scans the table on each segment and stops on a segment at the first misplaced row. On a large table with a correct distribution, it reads the whole table. +- The result tells you which segments contain misplaced rows. It does not identify the rows. diff --git a/docs/modules/gp-exttable-fdw.md b/docs/modules/gp-exttable-fdw.md new file mode 100644 index 0000000000..033e37e359 --- /dev/null +++ b/docs/modules/gp-exttable-fdw.md @@ -0,0 +1,72 @@ +--- +title: gp_exttable_fdw +--- + +# gp_exttable_fdw + +The `gp_exttable_fdw` module is the foreign-data wrapper that Apache Cloudberry uses to implement external tables. An external table is stored in the catalog as a foreign table that belongs to the `gp_exttable_server` foreign server, and `gp_exttable_fdw` scans it or inserts into it. + +:::note +`gp_exttable_fdw` and `gp_exttable_server` are special built-in objects that exist for backward compatibility with external tables. Do not use the wrapper directly in normal operation. Define external tables with [`CREATE EXTERNAL TABLE`](../sql-stmts/create-external-table.md) instead. +::: + +## Installation + +`gp_exttable_fdw` is always built and installed with Apache Cloudberry. It is part of the default `gpcontrib` build targets and does not depend on any `configure` option. + +The module is packaged as an extension (version `1.0`, not relocatable) whose objects are created in the `pg_catalog` schema. `initdb` registers the extension when the cluster is initialized, so you do not need to run `CREATE EXTENSION`. It does not require `shared_preload_libraries` or a restart. + +## Objects provided by the module + +| Object | Type | Description | +|---|---|---| +| `gp_exttable_fdw` | Foreign-data wrapper | The wrapper for external tables. Uses `gp_exttable_fdw_handler()` as its handler and `gp_exttable_permission_check()` as its validator. | +| `gp_exttable_server` | Foreign server | The server that all external tables belong to. | +| `gp_exttable_fdw_handler()` | Function | The FDW handler. Supports foreign scans and foreign inserts. | +| `gp_exttable_permission_check(text[], oid)` | Function | The FDW validator. Checks the table options and the permissions of the current role. | +| `pg_exttable()` | Function | Returns one row for each external table. Runs on the coordinator. | +| `pg_exttable` | View | Defined as `SELECT * FROM pg_exttable()`. | + +### The pg_exttable view + +The `pg_exttable` view shows the external table definition that is stored in the options of each foreign table of `gp_exttable_server`. + +| Column | Type | Description | +|---|---|---| +| `reloid` | `oid` | The OID of the external table. | +| `urilocation` | `text[]` | The location URIs of the external table. | +| `execlocation` | `text[]` | The segment locations where the external table runs. | +| `fmttype` | `"char"` | The format type of the external data. | +| `fmtopts` | `text` | The formatting options. | +| `options` | `text[]` | `{error_log_persistent=true}` when the table was created with `LOG ERRORS PERSISTENTLY`. Otherwise `NULL`. | +| `command` | `text` | The OS command to run for an `EXECUTE` external table. | +| `rejectlimit` | `int4` | The reject limit for single row error handling. `NULL` when single row error handling is not used. | +| `rejectlimittype` | `"char"` | The type of the reject limit: `r` for rows or `p` for percent. `NULL` when single row error handling is not used. | +| `logerrors` | `bool` | Whether errors are logged. | +| `encoding` | `int4` | The ID of the encoding of the external data. Use `pg_encoding_to_char(encoding)` to get the name. | +| `writable` | `bool` | Whether the table is a writable external table. | + +For example: + +```sql +SELECT reloid::regclass, urilocation, pg_encoding_to_char(encoding) +FROM pg_exttable; +``` + +## Foreign table options + +The validator of `gp_exttable_fdw` checks the following options of a foreign table that belongs to `gp_exttable_server`: + +| Option | Description | +|---|---| +| `format_type` | Required. One of `t` (text), `c` (CSV), or `b` (custom). | +| `location_uris` | The location URIs, separated by the `\|` character. You must specify either `location_uris` or `command`, and you cannot specify both. | +| `command` | The command of an `EXECUTE` external web table. Only superusers can set this option. | +| `is_writable` | A Boolean value that marks the table as writable. | +| `reject_limit_type` | `r` (rows, the default) or `p` (percent). | +| `reject_limit` | The reject limit for single row error handling. It cannot be used with a writable external table. | +| `encoding` | An encoding name (for example `UTF8` or `GBK`) or a numeric encoding ID. | + +When a role that is not a superuser creates a table, the validator also checks whether the role is allowed to create an external table for the protocol of the location URIs. + +A literal `|` or `\` character inside a location URI must be escaped as `\|` or `\\` in the `location_uris` option value. diff --git a/docs/modules/gp-internal-tools.md b/docs/modules/gp-internal-tools.md new file mode 100644 index 0000000000..17ff1c9f24 --- /dev/null +++ b/docs/modules/gp-internal-tools.md @@ -0,0 +1,100 @@ +--- +title: gp_internal_tools +--- + +# gp_internal_tools + +The `gp_internal_tools` module is a set of diagnostic tools for Apache Cloudberry. The extension installs the `session_state` schema with a view that shows the memory consumption of each session on the coordinator and on every segment. The module also ships shared libraries with low-level diagnostic functions for the workfile manager, the instrumentation shared memory, and append-optimized tables. + +:::note +This module is meant for troubleshooting. It exposes internal server state and is not needed for normal operation. +::: + +## Build and install + +`gp_internal_tools` is built and installed by default with Apache Cloudberry. No `configure` option is required. + +The build installs four shared libraries in the library directory of the installation: `gp_session_state_memory_stats`, `gp_workfile_mgr`, `gp_instrument_shmem`, and `gp_ao_co_diagnostics`. + +## Enable the module + +Create the extension in each database where you want to use it. Creating the extension requires superuser privileges. It does not need `shared_preload_libraries`. + +```sql +CREATE EXTENSION gp_internal_tools; +``` + +The extension creates its objects in the `session_state` schema. + +## Objects installed by the extension + +### session_state.session_level_memory_consumption + +A view with one row per session and per instance (the coordinator and each segment) that shows the memory used by the session on that instance. `SELECT` on the view is granted to `PUBLIC`. + +| Column | Type | Description | +|---|---|---| +| `datname` | `name` | Name of the database that the session is connected to. | +| `sess_id` | `integer` | Session ID. | +| `usename` | `name` | Name of the session user. | +| `query` | `text` | Current query of the session, as shown in `pg_stat_activity`. | +| `segid` | `integer` | Segment content ID. `-1` is the coordinator. | +| `vmem_mb` | `integer` | Virtual memory used by the session on this instance, in MB. | +| `is_runaway` | `boolean` | Whether the session is marked as a runaway session on this instance. | +| `qe_count` | `integer` | Number of query executor processes of the session on this instance. | +| `active_qe_count` | `integer` | Number of active query executor processes of the session on this instance. | +| `dirty_qe_count` | `integer` | Internal cleanup counter of the session (the `cleanupCountdown` field of the session state). | +| `runaway_vmem_mb` | `integer` | Value of the virtual memory counter of the session when it was marked as a runaway session. | +| `runaway_command_cnt` | `integer` | Command count of the session when it was marked as a runaway session. | +| `idle_start` | `timestamp with time zone` | Last time a query executor process of this session became idle. | + +The `datname`, `usename`, and `query` columns come from a left join with `pg_stat_activity` on the session ID, so they are `NULL` if the session is not found there. + +### Functions + +The view is built on two set-returning functions. `EXECUTE` on both is granted to `PUBLIC`. + +| Function | Runs on | Description | +|---|---|---| +| `session_state.session_state_memory_entries_f_on_master()` | Coordinator | Returns the session memory entries of the coordinator. | +| `session_state.session_state_memory_entries_f_on_segments()` | All segments | Returns the session memory entries of each segment. | + +Both functions return `SETOF record` with the columns `(segid int, sessionid int, vmem_mb int, runaway_status int, qe_count int, active_qe_count int, dirty_qe_count int, runaway_vmem_mb int, runaway_command_cnt int, idle_start timestamp with time zone)`. Query the view instead of calling them directly. + +## Example + +```sql +CREATE EXTENSION gp_internal_tools; + +SELECT * FROM session_state.session_level_memory_consumption LIMIT 0; +``` + +```text + datname | sess_id | usename | query | segid | vmem_mb | is_runaway | qe_count | active_qe_count | dirty_qe_count | runaway_vmem_mb | runaway_command_cnt | idle_start +---------+---------+---------+-------+-------+---------+------------+----------+-----------------+----------------+-----------------+---------------------+------------ +(0 rows) +``` + +To see the memory used by the current session on each instance: + +```sql +SELECT m.segid, m.vmem_mb, m.is_runaway +FROM session_state.session_level_memory_consumption m, pg_stat_activity a +WHERE a.pid = pg_backend_pid() + AND m.sess_id = a.sess_id; +``` + +## Additional diagnostic libraries + +The module also builds and installs the following shared libraries. The extension script does not create SQL functions for them: `CREATE EXTENSION gp_internal_tools` installs only the `session_state` objects described above. The C functions in these libraries are thin wrappers around internal server functions and are intended for support and development use. + +| Library | C functions | Purpose | +|---|---|---| +| `gp_workfile_mgr` | `gp_workfile_mgr_cache_entries`, `gp_workfile_mgr_used_diskspace` | Workfile manager diagnostics. `gp_workfile_mgr_cache_entries` returns the workfile cache entries of one segment. `gp_workfile_mgr_used_diskspace` returns a `(segid int4, bytes int8)` row with the number of bytes used for workfiles on a segment. | +| `gp_instrument_shmem` | `gp_instrument_shmem_summary`, `gp_instrument_shmem_detail` | Diagnostics of the instrumentation slots in shared memory. `gp_instrument_shmem_summary` returns `(segid int4, num_free int8, num_used int8)`. `gp_instrument_shmem_detail` returns one row per used slot with `(tmid int4, ssid int4, ccnt int2, segid int2, pid int4, nid int2, tuplecount int8, nloops int8, ntuples int8)`. | +| `gp_ao_co_diagnostics` | `gp_aoseg_wrapper`, `gp_aoseg_history_wrapper`, `gp_aocsseg_wrapper`, `gp_aocsseg_history_wrapper`, `gp_aoblkdir_wrapper`, `gp_aovisimap_wrapper`, `gp_aovisimap_entry_wrapper`, `gp_aovisimap_hidden_info_wrapper`, among others | Diagnostics for append-optimized row-oriented (AO) and column-oriented (CO) tables: segment file metadata including historic (non-visible) entries, block directory entries, and visibility map information such as the number of hidden tuples per segment file. | + +## Usage notes + +- The view calls a function on every segment, so each query on it dispatches to the whole cluster. +- Dropping the extension removes the view and the functions in the `session_state` schema. diff --git a/docs/modules/gp-legacy-string-agg.md b/docs/modules/gp-legacy-string-agg.md new file mode 100644 index 0000000000..14565d3203 --- /dev/null +++ b/docs/modules/gp-legacy-string-agg.md @@ -0,0 +1,83 @@ +--- +title: gp_legacy_string_agg +--- + +# gp_legacy_string_agg + +The `gp_legacy_string_agg` module provides a single-argument `string_agg(text)` aggregate function. Apache Cloudberry, like PostgreSQL, provides only two-argument forms of `string_agg()` as built-in functions. + +:::note +The single-argument function is deprecated. Use this module only to keep legacy SQL running while you migrate it to the built-in two-argument `string_agg()` function. +::: + +## Installation + +`gp_legacy_string_agg` is always built and installed with Apache Cloudberry. It is part of the default `gpcontrib` build targets and does not depend on any `configure` option. + +The module is an extension (version `1.0.0`, relocatable). Before you can use the function, create the extension in each database where you want to use it: + +```sql +CREATE EXTENSION gp_legacy_string_agg; +``` + +The module does not require `shared_preload_libraries` or a restart. + +## Usage + +The single-argument aggregate function has the following signature: + +```sql +string_agg(text) +``` + +The function concatenates the non-null input values into a string without a delimiter. Each aggregate result (each group) raises a deprecation warning. The result is `NULL` when there are no non-null input values. For example: + +```sql +SELECT string_agg(a) FROM (VALUES('aaaa'),('bbbb'),('cccc'),(NULL)) g(a); +``` + +``` +WARNING: Deprecated call to string_agg(text), use string_agg(text, text) instead + string_agg +-------------- + aaaabbbbcccc +(1 row) +``` + +The result depends on the order of the input rows. The order is unspecified by default. To control it, use an `ORDER BY` clause within the aggregate call. For example: + +```sql +CREATE TABLE foo(a int, b text); +INSERT INTO foo VALUES(1, 'aaaa'),(2, 'bbbb'),(3, 'cccc'), (4, NULL); +SELECT string_agg(b ORDER BY a) FROM foo; +``` + +``` +WARNING: Deprecated call to string_agg(text), use string_agg(text, text) instead + string_agg +-------------- + aaaabbbbcccc +(1 row) +``` + +## Migrate to the two-argument string_agg() function + +The built-in aggregate function takes a delimiter as its second argument: + +```sql +string_agg(text, text) +``` + +A call with an empty delimiter is equivalent to the single-argument function that this module provides. For example, replace `string_agg(a)` with: + +```sql +string_agg(a, '') +``` + +After you convert all the calls, you can drop the extension: + +```sql +DROP EXTENSION gp_legacy_string_agg; +``` + +For more information about the built-in function, see [Aggregate Functions](https://www.postgresql.org/docs/14/functions-aggregate.html) in the PostgreSQL documentation. diff --git a/docs/modules/gp-relsizes-stats.md b/docs/modules/gp-relsizes-stats.md new file mode 100644 index 0000000000..1582460263 --- /dev/null +++ b/docs/modules/gp-relsizes-stats.md @@ -0,0 +1,192 @@ +--- +title: gp_relsizes_stats +--- + +# gp_relsizes_stats + +The `gp_relsizes_stats` module collects and stores the on-disk sizes of relation files in Apache Cloudberry. It scans the database directories on the segments, records the size and modification time of each relation file, and exposes views that aggregate these figures per table and per schema. + +Statistics can be collected periodically by a background worker, or on demand by calling a function. Naptime parameters let you spread the file scan over time to limit the I/O load. + +## Build and install + +`gp_relsizes_stats` is built and installed by default with Apache Cloudberry. No `configure` option is required. + +To build and install only this module from the source tree: + +```shell +make -C gpcontrib/gp_relsizes_stats +make -C gpcontrib/gp_relsizes_stats install +``` + +## Enable the module + +1. Add the library to `shared_preload_libraries` and restart the cluster. The periodic background worker is registered only when the library is preloaded: + + ```shell + gpconfig -c shared_preload_libraries -v "'gp_relsizes_stats'" --skipvalidation + gpstop -ra + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +2. Create the extension in each database for which you want to collect statistics: + + ```sql + CREATE EXTENSION gp_relsizes_stats; + ``` + + The extension is marked as trusted, so a non-superuser who has the `CREATE` privilege on the database can create it. The extension creates the `relsizes_stats_schema` schema. The role that creates the extension is granted `USAGE` on the schema and `SELECT` on its tables and views, with grant option. + +3. Optional: enable periodic collection. It is off by default: + + ```shell + gpconfig -c gp_relsizes_stats.enabled -v on --skipvalidation + gpstop -u + ``` + +## How collection works + +- The main background worker connects to the `postgres` database. When `gp_relsizes_stats.enabled` is on, it runs a collection cycle and then sleeps for `gp_relsizes_stats.restart_naptime`. +- A collection cycle goes through all databases except `template0`, `template1`, `diskquota`, and `gpperfmon`. For each database it starts a dynamic background worker. Databases in which the extension is not created are skipped. +- For each database, the worker rebuilds `segment_file_map`, replaces the content of `segment_file_sizes` with a fresh scan of the database directories on all segments (the default tablespace and user tablespaces), and, if `gp_relsizes_stats.save_history` is on, refreshes `table_sizes_history`. +- Each collection cycle needs free background worker slots. If a worker cannot be registered, the error message hints at increasing `max_worker_processes`. If this error occurs in the periodic worker, the worker exits and does not start again until the cluster is restarted. + +## Configuration parameters + +All parameters can be set only in the server configuration and take effect after a configuration reload. The naptime parameters take plain integers in milliseconds. Values with a unit, such as `'6h'`, are not accepted. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `gp_relsizes_stats.enabled` | boolean | `off` | Enables periodic collection by the background worker. | +| `gp_relsizes_stats.save_history` | boolean | `on` | Saves table sizes to `relsizes_stats_schema.table_sizes_history` during collection. | +| `gp_relsizes_stats.restart_naptime` | integer (ms) | `21600000` | Time between two collection cycles. The default is 6 hours. Range: `0` to `2147483647`. | +| `gp_relsizes_stats.database_naptime` | integer (ms) | `0` | Total pause that is spread between databases in one cycle. The value is divided by the number of databases, and the worker sleeps for the result after each database. Range: `0` to `2147483647`. | +| `gp_relsizes_stats.file_naptime` | integer (ms) | `1` | Pause after each file during the file scan. Range: `0` to `2147483647`. | + +## SQL objects + +All objects are created in the `relsizes_stats_schema` schema. + +### Functions + +| Function | Returns | Description | +|---|---|---| +| `relsizes_collect_stats_once()` | `void` | Runs one collection cycle for all databases, without the naptime pauses. It does not check `gp_relsizes_stats.enabled`. | +| `get_stats_for_database(dboid oid, fast bool)` | `TABLE (segment integer, relfilenode oid, filepath text, size bigint, mtime bigint)` | Runs on all segments. Scans the directories of the database `dboid` and returns one row per relation file. When `fast` is `true`, the scan does not pause between files. It is used by the collection process. | + +### Tables + +`segment_file_map`: maps relations to their relation file nodes on each segment. Append-optimized, randomly distributed. + +| Column | Type | Description | +|---|---|---| +| `segment` | `integer` | Segment content ID. | +| `reloid` | `oid` | OID of the relation. | +| `relfilenode` | `oid` | Relation file node on that segment. | + +`segment_file_sizes`: the result of the last file scan. Append-optimized, randomly distributed. + +| Column | Type | Description | +|---|---|---| +| `segment` | `integer` | Segment content ID. | +| `relfilenode` | `oid` | Relation file node, parsed from the file name. | +| `filepath` | `text` | Path of the file. | +| `size` | `bigint` | File size in bytes. | +| `mtime` | `bigint` | Last modification time of the file, as a Unix timestamp. | + +`table_sizes_history`: table sizes with the date of collection. Randomly distributed. + +| Column | Type | Description | +|---|---|---| +| `insert_date` | `date` | Date on which the row was collected. | +| `nspname` | `text` | Schema name. | +| `relname` | `text` | Table name. | +| `size` | `bigint` | Table size in bytes. | +| `mtime` | `timestamp` | Latest modification time among the files of the table. | + +### Views + +`table_files`: one row per file of each table (relation kinds `r` and `p`, excluding the `pg_global` tablespace), including the files of TOAST tables, TOAST indexes, and the auxiliary relations of append-optimized tables. Indexes on the table and the block directory of append-optimized tables are not included, so the sizes in `table_sizes` are not the total relation sizes. + +| Column | Type | Description | +|---|---|---| +| `nspname` | `name` | Schema name. | +| `relname` | `name` | Table name. For rows with `own_file = false`, this is the name of the root partitioned table. | +| `segment` | `integer` | Segment content ID. | +| `relfilenode` | `oid` | Relation file node. | +| `filepath` | `text` | Path of the file. `NULL` if the relation has no file on disk. | +| `kind` | `text` | One of `main`, `toast`, `toast_idx`, `ao`, `ao_vm`, `ao_vm_idx`. | +| `size` | `bigint` | File size in bytes, `0` if the relation has no file. | +| `mtime` | `bigint` | Modification time as a Unix timestamp, `0` if the relation has no file. | +| `own_file` | `boolean` | `true` if the file belongs to the relation named in `relname`. `false` if the file belongs to a descendant partition and is attributed to the root partitioned table. | + +`table_sizes`: total size per table. For a root partitioned table, the size includes all its descendant partitions. Each partition also appears as its own row. + +| Column | Type | Description | +|---|---|---| +| `nspname` | `name` | Schema name. | +| `relname` | `name` | Table name. | +| `size` | `numeric` | Sum of file sizes in bytes. | +| `mtime` | `timestamp with time zone` | Latest modification time among the files of the table. | + +`namespace_sizes`: total size per schema. Only rows with `own_file = true` are summed, so partitions are not counted twice. + +| Column | Type | Description | +|---|---|---| +| `nspname` | `name` | Schema name. | +| `size` | `numeric` | Sum of file sizes in bytes. | + +## Example + +Collect statistics on demand and check the size of a table: + +```sql +CREATE EXTENSION gp_relsizes_stats; + +CREATE TABLE employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + department_id INT, + date_of_birth DATE +); + +INSERT INTO employees (first_name, last_name, department_id, date_of_birth) VALUES +('John', 'Doe', 1, '1988-06-15'), +('Jane', 'Smith', 2, '1990-07-20'), +('Emily', 'Jones', 1, '1985-08-30'); + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; +``` + +```text + size +------- + 65536 +(1 row) +``` + +Check the size of a schema: + +```sql +SELECT size FROM relsizes_stats_schema.namespace_sizes WHERE nspname = 'public'; +``` + +To allow another role to read the statistics: + +```sql +GRANT USAGE ON SCHEMA relsizes_stats_schema TO user2; +GRANT SELECT ON ALL TABLES IN SCHEMA relsizes_stats_schema TO user2; +``` + +## Usage notes + +- The periodic background worker runs only when the library is loaded through `shared_preload_libraries`. `relsizes_collect_stats_once()` also works without preloading the library. +- File sizes are collected on the segments only. Files on the coordinator are not scanned. +- The sizes in `segment_file_sizes` and in the views reflect the last collection, not the current state of the files. +- Each collection deletes all rows of `table_sizes_history` in the database and inserts the current table sizes with the current date. The table therefore holds the result of the latest collection that ran with `gp_relsizes_stats.save_history` turned on. +- Partitioned tables (root and intermediate levels) have no files on disk. They appear in `table_files` with a size of `0`. +- The module is tested only on Linux. diff --git a/docs/modules/gp-sparse-vector.md b/docs/modules/gp-sparse-vector.md new file mode 100644 index 0000000000..1aa24d353d --- /dev/null +++ b/docs/modules/gp-sparse-vector.md @@ -0,0 +1,296 @@ +--- +title: gp_sparse_vector +--- + +# gp_sparse_vector + +The `gp_sparse_vector` module implements a sparse vector data type named `svec`, and the associated functions, operators, and aggregates. The type uses compressed storage of zeros to make vector computations on floating point numbers faster. + +## Installation + +`gp_sparse_vector` is always built and installed with Apache Cloudberry. It is part of the default `gpcontrib` build targets and does not depend on any `configure` option. + +The module is an extension (version `1.0.1`). Before you can use it, create the extension in each database where you want to use it: + +```sql +CREATE EXTENSION gp_sparse_vector; +``` + +The module does not require `shared_preload_libraries` or a restart. + +:::note +The extension creates all its objects in the `sparse_vector` schema. Either qualify the object names with the schema (for example `sparse_vector.svec`), or add the schema to the search path: + +```sql +SET search_path TO sparse_vector, public; +``` + +The examples in this document assume that `sparse_vector` is in the search path. +::: + +## About the svec data type + +When you use arrays of floating point numbers for various calculations, you often have long runs of zeros. This is common in scientific, retail optimization, and text processing applications. Each floating point number takes 8 bytes of storage in memory or on disk, so saving those zeros is often impractical. Many computations also benefit from skipping over the zeros. + +For example, suppose that the following array of doubles is stored as a `float8[]`: + +``` +'{0, 33, <40,000 zeros>, 12, 22 }'::float8[] +``` + +This type of array arises often in text processing, where a dictionary might have 40-100K terms and each vector stores the number of times that each word appears in a particular document. This array occupies slightly more than 320KB of memory or disk, most of it zeros. Any operation that you perform on this data works on 40,001 fields that are not important. + +The built-in array data type has a bitmap for null values, but it is a poor choice for this use case. It is not optimized for `float8[]` or for long runs of zeros instead of nulls, and the bitmap is not run-length-encoding (RLE) compressed. Even if each zero were stored as a `NULL` in the array, the bitmap for nulls would use 5KB to mark the nulls. + +The `svec` type uses a simple RLE-based scheme that is biased toward being efficient for zero values. With this scheme, the bitmap of the example vector needs only 6 bytes. + +:::note +`svec` supports only `float8` vector values. +::: + +### Create svec values + +You can construct an `svec` directly from a float array: + +```sql +SELECT ('{0, 13, 37, 53, 0, 71 }'::float8[])::svec; +``` + +The text representation of an `svec` has the form `{counts}:{values}`, where each count is the number of times that the corresponding value repeats. For example, `'{1,2,3,4}:{3,4,5,6}'::svec` is a vector of 10 elements: one `3`, two `4`s, three `5`s, and four `6`s. + +The module defines implicit casts between `svec` and `float8[]` in both directions, and implicit casts from `int2`, `integer`, `bigint`, `float4`, `float8`, and `numeric` to `svec`. + +## Operators + +The operators take advantage of the sparse storage format, which makes the computations on `svec` values faster. The `+`, `-`, `*`, `/`, and `%*%` operators accept `svec` and `float8[]` operands in any combination. A number that is cast to `svec` is a scalar and is applied to every element of the other operand. Otherwise, both operands must have the same dimension. + +| Operator | Description | +|---|---| +| `+` | Adds the corresponding elements of two vectors. | +| `-` | Subtracts the corresponding elements of two vectors. | +| `*` | Multiplies the corresponding elements of two vectors. | +| `/` | Divides the corresponding elements of two vectors. | +| `^` | Raises each element of the left `svec` to the power that the right operand gives. The right operand must be a scalar, for example `a ^ 2::svec`. A vector on the right side raises an error. | +| `%*%` | Returns the dot product of two vectors as a `float8`. | +| `\|\|` | Concatenates two `svec` values. | +| `*\|\|` | Concatenates an `svec` (right operand) to itself the number of times given by the `int4` left operand. | +| `=` | Tests two `svec` values, or two `float8[]` values, for equality. | +| `<`, `<=`, `==`, `>=`, `>` | Compare two `svec` values by their L2 norm. These operators are the members of the `svec_l2_ops` B-tree operator class. | +| `<>` | Defined, but not a member of the operator class. In this version it is bound to the same function as `==`, so it returns `true` when the L2 norms are equal. Do not rely on it. | + +The plus (`+`) operator adds each of the terms of two vectors of the same dimension. For example, if vector `a = {0,1,5}` and vector `b = {4,3,2}`, compute the vector addition as follows: + +```sql +SELECT ('{0,1,5}'::float8[]::svec + '{4,3,2}'::float8[]::svec)::float8[]; +``` + +``` + float8 +--------- + {4,4,7} +``` + +A vector dot product (`%*%`) between vectors `a` and `b` returns a scalar result of type `float8`. Compute the dot product (`0*4 + 1*3 + 5*2 = 13`) as follows: + +```sql +SELECT '{0,1,5}'::float8[]::svec %*% '{4,3,2}'::float8[]::svec; +``` + +``` + ?column? +---------- + 13 +``` + +## Functions and aggregates + +| Function | Description | +|---|---| +| `dot(a, b)` | Returns the dot product. Each argument is an `svec` or a `float8[]`. | +| `l1norm(svec)`, `l1norm(float8[])` | Returns the L1 norm of the vector. | +| `l2norm(svec)`, `l2norm(float8[])` | Returns the L2 norm of the vector. | +| `log(svec)` | Returns an `svec` that contains the natural logarithm of each element. | +| `dimension(svec)` | Returns the number of elements in the vector. | +| `unnest(svec)` | Returns the elements of the vector as a set of `float8` values. | +| `vec_sum(svec)`, `vec_sum(float8[])` | Returns the sum of the elements of the vector. | +| `vec_median(svec)`, `vec_median(float8[])` | Returns the median of the elements of the vector. | +| `vec_pivot(svec, float8)` | Appends a `float8` value to an `svec`. | +| `dmin(float8, float8)`, `dmax(float8, float8)` | Returns the smaller or the larger of two `float8` values. | +| `gp_extract_feature_histogram(text[], text[])` | Returns the sparse feature vector of a document (second argument) for a dictionary (first argument). | + +| Aggregate | Description | +|---|---| +| `sum(svec)` | Returns the element-wise sum of the input vectors. | +| `vec_count_nonzero(svec)` | Returns an `svec` that contains, for each position, the number of input vectors that have a non-zero element in the position. | +| `count_vec(svec)` | The legacy name of `vec_count_nonzero(svec)`. | +| `array_agg(float8)` | Aggregates `float8` values into an `svec`. | +| `median_inmemory(float8)` | Returns the median of the input `float8` values. | + +For example, for the set of vectors `{0,1,5}`, `{10,0,3}`, `{0,0,3}`, and `{0,1,0}`, the count of non-zero terms is `{1,2,3}`: + +```sql +CREATE TABLE listvecs(a svec); + +INSERT INTO listvecs VALUES ('{0,1,5}'::float8[]), + ('{10,0,3}'::float8[]), + ('{0,0,3}'::float8[]), + ('{0,1,0}'::float8[]); + +SELECT vec_count_nonzero(a)::float8[] AS count_nonzero FROM listvecs; +``` + +``` + count_nonzero +--------------- + {1,2,3} +(1 row) +``` + +## Example + +This text classification example creates tables that represent a dictionary and some documents, and then performs document classification with vector arithmetic on word counts and proportions of dictionary words in each document. + +Enable the extension and set the search path: + +```sql +CREATE EXTENSION gp_sparse_vector; +SET search_path TO sparse_vector, public; +``` + +Suppose that you have a dictionary that is composed of words in a text array. Create a table to store the dictionary and insert the words: + +```sql +CREATE TABLE features (dictionary text[][]) DISTRIBUTED RANDOMLY; +INSERT INTO features + VALUES ('{am,before,being,bothered,corpus,document,i,in,is,me,never,now,one,really,second,the,third,this,until}'); +``` + +You have a set of documents, each also defined as an array of words. Create a table to represent the documents and insert some data: + +```sql +CREATE TABLE documents (docnum int, document text[]) DISTRIBUTED RANDOMLY; +INSERT INTO documents VALUES + (1,'{this,is,one,document,in,the,corpus}'), + (2,'{i,am,the,second,document,in,the,corpus}'), + (3,'{being,third,never,really,bothered,me,until,now}'), + (4,'{the,document,before,me,is,the,third,document}'); +``` + +To find the dictionary words that are present in each document, prepare a Sparse Feature Vector (SFV) for each document. An SFV is a vector of dimension N, where N is the number of dictionary words, and each SFV contains a count of each dictionary word in the document. + +Use the `gp_extract_feature_histogram()` function to create an SFV from a document. The function returns an `svec` that contains the count of each of the dictionary words in the ordinal positions of the dictionary. The vectors that you generate for the documents are referred to as the corpus: + +```sql +CREATE TABLE corpus (docnum int, feature_vector svec) DISTRIBUTED RANDOMLY; + +INSERT INTO corpus + (SELECT docnum, + gp_extract_feature_histogram( + (SELECT dictionary FROM features LIMIT 1), document) + FROM documents); +``` + +Show the feature vector of each document: + +```sql +SELECT docnum AS document_number, feature_vector::float8[] FROM corpus ORDER BY 1; +``` + +``` + document_number | feature_vector +-----------------+----------------------------------------- + 1 | {0,0,0,0,1,1,0,1,1,0,0,0,1,0,0,1,0,1,0} + 2 | {1,0,0,0,1,1,1,1,0,0,0,0,0,0,1,2,0,0,0} + 3 | {0,0,1,1,0,0,0,0,0,1,1,1,0,1,0,0,1,0,1} + 4 | {0,1,0,0,0,2,0,0,1,1,0,0,0,0,0,2,1,0,0} +(4 rows) +``` + +The word "am" is the first word in the dictionary, and there is `1` instance of it in the SFV of the second document, "i am the second document in the corpus". The word "before" has no instances in the document, so its value is `0`, and so on. + +Count the number of documents in which each feature occurs at least once: + +```sql +SELECT (vec_count_nonzero(feature_vector))::float8[] AS count_in_document FROM corpus; +``` + +``` + count_in_document +----------------------------------------- + {1,1,1,1,2,3,1,2,2,2,1,1,1,1,1,3,2,1,1} +(1 row) +``` + +Count all occurrences of each term in all documents: + +```sql +SELECT (sum(feature_vector))::float8[] AS sum_in_document FROM corpus; +``` + +``` + sum_in_document +----------------------------------------- + {1,1,1,1,2,4,1,2,2,2,1,1,1,1,1,5,2,1,1} +(1 row) +``` + +The remainder of the classification process is vector math. The count is turned into a weight that reflects Term Frequency / Inverse Document Frequency (tf/idf). The calculation for a given term in a given document is: + +``` +#_times_term_appears_in_this_doc * log( #_docs / #_docs_the_term_appears_in ) +``` + +`#_docs` is the total number of documents (4 in this case). For example, the term "document" in document 1 has a weight of `1 * log(4/3)`. In document 4, the term has a weight of `2 * log(4/3)`. Terms that appear in every document have a weight of `0`. + +Calculate the tf/idf: + +```sql +SELECT docnum, (feature_vector*logidf)::float8[] AS tf_idf + FROM (SELECT log(count(feature_vector)/vec_count_nonzero(feature_vector)) AS logidf FROM corpus) + AS foo, corpus ORDER BY docnum; +``` + +``` + docnum | tf_idf +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 1 | {0,0,0,0,0.6931471805599453,0.28768207245178085,0,0.6931471805599453,0.6931471805599453,0,0,0,1.3862943611198906,0,0,0.28768207245178085,0,1.3862943611198906,0} + 2 | {1.3862943611198906,0,0,0,0.6931471805599453,0.28768207245178085,1.3862943611198906,0.6931471805599453,0,0,0,0,0,0,1.3862943611198906,0.5753641449035617,0,0,0} + 3 | {0,0,1.3862943611198906,1.3862943611198906,0,0,0,0,0,0.6931471805599453,1.3862943611198906,1.3862943611198906,0,1.3862943611198906,0,0,0.6931471805599453,0,1.3862943611198906} + 4 | {0,1.3862943611198906,0,0,0,0.5753641449035617,0,0,0.6931471805599453,0.6931471805599453,0,0,0,0,0,0.5753641449035617,0.6931471805599453,0,0} +(4 rows) +``` + +You can determine the angular distance between one document and the rest of the documents with the `ACOS` of the dot product of the document vectors. First, create a table that stores the tf/idf weighted vectors: + +```sql +CREATE TABLE weights AS + (SELECT docnum, (feature_vector*logidf) tf_idf + FROM (SELECT log(count(feature_vector)/vec_count_nonzero(feature_vector)) + AS logidf FROM corpus) foo, corpus ORDER BY docnum) + DISTRIBUTED RANDOMLY; +``` + +Then calculate the angular distance between the first document and every other document: + +```sql +SELECT docnum, trunc((180.*(ACOS(dmin(1.,(tf_idf%*%testdoc)/(l2norm(tf_idf)*l2norm(testdoc))))/(4.*ATAN(1.))))::numeric,2) + AS angular_distance FROM weights, + (SELECT tf_idf testdoc FROM weights WHERE docnum = 1 LIMIT 1) foo +ORDER BY 1; +``` + +``` + docnum | angular_distance +--------+------------------ + 1 | 0.00 + 2 | 78.82 + 3 | 90.00 + 4 | 80.02 +(4 rows) +``` + +The angular distance between document 1 and itself is 0 degrees, and between document 1 and document 3 is 90 degrees because they share no features at all. + +## Additional documentation + +For more information about this module, see the README files in the [`gpcontrib/gp_sparse_vector`](https://github.com/apache/cloudberry/tree/REL_2_STABLE/gpcontrib/gp_sparse_vector) directory of the Apache Cloudberry source code. diff --git a/docs/modules/gp-stats-collector.md b/docs/modules/gp-stats-collector.md new file mode 100644 index 0000000000..ff5e821f58 --- /dev/null +++ b/docs/modules/gp-stats-collector.md @@ -0,0 +1,284 @@ +--- +title: gp_stats_collector +--- + +# gp_stats_collector + +The `gp_stats_collector` module collects query execution metrics in Apache Cloudberry and reports them to an external monitoring agent. It hooks into the executor on the coordinator and on the segments, and for each query it sends lifecycle events (query text, plan, timestamps, user and database) together with system, instrumentation, interconnect, and spill metrics. Events are delivered over a Unix domain socket, or can be written to a log table inside the database. + +The module also provides an on-demand runtime query state interface (`pg_query_state`) that makes every backend of a running query report per-plan-node statistics without waiting for the query to finish. + +## Build and install + +`gp_stats_collector` is not built by default. To build it, configure Apache Cloudberry with the `--with-gp-stats-collector` option: + +```shell +./configure --with-gp-stats-collector [other options] +``` + +When this option is set, `configure` requires: + +- The protobuf library, version 3.0.0 or later (detected through `pkg-config`). +- The `protoc` compiler in `PATH`. + +The module contains C++ sources that are compiled with `-std=c++17`. + +## Enable the module + +1. Add the library to `shared_preload_libraries` and restart the cluster. The module allocates shared memory and registers signal handlers at server start, so it must be preloaded: + + ```shell + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' + gpstop -ra + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +2. Create the extension in the database where you want to use the SQL objects. The extension requires superuser privileges to create: + + ```sql + CREATE EXTENSION gp_stats_collector; + ``` + + The extension creates the `gpsc` schema. The default extension version is `1.2`. + +3. Turn on collection. Collection is off by default: + + ```sql + SET gpsc.enable TO on; + ``` + +## Collected data + +A query passes through four states, and the module sends one message per state from the dispatcher process (on the coordinator) and from the executor processes (on segments): + +```text +submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end -> ExecutorEnd() -> done +``` + +| Data | Description | Controlled by | +|---|---|---| +| Query lifecycle | Query text, normalized query text, timestamps (submit, start, end, done), user and database information. | `gpsc.enable` | +| `EXPLAIN` data | The output of `EXPLAIN (TEXT, COSTS, VERBOSE)` for the query. | `gpsc.enable` | +| `EXPLAIN ANALYZE` data | The output of `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)`. Sent only when the execution time reaches `gpsc.min_analyze_time`. | `gpsc.enable`, `gpsc.enable_analyze`, `gpsc.enable_cdbstats`, `gpsc.min_analyze_time` | +| Other metrics | Instrumentation, system, network, interconnect, and spill metrics. | `gpsc.enable` | + +Notes on the reported events: + +- Some queries skip the `end` state. In that case the `end` statistics are sent with `done`. +- If a query fails or is canceled, its statistics are sent with `done`. +- Some statistics are computed as the difference between the current value and a snapshot taken at `submit`. +- Each process (dispatcher or executor) sends its own statistics. +- A nested query on the dispatcher becomes a top-level query on the executors. + +## Configuration parameters + +### Collector parameters + +A parameter with the `superuser` context can be changed only by a superuser. A parameter with the `user` context can be changed by any role in its session. + +| Parameter | Type | Default | Context | Description | +|---|---|---|---|---| +| `gpsc.enable` | boolean | `off` | superuser | Enables the metrics collector. | +| `gpsc.uds_path` | string | `/tmp/gpsc_agent.sock` | superuser | File system path of the agent's Unix domain socket. | +| `gpsc.logging_mode` | enum | `uds` | superuser | Destination of collected events. `uds` sends events to the Unix domain socket. `tbl` writes events to the log table that is exposed through the `gpsc.log` view. Only superusers can view this setting. | +| `gpsc.enable_analyze` | boolean | `on` | superuser | Collects `EXPLAIN ANALYZE` data. When it is on, the dispatcher enables buffer, row, and timing instrumentation for the queries it runs. | +| `gpsc.enable_cdbstats` | boolean | `on` | superuser | Adds Cloudberry-specific executor statistics to the `EXPLAIN ANALYZE` data. Takes effect only when `gpsc.enable_analyze` is on. | +| `gpsc.min_analyze_time` | integer (ms) | `10000` | user | Minimum execution time above which the `EXPLAIN ANALYZE` text is reported. `0` reports it for all queries. `-1` turns the feature off. | +| `gpsc.report_nested_queries` | boolean | `on` | user | When on, both top-level and nested queries are reported from the coordinator, and segments report aggregates. When off, only top-level queries are reported from the coordinator and segments. | +| `gpsc.enable_utility` | boolean | `off` | user | Collects statistics for utility statements. | +| `gpsc.ignored_users_list` | string | empty | superuser | Comma-separated list of roles whose queries are not reported. | +| `gpsc.max_text_size` | integer (bytes) | `1048576` | superuser | Query texts and error messages longer than this size are trimmed. Range: `0` to `2147483647`. | +| `gpsc.max_plan_size` | integer (kB) | `1024` | superuser | Plan texts longer than this size are trimmed. Range: `0` to `2097151`. | + +:::note +It is not recommended to set `gpsc.max_text_size` or `gpsc.max_plan_size` higher than 1024 kB. +::: + +### Runtime query state parameters + +These parameters are registered only when the library is loaded through `shared_preload_libraries`. + +| Parameter | Type | Default | Context | Description | +|---|---|---|---|---| +| `pg_query_state.enable` | boolean | `on` | superuser | Enables the runtime query state module (executor hooks and signal handling). | +| `pg_query_state.enable_timing` | boolean | `on` | superuser | Collects timing data in addition to row counts. | +| `pg_query_state.enable_buffers` | boolean | `on` | superuser | Collects buffer usage. | + +## SQL objects + +All objects are created in the `gpsc` schema. `USAGE` on the schema is granted to `PUBLIC`. + +The `gpsc.log` and `gpsc.stat_messages` views are not granted to `PUBLIC`. All functions keep the default `EXECUTE` privilege for `PUBLIC`, and `gpsc.truncate_log()` and `gpsc.stat_messages_reset()` do not check the privileges of the caller. Revoke `EXECUTE` on these functions from `PUBLIC` if only administrators should run them. + +Interconnect statistics are collected only when the cluster uses the UDPIFC interconnect. + +### gpsc.stat_messages + +A view that shows, for the coordinator and for each segment, counters of the messages that the collector has tried to send. Querying it fails with an error if the library is not loaded through `shared_preload_libraries`. + +| Column | Type | Description | +|---|---|---| +| `segid` | `int` | Segment content ID. `-1` is the coordinator. | +| `total_messages` | `bigint` | Total number of messages, including failed ones. | +| `send_failures` | `bigint` | Number of messages that failed to be sent. | +| `connection_failures` | `bigint` | Number of failed connections to the agent socket. | +| `other_errors` | `bigint` | Number of other errors. | +| `max_message_size` | `int` | Size of the largest message. | + +### gpsc.stat_messages_reset() + +```sql +gpsc.stat_messages_reset() RETURNS SETOF void +``` + +Resets the counters shown in `gpsc.stat_messages` on the coordinator and on all segments. + +### gpsc.log + +A view over the log table on the coordinator and on all segments, ordered by `tmid`, `ssid`, `ccnt`. The table is filled when `gpsc.logging_mode` is `tbl`. Each row is one reported event. + +Rows are inserted in a way that keeps them even if the transaction that ran the query rolls back. + +| Column | Type | Description | +|---|---|---| +| `query_id` | `bigint` | Query ID. | +| `plan_id` | `bigint` | Hash of the normalized plan. | +| `nested_level` | `bigint` | Nesting level of the query. | +| `slice_id` | `bigint` | Slice ID. | +| `systemstat_vsize`, `systemstat_rss`, `systemstat_vmsizekb`, `systemstat_vmpeakkb` | `bigint` | Process memory figures from `/proc/pid/stat` and `/proc/pid/status`. | +| `systemstat_rchar`, `systemstat_wchar`, `systemstat_syscr`, `systemstat_syscw`, `systemstat_read_bytes`, `systemstat_write_bytes`, `systemstat_cancelled_write_bytes` | `bigint` | Process I/O counters from `/proc/pid/io`. | +| `instrumentation_ntuples`, `instrumentation_nloops`, `instrumentation_tuplecount` | `bigint` | Tuple and loop counters. | +| `instrumentation_shared_blks_hit`, `instrumentation_shared_blks_read`, `instrumentation_shared_blks_dirtied`, `instrumentation_shared_blks_written` | `bigint` | Shared buffer block counters. | +| `instrumentation_local_blks_hit`, `instrumentation_local_blks_read`, `instrumentation_local_blks_dirtied`, `instrumentation_local_blks_written` | `bigint` | Local buffer block counters. | +| `instrumentation_temp_blks_read`, `instrumentation_temp_blks_written` | `bigint` | Temporary file block counters. | +| `instrumentation_inherited_calls` | `bigint` | Number of nested queries. | +| `instrumentation_sent_total_bytes`, `instrumentation_sent_tuple_bytes`, `instrumentation_sent_chunks` | `bigint` | Network data sent. | +| `instrumentation_received_total_bytes`, `instrumentation_received_tuple_bytes`, `instrumentation_received_chunks` | `bigint` | Network data received. | +| `interconnect_total_recv_queue_size`, `interconnect_recv_queue_size_counting_time`, `interconnect_total_capacity`, `interconnect_capacity_counting_time`, `interconnect_total_buffers`, `interconnect_buffer_counting_time`, `interconnect_active_connections_num`, `interconnect_retransmits`, `interconnect_startup_cached_pkt_num`, `interconnect_mismatch_num`, `interconnect_crc_errors`, `interconnect_snd_pkt_num`, `interconnect_recv_pkt_num`, `interconnect_disordered_pkt_num`, `interconnect_duplicated_pkt_num`, `interconnect_recv_ack_num`, `interconnect_status_query_msg_num` | `bigint` | Interconnect statistics. | +| `spill_totalbytes` | `bigint` | Spill bytes written. | +| `systemstat_runningtimeseconds`, `systemstat_usertimeseconds`, `systemstat_kerneltimeseconds` | `double precision` | Wall clock, user, and kernel time in seconds. | +| `instrumentation_firsttuple`, `instrumentation_startup`, `instrumentation_total`, `instrumentation_blk_read_time`, `instrumentation_blk_write_time`, `instrumentation_startup_time`, `instrumentation_inherited_time` | `double precision` | Timing figures in seconds. | +| `datetime`, `submit_time`, `start_time`, `end_time` | `timestamp with time zone` | Event and query timestamps. | +| `tmid`, `ssid`, `ccnt` | `int` | Query key: a time-based ID from `gp_gettmid()`, the session ID, and the command counter. | +| `dbid` | `int` | The `dbid` of the reporting instance, as in `gp_segment_configuration`. | +| `segid` | `int` | Segment index. `-1` is the coordinator. | +| `spill_filecount` | `int` | Number of spill (temporary) files created. | +| `generator` | `text` | Planner or optimizer that produced the plan. | +| `query_text`, `template_query_text` | `text` | Query text and normalized query text. | +| `plan_text`, `template_plan_text` | `text` | `EXPLAIN` text and normalized plan text. | +| `analyze_text` | `text` | `EXPLAIN ANALYZE` text. | +| `user_name`, `database_name`, `rsgname` | `text` | Session user, database, and resource group. | +| `error_message` | `text` | Error message, if the query failed. | +| `query_status` | `text` | Event state, for example `QUERY_STATUS_SUBMIT`, `QUERY_STATUS_START`, `QUERY_STATUS_END`, `QUERY_STATUS_DONE`. | +| `utility` | `boolean` | Whether the event belongs to a utility statement. | + +### gpsc.truncate_log() + +```sql +gpsc.truncate_log() RETURNS SETOF void +``` + +Truncates the log table on the coordinator and on all segments. + +### gpsc.pg_query_state() + +```sql +gpsc.pg_query_state(pid int, trace_id bytea) RETURNS SETOF void +``` + +Triggers runtime per-node collection for the query that is running on the coordinator backend `pid`. The request is fanned out to the coordinator backend and to every segment backend that takes part in the query. Each backend walks its plan tree and pushes one batch of per-node statistics to the socket set in `gpsc.uds_path`. The coordinator also pushes the plan document, rate-limited so that repeated polls of a long query do not resend an unchanged plan. The function returns no rows and does not wait for the results. + +- `trace_id` must be exactly 16 bytes. The caller generates it, and the receiver uses it as the key of the collection. +- The function runs on the coordinator only. +- A backend cannot poll itself. + +### gpsc.pg_query_state_backends() + +```sql +gpsc.pg_query_state_backends(pid int) RETURNS TABLE(segid int, pid int) +``` + +Lists the segment backends that take part in the query running on backend `pid`, so that a collector knows how many batches to expect. A query that runs only on the coordinator is reported as a single row for the coordinator with `segid` less than `0`. The function returns an empty set when the target is not running a query or has `pg_query_state.enable` turned off. + +### Internal objects + +The following objects are created by the extension but are not meant to be called directly: + +- `gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[], bytea)` and the composite type `gpsc.gp_segment_pid (segid int, pid int)`: used by `gpsc.pg_query_state()` to dispatch the request to segments. +- Functions whose names start with two underscores, such as `gpsc.__stat_messages_f_on_master()` or `gpsc.__test_uds_start_server(text)`: helpers behind the views and functions above, and helpers for the regression tests. + +### Privileges + +`EXECUTE` on `gpsc.pg_query_state()`, `gpsc.pg_query_state_backends()`, and `gpsc.cbdb_mpp_query_state()` is granted to `PUBLIC`. Access is checked in the server: a caller can poll a backend only if the caller is a superuser or runs under the same role as the target backend. This lets a monitoring agent use a non-superuser role without being able to observe queries of other roles. + +## Example + +The following example writes events to the log table instead of the socket and then reads them back. + +```sql +CREATE EXTENSION gp_stats_collector; + +-- Helper to sort the events of one query in lifecycle order +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; + +SET gpsc.logging_mode to 'TBL'; + +SELECT 1; +SELECT COUNT(*) FROM generate_series(1,10); + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +``` + +```text + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE +(8 rows) +``` + +To clear the log table: + +```sql +SELECT gpsc.truncate_log(); +``` + +To poll a running query, generate a 16-byte trace ID and pass the process ID of the coordinator backend that runs the query: + +```sql +SELECT * FROM gpsc.pg_query_state_backends(12345); +SELECT gpsc.pg_query_state(12345, '\x00112233445566778899aabbccddeeff'::bytea); +``` + +## Usage notes + +- Delivery is best-effort. A message that cannot be written to the socket is dropped and is not retried, so a slow or absent agent does not add latency to queries. Use `gpsc.stat_messages` to check for failures. +- Runtime query state results are pushed to the agent socket. `gpsc.pg_query_state()` does not return them to the SQL caller. +- `gpsc.pg_query_state()` and `gpsc.pg_query_state_backends()` report an error if the library is not loaded through `shared_preload_libraries`, if no backend with the given `pid` exists, or if the caller is not allowed to poll the target backend. `gpsc.pg_query_state_backends()` also reports an error if `pid` belongs to a segment executor process instead of the coordinator backend of the session. +- The process-level system statistics are read from the `/proc` file system. diff --git a/docs/modules/hstore.md b/docs/modules/hstore.md new file mode 100644 index 0000000000..169f029507 --- /dev/null +++ b/docs/modules/hstore.md @@ -0,0 +1,21 @@ +--- +title: hstore +--- + +# hstore + +The `hstore` module implements the `hstore` data type for storing sets of key/value pairs within a single value. This can be useful in various scenarios, such as rows with many attributes that are rarely examined, or semi-structured data. + +The Apache Cloudberry `hstore` module is equivalent to the PostgreSQL `hstore` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`hstore` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION hstore; +``` + +## Module documentation + +See [hstore](https://www.postgresql.org/docs/14/hstore.html) in the PostgreSQL documentation for details about the data type, operators, and functions that this module defines. diff --git a/docs/modules/index.md b/docs/modules/index.md new file mode 100644 index 0000000000..28601e38f0 --- /dev/null +++ b/docs/modules/index.md @@ -0,0 +1,162 @@ +--- +title: Extension Modules Overview +--- + +# Extension Modules Overview + +Apache Cloudberry ships with a set of optional modules that add data types, functions, operators, foreign-data wrappers, storage formats, and diagnostic tools to the database. The modules live in two directories of the source tree: + +- `contrib/` holds the modules that come from PostgreSQL, plus several modules that Apache Cloudberry adds. +- `gpcontrib/` holds the modules that are specific to Apache Cloudberry. + +This page lists every module in both directories as of Apache Cloudberry 2.2, with its version, its history across the 2.x releases, and how it is built. + +## Use a module + +Most modules are packaged as extensions. An extension is registered once in every database in which you want to use it: + +```sql +CREATE EXTENSION ; +``` + +To list the extensions that are installed on the cluster and the ones that are registered in the current database: + +```sql +SELECT name, default_version, installed_version FROM pg_available_extensions ORDER BY name; +``` + +Some modules are plain shared libraries that do not have SQL objects. You load such a module with `LOAD`, or by adding it to `shared_preload_libraries` or `session_preload_libraries`. The page of each module tells you which method applies. + +## How to read the tables + +**Version** is the extension version (`default_version` in the control file of the module) in Apache Cloudberry 2.2. A dash means that the module is a library or a program without an extension version. + +**2.x history** compares the 2.0, 2.1, and 2.2 releases with each other, and 2.0 with the 1.6 release: + +- **New in 2.0**, **New in 2.1**, **New in 2.2**: the release that first includes the module. +- **Updated in ...**: the extension version changed in that release. +- A dash means that the module is present in 2.0, 2.1, and 2.2 with the same version. + +**Build** tells you how the module gets built when you build Apache Cloudberry from source: + +- **Default**: `make` and `make install` at the top of the source tree build and install the module. +- **On request**: the module is in the source tree, but you need to build it explicitly, for example with `make -C contrib/ install`, or by building the `world` target. The build scripts in `devops/build/automation/cloudberry/scripts` of the source tree also build and install `contrib/`, so a cluster built with them has these modules, provided that the required `configure` options are set. +- **A `configure` option**: the module is built only when you pass that option to `configure`. Where the option is on by default, the table says so. +- **Not built**: the source is in the tree, but no makefile target of Apache Cloudberry builds it. + +The module files must be installed on the coordinator and on every segment host. + +:::note +A module name that links to a page in this section has been reviewed against the Apache Cloudberry 2.x source code. For the other PostgreSQL modules, the link points to the PostgreSQL documentation. Those modules follow the PostgreSQL behavior, and some of them might have limitations in a distributed cluster. +::: + +## Modules in contrib/ + +### PostgreSQL modules + +| Module | Description | Version | 2.x history | Build | +|---|---|---|---|---| +| [adminpack](https://www.postgresql.org/docs/14/adminpack.html) | Support functions for administration tools such as pgAdmin. | 2.1 | — | Not built | +| [amcheck](https://www.postgresql.org/docs/14/amcheck.html) | Verifies the logical consistency of relations and indexes. | 1.3 | — | On request | +| [auth_delay](https://www.postgresql.org/docs/14/auth-delay.html) | Pauses before reporting an authentication failure. | — | — | On request | +| [auto_explain](./auto-explain.md) | Logs execution plans of slow statements automatically. | — | — | Default | +| [bloom](https://www.postgresql.org/docs/14/bloom.html) | Index access method based on Bloom filters. | 1.0 | — | On request | +| bool_plperl | Transform between `bool` and PL/Perl. | 1.0 | — | On request, with `--with-perl` | +| [btree_gin](./btree-gin.md) | GIN operator classes with B-tree behavior. | 1.3 | — | Default | +| [btree_gist](https://www.postgresql.org/docs/14/btree-gist.html) | GiST operator classes with B-tree behavior. | 1.6 | — | On request | +| [citext](./citext.md) | Case-insensitive character string type. | 1.6 | — | Default | +| [cube](https://www.postgresql.org/docs/14/cube.html) | Data type for multidimensional cubes. | 1.5 | — | Not built | +| [dblink](./dblink.md) | Connects to other databases from within a session. | 1.2 | — | Default | +| [dict_int](https://www.postgresql.org/docs/14/dict-int.html) | Text search dictionary template for integers. | 1.0 | — | On request | +| [dict_xsyn](https://www.postgresql.org/docs/14/dict-xsyn.html) | Text search dictionary template for extended synonym processing. | 1.0 | — | On request | +| [earthdistance](https://www.postgresql.org/docs/14/earthdistance.html) | Calculates great-circle distances on the surface of the Earth. | 1.1 | — | Not built | +| [file_fdw](https://www.postgresql.org/docs/14/file-fdw.html) | Foreign-data wrapper for files in the server file system. | 1.0 | — | Default | +| [fuzzystrmatch](./fuzzystrmatch.md) | Determines similarities and distance between strings. | 1.1 | — | Default | +| [hstore](./hstore.md) | Stores sets of key/value pairs in a single value. | 1.8 | — | Default | +| hstore_plperl | Transform between `hstore` and PL/Perl. | 1.0 | — | On request, with `--with-perl` | +| hstore_plpython | Transform between `hstore` and PL/Python. | 1.0 | — | On request, with `--with-python` (on by default) | +| [intagg](https://www.postgresql.org/docs/14/intagg.html) | Integer aggregator and enumerator (obsolete). | 1.1 | — | On request | +| [intarray](https://www.postgresql.org/docs/14/intarray.html) | Functions, operators, and index support for arrays of integers. | 1.5 | — | On request | +| [isn](./isn.md) | Data types for international product numbering standards. | 1.2 | — | On request | +| jsonb_plperl | Transform between `jsonb` and PL/Perl. | 1.0 | — | On request, with `--with-perl` | +| jsonb_plpython | Transform between `jsonb` and PL/Python. | 1.0 | — | On request, with `--with-python` (on by default) | +| [lo](https://www.postgresql.org/docs/14/lo.html) | Large object maintenance. | 1.1 | — | Not built | +| [ltree](./ltree.md) | Data type for hierarchical tree-like labels. | 1.2 | — | On request | +| ltree_plpython | Transform between `ltree` and PL/Python. | 1.0 | — | On request, with `--with-python` (on by default) | +| [oid2name](https://www.postgresql.org/docs/14/oid2name.html) | Utility program that resolves OIDs and file nodes to names. | — | — | On request | +| [old_snapshot](https://www.postgresql.org/docs/14/oldsnapshot.html) | Shows the XID/time mapping that `old_snapshot_threshold` uses. | 1.0 | — | On request | +| [pageinspect](./pageinspect.md) | Inspects the contents of database pages at a low level. | 1.9 | — | Default | +| [passwordcheck](https://www.postgresql.org/docs/14/passwordcheck.html) | Checks the strength of passwords. | — | — | Default | +| [pg_buffercache](./pg-buffercache.md) | Examines the shared buffer cache. | 1.4.1 | **Updated in 2.0** (1.3 to 1.4.1) | Default | +| [pg_freespacemap](https://www.postgresql.org/docs/14/pgfreespacemap.html) | Examines the free space map. | 1.2 | — | On request | +| [pg_prewarm](https://www.postgresql.org/docs/14/pgprewarm.html) | Loads relation data into the buffer cache. | 1.2 | — | On request | +| [pg_standby](https://www.postgresql.org/docs/14/pgstandby.html) | Utility program for warm standby servers. | — | — | Not built | +| [pg_stat_statements](https://www.postgresql.org/docs/14/pgstatstatements.html) | Tracks planning and execution statistics of SQL statements. | 1.9 | — | On request | +| [pg_surgery](https://www.postgresql.org/docs/14/pgsurgery.html) | Performs surgery on a damaged relation. | 1.0 | — | On request | +| [pg_trgm](./pg-trgm.md) | Text similarity measurement and index searching based on trigrams. | 1.6 | — | Default | +| [pg_visibility](https://www.postgresql.org/docs/14/pgvisibility.html) | Examines the visibility map and page-level visibility information. | 1.2 | — | On request | +| [pgcrypto](./pgcrypto.md) | Cryptographic functions. | 1.3 | — | Default | +| [pgrowlocks](https://www.postgresql.org/docs/14/pgrowlocks.html) | Shows row-level locking information. | 1.2 | — | On request | +| [pgstattuple](https://www.postgresql.org/docs/14/pgstattuple.html) | Shows tuple-level statistics. | 1.5 | — | On request | +| [postgres_fdw](./postgres-fdw.md) | Foreign-data wrapper for data stored in external PostgreSQL servers. | 1.1 | — | On request | +| [seg](https://www.postgresql.org/docs/14/seg.html) | Data type for line segments or floating-point intervals. | 1.4 | — | On request | +| [sepgsql](https://www.postgresql.org/docs/14/sepgsql.html) | Label-based mandatory access control based on SELinux. | — | — | On request, with `--with-selinux` | +| [spi](https://www.postgresql.org/docs/14/contrib-spi.html) | Example trigger functions: `autoinc`, `insert_username`, `moddatetime`, and `refint`. | 1.0 | — | On request | +| [sslinfo](./sslinfo.md) | Information about the SSL certificate of the current client. | 1.2 | — | On request, requires OpenSSL | +| start-scripts | Example scripts for starting the server at boot time. | — | — | Not built | +| [tablefunc](./tablefunc.md) | Functions that return tables, including `crosstab`. | 1.0 | — | Default | +| [tcn](https://www.postgresql.org/docs/14/tcn.html) | Trigger function that notifies listeners of changes to a table. | 1.0 | — | On request | +| [test_decoding](https://www.postgresql.org/docs/14/test-decoding.html) | Example output plugin for logical decoding. | — | — | On request | +| [tsm_system_rows](./tsm-system-rows.md) | `SYSTEM_ROWS` table sampling method. | 1.0 | — | On request | +| [tsm_system_time](./tsm-system-time.md) | `SYSTEM_TIME` table sampling method. | 1.0 | — | On request | +| [unaccent](https://www.postgresql.org/docs/14/unaccent.html) | Text search dictionary that removes accents. | 1.1 | — | On request | +| [uuid-ossp](./uuid-ossp.md) | Functions that generate universally unique identifiers (UUIDs). | 1.1 | — | On request, with `--with-uuid=LIB` | +| [vacuumlo](https://www.postgresql.org/docs/14/vacuumlo.html) | Utility program that removes orphaned large objects. | — | — | On request | +| [xml2](https://www.postgresql.org/docs/14/xml2.html) | XPath querying and XSLT functions (obsolete). | 1.1 | — | On request, with `--with-libxml` | + +### Apache Cloudberry modules + +| Module | Description | Version | 2.x history | Build | +|---|---|---|---|---| +| extprotocol | Example of a custom protocol for external tables. | — | — | Default | +| formatter | Example of a custom data formatter for external tables. | — | — | Default | +| formatter_fixedwidth | Custom formatter that reads and writes fixed-width data files. | — | — | Default | +| indexscan | The `readindex()` function, which reads the entries of an index for testing and diagnosis. | — | — | Default | +| interconnect | The interconnect implementations (UDP, TCP, and proxy) as a loadable module. The `interconnect` extension adds views with interconnect statistics. | 1.0 | **Updated in 2.2** (packaged as an extension) | Default | +| pax_storage | The PAX table format. See [PAX Storage Format](../operate-with-data/pax-table-format.md). | — | **New in 2.0** | `--enable-pax` | +| sasdemo | Example programs and formatter functions for exchanging data with SAS. | — | — | Not built | +| try_convert | The `try_convert()` function for type casts that return a default value instead of an error. | 1.0 | **New in 2.2** | Default | +| udp2 | A reworked implementation of the UDP interconnect that is separated from the database kernel. | — | **New in 2.1** | `--enable-ic-udp2` | + +## Modules in gpcontrib/ + +| Module | Description | Version | 2.x history | Build | +|---|---|---|---|---| +| [diskquota](./diskquota.md) | Limits the disk space that schemas, roles, and tablespaces can use. | 2.3 | **New in 2.2** | `--with-diskquota` | +| gp_debug_numsegments | Gets and sets the default number of segments that new tables are created on. For development and testing. | 1.0 | — | Default (debug extensions) | +| [gp_distribution_policy](./gp-distribution-policy.md) | Checks whether the rows of a table are stored on the segments that the distribution policy expects. | 1.0 | — | Default | +| [gp_exttable_fdw](./gp-exttable-fdw.md) | Foreign-data wrapper that implements external tables. | 1.0 | — | Default | +| gp_inject_fault | Fault injection framework. For development and testing. | 1.0 | — | Default (debug extensions) | +| [gp_internal_tools](./gp-internal-tools.md) | Diagnostic views and functions for troubleshooting. | 1.0.0 | — | Default | +| [gp_legacy_string_agg](./gp-legacy-string-agg.md) | The single-argument `string_agg()` aggregate. | 1.0.0 | — | Default | +| [gp_relsizes_stats](./gp-relsizes-stats.md) | Tracks the on-disk sizes of tables. | 1.0 | **New in 2.2** | Default | +| gp_replica_check | Compares primary and mirror segments. For development and testing. | 0.0.1 | — | Default (debug extensions) | +| [gp_sparse_vector](./gp-sparse-vector.md) | Sparse vector data type and its operators. | 1.0.1 | — | Default | +| [gp_stats_collector](./gp-stats-collector.md) | Collects query and plan execution metrics and reports them to a monitoring agent. | 1.2 | **New in 2.2** | `--with-gp-stats-collector` | +| [gp_toolkit](../sys-catalogs/gp_toolkit.md) | Administrative views and functions. | 1.7 | **Updated in 2.0** (1.4 to 1.6), **Updated in 2.2** (1.6 to 1.7) | Default | +| gpcloud | The `s3` protocol for external tables and the `gpcheckcloud` utility. See [Load Data from Amazon S3](../data-loading/load-data-from-s3.md). | — | — | `--enable-gpcloud` | +| gpmapreduce | The `gpmapreduce` utility. | — | — | `--enable-mapreduce` | +| [orafce](./orafce.md) | Oracle compatibility functions and packages. | 4.9 | — | `--enable-orafce` | +| pg_hint_plan | Controls execution plans with hints in SQL comments. See [Query Plan Hints](../performance/optimize-queries/query-hints.md). | 1.3.9 | **New in 2.0** | Default | +| pxf_fdw | Foreign-data wrapper for the Platform Extension Framework (PXF). See [Load Data Using PXF](../data-loading/load-data-using-pxf.md). | 1.0 | — | `--enable-pxf` (on by default) | +| [reject_partition_fullscan](./reject-partition-fullscan.md) | Rejects queries that scan all partitions of a table without pruning. | 1.0 | **New in 2.2** | Default (debug extensions) | +| [yezzey](./yezzey.md) | Offloads the data of append-optimized tables to S3-compatible object storage. The source is a Git submodule. | 2.0 | **New in 2.2** | `--with-yezzey` | +| zstd | Zstandard compression for append-optimized tables. | — | — | `--with-zstd` (on by default) | + +The modules that are marked as debug extensions are built unless you pass `--disable-debug-extensions` to `configure`. Do not use `gp_debug_numsegments`, `gp_inject_fault`, or `gp_replica_check` in production clusters. + +### Removed modules + +| Module | Description | Removed in | +|---|---|---| +| quicklz | QuickLZ compression for append-optimized tables. | 2.1 | diff --git a/docs/modules/isn.md b/docs/modules/isn.md new file mode 100644 index 0000000000..d18ae3f99c --- /dev/null +++ b/docs/modules/isn.md @@ -0,0 +1,29 @@ +--- +title: isn +--- + +# isn + +The `isn` module provides data types for the international product numbering standards EAN13, UPC, ISBN (books), ISMN (music), and ISSN (serials). + +The Apache Cloudberry `isn` module is equivalent to the PostgreSQL `isn` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`isn` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/isn install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION isn; +``` + +## Module documentation + +See [isn](https://www.postgresql.org/docs/14/isn.html) in the PostgreSQL documentation for details about the data types, casts, and functions that this module defines. diff --git a/docs/modules/ltree.md b/docs/modules/ltree.md new file mode 100644 index 0000000000..636d482ff3 --- /dev/null +++ b/docs/modules/ltree.md @@ -0,0 +1,33 @@ +--- +title: ltree +--- + +# ltree + +The `ltree` module implements a data type named `ltree` that you can use to represent labels of data stored in a hierarchical tree-like structure. The module also provides extensive facilities for searching through label trees. + +The Apache Cloudberry `ltree` module is based on the PostgreSQL `ltree` module. See [MPP considerations](#mpp-considerations) for the difference. + +## Install and register the module + +`ltree` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/ltree install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION ltree; +``` + +## MPP considerations + +The extension does not provide a hash operator class for the `ltree` data type. For this reason, you cannot use an `ltree` column as the distribution key of a table. A table whose only column is of type `ltree` is created with random distribution. + +## Module documentation + +See [ltree](https://www.postgresql.org/docs/14/ltree.html) in the PostgreSQL documentation for details about the data types, operators, and functions that this module defines. diff --git a/docs/modules/orafce.md b/docs/modules/orafce.md new file mode 100644 index 0000000000..67df12b70d --- /dev/null +++ b/docs/modules/orafce.md @@ -0,0 +1,119 @@ +--- +title: orafce +--- + +# orafce + +The `orafce` module provides Oracle compatibility SQL functions, data types, and packages in Apache Cloudberry. It implements a subset of the functions and packages of the Oracle RDBMS, which helps you port Oracle applications. + +The module in Apache Cloudberry is a modified version of the open source [Orafce](https://github.com/orafce/orafce) extension. The source files are in the `gpcontrib/orafce` directory of the Apache Cloudberry source code and are based on Orafce 4.9 (4.9.2). + +## Installation + +`orafce` is not built by default. To build and install the module with Apache Cloudberry, specify the `--enable-orafce` option when you run `configure`: + +```shell +./configure --enable-orafce [other options...] +make +make install +``` + +The module is an extension (version `4.9`, not relocatable). Before you can use it, create the extension in each database where you want to use it: + +```sql +CREATE EXTENSION orafce; +``` + +## Schemas + +The extension creates its objects in the following schemas: + +| Schema | Content | +|---|---| +| `oracle` | Oracle compatible functions (for example `oracle.substr()`, `oracle.add_months()`, `oracle.nvl2()`, and `oracle.sysdate()`), the `oracle.date` domain, the `oracle.varchar2` and `oracle.nvarchar2` data types, and the `oracle.dual` view. | +| `dbms_output` | The `DBMS_OUTPUT` package. | +| `dbms_pipe` | The `DBMS_PIPE` package. | +| `dbms_utility` | The `DBMS_UTILITY` package. | +| `dbms_assert` | The `DBMS_ASSERT` package. | +| `dbms_random` | The `DBMS_RANDOM` package. | +| `dbms_sql` | The `DBMS_SQL` package. | +| `utl_file` | The `UTL_FILE` package. | +| `plvstr`, `plvchr`, `plvdate`, `plvsubst`, `plvlex` | The PLVision string, character, date, substitution, and lexer packages. | +| `plunit` | The PLUnit assertion functions. | + +## Usage + +To call the functions in the `oracle` schema without a schema qualifier, add the schema to the search path. For example, this command sets the default search path of a database: + +```sql +ALTER DATABASE db_name SET search_path = "$user", public, oracle; +``` + +Some functions in the `oracle` schema have the same name as a built-in function, for example `oracle.substr()`. The built-in `pg_catalog` schema is searched first by default. To give precedence to the Oracle compatible version, either qualify the function name with the schema, or put `oracle` before `pg_catalog` in the search path: + +```sql +SET search_path TO oracle, "$user", public, pg_catalog; +``` + +For example: + +```sql +SET search_path TO public, oracle; +SELECT add_months(date '2003-08-01', 3); +``` + +``` + add_months +------------ + 2003-11-01 +(1 row) +``` + +You can use the `oracle.varchar2` and `oracle.nvarchar2` data types as the distribution key or as the partition key of a table: + +```sql +CREATE TABLE t1 (str oracle.varchar2(30)) DISTRIBUTED BY (str); +``` + +### Configuration parameters + +The module defines the following configuration parameters. You can set them in a session with `SET`. + +| Parameter | Default | Description | +|---|---|---| +| `orafce.nls_date_format` | Not set | Emulates the date output behavior of Oracle. | +| `orafce.timezone` | `GMT` | The time zone that the `oracle.sysdate()` function uses and that `oracle.dbtimezone()` returns. | +| `orafce.varchar2_null_safe_concat` | `false` | Whether the concatenation of `varchar2` values is null safe. | +| `orafce.sys_guid_source` | `uuid_generate_v1` | The function that `oracle.sys_guid()` uses to make its result: `uuid_generate_v1`, `uuid_generate_v1mc`, or `uuid_generate_v4` from the `uuid-ossp` extension, or the built-in `gen_random_uuid`. In this version, the value `uuid_generate_v4` is stored as `uuid_generate_v1`. | +| `orafce.using_substring_zero_width_in_substr` | `warning_oracle` | The behavior of the `substr()` function when the substring length argument is zero. Valid values are `warning_oracle`, `warning_orafce`, `oracle`, and `orafce`. | + +## Differences from the original Orafce extension + +The module in Apache Cloudberry differs from the original Orafce 4.9 extension as follows: + +- The `oracle.decode()` functions are not created. Apache Cloudberry has built-in support for `decode()`: the parser transforms a `decode()` call into a `CASE` expression, so you can use `decode()` without the `orafce` extension. For example: + + ```sql + SELECT decode(2, 1, 'ABC', 2, 'DEF'); + ``` + + ``` + case + ------ + DEF + (1 row) + ``` + +- The `dbms_alert` package is not available. The `dbms_alert` schema and its functions are not created, because the implementation cannot run in an MPP environment. + +:::note +PostgreSQL 14, on which Apache Cloudberry is based, provides `sinh()`, `cosh()`, and `tanh()` as built-in functions. See [Mathematical Functions and Operators](https://www.postgresql.org/docs/14/functions-math.html) in the PostgreSQL documentation. You do not need `orafce` to use them. +::: + +## Additional documentation + +For detailed information about the individual functions and supporting objects in this module, see the following files in the [`gpcontrib/orafce`](https://github.com/apache/cloudberry/tree/REL_2_STABLE/gpcontrib/orafce) directory of the Apache Cloudberry source code: + +- `README.asciidoc` +- The documents in the `doc/orafce_documentation` directory +- The Oracle to PostgreSQL migration notes in the `doc/sql_migration` directory diff --git a/docs/modules/pageinspect.md b/docs/modules/pageinspect.md new file mode 100644 index 0000000000..0e5c0cb421 --- /dev/null +++ b/docs/modules/pageinspect.md @@ -0,0 +1,94 @@ +--- +title: pageinspect +--- + +# pageinspect + +The `pageinspect` module provides functions for low-level inspection of the contents of database pages. Only superusers can use these functions. + +The Apache Cloudberry `pageinspect` module is based on the PostgreSQL `pageinspect` module. The differences are described in [MPP considerations](#mpp-considerations) and [Functions added by Apache Cloudberry](#functions-added-by-apache-cloudberry). + +## Install and register the module + +`pageinspect` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION pageinspect; +``` + +If an older version of the extension is already registered in a database, update it to get the functions of the current version: + +```sql +ALTER EXTENSION pageinspect UPDATE; +``` + +## MPP considerations + +- `get_raw_page()` does not read pages of append-optimized tables. It also rejects relations that have no page storage of their own, such as views, foreign tables, and partitioned tables. +- A function that reads data from a relation reads it only from the instance on which the function runs. For example, when you call `get_raw_page()` for a user table in a normal session, the function runs on the coordinator, which stores no data for the table, and the call fails with the error `block number 0 is out of range for relation "..."`. System catalog tables can be read on the coordinator. + + To inspect pages on a segment, run the function on that segment, for example in a utility-mode connection to the segment. + +## Functions + +The module provides the PostgreSQL 14 `pageinspect` functions: + +| Group | Functions | +|-------|-----------| +| General | `get_raw_page`, `page_header`, `page_checksum`, `fsm_page_contents` | +| Heap | `heap_page_items`, `tuple_data_split`, `heap_page_item_attrs`, `heap_tuple_infomask_flags` | +| B-tree | `bt_metap`, `bt_page_stats`, `bt_page_items` | +| BRIN | `brin_page_type`, `brin_metapage_info`, `brin_revmap_data`, `brin_page_items` | +| GIN | `gin_metapage_info`, `gin_page_opaque_info`, `gin_leafpage_items` | +| GiST | `gist_page_opaque_info`, `gist_page_items`, `gist_page_items_bytea` | +| Hash | `hash_page_type`, `hash_page_stats`, `hash_page_items`, `hash_bitmap_info`, `hash_metapage_info` | + +See [pageinspect](https://www.postgresql.org/docs/14/pageinspect.html) in the PostgreSQL documentation for details about these functions. + +## Functions added by Apache Cloudberry + +### Bitmap index functions + +| Function | Description | +|----------|-------------| +| `bm_metap(relname text) returns record` | Returns information about the meta page of a bitmap index: `magic`, `version`, `auxrelid`, `auxindexrelid`, and `lovlastblknum`. | +| `bm_bitmap_page_header(relname text, blkno int) returns record` | Returns the header information of a bitmap page: `num_words`, `next_blkno`, and `last_tid`. | +| `bm_lov_page_items(relname text, blkno int) returns setof record` | Returns the list of values (LOV) items of a bitmap LOV page. | +| `bm_bitmap_page_items(relname text, blkno int) returns setof record` | Returns the content words of a bitmap page and whether each word is compressed: `word_num`, `compressed`, and `content_word`. | +| `bm_bitmap_page_items(page bytea) returns setof record` | Same as the previous function, for a page image obtained with `get_raw_page()`. | + +`bm_lov_page_items()` returns the columns `itemoffset`, `lov_head_blkno`, `lov_tail_blkno`, `last_complete_word`, `last_word`, `last_tid`, `last_setbit_tid`, `is_last_complete_word_fill`, and `is_last_word_fill`. + +### BRIN functions + +Apache Cloudberry supports BRIN indexes on append-optimized tables, and `pageinspect` reflects this: + +- `brin_metapage_info(page bytea)` returns four columns in addition to the PostgreSQL ones: `isAO` (`boolean`), `firstrevmappages` (`bigint[]`), `lastrevmappages` (`bigint[]`), and `lastrevmappagenums` (`bigint[]`). The arrays describe the range map chains of an index on an append-optimized table. +- `brin_revmap_chain(indexrelid regclass, segno int) returns bigint[]` returns the chain of range map block numbers for a segment file number (block sequence) of an append-optimized table. `segno` must be in the range from 0 to 127. + +## Examples + +Get information about the meta page of the bitmap index named `i1`: + +```sql +SELECT * FROM bm_metap('i1'); +``` + +Display the header information of the second block of the bitmap index `i1`: + +```sql +SELECT * FROM bm_bitmap_page_header('i1', 2); +``` + +Display the LOV items in the first block of the bitmap index `i1`: + +```sql +SELECT * FROM bm_lov_page_items('i1', 1) ORDER BY itemoffset; +``` + +Return the content words in the second block of the bitmap index `i1`, either by index name and block number, or from a page image: + +```sql +SELECT * FROM bm_bitmap_page_items('i1', 2) ORDER BY word_num; +SELECT * FROM bm_bitmap_page_items(get_raw_page('i1', 2)) ORDER BY word_num; +``` diff --git a/docs/modules/pg-buffercache.md b/docs/modules/pg-buffercache.md new file mode 100644 index 0000000000..30a673d6eb --- /dev/null +++ b/docs/modules/pg-buffercache.md @@ -0,0 +1,104 @@ +--- +title: pg_buffercache +--- + +# pg_buffercache + +The `pg_buffercache` module provides a means for examining what is happening in the shared buffer cache in real time. + +The Apache Cloudberry `pg_buffercache` module is based on the PostgreSQL `pg_buffercache` module. In addition to the PostgreSQL views and functions, which report on the instance that you are connected to, it provides `gp_`-prefixed views that collect the same information from the coordinator and from all primary segments. + +## Install and register the module + +`pg_buffercache` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION pg_buffercache; +``` + +By default, access to the views and functions of this module is restricted to superusers and to members of the `pg_monitor` role. + +## Views and functions for a single instance + +These objects report on the shared buffers of the instance that runs the query. When you query them in a normal session, that instance is the coordinator. + +| Object | Description | +|--------|-------------| +| `pg_buffercache` view, `pg_buffercache_pages()` function | One row for each shared buffer. | +| `pg_buffercache_summary` view, `pg_buffercache_summary()` function | One row that summarizes the state of the shared buffer cache. | +| `pg_buffercache_usage_counts` view, `pg_buffercache_usage_counts()` function | One row for each usage count value. | + +The `pg_buffercache` view has the following columns: + +| Column | Type | Description | +|--------|------|-------------| +| `bufferid` | `integer` | ID of the buffer, in the range from 1 to `shared_buffers`. | +| `relfilenode` | `oid` | Filenode number of the relation. | +| `reltablespace` | `oid` | Tablespace OID of the relation. | +| `reldatabase` | `oid` | Database OID of the relation. | +| `relforknumber` | `smallint` | Fork number within the relation. | +| `relblocknumber` | `bigint` | Page number within the relation. | +| `isdirty` | `boolean` | Whether the page is dirty. | +| `usagecount` | `smallint` | Clock-sweep access count. | +| `pinning_backends` | `integer` | Number of backends that pin this buffer. | + +The `pg_buffercache_summary` view has the following columns: + +| Column | Type | Description | +|--------|------|-------------| +| `buffers_used` | `integer` | Number of used shared buffers. | +| `buffers_unused` | `integer` | Number of unused shared buffers. | +| `buffers_dirty` | `integer` | Number of dirty shared buffers. | +| `buffers_pinned` | `integer` | Number of pinned shared buffers. | +| `usagecount_avg` | `double precision` | Average usage count of the used shared buffers. | + +The `pg_buffercache_usage_counts` view has the following columns: + +| Column | Type | Description | +|--------|------|-------------| +| `usage_count` | `integer` | A possible buffer usage count. | +| `buffers` | `integer` | Number of buffers with this usage count. | +| `dirty` | `integer` | Number of dirty buffers with this usage count. | +| `pinned` | `integer` | Number of pinned buffers with this usage count. | + +The summary and usage count views are much cheaper to query than `pg_buffercache`. + +## Cluster-wide views + +| View | Description | +|------|-------------| +| `gp_buffercache` | The `pg_buffercache` rows of the coordinator and of every primary segment. | +| `gp_buffercache_summary` | The `pg_buffercache_summary` row of the coordinator and of every primary segment: one row per instance. | +| `gp_buffercache_usage_counts` | The `pg_buffercache_usage_counts` rows of the coordinator and of every primary segment. | +| `gp_buffercache_summary_aggregated` | A single row that aggregates `gp_buffercache_summary` over all instances: the sum of each buffer counter and the average of `usagecount_avg`. | +| `gp_buffercache_usage_counts_aggregated` | A single row that contains the sums of the columns of `gp_buffercache_usage_counts`, including the `usage_count` column itself. It is not a breakdown per usage count. | + +The first three views have the columns of the corresponding `pg_` view, preceded by a `gp_segment_id` column. The value of `gp_segment_id` is `-1` for the coordinator. + +## Examples + +Show the buffer cache summary of each instance: + +```sql +SELECT * FROM gp_buffercache_summary; +``` + +Show the totals for the whole cluster: + +```sql +SELECT * FROM gp_buffercache_summary_aggregated; +``` + +Count the dirty buffers on each instance: + +```sql +SELECT gp_segment_id, count(*) AS dirty_buffers +FROM gp_buffercache +WHERE isdirty +GROUP BY gp_segment_id +ORDER BY gp_segment_id; +``` + +## Module documentation + +See [pg_buffercache](https://www.postgresql.org/docs/14/pgbuffercache.html) in the PostgreSQL documentation for more information about the `pg_buffercache` view. The summary and usage count functions were added to PostgreSQL in version 16 and are described in the [PostgreSQL 16 documentation](https://www.postgresql.org/docs/16/pgbuffercache.html). diff --git a/docs/modules/pg-trgm.md b/docs/modules/pg-trgm.md new file mode 100644 index 0000000000..23e47c9b9c --- /dev/null +++ b/docs/modules/pg-trgm.md @@ -0,0 +1,21 @@ +--- +title: pg_trgm +--- + +# pg_trgm + +The `pg_trgm` module provides functions and operators for determining the similarity of alphanumeric text based on trigram matching. The module also provides GiST and GIN index operator classes that support fast searching for similar strings. + +The Apache Cloudberry `pg_trgm` module is equivalent to the PostgreSQL `pg_trgm` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`pg_trgm` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION pg_trgm; +``` + +## Module documentation + +See [pg_trgm](https://www.postgresql.org/docs/14/pgtrgm.html) in the PostgreSQL documentation for details about the functions, operators, and index operator classes in this module. diff --git a/docs/modules/pgcrypto.md b/docs/modules/pgcrypto.md new file mode 100644 index 0000000000..c4bf02d849 --- /dev/null +++ b/docs/modules/pgcrypto.md @@ -0,0 +1,83 @@ +--- +title: pgcrypto +--- + +# pgcrypto + +The `pgcrypto` module provides cryptographic functions: general hashing, password hashing, PGP encryption, raw encryption, and random data generation. With these functions, you can store certain columns of data in encrypted form. Data that is stored encrypted cannot be read by anyone who does not have the encryption key, and it cannot be read directly from the disks. + +:::note +The `pgcrypto` functions run inside the database server, which means that all the data and passwords move between `pgcrypto` and the client application in clear text. For better security, also use SSL connections between the client and the Apache Cloudberry coordinator. +::: + +## Install and register the module + +`pgcrypto` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION pgcrypto; +``` + +## Functions + +| Group | Functions | +|-------|-----------| +| General hashing | `digest()`, `hmac()` | +| Password hashing | `crypt()`, `gen_salt()` | +| PGP encryption | `pgp_sym_encrypt()`, `pgp_sym_encrypt_bytea()`, `pgp_sym_decrypt()`, `pgp_sym_decrypt_bytea()`, `pgp_pub_encrypt()`, `pgp_pub_encrypt_bytea()`, `pgp_pub_decrypt()`, `pgp_pub_decrypt_bytea()`, `pgp_key_id()`, `armor()`, `dearmor()`, `pgp_armor_headers()` | +| Raw encryption | `encrypt()`, `decrypt()`, `encrypt_iv()`, `decrypt_iv()` | +| Random data | `gen_random_bytes()`, `gen_random_uuid()` | + +Example: + +```sql +SELECT digest('santa claus', 'sha256'); +SELECT pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=aes256'), 'mypass'); +``` + +In addition to the ciphers of the PostgreSQL module, the raw encryption functions accept the SM4 ciphers `sm4-ecb` and `sm4-cbc`, and `digest()` and `hmac()` accept the `sm3` digest. In a build that uses OpenSSL, the SM4 ciphers are available only if Apache Cloudberry was configured with `--enable-openssl-redirect`. Otherwise, they fail with `Cannot use "sm4-ecb": No such cipher algorithm`. The `sm3` digest is available if the OpenSSL library provides it. + +## FIPS mode + +The module defines the configuration parameter `pgcrypto.fips`, which restricts `pgcrypto` to a limited set of functionality that is suitable for FIPS 140-2 mode. The parameter is a Boolean, its default value is `off`, and only superusers can change it. + +FIPS mode requires an OpenSSL library that provides the FIPS module, on all hosts of the cluster. If the OpenSSL version that Apache Cloudberry was compiled against has no FIPS support, turning the parameter on fails with the error `FIPS enabled OpenSSL is required for strict FIPS mode`. A build without OpenSSL does not support FIPS mode. + +When `pgcrypto.fips` is `on`: + +- FIPS mode is turned on in the OpenSSL library. +- `crypt()` and `gen_salt()` are not allowed. +- The raw encryption functions (`encrypt()`, `decrypt()`, `encrypt_iv()`, and `decrypt_iv()`) are not allowed. +- `digest()` and `hmac()` reject digests that are not FIPS-approved, such as `md5`, with an error that starts with `Cannot use "md5"`. +- The PGP functions are not usable either. They recognize only the `3des`, `aes128` (`aes`), `aes192`, and `aes256` cipher names and the `sha1`, `sha256`, `sha384`, and `sha512` digest names, and reject other names, such as `bf`, with the error `Unsupported cipher algorithm`. Encryption and decryption with the recognized ciphers then fail with `requested functionality not allowed in FIPS mode`, because `pgcrypto` reaches these ciphers through an OpenSSL interface that is not permitted in FIPS mode. + +A call to `crypt()`, `gen_salt()`, or a raw encryption function fails with the error `requested functionality not allowed in FIPS mode`. + +### Enable FIPS mode + +1. Add the `pgcrypto` library to `shared_preload_libraries`, and restart the cluster. This makes the `pgcrypto.fips` parameter known to the server before any `pgcrypto` function is called. + + ```shell + gpconfig -c shared_preload_libraries -v 'pgcrypto.so' + gpstop -ar + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +2. Set the parameter for each database that must use FIPS mode. For example, for the database `testdb`: + + ```sql + ALTER DATABASE testdb SET pgcrypto.fips TO on; + ``` + +3. Reconnect to the database, and check the setting: + + ```sql + SHOW pgcrypto.fips; + ``` + +To turn FIPS mode off, run `ALTER DATABASE testdb RESET pgcrypto.fips;`, and reconnect. + +## Module documentation + +See [pgcrypto](https://www.postgresql.org/docs/14/pgcrypto.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/docs/modules/postgres-fdw.md b/docs/modules/postgres-fdw.md new file mode 100644 index 0000000000..a28591ecd1 --- /dev/null +++ b/docs/modules/postgres-fdw.md @@ -0,0 +1,119 @@ +--- +title: postgres_fdw +--- + +# postgres_fdw + +The `postgres_fdw` module is a foreign-data wrapper (FDW) that you can use to access data stored in a remote PostgreSQL or Apache Cloudberry database. + +The Apache Cloudberry `postgres_fdw` module is a modified version of the PostgreSQL `postgres_fdw` module. It supports the MPP execution options of Apache Cloudberry foreign tables, and it has some restrictions that the PostgreSQL module does not have. Both are described below. + +## Install and register the module + +`postgres_fdw` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/postgres_fdw install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION postgres_fdw; +``` + +## Basic usage + +```sql +CREATE SERVER pgserver FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (host 'pghost', port '5432', dbname 'sales'); + +CREATE USER MAPPING FOR CURRENT_USER SERVER pgserver + OPTIONS (user 'remote_user', password 'secret'); + +CREATE FOREIGN TABLE orders_ft (id int, amount numeric) + SERVER pgserver + OPTIONS (schema_name 'public', table_name 'orders'); + +SELECT count(*) FROM orders_ft; +``` + +## Options + +The module accepts the connection options of `libpq` and the options of the PostgreSQL 14 `postgres_fdw` module: `schema_name`, `table_name`, `column_name`, `use_remote_estimate`, `fdw_startup_cost`, `fdw_tuple_cost`, `extensions`, `updatable`, `truncatable`, `fetch_size`, `batch_size`, `async_capable`, `keep_connections`, `password_required`, `sslcert`, and `sslkey`. + +The following options are specific to Apache Cloudberry: + +| Option | Where | Description | +|--------|-------|-------------| +| `mpp_execute` | Foreign table, server, or foreign-data wrapper | Apache Cloudberry option that determines where the foreign table is accessed: `'coordinator'` (the same as `'master'`), `'any'`, or `'all segments'`. | +| `num_segments` | Server | Number of segments that access a foreign table whose `mpp_execute` option is `'all segments'`. If you do not set this option, all segments of the cluster are used. | + +With `mpp_execute 'all segments'`, an `INSERT INTO SELECT ...` statement writes to the remote server from the segments in parallel. + +## Access multiple remote servers from one foreign table + +A foreign table whose `mpp_execute` option is `'all segments'` can read from several remote servers that hold tables of the same structure. Add each server to the foreign table as a foreign segment: + +```sql +CREATE SERVER mpps1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host 'pghost1', dbname 'fdw1'); +CREATE SERVER mpps2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host 'pghost2', dbname 'fdw2'); +CREATE USER MAPPING FOR CURRENT_USER SERVER mpps1; +CREATE USER MAPPING FOR CURRENT_USER SERVER mpps2; + +CREATE FOREIGN TABLE fs1 (a int, b text) + SERVER mpps1 + OPTIONS (schema_name 'public', table_name 't1', mpp_execute 'all segments'); + +ADD FOREIGN SEGMENT FROM SERVER mpps1 INTO fs1; +ADD FOREIGN SEGMENT FROM SERVER mpps2 INTO fs1; +``` + +A query on `fs1` then scans one remote server per query executor process and gathers the results on the coordinator: + +```sql +SET optimizer = off; +EXPLAIN (COSTS OFF) SELECT * FROM fs1; +``` + +```text + QUERY PLAN +------------------------------------------ + Gather Motion 2:1 (slice1; segments: 2) + -> Foreign Scan on fs1 + Optimizer: Postgres query optimizer +(3 rows) +``` + +For such a foreign table: + +- Aggregates can be pushed down to the remote servers as partial aggregates, which are finalized on the coordinator. +- A `LIMIT` clause is applied on each segment as a preliminary limit before the rows are gathered on the coordinator. +- The hidden column `gp_foreign_server` contains the OID of the foreign server that a row comes from. You can use it in join conditions, for example `fs1.gp_foreign_server = fs2.gp_foreign_server`, to join only rows that come from the same server. + +:::caution +Queries on a foreign table with foreign segments return correct results only with the Postgres-based planner. GPORCA ignores the foreign segments: it scans the table on all segments of the cluster, so some remote servers are read more than once and rows are duplicated, and `gp_foreign_server` is `0` for every row. Turn GPORCA off before you query such a table: + +```sql +SET optimizer = off; +``` +::: + +## Limitations + +- The `use_remote_estimate` option is not supported. If the option is present on a server or a foreign table, with any value, the module ignores it and reports the warning `fdw option 'use_remote_estimate' is not supported.` +- When the remote server is an Apache Cloudberry database, only `INSERT` is supported on the foreign table. `UPDATE` and `DELETE` are not supported. + +## Functions + +| Function | Description | +|----------|-------------| +| `postgres_fdw_get_connections(OUT server_name text, OUT valid boolean) returns setof record` | Returns the foreign server names of all open connections that `postgres_fdw` established from the local session, and whether each connection is valid. | +| `postgres_fdw_disconnect(server_name text) returns boolean` | Discards the open connections from the local session to the given foreign server. | +| `postgres_fdw_disconnect_all() returns boolean` | Discards all open connections from the local session to foreign servers. | + +## Module documentation + +See [postgres_fdw](https://www.postgresql.org/docs/14/postgres-fdw.html) in the PostgreSQL documentation for details about the options and behavior of this module. diff --git a/docs/modules/reject-partition-fullscan.md b/docs/modules/reject-partition-fullscan.md new file mode 100644 index 0000000000..2dc0ffd482 --- /dev/null +++ b/docs/modules/reject-partition-fullscan.md @@ -0,0 +1,128 @@ +--- +title: reject_partition_fullscan +--- + +# reject_partition_fullscan + +The `reject_partition_fullscan` module rejects queries that scan all partitions of a partitioned table without effective partition pruning. It installs a planner hook that inspects the finished plan, whether it comes from the Postgres-based planner or from GPORCA, and raises an error if a partitioned table would be scanned in full. + +Use it to prevent expensive queries that omit a filter on the partition key. + +## Build and install + +`reject_partition_fullscan` is one of the debug extensions under `gpcontrib`. These are built when `--enable-debug-extensions` is in effect, which is the default. If Apache Cloudberry is configured with `--disable-debug-extensions`, the module is not built. + +## Enable the module + +The module works by loading its shared library. It has no SQL objects. + +- To enable it for the whole cluster, add the library to `shared_preload_libraries` and restart the cluster: + + ```shell + gpconfig -c shared_preload_libraries -v 'reject_partition_fullscan' + gpstop -ra + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +- To enable it for the current session only, load the library with `LOAD`. This requires superuser privileges: + + ```sql + LOAD 'reject_partition_fullscan'; + ``` + +`CREATE EXTENSION reject_partition_fullscan` is available, but it creates no objects. The planner hook and the configuration parameters are registered when the library is loaded. + +:::note +The check is on by default once the library is loaded, because `reject_partition_fullscan` defaults to `on`. +::: + +## Configuration parameters + +Both parameters can be changed by any role in its session. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `reject_partition_fullscan` | boolean | `on` | Rejects queries on partitioned tables that scan all partitions without pruning. | +| `partition_fullscan_threshold` | integer | `0` | Maximum number of partitions that a query is allowed to scan after pruning. With `0`, a plan of the Postgres-based planner is rejected if more than one partition remains. See [Behavior](#behavior). Range: `0` to `2147483647`. | + +## Behavior + +After a statement is planned, the module walks the plan tree and checks every scan of a partitioned table: + +- With `partition_fullscan_threshold = 0`, a plan of the Postgres-based planner is rejected if more than one partition remains after plan-time pruning. A filter on the partition key that keeps two or more partitions is therefore rejected too. Set a positive threshold to allow such queries. +- With `partition_fullscan_threshold` greater than `0`, the statement is rejected if the number of partitions that remain after pruning is greater than the threshold. + +:::caution +The module works reliably with plans of the Postgres-based planner. GPORCA, which is the default optimizer, attaches run-time pruning information to its dynamic scans, and the module treats such scans as prunable at run time. As a result, `SELECT` statements that GPORCA plans are not rejected, even when they scan all partitions. `UPDATE` and `DELETE` statements on partitioned tables are planned by the Postgres-based planner and are checked. To enforce the check for `SELECT` statements, turn GPORCA off in the session with `SET optimizer = off`. +::: + +A rejected statement fails with an error of this form: + +```text +ERROR: partitioned table "." full partition scan is not allowed, partitions would be scanned +HINT: Add a WHERE clause on the partition key to enable partition pruning. +``` + +The check runs on every statement that is planned. It applies to `SELECT`, `UPDATE`, and `DELETE` statements, to partitioned tables referenced in subqueries, and to statements such as `EXPLAIN`, `INSERT ... SELECT`, and `CREATE TABLE AS` that read a partitioned table. + +The following cases are not rejected: + +- `reject_partition_fullscan` is `off`. +- `enable_partition_pruning` is `off`. +- The scan covers at most one partition, for example a partitioned table with a single partition. +- Partitions can be pruned at run time: plans with pruning steps that are evaluated at executor startup or during execution (for example a prepared statement with a parameter on the partition key), and GPORCA dynamic scans that use join-based partition pruning. + +## Example + +```sql +LOAD 'reject_partition_fullscan'; +SET optimizer = off; + +CREATE TABLE pfr_test (id int, dt date, val text) + PARTITION BY RANGE (dt); +CREATE TABLE pfr_test_p1 PARTITION OF pfr_test + FOR VALUES FROM ('2025-01-01') TO ('2025-04-01'); +CREATE TABLE pfr_test_p2 PARTITION OF pfr_test + FOR VALUES FROM ('2025-04-01') TO ('2025-07-01'); +CREATE TABLE pfr_test_p3 PARTITION OF pfr_test + FOR VALUES FROM ('2025-07-01') TO ('2025-10-01'); + +SET reject_partition_fullscan = on; +SET partition_fullscan_threshold = 0; + +-- Rejected: no filter on the partition key +SELECT * FROM pfr_test; +SELECT * FROM pfr_test WHERE val = 'x'; +UPDATE pfr_test SET val = 'y'; + +-- Allowed: the filter prunes the scan to one partition +SELECT * FROM pfr_test WHERE dt = '2025-02-01'; +DELETE FROM pfr_test WHERE dt = '2025-02-01'; + +-- Rejected: two partitions remain after pruning +SELECT * FROM pfr_test + WHERE dt >= '2025-01-01' AND dt < '2025-07-01'; + +-- Threshold mode: allow up to 2 partitions +SET partition_fullscan_threshold = 2; + +-- Allowed: pruned to 2 partitions +SELECT * FROM pfr_test + WHERE dt >= '2025-01-01' AND dt < '2025-07-01'; + +-- Rejected: 3 partitions exceed the threshold +SELECT * FROM pfr_test; +``` + +To allow a full scan for one session, turn the check off: + +```sql +SET reject_partition_fullscan = off; +``` + +## Usage notes + +- Both parameters can be changed by any role, so the module is a guard against accidental full scans, not an enforcement mechanism. +- The check runs at planning time on the plan that the optimizer produced. It does not consider how much data the partitions contain. +- When the library is loaded with `LOAD`, the check applies only to that session. diff --git a/docs/modules/sslinfo.md b/docs/modules/sslinfo.md new file mode 100644 index 0000000000..8c0d9119bb --- /dev/null +++ b/docs/modules/sslinfo.md @@ -0,0 +1,51 @@ +--- +title: sslinfo +--- + +# sslinfo + +The `sslinfo` module provides information about the SSL certificate that the current client provided when it connected to Apache Cloudberry. Most functions in this module return `NULL` if the current connection does not use SSL. + +The Apache Cloudberry `sslinfo` module is equivalent to the PostgreSQL `sslinfo` module. The functions report on the client connection to the coordinator. + +## Install and register the module + +`sslinfo` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry, and it can be built only when Apache Cloudberry is configured with OpenSSL support (`--with-ssl=openssl`, or the older spelling `--with-openssl`). If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/sslinfo install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION sslinfo; +``` + +## Functions + +| Function | Description | +|----------|-------------| +| `ssl_is_used() returns boolean` | Returns `true` if the current connection uses SSL. | +| `ssl_version() returns text` | Returns the name of the protocol used for the SSL connection. | +| `ssl_cipher() returns text` | Returns the name of the cipher used for the SSL connection. | +| `ssl_client_cert_present() returns boolean` | Returns `true` if the client presented a valid SSL certificate. | +| `ssl_client_serial() returns numeric` | Returns the serial number of the client certificate. | +| `ssl_client_dn() returns text` | Returns the full subject of the client certificate. | +| `ssl_issuer_dn() returns text` | Returns the full issuer name of the client certificate. | +| `ssl_client_dn_field(fieldname text) returns text` | Returns one field of the subject of the client certificate. | +| `ssl_issuer_field(fieldname text) returns text` | Returns one field of the issuer of the client certificate. | +| `ssl_extension_info() returns setof record` | Returns the name, value, and criticality of each extension of the client certificate. | + +## Example + +```sql +SELECT ssl_is_used(), ssl_version(), ssl_cipher(); +SELECT ssl_client_dn_field('CN'); +``` + +## Module documentation + +See [sslinfo](https://www.postgresql.org/docs/14/sslinfo.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/docs/modules/tablefunc.md b/docs/modules/tablefunc.md new file mode 100644 index 0000000000..f237496691 --- /dev/null +++ b/docs/modules/tablefunc.md @@ -0,0 +1,21 @@ +--- +title: tablefunc +--- + +# tablefunc + +The `tablefunc` module provides various functions that return tables (that is, multiple rows), such as `normal_rand()`, `crosstab()`, and `connectby()`. + +The Apache Cloudberry `tablefunc` module is equivalent to the PostgreSQL `tablefunc` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`tablefunc` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION tablefunc; +``` + +## Module documentation + +See [tablefunc](https://www.postgresql.org/docs/14/tablefunc.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/docs/modules/tsm-system-rows.md b/docs/modules/tsm-system-rows.md new file mode 100644 index 0000000000..bad2f45dd7 --- /dev/null +++ b/docs/modules/tsm-system-rows.md @@ -0,0 +1,47 @@ +--- +title: tsm_system_rows +--- + +# tsm_system_rows + +The `tsm_system_rows` module implements the `SYSTEM_ROWS` table sampling method, which you can use in the `TABLESAMPLE` clause of a `SELECT` command. + +## Install and register the module + +`tsm_system_rows` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/tsm_system_rows install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION tsm_system_rows; +``` + +## Use the module + +The `SYSTEM_ROWS` sampling method accepts a single integer (`bigint`) argument that is the maximum number of rows to read. If the table does not contain that many rows, the whole table is selected. A negative argument raises an error. + +Like the built-in `SYSTEM` sampling method, `SYSTEM_ROWS` performs block-level sampling, so the sample is not completely random and might be subject to clustering effects, especially if only a small number of rows are requested. + +`SYSTEM_ROWS` does not support the `REPEATABLE` clause. + +:::note +In Apache Cloudberry, each segment samples its own part of a distributed table, and the limit applies to each segment separately. A query on a distributed table can therefore return up to the given number of rows from each segment. For example, `SYSTEM_ROWS(75)` on a cluster with three primary segments returns up to 225 rows. +::: + +## Example + +The following command returns a sample of at most 75 rows from each segment for the table `tablea`: + +```sql +SELECT * FROM tablea TABLESAMPLE SYSTEM_ROWS(75); +``` + +## Module documentation + +See [tsm_system_rows](https://www.postgresql.org/docs/14/tsm-system-rows.html) in the PostgreSQL documentation. diff --git a/docs/modules/tsm-system-time.md b/docs/modules/tsm-system-time.md new file mode 100644 index 0000000000..493541904c --- /dev/null +++ b/docs/modules/tsm-system-time.md @@ -0,0 +1,47 @@ +--- +title: tsm_system_time +--- + +# tsm_system_time + +The `tsm_system_time` module implements the `SYSTEM_TIME` table sampling method, which you can use in the `TABLESAMPLE` clause of a `SELECT` command. + +## Install and register the module + +`tsm_system_time` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/tsm_system_time install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION tsm_system_time; +``` + +## Use the module + +The `SYSTEM_TIME` sampling method accepts a single floating-point argument that is the maximum number of milliseconds to spend reading the table. This lets you control how long the query takes; the tradeoff is that the size of the sample becomes hard to predict. The sample contains as many rows as could be read in the specified time, unless the whole table has been read first. A negative argument raises an error. + +Like the built-in `SYSTEM` sampling method, `SYSTEM_TIME` performs block-level sampling, so the sample is not completely random and might be subject to clustering effects, especially if only a small number of rows are selected. + +`SYSTEM_TIME` does not support the `REPEATABLE` clause. + +:::note +In Apache Cloudberry, each segment samples its own part of a distributed table, and the time limit applies to each segment separately. +::: + +## Example + +The following command returns as large a sample of `tablea` as each segment can read in 500 milliseconds. If the entire table can be read in less time, all of its rows are returned. + +```sql +SELECT * FROM tablea TABLESAMPLE SYSTEM_TIME(500); +``` + +## Module documentation + +See [tsm_system_time](https://www.postgresql.org/docs/14/tsm-system-time.html) in the PostgreSQL documentation. diff --git a/docs/modules/uuid-ossp.md b/docs/modules/uuid-ossp.md new file mode 100644 index 0000000000..e6da69d011 --- /dev/null +++ b/docs/modules/uuid-ossp.md @@ -0,0 +1,45 @@ +--- +title: uuid-ossp +--- + +# uuid-ossp + +The `uuid-ossp` module provides functions that generate universally unique identifiers (UUIDs) using one of several standard algorithms. The module also includes functions that produce certain special UUID constants. + +The Apache Cloudberry `uuid-ossp` module is equivalent to the PostgreSQL `uuid-ossp` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`uuid-ossp` is built only when Apache Cloudberry is configured with a UUID library, using the `--with-uuid=LIB` option of `configure` (`LIB` is `e2fs`, `bsd`, or `ossp`). It is also not part of the default build targets (`make` and `make install`). If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/uuid-ossp install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it. Because the extension name contains a hyphen, you must quote it: + +```sql +CREATE EXTENSION "uuid-ossp"; +``` + +## Functions + +| Function | Description | +|----------|-------------| +| `uuid_generate_v1()` | Generates a version 1 UUID from the MAC address of the host and a timestamp. | +| `uuid_generate_v1mc()` | Generates a version 1 UUID using a random multicast MAC address. | +| `uuid_generate_v3(namespace uuid, name text)` | Generates a version 3 UUID (MD5 hash) in the given namespace. | +| `uuid_generate_v4()` | Generates a version 4 UUID from random numbers. | +| `uuid_generate_v5(namespace uuid, name text)` | Generates a version 5 UUID (SHA-1 hash) in the given namespace. | +| `uuid_nil()` | Returns the nil UUID constant. | +| `uuid_ns_dns()`, `uuid_ns_url()`, `uuid_ns_oid()`, `uuid_ns_x500()` | Return the constants that designate the DNS, URL, ISO object identifier, and X.500 distinguished name namespaces. | + +:::note +If you only need randomly generated (version 4) UUIDs, you can use the `gen_random_uuid()` function of the [pgcrypto](./pgcrypto.md) module instead. +::: + +## Module documentation + +See [uuid-ossp](https://www.postgresql.org/docs/14/uuid-ossp.html) in the PostgreSQL documentation for details about this module. diff --git a/docs/modules/yezzey.md b/docs/modules/yezzey.md new file mode 100644 index 0000000000..326d6548c0 --- /dev/null +++ b/docs/modules/yezzey.md @@ -0,0 +1,275 @@ +--- +title: yezzey +--- + +# yezzey + +The `yezzey` module offloads the data of append-optimized tables (row-oriented and column-oriented) to S3-compatible object storage. An offloaded table keeps its definition and stays in the database. You continue to query and modify it with the same SQL statements, and Apache Cloudberry reads and writes its data files in the object storage instead of on the local disks of the segments. + +Use `yezzey` to move cold data to cheaper storage and to reduce the disk space that a cluster needs. + +:::note +`yezzey` is developed in the [open-gpdb/yezzey](https://github.com/open-gpdb/yezzey) project. The Apache Cloudberry source tree references it as a Git submodule in `gpcontrib/yezzey`. Apache Cloudberry 2.2 pins `yezzey` version 2.0. +::: + +## How it works + +`yezzey` consists of the following parts: + +- A storage manager for append-optimized tables. It is installed as a hook when the `yezzey` library is loaded, and it redirects the file operations of offloaded tables to the object storage. +- The `yezzey` extension, which provides the functions and the metadata tables that manage offloading. +- [YProxy](https://github.com/open-gpdb/yproxy), a separate service that runs on the cluster hosts. The segments do not connect to the object storage directly. They send all requests to YProxy through a Unix domain socket, and YProxy keeps a pool of connections to the object storage and schedules the requests. + +When a table is offloaded: + +1. `yezzey` takes an exclusive lock on the table. +2. The data files of the table are uploaded to the object storage on all segments. +3. The tablespace of the table changes to the `yezzey` virtual tablespace (OID `8555`). +4. The transaction commits. + +After offloading, new data that you insert into the table is written to the object storage. Files in the object storage are never modified in place. Each write creates new files, and `yezzey` records them in a metadata table, the virtual index, that follows the MVCC rules of the database. Files that are no longer referenced become garbage, which you remove with the cleanup functions described in [Remove obsolete files](#remove-obsolete-files). + +## Build and install + +`yezzey` is not built by default. + +1. Fetch the submodule in the Apache Cloudberry source tree: + + ```shell + git submodule update --init gpcontrib/yezzey + ``` + +2. Configure Apache Cloudberry with the `--with-yezzey` option, then build and install. `--with-yezzey` requires OpenSSL support (`--with-ssl=openssl`) and a C++11 compiler: + + ```shell + ./configure --with-yezzey --with-ssl=openssl [other options...] + make + make install + ``` + + To build only this module against an existing build tree: + + ```shell + make -C gpcontrib/yezzey + make -C gpcontrib/yezzey install + ``` + +3. Build and install YProxy on every host of the cluster. See the [YProxy](https://github.com/open-gpdb/yproxy) project for instructions. + +## Set up + +### Configure and start YProxy + +YProxy reads a YAML configuration file that defines its socket and the object storage. The following example is based on the configuration that the `yezzey` tests use: + +```yaml +socket_path: "/tmp/yproxy.sock" +interconnect_socket_path: "/tmp/ic.sock" +log_level: info + +storage: + access_key_id: "" + secret_access_key: "" + storage_endpoint: "" + storage_prefix: "" + storage_bucket: "" + storage_region: "" + storage_type: "s3" + tablespace_map: + "pg_default": "" + +backup_storage: + access_key_id: "" + secret_access_key: "" + storage_endpoint: "" + storage_prefix: "" + storage_bucket: "" + storage_region: "" + storage_type: "s3" +``` + +`tablespace_map` maps the tablespace that a table was in before offloading to the bucket that receives its data. Add an entry for every tablespace that contains tables you want to offload. + +Start YProxy on every host, and make sure that the operating system user that runs Apache Cloudberry can access the socket: + +```shell +yproxy -c /path/to/yproxy.yaml +``` + +For the complete list of YProxy settings, see the YProxy documentation. + +### Enable the module + +1. Add `yezzey` to `shared_preload_libraries` and restart the cluster. The library refuses to load in any other way: + + ```shell + gpconfig -c shared_preload_libraries -v yezzey + gpstop -ar + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +2. Set the configuration parameters that your environment needs, and reload the configuration. At least check `yezzey.yproxy_socket` and `yezzey.use_gpg_crypto`: + + ```shell + gpconfig -c yezzey.yproxy_socket -v "/tmp/yproxy.sock" + gpstop -u + ``` + + `yezzey.use_gpg_crypto` is `on` by default, which requires a GPG key in the `crypto` section of the YProxy configuration (`gpg_key_id` and `gpg_key_path`). Without a key, `yezzey.offload_relation()` fails with `failed to offload relation`. If you do not encrypt the data, turn the parameter off: + + ```shell + gpconfig -c yezzey.use_gpg_crypto -v false + gpstop -u + ``` + +3. Create the extension in each database in which you want to offload tables: + + ```sql + CREATE EXTENSION yezzey; + ``` + + The extension creates the `yezzey` schema and grants `USAGE` on it to `PUBLIC`. The extension is marked as trusted, so a non-superuser who has the `CREATE` privilege on the database can create it. + + :::note + The functions of the extension can be run by `PUBLIC`, and `yezzey.offload_relation()` and `yezzey.load_relation()` do not check the ownership of the table. Revoke `EXECUTE` on the functions from `PUBLIC` if only administrators should offload, load, or clean up tables. + ::: + +## Configuration parameters + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `yezzey.yproxy_socket` | string | `/tmp/yproxy.sock` | Path of the Unix domain socket of YProxy. | +| `yezzey.use_gpg_crypto` | boolean | `on` | Asks YProxy to encrypt the data with GPG when it is written to the object storage. The key is configured in the `crypto` section of the YProxy configuration. | +| `yezzey.storage_class` | string | `STANDARD` | Default storage class for the objects that `yezzey` writes. | +| `yezzey.multipart_chunksize` | integer | `16777216` | Default chunk size of a multipart upload, in bytes. | +| `yezzey.multipart_threshold` | integer | `67108864` | Default size threshold for multipart uploads, in bytes. | +| `yezzey.use_otm_feature` | boolean | `off` | Enables the offload tablespace map. When it is `off`, only tables in the default tablespace can be offloaded. When it is `on`, tables in other tablespaces can be offloaded too, and `yezzey` records the original tablespace of each table in `yezzey.offload_tablespace_map`. This parameter can be set only at connection start or in the server configuration. | +| `yezzey.log_level` | enum | `debug1` | Log level of the messages of the `yezzey` functions. | +| `yezzey.ao_log_level` | enum | `debug1` | Log level of the messages about the read and write operations on offloaded files. | +| `yezzey.autooffload` | boolean | `off` | Reserved. The parameter is defined, but it is not used in this version. | + +Except for `yezzey.use_otm_feature` and `yezzey.autooffload`, the parameters can be changed by superusers only. + +## Offload a table + +`yezzey` works with append-optimized tables only, both row-oriented and column-oriented. + +To offload an existing table, call `yezzey.offload_relation()`: + +```sql +CREATE TABLE sales_archive (id int, amount numeric) + WITH (appendonly=true) DISTRIBUTED BY (id); +INSERT INTO sales_archive SELECT i, i FROM generate_series(1, 100000) i; + +SELECT yezzey.offload_relation('sales_archive'); +``` + +The function reports its progress with `NOTICE` messages from each segment. The time that it takes depends on the size of the table. If the table is already offloaded, the function returns with the warning `The relation has already been offloaded`. + +You can also create a table that is offloaded from the start, by placing it in the `yezzey` virtual tablespace: + +```sql +CREATE TABLE regao_tt_alr_off (i int) + WITH (appendonly=true) + TABLESPACE yezzey + DISTRIBUTED BY (i); + +INSERT INTO regao_tt_alr_off SELECT generate_series(1,10010); + +SELECT segindex, local_bytes, external_bytes +FROM yezzey.offload_relation_status('regao_tt_alr_off'); +``` + +```text + segindex | local_bytes | external_bytes +----------+-------------+---------------- + 0 | 0 | 33816 + 1 | 0 | 33928 + 2 | 0 | 32560 +(3 rows) +``` + +After a table is offloaded, you can continue to run `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `VACUUM`, and `ALTER TABLE ... ADD COLUMN` on it. + +To offload a partitioned table, offload each leaf partition. Calling `yezzey.offload_relation()` on the partitioned table itself fails. + +## Load a table back to local storage + +To move the data of an offloaded table back to the local disks of the segments, call `yezzey.load_relation()`: + +```sql +SELECT yezzey.load_relation('sales_archive'); +``` + +The time that the download takes depends on the size of the table and the number of its segment files. The files that were uploaded earlier stay in the object storage, so `external_bytes` in `yezzey.offload_relation_status()` is not reset to `0`. + +## Check the status of a table + +`yezzey.offload_relation_status()` returns one row per segment: + +```sql +SELECT * FROM yezzey.offload_relation_status('sales_archive'); +``` + +| Column | Description | +|---|---| +| `reloid` | OID of the table. | +| `segindex` | Segment content ID. | +| `local_bytes` | Size of the data that is stored on the local disk. `0` means that the table is fully offloaded. | +| `local_commited_bytes` | Size of the committed data on the local disk. | +| `external_bytes` | Size of the data in the object storage. `0` on all segments means that the table has never been offloaded. | +| `external_bloat_bytes` | Size of the data in the object storage that is no longer in use but is not deleted yet. | + +For more detail, use `yezzey.offload_relation_status_per_filesegment()`, which returns one row per segment file, or `yezzey.relation_describe_external_storage_structure()`, which returns one row per file in the object storage, including its path. + +The `yezzey.offload_metadata` table lists the tables that were offloaded with `yezzey.offload_relation()`. Tables that were created directly in the `yezzey` tablespace are not listed. `SELECT` on this table is granted to `PUBLIC`. + +## Remove obsolete files + +Data in the object storage is not deleted when rows are deleted or when a table is dropped, because the files might still be needed to roll back a transaction or to restore a backup. The following functions send delete requests for obsolete files to YProxy. They run on all segments. + +| Function | Scope | +|---|---| +| `yezzey.vacuum(confirm boolean DEFAULT false, crazyDrop boolean DEFAULT false)` | All tablespaces. | +| `yezzey.vacuum_tablespace(tablespace oid, confirm boolean DEFAULT false, crazyDrop boolean DEFAULT false)` | One tablespace. | +| `yezzey.vacuum_relation(relation regclass, confirm boolean DEFAULT false, crazyDrop boolean DEFAULT false)` | One offloaded table. | + +- `confirm` controls whether YProxy actually deletes the objects. With the default `false`, the call is a dry run. Set it to `true` to confirm the deletion. +- `crazyDrop` enables a more aggressive cleanup mode. Only superusers can set it to `true`. + +A typical cleanup looks like this: + +1. Run `VACUUM` on the append-optimized table, so that its obsolete files become garbage. +2. Inspect the storage of the table with `yezzey.relation_describe_external_storage_structure('
')`. +3. Call `yezzey.vacuum_relation()`, `yezzey.vacuum_tablespace()`, or `yezzey.vacuum()`, depending on the scope that you want to clean. +4. Check the storage of the table again to verify that the obsolete files are removed. + +By default, YProxy keeps the files that a backup might still need. See the `vacuum` section of the YProxy configuration for the settings that control this behavior. + +## Function reference + +All functions are in the `yezzey` schema. + +| Function | Returns | Description | +|---|---|---| +| `offload_relation(relation regclass)` | `void` | Offloads an append-optimized table to the object storage. | +| `load_relation(relation regclass)` | `void` | Loads an offloaded table back to local storage. | +| `offload_relation_status(relation regclass)` | set of rows | Local and external size of a table, per segment. | +| `offload_relation_status_per_filesegment(relation regclass)` | set of rows | Local and external size of a table, per segment file. | +| `relation_describe_external_storage_structure(relation regclass)` | set of rows | The files of a table in the object storage, with their paths and sizes. | +| `vacuum(confirm boolean, crazyDrop boolean)` | set of `boolean` | Removes obsolete files in all tablespaces. | +| `vacuum_tablespace(tablespace oid, confirm boolean, crazyDrop boolean)` | set of `boolean` | Removes obsolete files in one tablespace. | +| `vacuum_relation(relation regclass, confirm boolean, crazyDrop boolean)` | set of `boolean` | Removes obsolete files of one offloaded table. | + +The extension also creates internal functions and metadata tables, such as `yezzey.yezzey_virtual_index`, `yezzey.yezzey_expire_index`, and `yezzey.offload_tablespace_map`. Do not call the internal functions or modify the metadata tables directly. + +## Usage notes + +- Only append-optimized tables can be offloaded. +- To offload a table that is not in the default tablespace, turn on `yezzey.use_otm_feature`. Otherwise the offload is rejected. +- You cannot move an offloaded table to another tablespace with `ALTER TABLE ... SET TABLESPACE`. The statement fails. Use `yezzey.load_relation()` to bring the data back to local storage. +- YProxy must be running on every host before you access offloaded tables. If YProxy or the object storage is unavailable, `yezzey` retries the read requests up to 100 times. +- Offloading takes an exclusive lock on the table for the duration of the upload. +- When you restore a cluster from a backup, the object storage must contain the files that the backup refers to. For this reason, obsolete files are kept until no backup needs them. +- For a standby cluster that receives WAL from the primary cluster, the object storage must be accessible from the standby cluster as well. diff --git a/sidebars.ts b/sidebars.ts index a35023519d..3ddc808b61 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -526,6 +526,46 @@ const sidebars: SidebarsConfig = { 'data-types', 'config-params-guc-list', + { + type: 'category', + label: 'Extension Modules', + link: { + type: "doc", + id: 'modules/index', + }, + items: [ + 'modules/auto-explain', + 'modules/btree-gin', + 'modules/citext', + 'modules/dblink', + 'modules/diskquota', + 'modules/fuzzystrmatch', + 'modules/gp-distribution-policy', + 'modules/gp-exttable-fdw', + 'modules/gp-internal-tools', + 'modules/gp-legacy-string-agg', + 'modules/gp-relsizes-stats', + 'modules/gp-sparse-vector', + 'modules/gp-stats-collector', + 'modules/hstore', + 'modules/isn', + 'modules/ltree', + 'modules/orafce', + 'modules/pageinspect', + 'modules/pg-buffercache', + 'modules/pg-trgm', + 'modules/pgcrypto', + 'modules/postgres-fdw', + 'modules/reject-partition-fullscan', + 'modules/sslinfo', + 'modules/tablefunc', + 'modules/tsm-system-rows', + 'modules/tsm-system-time', + 'modules/uuid-ossp', + 'modules/yezzey', + ] + }, + { type: 'category', label: 'Built-in Functions', diff --git a/versioned_docs/version-2.x/modules/auto-explain.md b/versioned_docs/version-2.x/modules/auto-explain.md new file mode 100644 index 0000000000..bd266e5130 --- /dev/null +++ b/versioned_docs/version-2.x/modules/auto-explain.md @@ -0,0 +1,55 @@ +--- +title: auto_explain +--- + +# auto_explain + +The `auto_explain` module logs the execution plans of slow statements automatically, without having to run `EXPLAIN` by hand. + +In Apache Cloudberry, `auto_explain` takes effect only in regular sessions on the coordinator. When the library is loaded in a segment process or in a utility-mode connection, it does nothing and does not define its configuration parameters. Otherwise, it works like the PostgreSQL `auto_explain` module. With `auto_explain.log_analyze` enabled, the logged plan also includes the per-slice execution statistics collected from the segments. + +## Load the module + +`auto_explain` is built and installed by a default Apache Cloudberry build. It is a library, not an extension: it provides no SQL objects, and you do not run `CREATE EXTENSION` for it. + +To load the module into a single session, run the following command as a superuser: + +```sql +LOAD 'auto_explain'; +``` + +To load it into all sessions, add `auto_explain` to `session_preload_libraries` or `shared_preload_libraries`. A change to `shared_preload_libraries` requires a restart of the cluster. Preloading lets you track slow queries whenever they happen, at the cost of some overhead for all queries. + +## Configuration parameters + +The module does nothing until `auto_explain.log_min_duration` is set to a value other than `-1`. Only superusers can change these parameters. + +| Parameter | Description | +|-----------|-------------| +| `auto_explain.log_min_duration` | Minimum statement execution time, in milliseconds, above which the plan is logged. `0` logs all plans. `-1` (the default) turns logging off. | +| `auto_explain.log_analyze` | Logs `EXPLAIN ANALYZE` output instead of `EXPLAIN` output. | +| `auto_explain.log_buffers` | Logs buffer usage. Takes effect only when `log_analyze` is on. | +| `auto_explain.log_wal` | Logs WAL usage. Takes effect only when `log_analyze` is on. | +| `auto_explain.log_timing` | Collects per-node timing. Takes effect only when `log_analyze` is on. | +| `auto_explain.log_triggers` | Includes trigger statistics. Takes effect only when `log_analyze` is on. | +| `auto_explain.log_verbose` | Logs `EXPLAIN VERBOSE` output. | +| `auto_explain.log_settings` | Logs the modified configuration parameters that affect query planning. | +| `auto_explain.log_format` | Output format: `text`, `xml`, `json`, or `yaml`. | +| `auto_explain.log_level` | Log level for the plan. | +| `auto_explain.log_nested_statements` | Logs statements that are run inside functions. | +| `auto_explain.sample_rate` | Fraction of statements to process in each session. | + +## Example + +```sql +LOAD 'auto_explain'; +SET auto_explain.log_min_duration = 0; +SET auto_explain.log_analyze = true; +SELECT count(*) FROM pg_class; +``` + +The plan of the `SELECT` statement is written to the coordinator log. + +## Module documentation + +See [auto_explain](https://www.postgresql.org/docs/14/auto-explain.html) in the PostgreSQL documentation for details about each configuration parameter. diff --git a/versioned_docs/version-2.x/modules/btree-gin.md b/versioned_docs/version-2.x/modules/btree-gin.md new file mode 100644 index 0000000000..e0f3c6082b --- /dev/null +++ b/versioned_docs/version-2.x/modules/btree-gin.md @@ -0,0 +1,21 @@ +--- +title: btree_gin +--- + +# btree_gin + +The `btree_gin` module provides sample generalized inverted index (GIN) operator classes that implement B-tree equivalent behavior for a number of data types. + +The Apache Cloudberry `btree_gin` module is equivalent to the PostgreSQL `btree_gin` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`btree_gin` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION btree_gin; +``` + +## Module documentation + +See [btree_gin](https://www.postgresql.org/docs/14/btree-gin.html) in the PostgreSQL documentation for the list of supported data types and usage examples. diff --git a/versioned_docs/version-2.x/modules/citext.md b/versioned_docs/version-2.x/modules/citext.md new file mode 100644 index 0000000000..51631c4681 --- /dev/null +++ b/versioned_docs/version-2.x/modules/citext.md @@ -0,0 +1,21 @@ +--- +title: citext +--- + +# citext + +The `citext` module provides a case-insensitive character string data type, `citext`. Essentially, it internally calls the `lower()` function when comparing values. Otherwise, it behaves almost exactly like the `text` data type. + +The Apache Cloudberry `citext` module is equivalent to the PostgreSQL `citext` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`citext` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION citext; +``` + +## Module documentation + +See [citext](https://www.postgresql.org/docs/14/citext.html) in the PostgreSQL documentation for details about the data type, operators, and functions that this module defines. diff --git a/versioned_docs/version-2.x/modules/dblink.md b/versioned_docs/version-2.x/modules/dblink.md new file mode 100644 index 0000000000..277213883e --- /dev/null +++ b/versioned_docs/version-2.x/modules/dblink.md @@ -0,0 +1,135 @@ +--- +title: dblink +--- + +# dblink + +The `dblink` module supports connections to other databases from within a database session. The other database can be in the same Apache Cloudberry cluster, in another Apache Cloudberry cluster, or in a PostgreSQL server. + +:::note +`dblink` is intended for short ad hoc queries in other databases. It is not a replacement for external tables, foreign tables, or data migration tools. +::: + +The Apache Cloudberry `dblink` module is a modified version of the PostgreSQL `dblink` module. See [MPP considerations](#mpp-considerations) for the restrictions that apply. + +## Install and register the module + +`dblink` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION dblink; +``` + +## MPP considerations + +- A named or implicit (unnamed) `dblink` connection exists only in the coordinator process that serves your session. A statement that modifies table data, such as `INSERT INTO ... SELECT ... FROM dblink(...)`, runs the `dblink()` call in a different process, which cannot see that connection. In such statements, pass the connection string directly to `dblink()`: + + ```sql + CREATE TABLE testdbllocal (a int, b text) DISTRIBUTED BY (a); + INSERT INTO testdbllocal + SELECT * FROM dblink('dbname=postgres', 'SELECT * FROM testdblink') AS dbltab(id int, product text); + ``` + + If you use a connection name instead, the statement fails with the error `could not establish connection`, and the detail `missing "=" after "" in connection info string`. + +- Asynchronous queries are not supported. The extension does not create the `dblink_send_query()`, `dblink_is_busy()`, and `dblink_get_result()` functions. + +- A non-superuser must specify both `host` and `password` in the connection string of `dblink_connect()` and `dblink()`, even to connect to a local database. Otherwise, the call fails with `host is required` or `password is required`. If a non-superuser does not specify `user`, the name of the current database user is used. + +## Use dblink + +The following steps show the basic use of `dblink`. The examples use `dblink_connect()` to create a connection to a database and `dblink()` to run an SQL query. + +1. Create a sample table in the `postgres` database: + + ```sql + CREATE TABLE testdblink (a int, b text) DISTRIBUTED BY (a); + INSERT INTO testdblink VALUES (1, 'Cheese'), (2, 'Fish'); + ``` + +2. Log in to a different database, for example `testdb`, as a superuser, and register the extension if you have not done so: + + ```sql + CREATE EXTENSION dblink; + ``` + +3. Use `dblink_connect()` to create either an implicit or a named connection to another database. The connection string is a `libpq`-style keyword/value string. This example creates a connection named `mylocalconn` to the `postgres` database on the local cluster: + + ```sql + SELECT dblink_connect('mylocalconn', 'dbname=postgres user=gpadmin'); + ``` + + ```text + dblink_connect + ---------------- + OK + (1 row) + ``` + +4. Use `dblink()` to run a query over the connection. The function returns `record`, so you must specify the names and types of the returned columns: + + ```sql + SELECT * FROM dblink('mylocalconn', 'SELECT * FROM testdblink') AS dbltab(id int, product text); + ``` + + ```text + id | product + ----+--------- + 1 | Cheese + 2 | Fish + (2 rows) + ``` + +5. Close the connection when you no longer need it: + + ```sql + SELECT dblink_disconnect('mylocalconn'); + ``` + +To connect to a remote system, include the host and password information in the connection string. For example, to create an implicit connection: + +```sql +SELECT dblink_connect('host=remotehost port=5432 dbname=postgres user=gpadmin password=secret'); +``` + +### Use dblink as a non-superuser + +A non-superuser must include the host and the password in the connection string of `dblink_connect()`. For example, the user `test_user` can create a connection to the local coordinator host `cdw` with this command: + +```sql +SELECT dblink_connect('host=cdw port=5432 dbname=postgres user=test_user password=secret'); +``` + +In addition, the remote server must actually request a password for the connection. If it does not (for example, with `trust` authentication), the call fails with `password is required`. + +To let non-superusers create connections that do not require a password, use the `dblink_connect_u()` function. It is identical to `dblink_connect()`, except that it skips these checks. `dblink_connect_u()` is installed with all privileges revoked from `PUBLIC`, so only superusers can call it until you grant the `EXECUTE` privilege on it. + +:::caution +Grant `EXECUTE` on `dblink_connect_u()` only to users that you trust. If the target system has users with an authentication method that does not involve a password, a non-superuser who can call `dblink_connect_u()` can impersonate such a user and escalate privileges. Also, a password can be supplied from the server environment, such as a `~/.pgpass` file that belongs to the operating system user that runs the server. +::: + +1. As a superuser, grant the `EXECUTE` privilege on the `dblink_connect_u()` functions: + + ```sql + GRANT EXECUTE ON FUNCTION dblink_connect_u(text) TO test_user; + GRANT EXECUTE ON FUNCTION dblink_connect_u(text, text) TO test_user; + ``` + +2. As `test_user`, create a connection without a password, and use it: + + ```sql + SELECT dblink_connect_u('testconn', 'dbname=postgres user=test_user'); + SELECT * FROM dblink('testconn', 'SELECT * FROM testdblink') AS dbltab(id int, product text); + ``` + +### Use dblink with SSL-encrypted connections + +To connect over an encrypted connection, specify the `sslmode` keyword in the connection string. Set `sslmode` to at least `require` to disallow unencrypted connections. For example: + +```sql +SELECT dblink_connect('con_sales', 'dbname=sales host=cdw user=gpadmin sslmode=require'); +``` + +## Module documentation + +See [dblink](https://www.postgresql.org/docs/14/dblink.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/versioned_docs/version-2.x/modules/diskquota.md b/versioned_docs/version-2.x/modules/diskquota.md new file mode 100644 index 0000000000..a7c89da165 --- /dev/null +++ b/versioned_docs/version-2.x/modules/diskquota.md @@ -0,0 +1,499 @@ +--- +title: diskquota +--- + +# diskquota + +The `diskquota` module allows Apache Cloudberry administrators to limit the amount of disk space that schemas, roles, or tablespaces use in a database. + +Apache Cloudberry includes `diskquota` 2.3 in the `gpcontrib/diskquota` directory of the source code. + +## Installation + +### Build the module + +`diskquota` is not built by default. To build and install the module with Apache Cloudberry, specify the `--with-diskquota` option when you run `configure`: + +```shell +./configure --with-diskquota [other options...] +make +make install +``` + +The module is built with CMake, so `cmake` must be installed on the build host. + +### Enable the module + +Before you can use the module, perform these steps: + +1. Create the `diskquota` database. The module uses this database to store the list of the databases in which the module is enabled. + + ```shell + createdb diskquota + ``` + +2. Add the `diskquota-2.3` shared library to the `shared_preload_libraries` server configuration parameter and restart Apache Cloudberry. Be sure to retain the previous setting of the configuration parameter. For example: + + ```shell + gpconfig -s shared_preload_libraries + gpconfig -c shared_preload_libraries -v 'diskquota-2.3' + gpstop -ar + ``` + + If `shared_preload_libraries` already contains other libraries, specify a comma-separated list, for example `'auto_explain,diskquota-2.3'`. + +3. Create the `diskquota` extension in each database in which you want to enforce disk usage quotas. The `diskquota.max_monitored_databases` parameter limits the number of databases in which you can create the extension (`50` by default). + + ```sql + CREATE EXTENSION diskquota; + ``` + +4. If the database is not empty when you create the extension, `diskquota` raises the following warning. Run the `diskquota.init_table_size_table()` function to size the existing tables. In a database with many files, this can take some time. + + ``` + WARNING: [diskquota] diskquota is not ready because current database is not empty + HINT: please run 'SELECT diskquota.init_table_size_table();' to initialize diskquota + ``` + + ```sql + SELECT diskquota.init_table_size_table(); + ``` + +:::note +The library must be loaded through `shared_preload_libraries`. If it is not, the functions of the module fail with an error that reports that `diskquota-2.3` is not in `shared_preload_libraries`. +::: + +For more information about the `gpconfig` and `gpstop` utilities, see [gpconfig](../sys-utilities/gpconfig.md) and [gpstop](../sys-utilities/gpstop.md). + +## About the diskquota module + +The disk usage of a table includes the table data, indexes, toast tables, and free space map. For append-optimized tables, the calculation also includes the auxiliary tables, such as the visibility map and the block directory table. + +You can set the following quotas with the `diskquota` module: + +- A schema disk quota sets a limit on the disk space that all the tables in a specific schema of a database can use. The disk usage of a schema is the total of the disk usage on all segments for all tables in the schema. +- A role disk quota sets a limit on the disk space that all the tables that a specific role owns in a database can use. The disk usage of a role is the total of the disk usage on all segments for all tables that the role owns. Although a role is a cluster-level object, the quota limit and the disk usage of a role are specific to each database. +- A schema tablespace disk quota sets a limit on the disk space that all the tables in a specific schema and tablespace of a database can use. +- A role tablespace disk quota sets a limit on the disk space that all the tables that a specific role owns in a specific tablespace of a database can use. +- A per-segment tablespace disk quota sets a limit on the disk space that a single segment can use when a tablespace quota is set for a schema or role. + +:::note +You cannot set a role disk quota for the system owner, that is, the bootstrap superuser that initialized the cluster. +::: + +Each database stores and manages its own quota configuration in the `diskquota.quota_config` table. + +## How diskquota monitors disk usage + +`diskquota` is based on the background worker framework. A single `diskquota` launcher process runs on the coordinator. The launcher starts `diskquota` worker processes on the coordinator for the databases in which the extension is created. No launcher or worker processes run on the segments. + +A worker process monitors the disk usage of the schemas, roles, and tablespaces in its database. On each segment, hooks in the storage manager record the tables whose files are created, extended, or truncated. These tables are called active tables. Periodically, the worker process queries the segments for the sizes of the active tables, updates the disk usage of the corresponding schemas and roles, and compares the usage with the quota limits. When a quota is exceeded, the worker puts the schema, role, or tablespace into the reject map, which is stored in shared memory. + +`diskquota` can enforce both soft limits and hard limits for disk usage: + +- `diskquota` always enforces soft limits. It checks the reject map before a query runs. If the target schema, role, or tablespace of a query that adds data is in the reject map, the query is cancelled before it starts, with an error message that reports that the quota is exceeded. If the quota is not exceeded when a query starts, the query is allowed to run, even if it eventually causes the quota to be exceeded. +- When hard limit enforcement is enabled, `diskquota` also checks disk usage during query execution. If a query exceeds a disk quota while it is running, `diskquota` terminates the query. Hard limit enforcement is deactivated by default. See [diskquota.hard_limit](#diskquotahard_limit). + +There is some delay after a quota is reached before the schema or role is added to the reject map, because the worker processes recalculate the table sizes periodically (every two seconds by default). Other queries can add more data during the delay. The same delay occurs when the disk usage falls beneath a quota because of operations that remove data, such as `DROP`, `TRUNCATE`, or `VACUUM FULL`. See [diskquota.naptime](#diskquotanaptime). + +Deleting rows or running `VACUUM` on a table does not release disk space, so these operations alone cannot remove a schema or role from the reject map. To reduce the disk space that a table uses, run `VACUUM FULL` or `TRUNCATE TABLE`. + +If the number of databases in which the extension is created is larger than `diskquota.max_workers`, the launcher starts and stops worker processes dynamically, so that the databases share the available workers. + +## Functions and views + +The functions and views of the module are in the `diskquota` schema. + +Only superusers can run the functions that set quotas, and the `pause()` and `resume()` functions. + +| Function | Description | +|---|---| +| `void init_table_size_table()` | Sizes the existing tables in the current database. | +| `void set_schema_quota(schema_name text, quota text)` | Sets a disk quota for a schema in the current database. | +| `void set_role_quota(role_name text, quota text)` | Sets a disk quota for a role in the current database. | +| `void set_schema_tablespace_quota(schema_name text, tablespace_name text, quota text)` | Sets a disk quota for a schema and tablespace combination in the current database. | +| `void set_role_tablespace_quota(role_name text, tablespace_name text, quota text)` | Sets a disk quota for a role and tablespace combination in the current database. | +| `void set_per_segment_quota(tablespace_name text, ratio float4)` | Sets a per-segment disk quota ratio for a tablespace in the current database. | +| `void pause()` | Continues to count the disk usage of the current database, but stops raising errors when a limit is exceeded. | +| `void resume()` | Resumes raising errors when a disk usage limit is exceeded in the current database. | +| `status()` | Returns a table with the `name` and `status` columns that shows the state of soft and hard limit enforcement in the current database, and the binary and schema versions of `diskquota`. | +| `bigint relation_size(relation regclass)` | Returns the size of a relation on all segments as `diskquota` calculates it. | +| `boolean wait_for_worker_new_epoch()` | Waits until the worker process of the current database finishes a new round of disk usage refresh. | + +| View | Description | +|---|---| +| `show_fast_database_size_view` | Displays the disk space usage of the current database. | +| `show_fast_schema_quota_view` | Lists the quotas and the disk usage for schemas in the current database. | +| `show_fast_role_quota_view` | Lists the quotas and the disk usage for roles in the current database. | +| `show_fast_schema_tablespace_quota_view` | Lists the quotas and the disk usage for schemas per tablespace in the current database. | +| `show_fast_role_tablespace_quota_view` | Lists the quotas and the disk usage for roles per tablespace in the current database. | +| `show_segment_ratio_quota_view` | Displays the per-segment quota ratio of the tablespaces for which a ratio is set in the current database. | +| `rejectmap` | Lists the entries in the reject map. | + +## Configuration parameters + +The module defines the following server configuration parameters. Use the `gpconfig` utility to set them. A parameter that is marked "restart" takes effect after you restart the cluster (`gpstop -ar`). A parameter that is marked "reload" takes effect after you reload the configuration (`gpstop -u`). + +| Parameter | Default | Takes effect | Description | +|---|---|---|---| +| `diskquota.naptime` | `2` | reload | The interval, in seconds, at which `diskquota` recalculates the table sizes. | +| `diskquota.hard_limit` | `off` | reload | Activates or deactivates hard limit enforcement of disk usage. | +| `diskquota.worker_timeout` | `60` | reload | The time, in seconds, after which `diskquota.wait_for_worker_new_epoch()` raises a notice that it timed out while waiting for the worker process. | +| `diskquota.max_workers` | `10` | restart | The maximum number of `diskquota` worker processes that can run at the same time. The maximum value is `20`. | +| `diskquota.max_active_tables` | `307200` (`300 * 1024`) | restart | The maximum number of active relations (tables, indexes, and so on) that `diskquota` can track at the same time. | +| `diskquota.max_table_segments` | `10485760` (`10 * 1024 * 1024`) | restart | The maximum number of table segments in the cluster. | +| `diskquota.max_monitored_databases` | `50` | restart | The maximum number of databases that `diskquota` can monitor. The maximum value is `1024`. | +| `diskquota.max_quota_probes` | `1048576` (`1024 * 1024`) | restart | The maximum number of quota probes in the cluster. | + +### diskquota.naptime + +The smaller the `diskquota.naptime` value, the less delay in detecting changes in disk usage. This example sets the value to ten seconds: + +```shell +gpconfig -c diskquota.naptime -v 10 +gpstop -u +``` + +### diskquota.hard_limit + +When hard limit enforcement is enabled, `diskquota` checks the quotas during query execution. If a running query exceeds a quota limit, `diskquota` terminates the query. + +To activate hard limit enforcement for all databases, set the parameter to `on` and reload the configuration: + +```shell +gpconfig -c diskquota.hard_limit -v 'on' +gpstop -u +``` + +Run the following query to view the hard limit enforcement setting: + +```sql +SELECT * FROM diskquota.status(); +``` + +### diskquota.max_workers + +The parameter specifies the maximum number of worker processes that can run at the same time, not including the launcher process. The `diskquota` workers are taken from the pool of worker processes that the `max_worker_processes` server configuration parameter establishes, so `diskquota.max_workers` should be less than `max_worker_processes`. + +### diskquota.max_active_tables + +Active tables are the tables whose sizes might have changed since `diskquota` last recalculated the table sizes. The active table list is stored in shared memory that is allocated when the server starts. If the active table shared memory is full, `diskquota` might fail to detect the corresponding disk usage changes in time. + +### diskquota.max_table_segments + +A table (including each child table of a partitioned table) is distributed to all segments. `diskquota` counts the part of a table on each segment as a table segment. The parameter therefore limits the number of tables that `diskquota` can monitor. + +When you expand the cluster, each table consumes more table segments, which reduces the number of tables that `diskquota` can support. If the following warning is reported, increase the `diskquota.max_table_segments` value, and then restart Apache Cloudberry: + +``` +WARNING: [diskquota] the number of tables exceeds the limit, please increase the GUC value for diskquota.max_table_segments. Current diskquota.max_table_segments value: 10485760 +``` + +## Use the diskquota module + +### View the diskquota status + +To view the state of soft and hard limit enforcement in the current database and the `diskquota` version numbers, call the `status()` function: + +```sql +SELECT * FROM diskquota.status(); +``` + +``` + name | status +------------------------+-------- + soft limits | on + hard limits | off + current binary version | 2.3.0 + current schema version | 2.3 +(4 rows) +``` + +When `diskquota` is paused in the database, the status of `soft limits` is `paused`. The status of `hard limits` is also `paused` if hard limit enforcement is enabled. + +### Pause and resume quota enforcement + +To stop raising quota exceeded errors in the current database for a period of time, pause and then resume `diskquota`: + +```sql +SELECT diskquota.pause(); +-- Perform the operations for which quotas should not be enforced. +SELECT diskquota.resume(); +``` + +`diskquota` continues to count disk usage while it is paused. + +### Set a schema or role disk quota + +Use the `diskquota.set_schema_quota()` and `diskquota.set_role_quota()` functions to set, update, or delete the disk quota limits for schemas and roles in the current database. The functions take two arguments: the schema or role name, and the quota to set. Specify the quota in units of MB, GB, TB, or PB, for example `'2TB'`. A number without a unit is in MB. A quota of `0` is not allowed. + +The functions convert the schema or role name to lower case, unless you enclose the name in double quotes inside the string, for example `'"MyRole"'`. + +The following example sets a 250GB quota for the `acct` schema: + +```sql +SELECT diskquota.set_schema_quota('acct', '250GB'); +``` + +This example sets a 500MB quota for the `nickd` role: + +```sql +SELECT diskquota.set_role_quota('nickd', '500MB'); +``` + +To change a quota, call the function again with the new quota value. + +To remove a schema or role quota, set the quota value to `'-1'`. + +### Set a tablespace disk quota for a schema or role + +Use the `diskquota.set_schema_tablespace_quota()` and `diskquota.set_role_tablespace_quota()` functions to set, update, or delete per-tablespace disk quota limits for schemas and roles in the current database. The functions take three arguments: the schema or role name, the tablespace name, and the quota to set. + +The following example sets a 250GB disk quota for the `acct` schema in the `tspaced1` tablespace: + +```sql +SELECT diskquota.set_schema_tablespace_quota('acct', 'tspaced1', '250GB'); +``` + +This example sets a 500MB disk quota for the `nickd` role in the `tspaced2` tablespace: + +```sql +SELECT diskquota.set_role_tablespace_quota('nickd', 'tspaced2', '500MB'); +``` + +To change a quota, call the function again with the new quota value. + +To remove a schema or role tablespace quota, set the quota value to `'-1'`. + +### Set a per-segment tablespace disk quota + +When you set a tablespace quota for a schema or a role, you can also define a per-segment quota for the tablespace. A per-segment quota limits the amount of disk space that the tablespace can consume on a single segment, which helps to prevent the disk of a segment from filling because of data skew. + +Use the `diskquota.set_per_segment_quota()` function to set, update, or delete a per-segment tablespace quota. The function takes two arguments: the tablespace name and a ratio. The ratio identifies how much more of the disk quota a single segment can use than the average segment quota. A ratio of `0` is not allowed. + +The average segment quota is calculated as follows: + +``` +avg_seg_quota = tablespace_quota / number_of_segments +``` + +For example, if the cluster has 8 segments and you set the following schema tablespace quota, the average segment quota for the `tspaced1` tablespace is `800GB / 8 = 100GB`: + +```sql +SELECT diskquota.set_schema_tablespace_quota('accts', 'tspaced1', '800GB'); +``` + +If you then set the following per-segment ratio, the maximum disk usage that is allowed per segment is `100GB * 2.0 = 200GB`: + +```sql +SELECT diskquota.set_per_segment_quota('tspaced1', '2.0'); +``` + +To change the ratio, call the function again with the new value. To remove a per-segment tablespace quota, set the ratio to `'-1'`. + +To view the per-segment quota ratio that is set for a tablespace, query the `show_segment_ratio_quota_view` view: + +```sql +SELECT tablespace_name, per_seg_quota_ratio + FROM diskquota.show_segment_ratio_quota_view WHERE tablespace_name IN ('tspaced1'); +``` + +``` + tablespace_name | per_seg_quota_ratio +-----------------+--------------------- + tspaced1 | 2 +(1 row) +``` + +### Display disk quotas and disk usage + +The `diskquota.show_fast_schema_quota_view` view lists the quotas for schemas in the current database. The `nspsize_in_bytes` column contains the calculated size of all the tables that belong to the schema. + +```sql +SELECT * FROM diskquota.show_fast_schema_quota_view; +``` + +``` + schema_name | schema_oid | quota_in_mb | nspsize_in_bytes +-------------+------------+-------------+------------------ + acct | 16561 | 256000 | 131072 + public | 2200 | 250 | 3014656 +(2 rows) +``` + +The `diskquota.show_fast_role_quota_view` view lists the quotas for roles in the current database. The `rolsize_in_bytes` column contains the calculated size of all the tables that the role owns. + +```sql +SELECT * FROM diskquota.show_fast_role_quota_view; +``` + +``` + role_name | role_oid | quota_in_mb | rolsize_in_bytes +-----------+----------+-------------+------------------ + mdach | 16558 | 500 | 131072 + nickd | 16577 | 500 | 144670720 +(2 rows) +``` + +To view the per-tablespace disk quotas for schemas and roles, query the `diskquota.show_fast_schema_tablespace_quota_view` and `diskquota.show_fast_role_tablespace_quota_view` views. For example: + +```sql +SELECT schema_name, tablespace_name, quota_in_mb, nspsize_tablespace_in_bytes + FROM diskquota.show_fast_schema_tablespace_quota_view + WHERE schema_name = 'acct' AND tablespace_name = 'tspaced1'; +``` + +### Identify the diskquota-monitored databases + +Run the following commands to obtain the list of the databases that `diskquota` monitors in the cluster: + +```sql +\c diskquota +SELECT d.datname FROM diskquota_namespace.database_list q, pg_database d + WHERE q.dbid = d.oid ORDER BY d.datname; +``` + +## Notes and limitations + +- Drop the `diskquota` extension from a database before you drop the database. A `diskquota` worker process is connected to each database in which the extension is created. +- `diskquota` also counts the disk usage of temporary tables. The size of a temporary table is added to the usage of the role that owns the table. It is not added to the usage of the current schema, because a temporary table is located in a temporary schema. +- When the reject map shared memory is full, data might be loaded into some schemas or roles after they have reached their quota limits. + +## Examples + +### Set a schema quota + +This example configures a schema quota and shows the soft limit behavior as data is added to the schema. The example assumes that the `diskquota` library is preloaded and that the `diskquota` database exists. + +1. Create a database named `testdb` and connect to it. + + ```shell + createdb testdb + psql -d testdb + ``` + +2. Create the `diskquota` extension in the database. + + ```sql + CREATE EXTENSION diskquota; + ``` + +3. Create a schema named `s1`. + + ```sql + CREATE SCHEMA s1; + ``` + +4. Set a 1MB disk quota for the `s1` schema. + + ```sql + SELECT diskquota.set_schema_quota('s1', '1 MB'); + ``` + +5. Create a table in the `s1` schema and insert a small amount of data into it. The schema has no data yet, so it is not in the reject map. + + ```sql + SET search_path TO s1; + CREATE TABLE a(i int) DISTRIBUTED BY (i); + INSERT INTO a SELECT generate_series(1,100); + ``` + +6. Insert a large amount of data, enough to exceed the 1MB quota of the schema. Before the `INSERT` command, the `s1` schema is not in the reject map, so with only soft limit enforcement in effect the command is allowed to run, even though it exceeds the limit of the schema. + + ```sql + INSERT INTO a SELECT generate_series(1,10000000); + ``` + +7. Try to insert a small amount of data. Because the previous command exceeded the disk quota of the schema, the schema is in the reject map and any command that loads data is cancelled. + + ```sql + INSERT INTO a SELECT generate_series(1,100); + ``` + + ``` + ERROR: schema's disk space quota exceeded with name: s1 + ``` + +8. Remove the quota from the `s1` schema by setting it to `-1`, and insert a small amount of data again. The 5-second sleep before the `INSERT` command ensures that the reject map is updated before the command runs. + + ```sql + SELECT diskquota.set_schema_quota('s1', '-1'); + SELECT pg_sleep(5); + INSERT INTO a SELECT generate_series(1,100); + ``` + +### Enable hard limit enforcement + +This example enables hard limit enforcement of disk usage, and runs the commands of the previous example again. Before you start, empty the table from the previous example so that the schema is under its quota again: + +```sql +TRUNCATE s1.a; +SELECT diskquota.wait_for_worker_new_epoch(); +``` + +1. Enable hard limit enforcement. + + ```shell + gpconfig -c diskquota.hard_limit -v 'on' + gpstop -u + ``` + +2. View the hard limit enforcement setting. + + ```sql + SELECT * FROM diskquota.status(); + ``` + +3. Set a 1MB disk quota for the `s1` schema again. + + ```sql + SELECT diskquota.set_schema_quota('s1', '1 MB'); + ``` + +4. Insert a large amount of data, enough to exceed the 1MB quota of the schema. Before the `INSERT` command, the `s1` schema is not in the reject map, so the command is allowed to start. When the operation exceeds the schema quota, `diskquota` terminates the query with an error like `schema's disk space quota exceeded with name: 17952`. With hard limit enforcement, the error reports the OID of the schema instead of its name. + + ```sql + INSERT INTO a SELECT generate_series(1,10000000); + ``` + +5. Remove the quota from the `s1` schema, and wait until the reject map is updated: + + ```sql + SELECT diskquota.set_schema_quota('s1', '-1'); + SELECT diskquota.wait_for_worker_new_epoch(); + ``` + +### Set a per-segment tablespace quota + +This example configures a tablespace quota and a per-segment tablespace quota. It uses the `testdb` database and the `s1` schema of the previous examples, and assumes that a tablespace named `tbsp1` exists in the cluster. + +1. Set a disk quota of 10MB for the `s1` schema in the `tbsp1` tablespace. + + ```sql + SELECT diskquota.set_schema_tablespace_quota('s1', 'tbsp1', '10 MB'); + ``` + +2. Create a table named `b` in the `tbsp1` tablespace and insert some data. The data does not exceed the 10MB quota. + + ```sql + SET search_path TO s1; + CREATE TABLE b(i int) TABLESPACE tbsp1 DISTRIBUTED BY (i); + INSERT INTO b SELECT generate_series(1,100000); + ``` + +3. Set a per-segment ratio of `0.1` for the `tbsp1` tablespace. With this setting, a single segment can use at most the average segment quota multiplied by `0.1`, which is less than the space that the table already uses on a segment. + + ```sql + SELECT diskquota.set_per_segment_quota('tbsp1', 0.1); + ``` + +4. Wait for `diskquota` to refresh the disk usage, and then try to insert more data. Because the per-segment tablespace quota is exceeded, the command is cancelled. + + ```sql + SELECT diskquota.wait_for_worker_new_epoch(); + INSERT INTO b SELECT generate_series(1,100); + ``` + + ``` + ERROR: tablespace: tbsp1, schema: s1 diskquota exceeded per segment quota + ``` diff --git a/versioned_docs/version-2.x/modules/fuzzystrmatch.md b/versioned_docs/version-2.x/modules/fuzzystrmatch.md new file mode 100644 index 0000000000..930a166c48 --- /dev/null +++ b/versioned_docs/version-2.x/modules/fuzzystrmatch.md @@ -0,0 +1,21 @@ +--- +title: fuzzystrmatch +--- + +# fuzzystrmatch + +The `fuzzystrmatch` module provides functions that determine similarities and distance between strings, based on the Soundex, Levenshtein, Metaphone, and Double Metaphone algorithms. + +The Apache Cloudberry `fuzzystrmatch` module is equivalent to the PostgreSQL `fuzzystrmatch` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`fuzzystrmatch` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION fuzzystrmatch; +``` + +## Module documentation + +See [fuzzystrmatch](https://www.postgresql.org/docs/14/fuzzystrmatch.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/versioned_docs/version-2.x/modules/gp-distribution-policy.md b/versioned_docs/version-2.x/modules/gp-distribution-policy.md new file mode 100644 index 0000000000..4c6ecaa326 --- /dev/null +++ b/versioned_docs/version-2.x/modules/gp-distribution-policy.md @@ -0,0 +1,77 @@ +--- +title: gp_distribution_policy +--- + +# gp_distribution_policy + +The `gp_distribution_policy` module checks whether the rows of a table are stored on the segments that the table's distribution policy assigns them to. On each segment, it scans the table, computes the hash of the distribution key columns of every row, and verifies that the row belongs to that segment. + +Use it to detect rows that ended up on a wrong segment. + +## Build and install + +`gp_distribution_policy` is built and installed by default with Apache Cloudberry. No `configure` option is required. + +## Enable the module + +Create the extension in each database where you want to use it. Creating the extension requires superuser privileges. It does not need `shared_preload_libraries`. + +```sql +CREATE EXTENSION gp_distribution_policy; +``` + +The extension installs its functions in the `public` schema. + +## Functions + +| Function | Returns | Description | +|---|---|---| +| `gp_table_distribution_check(regclass)` | `TABLE (gp_segment_id int, is_distribution_correct boolean)` | Runs the check on all segments and returns one row per segment. `is_distribution_correct` is `true` if all rows stored on that segment belong to it, and `false` if at least one row is on the wrong segment. | +| `gp_distribution_policy_table_check(relid regclass)` | `boolean` | Checks the rows of the table on the instance where the function is executed. It is the per-segment function behind `gp_table_distribution_check()`. Call `gp_table_distribution_check()` instead of calling this function directly. | + +## Example + +```sql +CREATE EXTENSION gp_distribution_policy; + +CREATE TABLE some_other_table(id int, some_other_data int) DISTRIBUTED BY (id); +INSERT INTO some_other_table SELECT i, i FROM generate_series(1,12) i; + +SELECT gp_table_distribution_check('some_other_table'); +``` + +On a cluster with three segments where all rows are placed correctly, the output is: + +```text + gp_table_distribution_check +----------------------------- + (0,t) + (1,t) + (2,t) +(3 rows) +``` + +If a segment holds rows that hash to a different segment, that segment is reported with `f`. For example, after the same row has been inserted directly on every segment, only one segment is correct: + +```text + gp_table_distribution_check +----------------------------- + (0,t) + (1,f) + (2,f) +(3 rows) +``` + +To get the result as separate columns: + +```sql +SELECT * FROM gp_table_distribution_check('some_other_table'); +``` + +## Usage notes + +- The input relation must be a table, a materialized view, or a directory table. For views, foreign tables, and partitioned tables, the function reports the error `input relation is not a table`. To check a partitioned table, run the check on each leaf partition. +- Heap tables and append-optimized tables are supported. +- The check is meaningful only for hash-distributed tables. For randomly distributed and replicated tables, the function reports `false` for some segments even though the data is placed correctly, so do not use it for such tables. +- The check scans the table on each segment and stops on a segment at the first misplaced row. On a large table with a correct distribution, it reads the whole table. +- The result tells you which segments contain misplaced rows. It does not identify the rows. diff --git a/versioned_docs/version-2.x/modules/gp-exttable-fdw.md b/versioned_docs/version-2.x/modules/gp-exttable-fdw.md new file mode 100644 index 0000000000..033e37e359 --- /dev/null +++ b/versioned_docs/version-2.x/modules/gp-exttable-fdw.md @@ -0,0 +1,72 @@ +--- +title: gp_exttable_fdw +--- + +# gp_exttable_fdw + +The `gp_exttable_fdw` module is the foreign-data wrapper that Apache Cloudberry uses to implement external tables. An external table is stored in the catalog as a foreign table that belongs to the `gp_exttable_server` foreign server, and `gp_exttable_fdw` scans it or inserts into it. + +:::note +`gp_exttable_fdw` and `gp_exttable_server` are special built-in objects that exist for backward compatibility with external tables. Do not use the wrapper directly in normal operation. Define external tables with [`CREATE EXTERNAL TABLE`](../sql-stmts/create-external-table.md) instead. +::: + +## Installation + +`gp_exttable_fdw` is always built and installed with Apache Cloudberry. It is part of the default `gpcontrib` build targets and does not depend on any `configure` option. + +The module is packaged as an extension (version `1.0`, not relocatable) whose objects are created in the `pg_catalog` schema. `initdb` registers the extension when the cluster is initialized, so you do not need to run `CREATE EXTENSION`. It does not require `shared_preload_libraries` or a restart. + +## Objects provided by the module + +| Object | Type | Description | +|---|---|---| +| `gp_exttable_fdw` | Foreign-data wrapper | The wrapper for external tables. Uses `gp_exttable_fdw_handler()` as its handler and `gp_exttable_permission_check()` as its validator. | +| `gp_exttable_server` | Foreign server | The server that all external tables belong to. | +| `gp_exttable_fdw_handler()` | Function | The FDW handler. Supports foreign scans and foreign inserts. | +| `gp_exttable_permission_check(text[], oid)` | Function | The FDW validator. Checks the table options and the permissions of the current role. | +| `pg_exttable()` | Function | Returns one row for each external table. Runs on the coordinator. | +| `pg_exttable` | View | Defined as `SELECT * FROM pg_exttable()`. | + +### The pg_exttable view + +The `pg_exttable` view shows the external table definition that is stored in the options of each foreign table of `gp_exttable_server`. + +| Column | Type | Description | +|---|---|---| +| `reloid` | `oid` | The OID of the external table. | +| `urilocation` | `text[]` | The location URIs of the external table. | +| `execlocation` | `text[]` | The segment locations where the external table runs. | +| `fmttype` | `"char"` | The format type of the external data. | +| `fmtopts` | `text` | The formatting options. | +| `options` | `text[]` | `{error_log_persistent=true}` when the table was created with `LOG ERRORS PERSISTENTLY`. Otherwise `NULL`. | +| `command` | `text` | The OS command to run for an `EXECUTE` external table. | +| `rejectlimit` | `int4` | The reject limit for single row error handling. `NULL` when single row error handling is not used. | +| `rejectlimittype` | `"char"` | The type of the reject limit: `r` for rows or `p` for percent. `NULL` when single row error handling is not used. | +| `logerrors` | `bool` | Whether errors are logged. | +| `encoding` | `int4` | The ID of the encoding of the external data. Use `pg_encoding_to_char(encoding)` to get the name. | +| `writable` | `bool` | Whether the table is a writable external table. | + +For example: + +```sql +SELECT reloid::regclass, urilocation, pg_encoding_to_char(encoding) +FROM pg_exttable; +``` + +## Foreign table options + +The validator of `gp_exttable_fdw` checks the following options of a foreign table that belongs to `gp_exttable_server`: + +| Option | Description | +|---|---| +| `format_type` | Required. One of `t` (text), `c` (CSV), or `b` (custom). | +| `location_uris` | The location URIs, separated by the `\|` character. You must specify either `location_uris` or `command`, and you cannot specify both. | +| `command` | The command of an `EXECUTE` external web table. Only superusers can set this option. | +| `is_writable` | A Boolean value that marks the table as writable. | +| `reject_limit_type` | `r` (rows, the default) or `p` (percent). | +| `reject_limit` | The reject limit for single row error handling. It cannot be used with a writable external table. | +| `encoding` | An encoding name (for example `UTF8` or `GBK`) or a numeric encoding ID. | + +When a role that is not a superuser creates a table, the validator also checks whether the role is allowed to create an external table for the protocol of the location URIs. + +A literal `|` or `\` character inside a location URI must be escaped as `\|` or `\\` in the `location_uris` option value. diff --git a/versioned_docs/version-2.x/modules/gp-internal-tools.md b/versioned_docs/version-2.x/modules/gp-internal-tools.md new file mode 100644 index 0000000000..17ff1c9f24 --- /dev/null +++ b/versioned_docs/version-2.x/modules/gp-internal-tools.md @@ -0,0 +1,100 @@ +--- +title: gp_internal_tools +--- + +# gp_internal_tools + +The `gp_internal_tools` module is a set of diagnostic tools for Apache Cloudberry. The extension installs the `session_state` schema with a view that shows the memory consumption of each session on the coordinator and on every segment. The module also ships shared libraries with low-level diagnostic functions for the workfile manager, the instrumentation shared memory, and append-optimized tables. + +:::note +This module is meant for troubleshooting. It exposes internal server state and is not needed for normal operation. +::: + +## Build and install + +`gp_internal_tools` is built and installed by default with Apache Cloudberry. No `configure` option is required. + +The build installs four shared libraries in the library directory of the installation: `gp_session_state_memory_stats`, `gp_workfile_mgr`, `gp_instrument_shmem`, and `gp_ao_co_diagnostics`. + +## Enable the module + +Create the extension in each database where you want to use it. Creating the extension requires superuser privileges. It does not need `shared_preload_libraries`. + +```sql +CREATE EXTENSION gp_internal_tools; +``` + +The extension creates its objects in the `session_state` schema. + +## Objects installed by the extension + +### session_state.session_level_memory_consumption + +A view with one row per session and per instance (the coordinator and each segment) that shows the memory used by the session on that instance. `SELECT` on the view is granted to `PUBLIC`. + +| Column | Type | Description | +|---|---|---| +| `datname` | `name` | Name of the database that the session is connected to. | +| `sess_id` | `integer` | Session ID. | +| `usename` | `name` | Name of the session user. | +| `query` | `text` | Current query of the session, as shown in `pg_stat_activity`. | +| `segid` | `integer` | Segment content ID. `-1` is the coordinator. | +| `vmem_mb` | `integer` | Virtual memory used by the session on this instance, in MB. | +| `is_runaway` | `boolean` | Whether the session is marked as a runaway session on this instance. | +| `qe_count` | `integer` | Number of query executor processes of the session on this instance. | +| `active_qe_count` | `integer` | Number of active query executor processes of the session on this instance. | +| `dirty_qe_count` | `integer` | Internal cleanup counter of the session (the `cleanupCountdown` field of the session state). | +| `runaway_vmem_mb` | `integer` | Value of the virtual memory counter of the session when it was marked as a runaway session. | +| `runaway_command_cnt` | `integer` | Command count of the session when it was marked as a runaway session. | +| `idle_start` | `timestamp with time zone` | Last time a query executor process of this session became idle. | + +The `datname`, `usename`, and `query` columns come from a left join with `pg_stat_activity` on the session ID, so they are `NULL` if the session is not found there. + +### Functions + +The view is built on two set-returning functions. `EXECUTE` on both is granted to `PUBLIC`. + +| Function | Runs on | Description | +|---|---|---| +| `session_state.session_state_memory_entries_f_on_master()` | Coordinator | Returns the session memory entries of the coordinator. | +| `session_state.session_state_memory_entries_f_on_segments()` | All segments | Returns the session memory entries of each segment. | + +Both functions return `SETOF record` with the columns `(segid int, sessionid int, vmem_mb int, runaway_status int, qe_count int, active_qe_count int, dirty_qe_count int, runaway_vmem_mb int, runaway_command_cnt int, idle_start timestamp with time zone)`. Query the view instead of calling them directly. + +## Example + +```sql +CREATE EXTENSION gp_internal_tools; + +SELECT * FROM session_state.session_level_memory_consumption LIMIT 0; +``` + +```text + datname | sess_id | usename | query | segid | vmem_mb | is_runaway | qe_count | active_qe_count | dirty_qe_count | runaway_vmem_mb | runaway_command_cnt | idle_start +---------+---------+---------+-------+-------+---------+------------+----------+-----------------+----------------+-----------------+---------------------+------------ +(0 rows) +``` + +To see the memory used by the current session on each instance: + +```sql +SELECT m.segid, m.vmem_mb, m.is_runaway +FROM session_state.session_level_memory_consumption m, pg_stat_activity a +WHERE a.pid = pg_backend_pid() + AND m.sess_id = a.sess_id; +``` + +## Additional diagnostic libraries + +The module also builds and installs the following shared libraries. The extension script does not create SQL functions for them: `CREATE EXTENSION gp_internal_tools` installs only the `session_state` objects described above. The C functions in these libraries are thin wrappers around internal server functions and are intended for support and development use. + +| Library | C functions | Purpose | +|---|---|---| +| `gp_workfile_mgr` | `gp_workfile_mgr_cache_entries`, `gp_workfile_mgr_used_diskspace` | Workfile manager diagnostics. `gp_workfile_mgr_cache_entries` returns the workfile cache entries of one segment. `gp_workfile_mgr_used_diskspace` returns a `(segid int4, bytes int8)` row with the number of bytes used for workfiles on a segment. | +| `gp_instrument_shmem` | `gp_instrument_shmem_summary`, `gp_instrument_shmem_detail` | Diagnostics of the instrumentation slots in shared memory. `gp_instrument_shmem_summary` returns `(segid int4, num_free int8, num_used int8)`. `gp_instrument_shmem_detail` returns one row per used slot with `(tmid int4, ssid int4, ccnt int2, segid int2, pid int4, nid int2, tuplecount int8, nloops int8, ntuples int8)`. | +| `gp_ao_co_diagnostics` | `gp_aoseg_wrapper`, `gp_aoseg_history_wrapper`, `gp_aocsseg_wrapper`, `gp_aocsseg_history_wrapper`, `gp_aoblkdir_wrapper`, `gp_aovisimap_wrapper`, `gp_aovisimap_entry_wrapper`, `gp_aovisimap_hidden_info_wrapper`, among others | Diagnostics for append-optimized row-oriented (AO) and column-oriented (CO) tables: segment file metadata including historic (non-visible) entries, block directory entries, and visibility map information such as the number of hidden tuples per segment file. | + +## Usage notes + +- The view calls a function on every segment, so each query on it dispatches to the whole cluster. +- Dropping the extension removes the view and the functions in the `session_state` schema. diff --git a/versioned_docs/version-2.x/modules/gp-legacy-string-agg.md b/versioned_docs/version-2.x/modules/gp-legacy-string-agg.md new file mode 100644 index 0000000000..14565d3203 --- /dev/null +++ b/versioned_docs/version-2.x/modules/gp-legacy-string-agg.md @@ -0,0 +1,83 @@ +--- +title: gp_legacy_string_agg +--- + +# gp_legacy_string_agg + +The `gp_legacy_string_agg` module provides a single-argument `string_agg(text)` aggregate function. Apache Cloudberry, like PostgreSQL, provides only two-argument forms of `string_agg()` as built-in functions. + +:::note +The single-argument function is deprecated. Use this module only to keep legacy SQL running while you migrate it to the built-in two-argument `string_agg()` function. +::: + +## Installation + +`gp_legacy_string_agg` is always built and installed with Apache Cloudberry. It is part of the default `gpcontrib` build targets and does not depend on any `configure` option. + +The module is an extension (version `1.0.0`, relocatable). Before you can use the function, create the extension in each database where you want to use it: + +```sql +CREATE EXTENSION gp_legacy_string_agg; +``` + +The module does not require `shared_preload_libraries` or a restart. + +## Usage + +The single-argument aggregate function has the following signature: + +```sql +string_agg(text) +``` + +The function concatenates the non-null input values into a string without a delimiter. Each aggregate result (each group) raises a deprecation warning. The result is `NULL` when there are no non-null input values. For example: + +```sql +SELECT string_agg(a) FROM (VALUES('aaaa'),('bbbb'),('cccc'),(NULL)) g(a); +``` + +``` +WARNING: Deprecated call to string_agg(text), use string_agg(text, text) instead + string_agg +-------------- + aaaabbbbcccc +(1 row) +``` + +The result depends on the order of the input rows. The order is unspecified by default. To control it, use an `ORDER BY` clause within the aggregate call. For example: + +```sql +CREATE TABLE foo(a int, b text); +INSERT INTO foo VALUES(1, 'aaaa'),(2, 'bbbb'),(3, 'cccc'), (4, NULL); +SELECT string_agg(b ORDER BY a) FROM foo; +``` + +``` +WARNING: Deprecated call to string_agg(text), use string_agg(text, text) instead + string_agg +-------------- + aaaabbbbcccc +(1 row) +``` + +## Migrate to the two-argument string_agg() function + +The built-in aggregate function takes a delimiter as its second argument: + +```sql +string_agg(text, text) +``` + +A call with an empty delimiter is equivalent to the single-argument function that this module provides. For example, replace `string_agg(a)` with: + +```sql +string_agg(a, '') +``` + +After you convert all the calls, you can drop the extension: + +```sql +DROP EXTENSION gp_legacy_string_agg; +``` + +For more information about the built-in function, see [Aggregate Functions](https://www.postgresql.org/docs/14/functions-aggregate.html) in the PostgreSQL documentation. diff --git a/versioned_docs/version-2.x/modules/gp-relsizes-stats.md b/versioned_docs/version-2.x/modules/gp-relsizes-stats.md new file mode 100644 index 0000000000..1582460263 --- /dev/null +++ b/versioned_docs/version-2.x/modules/gp-relsizes-stats.md @@ -0,0 +1,192 @@ +--- +title: gp_relsizes_stats +--- + +# gp_relsizes_stats + +The `gp_relsizes_stats` module collects and stores the on-disk sizes of relation files in Apache Cloudberry. It scans the database directories on the segments, records the size and modification time of each relation file, and exposes views that aggregate these figures per table and per schema. + +Statistics can be collected periodically by a background worker, or on demand by calling a function. Naptime parameters let you spread the file scan over time to limit the I/O load. + +## Build and install + +`gp_relsizes_stats` is built and installed by default with Apache Cloudberry. No `configure` option is required. + +To build and install only this module from the source tree: + +```shell +make -C gpcontrib/gp_relsizes_stats +make -C gpcontrib/gp_relsizes_stats install +``` + +## Enable the module + +1. Add the library to `shared_preload_libraries` and restart the cluster. The periodic background worker is registered only when the library is preloaded: + + ```shell + gpconfig -c shared_preload_libraries -v "'gp_relsizes_stats'" --skipvalidation + gpstop -ra + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +2. Create the extension in each database for which you want to collect statistics: + + ```sql + CREATE EXTENSION gp_relsizes_stats; + ``` + + The extension is marked as trusted, so a non-superuser who has the `CREATE` privilege on the database can create it. The extension creates the `relsizes_stats_schema` schema. The role that creates the extension is granted `USAGE` on the schema and `SELECT` on its tables and views, with grant option. + +3. Optional: enable periodic collection. It is off by default: + + ```shell + gpconfig -c gp_relsizes_stats.enabled -v on --skipvalidation + gpstop -u + ``` + +## How collection works + +- The main background worker connects to the `postgres` database. When `gp_relsizes_stats.enabled` is on, it runs a collection cycle and then sleeps for `gp_relsizes_stats.restart_naptime`. +- A collection cycle goes through all databases except `template0`, `template1`, `diskquota`, and `gpperfmon`. For each database it starts a dynamic background worker. Databases in which the extension is not created are skipped. +- For each database, the worker rebuilds `segment_file_map`, replaces the content of `segment_file_sizes` with a fresh scan of the database directories on all segments (the default tablespace and user tablespaces), and, if `gp_relsizes_stats.save_history` is on, refreshes `table_sizes_history`. +- Each collection cycle needs free background worker slots. If a worker cannot be registered, the error message hints at increasing `max_worker_processes`. If this error occurs in the periodic worker, the worker exits and does not start again until the cluster is restarted. + +## Configuration parameters + +All parameters can be set only in the server configuration and take effect after a configuration reload. The naptime parameters take plain integers in milliseconds. Values with a unit, such as `'6h'`, are not accepted. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `gp_relsizes_stats.enabled` | boolean | `off` | Enables periodic collection by the background worker. | +| `gp_relsizes_stats.save_history` | boolean | `on` | Saves table sizes to `relsizes_stats_schema.table_sizes_history` during collection. | +| `gp_relsizes_stats.restart_naptime` | integer (ms) | `21600000` | Time between two collection cycles. The default is 6 hours. Range: `0` to `2147483647`. | +| `gp_relsizes_stats.database_naptime` | integer (ms) | `0` | Total pause that is spread between databases in one cycle. The value is divided by the number of databases, and the worker sleeps for the result after each database. Range: `0` to `2147483647`. | +| `gp_relsizes_stats.file_naptime` | integer (ms) | `1` | Pause after each file during the file scan. Range: `0` to `2147483647`. | + +## SQL objects + +All objects are created in the `relsizes_stats_schema` schema. + +### Functions + +| Function | Returns | Description | +|---|---|---| +| `relsizes_collect_stats_once()` | `void` | Runs one collection cycle for all databases, without the naptime pauses. It does not check `gp_relsizes_stats.enabled`. | +| `get_stats_for_database(dboid oid, fast bool)` | `TABLE (segment integer, relfilenode oid, filepath text, size bigint, mtime bigint)` | Runs on all segments. Scans the directories of the database `dboid` and returns one row per relation file. When `fast` is `true`, the scan does not pause between files. It is used by the collection process. | + +### Tables + +`segment_file_map`: maps relations to their relation file nodes on each segment. Append-optimized, randomly distributed. + +| Column | Type | Description | +|---|---|---| +| `segment` | `integer` | Segment content ID. | +| `reloid` | `oid` | OID of the relation. | +| `relfilenode` | `oid` | Relation file node on that segment. | + +`segment_file_sizes`: the result of the last file scan. Append-optimized, randomly distributed. + +| Column | Type | Description | +|---|---|---| +| `segment` | `integer` | Segment content ID. | +| `relfilenode` | `oid` | Relation file node, parsed from the file name. | +| `filepath` | `text` | Path of the file. | +| `size` | `bigint` | File size in bytes. | +| `mtime` | `bigint` | Last modification time of the file, as a Unix timestamp. | + +`table_sizes_history`: table sizes with the date of collection. Randomly distributed. + +| Column | Type | Description | +|---|---|---| +| `insert_date` | `date` | Date on which the row was collected. | +| `nspname` | `text` | Schema name. | +| `relname` | `text` | Table name. | +| `size` | `bigint` | Table size in bytes. | +| `mtime` | `timestamp` | Latest modification time among the files of the table. | + +### Views + +`table_files`: one row per file of each table (relation kinds `r` and `p`, excluding the `pg_global` tablespace), including the files of TOAST tables, TOAST indexes, and the auxiliary relations of append-optimized tables. Indexes on the table and the block directory of append-optimized tables are not included, so the sizes in `table_sizes` are not the total relation sizes. + +| Column | Type | Description | +|---|---|---| +| `nspname` | `name` | Schema name. | +| `relname` | `name` | Table name. For rows with `own_file = false`, this is the name of the root partitioned table. | +| `segment` | `integer` | Segment content ID. | +| `relfilenode` | `oid` | Relation file node. | +| `filepath` | `text` | Path of the file. `NULL` if the relation has no file on disk. | +| `kind` | `text` | One of `main`, `toast`, `toast_idx`, `ao`, `ao_vm`, `ao_vm_idx`. | +| `size` | `bigint` | File size in bytes, `0` if the relation has no file. | +| `mtime` | `bigint` | Modification time as a Unix timestamp, `0` if the relation has no file. | +| `own_file` | `boolean` | `true` if the file belongs to the relation named in `relname`. `false` if the file belongs to a descendant partition and is attributed to the root partitioned table. | + +`table_sizes`: total size per table. For a root partitioned table, the size includes all its descendant partitions. Each partition also appears as its own row. + +| Column | Type | Description | +|---|---|---| +| `nspname` | `name` | Schema name. | +| `relname` | `name` | Table name. | +| `size` | `numeric` | Sum of file sizes in bytes. | +| `mtime` | `timestamp with time zone` | Latest modification time among the files of the table. | + +`namespace_sizes`: total size per schema. Only rows with `own_file = true` are summed, so partitions are not counted twice. + +| Column | Type | Description | +|---|---|---| +| `nspname` | `name` | Schema name. | +| `size` | `numeric` | Sum of file sizes in bytes. | + +## Example + +Collect statistics on demand and check the size of a table: + +```sql +CREATE EXTENSION gp_relsizes_stats; + +CREATE TABLE employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + department_id INT, + date_of_birth DATE +); + +INSERT INTO employees (first_name, last_name, department_id, date_of_birth) VALUES +('John', 'Doe', 1, '1988-06-15'), +('Jane', 'Smith', 2, '1990-07-20'), +('Emily', 'Jones', 1, '1985-08-30'); + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; +``` + +```text + size +------- + 65536 +(1 row) +``` + +Check the size of a schema: + +```sql +SELECT size FROM relsizes_stats_schema.namespace_sizes WHERE nspname = 'public'; +``` + +To allow another role to read the statistics: + +```sql +GRANT USAGE ON SCHEMA relsizes_stats_schema TO user2; +GRANT SELECT ON ALL TABLES IN SCHEMA relsizes_stats_schema TO user2; +``` + +## Usage notes + +- The periodic background worker runs only when the library is loaded through `shared_preload_libraries`. `relsizes_collect_stats_once()` also works without preloading the library. +- File sizes are collected on the segments only. Files on the coordinator are not scanned. +- The sizes in `segment_file_sizes` and in the views reflect the last collection, not the current state of the files. +- Each collection deletes all rows of `table_sizes_history` in the database and inserts the current table sizes with the current date. The table therefore holds the result of the latest collection that ran with `gp_relsizes_stats.save_history` turned on. +- Partitioned tables (root and intermediate levels) have no files on disk. They appear in `table_files` with a size of `0`. +- The module is tested only on Linux. diff --git a/versioned_docs/version-2.x/modules/gp-sparse-vector.md b/versioned_docs/version-2.x/modules/gp-sparse-vector.md new file mode 100644 index 0000000000..1aa24d353d --- /dev/null +++ b/versioned_docs/version-2.x/modules/gp-sparse-vector.md @@ -0,0 +1,296 @@ +--- +title: gp_sparse_vector +--- + +# gp_sparse_vector + +The `gp_sparse_vector` module implements a sparse vector data type named `svec`, and the associated functions, operators, and aggregates. The type uses compressed storage of zeros to make vector computations on floating point numbers faster. + +## Installation + +`gp_sparse_vector` is always built and installed with Apache Cloudberry. It is part of the default `gpcontrib` build targets and does not depend on any `configure` option. + +The module is an extension (version `1.0.1`). Before you can use it, create the extension in each database where you want to use it: + +```sql +CREATE EXTENSION gp_sparse_vector; +``` + +The module does not require `shared_preload_libraries` or a restart. + +:::note +The extension creates all its objects in the `sparse_vector` schema. Either qualify the object names with the schema (for example `sparse_vector.svec`), or add the schema to the search path: + +```sql +SET search_path TO sparse_vector, public; +``` + +The examples in this document assume that `sparse_vector` is in the search path. +::: + +## About the svec data type + +When you use arrays of floating point numbers for various calculations, you often have long runs of zeros. This is common in scientific, retail optimization, and text processing applications. Each floating point number takes 8 bytes of storage in memory or on disk, so saving those zeros is often impractical. Many computations also benefit from skipping over the zeros. + +For example, suppose that the following array of doubles is stored as a `float8[]`: + +``` +'{0, 33, <40,000 zeros>, 12, 22 }'::float8[] +``` + +This type of array arises often in text processing, where a dictionary might have 40-100K terms and each vector stores the number of times that each word appears in a particular document. This array occupies slightly more than 320KB of memory or disk, most of it zeros. Any operation that you perform on this data works on 40,001 fields that are not important. + +The built-in array data type has a bitmap for null values, but it is a poor choice for this use case. It is not optimized for `float8[]` or for long runs of zeros instead of nulls, and the bitmap is not run-length-encoding (RLE) compressed. Even if each zero were stored as a `NULL` in the array, the bitmap for nulls would use 5KB to mark the nulls. + +The `svec` type uses a simple RLE-based scheme that is biased toward being efficient for zero values. With this scheme, the bitmap of the example vector needs only 6 bytes. + +:::note +`svec` supports only `float8` vector values. +::: + +### Create svec values + +You can construct an `svec` directly from a float array: + +```sql +SELECT ('{0, 13, 37, 53, 0, 71 }'::float8[])::svec; +``` + +The text representation of an `svec` has the form `{counts}:{values}`, where each count is the number of times that the corresponding value repeats. For example, `'{1,2,3,4}:{3,4,5,6}'::svec` is a vector of 10 elements: one `3`, two `4`s, three `5`s, and four `6`s. + +The module defines implicit casts between `svec` and `float8[]` in both directions, and implicit casts from `int2`, `integer`, `bigint`, `float4`, `float8`, and `numeric` to `svec`. + +## Operators + +The operators take advantage of the sparse storage format, which makes the computations on `svec` values faster. The `+`, `-`, `*`, `/`, and `%*%` operators accept `svec` and `float8[]` operands in any combination. A number that is cast to `svec` is a scalar and is applied to every element of the other operand. Otherwise, both operands must have the same dimension. + +| Operator | Description | +|---|---| +| `+` | Adds the corresponding elements of two vectors. | +| `-` | Subtracts the corresponding elements of two vectors. | +| `*` | Multiplies the corresponding elements of two vectors. | +| `/` | Divides the corresponding elements of two vectors. | +| `^` | Raises each element of the left `svec` to the power that the right operand gives. The right operand must be a scalar, for example `a ^ 2::svec`. A vector on the right side raises an error. | +| `%*%` | Returns the dot product of two vectors as a `float8`. | +| `\|\|` | Concatenates two `svec` values. | +| `*\|\|` | Concatenates an `svec` (right operand) to itself the number of times given by the `int4` left operand. | +| `=` | Tests two `svec` values, or two `float8[]` values, for equality. | +| `<`, `<=`, `==`, `>=`, `>` | Compare two `svec` values by their L2 norm. These operators are the members of the `svec_l2_ops` B-tree operator class. | +| `<>` | Defined, but not a member of the operator class. In this version it is bound to the same function as `==`, so it returns `true` when the L2 norms are equal. Do not rely on it. | + +The plus (`+`) operator adds each of the terms of two vectors of the same dimension. For example, if vector `a = {0,1,5}` and vector `b = {4,3,2}`, compute the vector addition as follows: + +```sql +SELECT ('{0,1,5}'::float8[]::svec + '{4,3,2}'::float8[]::svec)::float8[]; +``` + +``` + float8 +--------- + {4,4,7} +``` + +A vector dot product (`%*%`) between vectors `a` and `b` returns a scalar result of type `float8`. Compute the dot product (`0*4 + 1*3 + 5*2 = 13`) as follows: + +```sql +SELECT '{0,1,5}'::float8[]::svec %*% '{4,3,2}'::float8[]::svec; +``` + +``` + ?column? +---------- + 13 +``` + +## Functions and aggregates + +| Function | Description | +|---|---| +| `dot(a, b)` | Returns the dot product. Each argument is an `svec` or a `float8[]`. | +| `l1norm(svec)`, `l1norm(float8[])` | Returns the L1 norm of the vector. | +| `l2norm(svec)`, `l2norm(float8[])` | Returns the L2 norm of the vector. | +| `log(svec)` | Returns an `svec` that contains the natural logarithm of each element. | +| `dimension(svec)` | Returns the number of elements in the vector. | +| `unnest(svec)` | Returns the elements of the vector as a set of `float8` values. | +| `vec_sum(svec)`, `vec_sum(float8[])` | Returns the sum of the elements of the vector. | +| `vec_median(svec)`, `vec_median(float8[])` | Returns the median of the elements of the vector. | +| `vec_pivot(svec, float8)` | Appends a `float8` value to an `svec`. | +| `dmin(float8, float8)`, `dmax(float8, float8)` | Returns the smaller or the larger of two `float8` values. | +| `gp_extract_feature_histogram(text[], text[])` | Returns the sparse feature vector of a document (second argument) for a dictionary (first argument). | + +| Aggregate | Description | +|---|---| +| `sum(svec)` | Returns the element-wise sum of the input vectors. | +| `vec_count_nonzero(svec)` | Returns an `svec` that contains, for each position, the number of input vectors that have a non-zero element in the position. | +| `count_vec(svec)` | The legacy name of `vec_count_nonzero(svec)`. | +| `array_agg(float8)` | Aggregates `float8` values into an `svec`. | +| `median_inmemory(float8)` | Returns the median of the input `float8` values. | + +For example, for the set of vectors `{0,1,5}`, `{10,0,3}`, `{0,0,3}`, and `{0,1,0}`, the count of non-zero terms is `{1,2,3}`: + +```sql +CREATE TABLE listvecs(a svec); + +INSERT INTO listvecs VALUES ('{0,1,5}'::float8[]), + ('{10,0,3}'::float8[]), + ('{0,0,3}'::float8[]), + ('{0,1,0}'::float8[]); + +SELECT vec_count_nonzero(a)::float8[] AS count_nonzero FROM listvecs; +``` + +``` + count_nonzero +--------------- + {1,2,3} +(1 row) +``` + +## Example + +This text classification example creates tables that represent a dictionary and some documents, and then performs document classification with vector arithmetic on word counts and proportions of dictionary words in each document. + +Enable the extension and set the search path: + +```sql +CREATE EXTENSION gp_sparse_vector; +SET search_path TO sparse_vector, public; +``` + +Suppose that you have a dictionary that is composed of words in a text array. Create a table to store the dictionary and insert the words: + +```sql +CREATE TABLE features (dictionary text[][]) DISTRIBUTED RANDOMLY; +INSERT INTO features + VALUES ('{am,before,being,bothered,corpus,document,i,in,is,me,never,now,one,really,second,the,third,this,until}'); +``` + +You have a set of documents, each also defined as an array of words. Create a table to represent the documents and insert some data: + +```sql +CREATE TABLE documents (docnum int, document text[]) DISTRIBUTED RANDOMLY; +INSERT INTO documents VALUES + (1,'{this,is,one,document,in,the,corpus}'), + (2,'{i,am,the,second,document,in,the,corpus}'), + (3,'{being,third,never,really,bothered,me,until,now}'), + (4,'{the,document,before,me,is,the,third,document}'); +``` + +To find the dictionary words that are present in each document, prepare a Sparse Feature Vector (SFV) for each document. An SFV is a vector of dimension N, where N is the number of dictionary words, and each SFV contains a count of each dictionary word in the document. + +Use the `gp_extract_feature_histogram()` function to create an SFV from a document. The function returns an `svec` that contains the count of each of the dictionary words in the ordinal positions of the dictionary. The vectors that you generate for the documents are referred to as the corpus: + +```sql +CREATE TABLE corpus (docnum int, feature_vector svec) DISTRIBUTED RANDOMLY; + +INSERT INTO corpus + (SELECT docnum, + gp_extract_feature_histogram( + (SELECT dictionary FROM features LIMIT 1), document) + FROM documents); +``` + +Show the feature vector of each document: + +```sql +SELECT docnum AS document_number, feature_vector::float8[] FROM corpus ORDER BY 1; +``` + +``` + document_number | feature_vector +-----------------+----------------------------------------- + 1 | {0,0,0,0,1,1,0,1,1,0,0,0,1,0,0,1,0,1,0} + 2 | {1,0,0,0,1,1,1,1,0,0,0,0,0,0,1,2,0,0,0} + 3 | {0,0,1,1,0,0,0,0,0,1,1,1,0,1,0,0,1,0,1} + 4 | {0,1,0,0,0,2,0,0,1,1,0,0,0,0,0,2,1,0,0} +(4 rows) +``` + +The word "am" is the first word in the dictionary, and there is `1` instance of it in the SFV of the second document, "i am the second document in the corpus". The word "before" has no instances in the document, so its value is `0`, and so on. + +Count the number of documents in which each feature occurs at least once: + +```sql +SELECT (vec_count_nonzero(feature_vector))::float8[] AS count_in_document FROM corpus; +``` + +``` + count_in_document +----------------------------------------- + {1,1,1,1,2,3,1,2,2,2,1,1,1,1,1,3,2,1,1} +(1 row) +``` + +Count all occurrences of each term in all documents: + +```sql +SELECT (sum(feature_vector))::float8[] AS sum_in_document FROM corpus; +``` + +``` + sum_in_document +----------------------------------------- + {1,1,1,1,2,4,1,2,2,2,1,1,1,1,1,5,2,1,1} +(1 row) +``` + +The remainder of the classification process is vector math. The count is turned into a weight that reflects Term Frequency / Inverse Document Frequency (tf/idf). The calculation for a given term in a given document is: + +``` +#_times_term_appears_in_this_doc * log( #_docs / #_docs_the_term_appears_in ) +``` + +`#_docs` is the total number of documents (4 in this case). For example, the term "document" in document 1 has a weight of `1 * log(4/3)`. In document 4, the term has a weight of `2 * log(4/3)`. Terms that appear in every document have a weight of `0`. + +Calculate the tf/idf: + +```sql +SELECT docnum, (feature_vector*logidf)::float8[] AS tf_idf + FROM (SELECT log(count(feature_vector)/vec_count_nonzero(feature_vector)) AS logidf FROM corpus) + AS foo, corpus ORDER BY docnum; +``` + +``` + docnum | tf_idf +--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 1 | {0,0,0,0,0.6931471805599453,0.28768207245178085,0,0.6931471805599453,0.6931471805599453,0,0,0,1.3862943611198906,0,0,0.28768207245178085,0,1.3862943611198906,0} + 2 | {1.3862943611198906,0,0,0,0.6931471805599453,0.28768207245178085,1.3862943611198906,0.6931471805599453,0,0,0,0,0,0,1.3862943611198906,0.5753641449035617,0,0,0} + 3 | {0,0,1.3862943611198906,1.3862943611198906,0,0,0,0,0,0.6931471805599453,1.3862943611198906,1.3862943611198906,0,1.3862943611198906,0,0,0.6931471805599453,0,1.3862943611198906} + 4 | {0,1.3862943611198906,0,0,0,0.5753641449035617,0,0,0.6931471805599453,0.6931471805599453,0,0,0,0,0,0.5753641449035617,0.6931471805599453,0,0} +(4 rows) +``` + +You can determine the angular distance between one document and the rest of the documents with the `ACOS` of the dot product of the document vectors. First, create a table that stores the tf/idf weighted vectors: + +```sql +CREATE TABLE weights AS + (SELECT docnum, (feature_vector*logidf) tf_idf + FROM (SELECT log(count(feature_vector)/vec_count_nonzero(feature_vector)) + AS logidf FROM corpus) foo, corpus ORDER BY docnum) + DISTRIBUTED RANDOMLY; +``` + +Then calculate the angular distance between the first document and every other document: + +```sql +SELECT docnum, trunc((180.*(ACOS(dmin(1.,(tf_idf%*%testdoc)/(l2norm(tf_idf)*l2norm(testdoc))))/(4.*ATAN(1.))))::numeric,2) + AS angular_distance FROM weights, + (SELECT tf_idf testdoc FROM weights WHERE docnum = 1 LIMIT 1) foo +ORDER BY 1; +``` + +``` + docnum | angular_distance +--------+------------------ + 1 | 0.00 + 2 | 78.82 + 3 | 90.00 + 4 | 80.02 +(4 rows) +``` + +The angular distance between document 1 and itself is 0 degrees, and between document 1 and document 3 is 90 degrees because they share no features at all. + +## Additional documentation + +For more information about this module, see the README files in the [`gpcontrib/gp_sparse_vector`](https://github.com/apache/cloudberry/tree/REL_2_STABLE/gpcontrib/gp_sparse_vector) directory of the Apache Cloudberry source code. diff --git a/versioned_docs/version-2.x/modules/gp-stats-collector.md b/versioned_docs/version-2.x/modules/gp-stats-collector.md new file mode 100644 index 0000000000..ff5e821f58 --- /dev/null +++ b/versioned_docs/version-2.x/modules/gp-stats-collector.md @@ -0,0 +1,284 @@ +--- +title: gp_stats_collector +--- + +# gp_stats_collector + +The `gp_stats_collector` module collects query execution metrics in Apache Cloudberry and reports them to an external monitoring agent. It hooks into the executor on the coordinator and on the segments, and for each query it sends lifecycle events (query text, plan, timestamps, user and database) together with system, instrumentation, interconnect, and spill metrics. Events are delivered over a Unix domain socket, or can be written to a log table inside the database. + +The module also provides an on-demand runtime query state interface (`pg_query_state`) that makes every backend of a running query report per-plan-node statistics without waiting for the query to finish. + +## Build and install + +`gp_stats_collector` is not built by default. To build it, configure Apache Cloudberry with the `--with-gp-stats-collector` option: + +```shell +./configure --with-gp-stats-collector [other options] +``` + +When this option is set, `configure` requires: + +- The protobuf library, version 3.0.0 or later (detected through `pkg-config`). +- The `protoc` compiler in `PATH`. + +The module contains C++ sources that are compiled with `-std=c++17`. + +## Enable the module + +1. Add the library to `shared_preload_libraries` and restart the cluster. The module allocates shared memory and registers signal handlers at server start, so it must be preloaded: + + ```shell + gpconfig -c shared_preload_libraries -v 'gp_stats_collector' + gpstop -ra + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +2. Create the extension in the database where you want to use the SQL objects. The extension requires superuser privileges to create: + + ```sql + CREATE EXTENSION gp_stats_collector; + ``` + + The extension creates the `gpsc` schema. The default extension version is `1.2`. + +3. Turn on collection. Collection is off by default: + + ```sql + SET gpsc.enable TO on; + ``` + +## Collected data + +A query passes through four states, and the module sends one message per state from the dispatcher process (on the coordinator) and from the executor processes (on segments): + +```text +submit -> ExecutorStart() -> start -> ExecutorRun() -> ExecutorFinish() -> end -> ExecutorEnd() -> done +``` + +| Data | Description | Controlled by | +|---|---|---| +| Query lifecycle | Query text, normalized query text, timestamps (submit, start, end, done), user and database information. | `gpsc.enable` | +| `EXPLAIN` data | The output of `EXPLAIN (TEXT, COSTS, VERBOSE)` for the query. | `gpsc.enable` | +| `EXPLAIN ANALYZE` data | The output of `EXPLAIN (TEXT, ANALYZE, BUFFERS, TIMING, VERBOSE)`. Sent only when the execution time reaches `gpsc.min_analyze_time`. | `gpsc.enable`, `gpsc.enable_analyze`, `gpsc.enable_cdbstats`, `gpsc.min_analyze_time` | +| Other metrics | Instrumentation, system, network, interconnect, and spill metrics. | `gpsc.enable` | + +Notes on the reported events: + +- Some queries skip the `end` state. In that case the `end` statistics are sent with `done`. +- If a query fails or is canceled, its statistics are sent with `done`. +- Some statistics are computed as the difference between the current value and a snapshot taken at `submit`. +- Each process (dispatcher or executor) sends its own statistics. +- A nested query on the dispatcher becomes a top-level query on the executors. + +## Configuration parameters + +### Collector parameters + +A parameter with the `superuser` context can be changed only by a superuser. A parameter with the `user` context can be changed by any role in its session. + +| Parameter | Type | Default | Context | Description | +|---|---|---|---|---| +| `gpsc.enable` | boolean | `off` | superuser | Enables the metrics collector. | +| `gpsc.uds_path` | string | `/tmp/gpsc_agent.sock` | superuser | File system path of the agent's Unix domain socket. | +| `gpsc.logging_mode` | enum | `uds` | superuser | Destination of collected events. `uds` sends events to the Unix domain socket. `tbl` writes events to the log table that is exposed through the `gpsc.log` view. Only superusers can view this setting. | +| `gpsc.enable_analyze` | boolean | `on` | superuser | Collects `EXPLAIN ANALYZE` data. When it is on, the dispatcher enables buffer, row, and timing instrumentation for the queries it runs. | +| `gpsc.enable_cdbstats` | boolean | `on` | superuser | Adds Cloudberry-specific executor statistics to the `EXPLAIN ANALYZE` data. Takes effect only when `gpsc.enable_analyze` is on. | +| `gpsc.min_analyze_time` | integer (ms) | `10000` | user | Minimum execution time above which the `EXPLAIN ANALYZE` text is reported. `0` reports it for all queries. `-1` turns the feature off. | +| `gpsc.report_nested_queries` | boolean | `on` | user | When on, both top-level and nested queries are reported from the coordinator, and segments report aggregates. When off, only top-level queries are reported from the coordinator and segments. | +| `gpsc.enable_utility` | boolean | `off` | user | Collects statistics for utility statements. | +| `gpsc.ignored_users_list` | string | empty | superuser | Comma-separated list of roles whose queries are not reported. | +| `gpsc.max_text_size` | integer (bytes) | `1048576` | superuser | Query texts and error messages longer than this size are trimmed. Range: `0` to `2147483647`. | +| `gpsc.max_plan_size` | integer (kB) | `1024` | superuser | Plan texts longer than this size are trimmed. Range: `0` to `2097151`. | + +:::note +It is not recommended to set `gpsc.max_text_size` or `gpsc.max_plan_size` higher than 1024 kB. +::: + +### Runtime query state parameters + +These parameters are registered only when the library is loaded through `shared_preload_libraries`. + +| Parameter | Type | Default | Context | Description | +|---|---|---|---|---| +| `pg_query_state.enable` | boolean | `on` | superuser | Enables the runtime query state module (executor hooks and signal handling). | +| `pg_query_state.enable_timing` | boolean | `on` | superuser | Collects timing data in addition to row counts. | +| `pg_query_state.enable_buffers` | boolean | `on` | superuser | Collects buffer usage. | + +## SQL objects + +All objects are created in the `gpsc` schema. `USAGE` on the schema is granted to `PUBLIC`. + +The `gpsc.log` and `gpsc.stat_messages` views are not granted to `PUBLIC`. All functions keep the default `EXECUTE` privilege for `PUBLIC`, and `gpsc.truncate_log()` and `gpsc.stat_messages_reset()` do not check the privileges of the caller. Revoke `EXECUTE` on these functions from `PUBLIC` if only administrators should run them. + +Interconnect statistics are collected only when the cluster uses the UDPIFC interconnect. + +### gpsc.stat_messages + +A view that shows, for the coordinator and for each segment, counters of the messages that the collector has tried to send. Querying it fails with an error if the library is not loaded through `shared_preload_libraries`. + +| Column | Type | Description | +|---|---|---| +| `segid` | `int` | Segment content ID. `-1` is the coordinator. | +| `total_messages` | `bigint` | Total number of messages, including failed ones. | +| `send_failures` | `bigint` | Number of messages that failed to be sent. | +| `connection_failures` | `bigint` | Number of failed connections to the agent socket. | +| `other_errors` | `bigint` | Number of other errors. | +| `max_message_size` | `int` | Size of the largest message. | + +### gpsc.stat_messages_reset() + +```sql +gpsc.stat_messages_reset() RETURNS SETOF void +``` + +Resets the counters shown in `gpsc.stat_messages` on the coordinator and on all segments. + +### gpsc.log + +A view over the log table on the coordinator and on all segments, ordered by `tmid`, `ssid`, `ccnt`. The table is filled when `gpsc.logging_mode` is `tbl`. Each row is one reported event. + +Rows are inserted in a way that keeps them even if the transaction that ran the query rolls back. + +| Column | Type | Description | +|---|---|---| +| `query_id` | `bigint` | Query ID. | +| `plan_id` | `bigint` | Hash of the normalized plan. | +| `nested_level` | `bigint` | Nesting level of the query. | +| `slice_id` | `bigint` | Slice ID. | +| `systemstat_vsize`, `systemstat_rss`, `systemstat_vmsizekb`, `systemstat_vmpeakkb` | `bigint` | Process memory figures from `/proc/pid/stat` and `/proc/pid/status`. | +| `systemstat_rchar`, `systemstat_wchar`, `systemstat_syscr`, `systemstat_syscw`, `systemstat_read_bytes`, `systemstat_write_bytes`, `systemstat_cancelled_write_bytes` | `bigint` | Process I/O counters from `/proc/pid/io`. | +| `instrumentation_ntuples`, `instrumentation_nloops`, `instrumentation_tuplecount` | `bigint` | Tuple and loop counters. | +| `instrumentation_shared_blks_hit`, `instrumentation_shared_blks_read`, `instrumentation_shared_blks_dirtied`, `instrumentation_shared_blks_written` | `bigint` | Shared buffer block counters. | +| `instrumentation_local_blks_hit`, `instrumentation_local_blks_read`, `instrumentation_local_blks_dirtied`, `instrumentation_local_blks_written` | `bigint` | Local buffer block counters. | +| `instrumentation_temp_blks_read`, `instrumentation_temp_blks_written` | `bigint` | Temporary file block counters. | +| `instrumentation_inherited_calls` | `bigint` | Number of nested queries. | +| `instrumentation_sent_total_bytes`, `instrumentation_sent_tuple_bytes`, `instrumentation_sent_chunks` | `bigint` | Network data sent. | +| `instrumentation_received_total_bytes`, `instrumentation_received_tuple_bytes`, `instrumentation_received_chunks` | `bigint` | Network data received. | +| `interconnect_total_recv_queue_size`, `interconnect_recv_queue_size_counting_time`, `interconnect_total_capacity`, `interconnect_capacity_counting_time`, `interconnect_total_buffers`, `interconnect_buffer_counting_time`, `interconnect_active_connections_num`, `interconnect_retransmits`, `interconnect_startup_cached_pkt_num`, `interconnect_mismatch_num`, `interconnect_crc_errors`, `interconnect_snd_pkt_num`, `interconnect_recv_pkt_num`, `interconnect_disordered_pkt_num`, `interconnect_duplicated_pkt_num`, `interconnect_recv_ack_num`, `interconnect_status_query_msg_num` | `bigint` | Interconnect statistics. | +| `spill_totalbytes` | `bigint` | Spill bytes written. | +| `systemstat_runningtimeseconds`, `systemstat_usertimeseconds`, `systemstat_kerneltimeseconds` | `double precision` | Wall clock, user, and kernel time in seconds. | +| `instrumentation_firsttuple`, `instrumentation_startup`, `instrumentation_total`, `instrumentation_blk_read_time`, `instrumentation_blk_write_time`, `instrumentation_startup_time`, `instrumentation_inherited_time` | `double precision` | Timing figures in seconds. | +| `datetime`, `submit_time`, `start_time`, `end_time` | `timestamp with time zone` | Event and query timestamps. | +| `tmid`, `ssid`, `ccnt` | `int` | Query key: a time-based ID from `gp_gettmid()`, the session ID, and the command counter. | +| `dbid` | `int` | The `dbid` of the reporting instance, as in `gp_segment_configuration`. | +| `segid` | `int` | Segment index. `-1` is the coordinator. | +| `spill_filecount` | `int` | Number of spill (temporary) files created. | +| `generator` | `text` | Planner or optimizer that produced the plan. | +| `query_text`, `template_query_text` | `text` | Query text and normalized query text. | +| `plan_text`, `template_plan_text` | `text` | `EXPLAIN` text and normalized plan text. | +| `analyze_text` | `text` | `EXPLAIN ANALYZE` text. | +| `user_name`, `database_name`, `rsgname` | `text` | Session user, database, and resource group. | +| `error_message` | `text` | Error message, if the query failed. | +| `query_status` | `text` | Event state, for example `QUERY_STATUS_SUBMIT`, `QUERY_STATUS_START`, `QUERY_STATUS_END`, `QUERY_STATUS_DONE`. | +| `utility` | `boolean` | Whether the event belongs to a utility statement. | + +### gpsc.truncate_log() + +```sql +gpsc.truncate_log() RETURNS SETOF void +``` + +Truncates the log table on the coordinator and on all segments. + +### gpsc.pg_query_state() + +```sql +gpsc.pg_query_state(pid int, trace_id bytea) RETURNS SETOF void +``` + +Triggers runtime per-node collection for the query that is running on the coordinator backend `pid`. The request is fanned out to the coordinator backend and to every segment backend that takes part in the query. Each backend walks its plan tree and pushes one batch of per-node statistics to the socket set in `gpsc.uds_path`. The coordinator also pushes the plan document, rate-limited so that repeated polls of a long query do not resend an unchanged plan. The function returns no rows and does not wait for the results. + +- `trace_id` must be exactly 16 bytes. The caller generates it, and the receiver uses it as the key of the collection. +- The function runs on the coordinator only. +- A backend cannot poll itself. + +### gpsc.pg_query_state_backends() + +```sql +gpsc.pg_query_state_backends(pid int) RETURNS TABLE(segid int, pid int) +``` + +Lists the segment backends that take part in the query running on backend `pid`, so that a collector knows how many batches to expect. A query that runs only on the coordinator is reported as a single row for the coordinator with `segid` less than `0`. The function returns an empty set when the target is not running a query or has `pg_query_state.enable` turned off. + +### Internal objects + +The following objects are created by the extension but are not meant to be called directly: + +- `gpsc.cbdb_mpp_query_state(gpsc.gp_segment_pid[], bytea)` and the composite type `gpsc.gp_segment_pid (segid int, pid int)`: used by `gpsc.pg_query_state()` to dispatch the request to segments. +- Functions whose names start with two underscores, such as `gpsc.__stat_messages_f_on_master()` or `gpsc.__test_uds_start_server(text)`: helpers behind the views and functions above, and helpers for the regression tests. + +### Privileges + +`EXECUTE` on `gpsc.pg_query_state()`, `gpsc.pg_query_state_backends()`, and `gpsc.cbdb_mpp_query_state()` is granted to `PUBLIC`. Access is checked in the server: a caller can poll a backend only if the caller is a superuser or runs under the same role as the target backend. This lets a monitoring agent use a non-superuser role without being able to observe queries of other roles. + +## Example + +The following example writes events to the log table instead of the socket and then reads them back. + +```sql +CREATE EXTENSION gp_stats_collector; + +-- Helper to sort the events of one query in lifecycle order +CREATE OR REPLACE FUNCTION gpsc_status_order(status text) +RETURNS integer +AS $$ +BEGIN + RETURN CASE status + WHEN 'QUERY_STATUS_SUBMIT' THEN 1 + WHEN 'QUERY_STATUS_START' THEN 2 + WHEN 'QUERY_STATUS_END' THEN 3 + WHEN 'QUERY_STATUS_DONE' THEN 4 + ELSE 999 + END; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +SET gpsc.ignored_users_list TO ''; +SET gpsc.enable TO TRUE; +SET gpsc.report_nested_queries TO TRUE; +SET gpsc.enable_utility TO FALSE; + +SET gpsc.logging_mode to 'TBL'; + +SELECT 1; +SELECT COUNT(*) FROM generate_series(1,10); + +RESET gpsc.logging_mode; + +SELECT segid, query_text, query_status FROM gpsc.log ORDER BY segid, ccnt, gpsc_status_order(query_status) ASC; +``` + +```text + segid | query_text | query_status +-------+---------------------------------------------+--------------------- + -1 | SELECT 1; | QUERY_STATUS_SUBMIT + -1 | SELECT 1; | QUERY_STATUS_START + -1 | SELECT 1; | QUERY_STATUS_END + -1 | SELECT 1; | QUERY_STATUS_DONE + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_SUBMIT + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_START + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_END + -1 | SELECT COUNT(*) FROM generate_series(1,10); | QUERY_STATUS_DONE +(8 rows) +``` + +To clear the log table: + +```sql +SELECT gpsc.truncate_log(); +``` + +To poll a running query, generate a 16-byte trace ID and pass the process ID of the coordinator backend that runs the query: + +```sql +SELECT * FROM gpsc.pg_query_state_backends(12345); +SELECT gpsc.pg_query_state(12345, '\x00112233445566778899aabbccddeeff'::bytea); +``` + +## Usage notes + +- Delivery is best-effort. A message that cannot be written to the socket is dropped and is not retried, so a slow or absent agent does not add latency to queries. Use `gpsc.stat_messages` to check for failures. +- Runtime query state results are pushed to the agent socket. `gpsc.pg_query_state()` does not return them to the SQL caller. +- `gpsc.pg_query_state()` and `gpsc.pg_query_state_backends()` report an error if the library is not loaded through `shared_preload_libraries`, if no backend with the given `pid` exists, or if the caller is not allowed to poll the target backend. `gpsc.pg_query_state_backends()` also reports an error if `pid` belongs to a segment executor process instead of the coordinator backend of the session. +- The process-level system statistics are read from the `/proc` file system. diff --git a/versioned_docs/version-2.x/modules/hstore.md b/versioned_docs/version-2.x/modules/hstore.md new file mode 100644 index 0000000000..169f029507 --- /dev/null +++ b/versioned_docs/version-2.x/modules/hstore.md @@ -0,0 +1,21 @@ +--- +title: hstore +--- + +# hstore + +The `hstore` module implements the `hstore` data type for storing sets of key/value pairs within a single value. This can be useful in various scenarios, such as rows with many attributes that are rarely examined, or semi-structured data. + +The Apache Cloudberry `hstore` module is equivalent to the PostgreSQL `hstore` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`hstore` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION hstore; +``` + +## Module documentation + +See [hstore](https://www.postgresql.org/docs/14/hstore.html) in the PostgreSQL documentation for details about the data type, operators, and functions that this module defines. diff --git a/versioned_docs/version-2.x/modules/index.md b/versioned_docs/version-2.x/modules/index.md new file mode 100644 index 0000000000..28601e38f0 --- /dev/null +++ b/versioned_docs/version-2.x/modules/index.md @@ -0,0 +1,162 @@ +--- +title: Extension Modules Overview +--- + +# Extension Modules Overview + +Apache Cloudberry ships with a set of optional modules that add data types, functions, operators, foreign-data wrappers, storage formats, and diagnostic tools to the database. The modules live in two directories of the source tree: + +- `contrib/` holds the modules that come from PostgreSQL, plus several modules that Apache Cloudberry adds. +- `gpcontrib/` holds the modules that are specific to Apache Cloudberry. + +This page lists every module in both directories as of Apache Cloudberry 2.2, with its version, its history across the 2.x releases, and how it is built. + +## Use a module + +Most modules are packaged as extensions. An extension is registered once in every database in which you want to use it: + +```sql +CREATE EXTENSION ; +``` + +To list the extensions that are installed on the cluster and the ones that are registered in the current database: + +```sql +SELECT name, default_version, installed_version FROM pg_available_extensions ORDER BY name; +``` + +Some modules are plain shared libraries that do not have SQL objects. You load such a module with `LOAD`, or by adding it to `shared_preload_libraries` or `session_preload_libraries`. The page of each module tells you which method applies. + +## How to read the tables + +**Version** is the extension version (`default_version` in the control file of the module) in Apache Cloudberry 2.2. A dash means that the module is a library or a program without an extension version. + +**2.x history** compares the 2.0, 2.1, and 2.2 releases with each other, and 2.0 with the 1.6 release: + +- **New in 2.0**, **New in 2.1**, **New in 2.2**: the release that first includes the module. +- **Updated in ...**: the extension version changed in that release. +- A dash means that the module is present in 2.0, 2.1, and 2.2 with the same version. + +**Build** tells you how the module gets built when you build Apache Cloudberry from source: + +- **Default**: `make` and `make install` at the top of the source tree build and install the module. +- **On request**: the module is in the source tree, but you need to build it explicitly, for example with `make -C contrib/ install`, or by building the `world` target. The build scripts in `devops/build/automation/cloudberry/scripts` of the source tree also build and install `contrib/`, so a cluster built with them has these modules, provided that the required `configure` options are set. +- **A `configure` option**: the module is built only when you pass that option to `configure`. Where the option is on by default, the table says so. +- **Not built**: the source is in the tree, but no makefile target of Apache Cloudberry builds it. + +The module files must be installed on the coordinator and on every segment host. + +:::note +A module name that links to a page in this section has been reviewed against the Apache Cloudberry 2.x source code. For the other PostgreSQL modules, the link points to the PostgreSQL documentation. Those modules follow the PostgreSQL behavior, and some of them might have limitations in a distributed cluster. +::: + +## Modules in contrib/ + +### PostgreSQL modules + +| Module | Description | Version | 2.x history | Build | +|---|---|---|---|---| +| [adminpack](https://www.postgresql.org/docs/14/adminpack.html) | Support functions for administration tools such as pgAdmin. | 2.1 | — | Not built | +| [amcheck](https://www.postgresql.org/docs/14/amcheck.html) | Verifies the logical consistency of relations and indexes. | 1.3 | — | On request | +| [auth_delay](https://www.postgresql.org/docs/14/auth-delay.html) | Pauses before reporting an authentication failure. | — | — | On request | +| [auto_explain](./auto-explain.md) | Logs execution plans of slow statements automatically. | — | — | Default | +| [bloom](https://www.postgresql.org/docs/14/bloom.html) | Index access method based on Bloom filters. | 1.0 | — | On request | +| bool_plperl | Transform between `bool` and PL/Perl. | 1.0 | — | On request, with `--with-perl` | +| [btree_gin](./btree-gin.md) | GIN operator classes with B-tree behavior. | 1.3 | — | Default | +| [btree_gist](https://www.postgresql.org/docs/14/btree-gist.html) | GiST operator classes with B-tree behavior. | 1.6 | — | On request | +| [citext](./citext.md) | Case-insensitive character string type. | 1.6 | — | Default | +| [cube](https://www.postgresql.org/docs/14/cube.html) | Data type for multidimensional cubes. | 1.5 | — | Not built | +| [dblink](./dblink.md) | Connects to other databases from within a session. | 1.2 | — | Default | +| [dict_int](https://www.postgresql.org/docs/14/dict-int.html) | Text search dictionary template for integers. | 1.0 | — | On request | +| [dict_xsyn](https://www.postgresql.org/docs/14/dict-xsyn.html) | Text search dictionary template for extended synonym processing. | 1.0 | — | On request | +| [earthdistance](https://www.postgresql.org/docs/14/earthdistance.html) | Calculates great-circle distances on the surface of the Earth. | 1.1 | — | Not built | +| [file_fdw](https://www.postgresql.org/docs/14/file-fdw.html) | Foreign-data wrapper for files in the server file system. | 1.0 | — | Default | +| [fuzzystrmatch](./fuzzystrmatch.md) | Determines similarities and distance between strings. | 1.1 | — | Default | +| [hstore](./hstore.md) | Stores sets of key/value pairs in a single value. | 1.8 | — | Default | +| hstore_plperl | Transform between `hstore` and PL/Perl. | 1.0 | — | On request, with `--with-perl` | +| hstore_plpython | Transform between `hstore` and PL/Python. | 1.0 | — | On request, with `--with-python` (on by default) | +| [intagg](https://www.postgresql.org/docs/14/intagg.html) | Integer aggregator and enumerator (obsolete). | 1.1 | — | On request | +| [intarray](https://www.postgresql.org/docs/14/intarray.html) | Functions, operators, and index support for arrays of integers. | 1.5 | — | On request | +| [isn](./isn.md) | Data types for international product numbering standards. | 1.2 | — | On request | +| jsonb_plperl | Transform between `jsonb` and PL/Perl. | 1.0 | — | On request, with `--with-perl` | +| jsonb_plpython | Transform between `jsonb` and PL/Python. | 1.0 | — | On request, with `--with-python` (on by default) | +| [lo](https://www.postgresql.org/docs/14/lo.html) | Large object maintenance. | 1.1 | — | Not built | +| [ltree](./ltree.md) | Data type for hierarchical tree-like labels. | 1.2 | — | On request | +| ltree_plpython | Transform between `ltree` and PL/Python. | 1.0 | — | On request, with `--with-python` (on by default) | +| [oid2name](https://www.postgresql.org/docs/14/oid2name.html) | Utility program that resolves OIDs and file nodes to names. | — | — | On request | +| [old_snapshot](https://www.postgresql.org/docs/14/oldsnapshot.html) | Shows the XID/time mapping that `old_snapshot_threshold` uses. | 1.0 | — | On request | +| [pageinspect](./pageinspect.md) | Inspects the contents of database pages at a low level. | 1.9 | — | Default | +| [passwordcheck](https://www.postgresql.org/docs/14/passwordcheck.html) | Checks the strength of passwords. | — | — | Default | +| [pg_buffercache](./pg-buffercache.md) | Examines the shared buffer cache. | 1.4.1 | **Updated in 2.0** (1.3 to 1.4.1) | Default | +| [pg_freespacemap](https://www.postgresql.org/docs/14/pgfreespacemap.html) | Examines the free space map. | 1.2 | — | On request | +| [pg_prewarm](https://www.postgresql.org/docs/14/pgprewarm.html) | Loads relation data into the buffer cache. | 1.2 | — | On request | +| [pg_standby](https://www.postgresql.org/docs/14/pgstandby.html) | Utility program for warm standby servers. | — | — | Not built | +| [pg_stat_statements](https://www.postgresql.org/docs/14/pgstatstatements.html) | Tracks planning and execution statistics of SQL statements. | 1.9 | — | On request | +| [pg_surgery](https://www.postgresql.org/docs/14/pgsurgery.html) | Performs surgery on a damaged relation. | 1.0 | — | On request | +| [pg_trgm](./pg-trgm.md) | Text similarity measurement and index searching based on trigrams. | 1.6 | — | Default | +| [pg_visibility](https://www.postgresql.org/docs/14/pgvisibility.html) | Examines the visibility map and page-level visibility information. | 1.2 | — | On request | +| [pgcrypto](./pgcrypto.md) | Cryptographic functions. | 1.3 | — | Default | +| [pgrowlocks](https://www.postgresql.org/docs/14/pgrowlocks.html) | Shows row-level locking information. | 1.2 | — | On request | +| [pgstattuple](https://www.postgresql.org/docs/14/pgstattuple.html) | Shows tuple-level statistics. | 1.5 | — | On request | +| [postgres_fdw](./postgres-fdw.md) | Foreign-data wrapper for data stored in external PostgreSQL servers. | 1.1 | — | On request | +| [seg](https://www.postgresql.org/docs/14/seg.html) | Data type for line segments or floating-point intervals. | 1.4 | — | On request | +| [sepgsql](https://www.postgresql.org/docs/14/sepgsql.html) | Label-based mandatory access control based on SELinux. | — | — | On request, with `--with-selinux` | +| [spi](https://www.postgresql.org/docs/14/contrib-spi.html) | Example trigger functions: `autoinc`, `insert_username`, `moddatetime`, and `refint`. | 1.0 | — | On request | +| [sslinfo](./sslinfo.md) | Information about the SSL certificate of the current client. | 1.2 | — | On request, requires OpenSSL | +| start-scripts | Example scripts for starting the server at boot time. | — | — | Not built | +| [tablefunc](./tablefunc.md) | Functions that return tables, including `crosstab`. | 1.0 | — | Default | +| [tcn](https://www.postgresql.org/docs/14/tcn.html) | Trigger function that notifies listeners of changes to a table. | 1.0 | — | On request | +| [test_decoding](https://www.postgresql.org/docs/14/test-decoding.html) | Example output plugin for logical decoding. | — | — | On request | +| [tsm_system_rows](./tsm-system-rows.md) | `SYSTEM_ROWS` table sampling method. | 1.0 | — | On request | +| [tsm_system_time](./tsm-system-time.md) | `SYSTEM_TIME` table sampling method. | 1.0 | — | On request | +| [unaccent](https://www.postgresql.org/docs/14/unaccent.html) | Text search dictionary that removes accents. | 1.1 | — | On request | +| [uuid-ossp](./uuid-ossp.md) | Functions that generate universally unique identifiers (UUIDs). | 1.1 | — | On request, with `--with-uuid=LIB` | +| [vacuumlo](https://www.postgresql.org/docs/14/vacuumlo.html) | Utility program that removes orphaned large objects. | — | — | On request | +| [xml2](https://www.postgresql.org/docs/14/xml2.html) | XPath querying and XSLT functions (obsolete). | 1.1 | — | On request, with `--with-libxml` | + +### Apache Cloudberry modules + +| Module | Description | Version | 2.x history | Build | +|---|---|---|---|---| +| extprotocol | Example of a custom protocol for external tables. | — | — | Default | +| formatter | Example of a custom data formatter for external tables. | — | — | Default | +| formatter_fixedwidth | Custom formatter that reads and writes fixed-width data files. | — | — | Default | +| indexscan | The `readindex()` function, which reads the entries of an index for testing and diagnosis. | — | — | Default | +| interconnect | The interconnect implementations (UDP, TCP, and proxy) as a loadable module. The `interconnect` extension adds views with interconnect statistics. | 1.0 | **Updated in 2.2** (packaged as an extension) | Default | +| pax_storage | The PAX table format. See [PAX Storage Format](../operate-with-data/pax-table-format.md). | — | **New in 2.0** | `--enable-pax` | +| sasdemo | Example programs and formatter functions for exchanging data with SAS. | — | — | Not built | +| try_convert | The `try_convert()` function for type casts that return a default value instead of an error. | 1.0 | **New in 2.2** | Default | +| udp2 | A reworked implementation of the UDP interconnect that is separated from the database kernel. | — | **New in 2.1** | `--enable-ic-udp2` | + +## Modules in gpcontrib/ + +| Module | Description | Version | 2.x history | Build | +|---|---|---|---|---| +| [diskquota](./diskquota.md) | Limits the disk space that schemas, roles, and tablespaces can use. | 2.3 | **New in 2.2** | `--with-diskquota` | +| gp_debug_numsegments | Gets and sets the default number of segments that new tables are created on. For development and testing. | 1.0 | — | Default (debug extensions) | +| [gp_distribution_policy](./gp-distribution-policy.md) | Checks whether the rows of a table are stored on the segments that the distribution policy expects. | 1.0 | — | Default | +| [gp_exttable_fdw](./gp-exttable-fdw.md) | Foreign-data wrapper that implements external tables. | 1.0 | — | Default | +| gp_inject_fault | Fault injection framework. For development and testing. | 1.0 | — | Default (debug extensions) | +| [gp_internal_tools](./gp-internal-tools.md) | Diagnostic views and functions for troubleshooting. | 1.0.0 | — | Default | +| [gp_legacy_string_agg](./gp-legacy-string-agg.md) | The single-argument `string_agg()` aggregate. | 1.0.0 | — | Default | +| [gp_relsizes_stats](./gp-relsizes-stats.md) | Tracks the on-disk sizes of tables. | 1.0 | **New in 2.2** | Default | +| gp_replica_check | Compares primary and mirror segments. For development and testing. | 0.0.1 | — | Default (debug extensions) | +| [gp_sparse_vector](./gp-sparse-vector.md) | Sparse vector data type and its operators. | 1.0.1 | — | Default | +| [gp_stats_collector](./gp-stats-collector.md) | Collects query and plan execution metrics and reports them to a monitoring agent. | 1.2 | **New in 2.2** | `--with-gp-stats-collector` | +| [gp_toolkit](../sys-catalogs/gp_toolkit.md) | Administrative views and functions. | 1.7 | **Updated in 2.0** (1.4 to 1.6), **Updated in 2.2** (1.6 to 1.7) | Default | +| gpcloud | The `s3` protocol for external tables and the `gpcheckcloud` utility. See [Load Data from Amazon S3](../data-loading/load-data-from-s3.md). | — | — | `--enable-gpcloud` | +| gpmapreduce | The `gpmapreduce` utility. | — | — | `--enable-mapreduce` | +| [orafce](./orafce.md) | Oracle compatibility functions and packages. | 4.9 | — | `--enable-orafce` | +| pg_hint_plan | Controls execution plans with hints in SQL comments. See [Query Plan Hints](../performance/optimize-queries/query-hints.md). | 1.3.9 | **New in 2.0** | Default | +| pxf_fdw | Foreign-data wrapper for the Platform Extension Framework (PXF). See [Load Data Using PXF](../data-loading/load-data-using-pxf.md). | 1.0 | — | `--enable-pxf` (on by default) | +| [reject_partition_fullscan](./reject-partition-fullscan.md) | Rejects queries that scan all partitions of a table without pruning. | 1.0 | **New in 2.2** | Default (debug extensions) | +| [yezzey](./yezzey.md) | Offloads the data of append-optimized tables to S3-compatible object storage. The source is a Git submodule. | 2.0 | **New in 2.2** | `--with-yezzey` | +| zstd | Zstandard compression for append-optimized tables. | — | — | `--with-zstd` (on by default) | + +The modules that are marked as debug extensions are built unless you pass `--disable-debug-extensions` to `configure`. Do not use `gp_debug_numsegments`, `gp_inject_fault`, or `gp_replica_check` in production clusters. + +### Removed modules + +| Module | Description | Removed in | +|---|---|---| +| quicklz | QuickLZ compression for append-optimized tables. | 2.1 | diff --git a/versioned_docs/version-2.x/modules/isn.md b/versioned_docs/version-2.x/modules/isn.md new file mode 100644 index 0000000000..d18ae3f99c --- /dev/null +++ b/versioned_docs/version-2.x/modules/isn.md @@ -0,0 +1,29 @@ +--- +title: isn +--- + +# isn + +The `isn` module provides data types for the international product numbering standards EAN13, UPC, ISBN (books), ISMN (music), and ISSN (serials). + +The Apache Cloudberry `isn` module is equivalent to the PostgreSQL `isn` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`isn` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/isn install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION isn; +``` + +## Module documentation + +See [isn](https://www.postgresql.org/docs/14/isn.html) in the PostgreSQL documentation for details about the data types, casts, and functions that this module defines. diff --git a/versioned_docs/version-2.x/modules/ltree.md b/versioned_docs/version-2.x/modules/ltree.md new file mode 100644 index 0000000000..636d482ff3 --- /dev/null +++ b/versioned_docs/version-2.x/modules/ltree.md @@ -0,0 +1,33 @@ +--- +title: ltree +--- + +# ltree + +The `ltree` module implements a data type named `ltree` that you can use to represent labels of data stored in a hierarchical tree-like structure. The module also provides extensive facilities for searching through label trees. + +The Apache Cloudberry `ltree` module is based on the PostgreSQL `ltree` module. See [MPP considerations](#mpp-considerations) for the difference. + +## Install and register the module + +`ltree` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/ltree install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION ltree; +``` + +## MPP considerations + +The extension does not provide a hash operator class for the `ltree` data type. For this reason, you cannot use an `ltree` column as the distribution key of a table. A table whose only column is of type `ltree` is created with random distribution. + +## Module documentation + +See [ltree](https://www.postgresql.org/docs/14/ltree.html) in the PostgreSQL documentation for details about the data types, operators, and functions that this module defines. diff --git a/versioned_docs/version-2.x/modules/orafce.md b/versioned_docs/version-2.x/modules/orafce.md new file mode 100644 index 0000000000..67df12b70d --- /dev/null +++ b/versioned_docs/version-2.x/modules/orafce.md @@ -0,0 +1,119 @@ +--- +title: orafce +--- + +# orafce + +The `orafce` module provides Oracle compatibility SQL functions, data types, and packages in Apache Cloudberry. It implements a subset of the functions and packages of the Oracle RDBMS, which helps you port Oracle applications. + +The module in Apache Cloudberry is a modified version of the open source [Orafce](https://github.com/orafce/orafce) extension. The source files are in the `gpcontrib/orafce` directory of the Apache Cloudberry source code and are based on Orafce 4.9 (4.9.2). + +## Installation + +`orafce` is not built by default. To build and install the module with Apache Cloudberry, specify the `--enable-orafce` option when you run `configure`: + +```shell +./configure --enable-orafce [other options...] +make +make install +``` + +The module is an extension (version `4.9`, not relocatable). Before you can use it, create the extension in each database where you want to use it: + +```sql +CREATE EXTENSION orafce; +``` + +## Schemas + +The extension creates its objects in the following schemas: + +| Schema | Content | +|---|---| +| `oracle` | Oracle compatible functions (for example `oracle.substr()`, `oracle.add_months()`, `oracle.nvl2()`, and `oracle.sysdate()`), the `oracle.date` domain, the `oracle.varchar2` and `oracle.nvarchar2` data types, and the `oracle.dual` view. | +| `dbms_output` | The `DBMS_OUTPUT` package. | +| `dbms_pipe` | The `DBMS_PIPE` package. | +| `dbms_utility` | The `DBMS_UTILITY` package. | +| `dbms_assert` | The `DBMS_ASSERT` package. | +| `dbms_random` | The `DBMS_RANDOM` package. | +| `dbms_sql` | The `DBMS_SQL` package. | +| `utl_file` | The `UTL_FILE` package. | +| `plvstr`, `plvchr`, `plvdate`, `plvsubst`, `plvlex` | The PLVision string, character, date, substitution, and lexer packages. | +| `plunit` | The PLUnit assertion functions. | + +## Usage + +To call the functions in the `oracle` schema without a schema qualifier, add the schema to the search path. For example, this command sets the default search path of a database: + +```sql +ALTER DATABASE db_name SET search_path = "$user", public, oracle; +``` + +Some functions in the `oracle` schema have the same name as a built-in function, for example `oracle.substr()`. The built-in `pg_catalog` schema is searched first by default. To give precedence to the Oracle compatible version, either qualify the function name with the schema, or put `oracle` before `pg_catalog` in the search path: + +```sql +SET search_path TO oracle, "$user", public, pg_catalog; +``` + +For example: + +```sql +SET search_path TO public, oracle; +SELECT add_months(date '2003-08-01', 3); +``` + +``` + add_months +------------ + 2003-11-01 +(1 row) +``` + +You can use the `oracle.varchar2` and `oracle.nvarchar2` data types as the distribution key or as the partition key of a table: + +```sql +CREATE TABLE t1 (str oracle.varchar2(30)) DISTRIBUTED BY (str); +``` + +### Configuration parameters + +The module defines the following configuration parameters. You can set them in a session with `SET`. + +| Parameter | Default | Description | +|---|---|---| +| `orafce.nls_date_format` | Not set | Emulates the date output behavior of Oracle. | +| `orafce.timezone` | `GMT` | The time zone that the `oracle.sysdate()` function uses and that `oracle.dbtimezone()` returns. | +| `orafce.varchar2_null_safe_concat` | `false` | Whether the concatenation of `varchar2` values is null safe. | +| `orafce.sys_guid_source` | `uuid_generate_v1` | The function that `oracle.sys_guid()` uses to make its result: `uuid_generate_v1`, `uuid_generate_v1mc`, or `uuid_generate_v4` from the `uuid-ossp` extension, or the built-in `gen_random_uuid`. In this version, the value `uuid_generate_v4` is stored as `uuid_generate_v1`. | +| `orafce.using_substring_zero_width_in_substr` | `warning_oracle` | The behavior of the `substr()` function when the substring length argument is zero. Valid values are `warning_oracle`, `warning_orafce`, `oracle`, and `orafce`. | + +## Differences from the original Orafce extension + +The module in Apache Cloudberry differs from the original Orafce 4.9 extension as follows: + +- The `oracle.decode()` functions are not created. Apache Cloudberry has built-in support for `decode()`: the parser transforms a `decode()` call into a `CASE` expression, so you can use `decode()` without the `orafce` extension. For example: + + ```sql + SELECT decode(2, 1, 'ABC', 2, 'DEF'); + ``` + + ``` + case + ------ + DEF + (1 row) + ``` + +- The `dbms_alert` package is not available. The `dbms_alert` schema and its functions are not created, because the implementation cannot run in an MPP environment. + +:::note +PostgreSQL 14, on which Apache Cloudberry is based, provides `sinh()`, `cosh()`, and `tanh()` as built-in functions. See [Mathematical Functions and Operators](https://www.postgresql.org/docs/14/functions-math.html) in the PostgreSQL documentation. You do not need `orafce` to use them. +::: + +## Additional documentation + +For detailed information about the individual functions and supporting objects in this module, see the following files in the [`gpcontrib/orafce`](https://github.com/apache/cloudberry/tree/REL_2_STABLE/gpcontrib/orafce) directory of the Apache Cloudberry source code: + +- `README.asciidoc` +- The documents in the `doc/orafce_documentation` directory +- The Oracle to PostgreSQL migration notes in the `doc/sql_migration` directory diff --git a/versioned_docs/version-2.x/modules/pageinspect.md b/versioned_docs/version-2.x/modules/pageinspect.md new file mode 100644 index 0000000000..0e5c0cb421 --- /dev/null +++ b/versioned_docs/version-2.x/modules/pageinspect.md @@ -0,0 +1,94 @@ +--- +title: pageinspect +--- + +# pageinspect + +The `pageinspect` module provides functions for low-level inspection of the contents of database pages. Only superusers can use these functions. + +The Apache Cloudberry `pageinspect` module is based on the PostgreSQL `pageinspect` module. The differences are described in [MPP considerations](#mpp-considerations) and [Functions added by Apache Cloudberry](#functions-added-by-apache-cloudberry). + +## Install and register the module + +`pageinspect` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION pageinspect; +``` + +If an older version of the extension is already registered in a database, update it to get the functions of the current version: + +```sql +ALTER EXTENSION pageinspect UPDATE; +``` + +## MPP considerations + +- `get_raw_page()` does not read pages of append-optimized tables. It also rejects relations that have no page storage of their own, such as views, foreign tables, and partitioned tables. +- A function that reads data from a relation reads it only from the instance on which the function runs. For example, when you call `get_raw_page()` for a user table in a normal session, the function runs on the coordinator, which stores no data for the table, and the call fails with the error `block number 0 is out of range for relation "..."`. System catalog tables can be read on the coordinator. + + To inspect pages on a segment, run the function on that segment, for example in a utility-mode connection to the segment. + +## Functions + +The module provides the PostgreSQL 14 `pageinspect` functions: + +| Group | Functions | +|-------|-----------| +| General | `get_raw_page`, `page_header`, `page_checksum`, `fsm_page_contents` | +| Heap | `heap_page_items`, `tuple_data_split`, `heap_page_item_attrs`, `heap_tuple_infomask_flags` | +| B-tree | `bt_metap`, `bt_page_stats`, `bt_page_items` | +| BRIN | `brin_page_type`, `brin_metapage_info`, `brin_revmap_data`, `brin_page_items` | +| GIN | `gin_metapage_info`, `gin_page_opaque_info`, `gin_leafpage_items` | +| GiST | `gist_page_opaque_info`, `gist_page_items`, `gist_page_items_bytea` | +| Hash | `hash_page_type`, `hash_page_stats`, `hash_page_items`, `hash_bitmap_info`, `hash_metapage_info` | + +See [pageinspect](https://www.postgresql.org/docs/14/pageinspect.html) in the PostgreSQL documentation for details about these functions. + +## Functions added by Apache Cloudberry + +### Bitmap index functions + +| Function | Description | +|----------|-------------| +| `bm_metap(relname text) returns record` | Returns information about the meta page of a bitmap index: `magic`, `version`, `auxrelid`, `auxindexrelid`, and `lovlastblknum`. | +| `bm_bitmap_page_header(relname text, blkno int) returns record` | Returns the header information of a bitmap page: `num_words`, `next_blkno`, and `last_tid`. | +| `bm_lov_page_items(relname text, blkno int) returns setof record` | Returns the list of values (LOV) items of a bitmap LOV page. | +| `bm_bitmap_page_items(relname text, blkno int) returns setof record` | Returns the content words of a bitmap page and whether each word is compressed: `word_num`, `compressed`, and `content_word`. | +| `bm_bitmap_page_items(page bytea) returns setof record` | Same as the previous function, for a page image obtained with `get_raw_page()`. | + +`bm_lov_page_items()` returns the columns `itemoffset`, `lov_head_blkno`, `lov_tail_blkno`, `last_complete_word`, `last_word`, `last_tid`, `last_setbit_tid`, `is_last_complete_word_fill`, and `is_last_word_fill`. + +### BRIN functions + +Apache Cloudberry supports BRIN indexes on append-optimized tables, and `pageinspect` reflects this: + +- `brin_metapage_info(page bytea)` returns four columns in addition to the PostgreSQL ones: `isAO` (`boolean`), `firstrevmappages` (`bigint[]`), `lastrevmappages` (`bigint[]`), and `lastrevmappagenums` (`bigint[]`). The arrays describe the range map chains of an index on an append-optimized table. +- `brin_revmap_chain(indexrelid regclass, segno int) returns bigint[]` returns the chain of range map block numbers for a segment file number (block sequence) of an append-optimized table. `segno` must be in the range from 0 to 127. + +## Examples + +Get information about the meta page of the bitmap index named `i1`: + +```sql +SELECT * FROM bm_metap('i1'); +``` + +Display the header information of the second block of the bitmap index `i1`: + +```sql +SELECT * FROM bm_bitmap_page_header('i1', 2); +``` + +Display the LOV items in the first block of the bitmap index `i1`: + +```sql +SELECT * FROM bm_lov_page_items('i1', 1) ORDER BY itemoffset; +``` + +Return the content words in the second block of the bitmap index `i1`, either by index name and block number, or from a page image: + +```sql +SELECT * FROM bm_bitmap_page_items('i1', 2) ORDER BY word_num; +SELECT * FROM bm_bitmap_page_items(get_raw_page('i1', 2)) ORDER BY word_num; +``` diff --git a/versioned_docs/version-2.x/modules/pg-buffercache.md b/versioned_docs/version-2.x/modules/pg-buffercache.md new file mode 100644 index 0000000000..30a673d6eb --- /dev/null +++ b/versioned_docs/version-2.x/modules/pg-buffercache.md @@ -0,0 +1,104 @@ +--- +title: pg_buffercache +--- + +# pg_buffercache + +The `pg_buffercache` module provides a means for examining what is happening in the shared buffer cache in real time. + +The Apache Cloudberry `pg_buffercache` module is based on the PostgreSQL `pg_buffercache` module. In addition to the PostgreSQL views and functions, which report on the instance that you are connected to, it provides `gp_`-prefixed views that collect the same information from the coordinator and from all primary segments. + +## Install and register the module + +`pg_buffercache` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION pg_buffercache; +``` + +By default, access to the views and functions of this module is restricted to superusers and to members of the `pg_monitor` role. + +## Views and functions for a single instance + +These objects report on the shared buffers of the instance that runs the query. When you query them in a normal session, that instance is the coordinator. + +| Object | Description | +|--------|-------------| +| `pg_buffercache` view, `pg_buffercache_pages()` function | One row for each shared buffer. | +| `pg_buffercache_summary` view, `pg_buffercache_summary()` function | One row that summarizes the state of the shared buffer cache. | +| `pg_buffercache_usage_counts` view, `pg_buffercache_usage_counts()` function | One row for each usage count value. | + +The `pg_buffercache` view has the following columns: + +| Column | Type | Description | +|--------|------|-------------| +| `bufferid` | `integer` | ID of the buffer, in the range from 1 to `shared_buffers`. | +| `relfilenode` | `oid` | Filenode number of the relation. | +| `reltablespace` | `oid` | Tablespace OID of the relation. | +| `reldatabase` | `oid` | Database OID of the relation. | +| `relforknumber` | `smallint` | Fork number within the relation. | +| `relblocknumber` | `bigint` | Page number within the relation. | +| `isdirty` | `boolean` | Whether the page is dirty. | +| `usagecount` | `smallint` | Clock-sweep access count. | +| `pinning_backends` | `integer` | Number of backends that pin this buffer. | + +The `pg_buffercache_summary` view has the following columns: + +| Column | Type | Description | +|--------|------|-------------| +| `buffers_used` | `integer` | Number of used shared buffers. | +| `buffers_unused` | `integer` | Number of unused shared buffers. | +| `buffers_dirty` | `integer` | Number of dirty shared buffers. | +| `buffers_pinned` | `integer` | Number of pinned shared buffers. | +| `usagecount_avg` | `double precision` | Average usage count of the used shared buffers. | + +The `pg_buffercache_usage_counts` view has the following columns: + +| Column | Type | Description | +|--------|------|-------------| +| `usage_count` | `integer` | A possible buffer usage count. | +| `buffers` | `integer` | Number of buffers with this usage count. | +| `dirty` | `integer` | Number of dirty buffers with this usage count. | +| `pinned` | `integer` | Number of pinned buffers with this usage count. | + +The summary and usage count views are much cheaper to query than `pg_buffercache`. + +## Cluster-wide views + +| View | Description | +|------|-------------| +| `gp_buffercache` | The `pg_buffercache` rows of the coordinator and of every primary segment. | +| `gp_buffercache_summary` | The `pg_buffercache_summary` row of the coordinator and of every primary segment: one row per instance. | +| `gp_buffercache_usage_counts` | The `pg_buffercache_usage_counts` rows of the coordinator and of every primary segment. | +| `gp_buffercache_summary_aggregated` | A single row that aggregates `gp_buffercache_summary` over all instances: the sum of each buffer counter and the average of `usagecount_avg`. | +| `gp_buffercache_usage_counts_aggregated` | A single row that contains the sums of the columns of `gp_buffercache_usage_counts`, including the `usage_count` column itself. It is not a breakdown per usage count. | + +The first three views have the columns of the corresponding `pg_` view, preceded by a `gp_segment_id` column. The value of `gp_segment_id` is `-1` for the coordinator. + +## Examples + +Show the buffer cache summary of each instance: + +```sql +SELECT * FROM gp_buffercache_summary; +``` + +Show the totals for the whole cluster: + +```sql +SELECT * FROM gp_buffercache_summary_aggregated; +``` + +Count the dirty buffers on each instance: + +```sql +SELECT gp_segment_id, count(*) AS dirty_buffers +FROM gp_buffercache +WHERE isdirty +GROUP BY gp_segment_id +ORDER BY gp_segment_id; +``` + +## Module documentation + +See [pg_buffercache](https://www.postgresql.org/docs/14/pgbuffercache.html) in the PostgreSQL documentation for more information about the `pg_buffercache` view. The summary and usage count functions were added to PostgreSQL in version 16 and are described in the [PostgreSQL 16 documentation](https://www.postgresql.org/docs/16/pgbuffercache.html). diff --git a/versioned_docs/version-2.x/modules/pg-trgm.md b/versioned_docs/version-2.x/modules/pg-trgm.md new file mode 100644 index 0000000000..23e47c9b9c --- /dev/null +++ b/versioned_docs/version-2.x/modules/pg-trgm.md @@ -0,0 +1,21 @@ +--- +title: pg_trgm +--- + +# pg_trgm + +The `pg_trgm` module provides functions and operators for determining the similarity of alphanumeric text based on trigram matching. The module also provides GiST and GIN index operator classes that support fast searching for similar strings. + +The Apache Cloudberry `pg_trgm` module is equivalent to the PostgreSQL `pg_trgm` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`pg_trgm` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION pg_trgm; +``` + +## Module documentation + +See [pg_trgm](https://www.postgresql.org/docs/14/pgtrgm.html) in the PostgreSQL documentation for details about the functions, operators, and index operator classes in this module. diff --git a/versioned_docs/version-2.x/modules/pgcrypto.md b/versioned_docs/version-2.x/modules/pgcrypto.md new file mode 100644 index 0000000000..c4bf02d849 --- /dev/null +++ b/versioned_docs/version-2.x/modules/pgcrypto.md @@ -0,0 +1,83 @@ +--- +title: pgcrypto +--- + +# pgcrypto + +The `pgcrypto` module provides cryptographic functions: general hashing, password hashing, PGP encryption, raw encryption, and random data generation. With these functions, you can store certain columns of data in encrypted form. Data that is stored encrypted cannot be read by anyone who does not have the encryption key, and it cannot be read directly from the disks. + +:::note +The `pgcrypto` functions run inside the database server, which means that all the data and passwords move between `pgcrypto` and the client application in clear text. For better security, also use SSL connections between the client and the Apache Cloudberry coordinator. +::: + +## Install and register the module + +`pgcrypto` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION pgcrypto; +``` + +## Functions + +| Group | Functions | +|-------|-----------| +| General hashing | `digest()`, `hmac()` | +| Password hashing | `crypt()`, `gen_salt()` | +| PGP encryption | `pgp_sym_encrypt()`, `pgp_sym_encrypt_bytea()`, `pgp_sym_decrypt()`, `pgp_sym_decrypt_bytea()`, `pgp_pub_encrypt()`, `pgp_pub_encrypt_bytea()`, `pgp_pub_decrypt()`, `pgp_pub_decrypt_bytea()`, `pgp_key_id()`, `armor()`, `dearmor()`, `pgp_armor_headers()` | +| Raw encryption | `encrypt()`, `decrypt()`, `encrypt_iv()`, `decrypt_iv()` | +| Random data | `gen_random_bytes()`, `gen_random_uuid()` | + +Example: + +```sql +SELECT digest('santa claus', 'sha256'); +SELECT pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=aes256'), 'mypass'); +``` + +In addition to the ciphers of the PostgreSQL module, the raw encryption functions accept the SM4 ciphers `sm4-ecb` and `sm4-cbc`, and `digest()` and `hmac()` accept the `sm3` digest. In a build that uses OpenSSL, the SM4 ciphers are available only if Apache Cloudberry was configured with `--enable-openssl-redirect`. Otherwise, they fail with `Cannot use "sm4-ecb": No such cipher algorithm`. The `sm3` digest is available if the OpenSSL library provides it. + +## FIPS mode + +The module defines the configuration parameter `pgcrypto.fips`, which restricts `pgcrypto` to a limited set of functionality that is suitable for FIPS 140-2 mode. The parameter is a Boolean, its default value is `off`, and only superusers can change it. + +FIPS mode requires an OpenSSL library that provides the FIPS module, on all hosts of the cluster. If the OpenSSL version that Apache Cloudberry was compiled against has no FIPS support, turning the parameter on fails with the error `FIPS enabled OpenSSL is required for strict FIPS mode`. A build without OpenSSL does not support FIPS mode. + +When `pgcrypto.fips` is `on`: + +- FIPS mode is turned on in the OpenSSL library. +- `crypt()` and `gen_salt()` are not allowed. +- The raw encryption functions (`encrypt()`, `decrypt()`, `encrypt_iv()`, and `decrypt_iv()`) are not allowed. +- `digest()` and `hmac()` reject digests that are not FIPS-approved, such as `md5`, with an error that starts with `Cannot use "md5"`. +- The PGP functions are not usable either. They recognize only the `3des`, `aes128` (`aes`), `aes192`, and `aes256` cipher names and the `sha1`, `sha256`, `sha384`, and `sha512` digest names, and reject other names, such as `bf`, with the error `Unsupported cipher algorithm`. Encryption and decryption with the recognized ciphers then fail with `requested functionality not allowed in FIPS mode`, because `pgcrypto` reaches these ciphers through an OpenSSL interface that is not permitted in FIPS mode. + +A call to `crypt()`, `gen_salt()`, or a raw encryption function fails with the error `requested functionality not allowed in FIPS mode`. + +### Enable FIPS mode + +1. Add the `pgcrypto` library to `shared_preload_libraries`, and restart the cluster. This makes the `pgcrypto.fips` parameter known to the server before any `pgcrypto` function is called. + + ```shell + gpconfig -c shared_preload_libraries -v 'pgcrypto.so' + gpstop -ar + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +2. Set the parameter for each database that must use FIPS mode. For example, for the database `testdb`: + + ```sql + ALTER DATABASE testdb SET pgcrypto.fips TO on; + ``` + +3. Reconnect to the database, and check the setting: + + ```sql + SHOW pgcrypto.fips; + ``` + +To turn FIPS mode off, run `ALTER DATABASE testdb RESET pgcrypto.fips;`, and reconnect. + +## Module documentation + +See [pgcrypto](https://www.postgresql.org/docs/14/pgcrypto.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/versioned_docs/version-2.x/modules/postgres-fdw.md b/versioned_docs/version-2.x/modules/postgres-fdw.md new file mode 100644 index 0000000000..a28591ecd1 --- /dev/null +++ b/versioned_docs/version-2.x/modules/postgres-fdw.md @@ -0,0 +1,119 @@ +--- +title: postgres_fdw +--- + +# postgres_fdw + +The `postgres_fdw` module is a foreign-data wrapper (FDW) that you can use to access data stored in a remote PostgreSQL or Apache Cloudberry database. + +The Apache Cloudberry `postgres_fdw` module is a modified version of the PostgreSQL `postgres_fdw` module. It supports the MPP execution options of Apache Cloudberry foreign tables, and it has some restrictions that the PostgreSQL module does not have. Both are described below. + +## Install and register the module + +`postgres_fdw` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/postgres_fdw install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION postgres_fdw; +``` + +## Basic usage + +```sql +CREATE SERVER pgserver FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (host 'pghost', port '5432', dbname 'sales'); + +CREATE USER MAPPING FOR CURRENT_USER SERVER pgserver + OPTIONS (user 'remote_user', password 'secret'); + +CREATE FOREIGN TABLE orders_ft (id int, amount numeric) + SERVER pgserver + OPTIONS (schema_name 'public', table_name 'orders'); + +SELECT count(*) FROM orders_ft; +``` + +## Options + +The module accepts the connection options of `libpq` and the options of the PostgreSQL 14 `postgres_fdw` module: `schema_name`, `table_name`, `column_name`, `use_remote_estimate`, `fdw_startup_cost`, `fdw_tuple_cost`, `extensions`, `updatable`, `truncatable`, `fetch_size`, `batch_size`, `async_capable`, `keep_connections`, `password_required`, `sslcert`, and `sslkey`. + +The following options are specific to Apache Cloudberry: + +| Option | Where | Description | +|--------|-------|-------------| +| `mpp_execute` | Foreign table, server, or foreign-data wrapper | Apache Cloudberry option that determines where the foreign table is accessed: `'coordinator'` (the same as `'master'`), `'any'`, or `'all segments'`. | +| `num_segments` | Server | Number of segments that access a foreign table whose `mpp_execute` option is `'all segments'`. If you do not set this option, all segments of the cluster are used. | + +With `mpp_execute 'all segments'`, an `INSERT INTO SELECT ...` statement writes to the remote server from the segments in parallel. + +## Access multiple remote servers from one foreign table + +A foreign table whose `mpp_execute` option is `'all segments'` can read from several remote servers that hold tables of the same structure. Add each server to the foreign table as a foreign segment: + +```sql +CREATE SERVER mpps1 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host 'pghost1', dbname 'fdw1'); +CREATE SERVER mpps2 FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host 'pghost2', dbname 'fdw2'); +CREATE USER MAPPING FOR CURRENT_USER SERVER mpps1; +CREATE USER MAPPING FOR CURRENT_USER SERVER mpps2; + +CREATE FOREIGN TABLE fs1 (a int, b text) + SERVER mpps1 + OPTIONS (schema_name 'public', table_name 't1', mpp_execute 'all segments'); + +ADD FOREIGN SEGMENT FROM SERVER mpps1 INTO fs1; +ADD FOREIGN SEGMENT FROM SERVER mpps2 INTO fs1; +``` + +A query on `fs1` then scans one remote server per query executor process and gathers the results on the coordinator: + +```sql +SET optimizer = off; +EXPLAIN (COSTS OFF) SELECT * FROM fs1; +``` + +```text + QUERY PLAN +------------------------------------------ + Gather Motion 2:1 (slice1; segments: 2) + -> Foreign Scan on fs1 + Optimizer: Postgres query optimizer +(3 rows) +``` + +For such a foreign table: + +- Aggregates can be pushed down to the remote servers as partial aggregates, which are finalized on the coordinator. +- A `LIMIT` clause is applied on each segment as a preliminary limit before the rows are gathered on the coordinator. +- The hidden column `gp_foreign_server` contains the OID of the foreign server that a row comes from. You can use it in join conditions, for example `fs1.gp_foreign_server = fs2.gp_foreign_server`, to join only rows that come from the same server. + +:::caution +Queries on a foreign table with foreign segments return correct results only with the Postgres-based planner. GPORCA ignores the foreign segments: it scans the table on all segments of the cluster, so some remote servers are read more than once and rows are duplicated, and `gp_foreign_server` is `0` for every row. Turn GPORCA off before you query such a table: + +```sql +SET optimizer = off; +``` +::: + +## Limitations + +- The `use_remote_estimate` option is not supported. If the option is present on a server or a foreign table, with any value, the module ignores it and reports the warning `fdw option 'use_remote_estimate' is not supported.` +- When the remote server is an Apache Cloudberry database, only `INSERT` is supported on the foreign table. `UPDATE` and `DELETE` are not supported. + +## Functions + +| Function | Description | +|----------|-------------| +| `postgres_fdw_get_connections(OUT server_name text, OUT valid boolean) returns setof record` | Returns the foreign server names of all open connections that `postgres_fdw` established from the local session, and whether each connection is valid. | +| `postgres_fdw_disconnect(server_name text) returns boolean` | Discards the open connections from the local session to the given foreign server. | +| `postgres_fdw_disconnect_all() returns boolean` | Discards all open connections from the local session to foreign servers. | + +## Module documentation + +See [postgres_fdw](https://www.postgresql.org/docs/14/postgres-fdw.html) in the PostgreSQL documentation for details about the options and behavior of this module. diff --git a/versioned_docs/version-2.x/modules/reject-partition-fullscan.md b/versioned_docs/version-2.x/modules/reject-partition-fullscan.md new file mode 100644 index 0000000000..2dc0ffd482 --- /dev/null +++ b/versioned_docs/version-2.x/modules/reject-partition-fullscan.md @@ -0,0 +1,128 @@ +--- +title: reject_partition_fullscan +--- + +# reject_partition_fullscan + +The `reject_partition_fullscan` module rejects queries that scan all partitions of a partitioned table without effective partition pruning. It installs a planner hook that inspects the finished plan, whether it comes from the Postgres-based planner or from GPORCA, and raises an error if a partitioned table would be scanned in full. + +Use it to prevent expensive queries that omit a filter on the partition key. + +## Build and install + +`reject_partition_fullscan` is one of the debug extensions under `gpcontrib`. These are built when `--enable-debug-extensions` is in effect, which is the default. If Apache Cloudberry is configured with `--disable-debug-extensions`, the module is not built. + +## Enable the module + +The module works by loading its shared library. It has no SQL objects. + +- To enable it for the whole cluster, add the library to `shared_preload_libraries` and restart the cluster: + + ```shell + gpconfig -c shared_preload_libraries -v 'reject_partition_fullscan' + gpstop -ra + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +- To enable it for the current session only, load the library with `LOAD`. This requires superuser privileges: + + ```sql + LOAD 'reject_partition_fullscan'; + ``` + +`CREATE EXTENSION reject_partition_fullscan` is available, but it creates no objects. The planner hook and the configuration parameters are registered when the library is loaded. + +:::note +The check is on by default once the library is loaded, because `reject_partition_fullscan` defaults to `on`. +::: + +## Configuration parameters + +Both parameters can be changed by any role in its session. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `reject_partition_fullscan` | boolean | `on` | Rejects queries on partitioned tables that scan all partitions without pruning. | +| `partition_fullscan_threshold` | integer | `0` | Maximum number of partitions that a query is allowed to scan after pruning. With `0`, a plan of the Postgres-based planner is rejected if more than one partition remains. See [Behavior](#behavior). Range: `0` to `2147483647`. | + +## Behavior + +After a statement is planned, the module walks the plan tree and checks every scan of a partitioned table: + +- With `partition_fullscan_threshold = 0`, a plan of the Postgres-based planner is rejected if more than one partition remains after plan-time pruning. A filter on the partition key that keeps two or more partitions is therefore rejected too. Set a positive threshold to allow such queries. +- With `partition_fullscan_threshold` greater than `0`, the statement is rejected if the number of partitions that remain after pruning is greater than the threshold. + +:::caution +The module works reliably with plans of the Postgres-based planner. GPORCA, which is the default optimizer, attaches run-time pruning information to its dynamic scans, and the module treats such scans as prunable at run time. As a result, `SELECT` statements that GPORCA plans are not rejected, even when they scan all partitions. `UPDATE` and `DELETE` statements on partitioned tables are planned by the Postgres-based planner and are checked. To enforce the check for `SELECT` statements, turn GPORCA off in the session with `SET optimizer = off`. +::: + +A rejected statement fails with an error of this form: + +```text +ERROR: partitioned table ".
" full partition scan is not allowed, partitions would be scanned +HINT: Add a WHERE clause on the partition key to enable partition pruning. +``` + +The check runs on every statement that is planned. It applies to `SELECT`, `UPDATE`, and `DELETE` statements, to partitioned tables referenced in subqueries, and to statements such as `EXPLAIN`, `INSERT ... SELECT`, and `CREATE TABLE AS` that read a partitioned table. + +The following cases are not rejected: + +- `reject_partition_fullscan` is `off`. +- `enable_partition_pruning` is `off`. +- The scan covers at most one partition, for example a partitioned table with a single partition. +- Partitions can be pruned at run time: plans with pruning steps that are evaluated at executor startup or during execution (for example a prepared statement with a parameter on the partition key), and GPORCA dynamic scans that use join-based partition pruning. + +## Example + +```sql +LOAD 'reject_partition_fullscan'; +SET optimizer = off; + +CREATE TABLE pfr_test (id int, dt date, val text) + PARTITION BY RANGE (dt); +CREATE TABLE pfr_test_p1 PARTITION OF pfr_test + FOR VALUES FROM ('2025-01-01') TO ('2025-04-01'); +CREATE TABLE pfr_test_p2 PARTITION OF pfr_test + FOR VALUES FROM ('2025-04-01') TO ('2025-07-01'); +CREATE TABLE pfr_test_p3 PARTITION OF pfr_test + FOR VALUES FROM ('2025-07-01') TO ('2025-10-01'); + +SET reject_partition_fullscan = on; +SET partition_fullscan_threshold = 0; + +-- Rejected: no filter on the partition key +SELECT * FROM pfr_test; +SELECT * FROM pfr_test WHERE val = 'x'; +UPDATE pfr_test SET val = 'y'; + +-- Allowed: the filter prunes the scan to one partition +SELECT * FROM pfr_test WHERE dt = '2025-02-01'; +DELETE FROM pfr_test WHERE dt = '2025-02-01'; + +-- Rejected: two partitions remain after pruning +SELECT * FROM pfr_test + WHERE dt >= '2025-01-01' AND dt < '2025-07-01'; + +-- Threshold mode: allow up to 2 partitions +SET partition_fullscan_threshold = 2; + +-- Allowed: pruned to 2 partitions +SELECT * FROM pfr_test + WHERE dt >= '2025-01-01' AND dt < '2025-07-01'; + +-- Rejected: 3 partitions exceed the threshold +SELECT * FROM pfr_test; +``` + +To allow a full scan for one session, turn the check off: + +```sql +SET reject_partition_fullscan = off; +``` + +## Usage notes + +- Both parameters can be changed by any role, so the module is a guard against accidental full scans, not an enforcement mechanism. +- The check runs at planning time on the plan that the optimizer produced. It does not consider how much data the partitions contain. +- When the library is loaded with `LOAD`, the check applies only to that session. diff --git a/versioned_docs/version-2.x/modules/sslinfo.md b/versioned_docs/version-2.x/modules/sslinfo.md new file mode 100644 index 0000000000..8c0d9119bb --- /dev/null +++ b/versioned_docs/version-2.x/modules/sslinfo.md @@ -0,0 +1,51 @@ +--- +title: sslinfo +--- + +# sslinfo + +The `sslinfo` module provides information about the SSL certificate that the current client provided when it connected to Apache Cloudberry. Most functions in this module return `NULL` if the current connection does not use SSL. + +The Apache Cloudberry `sslinfo` module is equivalent to the PostgreSQL `sslinfo` module. The functions report on the client connection to the coordinator. + +## Install and register the module + +`sslinfo` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry, and it can be built only when Apache Cloudberry is configured with OpenSSL support (`--with-ssl=openssl`, or the older spelling `--with-openssl`). If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/sslinfo install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION sslinfo; +``` + +## Functions + +| Function | Description | +|----------|-------------| +| `ssl_is_used() returns boolean` | Returns `true` if the current connection uses SSL. | +| `ssl_version() returns text` | Returns the name of the protocol used for the SSL connection. | +| `ssl_cipher() returns text` | Returns the name of the cipher used for the SSL connection. | +| `ssl_client_cert_present() returns boolean` | Returns `true` if the client presented a valid SSL certificate. | +| `ssl_client_serial() returns numeric` | Returns the serial number of the client certificate. | +| `ssl_client_dn() returns text` | Returns the full subject of the client certificate. | +| `ssl_issuer_dn() returns text` | Returns the full issuer name of the client certificate. | +| `ssl_client_dn_field(fieldname text) returns text` | Returns one field of the subject of the client certificate. | +| `ssl_issuer_field(fieldname text) returns text` | Returns one field of the issuer of the client certificate. | +| `ssl_extension_info() returns setof record` | Returns the name, value, and criticality of each extension of the client certificate. | + +## Example + +```sql +SELECT ssl_is_used(), ssl_version(), ssl_cipher(); +SELECT ssl_client_dn_field('CN'); +``` + +## Module documentation + +See [sslinfo](https://www.postgresql.org/docs/14/sslinfo.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/versioned_docs/version-2.x/modules/tablefunc.md b/versioned_docs/version-2.x/modules/tablefunc.md new file mode 100644 index 0000000000..f237496691 --- /dev/null +++ b/versioned_docs/version-2.x/modules/tablefunc.md @@ -0,0 +1,21 @@ +--- +title: tablefunc +--- + +# tablefunc + +The `tablefunc` module provides various functions that return tables (that is, multiple rows), such as `normal_rand()`, `crosstab()`, and `connectby()`. + +The Apache Cloudberry `tablefunc` module is equivalent to the PostgreSQL `tablefunc` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`tablefunc` is built and installed by a default Apache Cloudberry build. Register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION tablefunc; +``` + +## Module documentation + +See [tablefunc](https://www.postgresql.org/docs/14/tablefunc.html) in the PostgreSQL documentation for details about the individual functions in this module. diff --git a/versioned_docs/version-2.x/modules/tsm-system-rows.md b/versioned_docs/version-2.x/modules/tsm-system-rows.md new file mode 100644 index 0000000000..bad2f45dd7 --- /dev/null +++ b/versioned_docs/version-2.x/modules/tsm-system-rows.md @@ -0,0 +1,47 @@ +--- +title: tsm_system_rows +--- + +# tsm_system_rows + +The `tsm_system_rows` module implements the `SYSTEM_ROWS` table sampling method, which you can use in the `TABLESAMPLE` clause of a `SELECT` command. + +## Install and register the module + +`tsm_system_rows` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/tsm_system_rows install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION tsm_system_rows; +``` + +## Use the module + +The `SYSTEM_ROWS` sampling method accepts a single integer (`bigint`) argument that is the maximum number of rows to read. If the table does not contain that many rows, the whole table is selected. A negative argument raises an error. + +Like the built-in `SYSTEM` sampling method, `SYSTEM_ROWS` performs block-level sampling, so the sample is not completely random and might be subject to clustering effects, especially if only a small number of rows are requested. + +`SYSTEM_ROWS` does not support the `REPEATABLE` clause. + +:::note +In Apache Cloudberry, each segment samples its own part of a distributed table, and the limit applies to each segment separately. A query on a distributed table can therefore return up to the given number of rows from each segment. For example, `SYSTEM_ROWS(75)` on a cluster with three primary segments returns up to 225 rows. +::: + +## Example + +The following command returns a sample of at most 75 rows from each segment for the table `tablea`: + +```sql +SELECT * FROM tablea TABLESAMPLE SYSTEM_ROWS(75); +``` + +## Module documentation + +See [tsm_system_rows](https://www.postgresql.org/docs/14/tsm-system-rows.html) in the PostgreSQL documentation. diff --git a/versioned_docs/version-2.x/modules/tsm-system-time.md b/versioned_docs/version-2.x/modules/tsm-system-time.md new file mode 100644 index 0000000000..493541904c --- /dev/null +++ b/versioned_docs/version-2.x/modules/tsm-system-time.md @@ -0,0 +1,47 @@ +--- +title: tsm_system_time +--- + +# tsm_system_time + +The `tsm_system_time` module implements the `SYSTEM_TIME` table sampling method, which you can use in the `TABLESAMPLE` clause of a `SELECT` command. + +## Install and register the module + +`tsm_system_time` is not part of the default build targets (`make` and `make install`) of Apache Cloudberry. If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/tsm_system_time install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it: + +```sql +CREATE EXTENSION tsm_system_time; +``` + +## Use the module + +The `SYSTEM_TIME` sampling method accepts a single floating-point argument that is the maximum number of milliseconds to spend reading the table. This lets you control how long the query takes; the tradeoff is that the size of the sample becomes hard to predict. The sample contains as many rows as could be read in the specified time, unless the whole table has been read first. A negative argument raises an error. + +Like the built-in `SYSTEM` sampling method, `SYSTEM_TIME` performs block-level sampling, so the sample is not completely random and might be subject to clustering effects, especially if only a small number of rows are selected. + +`SYSTEM_TIME` does not support the `REPEATABLE` clause. + +:::note +In Apache Cloudberry, each segment samples its own part of a distributed table, and the time limit applies to each segment separately. +::: + +## Example + +The following command returns as large a sample of `tablea` as each segment can read in 500 milliseconds. If the entire table can be read in less time, all of its rows are returned. + +```sql +SELECT * FROM tablea TABLESAMPLE SYSTEM_TIME(500); +``` + +## Module documentation + +See [tsm_system_time](https://www.postgresql.org/docs/14/tsm-system-time.html) in the PostgreSQL documentation. diff --git a/versioned_docs/version-2.x/modules/uuid-ossp.md b/versioned_docs/version-2.x/modules/uuid-ossp.md new file mode 100644 index 0000000000..e6da69d011 --- /dev/null +++ b/versioned_docs/version-2.x/modules/uuid-ossp.md @@ -0,0 +1,45 @@ +--- +title: uuid-ossp +--- + +# uuid-ossp + +The `uuid-ossp` module provides functions that generate universally unique identifiers (UUIDs) using one of several standard algorithms. The module also includes functions that produce certain special UUID constants. + +The Apache Cloudberry `uuid-ossp` module is equivalent to the PostgreSQL `uuid-ossp` module. There are no MPP-specific considerations for the module. + +## Install and register the module + +`uuid-ossp` is built only when Apache Cloudberry is configured with a UUID library, using the `--with-uuid=LIB` option of `configure` (`LIB` is `e2fs`, `bsd`, or `ossp`). It is also not part of the default build targets (`make` and `make install`). If `CREATE EXTENSION` reports that the extension is not available, build and install the module from the Apache Cloudberry source tree that your installation was configured from: + +```shell +make -C contrib/uuid-ossp install +``` + +The module files must be present on every host of the cluster (coordinator, standby coordinator, and all segment hosts). The module is also installed by `make install-world`. + +Then register the extension in each database in which you want to use it. Because the extension name contains a hyphen, you must quote it: + +```sql +CREATE EXTENSION "uuid-ossp"; +``` + +## Functions + +| Function | Description | +|----------|-------------| +| `uuid_generate_v1()` | Generates a version 1 UUID from the MAC address of the host and a timestamp. | +| `uuid_generate_v1mc()` | Generates a version 1 UUID using a random multicast MAC address. | +| `uuid_generate_v3(namespace uuid, name text)` | Generates a version 3 UUID (MD5 hash) in the given namespace. | +| `uuid_generate_v4()` | Generates a version 4 UUID from random numbers. | +| `uuid_generate_v5(namespace uuid, name text)` | Generates a version 5 UUID (SHA-1 hash) in the given namespace. | +| `uuid_nil()` | Returns the nil UUID constant. | +| `uuid_ns_dns()`, `uuid_ns_url()`, `uuid_ns_oid()`, `uuid_ns_x500()` | Return the constants that designate the DNS, URL, ISO object identifier, and X.500 distinguished name namespaces. | + +:::note +If you only need randomly generated (version 4) UUIDs, you can use the `gen_random_uuid()` function of the [pgcrypto](./pgcrypto.md) module instead. +::: + +## Module documentation + +See [uuid-ossp](https://www.postgresql.org/docs/14/uuid-ossp.html) in the PostgreSQL documentation for details about this module. diff --git a/versioned_docs/version-2.x/modules/yezzey.md b/versioned_docs/version-2.x/modules/yezzey.md new file mode 100644 index 0000000000..326d6548c0 --- /dev/null +++ b/versioned_docs/version-2.x/modules/yezzey.md @@ -0,0 +1,275 @@ +--- +title: yezzey +--- + +# yezzey + +The `yezzey` module offloads the data of append-optimized tables (row-oriented and column-oriented) to S3-compatible object storage. An offloaded table keeps its definition and stays in the database. You continue to query and modify it with the same SQL statements, and Apache Cloudberry reads and writes its data files in the object storage instead of on the local disks of the segments. + +Use `yezzey` to move cold data to cheaper storage and to reduce the disk space that a cluster needs. + +:::note +`yezzey` is developed in the [open-gpdb/yezzey](https://github.com/open-gpdb/yezzey) project. The Apache Cloudberry source tree references it as a Git submodule in `gpcontrib/yezzey`. Apache Cloudberry 2.2 pins `yezzey` version 2.0. +::: + +## How it works + +`yezzey` consists of the following parts: + +- A storage manager for append-optimized tables. It is installed as a hook when the `yezzey` library is loaded, and it redirects the file operations of offloaded tables to the object storage. +- The `yezzey` extension, which provides the functions and the metadata tables that manage offloading. +- [YProxy](https://github.com/open-gpdb/yproxy), a separate service that runs on the cluster hosts. The segments do not connect to the object storage directly. They send all requests to YProxy through a Unix domain socket, and YProxy keeps a pool of connections to the object storage and schedules the requests. + +When a table is offloaded: + +1. `yezzey` takes an exclusive lock on the table. +2. The data files of the table are uploaded to the object storage on all segments. +3. The tablespace of the table changes to the `yezzey` virtual tablespace (OID `8555`). +4. The transaction commits. + +After offloading, new data that you insert into the table is written to the object storage. Files in the object storage are never modified in place. Each write creates new files, and `yezzey` records them in a metadata table, the virtual index, that follows the MVCC rules of the database. Files that are no longer referenced become garbage, which you remove with the cleanup functions described in [Remove obsolete files](#remove-obsolete-files). + +## Build and install + +`yezzey` is not built by default. + +1. Fetch the submodule in the Apache Cloudberry source tree: + + ```shell + git submodule update --init gpcontrib/yezzey + ``` + +2. Configure Apache Cloudberry with the `--with-yezzey` option, then build and install. `--with-yezzey` requires OpenSSL support (`--with-ssl=openssl`) and a C++11 compiler: + + ```shell + ./configure --with-yezzey --with-ssl=openssl [other options...] + make + make install + ``` + + To build only this module against an existing build tree: + + ```shell + make -C gpcontrib/yezzey + make -C gpcontrib/yezzey install + ``` + +3. Build and install YProxy on every host of the cluster. See the [YProxy](https://github.com/open-gpdb/yproxy) project for instructions. + +## Set up + +### Configure and start YProxy + +YProxy reads a YAML configuration file that defines its socket and the object storage. The following example is based on the configuration that the `yezzey` tests use: + +```yaml +socket_path: "/tmp/yproxy.sock" +interconnect_socket_path: "/tmp/ic.sock" +log_level: info + +storage: + access_key_id: "" + secret_access_key: "" + storage_endpoint: "" + storage_prefix: "" + storage_bucket: "" + storage_region: "" + storage_type: "s3" + tablespace_map: + "pg_default": "" + +backup_storage: + access_key_id: "" + secret_access_key: "" + storage_endpoint: "" + storage_prefix: "" + storage_bucket: "" + storage_region: "" + storage_type: "s3" +``` + +`tablespace_map` maps the tablespace that a table was in before offloading to the bucket that receives its data. Add an entry for every tablespace that contains tables you want to offload. + +Start YProxy on every host, and make sure that the operating system user that runs Apache Cloudberry can access the socket: + +```shell +yproxy -c /path/to/yproxy.yaml +``` + +For the complete list of YProxy settings, see the YProxy documentation. + +### Enable the module + +1. Add `yezzey` to `shared_preload_libraries` and restart the cluster. The library refuses to load in any other way: + + ```shell + gpconfig -c shared_preload_libraries -v yezzey + gpstop -ar + ``` + + If `shared_preload_libraries` already lists other libraries, keep them in the value. + +2. Set the configuration parameters that your environment needs, and reload the configuration. At least check `yezzey.yproxy_socket` and `yezzey.use_gpg_crypto`: + + ```shell + gpconfig -c yezzey.yproxy_socket -v "/tmp/yproxy.sock" + gpstop -u + ``` + + `yezzey.use_gpg_crypto` is `on` by default, which requires a GPG key in the `crypto` section of the YProxy configuration (`gpg_key_id` and `gpg_key_path`). Without a key, `yezzey.offload_relation()` fails with `failed to offload relation`. If you do not encrypt the data, turn the parameter off: + + ```shell + gpconfig -c yezzey.use_gpg_crypto -v false + gpstop -u + ``` + +3. Create the extension in each database in which you want to offload tables: + + ```sql + CREATE EXTENSION yezzey; + ``` + + The extension creates the `yezzey` schema and grants `USAGE` on it to `PUBLIC`. The extension is marked as trusted, so a non-superuser who has the `CREATE` privilege on the database can create it. + + :::note + The functions of the extension can be run by `PUBLIC`, and `yezzey.offload_relation()` and `yezzey.load_relation()` do not check the ownership of the table. Revoke `EXECUTE` on the functions from `PUBLIC` if only administrators should offload, load, or clean up tables. + ::: + +## Configuration parameters + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `yezzey.yproxy_socket` | string | `/tmp/yproxy.sock` | Path of the Unix domain socket of YProxy. | +| `yezzey.use_gpg_crypto` | boolean | `on` | Asks YProxy to encrypt the data with GPG when it is written to the object storage. The key is configured in the `crypto` section of the YProxy configuration. | +| `yezzey.storage_class` | string | `STANDARD` | Default storage class for the objects that `yezzey` writes. | +| `yezzey.multipart_chunksize` | integer | `16777216` | Default chunk size of a multipart upload, in bytes. | +| `yezzey.multipart_threshold` | integer | `67108864` | Default size threshold for multipart uploads, in bytes. | +| `yezzey.use_otm_feature` | boolean | `off` | Enables the offload tablespace map. When it is `off`, only tables in the default tablespace can be offloaded. When it is `on`, tables in other tablespaces can be offloaded too, and `yezzey` records the original tablespace of each table in `yezzey.offload_tablespace_map`. This parameter can be set only at connection start or in the server configuration. | +| `yezzey.log_level` | enum | `debug1` | Log level of the messages of the `yezzey` functions. | +| `yezzey.ao_log_level` | enum | `debug1` | Log level of the messages about the read and write operations on offloaded files. | +| `yezzey.autooffload` | boolean | `off` | Reserved. The parameter is defined, but it is not used in this version. | + +Except for `yezzey.use_otm_feature` and `yezzey.autooffload`, the parameters can be changed by superusers only. + +## Offload a table + +`yezzey` works with append-optimized tables only, both row-oriented and column-oriented. + +To offload an existing table, call `yezzey.offload_relation()`: + +```sql +CREATE TABLE sales_archive (id int, amount numeric) + WITH (appendonly=true) DISTRIBUTED BY (id); +INSERT INTO sales_archive SELECT i, i FROM generate_series(1, 100000) i; + +SELECT yezzey.offload_relation('sales_archive'); +``` + +The function reports its progress with `NOTICE` messages from each segment. The time that it takes depends on the size of the table. If the table is already offloaded, the function returns with the warning `The relation has already been offloaded`. + +You can also create a table that is offloaded from the start, by placing it in the `yezzey` virtual tablespace: + +```sql +CREATE TABLE regao_tt_alr_off (i int) + WITH (appendonly=true) + TABLESPACE yezzey + DISTRIBUTED BY (i); + +INSERT INTO regao_tt_alr_off SELECT generate_series(1,10010); + +SELECT segindex, local_bytes, external_bytes +FROM yezzey.offload_relation_status('regao_tt_alr_off'); +``` + +```text + segindex | local_bytes | external_bytes +----------+-------------+---------------- + 0 | 0 | 33816 + 1 | 0 | 33928 + 2 | 0 | 32560 +(3 rows) +``` + +After a table is offloaded, you can continue to run `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `VACUUM`, and `ALTER TABLE ... ADD COLUMN` on it. + +To offload a partitioned table, offload each leaf partition. Calling `yezzey.offload_relation()` on the partitioned table itself fails. + +## Load a table back to local storage + +To move the data of an offloaded table back to the local disks of the segments, call `yezzey.load_relation()`: + +```sql +SELECT yezzey.load_relation('sales_archive'); +``` + +The time that the download takes depends on the size of the table and the number of its segment files. The files that were uploaded earlier stay in the object storage, so `external_bytes` in `yezzey.offload_relation_status()` is not reset to `0`. + +## Check the status of a table + +`yezzey.offload_relation_status()` returns one row per segment: + +```sql +SELECT * FROM yezzey.offload_relation_status('sales_archive'); +``` + +| Column | Description | +|---|---| +| `reloid` | OID of the table. | +| `segindex` | Segment content ID. | +| `local_bytes` | Size of the data that is stored on the local disk. `0` means that the table is fully offloaded. | +| `local_commited_bytes` | Size of the committed data on the local disk. | +| `external_bytes` | Size of the data in the object storage. `0` on all segments means that the table has never been offloaded. | +| `external_bloat_bytes` | Size of the data in the object storage that is no longer in use but is not deleted yet. | + +For more detail, use `yezzey.offload_relation_status_per_filesegment()`, which returns one row per segment file, or `yezzey.relation_describe_external_storage_structure()`, which returns one row per file in the object storage, including its path. + +The `yezzey.offload_metadata` table lists the tables that were offloaded with `yezzey.offload_relation()`. Tables that were created directly in the `yezzey` tablespace are not listed. `SELECT` on this table is granted to `PUBLIC`. + +## Remove obsolete files + +Data in the object storage is not deleted when rows are deleted or when a table is dropped, because the files might still be needed to roll back a transaction or to restore a backup. The following functions send delete requests for obsolete files to YProxy. They run on all segments. + +| Function | Scope | +|---|---| +| `yezzey.vacuum(confirm boolean DEFAULT false, crazyDrop boolean DEFAULT false)` | All tablespaces. | +| `yezzey.vacuum_tablespace(tablespace oid, confirm boolean DEFAULT false, crazyDrop boolean DEFAULT false)` | One tablespace. | +| `yezzey.vacuum_relation(relation regclass, confirm boolean DEFAULT false, crazyDrop boolean DEFAULT false)` | One offloaded table. | + +- `confirm` controls whether YProxy actually deletes the objects. With the default `false`, the call is a dry run. Set it to `true` to confirm the deletion. +- `crazyDrop` enables a more aggressive cleanup mode. Only superusers can set it to `true`. + +A typical cleanup looks like this: + +1. Run `VACUUM` on the append-optimized table, so that its obsolete files become garbage. +2. Inspect the storage of the table with `yezzey.relation_describe_external_storage_structure('
')`. +3. Call `yezzey.vacuum_relation()`, `yezzey.vacuum_tablespace()`, or `yezzey.vacuum()`, depending on the scope that you want to clean. +4. Check the storage of the table again to verify that the obsolete files are removed. + +By default, YProxy keeps the files that a backup might still need. See the `vacuum` section of the YProxy configuration for the settings that control this behavior. + +## Function reference + +All functions are in the `yezzey` schema. + +| Function | Returns | Description | +|---|---|---| +| `offload_relation(relation regclass)` | `void` | Offloads an append-optimized table to the object storage. | +| `load_relation(relation regclass)` | `void` | Loads an offloaded table back to local storage. | +| `offload_relation_status(relation regclass)` | set of rows | Local and external size of a table, per segment. | +| `offload_relation_status_per_filesegment(relation regclass)` | set of rows | Local and external size of a table, per segment file. | +| `relation_describe_external_storage_structure(relation regclass)` | set of rows | The files of a table in the object storage, with their paths and sizes. | +| `vacuum(confirm boolean, crazyDrop boolean)` | set of `boolean` | Removes obsolete files in all tablespaces. | +| `vacuum_tablespace(tablespace oid, confirm boolean, crazyDrop boolean)` | set of `boolean` | Removes obsolete files in one tablespace. | +| `vacuum_relation(relation regclass, confirm boolean, crazyDrop boolean)` | set of `boolean` | Removes obsolete files of one offloaded table. | + +The extension also creates internal functions and metadata tables, such as `yezzey.yezzey_virtual_index`, `yezzey.yezzey_expire_index`, and `yezzey.offload_tablespace_map`. Do not call the internal functions or modify the metadata tables directly. + +## Usage notes + +- Only append-optimized tables can be offloaded. +- To offload a table that is not in the default tablespace, turn on `yezzey.use_otm_feature`. Otherwise the offload is rejected. +- You cannot move an offloaded table to another tablespace with `ALTER TABLE ... SET TABLESPACE`. The statement fails. Use `yezzey.load_relation()` to bring the data back to local storage. +- YProxy must be running on every host before you access offloaded tables. If YProxy or the object storage is unavailable, `yezzey` retries the read requests up to 100 times. +- Offloading takes an exclusive lock on the table for the duration of the upload. +- When you restore a cluster from a backup, the object storage must contain the files that the backup refers to. For this reason, obsolete files are kept until no backup needs them. +- For a standby cluster that receives WAL from the primary cluster, the object storage must be accessible from the standby cluster as well. diff --git a/versioned_sidebars/version-2.x-sidebars.json b/versioned_sidebars/version-2.x-sidebars.json index 0d50115279..a9e0821ddd 100644 --- a/versioned_sidebars/version-2.x-sidebars.json +++ b/versioned_sidebars/version-2.x-sidebars.json @@ -599,6 +599,45 @@ }, "data-types", "config-params-guc-list", + { + "type": "category", + "label": "Extension Modules", + "link": { + "type": "doc", + "id": "modules/index" + }, + "items": [ + "modules/auto-explain", + "modules/btree-gin", + "modules/citext", + "modules/dblink", + "modules/diskquota", + "modules/fuzzystrmatch", + "modules/gp-distribution-policy", + "modules/gp-exttable-fdw", + "modules/gp-internal-tools", + "modules/gp-legacy-string-agg", + "modules/gp-relsizes-stats", + "modules/gp-sparse-vector", + "modules/gp-stats-collector", + "modules/hstore", + "modules/isn", + "modules/ltree", + "modules/orafce", + "modules/pageinspect", + "modules/pg-buffercache", + "modules/pg-trgm", + "modules/pgcrypto", + "modules/postgres-fdw", + "modules/reject-partition-fullscan", + "modules/sslinfo", + "modules/tablefunc", + "modules/tsm-system-rows", + "modules/tsm-system-time", + "modules/uuid-ossp", + "modules/yezzey" + ] + }, { "type": "category", "label": "Built-in Functions",