diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05d3a5e..6c8bc82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,10 @@ jobs: crate=$(grep -m1 '^version' Cargo.toml | cut -d'"' -f2) binding=$(grep -m1 '^version' datahub_python_bindings/Cargo.toml | cut -d'"' -f2) py=$(grep -m1 '^version' datahub_python_bindings/pyproject.toml | cut -d'"' -f2) - echo "crate=$crate binding=$binding pyproject=$py" - if [ "$crate" != "$binding" ] || [ "$crate" != "$py" ]; then - echo "::error::manifest versions disagree: $crate / $binding / $py" + c=$(grep -m1 '^version' datahub_c_bindings/Cargo.toml | cut -d'"' -f2) + echo "crate=$crate binding=$binding pyproject=$py c=$c" + if [ "$crate" != "$binding" ] || [ "$crate" != "$py" ] || [ "$crate" != "$c" ]; then + echo "::error::manifest versions disagree: $crate / $binding / $py / $c" exit 1 fi @@ -42,3 +43,17 @@ jobs: name: Wheels needs: versions uses: ./.github/workflows/build-wheels.yml + + c-sdk: + name: C bindings build, tests and header + runs-on: ubuntu-latest + needs: versions + steps: + - uses: actions/checkout@v6 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: datahub_c_bindings + # Builds the crate (which regenerates include/intellistream_datahub.h), runs its Rust + # tests, fails if the committed header is stale, then compiles tests/c/smoke.c with the + # system C compiler against the built library and runs it. No backend is involved. + - run: ./run_c_tests.sh --check-header diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 58d10eb..c8adf59 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,7 @@ on: - Cargo.toml - datahub_python_bindings/Cargo.toml - datahub_python_bindings/pyproject.toml + - datahub_c_bindings/Cargo.toml - .github/workflows/release.yml - .github/workflows/build-wheels.yml @@ -27,9 +28,10 @@ jobs: crate=$(grep -m1 '^version' Cargo.toml | cut -d'"' -f2) binding=$(grep -m1 '^version' datahub_python_bindings/Cargo.toml | cut -d'"' -f2) py=$(grep -m1 '^version' datahub_python_bindings/pyproject.toml | cut -d'"' -f2) - echo "crate=$crate binding=$binding pyproject=$py ref=${GITHUB_REF_NAME:-}" - if [ "$crate" != "$binding" ] || [ "$crate" != "$py" ]; then - echo "::error::manifest versions disagree: $crate / $binding / $py" + c=$(grep -m1 '^version' datahub_c_bindings/Cargo.toml | cut -d'"' -f2) + echo "crate=$crate binding=$binding pyproject=$py c=$c ref=${GITHUB_REF_NAME:-}" + if [ "$crate" != "$binding" ] || [ "$crate" != "$py" ] || [ "$crate" != "$c" ]; then + echo "::error::manifest versions disagree: $crate / $binding / $py / $c" exit 1 fi # Only a tag push carries a version to check against; a rehearsal has none. @@ -61,6 +63,37 @@ jobs: needs: versions uses: ./.github/workflows/build-wheels.yml + c-sdk: + # The C library is not published to a registry: each OS build is kept as a workflow + # artifact (header + shared + static library) for attaching to the release by hand. It does + # not gate the crate or wheel publication. + name: C bindings (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + needs: versions + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v6 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: datahub_c_bindings + - run: cargo build --release + working-directory: datahub_c_bindings + - uses: actions/upload-artifact@v6 + with: + name: c-sdk-${{ matrix.os }} + if-no-files-found: error + path: | + datahub_c_bindings/include/intellistream_datahub.h + datahub_c_bindings/target/release/libintellistream_datahub.so + datahub_c_bindings/target/release/libintellistream_datahub.dylib + datahub_c_bindings/target/release/libintellistream_datahub.a + datahub_c_bindings/target/release/intellistream_datahub.dll + datahub_c_bindings/target/release/intellistream_datahub.dll.lib + datahub_c_bindings/target/release/intellistream_datahub.lib + pypi: name: Publish to PyPI runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 828f237..cd71032 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ Cargo.lock .env CLAUDE.local.md /datahub_python_bindings/target +/datahub_c_bindings/target /datahub_python_bindings/python/intellistream_datahub_sdk/_core.abi3.so *.py[cod] diff --git a/AGENTS.md b/AGENTS.md index 7b74ef2..9959d33 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,7 @@ cargo test -- --ignored # run tests marked #[ignore] (e.g. long cargo test ::tests:: # e.g. `events::tests::test_events_full` cargo test -- --nocapture # show println! from tests (the SDK prints response bodies) ./run_python_tests.sh # Python-bindings suite (rebuilds the PyO3 module first — see below) +./run_c_tests.sh # C-bindings suite: cargo test + the C smoke test, no backend needed (see below) ``` Most tests are integration tests that call a live backend via `create_api_service()`. They read configuration from a local `.env` file (gitignored). Required: @@ -108,6 +109,8 @@ Synchronous mirror of the async API behind the `blocking` cargo feature — the ### Durable ingest buffering (`src/buffer.rs`, integration tests in `src/buffer_integration.rs`) +`TimeSeriesService::flush_buffer` / `EventsService::flush_buffer` drain a spool without ingesting anything new (for a controlled shutdown, or a host retrying on its own clock); `buffered_count` reports what is held. Retention is measured on each record's own timestamp, so a backfill older than the window is not kept. + When a datapoint/event send can't get through, ingestion spools to a segmented, zstd-compressed NDJSON log on disk and flushes automatically on a later ingest call. Invariants to preserve: memory use is bounded by a single segment (plain append-only active segment, zstd-sealed at ~50 MiB rollover via temp file + atomic rename, drained oldest-first one segment at a time); bounded by time retention (whole segments past the window dropped, expired records skipped on read) and a size cap (oldest segment deleted); a torn trailing line from an unclean shutdown is skipped on read. Each on-disk line is `\t`; the spool is content-agnostic. ### The `ApiServiceProvider` trait (`src/generic.rs`) @@ -372,10 +375,43 @@ behind it — which is exactly how the label table filled up. Mint a unique name owns the definition's lifecycle (create/rename/delete), where a shared row would be pulled out from under another test, and use a fixed *pair* when a test has to tell two labels apart. +## C bindings (`datahub_c_bindings/`) + +A thin FFI crate that builds the SDK as `libintellistream_datahub` (cdylib + staticlib) with a +cbindgen-generated header at `datahub_c_bindings/include/intellistream_datahub.h`. `docs/c-sdk-design.md` +records why it exists and what is deliberately not in it. Things to know when touching it: + +- **Every export is written out by hand.** cbindgen does not expand `macro_rules!`, so a + macro-generated `extern "C"` function ends up in the library but not in the header — and a C + caller cannot see it. Share bodies through private helper functions instead. +- **The header is generated by `build.rs` and committed.** `cargo build` in the crate rewrites it; + CI runs `./run_c_tests.sh --check-header`, which fails on a diff. Commit the regenerated header + with the change that caused it. The version macros and `DATAHUB_TIME_UNSET` are added by + `build.rs`, not by cbindgen. +- **Every export runs inside `error::guard`** (`catch_unwind`): a panic becomes `DATAHUB_PANIC` + with the message in the thread-local `datahub_last_error()`. Never let a panic reach C. +- **Runtime model:** a `datahub_client` owns a Tokio runtime and `block_on`s the async + `ApiService` directly — not the blocking client, which has no subscriptions. A listener shares + the runtime (`Arc`), so client and listener may be freed in either order. +- **Typed on the datapoint hot path, JSON everywhere else.** A `..._json` function takes exactly + the REST request body and returns exactly the response body (`items` plus `nextCursor`, which + `DataWrapper` itself skips when serializing); `datahub_request_json` is the raw authenticated + escape hatch for any endpoint without a dedicated function. +- **Core switches it depends on:** `http::set_debug_output(false)` (the core's stdout/stderr + tracing, on by default for Rust and Python users), `DataHubConfig::from_map` (one config path + for env, env file and typed setters; the C layer never reads `.env` from the host's cwd), and + `TimeSeriesService::flush_buffer` / `EventsService::flush_buffer` behind `datahub_client_flush`. +- **Tests:** unit tests in the crate; `tests/offline.rs` (boundary and spool paths, no network); + `tests/mock_api.rs` (a tiny HTTP mock asserting what goes on the wire); `tests/live.rs` (skips + without a `BASE_URL`); `tests/c/smoke.c`, compiled and run by `run_c_tests.sh`. The spool's + retention window is measured on each record's own timestamp, so a test that expects a record + to survive in the spool must stamp it with a recent time. +- The crate's version is locked to the other three manifests by the CI `versions` job. + ## Conventions - `#[serde(rename = "camelCase")]` or explicit `#[serde(rename = "...")]` on fields — the backend is camelCase, Rust is snake_case. - **A request body naming a field the api does not have is a 400.** Jackson used to drop unknown properties, so a stale or misspelled key was answered with 200 and no effect; a strict converter now rejects the body and names every offender alongside the fields the endpoint accepts. Two consequences for this SDK: a struct that doubles as request *and* response must `#[serde(skip_serializing)]` its response-only fields — `GraphDataWrapper`'s `errorBody`/`httpStatusCode` reached `/resources/create` and made every resource and function create and update a 400 — and one Rust type may not stand in for two endpoints that disagree on their fields (see the search forms above). Reading is unaffected: responses stay lenient in both directions. - `externalId` (string, user-supplied) and numeric `id` are both valid identifiers across the API. `IdAndExtId` / `IdAndExtIdCollection` model this choice. -- `process_response` (`src/http.rs`) prints response bodies to stdout (truncated to 2000 chars). This is deliberate for debugging — don't silently remove it. +- `process_response` (`src/http.rs`) prints response bodies to stdout (truncated to 2000 chars). This is deliberate for debugging — don't silently remove it. It and the other request-path prints go through the `debug_println!`/`debug_eprintln!` macros, gated by `http::set_debug_output` (on by default; the C bindings turn it off, since a library must not write to streams it does not own). - Tests that depend on backend state being empty are brittle; recent fixes moved away from exact-count assertions (see commit `7f0a059`). Don't add new ones. diff --git a/Cargo.toml b/Cargo.toml index 34e6569..b42efcc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,8 @@ exclude = [ "/datahub_python_bindings", "/python_tests", "/run_python_tests.sh", + "/datahub_c_bindings", + "/run_c_tests.sh", "/resources", ] diff --git a/README.md b/README.md index 1fe7073..f746c60 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,20 @@ the **compiled** module — always use the wrapper script, which rebuilds the bi ./run_python_tests.sh -k timeseries # extra args are forwarded to pytest ``` +## C bindings + +`datahub_c_bindings/` builds this SDK as a C library — `libintellistream_datahub` (shared and +static) with a cbindgen-generated header, `include/intellistream_datahub.h` — for C, C++, and +anything with C interop (.NET P/Invoke, Go cgo, LabVIEW, MATLAB). It is ingest-first: typed +datapoint ingest with the same durable buffering as the Rust crate, time series lookup, event +creation, the subscription listener, and a JSON convention for every other endpoint. See +[`datahub_c_bindings/README.md`](datahub_c_bindings/README.md) for usage and +[`docs/c-sdk-design.md`](docs/c-sdk-design.md) for the reasoning. + +```bash +./run_c_tests.sh # build the library, run its tests, compile and run the C smoke test +``` + ## Building and testing ```bash diff --git a/datahub_c_bindings/Cargo.toml b/datahub_c_bindings/Cargo.toml new file mode 100644 index 0000000..9918c89 --- /dev/null +++ b/datahub_c_bindings/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "datahub_c_bindings" +version = "0.3.0" +edition = "2021" +rust-version = "1.85" +description = "C ABI for the IntelliStream DataHub SDK: libintellistream_datahub plus a cbindgen-generated header." +license = "Apache-2.0" +repository = "https://github.com/IntelliStream-DataHub/dataplatform-rust-sdk" +publish = false + +[lib] +name = "intellistream_datahub" +# cdylib + staticlib are what C links against; rlib is what the crate's own Rust tests link against. +crate-type = ["cdylib", "staticlib", "rlib"] + +[dependencies] +intellistream-datahub-sdk = { path = ".." } +tokio = { version = "1", features = ["rt-multi-thread", "time"] } +serde = "1" +serde_json = "1" +chrono = "0.4" + +[build-dependencies] +cbindgen = "0.29" + +[dev-dependencies] +tempfile = "3" diff --git a/datahub_c_bindings/README.md b/datahub_c_bindings/README.md new file mode 100644 index 0000000..3e4f788 --- /dev/null +++ b/datahub_c_bindings/README.md @@ -0,0 +1,263 @@ +# IntelliStream DataHub — C bindings + +`libintellistream_datahub` is the [DataHub Rust SDK](../README.md) built as a C library: a shared +and a static library plus one header, [`include/intellistream_datahub.h`](include/intellistream_datahub.h). +It is for C and C++ directly, and for anything with C interop — .NET P/Invoke, Go cgo, LabVIEW's +Call Library Function Node, MATLAB's `loadlibrary`, Swift, Zig. + +There is exactly one implementation of every call: the Rust core. This crate adds no HTTP, auth, +buffering or WebSocket code of its own, so token refresh across every supported OAuth2 flow, the +durable on-disk spool, and the reconnecting subscription listener all behave exactly as they do +for Rust and Python callers. + +It is a **hosted-OS** library — Linux (glibc and musl), Windows, macOS. It needs an operating +system; a microcontroller without one is out of scope. + +## Build + +```bash +cd datahub_c_bindings +cargo build --release +``` + +produces, under `target/release/`: + +| File | What | +|---|---| +| `libintellistream_datahub.so` / `.dylib` / `intellistream_datahub.dll` | shared library | +| `libintellistream_datahub.a` / `intellistream_datahub.lib` | static library | + +and regenerates `include/intellistream_datahub.h` from the sources. The header is committed, so a +download of the repository has one without building anything. + +Link the shared library the usual way: + +```bash +cc gateway.c -I datahub_c_bindings/include -L target/release -lintellistream_datahub -o gateway +``` + +The static library also needs the system libraries the Rust runtime links against — on Linux +`-lpthread -ldl -lm` — and is several megabytes, since it carries reqwest, rustls and Tokio. + +TLS is rustls with the OS trust store (no OpenSSL dependency, so the library does not care which +`libssl` the host process has). On a stripped-down image with no CA bundle, point it at one with +`SSL_CERT_FILE` or `SSL_CERT_DIR`. + +## Conventions + +- **Handles are opaque pointers** — `datahub_config`, `datahub_client`, `datahub_listener`, + `datahub_message`, `datahub_timeseries` — each released with its own `_free` (or `_close`) and + never any other way. +- **Strings are NUL-terminated UTF-8.** Input strings are borrowed for the duration of the call. + A `char **` out-parameter hands you an owned string: release it with `datahub_string_free`. A + `const char *` returned by an accessor is borrowed and valid until the handle it came from is + freed. Never pass a library allocation to `free()`. +- **Every function that can fail returns a `datahub_status`.** Anything other than `DATAHUB_OK` + and `DATAHUB_BUFFERED` leaves a message in `datahub_last_error()` (per thread; overwritten by + the next failure on that thread) and, for HTTP failures, the code in `datahub_last_http_status()`. + `datahub_status_name()` gives the enum's name for logging. +- **A `datahub_client` may be used from any number of threads at once.** A listener and a message + belong to one thread at a time. +- **A Rust panic never reaches you.** It becomes `DATAHUB_PANIC` with the message in + `datahub_last_error()`; please report one, it is a bug in the SDK. + +### Status codes + +| Status | Meaning | +|---|---| +| `DATAHUB_OK` | Success. For an ingest call, the data reached the server. | +| `DATAHUB_BUFFERED` | The data went to the on-disk spool (server unreachable, or credential refused) and will be sent on a later ingest call or `datahub_client_flush`. Not an error. | +| `DATAHUB_TIMEOUT` | `datahub_listener_next`: nothing arrived within the timeout. | +| `DATAHUB_CLOSED` | `datahub_listener_next`: the stream has ended. | +| `DATAHUB_NOT_FOUND` | The api answered with no item where one was asked for. | +| `DATAHUB_INVALID_ARGUMENT` | NULL where a value was required, invalid UTF-8, an empty id, a non-finite value, a JSON body of the wrong shape. Nothing was sent. | +| `DATAHUB_CONFIG` | Incomplete or contradictory configuration (no `BASE_URL`, malformed URL, …). | +| `DATAHUB_AUTH` | A token could not be obtained, or the api answered 401/403. For a 401 the message carries the SDK's diagnosis of the token's `organization` claim when it has one. | +| `DATAHUB_HTTP` | Another non-2xx answer, or no answer at all; see `datahub_last_http_status()`. A transport failure (refused, DNS, timeout) is reported as 503, the same way the core treats it: retryable. | +| `DATAHUB_IO` | A local failure: unreadable env file, runtime could not start, WebSocket lost for good. | +| `DATAHUB_SUBSCRIPTION` | The server rejected one subscription (unknown id, no read access); the connection stays open, call `datahub_listener_next` again. | +| `DATAHUB_PANIC` | A Rust panic was caught at the boundary. | + +## Ingesting datapoints + +```c +#include +#include + +int main(void) { + datahub_config *cfg = datahub_config_new(); + datahub_config_set_base_url(cfg, "https://datahub.example.com"); + datahub_config_set_client_credentials(cfg, "gateway-7", "…secret…", + "https://sso.example.com/realms/datahub/protocol/openid-connect/token"); + datahub_config_set_scope(cfg, "organization:*"); /* Keycloak Organizations realms need it */ + datahub_config_set_buffer_dir(cfg, "/var/lib/gateway/datahub-spool"); /* also enables buffering */ + + datahub_client *client = NULL; + if (datahub_client_new(cfg, &client) != DATAHUB_OK) { + fprintf(stderr, "datahub: %s\n", datahub_last_error()); + datahub_config_free(cfg); + return 1; + } + datahub_config_free(cfg); /* the client copied what it needs */ + + datahub_datapoint points[] = { + { .timestamp_ms = 1789000000000, .value = 21.5 }, + { .timestamp_ms = 1789000001000, .value = 21.6 }, + }; + switch (datahub_datapoints_insert(client, "pump-1/temperature", points, 2)) { + case DATAHUB_OK: break; /* on the server */ + case DATAHUB_BUFFERED: break; /* on disk; sent later */ + default: fprintf(stderr, "datahub: %s\n", datahub_last_error()); /* fix and retry */ + } + + datahub_client_flush(client); /* push any backlog before a controlled shutdown */ + datahub_client_free(client); + return 0; +} +``` + +Configuration can also come from the process environment (`datahub_config_from_env()`), from a +dotenv-style file (`datahub_config_load_envfile`), or from `datahub_config_set(cfg, "KEY", "value")` +with any of the keys below. Nothing ever reads a `.env` file from the working directory: a library +must not pick up a dotfile from its host's cwd. + +| Key | Setter | Meaning | +|---|---|---| +| `BASE_URL` | `datahub_config_set_base_url` | The api's root URL. Required. | +| `TOKEN` | `datahub_config_set_token` | A bearer token used as-is, never refreshed. | +| `CLIENT_ID`, `CLIENT_SECRET`, `TOKEN_URI` | `datahub_config_set_client_credentials` | OAuth2 client credentials; tokens are minted and refreshed automatically. | +| `SCOPE` | `datahub_config_set_scope` | Added to the token request. `organization:*` (or `organization:`) on Keycloak Organizations realms. | +| `AUDIENCE` | `datahub_config_set_audience` | Token request audience (Auth0). | +| `ASSERTION`, `ASSERTION_CLIENT_ID`, `ASSERTION_CLIENT_SECRET`, `ASSERTION_TOKEN_URI`, `ASSERTION_SCOPE`, `ASSERTION_AUDIENCE`, `ASSERTION_GRANT` | `datahub_config_set_assertion*` | The RFC 7523 `jwt-bearer` and federated flows; see the crate README. | +| `ENABLE_BUFFERING`, `BUFFER_DIR`, `BUFFER_RETENTION_SECS`, `BUFFER_MAX_BYTES` | `datahub_config_enable_buffering`, `datahub_config_set_buffer_*` | Durable ingest buffering: off by default; 72 h window and 5 GiB cap when on. | + +### What buffering does and does not do + +With buffering on, an ingest that cannot reach the server — or whose credential is refused, which +is recoverable out of band — returns `DATAHUB_BUFFERED` and the data is on disk. The next ingest +call, or `datahub_client_flush`, sends the backlog first, so ordering holds; retries are safe +because the server dedups datapoints on (series, timestamp) and events on their client-stamped id. +`datahub_client_buffered_count` reports what is held. The spool survives the process: a new +client on the same directory picks the backlog up. + +The retention window is measured on each record's **own** timestamp, not on when it was spooled. +A backfill older than the window is reported `DATAHUB_BUFFERED` but does not survive in the spool. +Backfill old data with buffering off, or widen the window with `datahub_config_set_buffer_retention_secs`. + +Reads are never buffered. + +## Reading + +```c +datahub_timeseries *ts = NULL; +if (datahub_timeseries_get_by_external_id(client, "pump-1/temperature", &ts) == DATAHUB_OK) { + printf("%s (%s), unit %s\n", datahub_timeseries_name(ts), datahub_timeseries_external_id(ts), + datahub_timeseries_unit(ts) ? datahub_timeseries_unit(ts) : "-"); + datahub_timeseries_free(ts); +} + +datahub_datapoint_agg latest; +if (datahub_datapoints_latest(client, "pump-1/temperature", &latest) == DATAHUB_OK) { + printf("latest: %lld -> %g\n", (long long)latest.timestamp_ms, latest.value); +} + +datahub_datapoint_agg *points = NULL; +size_t count = 0; +if (datahub_datapoints_retrieve(client, "pump-1/temperature", + start_ms, DATAHUB_TIME_UNSET, 1000, &points, &count) == DATAHUB_OK) { + for (size_t i = 0; i < count; i++) { /* points[i].value; aggregates are NaN on a raw read */ } + datahub_datapoints_free(points, count); +} +``` + +## Everything else: JSON + +The datapoint path is typed because it is the path a gateway calls a thousand times a second. +Everything else crosses the boundary as **JSON text**: a `..._json` function takes exactly the +request body the REST endpoint takes and hands back exactly the response body it answers with +(`items`, plus `nextCursor` when there is another page). The REST API reference is therefore the +documentation for every one of them, and a C++ caller uses whatever JSON library it already has. + +```c +char *response = NULL; +datahub_status st = datahub_events_create_json(client, + "{\"items\":[{\"externalId\":\"alarm-17\",\"type\":\"Alarm\",\"eventTime\":\"2026-09-06T10:00:00Z\"}]}", + &response); +if (st == DATAHUB_OK || st == DATAHUB_BUFFERED) { /* … */ } +datahub_string_free(response); +``` + +These are not raw pass-throughs: the body is parsed into the SDK's own types and sent through the +same service method Rust and Python use (so buffering applies to `datahub_events_create_json`, and +a body whose fields have the wrong type is rejected as `DATAHUB_INVALID_ARGUMENT` before anything +is sent). + +For any endpoint without a dedicated function there is `datahub_request_json(client, "POST", +"/resources/filter", body, &response)`: an authenticated raw call, no buffering, response body +returned as sent. `GET` and `POST` are supported; the path is relative to the base URL and may +carry a query string. + +## Listening to subscriptions + +```c +const char *subs[] = { "pump-1-alarms" }; +datahub_listener *listener = NULL; +if (datahub_listener_open(client, subs, 1, &listener) != DATAHUB_OK) { /* … */ } + +datahub_message *msg = NULL; +for (;;) { + switch (datahub_listener_next(listener, 5000, &msg)) { + case DATAHUB_TIMEOUT: continue; /* idle; check a stop flag here */ + case DATAHUB_SUBSCRIPTION: fprintf(stderr, "%s\n", datahub_last_error()); continue; + case DATAHUB_OK: break; + default: goto done; /* DATAHUB_IO after failed reconnects */ + } + if (datahub_message_series_count(msg) > 0) { /* a DATAPOINTS message */ + const datahub_datapoint *points; size_t n; + datahub_message_series_datapoints(msg, 0, &points, &n); + /* points[i].timestamp_ms, points[i].value */ + } else { + handle_json(datahub_message_json(msg)); /* TIMESERIES, EVENT, RESOURCE, … */ + } + const char *id = datahub_message_id(msg); + datahub_listener_ack(listener, &id, 1); + datahub_message_free(msg); +} +done: +datahub_listener_close(listener); +``` + +The listener is pull-based on purpose: a callback API has to define which thread it runs on, what +it may call and what happens if it blocks, while a `next(timeout)` loop is what every C event loop +already knows how to drive. Call it often enough for the server's 15 s pings to be answered. On a +dropped connection the listener reconnects on its own with backoff and resumes the same +subscriptions; anything not acked is redelivered. + +## Console output + +The Rust core prints request/response tracing to stdout and stderr by default, which is useful in +a developer's terminal and wrong inside someone else's daemon. This library turns it **off**; +`datahub_set_debug_output(true)` turns it back on for the whole process. + +## Version + +`datahub_version()` returns the library version; `DATAHUB_VERSION` (and `_MAJOR`/`_MINOR`/`_PATCH`) +in the header say what it was generated from. The two are released together with the Rust crate +and the Python package, under one version number. + +## Testing + +```bash +../run_c_tests.sh # cargo build + cargo test + the C smoke test +../run_c_tests.sh --check-header # additionally fail if the committed header is stale (CI) +../run_c_tests.sh --smoke-only # just compile and run tests/c/smoke.c +``` + +`tests/offline.rs` covers the boundary and the spool paths with no network; `tests/mock_api.rs` +runs against a tiny HTTP mock and asserts what goes on the wire; `tests/live.rs` runs against a +real backend when the repository's `.env` names a `BASE_URL`, and prints `SKIP` otherwise; +`tests/c/smoke.c` is the header and library as a C compiler sees them. + +## Licence + +Apache-2.0, like the rest of this repository. diff --git a/datahub_c_bindings/build.rs b/datahub_c_bindings/build.rs new file mode 100644 index 0000000..4fad035 --- /dev/null +++ b/datahub_c_bindings/build.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Regenerates `include/intellistream_datahub.h` from the crate's sources on every build. +//! +//! The header is committed so a C user never needs cbindgen; CI regenerates it and fails on a +//! diff, the same way a stale `.so` is treated on the Python side. The version macros and the +//! `DATAHUB_TIME_UNSET` sentinel are added here rather than as Rust constants: cbindgen renders a +//! Rust `i64::MIN` path literally, which is not C, and a `-9223372036854775808` literal does not +//! fit a C `long long` before negation. + +use std::env; +use std::path::PathBuf; + +fn main() { + let crate_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + let version = env::var("CARGO_PKG_VERSION").expect("CARGO_PKG_VERSION"); + let major = env::var("CARGO_PKG_VERSION_MAJOR").expect("CARGO_PKG_VERSION_MAJOR"); + let minor = env::var("CARGO_PKG_VERSION_MINOR").expect("CARGO_PKG_VERSION_MINOR"); + let patch = env::var("CARGO_PKG_VERSION_PATCH").expect("CARGO_PKG_VERSION_PATCH"); + + println!("cargo:rerun-if-changed=src"); + println!("cargo:rerun-if-changed=cbindgen.toml"); + println!("cargo:rerun-if-changed=build.rs"); + + let mut config = + cbindgen::Config::from_file(crate_dir.join("cbindgen.toml")).expect("cbindgen.toml"); + config.after_includes = Some(format!( + "\n/* The library version this header was generated from. datahub_version() reports the\n \ + * version actually loaded; the two should agree. */\n\ + #define DATAHUB_VERSION \"{version}\"\n\ + #define DATAHUB_VERSION_MAJOR {major}\n\ + #define DATAHUB_VERSION_MINOR {minor}\n\ + #define DATAHUB_VERSION_PATCH {patch}\n\n\ + /* Pass as start_ms / end_ms to leave that end of a datapoint window open. */\n\ + #define DATAHUB_TIME_UNSET INT64_MIN\n" + )); + + let header = crate_dir.join("include").join("intellistream_datahub.h"); + match cbindgen::Builder::new() + .with_crate(&crate_dir) + .with_config(config) + .generate() + { + Ok(bindings) => { + bindings.write_to_file(&header); + } + // A syntax error in the sources is rustc's to report, with a far better message. + Err(cbindgen::Error::ParseSyntaxError { .. }) => {} + Err(e) => panic!("cbindgen failed to generate {}: {e}", header.display()), + } +} diff --git a/datahub_c_bindings/cbindgen.toml b/datahub_c_bindings/cbindgen.toml new file mode 100644 index 0000000..22abfbc --- /dev/null +++ b/datahub_c_bindings/cbindgen.toml @@ -0,0 +1,42 @@ +# cbindgen configuration for include/intellistream_datahub.h. build.rs runs cbindgen with this file +# and adds the version macros; the generated header is committed, and CI fails if it is stale. +language = "C" +include_guard = "INTELLISTREAM_DATAHUB_H" +include_version = false +cpp_compat = true +style = "type" +usize_is_size_t = true +documentation = true +documentation_style = "doxy" +header = """/* SPDX-License-Identifier: Apache-2.0 */ +/* + * IntelliStream DataHub SDK — C API. + * + * Conventions: + * - Handles (datahub_config, datahub_client, datahub_listener, datahub_message, + * datahub_timeseries) are opaque. Each has a matching _free (or _close) and must not be + * released any other way. + * - Strings are NUL-terminated UTF-8. Input strings are borrowed for the duration of the call. + * A `char **` out-parameter hands the caller an owned string, released with + * datahub_string_free(); a `const char *` returned by an accessor is borrowed and valid until + * the handle it came from is freed. Never pass a library allocation to free(). + * - Functions that can fail return a datahub_status. Anything other than DATAHUB_OK and + * DATAHUB_BUFFERED leaves a message in datahub_last_error() (per thread; overwritten by the + * next failure on that thread) and, for HTTP failures, the status in datahub_last_http_status(). + * - A datahub_client may be used from any number of threads at once. A datahub_listener and a + * datahub_message belong to one thread at a time. + * - Nothing here may be called from inside a Tokio runtime thread. + */""" +autogen_warning = "/* Generated by cbindgen from datahub_c_bindings/src. Edit the Rust sources; `cargo build` regenerates this file. */" + +[export] +include = ["datahub_status", "datahub_datapoint", "datahub_datapoint_agg"] + +[enum] +prefix_with_name = false + +[fn] +sort_by = "None" + +[parse] +parse_deps = false diff --git a/datahub_c_bindings/include/intellistream_datahub.h b/datahub_c_bindings/include/intellistream_datahub.h new file mode 100644 index 0000000..475a949 --- /dev/null +++ b/datahub_c_bindings/include/intellistream_datahub.h @@ -0,0 +1,647 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * IntelliStream DataHub SDK — C API. + * + * Conventions: + * - Handles (datahub_config, datahub_client, datahub_listener, datahub_message, + * datahub_timeseries) are opaque. Each has a matching _free (or _close) and must not be + * released any other way. + * - Strings are NUL-terminated UTF-8. Input strings are borrowed for the duration of the call. + * A `char **` out-parameter hands the caller an owned string, released with + * datahub_string_free(); a `const char *` returned by an accessor is borrowed and valid until + * the handle it came from is freed. Never pass a library allocation to free(). + * - Functions that can fail return a datahub_status. Anything other than DATAHUB_OK and + * DATAHUB_BUFFERED leaves a message in datahub_last_error() (per thread; overwritten by the + * next failure on that thread) and, for HTTP failures, the status in datahub_last_http_status(). + * - A datahub_client may be used from any number of threads at once. A datahub_listener and a + * datahub_message belong to one thread at a time. + * - Nothing here may be called from inside a Tokio runtime thread. + */ + +#ifndef INTELLISTREAM_DATAHUB_H +#define INTELLISTREAM_DATAHUB_H + +/* Generated by cbindgen from datahub_c_bindings/src. Edit the Rust sources; `cargo build` regenerates this file. */ + +#include +#include +#include +#include +#include + +/* The library version this header was generated from. datahub_version() reports the + * version actually loaded; the two should agree. */ +#define DATAHUB_VERSION "0.3.0" +#define DATAHUB_VERSION_MAJOR 0 +#define DATAHUB_VERSION_MINOR 3 +#define DATAHUB_VERSION_PATCH 0 + +/* Pass as start_ms / end_ms to leave that end of a datapoint window open. */ +#define DATAHUB_TIME_UNSET INT64_MIN + + +/** + * Outcome of a call. Everything except `DATAHUB_OK` and `DATAHUB_BUFFERED` leaves a message + * in `datahub_last_error()`. + */ +typedef enum { + /** + * The call succeeded. For an ingest call this means the data reached the server. + */ + DATAHUB_OK = 0, + /** + * The data went into the on-disk spool because the server could not be reached, or refused + * the credential. It is sent on a later ingest call or by `datahub_client_flush`. Not an + * error: this is the answer an edge device wants when the network is down. + */ + DATAHUB_BUFFERED = 1, + /** + * `datahub_listener_next`: nothing arrived within the timeout. + */ + DATAHUB_TIMEOUT = 2, + /** + * `datahub_listener_next`: the stream has ended. + */ + DATAHUB_CLOSED = 3, + /** + * The api answered with no item where exactly one was asked for. + */ + DATAHUB_NOT_FOUND = 4, + /** + * A NULL where a value was required, invalid UTF-8, an empty id, a non-finite value, or a + * JSON body that does not parse as the request the endpoint takes. + */ + DATAHUB_INVALID_ARGUMENT = 10, + /** + * The configuration is incomplete or contradictory: no BASE_URL, no usable credential set, + * a malformed URL. + */ + DATAHUB_CONFIG = 11, + /** + * A token could not be obtained, or the api answered 401 or 403. For a 401 the message + * includes the SDK's diagnosis of the token's `organization` claim when it has one. + */ + DATAHUB_AUTH = 12, + /** + * The api answered another non-2xx status, or the request got no response at all; + * `datahub_last_http_status()` says which. A transport failure (connection refused, DNS, + * timeout) is reported as 503, the same way the core treats it: retryable. + */ + DATAHUB_HTTP = 13, + /** + * A local failure: an env file that cannot be read, a runtime that could not start, a + * WebSocket that could not be opened or was lost for good. + */ + DATAHUB_IO = 14, + /** + * The listener reported an error for one subscription (unknown id, no read access). The + * connection stays open and the other subscriptions keep delivering; call + * `datahub_listener_next` again. + */ + DATAHUB_SUBSCRIPTION = 15, + /** + * A Rust panic was caught at the boundary. The message is in `datahub_last_error()`; + * please report it, it is a bug in the SDK. + */ + DATAHUB_PANIC = 99, +} datahub_status; + +/** + * A connection to one DataHub api. Opaque; may be shared between threads; free with + * `datahub_client_free` after every listener opened from it has been closed. + */ +typedef struct datahub_client datahub_client; + +/** + * Configuration for a `datahub_client`. Opaque; free with `datahub_config_free`. + */ +typedef struct datahub_config datahub_config; + +/** + * A live WebSocket listener over one or more subscriptions. Opaque; one thread at a time; close + * (and free) with `datahub_listener_close`. + */ +typedef struct datahub_listener datahub_listener; + +/** + * One message delivered to a listener. Opaque; free with `datahub_message_free` after acking. + */ +typedef struct datahub_message datahub_message; + +/** + * One time series definition. Opaque; read it through the `datahub_timeseries_*` accessors and + * free it with `datahub_timeseries_free`. + */ +typedef struct datahub_timeseries datahub_timeseries; + +/** + * One numeric datapoint to ingest. + */ +typedef struct { + /** + * Unix epoch milliseconds, UTC. + */ + int64_t timestamp_ms; + /** + * Must be finite. + */ + double value; +} datahub_datapoint; + +/** + * One datapoint as read back. Aggregates the api did not send are NaN, so a plain read never + * needs a presence flag: a raw read carries `value`, an aggregated read the others. + */ +typedef struct { + /** + * Unix epoch milliseconds, UTC. + */ + int64_t timestamp_ms; + double value; + double min; + double max; + double average; + double sum; +} datahub_datapoint_agg; + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +/** + * The library version, e.g. `"0.3.0"`. Static; never NULL. Compare with `DATAHUB_VERSION`. + */ +const char *datahub_version(void); + +/** + * Turn the SDK's console tracing (response bodies, batch progress, failed-request notices on + * stdout/stderr) on or off for the whole process. Off by default in this library. + */ +void datahub_set_debug_output(bool enabled); + +/** + * Build a client from a config. The config is copied and may be freed afterwards. On success + * `*out` is the client; on failure it is NULL and the status says why (`DATAHUB_CONFIG` for an + * incomplete config, `DATAHUB_IO` if the runtime could not start). No request is made here: + * tokens are fetched lazily on the first call that needs one. + */ +datahub_status datahub_client_new(const datahub_config *config, datahub_client **out); + +/** + * Release a client. Spooled data stays on disk for the next client that opens the same buffer + * directory; call `datahub_client_flush` first if it should go out now. NULL is ignored. + */ +void datahub_client_free(datahub_client *client); + +/** + * Send whatever the datapoint and event spools hold, oldest first, without ingesting anything + * new. `DATAHUB_OK` when both spools are empty afterwards; `DATAHUB_BUFFERED` when the server is + * still unreachable and a backlog remains on disk. Always `DATAHUB_OK` when buffering is off. + */ +datahub_status datahub_client_flush(const datahub_client *client); + +/** + * Records (datapoints plus events) currently held in this client's on-disk spools. 0 when + * buffering is off, or when nothing has been spooled by this client yet. + */ +uint64_t datahub_client_buffered_count(const datahub_client *client); + +/** + * An authenticated raw request to any api endpoint — the escape hatch for everything this + * header has no dedicated function for. `method` is `GET` or `POST`; `path` is relative to the + * base URL (`/events/count`, `/resources/filter`), a query string included; `body` is the JSON + * request body (NULL sends `{}` for POST, nothing for GET). `*out` receives the raw response + * body (empty for 204). No buffering applies here; the typed ingest functions have it. + */ +datahub_status datahub_request_json(const datahub_client *client, + const char *method, + const char *path, + const char *body, + char **out); + +/** + * A new, empty config. Never NULL. + */ +datahub_config *datahub_config_new(void); + +/** + * A config holding a snapshot of the process environment (every variable, so the usual + * `BASE_URL`, `TOKEN`, `CLIENT_ID`, `BUFFER_DIR`, … are picked up). Never NULL. Does **not** read + * a `.env` file; see `datahub_config_load_envfile`. + */ +datahub_config *datahub_config_from_env(void); + +/** + * Read a dotenv-style file (`KEY=value` lines) into the config, overriding any keys already + * set. The process environment is left untouched. `DATAHUB_IO` when the file cannot be read, + * `DATAHUB_INVALID_ARGUMENT` when a line does not parse. + */ +datahub_status datahub_config_load_envfile(datahub_config *config, const char *path); + +/** + * Set any configuration key by its environment name (`SCOPE`, `AUDIENCE`, `ASSERTION_GRANT`, …). + * The typed setters below are conveniences over this. A NULL value removes the key. + */ +datahub_status datahub_config_set(datahub_config *config, const char *key, const char *value); + +/** + * Read a key back, or NULL when it is unset. Borrowed; valid until the same key is read again + * on this thread. Meant for diagnostics, not for hot paths. + */ +const char *datahub_config_get(const datahub_config *config, const char *key); + +/** + * The api's root URL, e.g. `https://datahub.example.com`. Required. + */ +datahub_status datahub_config_set_base_url(datahub_config *config, const char *value); + +/** + * A bearer token used as-is and never refreshed; the alternative to client credentials. + */ +datahub_status datahub_config_set_token(datahub_config *config, const char *value); + +/** + * OAuth2 scope added to the token request. Against a realm using Keycloak Organizations this + * must name the tenant: `organization:*`, or `organization:` to pin one. + */ +datahub_status datahub_config_set_scope(datahub_config *config, const char *value); + +/** + * OAuth2 audience for the token request. Required by Auth0, unused by Keycloak. + */ +datahub_status datahub_config_set_audience(datahub_config *config, const char *value); + +/** + * Optional project name. + */ +datahub_status datahub_config_set_project_name(datahub_config *config, const char *value); + +/** + * A ready-made JWT to exchange with the RFC 7523 `jwt-bearer` grant. Never refreshed; prefer + * `datahub_config_set_assertion_credentials`. + */ +datahub_status datahub_config_set_assertion(datahub_config *config, const char *value); + +/** + * Scope for the assertion request; Entra ID needs `api:///.default`. + */ +datahub_status datahub_config_set_assertion_scope(datahub_config *config, const char *value); + +/** + * Audience for the assertion request. + */ +datahub_status datahub_config_set_assertion_audience(datahub_config *config, const char *value); + +/** + * Grant used in the secretless assertion mode: `client_credentials` (default) or `jwt-bearer`. + */ +datahub_status datahub_config_set_assertion_grant(datahub_config *config, const char *value); + +/** + * OAuth2 client credentials: the token endpoint mints and refreshes tokens with these. All three + * are required for the flow to be configured at all. + */ +datahub_status datahub_config_set_client_credentials(datahub_config *config, + const char *client_id, + const char *client_secret, + const char *token_uri); + +/** + * Fetch the `jwt-bearer` assertion with client credentials from another provider (an Entra ID + * app registration, say) and exchange it at the token URI. All three are required. + */ +datahub_status datahub_config_set_assertion_credentials(datahub_config *config, + const char *client_id, + const char *client_secret, + const char *token_uri); + +/** + * Turn on durable ingest buffering with the default bounds (72 h window, 5 GiB cap, directory + * `.datahub-spool` under the working directory unless `datahub_config_set_buffer_dir` says + * otherwise). Off by default. + */ +datahub_status datahub_config_enable_buffering(datahub_config *config); + +/** + * Directory for the on-disk spools. Also enables buffering: a host that names a spool directory + * wants the spool. + */ +datahub_status datahub_config_set_buffer_dir(datahub_config *config, const char *dir); + +/** + * How long spooled records are kept, in seconds, measured on each record's own timestamp + * (default 72 h). Also enables buffering. + */ +datahub_status datahub_config_set_buffer_retention_secs(datahub_config *config, int64_t seconds); + +/** + * Size cap for each spool in bytes; the oldest segment is dropped past it (default 5 GiB). + * Also enables buffering. + */ +datahub_status datahub_config_set_buffer_max_bytes(datahub_config *config, uint64_t bytes); + +/** + * Release a config. The client it was used to build keeps its own copy. NULL is ignored. + */ +void datahub_config_free(datahub_config *config); + +/** + * Ingest `count` numeric datapoints into the series with this external id. + * + * `DATAHUB_OK` means they reached the server. With buffering enabled, `DATAHUB_BUFFERED` means + * the server could not be reached (or refused the credential) and they are on disk, to be sent + * on a later call: the on-disk backlog always goes first, so ordering holds. Retries are safe — + * the server dedups on (series, timestamp). A non-finite value is `DATAHUB_INVALID_ARGUMENT` + * before anything is sent. `count` of 0 is a no-op. + * + * The spool's retention window is measured on each datapoint's own timestamp, not on when it + * was spooled: a backfill older than the window (72 h by default) is reported `DATAHUB_BUFFERED` + * but does not survive in the spool. Backfill old data with buffering off, or widen the window. + */ +datahub_status datahub_datapoints_insert(const datahub_client *client, + const char *external_id, + const datahub_datapoint *points, + size_t count); + +/** + * Ingest `count` string-valued datapoints (for text-typed series). `timestamps_ms[i]` pairs + * with `values[i]`. Same status contract as `datahub_datapoints_insert`. + */ +datahub_status datahub_datapoints_insert_str(const datahub_client *client, + const char *external_id, + const int64_t *timestamps_ms, + const char *const *values, + size_t count); + +/** + * The most recent datapoint of a series, written into `*out`. `DATAHUB_NOT_FOUND` when the + * series has no datapoints (or does not exist). + */ +datahub_status datahub_datapoints_latest(const datahub_client *client, + const char *external_id, + datahub_datapoint_agg *out); + +/** + * Raw datapoints of a series inside a window. `start_ms` is inclusive, `end_ms` exclusive; + * pass `DATAHUB_TIME_UNSET` to leave either end open. `limit` of 0 means the server default. + * `*out` receives an array of `*out_count` points, newest last, released with + * `datahub_datapoints_free`; both are 0/NULL when the window is empty. For aggregated reads + * (`aggregates`, `granularity`, paging with `cursor`) use `datahub_datapoints_retrieve_json`. + */ +datahub_status datahub_datapoints_retrieve(const datahub_client *client, + const char *external_id, + int64_t start_ms, + int64_t end_ms, + uint64_t limit, + datahub_datapoint_agg **out, + size_t *out_count); + +/** + * Release an array from `datahub_datapoints_retrieve`, with the count it came with. NULL is ignored. + */ +void datahub_datapoints_free(datahub_datapoint_agg *points, + size_t count); + +/** + * `POST /timeseries/data/list` with the full request: `body` is + * `{"items":[{"externalId":…,"start":…,"end":…,"limit":…,"aggregates":[…],"granularity":…,"cursor":…}]}` + * and `*out` receives the response envelope as the api sent it. + */ +datahub_status datahub_datapoints_retrieve_json(const datahub_client *client, + const char *body, + char **out); + +/** + * The message left by the last failing call on this thread, or `""` when there has been none. + * Borrowed: valid until the next failing call on the same thread. + */ +const char *datahub_last_error(void); + +/** + * The HTTP status of the last `DATAHUB_HTTP` / `DATAHUB_AUTH` / `DATAHUB_NOT_FOUND` failure on + * this thread, or 0 when the last failure was not an HTTP response. + */ +int datahub_last_http_status(void); + +/** + * The name of a status, e.g. `"DATAHUB_BUFFERED"`, for logging. Static; never NULL. + */ +const char *datahub_status_name(datahub_status status); + +/** + * `POST /events/create`. `body` is `{"items":[{"externalId":…,"type":…,"eventTime":…}, …]}`. + * `DATAHUB_OK` with the created events in `*out`; with buffering enabled, `DATAHUB_BUFFERED` + * (and `{"items":[]}`) when they went to the spool instead. Each event is stamped with a + * time-ordered UUID before the first attempt, so a retry from the spool is not a duplicate. + */ +datahub_status datahub_events_create_json(const datahub_client *client, + const char *body, + char **out); + +/** + * `POST /events/filter`. `body` is `{"filter":{…},"limit":…,"cursor":…}`; page with the + * response's `nextCursor`. + */ +datahub_status datahub_events_filter_json(const datahub_client *client, + const char *body, + char **out); + +/** + * Open a listener on `count` subscription external ids (0 is allowed; add them later with + * `datahub_listener_subscribe`). The handshake fetches a token through the client. On a dropped + * connection the listener reconnects on its own with backoff and resumes the same + * subscriptions; anything not acked is redelivered. + */ +datahub_status datahub_listener_open(const datahub_client *client, + const char *const *subscription_external_ids, + size_t count, + datahub_listener **out); + +/** + * Wait up to `timeout_ms` for the next message (negative waits indefinitely, 0 only takes what + * has already arrived). `DATAHUB_OK` with the message in `*out`; `DATAHUB_TIMEOUT` with `*out` + * NULL when nothing came; `DATAHUB_SUBSCRIPTION` when the server rejected one subscription (the + * others keep delivering, call again); `DATAHUB_IO` when the connection was lost and could not + * be re-established after several attempts (calling again retries). Call this often enough for + * the server's 15 s pings to be answered, or the session is closed as idle and reconnected. + */ +datahub_status datahub_listener_next(datahub_listener *listener, + int64_t timeout_ms, + datahub_message **out); + +/** + * Acknowledge `count` message ids so they are not redelivered. + */ +datahub_status datahub_listener_ack(datahub_listener *listener, + const char *const *message_ids, + size_t count); + +/** + * Negative-acknowledge `count` message ids so they are redelivered. + */ +datahub_status datahub_listener_nack(datahub_listener *listener, + const char *const *message_ids, + size_t count); + +/** + * Add `count` subscriptions to the live set without reconnecting. + */ +datahub_status datahub_listener_subscribe(datahub_listener *listener, + const char *const *subscription_external_ids, + size_t count); + +/** + * Remove `count` subscriptions from the live set. + */ +datahub_status datahub_listener_unsubscribe(datahub_listener *listener, + const char *const *subscription_external_ids, + size_t count); + +/** + * Send a close frame, wait for the server's, and free the listener. Unacked messages are + * redelivered to the next listener on the same subscriptions. NULL is ignored. + */ +datahub_status datahub_listener_close(datahub_listener *listener); + +/** + * The id to pass to `datahub_listener_ack`/`_nack`. Borrowed; valid until the message is freed. + */ +const char *datahub_message_id(const datahub_message *message); + +/** + * The subscription this message was delivered for. Borrowed. + */ +const char *datahub_message_subscription(const datahub_message *message); + +/** + * What happened: `CREATE`, `UPDATE`, `DELETE` or `RENAME`. Borrowed. + */ +const char *datahub_message_action(const datahub_message *message); + +/** + * What it happened to: `DATAPOINTS`, `TIMESERIES`, `EVENT`, `RESOURCE`, …. Borrowed. + */ +const char *datahub_message_object(const datahub_message *message); + +/** + * The whole message as JSON (`subscriptionExternalId`, `messageId`, `payload`). Borrowed. + */ +const char *datahub_message_json(const datahub_message *message); + +/** + * How many series this message carries datapoints for (0 unless the object is `DATAPOINTS`). + */ +size_t datahub_message_series_count(const datahub_message *message); + +/** + * External id of the `index`-th series, or NULL when the message named it by id only. Borrowed. + */ +const char *datahub_message_series_external_id(const datahub_message *message, size_t index); + +/** + * Numeric id of the `index`-th series, or 0 when the message did not carry one. + */ +uint64_t datahub_message_series_id(const datahub_message *message, size_t index); + +/** + * The datapoints of the `index`-th series as a borrowed array (valid until the message is + * freed). A value that is not numeric reads as NaN, a timestamp that could not be parsed as + * `INT64_MIN`; the JSON has the originals. + */ +datahub_status datahub_message_series_datapoints(const datahub_message *message, + size_t index, + const datahub_datapoint **out, + size_t *out_count); + +/** + * Release a message. NULL is ignored. + */ +void datahub_message_free(datahub_message *message); + +/** + * Look one time series up by external id. `DATAHUB_NOT_FOUND` when the api knows no such series + * (or the caller may not read it — the api does not distinguish). + */ +datahub_status datahub_timeseries_get_by_external_id(const datahub_client *client, + const char *external_id, + datahub_timeseries **out); + +/** + * Release a time series handle. NULL is ignored. + */ +void datahub_timeseries_free(datahub_timeseries *ts); + +/** + * Numeric id; 0 when the api did not send one. + */ +uint64_t datahub_timeseries_id(const datahub_timeseries *ts); + +/** + * Numeric id of the data set the series belongs to; 0 when it has none. + */ +uint64_t datahub_timeseries_data_set_id(const datahub_timeseries *ts); + +/** + * External id. Borrowed; valid until the handle is freed. + */ +const char *datahub_timeseries_external_id(const datahub_timeseries *ts); + +/** + * Display name. Borrowed. + */ +const char *datahub_timeseries_name(const datahub_timeseries *ts); + +/** + * Unit symbol, or NULL when the series has none. Borrowed. + */ +const char *datahub_timeseries_unit(const datahub_timeseries *ts); + +/** + * Unit catalogue id, or NULL. Borrowed. + */ +const char *datahub_timeseries_unit_external_id(const datahub_timeseries *ts); + +/** + * Value type (`float`, `bigint`, `text`, …), or NULL when the endpoint did not say. Borrowed. + */ +const char *datahub_timeseries_value_type(const datahub_timeseries *ts); + +/** + * The whole definition as the api sent it, as JSON. Borrowed. + */ +const char *datahub_timeseries_json(const datahub_timeseries *ts); + +/** + * `POST /timeseries/create`. `body` is `{"items":[{"externalId":…,"name":…,"unit":…}, …]}`; + * `*out` receives the created definitions in the same envelope. + */ +datahub_status datahub_timeseries_create_json(const datahub_client *client, + const char *body, + char **out); + +/** + * `POST /timeseries/search`: free-text search with optional narrowing. `body` is + * `{"search":{"query":…},"filter":{…},"limit":…}`. + */ +datahub_status datahub_timeseries_search_json(const datahub_client *client, + const char *body, + char **out); + +/** + * `POST /timeseries/filter`: structured, AND-combined filtering. `body` is + * `{"filter":{…},"limit":…,"cursor":…}`; page with the response's `nextCursor`. + */ +datahub_status datahub_timeseries_filter_json(const datahub_client *client, + const char *body, + char **out); + +/** + * Release a string the library handed out through a `char **` out-parameter. NULL is ignored. + */ +void datahub_string_free(char *text); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif /* INTELLISTREAM_DATAHUB_H */ diff --git a/datahub_c_bindings/src/client.rs b/datahub_c_bindings/src/client.rs new file mode 100644 index 0000000..1019118 --- /dev/null +++ b/datahub_c_bindings/src/client.rs @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: Apache-2.0 +//! `datahub_client`: the async `ApiService` plus the Tokio runtime that drives it. + +use std::ffi::c_char; +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use intellistream_datahub_sdk::generic::ApiServiceProvider; +use intellistream_datahub_sdk::http::set_debug_output; +use intellistream_datahub_sdk::ApiService; +use tokio::runtime::{Builder, Runtime}; + +use crate::config::{config_value, datahub_config}; +use crate::error::{datahub_status, fail, from_config_error, from_response_error, guard}; +use crate::util::{mut_arg, opt_str_arg, out_str, ref_arg, str_arg}; + +/// A connection to one DataHub api. Opaque; may be shared between threads; free with +/// `datahub_client_free` after every listener opened from it has been closed. +pub struct datahub_client { + pub(crate) api: Arc, + pub(crate) rt: Arc, + pub(crate) base_url: String, +} + +impl datahub_client { + /// Drive a core future to completion on this client's runtime, blocking the calling thread. + pub(crate) fn run(&self, future: F) -> F::Output { + self.rt.block_on(future) + } +} + +/// Whether the host asked for the core's console tracing. Off by default for a library: the +/// core defaults it on, so every client construction re-applies this. +static DEBUG_OUTPUT: AtomicBool = AtomicBool::new(false); + +/// The library version, e.g. `"0.3.0"`. Static; never NULL. Compare with `DATAHUB_VERSION`. +#[no_mangle] +pub extern "C" fn datahub_version() -> *const c_char { + concat!(env!("CARGO_PKG_VERSION"), "\0").as_ptr() as *const c_char +} + +/// Turn the SDK's console tracing (response bodies, batch progress, failed-request notices on +/// stdout/stderr) on or off for the whole process. Off by default in this library. +#[no_mangle] +pub extern "C" fn datahub_set_debug_output(enabled: bool) { + DEBUG_OUTPUT.store(enabled, Ordering::Relaxed); + set_debug_output(enabled); +} + +/// Build a client from a config. The config is copied and may be freed afterwards. On success +/// `*out` is the client; on failure it is NULL and the status says why (`DATAHUB_CONFIG` for an +/// incomplete config, `DATAHUB_IO` if the runtime could not start). No request is made here: +/// tokens are fetched lazily on the first call that needs one. +#[no_mangle] +pub unsafe extern "C" fn datahub_client_new( + config: *const datahub_config, + out: *mut *mut datahub_client, +) -> datahub_status { + guard(|| { + let out = ffi_try!(mut_arg(out, "out")); + *out = std::ptr::null_mut(); + let config_ref = ffi_try!(ref_arg(config, "config")); + + // The core's `TokenUrl::new(...).expect(...)` would panic on a malformed token URI; say + // so as a config error instead. + if let Some(uri) = ffi_try!(config_value(config, "TOKEN_URI")) { + if !(uri.starts_with("http://") || uri.starts_with("https://")) { + return fail( + datahub_status::DATAHUB_CONFIG, + 0, + format!("TOKEN_URI must be an http(s) URL, got {uri:?}"), + ); + } + } + let core_config = match config_ref.build() { + Ok(core_config) => core_config, + Err(e) => return from_config_error(&e), + }; + let base_url = config_ref + .get("BASE_URL") + .unwrap_or_default() + .trim_end_matches('/') + .to_string(); + + set_debug_output(DEBUG_OUTPUT.load(Ordering::Relaxed)); + + let rt = match Builder::new_multi_thread() + .enable_all() + .worker_threads(2) + .thread_name("datahub-sdk") + .build() + { + Ok(rt) => rt, + Err(e) => { + return fail( + datahub_status::DATAHUB_IO, + 0, + format!("could not start the SDK runtime: {e}"), + ) + } + }; + let api = ApiService::new(core_config); + *out = Box::into_raw(Box::new(datahub_client { + api, + rt: Arc::new(rt), + base_url, + })); + datahub_status::DATAHUB_OK + }) +} + +/// Release a client. Spooled data stays on disk for the next client that opens the same buffer +/// directory; call `datahub_client_flush` first if it should go out now. NULL is ignored. +#[no_mangle] +pub unsafe extern "C" fn datahub_client_free(client: *mut datahub_client) { + if !client.is_null() { + let _ = guard(|| { + drop(Box::from_raw(client)); + datahub_status::DATAHUB_OK + }); + } +} + +/// Send whatever the datapoint and event spools hold, oldest first, without ingesting anything +/// new. `DATAHUB_OK` when both spools are empty afterwards; `DATAHUB_BUFFERED` when the server is +/// still unreachable and a backlog remains on disk. Always `DATAHUB_OK` when buffering is off. +#[no_mangle] +pub unsafe extern "C" fn datahub_client_flush(client: *const datahub_client) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let datapoints_done = client.run(client.api.time_series.flush_buffer()); + let events_done = client.run(client.api.events.flush_buffer()); + if datapoints_done && events_done { + datahub_status::DATAHUB_OK + } else { + datahub_status::DATAHUB_BUFFERED + } + }) +} + +/// Records (datapoints plus events) currently held in this client's on-disk spools. 0 when +/// buffering is off, or when nothing has been spooled by this client yet. +#[no_mangle] +pub unsafe extern "C" fn datahub_client_buffered_count(client: *const datahub_client) -> u64 { + match client.as_ref() { + Some(client) => { + client.api.time_series.buffered_count() + client.api.events.buffered_count() + } + None => 0, + } +} + +/// An authenticated raw request to any api endpoint — the escape hatch for everything this +/// header has no dedicated function for. `method` is `GET` or `POST`; `path` is relative to the +/// base URL (`/events/count`, `/resources/filter`), a query string included; `body` is the JSON +/// request body (NULL sends `{}` for POST, nothing for GET). `*out` receives the raw response +/// body (empty for 204). No buffering applies here; the typed ingest functions have it. +#[no_mangle] +pub unsafe extern "C" fn datahub_request_json( + client: *const datahub_client, + method: *const c_char, + path: *const c_char, + body: *const c_char, + out: *mut *mut c_char, +) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let method = ffi_try!(str_arg(method, "method")).to_ascii_uppercase(); + let path = ffi_try!(str_arg(path, "path")); + let body = ffi_try!(opt_str_arg(body, "body")); + let _ = ffi_try!(mut_arg(out, "out")); + + let url = format!( + "{}{}{}", + client.base_url, + if path.starts_with('/') { "" } else { "/" }, + path + ); + let json: serde_json::Value = match body { + Some(text) => match serde_json::from_str(text) { + Ok(value) => value, + Err(e) => { + return fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("body is not valid JSON: {e}"), + ) + } + }, + None => serde_json::json!({}), + }; + // Any service carries the request plumbing; the timeseries one is as good as any. + let service = &client.api.time_series; + let result = match method.as_str() { + "GET" => client.run(service.execute_get_request::(&url, None)), + "POST" => { + client.run(service.execute_post_request::(&url, &json)) + } + other => { + return fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("method must be GET or POST, got {other:?}"), + ) + } + }; + match result { + Ok(text) => { + ffi_try!(out_str(out, text)); + datahub_status::DATAHUB_OK + } + Err(e) => from_response_error(&e), + } + }) +} diff --git a/datahub_c_bindings/src/config.rs b/datahub_c_bindings/src/config.rs new file mode 100644 index 0000000..d14acee --- /dev/null +++ b/datahub_c_bindings/src/config.rs @@ -0,0 +1,448 @@ +// SPDX-License-Identifier: Apache-2.0 +//! `datahub_config`: the values a client is built from. +//! +//! A config is a map keyed by the same names the environment uses (`BASE_URL`, `TOKEN`, +//! `CLIENT_ID`, …, `BUFFER_DIR`), so the environment, an env file and the typed setters all feed +//! one code path: `DataHubConfig::from_map` in the core. Nothing here reads or writes the process +//! environment unless `datahub_config_from_env` is called, and nothing ever reads a `.env` file +//! from the working directory — a library must not pick up a dotfile from the host's cwd. +//! +//! Every export is written out by hand rather than generated by a macro: cbindgen does not expand +//! `macro_rules!`, so a macro-generated function would exist in the library but not in the header. + +use std::collections::HashMap; +use std::env; +use std::ffi::c_char; +use std::path::Path; + +use intellistream_datahub_sdk::datahub::DataHubConfig; +use intellistream_datahub_sdk::errors::DataHubError; + +use crate::error::{datahub_status, fail, guard}; +use crate::util::{mut_arg, opt_str_arg, ref_arg, str_arg}; + +/// Configuration for a `datahub_client`. Opaque; free with `datahub_config_free`. +pub struct datahub_config { + pub(crate) values: HashMap, +} + +impl datahub_config { + pub(crate) fn build(&self) -> Result { + DataHubConfig::from_map(self.values.clone()) + } + + pub(crate) fn get(&self, key: &str) -> Option<&str> { + self.values.get(key).map(String::as_str) + } + + fn set(&mut self, key: &str, value: &str) { + self.values.insert(key.to_string(), value.to_string()); + } +} + +/// A new, empty config. Never NULL. +#[no_mangle] +pub extern "C" fn datahub_config_new() -> *mut datahub_config { + Box::into_raw(Box::new(datahub_config { + values: HashMap::new(), + })) +} + +/// A config holding a snapshot of the process environment (every variable, so the usual +/// `BASE_URL`, `TOKEN`, `CLIENT_ID`, `BUFFER_DIR`, … are picked up). Never NULL. Does **not** read +/// a `.env` file; see `datahub_config_load_envfile`. +#[no_mangle] +pub extern "C" fn datahub_config_from_env() -> *mut datahub_config { + Box::into_raw(Box::new(datahub_config { + values: env::vars().collect(), + })) +} + +/// Parse dotenv-style text: one `KEY=value` per line, `#` comments, blank lines, an optional +/// `export ` prefix, and single or double quotes around a value. Nothing fancier — the same +/// subset every `.env` in this project uses. +pub(crate) fn parse_env_text(text: &str) -> Result, String> { + let mut entries = Vec::new(); + for (index, raw) in text.lines().enumerate() { + let number = index + 1; + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let line = line.strip_prefix("export ").unwrap_or(line).trim_start(); + let Some((key, value)) = line.split_once('=') else { + return Err(format!("line {number}: expected KEY=value")); + }; + let key = key.trim(); + if key.is_empty() || !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(format!("line {number}: {key:?} is not a valid key")); + } + let value = value.trim(); + let value = match value.chars().next() { + Some(quote @ ('"' | '\'')) => match value[1..].find(quote) { + Some(end) => &value[1..1 + end], + None => return Err(format!("line {number}: unterminated quote")), + }, + _ => value.split(" #").next().unwrap_or(value).trim(), + }; + entries.push((key.to_string(), value.to_string())); + } + Ok(entries) +} + +/// Read a dotenv-style file (`KEY=value` lines) into the config, overriding any keys already +/// set. The process environment is left untouched. `DATAHUB_IO` when the file cannot be read, +/// `DATAHUB_INVALID_ARGUMENT` when a line does not parse. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_load_envfile( + config: *mut datahub_config, + path: *const c_char, +) -> datahub_status { + guard(|| { + let config = ffi_try!(mut_arg(config, "config")); + let path = ffi_try!(str_arg(path, "path")); + let text = match std::fs::read_to_string(Path::new(path)) { + Ok(text) => text, + Err(e) => { + return fail( + datahub_status::DATAHUB_IO, + 0, + format!("cannot read env file {path}: {e}"), + ) + } + }; + match parse_env_text(&text) { + Ok(entries) => { + for (key, value) in entries { + config.set(&key, &value); + } + datahub_status::DATAHUB_OK + } + Err(e) => fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("cannot parse env file {path}: {e}"), + ), + } + }) +} + +/// Set any configuration key by its environment name (`SCOPE`, `AUDIENCE`, `ASSERTION_GRANT`, …). +/// The typed setters below are conveniences over this. A NULL value removes the key. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set( + config: *mut datahub_config, + key: *const c_char, + value: *const c_char, +) -> datahub_status { + guard(|| { + let config = ffi_try!(mut_arg(config, "config")); + let key = ffi_try!(str_arg(key, "key")); + if key.trim().is_empty() { + return fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + "key must not be empty", + ); + } + match ffi_try!(opt_str_arg(value, "value")) { + Some(value) => config.set(key, value), + None => { + config.values.remove(key); + } + } + datahub_status::DATAHUB_OK + }) +} + +thread_local! { + /// `datahub_config_get` hands out NUL-terminated copies; they live here so the pointer stays + /// valid until the same key is read again on this thread, or the thread ends. + static GET_CACHE: std::cell::RefCell> = + std::cell::RefCell::new(HashMap::new()); +} + +/// Read a key back, or NULL when it is unset. Borrowed; valid until the same key is read again +/// on this thread. Meant for diagnostics, not for hot paths. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_get( + config: *const datahub_config, + key: *const c_char, +) -> *const c_char { + let (Some(config), false) = (config.as_ref(), key.is_null()) else { + return std::ptr::null(); + }; + let Ok(key) = std::ffi::CStr::from_ptr(key).to_str() else { + return std::ptr::null(); + }; + match config.values.get(key) { + Some(value) => GET_CACHE.with(|cache| { + let mut cache = cache.borrow_mut(); + cache.insert(key.to_string(), crate::util::to_cstring(value)); + cache[key].as_ptr() + }), + None => std::ptr::null(), + } +} + +/// The body every typed string setter shares. +unsafe fn set_key(config: *mut datahub_config, key: &str, value: *const c_char) -> datahub_status { + guard(|| { + let config = ffi_try!(mut_arg(config, "config")); + let value = ffi_try!(str_arg(value, "value")); + config.set(key, value); + datahub_status::DATAHUB_OK + }) +} + +/// The api's root URL, e.g. `https://datahub.example.com`. Required. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_base_url( + config: *mut datahub_config, + value: *const c_char, +) -> datahub_status { + set_key(config, "BASE_URL", value) +} + +/// A bearer token used as-is and never refreshed; the alternative to client credentials. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_token( + config: *mut datahub_config, + value: *const c_char, +) -> datahub_status { + set_key(config, "TOKEN", value) +} + +/// OAuth2 scope added to the token request. Against a realm using Keycloak Organizations this +/// must name the tenant: `organization:*`, or `organization:` to pin one. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_scope( + config: *mut datahub_config, + value: *const c_char, +) -> datahub_status { + set_key(config, "SCOPE", value) +} + +/// OAuth2 audience for the token request. Required by Auth0, unused by Keycloak. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_audience( + config: *mut datahub_config, + value: *const c_char, +) -> datahub_status { + set_key(config, "AUDIENCE", value) +} + +/// Optional project name. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_project_name( + config: *mut datahub_config, + value: *const c_char, +) -> datahub_status { + set_key(config, "PROJECT_NAME", value) +} + +/// A ready-made JWT to exchange with the RFC 7523 `jwt-bearer` grant. Never refreshed; prefer +/// `datahub_config_set_assertion_credentials`. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_assertion( + config: *mut datahub_config, + value: *const c_char, +) -> datahub_status { + set_key(config, "ASSERTION", value) +} + +/// Scope for the assertion request; Entra ID needs `api:///.default`. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_assertion_scope( + config: *mut datahub_config, + value: *const c_char, +) -> datahub_status { + set_key(config, "ASSERTION_SCOPE", value) +} + +/// Audience for the assertion request. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_assertion_audience( + config: *mut datahub_config, + value: *const c_char, +) -> datahub_status { + set_key(config, "ASSERTION_AUDIENCE", value) +} + +/// Grant used in the secretless assertion mode: `client_credentials` (default) or `jwt-bearer`. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_assertion_grant( + config: *mut datahub_config, + value: *const c_char, +) -> datahub_status { + set_key(config, "ASSERTION_GRANT", value) +} + +/// OAuth2 client credentials: the token endpoint mints and refreshes tokens with these. All three +/// are required for the flow to be configured at all. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_client_credentials( + config: *mut datahub_config, + client_id: *const c_char, + client_secret: *const c_char, + token_uri: *const c_char, +) -> datahub_status { + guard(|| { + let config = ffi_try!(mut_arg(config, "config")); + let client_id = ffi_try!(str_arg(client_id, "client_id")); + let client_secret = ffi_try!(str_arg(client_secret, "client_secret")); + let token_uri = ffi_try!(str_arg(token_uri, "token_uri")); + config.set("CLIENT_ID", client_id); + config.set("CLIENT_SECRET", client_secret); + config.set("TOKEN_URI", token_uri); + datahub_status::DATAHUB_OK + }) +} + +/// Fetch the `jwt-bearer` assertion with client credentials from another provider (an Entra ID +/// app registration, say) and exchange it at the token URI. All three are required. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_assertion_credentials( + config: *mut datahub_config, + client_id: *const c_char, + client_secret: *const c_char, + token_uri: *const c_char, +) -> datahub_status { + guard(|| { + let config = ffi_try!(mut_arg(config, "config")); + let client_id = ffi_try!(str_arg(client_id, "client_id")); + let client_secret = ffi_try!(str_arg(client_secret, "client_secret")); + let token_uri = ffi_try!(str_arg(token_uri, "token_uri")); + config.set("ASSERTION_CLIENT_ID", client_id); + config.set("ASSERTION_CLIENT_SECRET", client_secret); + config.set("ASSERTION_TOKEN_URI", token_uri); + datahub_status::DATAHUB_OK + }) +} + +/// Turn on durable ingest buffering with the default bounds (72 h window, 5 GiB cap, directory +/// `.datahub-spool` under the working directory unless `datahub_config_set_buffer_dir` says +/// otherwise). Off by default. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_enable_buffering( + config: *mut datahub_config, +) -> datahub_status { + guard(|| { + let config = ffi_try!(mut_arg(config, "config")); + config.set("ENABLE_BUFFERING", "true"); + datahub_status::DATAHUB_OK + }) +} + +/// Directory for the on-disk spools. Also enables buffering: a host that names a spool directory +/// wants the spool. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_buffer_dir( + config: *mut datahub_config, + dir: *const c_char, +) -> datahub_status { + guard(|| { + let config = ffi_try!(mut_arg(config, "config")); + let dir = ffi_try!(str_arg(dir, "dir")); + config.set("BUFFER_DIR", dir); + config.set("ENABLE_BUFFERING", "true"); + datahub_status::DATAHUB_OK + }) +} + +/// How long spooled records are kept, in seconds, measured on each record's own timestamp +/// (default 72 h). Also enables buffering. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_buffer_retention_secs( + config: *mut datahub_config, + seconds: i64, +) -> datahub_status { + guard(|| { + let config = ffi_try!(mut_arg(config, "config")); + if seconds <= 0 { + return fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + "buffer retention must be a positive number of seconds", + ); + } + config.set("BUFFER_RETENTION_SECS", &seconds.to_string()); + config.set("ENABLE_BUFFERING", "true"); + datahub_status::DATAHUB_OK + }) +} + +/// Size cap for each spool in bytes; the oldest segment is dropped past it (default 5 GiB). +/// Also enables buffering. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_set_buffer_max_bytes( + config: *mut datahub_config, + bytes: u64, +) -> datahub_status { + guard(|| { + let config = ffi_try!(mut_arg(config, "config")); + if bytes == 0 { + return fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + "buffer size cap must be positive", + ); + } + config.set("BUFFER_MAX_BYTES", &bytes.to_string()); + config.set("ENABLE_BUFFERING", "true"); + datahub_status::DATAHUB_OK + }) +} + +/// Release a config. The client it was used to build keeps its own copy. NULL is ignored. +#[no_mangle] +pub unsafe extern "C" fn datahub_config_free(config: *mut datahub_config) { + if !config.is_null() { + drop(Box::from_raw(config)); + } +} + +/// The value of a key, for the rest of the crate. +pub(crate) unsafe fn config_value<'a>( + config: *const datahub_config, + key: &str, +) -> Result, datahub_status> { + Ok(ref_arg(config, "config")?.get(key)) +} + +#[cfg(test)] +mod tests { + use super::parse_env_text; + + #[test] + fn parses_the_dotenv_subset_this_project_uses() { + let text = "# comment\n\nBASE_URL=https://h\nexport TOKEN='a b'\nSCOPE=\"organization:*\" # tenant\nX = plain # note\nY=\n"; + let entries = parse_env_text(text).unwrap(); + assert_eq!( + entries, + vec![ + ("BASE_URL".to_string(), "https://h".to_string()), + ("TOKEN".to_string(), "a b".to_string()), + ("SCOPE".to_string(), "organization:*".to_string()), + ("X".to_string(), "plain".to_string()), + ("Y".to_string(), String::new()), + ] + ); + } + + #[test] + fn rejects_what_it_cannot_read_and_names_the_line() { + assert_eq!( + parse_env_text("A=1\nnot a pair\n").unwrap_err(), + "line 2: expected KEY=value" + ); + assert_eq!( + parse_env_text("BAD KEY=1\n").unwrap_err(), + "line 1: \"BAD KEY\" is not a valid key" + ); + assert_eq!( + parse_env_text("A=\"open\n").unwrap_err(), + "line 1: unterminated quote" + ); + } +} diff --git a/datahub_c_bindings/src/datapoints.rs b/datahub_c_bindings/src/datapoints.rs new file mode 100644 index 0000000..0d9b7b8 --- /dev/null +++ b/datahub_c_bindings/src/datapoints.rs @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Datapoints: the typed hot path. Arrays of plain C structs in, arrays of plain C structs out, +//! and no JSON library needed on the caller's side. + +use std::ffi::c_char; + +use chrono::{DateTime, Utc}; +use intellistream_datahub_sdk::generic::{ + DataWrapper, Datapoint, DatapointString, DatapointsCollection, IdAndExtId, RetrieveFilter, +}; +use intellistream_datahub_sdk::http::ResponseError; + +use crate::client::datahub_client; +use crate::error::{datahub_status, fail, from_response_error, guard}; +use crate::json::{parse, respond}; +use crate::util::{mut_arg, nonempty, ref_arg, str_arg}; + +/// The `start_ms`/`end_ms` sentinel, exported to C as `DATAHUB_TIME_UNSET` by build.rs. +pub(crate) const TIME_UNSET: i64 = i64::MIN; + +/// One numeric datapoint to ingest. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct datahub_datapoint { + /// Unix epoch milliseconds, UTC. + pub timestamp_ms: i64, + /// Must be finite. + pub value: f64, +} + +/// One datapoint as read back. Aggregates the api did not send are NaN, so a plain read never +/// needs a presence flag: a raw read carries `value`, an aggregated read the others. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct datahub_datapoint_agg { + /// Unix epoch milliseconds, UTC. + pub timestamp_ms: i64, + pub value: f64, + pub min: f64, + pub max: f64, + pub average: f64, + pub sum: f64, +} + +impl datahub_datapoint_agg { + fn from_core(dp: &Datapoint) -> Self { + datahub_datapoint_agg { + timestamp_ms: dp.timestamp.timestamp_millis(), + value: dp.value.unwrap_or(f64::NAN), + min: dp.min.unwrap_or(f64::NAN), + max: dp.max.unwrap_or(f64::NAN), + average: dp.average.unwrap_or(f64::NAN), + sum: dp.sum.unwrap_or(f64::NAN), + } + } +} + +/// The core answers a spooled ingest with 202 and no items; the caller must be able to tell. +fn ingest_status(result: Result, ResponseError>) -> datahub_status { + match result { + Ok(wrapper) if wrapper.get_http_status_code() == Some(202) => { + datahub_status::DATAHUB_BUFFERED + } + Ok(_) => datahub_status::DATAHUB_OK, + Err(e) => from_response_error(&e), + } +} + +unsafe fn send( + client: &datahub_client, + external_id: &str, + datapoints: Vec, +) -> datahub_status { + if datapoints.is_empty() { + return datahub_status::DATAHUB_OK; + } + let mut collection = DatapointsCollection::from_external_id(external_id); + collection.datapoints = datapoints; + let mut request = DataWrapper::from_vec(vec![collection]); + ingest_status(client.run(client.api.time_series.insert_datapoints(&mut request))) +} + +/// Ingest `count` numeric datapoints into the series with this external id. +/// +/// `DATAHUB_OK` means they reached the server. With buffering enabled, `DATAHUB_BUFFERED` means +/// the server could not be reached (or refused the credential) and they are on disk, to be sent +/// on a later call: the on-disk backlog always goes first, so ordering holds. Retries are safe — +/// the server dedups on (series, timestamp). A non-finite value is `DATAHUB_INVALID_ARGUMENT` +/// before anything is sent. `count` of 0 is a no-op. +/// +/// The spool's retention window is measured on each datapoint's own timestamp, not on when it +/// was spooled: a backfill older than the window (72 h by default) is reported `DATAHUB_BUFFERED` +/// but does not survive in the spool. Backfill old data with buffering off, or widen the window. +#[no_mangle] +pub unsafe extern "C" fn datahub_datapoints_insert( + client: *const datahub_client, + external_id: *const c_char, + points: *const datahub_datapoint, + count: usize, +) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let external_id = ffi_try!(nonempty( + ffi_try!(str_arg(external_id, "external_id")), + "external_id" + )); + if count == 0 { + return datahub_status::DATAHUB_OK; + } + if points.is_null() { + return fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("points must not be NULL when count is {count}"), + ); + } + let points = std::slice::from_raw_parts(points, count); + let mut datapoints = Vec::with_capacity(count); + for (i, point) in points.iter().enumerate() { + if !point.value.is_finite() { + return fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("points[{i}].value is not finite"), + ); + } + datapoints.push(DatapointString::new( + &point.timestamp_ms.to_string(), + &point.value.to_string(), + )); + } + send(client, external_id, datapoints) + }) +} + +/// Ingest `count` string-valued datapoints (for text-typed series). `timestamps_ms[i]` pairs +/// with `values[i]`. Same status contract as `datahub_datapoints_insert`. +#[no_mangle] +pub unsafe extern "C" fn datahub_datapoints_insert_str( + client: *const datahub_client, + external_id: *const c_char, + timestamps_ms: *const i64, + values: *const *const c_char, + count: usize, +) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let external_id = ffi_try!(nonempty( + ffi_try!(str_arg(external_id, "external_id")), + "external_id" + )); + if count == 0 { + return datahub_status::DATAHUB_OK; + } + if timestamps_ms.is_null() || values.is_null() { + return fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("timestamps_ms and values must not be NULL when count is {count}"), + ); + } + let timestamps = std::slice::from_raw_parts(timestamps_ms, count); + let mut datapoints = Vec::with_capacity(count); + for (i, timestamp) in timestamps.iter().enumerate() { + let value = ffi_try!(str_arg(*values.add(i), &format!("values[{i}]"))); + datapoints.push(DatapointString::new(×tamp.to_string(), value)); + } + send(client, external_id, datapoints) + }) +} + +/// The most recent datapoint of a series, written into `*out`. `DATAHUB_NOT_FOUND` when the +/// series has no datapoints (or does not exist). +#[no_mangle] +pub unsafe extern "C" fn datahub_datapoints_latest( + client: *const datahub_client, + external_id: *const c_char, + out: *mut datahub_datapoint_agg, +) -> datahub_status { + guard(|| { + let out = ffi_try!(mut_arg(out, "out")); + let client = ffi_try!(ref_arg(client, "client")); + let external_id = ffi_try!(nonempty( + ffi_try!(str_arg(external_id, "external_id")), + "external_id" + )); + + let request = DataWrapper::from_vec(vec![IdAndExtId::from_external_id(external_id)]); + match client.run(client.api.time_series.retrieve_latest_datapoint(&request)) { + Ok(response) => { + let latest = response + .get_items() + .iter() + .flat_map(|collection| collection.datapoints.iter()) + .max_by_key(|dp| dp.timestamp); + match latest { + Some(dp) => { + *out = datahub_datapoint_agg::from_core(dp); + datahub_status::DATAHUB_OK + } + None => fail( + datahub_status::DATAHUB_NOT_FOUND, + response.get_http_status_code().unwrap_or(0), + format!("no datapoints in time series {external_id:?}"), + ), + } + } + Err(e) => from_response_error(&e), + } + }) +} + +/// Raw datapoints of a series inside a window. `start_ms` is inclusive, `end_ms` exclusive; +/// pass `DATAHUB_TIME_UNSET` to leave either end open. `limit` of 0 means the server default. +/// `*out` receives an array of `*out_count` points, newest last, released with +/// `datahub_datapoints_free`; both are 0/NULL when the window is empty. For aggregated reads +/// (`aggregates`, `granularity`, paging with `cursor`) use `datahub_datapoints_retrieve_json`. +#[no_mangle] +pub unsafe extern "C" fn datahub_datapoints_retrieve( + client: *const datahub_client, + external_id: *const c_char, + start_ms: i64, + end_ms: i64, + limit: u64, + out: *mut *mut datahub_datapoint_agg, + out_count: *mut usize, +) -> datahub_status { + guard(|| { + let out = ffi_try!(mut_arg(out, "out")); + let out_count = ffi_try!(mut_arg(out_count, "out_count")); + *out = std::ptr::null_mut(); + *out_count = 0; + let client = ffi_try!(ref_arg(client, "client")); + let external_id = ffi_try!(nonempty( + ffi_try!(str_arg(external_id, "external_id")), + "external_id" + )); + + let bound = |ms: i64, name: &str| -> Result>, datahub_status> { + if ms == TIME_UNSET { + return Ok(None); + } + DateTime::::from_timestamp_millis(ms) + .map(Some) + .ok_or_else(|| { + fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("{name} ({ms}) is not a valid epoch-millisecond timestamp"), + ) + }) + }; + let filter = RetrieveFilter { + start: ffi_try!(bound(start_ms, "start_ms")), + end: ffi_try!(bound(end_ms, "end_ms")), + limit: (limit > 0).then_some(limit), + external_id: Some(external_id.to_string()), + ..Default::default() + }; + let request = DataWrapper::from_vec(vec![filter]); + match client.run(client.api.time_series.retrieve_datapoints(&request)) { + Ok(response) => { + let points: Vec = response + .get_items() + .iter() + .flat_map(|collection| collection.datapoints.iter()) + .map(datahub_datapoint_agg::from_core) + .collect(); + if !points.is_empty() { + let boxed = points.into_boxed_slice(); + *out_count = boxed.len(); + *out = Box::into_raw(boxed) as *mut datahub_datapoint_agg; + } + datahub_status::DATAHUB_OK + } + Err(e) => from_response_error(&e), + } + }) +} + +/// Release an array from `datahub_datapoints_retrieve`, with the count it came with. NULL is ignored. +#[no_mangle] +pub unsafe extern "C" fn datahub_datapoints_free(points: *mut datahub_datapoint_agg, count: usize) { + if !points.is_null() && count > 0 { + let slice: *mut [datahub_datapoint_agg] = std::ptr::slice_from_raw_parts_mut(points, count); + drop(Box::from_raw(slice)); + } +} + +/// `POST /timeseries/data/list` with the full request: `body` is +/// `{"items":[{"externalId":…,"start":…,"end":…,"limit":…,"aggregates":[…],"granularity":…,"cursor":…}]}` +/// and `*out` receives the response envelope as the api sent it. +#[no_mangle] +pub unsafe extern "C" fn datahub_datapoints_retrieve_json( + client: *const datahub_client, + body: *const c_char, + out: *mut *mut c_char, +) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let body = ffi_try!(str_arg(body, "body")); + let request: DataWrapper = + ffi_try!(parse(body, "datapoint retrieve request")); + respond( + out, + client.run(client.api.time_series.retrieve_datapoints(&request)), + false, + ) + }) +} diff --git a/datahub_c_bindings/src/error.rs b/datahub_c_bindings/src/error.rs new file mode 100644 index 0000000..b24ebc4 --- /dev/null +++ b/datahub_c_bindings/src/error.rs @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Status codes, the per-thread last-error slot, and the panic guard every export runs inside. + +use std::cell::{Cell, RefCell}; +use std::ffi::{c_char, c_int, CString}; +use std::panic::{catch_unwind, AssertUnwindSafe}; + +use intellistream_datahub_sdk::errors::DataHubError; +use intellistream_datahub_sdk::http::ResponseError; +use intellistream_datahub_sdk::ListenError; + +use crate::util::to_cstring; + +/// Outcome of a call. Everything except `DATAHUB_OK` and `DATAHUB_BUFFERED` leaves a message +/// in `datahub_last_error()`. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum datahub_status { + /// The call succeeded. For an ingest call this means the data reached the server. + DATAHUB_OK = 0, + /// The data went into the on-disk spool because the server could not be reached, or refused + /// the credential. It is sent on a later ingest call or by `datahub_client_flush`. Not an + /// error: this is the answer an edge device wants when the network is down. + DATAHUB_BUFFERED = 1, + /// `datahub_listener_next`: nothing arrived within the timeout. + DATAHUB_TIMEOUT = 2, + /// `datahub_listener_next`: the stream has ended. + DATAHUB_CLOSED = 3, + /// The api answered with no item where exactly one was asked for. + DATAHUB_NOT_FOUND = 4, + /// A NULL where a value was required, invalid UTF-8, an empty id, a non-finite value, or a + /// JSON body that does not parse as the request the endpoint takes. + DATAHUB_INVALID_ARGUMENT = 10, + /// The configuration is incomplete or contradictory: no BASE_URL, no usable credential set, + /// a malformed URL. + DATAHUB_CONFIG = 11, + /// A token could not be obtained, or the api answered 401 or 403. For a 401 the message + /// includes the SDK's diagnosis of the token's `organization` claim when it has one. + DATAHUB_AUTH = 12, + /// The api answered another non-2xx status, or the request got no response at all; + /// `datahub_last_http_status()` says which. A transport failure (connection refused, DNS, + /// timeout) is reported as 503, the same way the core treats it: retryable. + DATAHUB_HTTP = 13, + /// A local failure: an env file that cannot be read, a runtime that could not start, a + /// WebSocket that could not be opened or was lost for good. + DATAHUB_IO = 14, + /// The listener reported an error for one subscription (unknown id, no read access). The + /// connection stays open and the other subscriptions keep delivering; call + /// `datahub_listener_next` again. + DATAHUB_SUBSCRIPTION = 15, + /// A Rust panic was caught at the boundary. The message is in `datahub_last_error()`; + /// please report it, it is a bug in the SDK. + DATAHUB_PANIC = 99, +} + +thread_local! { + static LAST_ERROR: RefCell = RefCell::new(CString::default()); + static LAST_HTTP_STATUS: Cell = const { Cell::new(0) }; +} + +/// Record a failure for the calling thread and hand back the status to return. +pub(crate) fn fail( + status: datahub_status, + http_status: u16, + message: impl Into, +) -> datahub_status { + let message = message.into(); + LAST_ERROR.with(|slot| *slot.borrow_mut() = to_cstring(&message)); + LAST_HTTP_STATUS.with(|slot| slot.set(http_status as c_int)); + status +} + +/// Map a core `ResponseError` (an HTTP-level failure, or a token that could not be obtained). +pub(crate) fn from_response_error(error: &ResponseError) -> datahub_status { + let code = error.get_status().as_u16(); + let status = if code == 401 || code == 403 { + datahub_status::DATAHUB_AUTH + } else { + datahub_status::DATAHUB_HTTP + }; + fail(status, code, error.to_string()) +} + +/// Map a core `DataHubError` (configuration and token acquisition). +pub(crate) fn from_config_error(error: &DataHubError) -> datahub_status { + let status = match error { + DataHubError::ConfigError(_) | DataHubError::UrlError(_) | DataHubError::JsonError(_) => { + datahub_status::DATAHUB_CONFIG + } + DataHubError::OAuthError(_) => datahub_status::DATAHUB_AUTH, + DataHubError::HttpError(_) => datahub_status::DATAHUB_IO, + }; + fail(status, 0, error.to_string()) +} + +/// Map a listener error. +pub(crate) fn from_listen_error(error: &ListenError) -> datahub_status { + let status = match error { + ListenError::Request(message) if message.contains("api token") => { + datahub_status::DATAHUB_AUTH + } + ListenError::Request(_) => datahub_status::DATAHUB_CONFIG, + ListenError::Handshake(message) if message.contains("401") || message.contains("403") => { + datahub_status::DATAHUB_AUTH + } + ListenError::Handshake(_) | ListenError::WebSocket(_) => datahub_status::DATAHUB_IO, + ListenError::Deserialize(_) | ListenError::Serialize(_) => datahub_status::DATAHUB_HTTP, + ListenError::Subscription { .. } => datahub_status::DATAHUB_SUBSCRIPTION, + }; + fail(status, 0, error.to_string()) +} + +/// Run an FFI function body with panics converted to `DATAHUB_PANIC`. +/// +/// Unwinding across an `extern "C"` boundary is undefined behaviour (and aborts the process on +/// current Rust), so nothing is allowed to escape: the payload's message goes into the last-error +/// slot and the caller gets a status it can log. +pub(crate) fn guard datahub_status>(body: F) -> datahub_status { + match catch_unwind(AssertUnwindSafe(body)) { + Ok(status) => status, + Err(payload) => { + let message = payload + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "panic with a non-string payload".to_string()); + fail( + datahub_status::DATAHUB_PANIC, + 0, + format!("panic inside the DataHub SDK: {message}"), + ) + } + } +} + +/// The message left by the last failing call on this thread, or `""` when there has been none. +/// Borrowed: valid until the next failing call on the same thread. +#[no_mangle] +pub extern "C" fn datahub_last_error() -> *const c_char { + LAST_ERROR.with(|slot| slot.borrow().as_ptr()) +} + +/// The HTTP status of the last `DATAHUB_HTTP` / `DATAHUB_AUTH` / `DATAHUB_NOT_FOUND` failure on +/// this thread, or 0 when the last failure was not an HTTP response. +#[no_mangle] +pub extern "C" fn datahub_last_http_status() -> c_int { + LAST_HTTP_STATUS.with(|slot| slot.get()) +} + +/// The name of a status, e.g. `"DATAHUB_BUFFERED"`, for logging. Static; never NULL. +#[no_mangle] +pub extern "C" fn datahub_status_name(status: datahub_status) -> *const c_char { + let name: &'static str = match status { + datahub_status::DATAHUB_OK => "DATAHUB_OK\0", + datahub_status::DATAHUB_BUFFERED => "DATAHUB_BUFFERED\0", + datahub_status::DATAHUB_TIMEOUT => "DATAHUB_TIMEOUT\0", + datahub_status::DATAHUB_CLOSED => "DATAHUB_CLOSED\0", + datahub_status::DATAHUB_NOT_FOUND => "DATAHUB_NOT_FOUND\0", + datahub_status::DATAHUB_INVALID_ARGUMENT => "DATAHUB_INVALID_ARGUMENT\0", + datahub_status::DATAHUB_CONFIG => "DATAHUB_CONFIG\0", + datahub_status::DATAHUB_AUTH => "DATAHUB_AUTH\0", + datahub_status::DATAHUB_HTTP => "DATAHUB_HTTP\0", + datahub_status::DATAHUB_IO => "DATAHUB_IO\0", + datahub_status::DATAHUB_SUBSCRIPTION => "DATAHUB_SUBSCRIPTION\0", + datahub_status::DATAHUB_PANIC => "DATAHUB_PANIC\0", + }; + name.as_ptr() as *const c_char +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CStr; + + fn last_error() -> String { + unsafe { CStr::from_ptr(datahub_last_error()) } + .to_string_lossy() + .into_owned() + } + + #[test] + fn a_panic_becomes_a_status_with_its_message() { + let status = guard(|| panic!("boom {}", 42)); + assert_eq!(status, datahub_status::DATAHUB_PANIC); + assert_eq!(last_error(), "panic inside the DataHub SDK: boom 42"); + assert_eq!(datahub_last_http_status(), 0); + } + + #[test] + fn a_static_str_panic_payload_is_read_too() { + let status = guard(|| panic!("static")); + assert_eq!(status, datahub_status::DATAHUB_PANIC); + assert!(last_error().ends_with("static")); + } + + #[test] + fn the_error_slot_is_per_thread() { + fail(datahub_status::DATAHUB_CONFIG, 0, "on the main thread"); + let seen_elsewhere = std::thread::spawn(last_error).join().unwrap(); + assert_eq!(seen_elsewhere, "", "a fresh thread starts with no error"); + assert_eq!(last_error(), "on the main thread"); + } + + #[test] + fn a_message_with_an_interior_nul_is_sanitised_not_lost() { + fail(datahub_status::DATAHUB_IO, 0, "before\0after"); + assert_eq!(last_error(), "before after"); + } + + #[test] + fn status_names_are_stable_strings() { + let name = unsafe { CStr::from_ptr(datahub_status_name(datahub_status::DATAHUB_BUFFERED)) }; + assert_eq!(name.to_str().unwrap(), "DATAHUB_BUFFERED"); + } +} diff --git a/datahub_c_bindings/src/events.rs b/datahub_c_bindings/src/events.rs new file mode 100644 index 0000000..99c48ee --- /dev/null +++ b/datahub_c_bindings/src/events.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Events, as JSON: create (buffered like datapoints) and filter. + +use std::ffi::c_char; + +use intellistream_datahub_sdk::filters::EventFilterForm; +use intellistream_datahub_sdk::generic::DataWrapper; +use intellistream_datahub_sdk::Event; + +use crate::client::datahub_client; +use crate::error::{datahub_status, guard}; +use crate::json::{parse, respond}; +use crate::util::{ref_arg, str_arg}; + +/// `POST /events/create`. `body` is `{"items":[{"externalId":…,"type":…,"eventTime":…}, …]}`. +/// `DATAHUB_OK` with the created events in `*out`; with buffering enabled, `DATAHUB_BUFFERED` +/// (and `{"items":[]}`) when they went to the spool instead. Each event is stamped with a +/// time-ordered UUID before the first attempt, so a retry from the spool is not a duplicate. +#[no_mangle] +pub unsafe extern "C" fn datahub_events_create_json( + client: *const datahub_client, + body: *const c_char, + out: *mut *mut c_char, +) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let body = ffi_try!(str_arg(body, "body")); + let request: DataWrapper = ffi_try!(parse(body, "event create request")); + let events: Vec = request.get_items().clone(); + respond(out, client.run(client.api.events.create(&events)), true) + }) +} + +/// `POST /events/filter`. `body` is `{"filter":{…},"limit":…,"cursor":…}`; page with the +/// response's `nextCursor`. +#[no_mangle] +pub unsafe extern "C" fn datahub_events_filter_json( + client: *const datahub_client, + body: *const c_char, + out: *mut *mut c_char, +) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let body = ffi_try!(str_arg(body, "body")); + let form: EventFilterForm = ffi_try!(parse(body, "event filter request")); + respond(out, client.run(client.api.events.filter(&form)), false) + }) +} diff --git a/datahub_c_bindings/src/json.rs b/datahub_c_bindings/src/json.rs new file mode 100644 index 0000000..3fd22ff --- /dev/null +++ b/datahub_c_bindings/src/json.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The JSON convention: a function taking `..._json` accepts exactly the request body the REST +//! endpoint takes and returns exactly the response body it answers with, so the REST API +//! reference doubles as the documentation for it. +//! +//! It is not a raw pass-through. The body is deserialized into the core's typed structs and sent +//! through the same service method Rust and Python callers use — the durable buffer, chunking and +//! auth apply, and a body whose fields have the wrong type is rejected before any request is +//! made — then the typed response is serialized back. + +use std::ffi::c_char; + +use intellistream_datahub_sdk::generic::DataWrapper; +use intellistream_datahub_sdk::http::ResponseError; +use serde::de::DeserializeOwned; +use serde::Serialize; + +use crate::error::{datahub_status, fail, from_response_error}; +use crate::util::out_str; + +/// Parse a request body into the type the endpoint takes. +pub(crate) fn parse(body: &str, what: &str) -> Result { + serde_json::from_str(body).map_err(|e| { + fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("body is not a valid {what}: {e}"), + ) + }) +} + +/// The wire shape of a response: `items`, plus `nextCursor` when there is another page. Built by +/// hand because the core's `DataWrapper` skips `nextCursor` when serializing — it doubles as a +/// request body, where the field does not exist — and a C caller paging through results needs it. +pub(crate) fn wrapper_to_json(wrapper: &DataWrapper) -> String { + let mut value = serde_json::json!({ "items": wrapper.get_items() }); + if let Some(cursor) = wrapper.next_cursor() { + value["nextCursor"] = serde_json::Value::String(cursor.to_string()); + } + value.to_string() +} + +/// Turn a service result into the out-parameter and a status. With `may_buffer`, the core's +/// "accepted into the spool" answer (202, no items) becomes `DATAHUB_BUFFERED`. +pub(crate) unsafe fn respond( + out: *mut *mut c_char, + result: Result, ResponseError>, + may_buffer: bool, +) -> datahub_status { + match result { + Ok(wrapper) => { + let buffered = may_buffer + && wrapper.get_http_status_code() == Some(202) + && wrapper.get_items().is_empty(); + ffi_try!(out_str(out, wrapper_to_json(&wrapper))); + if buffered { + datahub_status::DATAHUB_BUFFERED + } else { + datahub_status::DATAHUB_OK + } + } + Err(e) => from_response_error(&e), + } +} diff --git a/datahub_c_bindings/src/lib.rs b/datahub_c_bindings/src/lib.rs new file mode 100644 index 0000000..a2cd606 --- /dev/null +++ b/datahub_c_bindings/src/lib.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 +//! C ABI for the IntelliStream DataHub SDK. +//! +//! This crate is a thin FFI layer over `intellistream-datahub-sdk`, built as +//! `libintellistream_datahub` (shared and static) with a cbindgen-generated header in +//! `include/`. There is exactly one implementation of every call — the Rust core — which is why +//! this crate contains no HTTP, auth, buffering or WebSocket code of its own. +//! +//! Every exported function is `extern "C"`, `#[no_mangle]`, and runs inside [`error::guard`], so a +//! Rust panic never unwinds into the caller: it becomes `DATAHUB_PANIC` with the panic message +//! in `datahub_last_error()`. +//! +//! The runtime model copies the core's blocking client: a [`datahub_client`] owns a Tokio +//! runtime and every call is `block_on` — but it wraps the async `ApiService` directly rather +//! than the blocking client, because the blocking client deliberately omits the subscription +//! listener and that listener is one of the three things this ABI exists for. +#![allow( + non_camel_case_types, + clippy::missing_safety_doc, + clippy::not_unsafe_ptr_arg_deref +)] + +/// Unwrap a `Result` or return the status from the enclosing FFI function. +macro_rules! ffi_try { + ($e:expr) => { + match $e { + Ok(value) => value, + Err(status) => return status, + } + }; +} + +mod client; +mod config; +mod datapoints; +mod error; +mod events; +mod json; +mod listener; +mod timeseries; +mod util; + +pub use client::*; +pub use config::*; +pub use datapoints::*; +pub use error::*; +pub use events::*; +pub use listener::*; +pub use timeseries::*; +pub use util::*; diff --git a/datahub_c_bindings/src/listener.rs b/datahub_c_bindings/src/listener.rs new file mode 100644 index 0000000..2e4f254 --- /dev/null +++ b/datahub_c_bindings/src/listener.rs @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The subscription listener, pull-based. +//! +//! `datahub_listener_next(timeout)` mirrors the core's `SubscriptionListener::next` rather than +//! delivering through a callback: a callback API has to define which thread it runs on, what it +//! may call and what happens if it blocks; a `next` loop with a timeout defines none of that and +//! is what every C event loop already knows how to drive. +//! +//! A listener keeps the runtime and the api service alive on its own, so it does not matter in +//! which order it and its client are released. + +use std::ffi::{c_char, CString}; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use chrono::DateTime; +use intellistream_datahub_sdk::{ + ApiService, ListenError, SubscriptionListener, SubscriptionMessage, +}; +use tokio::runtime::Runtime; + +use crate::client::datahub_client; +use crate::datapoints::datahub_datapoint; +use crate::error::{datahub_status, fail, from_listen_error, guard}; +use crate::util::{mut_arg, ref_arg, str_array_arg, to_cstring}; + +/// A live WebSocket listener over one or more subscriptions. Opaque; one thread at a time; close +/// (and free) with `datahub_listener_close`. +pub struct datahub_listener { + inner: SubscriptionListener, + rt: Arc, + _api: Arc, +} + +struct Series { + external_id: Option, + id: u64, + points: Vec, +} + +/// One message delivered to a listener. Opaque; free with `datahub_message_free` after acking. +pub struct datahub_message { + message_id: CString, + subscription: CString, + action: CString, + object: CString, + json: CString, + series: Vec, +} + +/// Stream timestamps arrive as strings; both epoch milliseconds and RFC 3339 are understood. +fn parse_stream_timestamp(text: &str) -> i64 { + if let Ok(ms) = text.parse::() { + return ms; + } + DateTime::parse_from_rfc3339(text) + .map(|t| t.timestamp_millis()) + .unwrap_or(i64::MIN) +} + +/// The wire name of a serde enum value (`CREATE`, `DATAPOINTS`, …). +fn enum_name(value: &T) -> String { + serde_json::to_value(value) + .ok() + .and_then(|v| v.as_str().map(str::to_string)) + .unwrap_or_default() +} + +impl datahub_message { + fn wrap(message: SubscriptionMessage) -> Self { + let series = message + .payload + .items + .iter() + .map(|collection| Series { + external_id: collection.external_id.as_deref().map(to_cstring), + id: collection.id.unwrap_or(0), + points: collection + .datapoints + .iter() + .map(|dp| datahub_datapoint { + timestamp_ms: parse_stream_timestamp(&dp.timestamp), + value: dp.value.parse::().unwrap_or(f64::NAN), + }) + .collect(), + }) + .collect(); + let json = serde_json::json!({ + "subscriptionExternalId": message.subscription_external_id, + "messageId": message.message_id, + "payload": message.payload, + }); + datahub_message { + message_id: to_cstring(&message.message_id), + subscription: to_cstring(&message.subscription_external_id), + action: to_cstring(&enum_name(&message.payload.event_action)), + object: to_cstring(&enum_name(&message.payload.event_object)), + json: to_cstring(&json.to_string()), + series, + } + } +} + +/// Open a listener on `count` subscription external ids (0 is allowed; add them later with +/// `datahub_listener_subscribe`). The handshake fetches a token through the client. On a dropped +/// connection the listener reconnects on its own with backoff and resumes the same +/// subscriptions; anything not acked is redelivered. +#[no_mangle] +pub unsafe extern "C" fn datahub_listener_open( + client: *const datahub_client, + subscription_external_ids: *const *const c_char, + count: usize, + out: *mut *mut datahub_listener, +) -> datahub_status { + guard(|| { + let out = ffi_try!(mut_arg(out, "out")); + *out = std::ptr::null_mut(); + let client = ffi_try!(ref_arg(client, "client")); + let ids = ffi_try!(str_array_arg( + subscription_external_ids, + count, + "subscription_external_ids" + )); + match client.run(client.api.subscriptions.listen(&ids)) { + Ok(inner) => { + *out = Box::into_raw(Box::new(datahub_listener { + inner, + rt: client.rt.clone(), + _api: client.api.clone(), + })); + datahub_status::DATAHUB_OK + } + Err(e) => from_listen_error(&e), + } + }) +} + +/// Wait up to `timeout_ms` for the next message (negative waits indefinitely, 0 only takes what +/// has already arrived). `DATAHUB_OK` with the message in `*out`; `DATAHUB_TIMEOUT` with `*out` +/// NULL when nothing came; `DATAHUB_SUBSCRIPTION` when the server rejected one subscription (the +/// others keep delivering, call again); `DATAHUB_IO` when the connection was lost and could not +/// be re-established after several attempts (calling again retries). Call this often enough for +/// the server's 15 s pings to be answered, or the session is closed as idle and reconnected. +#[no_mangle] +pub unsafe extern "C" fn datahub_listener_next( + listener: *mut datahub_listener, + timeout_ms: i64, + out: *mut *mut datahub_message, +) -> datahub_status { + guard(|| { + let out = ffi_try!(mut_arg(out, "out")); + *out = std::ptr::null_mut(); + let listener = ffi_try!(mut_arg(listener, "listener")); + let rt = listener.rt.clone(); + let result = if timeout_ms < 0 { + rt.block_on(listener.inner.next()) + } else { + let wait = Duration::from_millis(timeout_ms as u64); + match rt.block_on(tokio::time::timeout(wait, listener.inner.next())) { + Ok(result) => result, + Err(_) => return datahub_status::DATAHUB_TIMEOUT, + } + }; + match result { + None => datahub_status::DATAHUB_CLOSED, + Some(Ok(message)) => { + *out = Box::into_raw(Box::new(datahub_message::wrap(message))); + datahub_status::DATAHUB_OK + } + Some(Err(e)) => from_listen_error(&e), + } + }) +} + +type IdCall = for<'a> fn( + &'a mut SubscriptionListener, + &'a [String], +) -> Pin> + 'a>>; + +/// The body the four id-list calls share: check the ids, run the call on the listener's runtime. +unsafe fn with_ids( + listener: *mut datahub_listener, + ids: *const *const c_char, + count: usize, + what: &str, + call: IdCall, +) -> datahub_status { + guard(|| { + let listener = ffi_try!(mut_arg(listener, "listener")); + let ids = ffi_try!(str_array_arg(ids, count, what)); + if ids.is_empty() { + return fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("{what} must name at least one id"), + ); + } + let rt = listener.rt.clone(); + match rt.block_on(call(&mut listener.inner, &ids)) { + Ok(()) => datahub_status::DATAHUB_OK, + Err(e) => from_listen_error(&e), + } + }) +} + +/// Acknowledge `count` message ids so they are not redelivered. +#[no_mangle] +pub unsafe extern "C" fn datahub_listener_ack( + listener: *mut datahub_listener, + message_ids: *const *const c_char, + count: usize, +) -> datahub_status { + with_ids(listener, message_ids, count, "message_ids", |l, ids| { + Box::pin(l.ack(ids)) + }) +} + +/// Negative-acknowledge `count` message ids so they are redelivered. +#[no_mangle] +pub unsafe extern "C" fn datahub_listener_nack( + listener: *mut datahub_listener, + message_ids: *const *const c_char, + count: usize, +) -> datahub_status { + with_ids(listener, message_ids, count, "message_ids", |l, ids| { + Box::pin(l.nack(ids)) + }) +} + +/// Add `count` subscriptions to the live set without reconnecting. +#[no_mangle] +pub unsafe extern "C" fn datahub_listener_subscribe( + listener: *mut datahub_listener, + subscription_external_ids: *const *const c_char, + count: usize, +) -> datahub_status { + with_ids( + listener, + subscription_external_ids, + count, + "subscription_external_ids", + |l, ids| Box::pin(l.subscribe(ids)), + ) +} + +/// Remove `count` subscriptions from the live set. +#[no_mangle] +pub unsafe extern "C" fn datahub_listener_unsubscribe( + listener: *mut datahub_listener, + subscription_external_ids: *const *const c_char, + count: usize, +) -> datahub_status { + with_ids( + listener, + subscription_external_ids, + count, + "subscription_external_ids", + |l, ids| Box::pin(l.unsubscribe(ids)), + ) +} + +/// Send a close frame, wait for the server's, and free the listener. Unacked messages are +/// redelivered to the next listener on the same subscriptions. NULL is ignored. +#[no_mangle] +pub unsafe extern "C" fn datahub_listener_close(listener: *mut datahub_listener) -> datahub_status { + guard(|| { + if listener.is_null() { + return datahub_status::DATAHUB_OK; + } + let datahub_listener { inner, rt, _api } = *Box::from_raw(listener); + match rt.block_on(inner.close()) { + Ok(()) => datahub_status::DATAHUB_OK, + Err(e) => from_listen_error(&e), + } + }) +} + +unsafe fn message_field( + message: *const datahub_message, + pick: fn(&datahub_message) -> &CString, +) -> *const c_char { + message + .as_ref() + .map_or(std::ptr::null(), |m| pick(m).as_ptr()) +} + +/// The id to pass to `datahub_listener_ack`/`_nack`. Borrowed; valid until the message is freed. +#[no_mangle] +pub unsafe extern "C" fn datahub_message_id(message: *const datahub_message) -> *const c_char { + message_field(message, |m| &m.message_id) +} + +/// The subscription this message was delivered for. Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_message_subscription( + message: *const datahub_message, +) -> *const c_char { + message_field(message, |m| &m.subscription) +} + +/// What happened: `CREATE`, `UPDATE`, `DELETE` or `RENAME`. Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_message_action(message: *const datahub_message) -> *const c_char { + message_field(message, |m| &m.action) +} + +/// What it happened to: `DATAPOINTS`, `TIMESERIES`, `EVENT`, `RESOURCE`, …. Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_message_object(message: *const datahub_message) -> *const c_char { + message_field(message, |m| &m.object) +} + +/// The whole message as JSON (`subscriptionExternalId`, `messageId`, `payload`). Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_message_json(message: *const datahub_message) -> *const c_char { + message_field(message, |m| &m.json) +} + +/// How many series this message carries datapoints for (0 unless the object is `DATAPOINTS`). +#[no_mangle] +pub unsafe extern "C" fn datahub_message_series_count(message: *const datahub_message) -> usize { + message.as_ref().map_or(0, |m| m.series.len()) +} + +/// External id of the `index`-th series, or NULL when the message named it by id only. Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_message_series_external_id( + message: *const datahub_message, + index: usize, +) -> *const c_char { + message + .as_ref() + .and_then(|m| m.series.get(index)) + .and_then(|s| s.external_id.as_ref()) + .map_or(std::ptr::null(), |id| id.as_ptr()) +} + +/// Numeric id of the `index`-th series, or 0 when the message did not carry one. +#[no_mangle] +pub unsafe extern "C" fn datahub_message_series_id( + message: *const datahub_message, + index: usize, +) -> u64 { + message + .as_ref() + .and_then(|m| m.series.get(index)) + .map_or(0, |s| s.id) +} + +/// The datapoints of the `index`-th series as a borrowed array (valid until the message is +/// freed). A value that is not numeric reads as NaN, a timestamp that could not be parsed as +/// `INT64_MIN`; the JSON has the originals. +#[no_mangle] +pub unsafe extern "C" fn datahub_message_series_datapoints( + message: *const datahub_message, + index: usize, + out: *mut *const datahub_datapoint, + out_count: *mut usize, +) -> datahub_status { + guard(|| { + let out = ffi_try!(mut_arg(out, "out")); + let out_count = ffi_try!(mut_arg(out_count, "out_count")); + *out = std::ptr::null(); + *out_count = 0; + let message = ffi_try!(ref_arg(message, "message")); + match message.series.get(index) { + Some(series) => { + *out = series.points.as_ptr(); + *out_count = series.points.len(); + datahub_status::DATAHUB_OK + } + None => fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!( + "series index {index} is out of range (message carries {})", + message.series.len() + ), + ), + } + }) +} + +/// Release a message. NULL is ignored. +#[no_mangle] +pub unsafe extern "C" fn datahub_message_free(message: *mut datahub_message) { + if !message.is_null() { + drop(Box::from_raw(message)); + } +} + +#[cfg(test)] +mod tests { + use super::parse_stream_timestamp; + + #[test] + fn stream_timestamps_come_as_millis_or_rfc3339() { + assert_eq!(parse_stream_timestamp("1700000000000"), 1_700_000_000_000); + assert_eq!(parse_stream_timestamp("1970-01-01T00:00:01Z"), 1000); + assert_eq!( + parse_stream_timestamp("1970-01-01T00:00:01.250+00:00"), + 1250 + ); + assert_eq!(parse_stream_timestamp("not a time"), i64::MIN); + } +} diff --git a/datahub_c_bindings/src/timeseries.rs b/datahub_c_bindings/src/timeseries.rs new file mode 100644 index 0000000..435a7e6 --- /dev/null +++ b/datahub_c_bindings/src/timeseries.rs @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Time series: a typed lookup by external id, and the JSON create/search/filter calls. + +use std::ffi::{c_char, CString}; + +use intellistream_datahub_sdk::generic::{DataWrapper, IdAndExtId, SearchAndFilterForm}; +use intellistream_datahub_sdk::{TimeSeries, TimeSeriesFilter, TimeSeriesFilterForm}; + +use crate::client::datahub_client; +use crate::error::{datahub_status, fail, from_response_error, guard}; +use crate::json::{parse, respond}; +use crate::util::{mut_arg, nonempty, ref_arg, str_arg, to_cstring}; + +/// One time series definition. Opaque; read it through the `datahub_timeseries_*` accessors and +/// free it with `datahub_timeseries_free`. +pub struct datahub_timeseries { + id: u64, + data_set_id: u64, + external_id: CString, + name: CString, + unit: Option, + unit_external_id: Option, + value_type: Option, + json: CString, +} + +impl datahub_timeseries { + fn wrap(ts: &TimeSeries) -> Self { + datahub_timeseries { + id: ts.id.unwrap_or(0), + data_set_id: ts.data_set_id.unwrap_or(0), + external_id: to_cstring(&ts.external_id), + name: to_cstring(&ts.name), + unit: ts.unit.as_deref().map(to_cstring), + unit_external_id: ts.unit_external_id.as_deref().map(to_cstring), + value_type: ts.value_type.as_deref().map(to_cstring), + json: to_cstring(&serde_json::to_string(ts).unwrap_or_default()), + } + } +} + +/// Look one time series up by external id. `DATAHUB_NOT_FOUND` when the api knows no such series +/// (or the caller may not read it — the api does not distinguish). +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_get_by_external_id( + client: *const datahub_client, + external_id: *const c_char, + out: *mut *mut datahub_timeseries, +) -> datahub_status { + guard(|| { + let out = ffi_try!(mut_arg(out, "out")); + *out = std::ptr::null_mut(); + let client = ffi_try!(ref_arg(client, "client")); + let external_id = ffi_try!(nonempty( + ffi_try!(str_arg(external_id, "external_id")), + "external_id" + )); + + let request = DataWrapper::from_vec(vec![IdAndExtId::from_external_id(external_id)]); + match client.run(client.api.time_series.by_ids(&request)) { + Ok(response) => match response.get_items().first() { + Some(ts) => { + *out = Box::into_raw(Box::new(datahub_timeseries::wrap(ts))); + datahub_status::DATAHUB_OK + } + None => fail( + datahub_status::DATAHUB_NOT_FOUND, + response.get_http_status_code().unwrap_or(0), + format!("no time series with external id {external_id:?}"), + ), + }, + Err(e) => from_response_error(&e), + } + }) +} + +/// Release a time series handle. NULL is ignored. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_free(ts: *mut datahub_timeseries) { + if !ts.is_null() { + drop(Box::from_raw(ts)); + } +} + +/// Numeric id; 0 when the api did not send one. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_id(ts: *const datahub_timeseries) -> u64 { + ts.as_ref().map_or(0, |ts| ts.id) +} + +/// Numeric id of the data set the series belongs to; 0 when it has none. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_data_set_id(ts: *const datahub_timeseries) -> u64 { + ts.as_ref().map_or(0, |ts| ts.data_set_id) +} + +unsafe fn required( + ts: *const datahub_timeseries, + pick: fn(&datahub_timeseries) -> &CString, +) -> *const c_char { + ts.as_ref().map_or(std::ptr::null(), |ts| pick(ts).as_ptr()) +} + +unsafe fn optional( + ts: *const datahub_timeseries, + pick: fn(&datahub_timeseries) -> Option<&CString>, +) -> *const c_char { + ts.as_ref() + .and_then(pick) + .map_or(std::ptr::null(), |value| value.as_ptr()) +} + +/// External id. Borrowed; valid until the handle is freed. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_external_id( + ts: *const datahub_timeseries, +) -> *const c_char { + required(ts, |ts| &ts.external_id) +} + +/// Display name. Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_name(ts: *const datahub_timeseries) -> *const c_char { + required(ts, |ts| &ts.name) +} + +/// Unit symbol, or NULL when the series has none. Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_unit(ts: *const datahub_timeseries) -> *const c_char { + optional(ts, |ts| ts.unit.as_ref()) +} + +/// Unit catalogue id, or NULL. Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_unit_external_id( + ts: *const datahub_timeseries, +) -> *const c_char { + optional(ts, |ts| ts.unit_external_id.as_ref()) +} + +/// Value type (`float`, `bigint`, `text`, …), or NULL when the endpoint did not say. Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_value_type( + ts: *const datahub_timeseries, +) -> *const c_char { + optional(ts, |ts| ts.value_type.as_ref()) +} + +/// The whole definition as the api sent it, as JSON. Borrowed. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_json(ts: *const datahub_timeseries) -> *const c_char { + required(ts, |ts| &ts.json) +} + +/// `POST /timeseries/create`. `body` is `{"items":[{"externalId":…,"name":…,"unit":…}, …]}`; +/// `*out` receives the created definitions in the same envelope. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_create_json( + client: *const datahub_client, + body: *const c_char, + out: *mut *mut c_char, +) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let body = ffi_try!(str_arg(body, "body")); + let request: DataWrapper = ffi_try!(parse(body, "timeseries create request")); + respond( + out, + client.run(client.api.time_series.create(&request)), + false, + ) + }) +} + +/// `POST /timeseries/search`: free-text search with optional narrowing. `body` is +/// `{"search":{"query":…},"filter":{…},"limit":…}`. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_search_json( + client: *const datahub_client, + body: *const c_char, + out: *mut *mut c_char, +) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let body = ffi_try!(str_arg(body, "body")); + let form: SearchAndFilterForm = + ffi_try!(parse(body, "timeseries search request")); + respond(out, client.run(client.api.time_series.search(&form)), false) + }) +} + +/// `POST /timeseries/filter`: structured, AND-combined filtering. `body` is +/// `{"filter":{…},"limit":…,"cursor":…}`; page with the response's `nextCursor`. +#[no_mangle] +pub unsafe extern "C" fn datahub_timeseries_filter_json( + client: *const datahub_client, + body: *const c_char, + out: *mut *mut c_char, +) -> datahub_status { + guard(|| { + let client = ffi_try!(ref_arg(client, "client")); + let body = ffi_try!(str_arg(body, "body")); + let form: TimeSeriesFilterForm = ffi_try!(parse(body, "timeseries filter request")); + respond(out, client.run(client.api.time_series.filter(&form)), false) + }) +} diff --git a/datahub_c_bindings/src/util.rs b/datahub_c_bindings/src/util.rs new file mode 100644 index 0000000..9cf037c --- /dev/null +++ b/datahub_c_bindings/src/util.rs @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Argument checking and string ownership at the boundary. + +use std::ffi::{c_char, CStr, CString}; + +use crate::error::{datahub_status, fail}; + +/// A `CString` from arbitrary text: an interior NUL would make `CString::new` fail, so it is +/// replaced rather than losing the message. +pub(crate) fn to_cstring(text: &str) -> CString { + CString::new(text.replace('\0', " ")).expect("NULs were just removed") +} + +/// Borrow a required C string as `&str`. +pub(crate) unsafe fn str_arg<'a>( + ptr: *const c_char, + name: &str, +) -> Result<&'a str, datahub_status> { + if ptr.is_null() { + return Err(fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("{name} must not be NULL"), + )); + } + CStr::from_ptr(ptr).to_str().map_err(|e| { + fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("{name} is not valid UTF-8: {e}"), + ) + }) +} + +/// Borrow an optional C string; NULL is `None`. +pub(crate) unsafe fn opt_str_arg<'a>( + ptr: *const c_char, + name: &str, +) -> Result, datahub_status> { + if ptr.is_null() { + Ok(None) + } else { + str_arg(ptr, name).map(Some) + } +} + +/// A required string that must also carry something. +pub(crate) fn nonempty<'a>(value: &'a str, name: &str) -> Result<&'a str, datahub_status> { + if value.trim().is_empty() { + Err(fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("{name} must not be empty"), + )) + } else { + Ok(value) + } +} + +/// Borrow an array of `count` C strings. +pub(crate) unsafe fn str_array_arg( + ptrs: *const *const c_char, + count: usize, + name: &str, +) -> Result, datahub_status> { + if count == 0 { + return Ok(Vec::new()); + } + if ptrs.is_null() { + return Err(fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("{name} must not be NULL when its count is {count}"), + )); + } + let mut out = Vec::with_capacity(count); + for i in 0..count { + let item = str_arg(*ptrs.add(i), &format!("{name}[{i}]"))?; + out.push(item.to_string()); + } + Ok(out) +} + +/// Borrow a required handle. +pub(crate) unsafe fn ref_arg<'a, T>(ptr: *const T, name: &str) -> Result<&'a T, datahub_status> { + ptr.as_ref().ok_or_else(|| { + fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("{name} must not be NULL"), + ) + }) +} + +/// Borrow a required handle mutably. +pub(crate) unsafe fn mut_arg<'a, T>(ptr: *mut T, name: &str) -> Result<&'a mut T, datahub_status> { + ptr.as_mut().ok_or_else(|| { + fail( + datahub_status::DATAHUB_INVALID_ARGUMENT, + 0, + format!("{name} must not be NULL"), + ) + }) +} + +/// Hand an owned string to the caller through a `char **` out-parameter. +pub(crate) unsafe fn out_str(out: *mut *mut c_char, text: String) -> Result<(), datahub_status> { + let slot = mut_arg(out, "out")?; + *slot = to_cstring(&text).into_raw(); + Ok(()) +} + +/// Release a string the library handed out through a `char **` out-parameter. NULL is ignored. +#[no_mangle] +pub unsafe extern "C" fn datahub_string_free(text: *mut c_char) { + if !text.is_null() { + drop(CString::from_raw(text)); + } +} diff --git a/datahub_c_bindings/tests/c/smoke.c b/datahub_c_bindings/tests/c/smoke.c new file mode 100644 index 0000000..9a706c7 --- /dev/null +++ b/datahub_c_bindings/tests/c/smoke.c @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: Apache-2.0 */ +/* + * Smoke test of the C ABI as C sees it: compiled by run_c_tests.sh against the built library and + * run without a backend. It exercises the header (types, macros, every documented NULL + * tolerance), the error channel, and the one behaviour an edge device relies on most — that an + * unreachable server spools to disk and reports DATAHUB_BUFFERED. + */ +#define _DEFAULT_SOURCE 1 /* mkdtemp under -std=c11 */ +#include + +#include +#include +#include +#include +#include +#include + +static int failures = 0; + +#define CHECK(cond) \ + do { \ + if (cond) { \ + printf("ok %s\n", #cond); \ + } else { \ + printf("FAIL %s (line %d): %s\n", #cond, __LINE__, datahub_last_error()); \ + failures++; \ + } \ + } while (0) + +/* A port nothing listens on: the connection is refused at once. */ +#define UNREACHABLE "http://127.0.0.1:9" + +int main(void) { + /* --- version and error channel ------------------------------------------------------ */ + CHECK(datahub_version() != NULL); + CHECK(strcmp(datahub_version(), DATAHUB_VERSION) == 0); + CHECK(DATAHUB_VERSION_MAJOR >= 0 && DATAHUB_VERSION_MINOR >= 0 && DATAHUB_VERSION_PATCH >= 0); + CHECK(datahub_last_error() != NULL); + CHECK(strcmp(datahub_last_error(), "") == 0); + CHECK(strcmp(datahub_status_name(DATAHUB_BUFFERED), "DATAHUB_BUFFERED") == 0); + CHECK(DATAHUB_TIME_UNSET == INT64_MIN); + + /* --- NULL tolerance ----------------------------------------------------------------- */ + datahub_client *client = NULL; + CHECK(datahub_client_new(NULL, &client) == DATAHUB_INVALID_ARGUMENT); + CHECK(client == NULL); + CHECK(strcmp(datahub_last_error(), "config must not be NULL") == 0); + datahub_string_free(NULL); + datahub_config_free(NULL); + datahub_client_free(NULL); + datahub_timeseries_free(NULL); + datahub_message_free(NULL); + datahub_datapoints_free(NULL, 0); + CHECK(datahub_listener_close(NULL) == DATAHUB_OK); + CHECK(datahub_client_buffered_count(NULL) == 0); + + /* --- configuration errors ----------------------------------------------------------- */ + datahub_config *cfg = datahub_config_new(); + CHECK(cfg != NULL); + CHECK(datahub_client_new(cfg, &client) == DATAHUB_CONFIG); + CHECK(client == NULL); + CHECK(strstr(datahub_last_error(), "BASE_URL") != NULL); + CHECK(datahub_config_set(cfg, "SCOPE", "organization:*") == DATAHUB_OK); + CHECK(strcmp(datahub_config_get(cfg, "SCOPE"), "organization:*") == 0); + CHECK(datahub_config_get(cfg, "AUDIENCE") == NULL); + CHECK(datahub_config_set_buffer_max_bytes(cfg, 0) == DATAHUB_INVALID_ARGUMENT); + CHECK(datahub_config_load_envfile(cfg, "/nonexistent/gateway.env") == DATAHUB_IO); + + /* --- a client against an unreachable server, spooling to a temp dir ----------------- */ + char spool_dir[] = "/tmp/datahub-c-smoke-XXXXXX"; + CHECK(mkdtemp(spool_dir) != NULL); + CHECK(datahub_config_set_base_url(cfg, UNREACHABLE) == DATAHUB_OK); + CHECK(datahub_config_set_token(cfg, "smoke-token") == DATAHUB_OK); + CHECK(datahub_config_set_buffer_dir(cfg, spool_dir) == DATAHUB_OK); + CHECK(datahub_client_new(cfg, &client) == DATAHUB_OK); + CHECK(client != NULL); + datahub_config_free(cfg); /* the client copied what it needs */ + + /* Recent timestamps: the spool keeps a record only while it is inside the retention window. */ + int64_t now_ms = (int64_t)time(NULL) * 1000; + datahub_datapoint points[2] = { + { .timestamp_ms = now_ms, .value = 21.5 }, + { .timestamp_ms = now_ms + 1000, .value = 21.6 }, + }; + CHECK(datahub_datapoints_insert(client, "pump-1/temperature", points, 2) == DATAHUB_BUFFERED); + CHECK(datahub_client_buffered_count(client) == 2); + CHECK(datahub_client_flush(client) == DATAHUB_BUFFERED); + CHECK(datahub_client_buffered_count(client) == 2); + + /* Argument validation happens before anything is spooled or sent. */ + datahub_datapoint bad = { .timestamp_ms = 1, .value = NAN }; + CHECK(datahub_datapoints_insert(client, "pump-1/temperature", &bad, 1) == DATAHUB_INVALID_ARGUMENT); + CHECK(strcmp(datahub_last_error(), "points[0].value is not finite") == 0); + CHECK(datahub_datapoints_insert(client, "", points, 2) == DATAHUB_INVALID_ARGUMENT); + CHECK(datahub_datapoints_insert(client, "pump-1/temperature", NULL, 2) == DATAHUB_INVALID_ARGUMENT); + CHECK(datahub_datapoints_insert(client, "pump-1/temperature", NULL, 0) == DATAHUB_OK); + CHECK(datahub_client_buffered_count(client) == 2); + + /* Reads are not buffered: a transport failure is an HTTP 503. */ + datahub_timeseries *ts = NULL; + CHECK(datahub_timeseries_get_by_external_id(client, "pump-1/temperature", &ts) == DATAHUB_HTTP); + CHECK(ts == NULL); + CHECK(datahub_last_http_status() == 503); + CHECK(strncmp(datahub_last_error(), "503", 3) == 0); + + char *body = NULL; + CHECK(datahub_request_json(client, "PUT", "/events", NULL, &body) == DATAHUB_INVALID_ARGUMENT); + CHECK(body == NULL); + CHECK(datahub_request_json(client, "POST", "/events/create", "{oops", &body) == DATAHUB_INVALID_ARGUMENT); + CHECK(datahub_events_create_json(client, "{\"items\":[{\"externalId\":\"e\"}]}", &body) == DATAHUB_INVALID_ARGUMENT); + CHECK(strstr(datahub_last_error(), "event create request") != NULL); + + datahub_client_free(client); + + /* The spool survived the client: a new one on the same directory still holds the backlog. */ + cfg = datahub_config_new(); + datahub_config_set_base_url(cfg, UNREACHABLE); + datahub_config_set_token(cfg, "smoke-token"); + datahub_config_set_buffer_dir(cfg, spool_dir); + CHECK(datahub_client_new(cfg, &client) == DATAHUB_OK); + datahub_config_free(cfg); + CHECK(datahub_client_flush(client) == DATAHUB_BUFFERED); + CHECK(datahub_client_buffered_count(client) == 2); + datahub_client_free(client); + + char cleanup[sizeof(spool_dir) + 16]; + snprintf(cleanup, sizeof cleanup, "rm -rf '%s'", spool_dir); + if (system(cleanup) != 0) { + printf("warning: could not remove %s\n", spool_dir); + } + + if (failures == 0) { + printf("smoke test passed\n"); + return 0; + } + printf("%d check(s) failed\n", failures); + return 1; +} diff --git a/datahub_c_bindings/tests/common/mod.rs b/datahub_c_bindings/tests/common/mod.rs new file mode 100644 index 0000000..17a581c --- /dev/null +++ b/datahub_c_bindings/tests/common/mod.rs @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Shared test scaffolding: a tiny HTTP/1.1 mock of the api, and RAII wrappers over the C handles +//! so a test reads like the C it stands in for without leaking on an assertion failure. +#![allow(dead_code)] + +use std::ffi::{c_char, CStr, CString}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::Duration; + +use intellistream_datahub::*; + +// --------------------------------------------------------------------------------------------- +// Mock api +// --------------------------------------------------------------------------------------------- + +/// One request the mock received. +#[derive(Clone, Debug)] +pub struct Request { + pub method: String, + pub path: String, + pub headers: Vec<(String, String)>, + pub body: String, +} + +impl Request { + pub fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.as_str()) + } + + pub fn json(&self) -> serde_json::Value { + serde_json::from_str(&self.body) + .unwrap_or_else(|e| panic!("body is not JSON ({e}): {}", self.body)) + } +} + +type Handler = dyn Fn(&Request) -> (u16, String) + Send + Sync; + +/// A single-threaded HTTP/1.1 server answering from a handler, recording every request. It +/// closes each connection after answering, which reqwest handles without complaint. +pub struct MockServer { + pub base_url: String, + requests: Arc>>, + stop: Arc, + thread: Option>, +} + +impl MockServer { + pub fn start(handler: impl Fn(&Request) -> (u16, String) + Send + Sync + 'static) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + listener.set_nonblocking(true).expect("nonblocking"); + let port = listener.local_addr().unwrap().port(); + let requests: Arc>> = Arc::default(); + let stop = Arc::new(AtomicBool::new(false)); + let handler: Arc = Arc::new(handler); + let thread = { + let (requests, stop) = (requests.clone(), stop.clone()); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + match listener.accept() { + Ok((stream, _)) => serve(stream, handler.as_ref(), &requests), + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(2)); + } + Err(_) => break, + } + } + }) + }; + MockServer { + base_url: format!("http://127.0.0.1:{port}"), + requests, + stop, + thread: Some(thread), + } + } + + pub fn requests(&self) -> Vec { + self.requests.lock().unwrap().clone() + } + + pub fn last_request(&self) -> Request { + self.requests().pop().expect("the mock received no request") + } +} + +impl Drop for MockServer { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn serve(mut stream: TcpStream, handler: &Handler, requests: &Mutex>) { + stream.set_nonblocking(false).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut line = String::new(); + if reader.read_line(&mut line).unwrap_or(0) == 0 { + return; + } + let mut parts = line.split_whitespace(); + let method = parts.next().unwrap_or_default().to_string(); + let path = parts.next().unwrap_or_default().to_string(); + let mut headers = Vec::new(); + let mut content_length = 0usize; + loop { + let mut header = String::new(); + if reader.read_line(&mut header).unwrap_or(0) == 0 || header.trim().is_empty() { + break; + } + if let Some((name, value)) = header.trim_end().split_once(':') { + let (name, value) = (name.trim().to_string(), value.trim().to_string()); + if name.eq_ignore_ascii_case("content-length") { + content_length = value.parse().unwrap_or(0); + } + headers.push((name, value)); + } + } + let mut body = vec![0u8; content_length]; + if content_length > 0 { + reader.read_exact(&mut body).unwrap(); + } + let request = Request { + method, + path, + headers, + body: String::from_utf8_lossy(&body).into_owned(), + }; + let (status, response_body) = handler(&request); + requests.lock().unwrap().push(request); + let reason = match status { + 204 => "No Content", + 200 => "OK", + _ => "Status", + }; + let _ = write!( + stream, + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + response_body.len(), + response_body + ); + let _ = stream.flush(); +} + +/// A handler answering every request the same way. +pub fn always( + status: u16, + body: &str, +) -> impl Fn(&Request) -> (u16, String) + Send + Sync + 'static { + let body = body.to_string(); + move |_| (status, body.clone()) +} + +// --------------------------------------------------------------------------------------------- +// C-side helpers +// --------------------------------------------------------------------------------------------- + +pub fn cstr(text: &str) -> CString { + CString::new(text).unwrap() +} + +/// The last error on this thread, as a Rust string. +pub fn last_error() -> String { + unsafe { CStr::from_ptr(datahub_last_error()) } + .to_string_lossy() + .into_owned() +} + +/// Take ownership of a string the library handed out and free it. +pub unsafe fn take_string(ptr: *mut c_char) -> String { + assert!(!ptr.is_null(), "the library handed out a NULL string"); + let text = CStr::from_ptr(ptr).to_string_lossy().into_owned(); + datahub_string_free(ptr); + text +} + +pub unsafe fn borrow_string(ptr: *const c_char) -> Option { + if ptr.is_null() { + None + } else { + Some(CStr::from_ptr(ptr).to_string_lossy().into_owned()) + } +} + +/// RAII over `datahub_config`. +pub struct Config(pub *mut datahub_config); + +impl Config { + pub fn new() -> Self { + Config(datahub_config_new()) + } + + /// A config pointed at `base_url` with a static token, optionally spooling to `buffer_dir`. + pub fn for_server(base_url: &str, buffer_dir: Option<&std::path::Path>) -> Self { + let config = Config::new(); + config.set("BASE_URL", base_url); + config.set("TOKEN", "test-token"); + if let Some(dir) = buffer_dir { + let dir = cstr(dir.to_str().unwrap()); + assert_eq!( + unsafe { datahub_config_set_buffer_dir(config.0, dir.as_ptr()) }, + datahub_status::DATAHUB_OK + ); + } + config + } + + pub fn set(&self, key: &str, value: &str) { + let (key, value) = (cstr(key), cstr(value)); + assert_eq!( + unsafe { datahub_config_set(self.0, key.as_ptr(), value.as_ptr()) }, + datahub_status::DATAHUB_OK + ); + } + + pub fn get(&self, key: &str) -> Option { + let key = cstr(key); + unsafe { borrow_string(datahub_config_get(self.0, key.as_ptr())) } + } + + pub fn build(&self) -> Result { + let mut out = std::ptr::null_mut(); + match unsafe { datahub_client_new(self.0, &mut out) } { + datahub_status::DATAHUB_OK => { + assert!(!out.is_null()); + Ok(Client(out)) + } + status => { + assert!( + out.is_null(), + "a failed datahub_client_new must leave *out NULL" + ); + Err(status) + } + } + } +} + +impl Drop for Config { + fn drop(&mut self) { + unsafe { datahub_config_free(self.0) } + } +} + +/// RAII over `datahub_client`. +#[derive(Debug)] +pub struct Client(pub *mut datahub_client); + +impl Client { + pub fn insert(&self, external_id: &str, points: &[datahub_datapoint]) -> datahub_status { + let id = cstr(external_id); + unsafe { datahub_datapoints_insert(self.0, id.as_ptr(), points.as_ptr(), points.len()) } + } + + pub fn request_json( + &self, + method: &str, + path: &str, + body: Option<&str>, + ) -> Result { + let (method, path) = (cstr(method), cstr(path)); + let body = body.map(cstr); + let mut out = std::ptr::null_mut(); + let status = unsafe { + datahub_request_json( + self.0, + method.as_ptr(), + path.as_ptr(), + body.as_ref().map_or(std::ptr::null(), |b| b.as_ptr()), + &mut out, + ) + }; + match status { + datahub_status::DATAHUB_OK => Ok(unsafe { take_string(out) }), + status => Err(status), + } + } + + /// Call one of the `..._json(client, body, out)` functions. + pub fn json_call( + &self, + call: unsafe extern "C" fn( + *const datahub_client, + *const c_char, + *mut *mut c_char, + ) -> datahub_status, + body: &str, + ) -> (datahub_status, Option) { + let body = cstr(body); + let mut out = std::ptr::null_mut(); + let status = unsafe { call(self.0, body.as_ptr(), &mut out) }; + let text = if out.is_null() { + None + } else { + Some(unsafe { take_string(out) }) + }; + (status, text) + } + + pub fn flush(&self) -> datahub_status { + unsafe { datahub_client_flush(self.0) } + } + + pub fn buffered_count(&self) -> u64 { + unsafe { datahub_client_buffered_count(self.0) } + } +} + +impl Drop for Client { + fn drop(&mut self) { + unsafe { datahub_client_free(self.0) } + } +} + +pub fn point(timestamp_ms: i64, value: f64) -> datahub_datapoint { + datahub_datapoint { + timestamp_ms, + value, + } +} + +/// Now, in epoch milliseconds. The spool's retention window is measured on each record's own +/// timestamp, so anything meant to survive in a spool has to be stamped with a recent time. +pub fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64 +} diff --git a/datahub_c_bindings/tests/live.rs b/datahub_c_bindings/tests/live.rs new file mode 100644 index 0000000..b568c05 --- /dev/null +++ b/datahub_c_bindings/tests/live.rs @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Against a live backend, configured by the repository's `.env` (or the process environment). +//! Each test prints `SKIP` and passes when there is no `BASE_URL`, so a checkout without a backend +//! is unaffected — the convention the core's `multi_tenant_integration.rs` follows. + +mod common; + +use std::ffi::CStr; +use std::time::{SystemTime, UNIX_EPOCH}; + +use common::*; +use intellistream_datahub::*; + +/// The prefix the core's Rust suite uses, so the same cleanup sweeps cover what these leave behind. +const TEST_PREFIX: &str = "rust_sdk_c_"; + +fn unique(name: &str) -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + format!("{TEST_PREFIX}{name}_{}_{nanos}", std::process::id()) +} + +/// A config from the process environment plus the repository `.env`, or `None` to skip. +fn live_config() -> Option { + let config = Config(datahub_config_from_env()); + let env_file = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(".env"); + if env_file.exists() { + let path = cstr(env_file.to_str().unwrap()); + assert_eq!( + unsafe { datahub_config_load_envfile(config.0, path.as_ptr()) }, + datahub_status::DATAHUB_OK, + "{}", + last_error() + ); + } + if config.get("BASE_URL").is_none() { + println!("SKIP: no BASE_URL in the environment or ../.env"); + return None; + } + Some(config) +} + +fn delete_series(client: &Client, external_id: &str) { + let _ = client.request_json( + "POST", + "/timeseries/delete", + Some(&format!( + r#"{{"items":[{{"externalId":"{external_id}"}}]}}"# + )), + ); +} + +#[test] +fn datapoints_round_trip_through_a_real_backend() { + let Some(config) = live_config() else { return }; + let client = config + .build() + .unwrap_or_else(|s| panic!("{s:?}: {}", last_error())); + let external_id = unique("temperature"); + + let (status, out) = client.json_call( + datahub_timeseries_create_json, + &format!( + r#"{{"items":[{{"externalId":"{external_id}","name":"C SDK live test","unit":"°C"}}]}}"# + ), + ); + assert_eq!(status, datahub_status::DATAHUB_OK, "{}", last_error()); + let created: serde_json::Value = serde_json::from_str(&out.unwrap()).unwrap(); + assert_eq!(created["items"][0]["externalId"], external_id); + + let base = 1_700_000_000_000i64; + let points = [ + point(base, 20.0), + point(base + 1000, 21.0), + point(base + 2000, 22.0), + ]; + assert_eq!( + client.insert(&external_id, &points), + datahub_status::DATAHUB_OK, + "{}", + last_error() + ); + + // Ingest is asynchronous server-side; poll briefly for the latest datapoint. + let id = cstr(&external_id); + let mut latest = datahub_datapoint_agg { + timestamp_ms: 0, + value: 0.0, + min: 0.0, + max: 0.0, + average: 0.0, + sum: 0.0, + }; + let mut status = datahub_status::DATAHUB_NOT_FOUND; + for _ in 0..40 { + status = unsafe { datahub_datapoints_latest(client.0, id.as_ptr(), &mut latest) }; + if status == datahub_status::DATAHUB_OK { + break; + } + std::thread::sleep(std::time::Duration::from_millis(250)); + } + assert_eq!(status, datahub_status::DATAHUB_OK, "{}", last_error()); + assert_eq!((latest.timestamp_ms, latest.value), (base + 2000, 22.0)); + + let mut out = std::ptr::null_mut(); + let mut count = 0usize; + let status = unsafe { + datahub_datapoints_retrieve( + client.0, + id.as_ptr(), + base, + base + 3000, + 0, + &mut out, + &mut count, + ) + }; + assert_eq!(status, datahub_status::DATAHUB_OK, "{}", last_error()); + assert_eq!(count, 3); + let read = unsafe { std::slice::from_raw_parts(out, count) }; + assert_eq!(read.iter().map(|p| p.value).sum::(), 63.0); + unsafe { datahub_datapoints_free(out, count) }; + + let mut handle = std::ptr::null_mut(); + assert_eq!( + unsafe { datahub_timeseries_get_by_external_id(client.0, id.as_ptr(), &mut handle) }, + datahub_status::DATAHUB_OK + ); + assert_eq!( + unsafe { borrow_string(datahub_timeseries_unit(handle)) }.as_deref(), + Some("°C") + ); + unsafe { datahub_timeseries_free(handle) }; + + delete_series(&client, &external_id); +} + +#[test] +fn a_listener_receives_what_is_ingested() { + let Some(config) = live_config() else { return }; + let client = config + .build() + .unwrap_or_else(|s| panic!("{s:?}: {}", last_error())); + let external_id = unique("stream"); + let subscription = unique("subscription"); + + let (status, _) = client.json_call( + datahub_timeseries_create_json, + &format!(r#"{{"items":[{{"externalId":"{external_id}","name":"C SDK listener test"}}]}}"#), + ); + assert_eq!(status, datahub_status::DATAHUB_OK, "{}", last_error()); + let created = client.request_json( + "POST", + "/subscriptions/create", + Some(&format!( + r#"{{"items":[{{"externalId":"{subscription}","name":"C SDK listener test","timeseries":[{{"externalId":"{external_id}"}}]}}]}}"# + )), + ); + assert!(created.is_ok(), "{:?}: {}", created, last_error()); + + let ids = [cstr(&subscription)]; + let id_ptrs = [ids[0].as_ptr()]; + let mut listener = std::ptr::null_mut(); + assert_eq!( + unsafe { datahub_listener_open(client.0, id_ptrs.as_ptr(), 1, &mut listener) }, + datahub_status::DATAHUB_OK, + "{}", + last_error() + ); + + let point_ts = 1_700_000_000_000i64; + assert_eq!( + client.insert(&external_id, &[point(point_ts, 42.0)]), + datahub_status::DATAHUB_OK + ); + + let mut message = std::ptr::null_mut(); + let mut received = None; + for _ in 0..12 { + match unsafe { datahub_listener_next(listener, 5000, &mut message) } { + datahub_status::DATAHUB_OK => { + let object = unsafe { borrow_string(datahub_message_object(message)) }.unwrap(); + let ids = [unsafe { datahub_message_id(message) }]; + assert_eq!( + unsafe { datahub_listener_ack(listener, ids.as_ptr(), 1) }, + datahub_status::DATAHUB_OK + ); + if object == "DATAPOINTS" { + let mut points = std::ptr::null(); + let mut count = 0usize; + assert_eq!(unsafe { datahub_message_series_count(message) }, 1); + assert_eq!( + unsafe { + datahub_message_series_datapoints(message, 0, &mut points, &mut count) + }, + datahub_status::DATAHUB_OK + ); + let points = unsafe { std::slice::from_raw_parts(points, count) }; + received = Some((points[0].timestamp_ms, points[0].value)); + unsafe { datahub_message_free(message) }; + break; + } + unsafe { datahub_message_free(message) }; + } + datahub_status::DATAHUB_TIMEOUT => continue, + other => panic!( + "{:?}: {}", + unsafe { CStr::from_ptr(datahub_status_name(other)) }, + last_error() + ), + } + } + assert_eq!( + unsafe { datahub_listener_close(listener) }, + datahub_status::DATAHUB_OK + ); + assert_eq!( + received, + Some((point_ts, 42.0)), + "the ingested datapoint was delivered" + ); + + let _ = client.request_json( + "POST", + "/subscriptions/delete", + Some(&format!( + r#"{{"items":[{{"externalId":"{subscription}"}}]}}"# + )), + ); + delete_series(&client, &external_id); +} diff --git a/datahub_c_bindings/tests/mock_api.rs b/datahub_c_bindings/tests/mock_api.rs new file mode 100644 index 0000000..4e46d74 --- /dev/null +++ b/datahub_c_bindings/tests/mock_api.rs @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: Apache-2.0 +//! End to end against a mock of the api: what goes on the wire, and how answers come back. + +mod common; + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use common::*; +use intellistream_datahub::*; + +const SERIES: &str = r#"{"id":"42","externalId":"pump-1/temperature","name":"Pump 1 temperature","unit":"°C","unitExternalId":"deg_c","valueType":"float","dataSetId":"7","createdTime":"2026-01-01T00:00:00Z","lastUpdatedTime":"2026-01-01T00:00:00Z"}"#; + +fn series_by_route(request: &Request) -> (u16, String) { + match (request.method.as_str(), request.path.as_str()) { + ("POST", "/timeseries/data") => (204, String::new()), + ("POST", "/timeseries/byids") => { + if request.body.contains("pump-1/temperature") { + (200, format!(r#"{{"items":[{SERIES}]}}"#)) + } else { + (200, r#"{"items":[]}"#.to_string()) + } + } + ("POST", "/timeseries/data/latest") => ( + 200, + r#"{"items":[{"externalId":"pump-1/temperature","datapoints":[{"timestamp":"2026-01-01T00:00:00Z","value":1.5}]}]}"#.to_string(), + ), + ("POST", "/timeseries/data/list") => ( + 200, + r#"{"items":[{"externalId":"pump-1/temperature","datapoints":[{"timestamp":"2026-01-01T00:00:00Z","value":1.5},{"timestamp":"2026-01-01T01:00:00Z","min":1.0,"max":2.0,"average":1.5,"sum":3.0}],"nextCursor":"page-2"}]}"#.to_string(), + ), + ("POST", "/timeseries/create") | ("POST", "/timeseries/search") | ("POST", "/timeseries/filter") => { + (200, format!(r#"{{"items":[{SERIES}],"nextCursor":"more"}}"#)) + } + ("POST", "/events/create") => (200, request.body.clone()), + ("POST", "/events/filter") => (200, r#"{"items":[]}"#.to_string()), + ("GET", "/events/count?type=Alarm") => (200, r#"{"count":3}"#.to_string()), + ("POST", "/anything") => (200, request.body.clone()), + ("POST", "/nothing") => (204, String::new()), + _ => (404, format!(r#"{{"error":"no route for {} {}"}}"#, request.method, request.path)), + } +} + +#[test] +fn insert_posts_epoch_millis_strings_with_the_bearer_token() { + let server = MockServer::start(series_by_route); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + let points = [ + point(1_700_000_000_000, 21.5), + point(1_700_000_001_000, -3.0), + ]; + assert_eq!( + client.insert("pump-1/temperature", &points), + datahub_status::DATAHUB_OK + ); + + let request = server.last_request(); + assert_eq!( + (request.method.as_str(), request.path.as_str()), + ("POST", "/timeseries/data") + ); + assert_eq!(request.header("authorization"), Some("Bearer test-token")); + let body = request.json(); + assert_eq!(body["items"][0]["externalId"], "pump-1/temperature"); + assert_eq!( + body["items"][0]["datapoints"][0]["timestamp"], + "1700000000000" + ); + assert_eq!(body["items"][0]["datapoints"][0]["value"], "21.5"); + assert_eq!(body["items"][0]["datapoints"][1]["value"], "-3"); + assert!(body["items"][0]["id"].is_null(), "no numeric id was given"); +} + +#[test] +fn insert_str_carries_text_values_unchanged() { + let server = MockServer::start(series_by_route); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + let id = cstr("pump-1/temperature"); + let timestamps = [1_700_000_000_000i64, 1_700_000_001_000]; + let values = [cstr("OPEN"), cstr("CLOSED")]; + let value_ptrs = [values[0].as_ptr(), values[1].as_ptr()]; + let status = unsafe { + datahub_datapoints_insert_str( + client.0, + id.as_ptr(), + timestamps.as_ptr(), + value_ptrs.as_ptr(), + 2, + ) + }; + assert_eq!(status, datahub_status::DATAHUB_OK); + let body = server.last_request().json(); + assert_eq!(body["items"][0]["datapoints"][1]["value"], "CLOSED"); + assert_eq!( + body["items"][0]["datapoints"][1]["timestamp"], + "1700000001000" + ); +} + +#[test] +fn get_by_external_id_exposes_the_definition() { + let server = MockServer::start(series_by_route); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + let id = cstr("pump-1/temperature"); + let mut out = std::ptr::null_mut(); + assert_eq!( + unsafe { datahub_timeseries_get_by_external_id(client.0, id.as_ptr(), &mut out) }, + datahub_status::DATAHUB_OK + ); + assert!(!out.is_null()); + unsafe { + assert_eq!(datahub_timeseries_id(out), 42); + assert_eq!(datahub_timeseries_data_set_id(out), 7); + assert_eq!( + borrow_string(datahub_timeseries_external_id(out)).as_deref(), + Some("pump-1/temperature") + ); + assert_eq!( + borrow_string(datahub_timeseries_name(out)).as_deref(), + Some("Pump 1 temperature") + ); + assert_eq!( + borrow_string(datahub_timeseries_unit(out)).as_deref(), + Some("°C") + ); + assert_eq!( + borrow_string(datahub_timeseries_unit_external_id(out)).as_deref(), + Some("deg_c") + ); + assert_eq!( + borrow_string(datahub_timeseries_value_type(out)).as_deref(), + Some("float") + ); + let json: serde_json::Value = + serde_json::from_str(&borrow_string(datahub_timeseries_json(out)).unwrap()).unwrap(); + assert_eq!(json["externalId"], "pump-1/temperature"); + datahub_timeseries_free(out); + } + let body = server.last_request().json(); + assert_eq!(body["items"][0]["externalId"], "pump-1/temperature"); + + let unknown = cstr("no-such-series"); + let mut out = std::ptr::null_mut(); + assert_eq!( + unsafe { datahub_timeseries_get_by_external_id(client.0, unknown.as_ptr(), &mut out) }, + datahub_status::DATAHUB_NOT_FOUND + ); + assert!(out.is_null()); + assert_eq!(datahub_last_http_status(), 200); + assert_eq!( + last_error(), + "no time series with external id \"no-such-series\"" + ); +} + +#[test] +fn latest_and_retrieve_map_absent_aggregates_to_nan() { + let server = MockServer::start(series_by_route); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + let id = cstr("pump-1/temperature"); + + let mut latest = datahub_datapoint_agg { + timestamp_ms: 0, + value: 0.0, + min: 0.0, + max: 0.0, + average: 0.0, + sum: 0.0, + }; + assert_eq!( + unsafe { datahub_datapoints_latest(client.0, id.as_ptr(), &mut latest) }, + datahub_status::DATAHUB_OK + ); + assert_eq!(latest.timestamp_ms, 1_767_225_600_000); + assert_eq!(latest.value, 1.5); + assert!( + latest.min.is_nan() + && latest.max.is_nan() + && latest.average.is_nan() + && latest.sum.is_nan() + ); + + let mut out = std::ptr::null_mut(); + let mut count = 0usize; + let status = unsafe { + datahub_datapoints_retrieve( + client.0, + id.as_ptr(), + 1_767_225_600_000, + DATAHUB_TIME_UNSET, + 500, + &mut out, + &mut count, + ) + }; + assert_eq!(status, datahub_status::DATAHUB_OK); + assert_eq!(count, 2); + let points = unsafe { std::slice::from_raw_parts(out, count) }; + assert_eq!(points[0].value, 1.5); + assert!(points[1].value.is_nan()); + assert_eq!( + ( + points[1].min, + points[1].max, + points[1].average, + points[1].sum + ), + (1.0, 2.0, 1.5, 3.0) + ); + unsafe { datahub_datapoints_free(out, count) }; + + let body = server.last_request().json(); + assert_eq!(body["items"][0]["externalId"], "pump-1/temperature"); + assert_eq!(body["items"][0]["start"], "2026-01-01T00:00:00Z"); + assert!( + body["items"][0]["end"].is_null(), + "an unset bound is omitted" + ); + assert_eq!(body["items"][0]["limit"], 500); + + // An out-of-range bound is caught before any request. + let requests_before = server.requests().len(); + let status = unsafe { + datahub_datapoints_retrieve( + client.0, + id.as_ptr(), + i64::MAX, + DATAHUB_TIME_UNSET, + 0, + &mut out, + &mut count, + ) + }; + assert_eq!(status, datahub_status::DATAHUB_INVALID_ARGUMENT); + assert_eq!(server.requests().len(), requests_before); +} + +/// `DATAHUB_TIME_UNSET` is defined in the header by build.rs; the Rust side uses the same value. +const DATAHUB_TIME_UNSET: i64 = i64::MIN; + +#[test] +fn retrieve_json_keeps_the_next_cursor() { + let server = MockServer::start(series_by_route); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + let (status, out) = client.json_call( + datahub_datapoints_retrieve_json, + r#"{"items":[{"externalId":"pump-1/temperature","aggregates":["average"],"granularity":"1h"}]}"#, + ); + assert_eq!(status, datahub_status::DATAHUB_OK); + let out: serde_json::Value = serde_json::from_str(&out.unwrap()).unwrap(); + assert_eq!(out["items"][0]["datapoints"][1]["average"], 1.5); + assert_eq!(out["items"][0]["nextCursor"], "page-2"); + let sent = server.last_request().json(); + assert_eq!(sent["items"][0]["aggregates"][0], "average"); + assert_eq!(sent["items"][0]["granularity"], "1h"); +} + +#[test] +fn events_create_json_stamps_ids_and_returns_the_envelope() { + let server = MockServer::start(series_by_route); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + let (status, out) = client.json_call( + datahub_events_create_json, + r#"{"items":[{"externalId":"alarm-1","type":"Alarm","eventTime":"2026-01-01T00:00:00Z","metadata":{"severity":"high"}}]}"#, + ); + assert_eq!(status, datahub_status::DATAHUB_OK); + let sent = server.last_request().json(); + assert_eq!(sent["items"][0]["type"], "Alarm"); + assert_eq!(sent["items"][0]["metadata"]["severity"], "high"); + let stamped = sent["items"][0]["id"] + .as_str() + .expect("a UUID was stamped before the first send"); + assert_eq!(stamped.len(), 36); + let out: serde_json::Value = serde_json::from_str(&out.unwrap()).unwrap(); + assert_eq!(out["items"][0]["id"], stamped); + assert!(out.get("nextCursor").is_none()); +} + +#[test] +fn events_filter_json_sends_the_form_as_given() { + let server = MockServer::start(series_by_route); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + let (status, out) = client.json_call( + datahub_events_filter_json, + r#"{"filter":{"type":["Alarm"]},"limit":10}"#, + ); + assert_eq!(status, datahub_status::DATAHUB_OK); + assert_eq!(out.as_deref(), Some(r#"{"items":[]}"#)); + let sent = server.last_request().json(); + assert_eq!(sent["limit"], 10); + assert_eq!(sent["filter"]["type"][0], "Alarm"); +} + +#[test] +fn timeseries_json_calls_hit_their_endpoints_and_keep_paging() { + let server = MockServer::start(series_by_route); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + + let (status, out) = client.json_call(datahub_timeseries_create_json, r#"{"items":[{"externalId":"pump-1/temperature","name":"Pump 1 temperature","unit":"°C"}]}"#); + assert_eq!(status, datahub_status::DATAHUB_OK); + assert_eq!(server.last_request().path, "/timeseries/create"); + let out: serde_json::Value = serde_json::from_str(&out.unwrap()).unwrap(); + assert_eq!(out["items"][0]["id"], "42"); + assert_eq!(out["nextCursor"], "more"); + + let (status, _) = client.json_call( + datahub_timeseries_search_json, + r#"{"search":{"query":"pump"},"limit":5}"#, + ); + assert_eq!(status, datahub_status::DATAHUB_OK); + let sent = server.last_request(); + assert_eq!(sent.path, "/timeseries/search"); + assert_eq!(sent.json()["search"]["query"], "pump"); + + let (status, _) = client.json_call( + datahub_timeseries_filter_json, + r#"{"filter":{"unit":["°C"]},"limit":5}"#, + ); + assert_eq!(status, datahub_status::DATAHUB_OK); + let sent = server.last_request(); + assert_eq!(sent.path, "/timeseries/filter"); + assert_eq!(sent.json()["filter"]["unit"][0], "°C"); +} + +#[test] +fn request_json_is_a_raw_authenticated_call() { + let server = MockServer::start(series_by_route); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + + assert_eq!( + client + .request_json("get", "events/count?type=Alarm", None) + .unwrap(), + r#"{"count":3}"# + ); + let sent = server.last_request(); + assert_eq!( + (sent.method.as_str(), sent.path.as_str()), + ("GET", "/events/count?type=Alarm") + ); + assert_eq!(sent.header("authorization"), Some("Bearer test-token")); + + assert_eq!( + client + .request_json("POST", "/anything", Some(r#"{"a":[1,2]}"#)) + .unwrap(), + r#"{"a":[1,2]}"# + ); + assert_eq!( + client.request_json("POST", "/nothing", None).unwrap(), + "", + "a 204 is an empty body" + ); + assert_eq!( + server.last_request().body, + "{}", + "a NULL POST body is sent as {{}}" + ); + + assert_eq!( + client.request_json("GET", "/missing", None).unwrap_err(), + datahub_status::DATAHUB_HTTP + ); + assert_eq!(datahub_last_http_status(), 404); + assert!( + last_error().contains("no route for GET /missing"), + "{}", + last_error() + ); +} + +#[test] +fn a_401_is_an_auth_failure_and_a_500_is_http() { + let server = MockServer::start(|request: &Request| { + if request.path.ends_with("/data") { + (401, String::new()) + } else { + (500, r#"{"error":"boom"}"#.to_string()) + } + }); + let client = Config::for_server(&server.base_url, None).build().unwrap(); + assert_eq!( + client.insert("pump", &[point(1, 1.0)]), + datahub_status::DATAHUB_AUTH + ); + assert_eq!(datahub_last_http_status(), 401); + assert_eq!( + client + .request_json("GET", "/events/count", None) + .unwrap_err(), + datahub_status::DATAHUB_HTTP + ); + assert_eq!(datahub_last_http_status(), 500); + assert!(last_error().contains("boom"), "{}", last_error()); +} + +#[test] +fn a_spooled_backlog_is_sent_first_once_the_server_is_back() { + let down = Arc::new(AtomicBool::new(true)); + let server = { + let down = down.clone(); + MockServer::start(move |request: &Request| { + if down.load(Ordering::Relaxed) { + (503, "down".to_string()) + } else { + series_by_route(request) + } + }) + }; + let spool = tempfile::tempdir().unwrap(); + let client = Config::for_server(&server.base_url, Some(spool.path())) + .build() + .unwrap(); + + // Retention is measured on the data's own timestamp, so spooled records must be recent. + let now = now_ms(); + assert_eq!( + client.insert("pump-1/temperature", &[point(now, 1.0)]), + datahub_status::DATAHUB_BUFFERED + ); + let (status, _) = client.json_call( + datahub_events_create_json, + &format!( + r#"{{"items":[{{"externalId":"alarm-1","type":"Alarm","eventTime":"{}"}}]}}"#, + chrono::Utc::now().to_rfc3339() + ), + ); + assert_eq!(status, datahub_status::DATAHUB_BUFFERED); + assert_eq!(client.buffered_count(), 2); + assert_eq!(client.flush(), datahub_status::DATAHUB_BUFFERED); + + down.store(false, Ordering::Relaxed); + assert_eq!(client.flush(), datahub_status::DATAHUB_OK); + assert_eq!(client.buffered_count(), 0); + let delivered: Vec = server + .requests() + .into_iter() + .filter(|r| r.header("content-length") != Some("0")) + .filter_map(|r| { + serde_json::from_str::(&r.body) + .ok() + .map(|_| r.path) + }) + .collect(); + assert!( + delivered.contains(&"/timeseries/data".to_string()), + "{delivered:?}" + ); + assert!( + delivered.contains(&"/events/create".to_string()), + "{delivered:?}" + ); + + // And a later insert goes straight through with nothing left behind. + assert_eq!( + client.insert("pump-1/temperature", &[point(now + 1000, 2.0)]), + datahub_status::DATAHUB_OK + ); + assert_eq!(client.buffered_count(), 0); +} + +#[test] +fn spooled_records_older_than_the_retention_window_are_dropped() { + let server = MockServer::start(always(503, "down")); + let spool = tempfile::tempdir().unwrap(); + let client = Config::for_server(&server.base_url, Some(spool.path())) + .build() + .unwrap(); + // Reported as buffered — the call succeeded — but a 1970 datapoint is outside the 72 h window + // and never survives in the spool. Backfills older than the window need buffering off. + assert_eq!( + client.insert("pump-1/temperature", &[point(1, 1.0)]), + datahub_status::DATAHUB_BUFFERED + ); + assert_eq!(client.buffered_count(), 0); +} diff --git a/datahub_c_bindings/tests/offline.rs b/datahub_c_bindings/tests/offline.rs new file mode 100644 index 0000000..ee361d5 --- /dev/null +++ b/datahub_c_bindings/tests/offline.rs @@ -0,0 +1,482 @@ +// SPDX-License-Identifier: Apache-2.0 +//! The boundary and the offline paths: nothing here needs a network. + +mod common; + +use std::ffi::CStr; + +use common::*; +use intellistream_datahub::*; + +/// A port nothing listens on: connection refused, immediately. +const UNREACHABLE: &str = "http://127.0.0.1:9"; + +#[test] +fn version_is_the_crate_version() { + let version = unsafe { CStr::from_ptr(datahub_version()) } + .to_str() + .unwrap(); + assert_eq!(version, env!("CARGO_PKG_VERSION")); +} + +#[test] +fn a_fresh_thread_has_an_empty_error_not_a_null_one() { + let text = std::thread::spawn(|| { + let ptr = datahub_last_error(); + assert!(!ptr.is_null()); + last_error() + }) + .join() + .unwrap(); + assert_eq!(text, ""); +} + +#[test] +fn null_handles_are_invalid_arguments_not_crashes() { + let mut out = std::ptr::null_mut(); + assert_eq!( + unsafe { datahub_client_new(std::ptr::null(), &mut out) }, + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert!(out.is_null()); + assert_eq!(last_error(), "config must not be NULL"); + + let config = Config::new(); + assert_eq!( + unsafe { datahub_client_new(config.0, std::ptr::null_mut()) }, + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert_eq!(last_error(), "out must not be NULL"); + + assert_eq!( + unsafe { + datahub_datapoints_insert(std::ptr::null(), cstr("x").as_ptr(), std::ptr::null(), 0) + }, + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert_eq!( + unsafe { datahub_client_buffered_count(std::ptr::null()) }, + 0 + ); + assert_eq!(unsafe { datahub_timeseries_id(std::ptr::null()) }, 0); + assert!(unsafe { datahub_timeseries_name(std::ptr::null()) }.is_null()); + + // Every free tolerates NULL. + unsafe { + datahub_string_free(std::ptr::null_mut()); + datahub_config_free(std::ptr::null_mut()); + datahub_client_free(std::ptr::null_mut()); + datahub_timeseries_free(std::ptr::null_mut()); + datahub_message_free(std::ptr::null_mut()); + datahub_datapoints_free(std::ptr::null_mut(), 0); + assert_eq!( + datahub_listener_close(std::ptr::null_mut()), + datahub_status::DATAHUB_OK + ); + } +} + +#[test] +fn invalid_utf8_and_empty_ids_are_rejected_before_any_request() { + let config = Config::for_server(UNREACHABLE, None); + let client = config.build().unwrap(); + let bad = b"pump-\xff\0"; + assert_eq!( + unsafe { + datahub_datapoints_insert(client.0, bad.as_ptr() as *const _, std::ptr::null(), 0) + }, + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert!( + last_error().starts_with("external_id is not valid UTF-8"), + "{}", + last_error() + ); + + assert_eq!( + client.insert(" ", &[point(1, 1.0)]), + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert_eq!(last_error(), "external_id must not be empty"); + + assert_eq!( + client.insert("pump", &[point(1, 1.0), point(2, f64::NAN)]), + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert_eq!(last_error(), "points[1].value is not finite"); + + // count > 0 with a NULL array + let id = cstr("pump"); + assert_eq!( + unsafe { datahub_datapoints_insert(client.0, id.as_ptr(), std::ptr::null(), 3) }, + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + // count == 0 is a no-op, not an error + assert_eq!(client.insert("pump", &[]), datahub_status::DATAHUB_OK); +} + +#[test] +fn a_config_without_base_url_is_a_config_error_with_the_cores_message() { + let config = Config::new(); + config.set("TOKEN", "t"); + assert_eq!(config.build().unwrap_err(), datahub_status::DATAHUB_CONFIG); + assert!( + last_error().contains("BASE_URL is not set"), + "{}", + last_error() + ); +} + +#[test] +fn a_malformed_token_uri_is_a_config_error_not_a_panic() { + let config = Config::for_server(UNREACHABLE, None); + let (id, secret, uri) = (cstr("id"), cstr("secret"), cstr("not a url")); + assert_eq!( + unsafe { + datahub_config_set_client_credentials( + config.0, + id.as_ptr(), + secret.as_ptr(), + uri.as_ptr(), + ) + }, + datahub_status::DATAHUB_OK + ); + assert_eq!(config.build().unwrap_err(), datahub_status::DATAHUB_CONFIG); + assert!(last_error().contains("TOKEN_URI"), "{}", last_error()); +} + +#[test] +fn config_set_with_a_null_value_removes_the_key() { + let config = Config::new(); + config.set("SCOPE", "organization:*"); + assert_eq!(config.get("SCOPE").as_deref(), Some("organization:*")); + let key = cstr("SCOPE"); + assert_eq!( + unsafe { datahub_config_set(config.0, key.as_ptr(), std::ptr::null()) }, + datahub_status::DATAHUB_OK + ); + assert_eq!(config.get("SCOPE"), None); + let empty = cstr(" "); + assert_eq!( + unsafe { datahub_config_set(config.0, empty.as_ptr(), key.as_ptr()) }, + datahub_status::DATAHUB_INVALID_ARGUMENT + ); +} + +#[test] +fn typed_setters_land_on_the_documented_keys() { + let config = Config::new(); + let value = cstr("v"); + unsafe { + assert_eq!( + datahub_config_set_base_url(config.0, value.as_ptr()), + datahub_status::DATAHUB_OK + ); + assert_eq!( + datahub_config_set_scope(config.0, value.as_ptr()), + datahub_status::DATAHUB_OK + ); + assert_eq!( + datahub_config_set_audience(config.0, value.as_ptr()), + datahub_status::DATAHUB_OK + ); + assert_eq!( + datahub_config_set_assertion_grant(config.0, value.as_ptr()), + datahub_status::DATAHUB_OK + ); + assert_eq!( + datahub_config_set_assertion_credentials( + config.0, + value.as_ptr(), + value.as_ptr(), + value.as_ptr() + ), + datahub_status::DATAHUB_OK + ); + assert_eq!( + datahub_config_set_buffer_retention_secs(config.0, 3600), + datahub_status::DATAHUB_OK + ); + assert_eq!( + datahub_config_set_buffer_max_bytes(config.0, 0), + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert_eq!( + datahub_config_set_buffer_retention_secs(config.0, -1), + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + } + for key in [ + "BASE_URL", + "SCOPE", + "AUDIENCE", + "ASSERTION_GRANT", + "ASSERTION_CLIENT_ID", + "ASSERTION_CLIENT_SECRET", + "ASSERTION_TOKEN_URI", + ] { + assert_eq!(config.get(key).as_deref(), Some("v"), "{key}"); + } + assert_eq!(config.get("BUFFER_RETENTION_SECS").as_deref(), Some("3600")); + assert_eq!(config.get("ENABLE_BUFFERING").as_deref(), Some("true")); +} + +#[test] +fn an_env_file_is_read_without_touching_the_process_environment() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("gateway.env"); + std::fs::write( + &file, + "# gateway\nBASE_URL=https://datahub.example.com\nexport TOKEN='abc def'\nSCOPE=\"organization:*\" # tenant\nBUFFER_DIR=/var/spool/datahub\n\n", + ) + .unwrap(); + let config = Config::new(); + let path = cstr(file.to_str().unwrap()); + assert_eq!( + unsafe { datahub_config_load_envfile(config.0, path.as_ptr()) }, + datahub_status::DATAHUB_OK + ); + assert_eq!( + config.get("BASE_URL").as_deref(), + Some("https://datahub.example.com") + ); + assert_eq!(config.get("TOKEN").as_deref(), Some("abc def")); + assert_eq!(config.get("SCOPE").as_deref(), Some("organization:*")); + assert_eq!( + config.get("BUFFER_DIR").as_deref(), + Some("/var/spool/datahub") + ); + assert!( + std::env::var("BUFFER_DIR").is_err(), + "the process environment must stay untouched" + ); + + let missing = cstr(dir.path().join("nope.env").to_str().unwrap()); + assert_eq!( + unsafe { datahub_config_load_envfile(config.0, missing.as_ptr()) }, + datahub_status::DATAHUB_IO + ); + assert!( + last_error().starts_with("cannot read env file"), + "{}", + last_error() + ); + + std::fs::write(&file, "BASE_URL\n").unwrap(); + assert_eq!( + unsafe { datahub_config_load_envfile(config.0, path.as_ptr()) }, + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert!(last_error().contains("line 1"), "{}", last_error()); +} + +#[test] +fn from_env_snapshots_the_process_environment() { + std::env::set_var("DATAHUB_C_TEST_MARKER", "present"); + let config = Config(datahub_config_from_env()); + assert_eq!( + config.get("DATAHUB_C_TEST_MARKER").as_deref(), + Some("present") + ); + std::env::remove_var("DATAHUB_C_TEST_MARKER"); + assert_eq!( + config.get("DATAHUB_C_TEST_MARKER").as_deref(), + Some("present"), + "a snapshot, not a live view" + ); +} + +#[test] +fn with_buffering_an_unreachable_server_spools_to_disk() { + let spool = tempfile::tempdir().unwrap(); + let config = Config::for_server(UNREACHABLE, Some(spool.path())); + let client = config.build().unwrap(); + + let now = now_ms(); + let points = [point(now, 21.5), point(now + 1000, 21.6)]; + assert_eq!( + client.insert("pump-1/temperature", &points), + datahub_status::DATAHUB_BUFFERED + ); + assert_eq!(client.buffered_count(), 2); + let datapoints_dir = spool.path().join("datapoints"); + let segments: Vec<_> = std::fs::read_dir(&datapoints_dir) + .expect("the spool directory exists") + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert!( + segments.iter().any(|name| name.ends_with(".ndjson")), + "an active segment is on disk: {segments:?}" + ); + + // Still down: the backlog stays, and so does the count. + assert_eq!(client.flush(), datahub_status::DATAHUB_BUFFERED); + assert_eq!(client.buffered_count(), 2); + + // Buffering is per client: another client on the same directory sees the backlog on flush. + drop(client); + let again = Config::for_server(UNREACHABLE, Some(spool.path())) + .build() + .unwrap(); + assert_eq!(again.flush(), datahub_status::DATAHUB_BUFFERED); + assert_eq!( + again.buffered_count(), + 2, + "the spool was recovered from disk" + ); +} + +#[test] +fn without_buffering_an_unreachable_server_is_an_http_503() { + let config = Config::for_server(UNREACHABLE, None); + let client = config.build().unwrap(); + assert_eq!( + client.insert("pump", &[point(1, 1.0)]), + datahub_status::DATAHUB_HTTP + ); + assert_eq!(datahub_last_http_status(), 503); + assert!(last_error().starts_with("503"), "{}", last_error()); + assert_eq!(client.buffered_count(), 0); + assert_eq!( + client.flush(), + datahub_status::DATAHUB_OK, + "nothing to flush when buffering is off" + ); +} + +#[test] +fn a_token_that_cannot_be_minted_is_an_auth_failure_or_a_buffered_ingest() { + let credentials = |config: &Config| { + let (id, secret, uri) = ( + cstr("gateway"), + cstr("secret"), + cstr("http://127.0.0.1:9/token"), + ); + assert_eq!( + unsafe { + datahub_config_set_client_credentials( + config.0, + id.as_ptr(), + secret.as_ptr(), + uri.as_ptr(), + ) + }, + datahub_status::DATAHUB_OK + ); + }; + + let config = Config::new(); + config.set("BASE_URL", UNREACHABLE); + credentials(&config); + let client = config.build().unwrap(); + assert_eq!( + client.insert("pump", &[point(1, 1.0)]), + datahub_status::DATAHUB_AUTH + ); + assert_eq!(datahub_last_http_status(), 401); + assert!( + last_error().contains("failed to get api token"), + "{}", + last_error() + ); + + // With buffering on, an auth failure is recoverable out of band, so the data is kept. + let spool = tempfile::tempdir().unwrap(); + let config = Config::new(); + config.set("BASE_URL", UNREACHABLE); + credentials(&config); + let dir = cstr(spool.path().to_str().unwrap()); + assert_eq!( + unsafe { datahub_config_set_buffer_dir(config.0, dir.as_ptr()) }, + datahub_status::DATAHUB_OK + ); + let client = config.build().unwrap(); + assert_eq!( + client.insert("pump", &[point(now_ms(), 1.0)]), + datahub_status::DATAHUB_BUFFERED + ); + assert_eq!(client.buffered_count(), 1); +} + +#[test] +fn request_json_validates_its_arguments_locally() { + let config = Config::for_server(UNREACHABLE, None); + let client = config.build().unwrap(); + assert_eq!( + client.request_json("PUT", "/events", None).unwrap_err(), + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert!(last_error().contains("GET or POST"), "{}", last_error()); + assert_eq!( + client + .request_json("POST", "/events/create", Some("{not json")) + .unwrap_err(), + datahub_status::DATAHUB_INVALID_ARGUMENT + ); + assert!( + last_error().starts_with("body is not valid JSON"), + "{}", + last_error() + ); + assert_eq!( + client + .request_json("GET", "/events/count", None) + .unwrap_err(), + datahub_status::DATAHUB_HTTP + ); + assert_eq!(datahub_last_http_status(), 503); +} + +#[test] +fn json_bodies_of_the_wrong_shape_never_leave_the_process() { + let config = Config::for_server(UNREACHABLE, None); + let client = config.build().unwrap(); + let (status, out) = client.json_call( + datahub_events_create_json, + r#"{"items":[{"externalId":"e"}]}"#, + ); + assert_eq!( + status, + datahub_status::DATAHUB_INVALID_ARGUMENT, + "type and eventTime are required" + ); + assert!(out.is_none()); + assert!( + last_error().starts_with("body is not a valid event create request"), + "{}", + last_error() + ); + + let (status, _) = client.json_call( + datahub_timeseries_create_json, + r#"{"items":[{"externalId":"t","name":"t","unit":7}]}"#, + ); + assert_eq!( + status, + datahub_status::DATAHUB_INVALID_ARGUMENT, + "a unit that is a number is the wrong type" + ); +} + +#[test] +fn a_client_is_usable_from_several_threads_at_once() { + let spool = tempfile::tempdir().unwrap(); + let config = Config::for_server(UNREACHABLE, Some(spool.path())); + let client = config.build().unwrap(); + let handle = client.0 as usize; + let threads: Vec<_> = (0..4) + .map(|n| { + std::thread::spawn(move || { + let client = handle as *mut datahub_client; + let id = cstr(&format!("series-{n}")); + let points = [point(now_ms() + n, n as f64)]; + unsafe { datahub_datapoints_insert(client, id.as_ptr(), points.as_ptr(), 1) } + }) + }) + .collect(); + for thread in threads { + assert_eq!(thread.join().unwrap(), datahub_status::DATAHUB_BUFFERED); + } + assert_eq!(client.buffered_count(), 4); +} diff --git a/docs/c-sdk-design.md b/docs/c-sdk-design.md new file mode 100644 index 0000000..dd85884 --- /dev/null +++ b/docs/c-sdk-design.md @@ -0,0 +1,402 @@ +# A C SDK, built as an FFI crate over this one + +Design note, written before the crate existed. It records what a C SDK for DataHub should be, +where it should live, and what had to be true before starting. **The crate now exists at +`datahub_c_bindings/`** — see its README for usage; where the implementation departs from this +note is listed under [Status](#status) at the end. + +Baseline: this repository at `origin/main` @ `79173af` (crate `intellistream-datahub-sdk` 0.3.0). + +## The decision + +**Build the C SDK as a third crate in this repository — `datahub_c_bindings/` beside +`datahub_python_bindings/` — exposing a small, ingest-first C ABI over the existing Rust core. +Do not start it until a named consumer exists.** + +Three questions were asked; the answers, in order: + +1. **Does the Rust SDK already cover C?** No. There is no `extern "C"` surface, no `cbindgen`, + and the only `cdylib` in the tree is the PyO3 crate. A Rust `rlib` is consumable from Rust + only. Nothing written in C, C++, C#, Go, MATLAB or LabVIEW can touch this SDK today. +2. **Should there be one?** Yes, on condition. The parts of the SDK worth having from C are the + parts nobody should reimplement in C: token acquisition and refresh across four OAuth2 flows + (client credentials, refresh token, RFC 7523 `jwt-bearer`, and the secretless federated + exchange), the durable zstd disk spool that keeps ingest alive through an outage, and the + WebSocket subscription listener with reconnect. A one-off datapoint push from C can be done + with libcurl and a JSON library; the SDK earns its keep once any of those three is needed. +3. **Where?** Here. The whole point is *one implementation, several ABIs* — exactly the + arrangement the Python bindings already prove. A hand-written C client would be a third copy + of the auth and buffering logic, and would drift. + +### Who it is for + +A C ABI is the universal bridge, which is the strongest argument for building it once: + +| Consumer | How it loads the library | +|---|---| +| Edge gateways and PLC-adjacent daemons in C or C++ | Link `libintellistream_datahub` directly | +| C++ applications | Same header; an RAII wrapper is a few dozen lines the consumer can own | +| .NET | P/Invoke | +| Go | cgo | +| LabVIEW, MATLAB | Call Library Function Node / `loadlibrary` with the header | +| Swift, Zig, Free Pascal, Ada, … | Their native C interop | + +Consumers it does **not** target: Java (the platform ships `datahub-java-sdk`), Python (PyO3 +is a better fit than going through C), and Node, which should use `napi-rs` directly from Rust +if it is ever wanted rather than a C-to-N-API shim. + +It is a **hosted-OS** library — Linux (glibc and musl), Windows, macOS — not a bare-metal one. +reqwest, rustls and Tokio need an operating system; a microcontroller without one is out of +scope and would need a different (much smaller, `no_std`) design that shares nothing with this +SDK beyond the wire format. + +### The go/no-go condition + +Do not build this "for completeness". The Python bindings are ~8k lines because they mirror +every service; an unscoped C SDK would be the same size again, in the one language where every +line of surface is also a line of ownership and lifetime contract to get wrong, with zero known +callers. Start when one of the following is concrete: a gateway or firmware team that will link +it, a .NET or Go integration, or a LabVIEW/MATLAB deployment. The first consumer also decides +what goes into the first slice beyond the ingest core described below. + +## What exists today, and what the C layer builds on + +- **The async core** (`src/lib.rs`, `ApiService`) with every service as a field, driven by + Tokio. The C crate wraps this directly. +- **The blocking client** (`src/blocking.rs`): owns a `tokio::runtime::Runtime` and `block_on`s + each async call, so there is exactly one implementation of every call. That is the runtime + model the C layer copies. It does **not** wrap the blocking client itself, for one reason: + the blocking client deliberately has no `subscriptions` field (`async_api()` is the escape + hatch for it), and the listener is one of the three things the C SDK exists for. The FFI + crate therefore holds an `Arc` plus its own runtime and drives the async API the + way `blocking::ApiService::wrap` does. +- **Durable buffering** (`src/buffer.rs`): `TimeSeriesService::insert_datapoints` drains the + on-disk backlog first, posts in ≤100k-datapoint chunks, and spools to disk on a transient + failure; retries are safe because the backend dedups on `(series, timestamp)`. Configured + through `DataHubConfig::enable_buffering` / `set_buffer_dir` / `set_buffer_retention_secs` / + `set_buffer_max_bytes` or the `ENABLE_BUFFERING` / `BUFFER_*` variables. This is the feature + an edge device with flaky connectivity is buying. +- **TLS is already rustls** (`rustls-tls-native-roots` on both reqwest and tokio-tungstenite). + That matters more for a C library than for the crate: a `cdylib` loaded into an arbitrary host + process must not dynamically link a `libssl` whose version the host also has opinions about. + Keep it that way. `native-roots` reads the OS trust store, which a stripped embedded image may + not have; the library must document `SSL_CERT_FILE` / `SSL_CERT_DIR` as the fix. +- **Auth diagnostics** (`src/auth_diagnostics.rs`) reconstructs the reason for an otherwise + blank 401 from the token's `organization` claim. The C layer should surface that text through + its error string; it is the single most common support question and C callers have even less + to go on than Rust ones. + +### Two things in the core that must change first + +Both are small, both are core changes rather than FFI-crate workarounds, and both are +blockers for shipping a shared library: + +1. **`process_response` prints response bodies to stdout** (`src/http.rs`). That is a + documented, deliberate debugging aid in a Rust crate a developer runs on purpose. Inside a + `cdylib` loaded into someone's gateway daemon it is a library writing to a stream it does not + own. It has to become opt-in — a config flag or cargo feature, default off for the C crate — + before the first release. AGENTS.md says not to silently remove it; this note is the + non-silent request to gate it. +2. **`create_api_service()` reads `.env` from the current directory** via `dotenv`. A shared + library picking up a dotfile from the *host process's* working directory is a surprise the + host did not ask for. The C constructor that reads configuration from the environment uses + `DataHubConfig::from_env()` (process environment only); loading a file is a separate, + explicit `datahub_config_load_envfile(path)` built on `from_envfile`. + +Also worth noting, not a blocker: `insert_datapoint` panics when given neither an id nor an +external id. The FFI validates arguments before calling into the core, and `catch_unwind` at +the boundary is the backstop, never the plan. + +## Shape of the crate + +``` +datahub_c_bindings/ + Cargo.toml package datahub_c_bindings, publish = false, version locked to the others + cbindgen.toml + build.rs runs cbindgen; CI fails if the committed header is stale + include/ + intellistream_datahub.h generated, committed — C users must not need cbindgen + src/ + lib.rs handle types, runtime, error state, catch_unwind wrapper + config.rs datahub_config_* + client.rs datahub_client_* + datapoints.rs datahub_datapoints_* + json.rs the JSON pass-through functions + listener.rs datahub_listener_* / datahub_message_* + tests/ + c/smoke.c compiled and run in CI against the built library, no backend needed +``` + +```toml +[lib] +name = "intellistream_datahub" # libintellistream_datahub.{so,dylib,a}, intellistream_datahub.dll +crate-type = ["cdylib", "staticlib"] +``` + +Symbol prefix `datahub_`, types `datahub_client`, `datahub_config`, `datahub_listener`, +`datahub_message`, `datahub_datapoint`, status enum `datahub_status`. Every handle is an opaque +pointer to a Rust `Box`; C never sees a Rust struct layout except the two `#[repr(C)]` datapoint +structs below. + +**Workspace.** This will be the third crate with its own `Cargo.toml` and its own target +directory. Landing it is the moment to make the repository a Cargo workspace (root `[workspace] +members = ["datahub_python_bindings", "datahub_c_bindings"]`) so there is one lockfile, one +target dir, and one `cargo build --workspace` in CI. The root package already `exclude`s the +binding directories from the published crate, so `cargo publish` is unaffected. Separable from +the C work; recommended alongside it. + +## The ABI + +### Runtime and threading + +- A `datahub_client` owns one multi-thread Tokio runtime, created in `datahub_client_new`, + dropped in `datahub_client_free`. Every call is `runtime.block_on(...)`, exactly as + `blocking.rs` does. The calling C thread blocks for the duration of the call. +- `datahub_client` is `Send + Sync`: any number of C threads may call it concurrently. The + client must be freed after every listener created from it. +- `datahub_listener` and `datahub_message` are single-owner: one thread at a time. +- Nothing may be called from inside a Tokio runtime thread. That is only possible if the host + is itself Rust, in which case it should use the crate, not the C ABI. + +### Errors + +Every function that can fail returns a `datahub_status`: + +```c +typedef enum datahub_status { + DATAHUB_OK = 0, + DATAHUB_BUFFERED, /* accepted into the disk spool; will be sent on a later call */ + DATAHUB_TIMEOUT, /* datahub_listener_next: nothing arrived within the timeout */ + DATAHUB_CLOSED, /* datahub_listener_next: the stream has ended */ + DATAHUB_INVALID_ARGUMENT,/* NULL where a value was required, bad UTF-8, empty id, … */ + DATAHUB_CONFIG, /* DataHubError: missing BASE_URL, incomplete credential set */ + DATAHUB_AUTH, /* token acquisition failed, or 401/403 from the api */ + DATAHUB_HTTP, /* any other non-2xx; see datahub_last_http_status() */ + DATAHUB_IO, /* spool directory unwritable, TLS setup, DNS, socket */ + DATAHUB_PANIC, /* a Rust panic was caught at the boundary; report it */ +} datahub_status; +``` + +`DATAHUB_BUFFERED` is the important non-error. It is the answer an edge device wants when the +network is down: the datapoints are safe on disk and the call returns immediately. Callers that +must know whether data has *reached* the server check for `DATAHUB_OK` specifically. + +Detail travels out of band, thread-locally, so the enum stays small and the message can be as +long as it likes: + +```c +const char *datahub_last_error(void); /* borrowed; valid until the next failing call on this thread */ +int datahub_last_http_status(void); /* 0 when the last failure was not an HTTP response */ +``` + +The message is the `ResponseError`/`DataHubError` text, with the `auth_diagnostics` explanation +appended for a 401. Every failing call overwrites it; a successful call leaves it alone. + +Every exported function body runs inside `std::panic::catch_unwind`. A panic becomes +`DATAHUB_PANIC` with the panic message as the last error. Unwinding into C is undefined +behaviour and is never allowed to happen, whatever the core does. + +### Memory + +- Strings cross the boundary as NUL-terminated UTF-8. Input strings are borrowed for the + duration of the call only; the library copies what it keeps. +- Every pointer the library hands out has a matching free: `datahub_string_free`, + `datahub_config_free`, `datahub_client_free`, `datahub_listener_close`, + `datahub_message_free`. C `free()` on a Rust allocation is undefined behaviour; the header + says so next to every out-parameter. +- Arrays the library returns come with their length in an out-parameter and are freed as a unit. +- The datapoint structs are plain C values with no hidden ownership: + +```c +typedef struct datahub_datapoint { + int64_t timestamp_ms; /* Unix epoch milliseconds, UTC */ + double value; +} datahub_datapoint; + +/* Read side. Absent aggregates are NaN, so a plain read never needs a presence flag. */ +typedef struct datahub_datapoint_agg { + int64_t timestamp_ms; + double value, min, max, average, sum; +} datahub_datapoint_agg; +``` + +### Typed on the hot path, JSON everywhere else + +The datapoint path is typed — arrays of the structs above — because it is the path a gateway +calls a thousand times a second and the one where a C caller should never touch a JSON library. +Everything else crosses the boundary as **JSON text**: the same request body the REST endpoint +accepts and the same response body it returns. + +```c +datahub_status datahub_events_create_json(datahub_client *c, const char *request_json, char **response_json); +``` + +This is not a raw pass-through. The body is deserialized into the SDK's typed structs and sent +through the same service method Rust and Python callers use — so the durable buffer, chunking +and auth apply, and a body whose fields have the wrong type is rejected before any request is made — +then the typed response is serialized back. The point is that the *C surface* stays small and +stable while the SDK's own types can keep changing underneath it, and the REST API reference +doubles as the documentation for every JSON function. A C++ caller uses whatever JSON library +it already has; a C caller on the hot path never needs one. + +### Version 1 surface + +Ingest-first: what a device that produces datapoints and events, and reacts to subscription +messages, needs. Nothing else. + +| Group | Functions | +|---|---| +| Version | `datahub_version()` returning the crate version string; `DATAHUB_VERSION_{MAJOR,MINOR,PATCH}` macros in the header | +| Config | `datahub_config_new` / `_free`; `_from_env` (process environment only); `_load_envfile(path)`; setters for base URL, bearer token, client credentials (id, secret, token URI), scope, audience, the assertion set, and the three buffer bounds plus `_enable_buffering` | +| Client | `datahub_client_new(config, &client)`, `datahub_client_free`; `datahub_client_flush` to drain the spool on demand (before a controlled shutdown), `datahub_client_spool_bytes` | +| Time series | `datahub_timeseries_get_by_external_id` → id, name, unit, value type; `datahub_timeseries_create_json`; `datahub_timeseries_search_json` | +| Datapoints | `datahub_datapoints_insert(client, external_id, points, n)`; `datahub_datapoints_insert_str` for string-valued series; `datahub_datapoints_latest`; `datahub_datapoints_retrieve(client, external_id, start_ms, end_ms, limit, &out, &n)` (raw); aggregated reads via `_retrieve_json` | +| Events | `datahub_events_create_json`, `datahub_events_filter_json` | +| Subscriptions | `datahub_listener_open(client, ids, n, &l)`; `datahub_listener_next(l, timeout_ms, &msg)`; `datahub_listener_ack` / `_nack(l, ids, n)`; `datahub_listener_subscribe` / `_unsubscribe`; `datahub_listener_close` | +| Messages | `datahub_message_id`, `datahub_message_json` (borrowed, valid until `_free`), `datahub_message_datapoints(msg, &points, &n)` for the typed fast path when the message is a datapoint batch; `datahub_message_free` | + +`datahub_client_flush` needs a public "drain now" method on the core's timeseries and events +services; today draining only happens as a side effect of the next insert. Small addition, made +in the core, not the FFI crate — the rule throughout is that anything the C layer needs and the +core lacks is added to the core, so Rust and Python callers get it too. + +The listener is **pull-based** with a timeout, mirroring `SubscriptionListener::next`, rather +than callback-based. A callback API has to define which thread the callback runs on, what the +callback may call, and what happens if it blocks; a `next(timeout)` loop defines none of that +and is what every C event loop already knows how to integrate. A callback variant can be added +later on top of the same handle if a consumer needs it. + +### Deliberately not in version 1 + +Resources and the graph, datasets, files, labels, functions, edges, policies, units. Each is a +JSON function pair away when a consumer asks — that is what the JSON convention buys — but not +before. Also out: a C++ header, async/callback APIs, and any registry packaging (vcpkg, Conan); +the library ships as release tarballs, below. + +## Illustration + +```c +#include + +datahub_config *cfg = datahub_config_new(); +datahub_config_set_base_url(cfg, "https://datahub.example.com"); +datahub_config_set_client_credentials(cfg, client_id, client_secret, token_uri); +datahub_config_set_scope(cfg, "organization:*"); +datahub_config_set_buffer_dir(cfg, "/var/lib/gateway/datahub-spool"); /* also enables buffering */ + +datahub_client *client = NULL; +if (datahub_client_new(cfg, &client) != DATAHUB_OK) { + fprintf(stderr, "datahub: %s\n", datahub_last_error()); + datahub_config_free(cfg); + return 1; +} +datahub_config_free(cfg); /* the client copied what it needs */ + +datahub_datapoint points[] = { { .timestamp_ms = now_ms(), .value = 21.5 } }; +switch (datahub_datapoints_insert(client, "pump-1/temperature", points, 1)) { + case DATAHUB_OK: break; /* on the server */ + case DATAHUB_BUFFERED: break; /* on disk, will flush */ + default: fprintf(stderr, "datahub: %s\n", datahub_last_error()); /* fix and retry */ +} + +const char *subs[] = { "pump-1-alarms" }; +datahub_listener *l = NULL; +if (datahub_listener_open(client, subs, 1, &l) == DATAHUB_OK) { + datahub_message *msg = NULL; + for (;;) { + datahub_status st = datahub_listener_next(l, 5000, &msg); + if (st == DATAHUB_TIMEOUT) continue; /* idle; check a stop flag here */ + if (st != DATAHUB_OK) break; /* DATAHUB_CLOSED or an error */ + handle(datahub_message_json(msg)); + const char *id = datahub_message_id(msg); + datahub_listener_ack(l, &id, 1); + datahub_message_free(msg); + } + datahub_listener_close(l); +} + +datahub_client_flush(client); /* push any spooled backlog before exit */ +datahub_client_free(client); +``` + +## Build, test, release + +**Header.** `cbindgen` generates `include/intellistream_datahub.h` from the `#[no_mangle] +extern "C"` items and `#[repr(C)]` types. The generated file is **committed**, so a C user +clones or downloads and has a header; CI regenerates it and fails on a diff, the same way a +stale `.so` is treated on the Python side. + +**Tests.** Three layers, cheapest first: + +1. Rust unit tests in the crate for the boundary itself: NULL handling, invalid UTF-8, the + panic-to-status conversion, thread-local error state, every `_free` on every path. +2. `tests/c/smoke.c`, compiled with the system C compiler against `target/` and run in + CI **without a backend**: `datahub_version`, config construction and its error cases, and the + offline buffering path — point the client at an unreachable base URL with a temp spool + directory, insert, assert `DATAHUB_BUFFERED` and that a spool segment exists. That exercises + the most important behaviour of the whole library with no network. +3. Live-backend tests following the Rust suite's `.env` convention and `rust_sdk_`-style + prefixing, skipped when `.env` is absent, as `multi_tenant_integration.rs` does. + +**Versions.** The CI `versions` job compares three manifests today; the C crate's +`Cargo.toml` becomes the fourth. A `vX.Y.Z` tag releases all four together, so the C library +version always names the crate version it was built from. + +**Artifacts.** The C SDK is not published to a registry. `release.yml` gains a job that builds, +per target, a tarball of the header plus the shared and static libraries, and attaches them to +the GitHub Release. The target list is the one `build-wheels.yml` already proves works, which +is exactly the list that matters for edge hardware: + +| OS | Targets | +|---|---| +| Linux glibc | x86_64, i686, aarch64, armv7 | +| Linux musl | x86_64, aarch64 | +| Windows | x64, arm64 (`.dll` + import library) | +| macOS | x86_64, arm64 | + +A static library built from reqwest + rustls + Tokio is several megabytes and links against +`pthread`, `dl` and `m` on Linux; the tarball ships a `pkg-config` file so the consumer does not +have to know that. Fine for a gateway, wrong for a microcontroller — see the hosted-OS note +above. + +**Licence.** The repository is Apache-2.0 (`LICENSE`, `NOTICE`, `Cargo.toml`), which is what a +library statically linked into a customer's firmware has to be; the C crate inherits it. + +## Open points to settle before starting + +- **First consumer and their first slice.** The version-1 surface above is a floor. Whoever + links this first tells us which JSON pairs to add on day one. +- **Symbol prefix.** `datahub_` is short and unlikely to collide. The alternative, + `intellistream_datahub_`, matches the library and Python names but is unpleasant at every + call site. Decide once; it cannot change later. +- **Gating `process_response`'s stdout printing.** Feature flag or runtime config, and whether + the Python bindings should also default it off. Needs a decision in the core before any of + this ships. +- **Windows toolchain.** MSVC-built `.dll` + `.lib` only, or a MinGW build too. Default to + MSVC only until someone asks. + +## Status + +Implemented in `datahub_c_bindings/`, together with the two core changes above — +`http::set_debug_output` (on by default for the crate, off in the C library) and +`DataHubConfig::from_map` (so the C layer never reads `.env` from the host's cwd) — and +`flush_buffer` on both spooling services. Where the crate departs from the plan: + +- **Every export is written out by hand.** cbindgen does not expand `macro_rules!`; a + macro-generated function ends up in the library but not in the header. +- **Two more statuses:** `DATAHUB_NOT_FOUND` (an empty answer to a single lookup) and + `DATAHUB_SUBSCRIPTION` (the listener's per-subscription error, which leaves the stream open). +- **`datahub_request_json`**, an authenticated raw `GET`/`POST` to any endpoint, means nothing + is "a JSON function pair away": the typed and `_json` functions cover ingest, lookup, events + and the listener, and every other endpoint is reachable today. +- **`datahub_client_buffered_count`** reports spooled records, which is what the core counts, + rather than bytes. +- **`datahub_config_get`** and a generic `datahub_config_set(key, value)` exist beside the typed + setters, so every configuration key has a C spelling. +- **A static `TOKEN` now survives a 401** in the core. It cannot be re-minted, and dropping it + turned every later call into "OAuth2 Client not configured". +- **Retention is measured on the data's own timestamp.** A backfill older than the window is + reported buffered but does not survive in the spool; the insert functions say so. +- **Not done yet:** the Cargo workspace (the crate is standalone, like the Python one), + cross-compiled release tarballs (the release workflow builds natively on Linux, macOS and + Windows and keeps the result as workflow artifacts), and a `pkg-config` file. diff --git a/run_c_tests.sh b/run_c_tests.sh new file mode 100755 index 0000000..a38a19b --- /dev/null +++ b/run_c_tests.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# +# Build-and-test wrapper for the C bindings (datahub_c_bindings/). +# +# Runs, in order: `cargo build` (which also regenerates include/intellistream_datahub.h), +# `cargo test` (the boundary tests, the mock-api tests, and the live tests — which print SKIP +# and pass without a BASE_URL), then compiles tests/c/smoke.c with the system C compiler against +# the freshly built shared library and runs it. No backend is needed for anything but the live +# tests. +# +# Usage: +# ./run_c_tests.sh # everything +# ./run_c_tests.sh --release # optimized build (what a release ships) +# ./run_c_tests.sh --check-header # additionally fail if the committed header is stale (CI) +# ./run_c_tests.sh --smoke-only # skip cargo test; just build and run the C smoke test +# ./run_c_tests.sh -- -k buffering # everything after `--` goes to `cargo test` +# +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CRATE_DIR="$REPO_ROOT/datahub_c_bindings" + +RELEASE=0 +CHECK_HEADER=0 +SMOKE_ONLY=0 +CARGO_TEST_ARGS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --release) RELEASE=1; shift ;; + --check-header) CHECK_HEADER=1; shift ;; + --smoke-only) SMOKE_ONLY=1; shift ;; + -h|--help) + sed -n '2,17p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit 0 + ;; + --) shift; CARGO_TEST_ARGS+=("$@"); break ;; + *) CARGO_TEST_ARGS+=("$1"); shift ;; + esac +done + +log() { printf '\033[1;34m==>\033[0m %s\n' "$*"; } +die() { printf '\033[1;31merror:\033[0m %s\n' "$*" >&2; exit 1; } + +command -v cargo >/dev/null 2>&1 || die "cargo not found on PATH" +command -v cc >/dev/null 2>&1 || die "no C compiler (cc) on PATH" + +cd "$CRATE_DIR" + +profile=debug +build_flags=() +if [[ $RELEASE -eq 1 ]]; then + profile=release + build_flags+=(--release) +fi +target_dir="${CARGO_TARGET_DIR:-$CRATE_DIR/target}" +lib_dir="$target_dir/$profile" + +log "Building datahub_c_bindings ($profile) — this also regenerates include/intellistream_datahub.h" +cargo build "${build_flags[@]}" + +if [[ $CHECK_HEADER -eq 1 ]]; then + log "Checking that the committed header matches the sources" + if ! git -C "$REPO_ROOT" diff --exit-code -- datahub_c_bindings/include; then + die "include/intellistream_datahub.h is stale: commit the regenerated header" + fi +fi + +if [[ $SMOKE_ONLY -eq 0 ]]; then + log "Running the crate's Rust tests (boundary, mock api, live-if-configured)" + cargo test "${build_flags[@]}" -- ${CARGO_TEST_ARGS[@]+"${CARGO_TEST_ARGS[@]}"} +fi + +log "Compiling tests/c/smoke.c against $lib_dir" +smoke_bin="$lib_dir/datahub_c_smoke" +extra_libs=(-lpthread -ldl -lm) +[[ "$(uname -s)" == "Darwin" ]] && extra_libs=(-lm) +cc -std=c11 -Wall -Wextra -Werror \ + -I include tests/c/smoke.c \ + -L "$lib_dir" -lintellistream_datahub "${extra_libs[@]}" \ + -o "$smoke_bin" + +log "Running the smoke test" +LD_LIBRARY_PATH="$lib_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ +DYLD_LIBRARY_PATH="$lib_dir${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" \ + "$smoke_bin" diff --git a/src/datahub.rs b/src/datahub.rs index bc26d96..01eb9e4 100644 --- a/src/datahub.rs +++ b/src/datahub.rs @@ -248,7 +248,11 @@ impl DataHubConfig { } } - pub(crate) fn from_map(map: HashMap) -> Result { + /// Build a config from a map of the same keys the environment uses (`BASE_URL`, `TOKEN`, + /// `CLIENT_ID`, …, `BUFFER_DIR`), without reading or touching the process environment. This is + /// what [`from_env`](Self::from_env) does with `std::env::vars()`; the map form is for hosts + /// that assemble configuration themselves, such as the C bindings. + pub fn from_map(map: HashMap) -> Result { let baseurl = map.get("BASE_URL") .ok_or_else(|| DataHubError::ConfigError( "BASE_URL is not set. Define it in your .env file or export it in the environment (e.g. BASE_URL=http://localhost:8081).".to_string() @@ -500,6 +504,12 @@ impl DataHubConfig { /// seconds too early stays broken until the token expires rather than recovering on its next /// attempt. pub async fn invalidate_token(&self) { + // A user-supplied `TOKEN` cannot be re-minted: dropping it would only replace the + // server's 401 with a misleading "OAuth2 Client not configured" on every later call. + // Keep it, so the caller keeps seeing the real answer until they supply a new one. + if self.oauth2_client.is_none() && !self.has_assertion_exchange() { + return; + } let mut auth_state = self.auth_state.write().await; auth_state.token = None; auth_state.expire_time = None; @@ -692,6 +702,46 @@ pub fn to_snake_lower_cased_allow_start_with_digits(s: &str) -> String { } } +#[cfg(test)] +mod invalidate_tests { + use super::*; + + /// A static `TOKEN` has nothing to fall back to, so a 401 must not discard it: every later + /// call would otherwise fail with "OAuth2 Client not configured" instead of the server's answer. + #[test] + fn a_static_token_survives_invalidation() { + let config = DataHubConfig::from_vars( + "http://127.0.0.1:9".to_string(), + Some("static-token".to_string()), + None, + None, + None, + None, + ); + crate::block_on(async { + config.invalidate_token().await; + assert_eq!(config.get_api_token().await.unwrap(), "static-token"); + }); + } + + /// With client credentials configured the cached token is dropped, so the next call mints one. + #[test] + fn a_minted_token_is_dropped_on_invalidation() { + let config = DataHubConfig::from_vars( + "http://127.0.0.1:9".to_string(), + Some("cached-token".to_string()), + Some("http://127.0.0.1:9/token".to_string()), + Some("id".to_string()), + Some("secret".to_string()), + None, + ); + crate::block_on(async { + config.invalidate_token().await; + assert!(config.auth_state.read().await.token.is_none()); + }); + } +} + #[cfg(test)] mod scope_tests { use super::*; diff --git a/src/events/mod.rs b/src/events/mod.rs index 353a056..f215841 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -83,6 +83,20 @@ impl EventsService { self.spool.lock().unwrap().as_ref().map_or(0, |s| s.size()) } + /// Send whatever the durable event spool holds, oldest segment first, without creating + /// anything new. Returns `true` when the spool is empty afterwards and `false` when the server + /// is still unreachable (the backlog stays on disk). Always `true` when buffering is off. + pub async fn flush_buffer(&self) -> bool { + let svc = self.get_api_service(); + if !svc.config.buffering_enabled() { + return true; + } + self.ensure_spool(&svc.config); + drop(svc); // don't hold the ApiService Arc across awaits + let path = format!("{}/create", self.base_url); + self.drain_spool(&path, Utc::now().timestamp_millis()).await + } + fn ensure_spool(&self, config: &DataHubConfig) { let mut guard = self.spool.lock().unwrap(); if guard.is_none() { diff --git a/src/files/mod.rs b/src/files/mod.rs index 1b8834d..6933366 100644 --- a/src/files/mod.rs +++ b/src/files/mod.rs @@ -152,7 +152,7 @@ impl FileService { let mime_type = header_value(&response, reqwest::header::CONTENT_TYPE); let status = response.status(); let bytes = response.bytes().await.map_err(|err| { - eprintln!("Failed to read download body: {}", err); + debug_eprintln!("Failed to read download body: {}", err); ResponseError { status, message: err.to_string(), @@ -187,7 +187,7 @@ impl FileService { let mut file = File::create(destination.as_ref()).await.map_err(io_error)?; let mut written: u64 = 0; while let Some(chunk) = response.chunk().await.map_err(|err| { - eprintln!("Download stream failed: {}", err); + debug_eprintln!("Download stream failed: {}", err); ResponseError { status, message: err.to_string(), @@ -398,11 +398,11 @@ impl FileUpload { let kind: Option = match infer::get_from_path(file_path) { Ok(Some(file_type)) => Some(file_type.mime_type().to_string()), Ok(None) => { - println!("Could not determine file type for: {}", file_path); + debug_println!("Could not determine file type for: {}", file_path); Some("application/octet-stream".to_string()) } Err(e) => { - eprintln!("Error detecting file type for {}: {}", file_path, e); + debug_eprintln!("Error detecting file type for {}: {}", file_path, e); None } }; diff --git a/src/generic.rs b/src/generic.rs index f324b6e..cfa1e3d 100644 --- a/src/generic.rs +++ b/src/generic.rs @@ -743,7 +743,7 @@ pub trait ApiServiceProvider { .send() .await .map_err(|err| { - eprintln!("HTTP request failed: {}", err); + debug_eprintln!("HTTP request failed: {}", err); ResponseError::from_err(err) })? } else { @@ -754,7 +754,7 @@ pub trait ApiServiceProvider { .send() .await .map_err(|err| { - eprintln!("HTTP request failed: {}", err); + debug_eprintln!("HTTP request failed: {}", err); ResponseError::from_err(err) })? }; @@ -782,13 +782,13 @@ pub trait ApiServiceProvider { .send() .await .map_err(|err| { - eprintln!("HTTP request failed: {}", err); + debug_eprintln!("HTTP request failed: {}", err); ResponseError::from_err(err) })?; if response.status() == 204 { // Return deserialized `T` with an empty body and the HTTP status code T::deserialize_and_set_status("", response.status().as_u16()).map_err(|err| { - eprintln!("Failed to create object from empty response: {}", err); + debug_eprintln!("Failed to create object from empty response: {}", err); ResponseError { status: response.status(), message: err.to_string(), @@ -825,7 +825,7 @@ pub trait ApiServiceProvider { } let response = request.send().await.map_err(|err| { - eprintln!("HTTP file upload request failed: {}", err); + debug_eprintln!("HTTP file upload request failed: {}", err); ResponseError::from_err(err) })?; match process_response::(response, path).await { @@ -855,7 +855,7 @@ pub trait ApiServiceProvider { .send() .await .map_err(|err| { - eprintln!("HTTP request failed: {}", err); + debug_eprintln!("HTTP request failed: {}", err); ResponseError::from_err(err) })?; @@ -870,7 +870,7 @@ pub trait ApiServiceProvider { if status == http::StatusCode::UNAUTHORIZED { self.get_api_service().config.invalidate_token().await; } - eprintln!("Request failed with status: {status}"); + debug_eprintln!("Request failed with status: {status}"); Err(explain_auth_failure( ResponseError { status, @@ -988,7 +988,7 @@ where }) } else { // For non-2xx responses (errors) - eprintln!( + debug_eprintln!( "HTTP request failed with status code {}: {}", status_code, body ); @@ -1002,7 +1002,7 @@ where }) { Ok(result) => Ok(result), Err(_) => { - eprintln!("Error parsing HTTP response body: {}", body); + debug_eprintln!("Error parsing HTTP response body: {}", body); let mut wrapper: DataWrapper = DataWrapper::new(); wrapper.error_body = Some(body.to_string()); wrapper.set_http_status_code(status_code); diff --git a/src/graph_data_wrapper.rs b/src/graph_data_wrapper.rs index ee5b283..93d71e7 100644 --- a/src/graph_data_wrapper.rs +++ b/src/graph_data_wrapper.rs @@ -81,7 +81,7 @@ impl DataWrapperDeserializ wrapper }) } else { - eprintln!( + debug_eprintln!( "HTTP request failed with status code {}: {}", status_code, body ); @@ -91,7 +91,7 @@ impl DataWrapperDeserializ }) { Ok(result) => Ok(result), Err(_) => { - eprintln!("Error parsing HTTP response body: {}", body); + debug_eprintln!("Error parsing HTTP response body: {}", body); Ok(GraphDataWrapper { nodes: None, relations: None, diff --git a/src/http.rs b/src/http.rs index 55d37d6..298070f 100644 --- a/src/http.rs +++ b/src/http.rs @@ -3,8 +3,25 @@ use oauth2::http::StatusCode; use reqwest::{Error, Response}; use serde::de::DeserializeOwned; use std::fmt; +use std::sync::atomic::{AtomicBool, Ordering}; use thiserror::Error; +/// Whether the SDK prints request/response tracing (response bodies, batch progress, failed-request +/// notices) to stdout/stderr. On by default, as it always has been: it is the quickest way to see +/// what the api actually answered. Off is for hosts that embed the SDK as a library — a library +/// must not write to streams it does not own — which is why the C bindings default it off. +static DEBUG_OUTPUT: AtomicBool = AtomicBool::new(true); + +/// Turn the SDK's console tracing on or off for the whole process. See [`DEBUG_OUTPUT`]. +pub fn set_debug_output(enabled: bool) { + DEBUG_OUTPUT.store(enabled, Ordering::Relaxed); +} + +/// Whether [`set_debug_output`] has left console tracing on (the default). +pub fn debug_output_enabled() -> bool { + DEBUG_OUTPUT.load(Ordering::Relaxed) +} + #[derive(Debug, Error, Clone)] pub struct ResponseError { pub(crate) status: StatusCode, @@ -93,7 +110,7 @@ where if (200..300).contains(&status.as_u16()) { // Read the response body and attempt to deserialize let body = response.text().await.map_err(|err| { - eprintln!("Failed to read response body: {err}",); + debug_eprintln!("Failed to read response body: {err}",); ResponseError { status, message: err.to_string(), @@ -102,11 +119,11 @@ where let max_chars = 2000; let truncated_body = &body[..body.len().min(max_chars)]; - println!("Response body for path: {}\n{}", path, &truncated_body); // Debug output + debug_println!("Response body for path: {}\n{}", path, &truncated_body); // Debug output // Conditionally apply custom or default logic let result: T = T::deserialize_and_set_status(&body, status.as_u16()).map_err(|err| { - eprintln!("Failed to deserialize JSON: {err}",); + debug_eprintln!("Failed to deserialize JSON: {err}",); ResponseError { status, message: err.to_string(), @@ -116,7 +133,7 @@ where Ok(result) } else { let status = response.status(); - eprintln!("Request failed with status: {status}",); + debug_eprintln!("Request failed with status: {status}",); Err(ResponseError { status, message: response diff --git a/src/lib.rs b/src/lib.rs index 0410a7e..5abb70a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,6 +16,26 @@ pub use crate::labels::LabelsService; pub use crate::relations::EdgesService; pub use crate::subscriptions::SubscriptionsService; +/// `println!` that honours [`http::set_debug_output`]. The SDK's request/response tracing — +/// response bodies, batch progress, failed-request notices — goes through these two macros so a +/// host that embeds the SDK as a library (the C bindings, say) can silence it in one call. +macro_rules! debug_println { + ($($arg:tt)*) => { + if $crate::http::debug_output_enabled() { + println!($($arg)*); + } + }; +} + +/// `eprintln!` counterpart of [`debug_println!`]. +macro_rules! debug_eprintln { + ($($arg:tt)*) => { + if $crate::http::debug_output_enabled() { + eprintln!($($arg)*); + } + }; +} + /// Explaining an unexplained 401 from the token the SDK already holds. pub(crate) mod auth_diagnostics; #[cfg(feature = "blocking")] diff --git a/src/nodes.rs b/src/nodes.rs index 644938a..ca9fbed 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -819,8 +819,6 @@ mod tests { assert_eq!(ts.unit_external_id.as_deref(), Some("deg_c")); assert_eq!(ts.value_type.as_deref(), Some("float")); assert_eq!(ts.table_engine.as_deref(), Some("MERGETREE")); - // Raw numbers on the wire, unlike every id in the family. - assert_eq!(ts.security_categories, Some(vec![1, 2])); assert_eq!(ts.data_set_id, Some(21)); assert_eq!(ts.labels.as_deref(), Some(&["TIMESERIES".to_string()][..])); } @@ -864,7 +862,6 @@ mod tests { assert_eq!(ts.value_type, None, "not told, rather than a wrong default"); assert_eq!(ts.unit, None); assert_eq!(ts.table_engine, None); - assert_eq!(ts.security_categories, None); } #[test] diff --git a/src/resources/tests.rs b/src/resources/tests.rs index 9be7278..06ea463 100644 --- a/src/resources/tests.rs +++ b/src/resources/tests.rs @@ -397,7 +397,6 @@ async fn neo4j_persists_expected_fields_per_node_type() -> Result<(), Box bool { + let svc = self.get_api_service(); + if !svc.config.buffering_enabled() { + return true; + } + self.ensure_spool(&svc.config); + drop(svc); // don't hold the ApiService Arc across awaits + let path = format!("{}/data", self.base_url); + self.drain_spool(&path, Utc::now().timestamp_millis()).await + } + fn ensure_spool(&self, config: &DataHubConfig) { let mut guard = self.spool.lock().unwrap(); if guard.is_none() { @@ -328,7 +344,7 @@ impl TimeSeriesService { if total_datapoints > MAX_DATAPOINTS_PER_REQUEST { while total_datapoints > MAX_DATAPOINTS_PER_REQUEST { - println!("Total datapoints left: {}", total_datapoints); + debug_println!("Total datapoints left: {}", total_datapoints); // Divide the request into multiple batch requests let mut new_json: DataWrapper> = DataWrapper::new(); @@ -345,7 +361,7 @@ impl TimeSeriesService { let batch_size: usize = MAX_DATAPOINTS_PER_REQUEST / active_timeseries_with_datapoints.len(); - println!("Current Batch size: {}", batch_size); + debug_println!("Current Batch size: {}", batch_size); if orig_dp_collection.datapoints.len() > batch_size { let chunk: Vec = orig_dp_collection.datapoints.drain(..batch_size).collect(); @@ -356,7 +372,7 @@ impl TimeSeriesService { .iter() .position(|&x| x == orig_dp_collection.hash()) { - println!("Remove datacollection: {}", orig_dp_collection.to_string()); + debug_println!("Remove datacollection: {}", orig_dp_collection.to_string()); active_timeseries_with_datapoints.remove(pos); } } else { @@ -366,14 +382,14 @@ impl TimeSeriesService { } new_json.add_item(new_dp_collection.clone()); total_datapoints = total_datapoints - new_dp_collection.datapoints.len(); - println!("Total datapoints left: {}", total_datapoints); + debug_println!("Total datapoints left: {}", total_datapoints); } let mut new_total_datapoints: usize = 0; for dp_collection in new_json.get_items().iter() { new_total_datapoints += dp_collection.datapoints.len(); } - println!( + debug_println!( "Sending insert datapoints request with {} datapoints.", new_total_datapoints ); @@ -390,10 +406,10 @@ impl TimeSeriesService { Ok(ref r) => { // The backend acknowledges a successful insert with 204 No Content. assert_eq!(r.get_http_status_code().unwrap(), 204); - println!("Successfully inserted datapoints."); + debug_println!("Successfully inserted datapoints."); } Err(e) => { - eprintln!("{}", e.message); + debug_eprintln!("{}", e.message); panic!("Error inserting datapoints: {:?}", e.get_message()); } }); @@ -405,7 +421,7 @@ impl TimeSeriesService { for dp_collection in json.get_items().iter() { total_datapoints += dp_collection.datapoints.len(); } - println!("Final request: Total datapoints left: {}", total_datapoints); + debug_println!("Final request: Total datapoints left: {}", total_datapoints); self.execute_post_request::, _>(path, json) .await }