[WIP]Feat/c sdk

.github/workflows/ci.yml

@@ -20,9 +20,10 @@ jobs:
           crate=$(grep -m1 '^version' Cargo.toml | cut -d'"' -f2)
           binding=$(grep -m1 '^version' datahub_python_bindings/Cargo.toml | cut -d'"' -f2)
           py=$(grep -m1 '^version' datahub_python_bindings/pyproject.toml | cut -d'"' -f2)
-          echo "crate=$crate binding=$binding pyproject=$py"
-          if [ "$crate" != "$binding" ] || [ "$crate" != "$py" ]; then
-            echo "::error::manifest versions disagree: $crate / $binding / $py"
+          c=$(grep -m1 '^version' datahub_c_bindings/Cargo.toml | cut -d'"' -f2)
+          echo "crate=$crate binding=$binding pyproject=$py c=$c"
+          if [ "$crate" != "$binding" ] || [ "$crate" != "$py" ] || [ "$crate" != "$c" ]; then
+            echo "::error::manifest versions disagree: $crate / $binding / $py / $c"
             exit 1
           fi
 
@@ -42,3 +43,17 @@ jobs:
     name: Wheels
     needs: versions
     uses: ./.github/workflows/build-wheels.yml
+
+  c-sdk:
+    name: C bindings build, tests and header
+    runs-on: ubuntu-latest
+    needs: versions
+    steps:
+      - uses: actions/checkout@v6
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: datahub_c_bindings
+      # Builds the crate (which regenerates include/intellistream_datahub.h), runs its Rust
+      # tests, fails if the committed header is stale, then compiles tests/c/smoke.c with the
+      # system C compiler against the built library and runs it. No backend is involved.
+      - run: ./run_c_tests.sh --check-header

.github/workflows/release.yml

@@ -10,6 +10,7 @@ on:
       - Cargo.toml
       - datahub_python_bindings/Cargo.toml
       - datahub_python_bindings/pyproject.toml
+      - datahub_c_bindings/Cargo.toml
       - .github/workflows/release.yml
       - .github/workflows/build-wheels.yml
 
@@ -27,9 +28,10 @@ jobs:
           crate=$(grep -m1 '^version' Cargo.toml | cut -d'"' -f2)
           binding=$(grep -m1 '^version' datahub_python_bindings/Cargo.toml | cut -d'"' -f2)
           py=$(grep -m1 '^version' datahub_python_bindings/pyproject.toml | cut -d'"' -f2)
-          echo "crate=$crate binding=$binding pyproject=$py ref=${GITHUB_REF_NAME:-}"
-          if [ "$crate" != "$binding" ] || [ "$crate" != "$py" ]; then
-            echo "::error::manifest versions disagree: $crate / $binding / $py"
+          c=$(grep -m1 '^version' datahub_c_bindings/Cargo.toml | cut -d'"' -f2)
+          echo "crate=$crate binding=$binding pyproject=$py c=$c ref=${GITHUB_REF_NAME:-}"
+          if [ "$crate" != "$binding" ] || [ "$crate" != "$py" ] || [ "$crate" != "$c" ]; then
+            echo "::error::manifest versions disagree: $crate / $binding / $py / $c"
             exit 1
           fi
           # Only a tag push carries a version to check against; a rehearsal has none.
@@ -61,6 +63,37 @@ jobs:
     needs: versions
     uses: ./.github/workflows/build-wheels.yml
 
+  c-sdk:
+    # The C library is not published to a registry: each OS build is kept as a workflow
+    # artifact (header + shared + static library) for attaching to the release by hand. It does
+    # not gate the crate or wheel publication.
+    name: C bindings (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    needs: versions
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v6
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: datahub_c_bindings
+      - run: cargo build --release
+        working-directory: datahub_c_bindings
+      - uses: actions/upload-artifact@v6
+        with:
+          name: c-sdk-${{ matrix.os }}
+          if-no-files-found: error
+          path: |
+            datahub_c_bindings/include/intellistream_datahub.h
+            datahub_c_bindings/target/release/libintellistream_datahub.so
+            datahub_c_bindings/target/release/libintellistream_datahub.dylib
+            datahub_c_bindings/target/release/libintellistream_datahub.a
+            datahub_c_bindings/target/release/intellistream_datahub.dll
+            datahub_c_bindings/target/release/intellistream_datahub.dll.lib
+            datahub_c_bindings/target/release/intellistream_datahub.lib
+
   pypi:
     name: Publish to PyPI
     runs-on: ubuntu-latest

.gitignore

@@ -4,6 +4,7 @@ Cargo.lock
 .env
 CLAUDE.local.md
 /datahub_python_bindings/target
+/datahub_c_bindings/target
 /datahub_python_bindings/python/intellistream_datahub_sdk/_core.abi3.so
 *.py[cod]
 

AGENTS.md

@@ -11,6 +11,7 @@ cargo test -- --ignored                  # run tests marked #[ignore] (e.g. long
 cargo test <path>::tests::<name>         # e.g. `events::tests::test_events_full`
 cargo test -- --nocapture                # show println! from tests (the SDK prints response bodies)
 ./run_python_tests.sh                    # Python-bindings suite (rebuilds the PyO3 module first — see below)
+./run_c_tests.sh                         # C-bindings suite: cargo test + the C smoke test, no backend needed (see below)
 ```
 
 Most tests are integration tests that call a live backend via `create_api_service()`. They read configuration from a local `.env` file (gitignored). Required:
@@ -108,6 +109,8 @@ Synchronous mirror of the async API behind the `blocking` cargo feature — the
 
 ### Durable ingest buffering (`src/buffer.rs`, integration tests in `src/buffer_integration.rs`)
 
+`TimeSeriesService::flush_buffer` / `EventsService::flush_buffer` drain a spool without ingesting anything new (for a controlled shutdown, or a host retrying on its own clock); `buffered_count` reports what is held. Retention is measured on each record's own timestamp, so a backfill older than the window is not kept.
+
 When a datapoint/event send can't get through, ingestion spools to a segmented, zstd-compressed NDJSON log on disk and flushes automatically on a later ingest call. Invariants to preserve: memory use is bounded by a single segment (plain append-only active segment, zstd-sealed at ~50 MiB rollover via temp file + atomic rename, drained oldest-first one segment at a time); bounded by time retention (whole segments past the window dropped, expired records skipped on read) and a size cap (oldest segment deleted); a torn trailing line from an unclean shutdown is skipped on read. Each on-disk line is `<epoch_millis>\t<json>`; the spool is content-agnostic.
 
 ### The `ApiServiceProvider` trait (`src/generic.rs`)
@@ -372,10 +375,43 @@ behind it — which is exactly how the label table filled up. Mint a unique name
 owns the definition's lifecycle (create/rename/delete), where a shared row would be pulled out from
 under another test, and use a fixed *pair* when a test has to tell two labels apart.
 
+## C bindings (`datahub_c_bindings/`)
+
+A thin FFI crate that builds the SDK as `libintellistream_datahub` (cdylib + staticlib) with a
+cbindgen-generated header at `datahub_c_bindings/include/intellistream_datahub.h`. `docs/c-sdk-design.md`
+records why it exists and what is deliberately not in it. Things to know when touching it:
+
+- **Every export is written out by hand.** cbindgen does not expand `macro_rules!`, so a
+  macro-generated `extern "C"` function ends up in the library but not in the header — and a C
+  caller cannot see it. Share bodies through private helper functions instead.
+- **The header is generated by `build.rs` and committed.** `cargo build` in the crate rewrites it;
+  CI runs `./run_c_tests.sh --check-header`, which fails on a diff. Commit the regenerated header
+  with the change that caused it. The version macros and `DATAHUB_TIME_UNSET` are added by
+  `build.rs`, not by cbindgen.
+- **Every export runs inside `error::guard`** (`catch_unwind`): a panic becomes `DATAHUB_PANIC`
+  with the message in the thread-local `datahub_last_error()`. Never let a panic reach C.
+- **Runtime model:** a `datahub_client` owns a Tokio runtime and `block_on`s the async
+  `ApiService` directly — not the blocking client, which has no subscriptions. A listener shares
+  the runtime (`Arc`), so client and listener may be freed in either order.
+- **Typed on the datapoint hot path, JSON everywhere else.** A `..._json` function takes exactly
+  the REST request body and returns exactly the response body (`items` plus `nextCursor`, which
+  `DataWrapper` itself skips when serializing); `datahub_request_json` is the raw authenticated
+  escape hatch for any endpoint without a dedicated function.
+- **Core switches it depends on:** `http::set_debug_output(false)` (the core's stdout/stderr
+  tracing, on by default for Rust and Python users), `DataHubConfig::from_map` (one config path
+  for env, env file and typed setters; the C layer never reads `.env` from the host's cwd), and
+  `TimeSeriesService::flush_buffer` / `EventsService::flush_buffer` behind `datahub_client_flush`.
+- **Tests:** unit tests in the crate; `tests/offline.rs` (boundary and spool paths, no network);
+  `tests/mock_api.rs` (a tiny HTTP mock asserting what goes on the wire); `tests/live.rs` (skips
+  without a `BASE_URL`); `tests/c/smoke.c`, compiled and run by `run_c_tests.sh`. The spool's
+  retention window is measured on each record's own timestamp, so a test that expects a record
+  to survive in the spool must stamp it with a recent time.
+- The crate's version is locked to the other three manifests by the CI `versions` job.
+
 ## Conventions
 
 - `#[serde(rename = "camelCase")]` or explicit `#[serde(rename = "...")]` on fields — the backend is camelCase, Rust is snake_case.
 - **A request body naming a field the api does not have is a 400.** Jackson used to drop unknown properties, so a stale or misspelled key was answered with 200 and no effect; a strict converter now rejects the body and names every offender alongside the fields the endpoint accepts. Two consequences for this SDK: a struct that doubles as request *and* response must `#[serde(skip_serializing)]` its response-only fields — `GraphDataWrapper`'s `errorBody`/`httpStatusCode` reached `/resources/create` and made every resource and function create and update a 400 — and one Rust type may not stand in for two endpoints that disagree on their fields (see the search forms above). Reading is unaffected: responses stay lenient in both directions.
 - `externalId` (string, user-supplied) and numeric `id` are both valid identifiers across the API. `IdAndExtId` / `IdAndExtIdCollection` model this choice.
-- `process_response` (`src/http.rs`) prints response bodies to stdout (truncated to 2000 chars). This is deliberate for debugging — don't silently remove it.
+- `process_response` (`src/http.rs`) prints response bodies to stdout (truncated to 2000 chars). This is deliberate for debugging — don't silently remove it. It and the other request-path prints go through the `debug_println!`/`debug_eprintln!` macros, gated by `http::set_debug_output` (on by default; the C bindings turn it off, since a library must not write to streams it does not own).
 - Tests that depend on backend state being empty are brittle; recent fixes moved away from exact-count assertions (see commit `7f0a059`). Don't add new ones.

Cargo.toml

@@ -13,6 +13,8 @@ exclude = [
     "/datahub_python_bindings",
     "/python_tests",
     "/run_python_tests.sh",
+    "/datahub_c_bindings",
+    "/run_c_tests.sh",
     "/resources",
 ]
 

README.md

@@ -128,6 +128,20 @@ the **compiled** module — always use the wrapper script, which rebuilds the bi
 ./run_python_tests.sh -k timeseries  # extra args are forwarded to pytest
 ```
 
+## C bindings
+
+`datahub_c_bindings/` builds this SDK as a C library — `libintellistream_datahub` (shared and
+static) with a cbindgen-generated header, `include/intellistream_datahub.h` — for C, C++, and
+anything with C interop (.NET P/Invoke, Go cgo, LabVIEW, MATLAB). It is ingest-first: typed
+datapoint ingest with the same durable buffering as the Rust crate, time series lookup, event
+creation, the subscription listener, and a JSON convention for every other endpoint. See
+[`datahub_c_bindings/README.md`](datahub_c_bindings/README.md) for usage and
+[`docs/c-sdk-design.md`](docs/c-sdk-design.md) for the reasoning.
+
+```bash
+./run_c_tests.sh   # build the library, run its tests, compile and run the C smoke test
+```
+
 ## Building and testing
 
 ```bash

datahub_c_bindings/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "datahub_c_bindings"
+version = "0.3.0"
+edition = "2021"
+rust-version = "1.85"
+description = "C ABI for the IntelliStream DataHub SDK: libintellistream_datahub plus a cbindgen-generated header."
+license = "Apache-2.0"
+repository = "https://github.com/IntelliStream-DataHub/dataplatform-rust-sdk"
+publish = false
+
+[lib]
+name = "intellistream_datahub"
+# cdylib + staticlib are what C links against; rlib is what the crate's own Rust tests link against.
+crate-type = ["cdylib", "staticlib", "rlib"]
+
+[dependencies]
+intellistream-datahub-sdk = { path = ".." }
+tokio = { version = "1", features = ["rt-multi-thread", "time"] }
+serde = "1"
+serde_json = "1"
+chrono = "0.4"
+
+[build-dependencies]
+cbindgen = "0.29"
+
+[dev-dependencies]
+tempfile = "3"