From 1de1ce5c85a5806d8597918a8cabe3ea007280ed Mon Sep 17 00:00:00 2001 From: Lambdius Date: Sun, 18 Oct 2026 17:56:33 +0200 Subject: [PATCH] feat: add containerized schema publishing & GHCR releases --- .dockerignore | 29 +++--- .env.example | 2 + .github/scripts/packages.sh | 149 ++++++++++++++++++++++++++++ .github/workflows/ci.yml | 6 ++ .github/workflows/publish.yml | 49 ++++++++++ .github/workflows/release.yml | 43 ++++++++ Dockerfile | 25 +++++ Makefile | 14 ++- README.md | 178 ++++++++++++++++++++++++---------- package.json | 2 +- scripts/apply.sh | 97 +++++------------- scripts/check.sh | 13 +-- scripts/logger.sh | 9 +- scripts/registry.sh | 83 ++++++++++++++++ scripts/verify.sh | 105 ++++++++------------ 15 files changed, 581 insertions(+), 223 deletions(-) create mode 100644 .github/scripts/packages.sh create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/release.yml create mode 100644 Dockerfile create mode 100644 scripts/registry.sh diff --git a/.dockerignore b/.dockerignore index a3ad708..c07843b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,14 @@ -node_modules -dist -*.log -pnpm-debug.log* - -.git -.github -.githooks - -.DS_Store -.idea -.vscode - -.env -.env.example +* +!Dockerfile +!catalog.json +!catalog.schema.json +!src/ +!src/grpc/ +!src/grpc/*.proto +!src/kafka/ +!src/kafka/*.json +!scripts/ +!scripts/check.sh +!scripts/registry.sh +!scripts/verify.sh +!scripts/apply.sh diff --git a/.env.example b/.env.example index cc079d1..024931c 100644 --- a/.env.example +++ b/.env.example @@ -5,3 +5,5 @@ SCHEMA_REGISTRY_URL="http://localhost:8081" SCHEMA_REGISTRY_API_PATH="/apis/registry/v3" SCHEMA_REGISTRY_CCOMPAT_PATH="/apis/ccompat/v7" +SCHEMA_REGISTRY_CONNECT_TIMEOUT_SECONDS="5" +SCHEMA_REGISTRY_REQUEST_TIMEOUT_SECONDS="30" diff --git a/.github/scripts/packages.sh b/.github/scripts/packages.sh new file mode 100644 index 0000000..83c7c94 --- /dev/null +++ b/.github/scripts/packages.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GITHUB_TOKEN:?GITHUB_TOKEN is required}" + +readonly RETAINED_BUILDS=10 +readonly PACKAGE="shared-schemas" +readonly IMAGE="ghcr.io/monadiam/$PACKAGE" +readonly LOCAL_IMAGE="monadiam/shared-schemas:ci" +readonly PACKAGE_API="/orgs/MonadIAM/packages/container/$PACKAGE" +readonly VERSION_PATTERN='^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$' +readonly DIGEST_PATTERN='^sha256:[0-9a-f]{64}$' +readonly SHA_TAG_PATTERN='^sha-[0-9a-f]{40}$' + +commit=$(git rev-parse HEAD) +sha_tag="sha-$commit" +versions='[]' +source_digest='' +published_tag='' + +load_versions() { + local packages + + packages=$(gh api --paginate --slurp \ + "/orgs/MonadIAM/packages?package_type=container&per_page=100") + + if jq -e --arg name "$PACKAGE" \ + 'any(.[][]; .name == $name)' <<< "$packages" >/dev/null; then + versions=$(gh api --paginate --slurp \ + "$PACKAGE_API/versions?per_page=100" | jq 'add') + + source_digest=$(jq -r --arg tag "$sha_tag" ' + .[] | select(.metadata.container.tags | index($tag)) | .name + ' <<< "$versions") + fi +} + +publish_commit() { + local revision + + if [[ -n "$source_digest" ]]; then + printf 'Reusing %s:%s\n' "$IMAGE" "$sha_tag" + else + revision=$(docker inspect "$LOCAL_IMAGE" \ + --format '{{index .Config.Labels "org.opencontainers.image.revision"}}') + + if [[ "$revision" == "$commit" ]]; then + docker tag "$LOCAL_IMAGE" "$IMAGE:$sha_tag" + docker push "$IMAGE:$sha_tag" + else + printf 'Image revision does not match %s\n' "$commit" >&2 + exit 1 + fi + fi + + published_tag="$sha_tag" +} + +publish_release() { + local version existing_digest + + : "${GITHUB_REF_NAME:?GITHUB_REF_NAME is required}" + version=$(jq -er '.version' package.json) + + if [[ "$version" =~ $VERSION_PATTERN && + "$GITHUB_REF_NAME" == "v$version" && + "$source_digest" =~ $DIGEST_PATTERN ]]; then + existing_digest=$(jq -r --arg tag "$version" ' + .[] | select(.metadata.container.tags | index($tag)) | .name + ' <<< "$versions") + + if [[ -z "$existing_digest" ]]; then + docker buildx imagetools create --prefer-index=false \ + --tag "$IMAGE:$version" "$IMAGE@$source_digest" + elif [[ "$existing_digest" == "$source_digest" ]]; then + printf 'Reusing release %s\n' "$version" + else + printf 'Release %s already points to another digest\n' "$version" >&2 + exit 1 + fi + else + printf 'Release requires a matching version tag and published SHA image\n' >&2 + exit 1 + fi + + published_tag="$version" +} + +cleanup() { + local candidates id + + candidates=$(jq -r \ + --arg pattern "$SHA_TAG_PATTERN" \ + --arg current "$sha_tag" \ + --argjson keep "$RETAINED_BUILDS" ' + map(select( + .metadata.container.tags | + length > 0 and all(.[]; test($pattern)) + )) + | sort_by(.created_at) | reverse + | .[$keep:][] + | select(.metadata.container.tags | index($current) == null) + | .id + ' <<< "$versions") + + if [[ -n "$candidates" ]]; then + while IFS= read -r id; do + gh api --method DELETE "$PACKAGE_API/versions/$id" + printf 'Deleted intermediate image %s\n' "$id" + done <<< "$candidates" + fi +} + +verify_publication() { + local digest + + digest=$(docker buildx imagetools inspect "$IMAGE:$published_tag" \ + --format '{{.Manifest.Digest}}') + + if [[ "$digest" =~ $DIGEST_PATTERN && + ( -z "$source_digest" || "$digest" == "$source_digest" ) ]]; then + printf 'Image: %s:%s\nDigest: %s@%s\n' \ + "$IMAGE" "$published_tag" "$IMAGE" "$digest" \ + | tee -a "$GITHUB_STEP_SUMMARY" + else + printf 'Unexpected digest: %s\n' "$digest" >&2 + exit 1 + fi +} + +case "${1:?Usage: packages.sh sha|release|cleanup}" in + sha|release) + : "${GITHUB_STEP_SUMMARY:?GITHUB_STEP_SUMMARY is required}" + load_versions + + if [[ "$1" == "sha" ]]; then + publish_commit + else + publish_release + fi + + verify_publication ;; + cleanup) + load_versions + cleanup ;; + *) + printf 'Unknown operation: %s\n' "$1" >&2 + exit 1 ;; +esac diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e2bc36..a1721c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,12 @@ jobs: - name: Build package run: pnpm run build + - name: Build schema publisher image + run: make image IMAGE=monadiam/shared-schemas:ci VCS_REF="$GITHUB_SHA" + + - name: Check schema publisher image + run: docker run --rm --read-only --network none monadiam/shared-schemas:ci scripts/check.sh + ci: name: CI passed if: ${{ always() }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..3ecc310 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,49 @@ +name: Publish schema image + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + publish: + name: Publish schema image + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + packages: write + concurrency: + group: shared-schemas-package + cancel-in-progress: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build image from merged commit + run: make image IMAGE=monadiam/shared-schemas:ci VCS_REF="$GITHUB_SHA" + + - name: Log in to GHCR + run: printf '%s' "$GITHUB_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + + - name: Publish commit image + run: make image-publish + + - name: Prune intermediate images + run: make image-cleanup + + - name: Log out of GHCR + if: ${{ always() }} + run: docker logout ghcr.io diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..4dd08da --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Release schema image + +on: + push: + tags: ["v*"] + +permissions: + contents: read + packages: write + +concurrency: + group: shared-schemas-package + cancel-in-progress: false + +defaults: + run: + shell: bash + +jobs: + release: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Check out release commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Require a commit from main + run: git merge-base --is-ancestor HEAD origin/main + + - name: Log in to GHCR + run: printf '%s' "$GITHUB_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + + - name: Tag published image with package version + run: make image-release + + - name: Log out of GHCR + if: ${{ always() }} + run: docker logout ghcr.io diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..015cb38 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM alpine:3.23 + +ARG VCS_REF +RUN : "${VCS_REF:?VCS_REF build argument is required}" \ + && apk add --no-cache bash ca-certificates curl jq + +LABEL org.opencontainers.image.source="https://github.com/MonadIAM/shared" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.description="MonadIAM schema catalog publisher" + +WORKDIR /opt/shared +ENV SCHEMA_REGISTRY_API_PATH=/apis/registry/v3 \ + SCHEMA_REGISTRY_CCOMPAT_PATH=/apis/ccompat/v7 \ + SCHEMA_REGISTRY_CONNECT_TIMEOUT_SECONDS=5 \ + SCHEMA_REGISTRY_REQUEST_TIMEOUT_SECONDS=30 +COPY catalog.json catalog.schema.json ./ +COPY src/grpc ./src/grpc +COPY src/kafka ./src/kafka +COPY scripts/check.sh scripts/registry.sh scripts/verify.sh scripts/apply.sh ./scripts/ +RUN chmod -R a+rX /opt/shared + +USER 65532:65532 +RUN ./scripts/check.sh +ENTRYPOINT ["/bin/bash"] +CMD ["scripts/apply.sh"] diff --git a/Makefile b/Makefile index fea7288..8143f17 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ -.PHONY: build lint lint-fix tsc knip secrets-check lsp env check apply verify graphify graphify-rebuild graphify-html +.PHONY: build lint lint-fix tsc knip secrets-check lsp env check apply verify image image-publish image-release image-cleanup graphify graphify-rebuild graphify-html GRAPHIFY ?= graphify +IMAGE ?= monadiam/shared-schemas:local +VCS_REF ?= $(shell git rev-parse HEAD) # Development build: @@ -27,6 +29,16 @@ apply: pnpm run apply verify: pnpm run verify +image: + docker build --build-arg VCS_REF="$(VCS_REF)" --tag "$(IMAGE)" . + +# GHCR (GitHub Actions) +image-publish: + bash .github/scripts/packages.sh sha +image-release: + bash .github/scripts/packages.sh release +image-cleanup: + bash .github/scripts/packages.sh cleanup # Local code graph graphify: diff --git a/README.md b/README.md index bdb4d06..4092bcc 100644 --- a/README.md +++ b/README.md @@ -91,32 +91,65 @@ package root exports only common definitions and does not load seed data.
Schema Registry -The catalog is applied through two APIs of the same Apicurio Registry. +Source: [catalog.json](catalog.json); artifact paths are relative to `src/`. -Kafka artifacts are published through the Confluent compatibility API, because runtime clients -resolve schemas by subject name and that namespace is the registry default group: +| Protocol | Format | API | Group | Artifact | +| :------- | :------- | :------------------ | :--------------- | :-------------- | +| Kafka | JSON | `/apis/ccompat/v7` | `default` | `-value` | +| gRPC | Protobuf | `/apis/registry/v3` | `monadiam.grpc` | `` | -| API | Subject shape | -|:--------------------|:----------------| -| `/apis/ccompat/v7` | `-value` | +Kafka clients load schemas from Apicurio. gRPC loads `.proto` from `@monadiam/shared`; +registry publication stores the contract. Catalog `groupId` applies only to gRPC. -gRPC artifacts use Protocol Buffers and the native API, since nothing resolves them at runtime: +| Script / image | Responsibility | +| :----------------------------------- | :---------------------------------------------------- | +| [check.sh](scripts/check.sh) | Check catalog, files, JSON syntax and proto3 headers | +| [verify.sh](scripts/verify.sh) | Check compatibility; create no groups or artifacts | +| [apply.sh](scripts/apply.sh) | Publish schemas; identical content reuses its version | +| [registry.sh](scripts/registry.sh) | Configuration, HTTP timeouts and error details | +| [Dockerfile](Dockerfile) | Catalog + scripts; Bash/curl/jq; UID/GID `65532` | -| Group ID | Artifact ID shape | -|:----------------|:------------------| -| `monadiam.grpc` | `` | +```bash +# Local: configure .env first (see Environment). +make check +make verify +make apply + +# Image: default command is scripts/apply.sh. +make image IMAGE=monadiam/shared-schemas:local +``` -The `groupId` field in the catalog applies to native API artifacts only. +In containers, provide `SCHEMA_REGISTRY_URL`; select `scripts/verify.sh` or +`scripts/apply.sh`. Read-only filesystem is supported. Planned stand order: +`Apicurio Ready -> Job (verify -> apply) -> services`. -Ordinary consumers use bounded inline retries and publish terminal failures to shared dead-letter topics. -Only dispatch uses delayed retry: `message-dispatch-retry-notification-service-value`. -Realm bootstrap, membership join and ownership transfer requests and outcomes share `realm-value`. -The separate membership topic/schema is removed; all realm workflow payloads carry `realm` for partition routing. -Dead-letter topics have no automatic replay consumers. +| Execution rule | Behavior | +| :-------------- | :---------------------------------------------------------------- | +| New gRPC group | Local check only; full server validity check on publication | +| Partial failure | Previous writes remain; rerun after correcting the error | +| HTTP failure | Nonzero exit, status and response body; no automatic retries | -Repeated runs are idempotent: the compatibility API returns the existing id for identical content, and native artifacts are created with `ifExists=FIND_OR_CREATE_VERSION`. +
-Artifact paths in `catalog.json` are resolved against `src`. Override with `SCHEMA_DIR` if needed. +---- + +
+Container package (GHCR) + +Image: `ghcr.io/monadiam/shared-schemas` (`linux/amd64`). +Workflows: [CI](.github/workflows/ci.yml), [publish](.github/workflows/publish.yml), [release](.github/workflows/release.yml). +Publication and retention: [packages.sh](.github/scripts/packages.sh). + +| Event / setting | Behavior | +| :----------------- | :------------------------------------------------------------- | +| PR / merge queue | Checks and image build; no publication | +| Merge into main | Build and publish image as `sha-` | +| Git tag `vX.Y.Z` | Require `package.json.version = X.Y.Z`; add image tag `X.Y.Z` | +| Release image | Reuse published SHA digest; no rebuild or tag overwrite | +| Stand | Pin release as `:@sha256:` | +| Retention | Keep last 10 SHA builds and current commit | +| Protected images | Keep releases, custom tags and untagged manifests | +| Authentication | CI uses `GITHUB_TOKEN`; public pulls require no credentials |
@@ -131,11 +164,17 @@ Create local variables from the example file: make env ``` -| Variable | Example | Purpose | -|:-------------------------------|:------------------------|:---------------------------------| -| `SCHEMA_REGISTRY_URL` | `http://localhost:8081` | Apicurio Registry base URL | -| `SCHEMA_REGISTRY_API_PATH` | `/apis/registry/v3` | Apicurio native API path | -| `SCHEMA_REGISTRY_CCOMPAT_PATH` | `/apis/ccompat/v7` | Confluent compatibility API path | +| Variable | Example | Purpose | +| :---------------------------------------- | :------------------------ | :------------------------------------- | +| `SCHEMA_REGISTRY_URL` | `http://localhost:8081` | Apicurio Registry base URL | +| `SCHEMA_REGISTRY_API_PATH` | `/apis/registry/v3` | Apicurio native API path | +| `SCHEMA_REGISTRY_CCOMPAT_PATH` | `/apis/ccompat/v7` | Confluent compatibility API path | +| `SCHEMA_REGISTRY_CONNECT_TIMEOUT_SECONDS` | `5` | Maximum connection time, seconds | +| `SCHEMA_REGISTRY_REQUEST_TIMEOUT_SECONDS` | `30` | Maximum complete request time, seconds | + +All five settings are required. Local scripts read `.env`; the image declares API +paths and timeouts through `ENV`, and requires `SCHEMA_REGISTRY_URL` at runtime. +Missing or empty values fail immediately; scripts do not supply fallback values. @@ -144,22 +183,32 @@ make env
Commands -| Makefile | Description | -|:------------------------|:-------------------------------------------------------------| -| `make build` | Compile the package into `dist`. | -| `make lint` | Run ESLint. | -| `make lint-fix` | Fix only Prettier violations and print changed file paths. | -| `make tsc` | Check TypeScript with tsc --noEmit. | -| `make knip` | Run Knip. | -| `make secrets-check` | Run the hook secret scan (staged changes). | -| `make lsp` | Start the local TypeScript MCP server over stdio. | -| `make graphify` | Create or update the local code graph and Markdown report. | -| `make graphify-rebuild` | Rescan all code files and regenerate the report. | -| `make graphify-html` | Update the graph and report, then export HTML visualization. | -| `make env` | Create `.env` from `.env.example`. | -| `make check` | Validate local catalog and referenced artifacts. | -| `make apply` | Publish catalog artifacts to Schema Registry. | -| `make verify` | Check catalog artifacts for compatibility. | +| Makefile | Description | +|:------------------------|:---------------------------------------------------------------------------| +| **Development** | | +| `make build` | Compile the package into `dist`. | +| `make lint` | Run ESLint. | +| `make lint-fix` | Fix only Prettier violations and print changed file paths. | +| `make tsc` | Check TypeScript with tsc --noEmit. | +| `make knip` | Run Knip. | +| `make secrets-check` | Run the hook secret scan (staged changes). | +| `make lsp` | Start the local TypeScript MCP server over stdio. | +| `make graphify` | Create or update the local code graph and Markdown report. | +| `make graphify-rebuild` | Rescan all code files and regenerate the report. | +| `make graphify-html` | Update the graph and report, then export HTML visualization. | +| **Schema Registry** | | +| `make env` | Create `.env` from `.env.example`. | +| `make check` | Check catalog, artifact files, JSON syntax and proto3 headers; no network. | +| `make verify` | Check schema compatibility; create no groups or artifacts. | +| `make apply` | Publish schemas to Apicurio; identical content reuses its version. | +| **Container / GHCR** | | +| `make image` | Build catalog/scripts image; configurable `IMAGE` and `VCS_REF`. | +| `make image-publish` | Publish existing `monadiam/shared-schemas:ci` as `sha-` in GHCR. | +| `make image-release` | Assign the package version to the published SHA image; no rebuild. | +| `make image-cleanup` | Delete old SHA images according to the GHCR retention policy. | + +GHCR targets run in Actions with `GITHUB_TOKEN`; publish/release also require +`GITHUB_STEP_SUMMARY`, and release requires `GITHUB_REF_NAME`.
@@ -182,19 +231,46 @@ pnpm add git+https://github.com/MonadIAM/shared.git#v2.0.0
Releasing a new version -1. Update `version` field in `package.json`. -2. Commit changes: -```bash -git add . -git commit -m "feat: add new message templates" -``` -3. Create tag and push: -```sh -git tag v2.0.0 -git push origin main --tags -``` +1. Create a branch from current `main`: + + ```bash + git switch main + git pull --ff-only origin main + git switch -c feat/message-templates + ``` + +2. Make changes and update `package.json.version` to the next release version. + Schema and TypeScript changes share this version. + +3. Commit and push the branch: + + ```bash + git add . + git commit -m "feat: add new message templates" + git push -u origin feat/message-templates + ``` + +4. Open a PR into `main`. Wait for required checks, including **CI passed**, then + merge. Wait for **Publish schema image** to succeed for the resulting commit: + it builds and publishes `ghcr.io/monadiam/shared-schemas:sha-`. + +5. Update local `main`. Its latest commit should be your published release commit: + + ```bash + git switch main + git pull --ff-only origin main + ``` + +6. Create and push the tag matching `package.json.version` (example: `2.0.20`): + + ```bash + git tag v2.0.20 + git push origin v2.0.20 + ``` -Schema changes and TypeScript changes share one version, so bump it for schema edits as well. +7. Wait for **Release schema image**. It verifies `vX.Y.Z` against + `package.json.version` and adds image tag `X.Y.Z` to the published SHA image + without rebuilding. Use the digest from the workflow summary in stand.
diff --git a/package.json b/package.json index 032f60d..c21e5b2 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "description": "Shared Kafka/gRPC contracts and common enums between MonadIAM services", "author": "Daniil Egorov (https://github.com/Lambdius)", "license": "UNLICENSED", - "version": "2.0.19", + "version": "3.0.0", "types": "dist/index.d.ts", "main": "dist/index.js", "exports": { diff --git a/scripts/apply.sh b/scripts/apply.sh index 1f9d186..22cbbfa 100755 --- a/scripts/apply.sh +++ b/scripts/apply.sh @@ -2,58 +2,20 @@ set -euo pipefail ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -CATALOG_FILE="${CATALOG_FILE:-"$ROOT_DIR/catalog.json"}" -SCHEMA_DIR="${SCHEMA_DIR:-"$ROOT_DIR/src"}" -ENV_FILE="${ENV_FILE:-"$ROOT_DIR/.env"}" - -if [ -f "$ENV_FILE" ]; then - set -a - . "$ENV_FILE" - set +a -fi - -REGISTRY_URL="${1:-${SCHEMA_REGISTRY_URL:-}}" -REGISTRY_API_PATH="${SCHEMA_REGISTRY_API_PATH:-/apis/registry/v3}" -REGISTRY_CCOMPAT_PATH="${SCHEMA_REGISTRY_CCOMPAT_PATH:-/apis/ccompat/v7}" - -if [ -z "$REGISTRY_URL" ]; then - echo "SCHEMA_REGISTRY_URL is required" >&2 - exit 1 -fi - -REGISTRY_API="${REGISTRY_URL%/}${REGISTRY_API_PATH}" -REGISTRY_CCOMPAT="${REGISTRY_URL%/}${REGISTRY_CCOMPAT_PATH}" - -command -v curl >/dev/null 2>&1 || { - echo "curl is required" >&2 - exit 1 -} - -command -v jq >/dev/null 2>&1 || { - echo "jq is required" >&2 - exit 1 -} - -"$ROOT_DIR/scripts/check.sh" >/dev/null +. "$ROOT_DIR/scripts/registry.sh" +registry_initialize "$@" create_group() { local group_id="$1" - local status + local payload - status="$( - jq -n --arg groupId "$group_id" '{ groupId: $groupId }' \ - | curl -sS -o /dev/null -w "%{http_code}" \ - -X POST "$REGISTRY_API/groups" \ - -H "Content-Type: application/json" \ - --data-binary @- - )" + payload="$(jq -n --arg groupId "$group_id" '{ groupId: $groupId }')" + registry_request POST "$REGISTRY_API/groups" application/json <<< "$payload" - case "$status" in + case "$REGISTRY_STATUS" in 200 | 204 | 409) ;; *) - echo "Failed to create registry group $group_id: HTTP $status" >&2 - exit 1 - ;; + registry_fail "Failed to create registry group $group_id" ;; esac } @@ -61,11 +23,11 @@ publish_grpc_artifact() { local group_id="$1" local artifact_id="$2" local artifact_path="$SCHEMA_DIR/$3" - local status + local payload create_group "$group_id" - status="$( + payload="$( jq -n \ --arg artifactId "$artifact_id" \ --rawfile content "$artifact_path" \ @@ -78,51 +40,38 @@ publish_grpc_artifact() { contentType: "text/plain" } } - }' \ - | curl -sS -o /dev/null -w "%{http_code}" \ - -X POST "$REGISTRY_API/groups/$group_id/artifacts?ifExists=FIND_OR_CREATE_VERSION&canonical=true" \ - -H "Content-Type: application/json" \ - --data-binary @- + }' )" + registry_request POST "$REGISTRY_API/groups/$group_id/artifacts?ifExists=FIND_OR_CREATE_VERSION&canonical=true" application/json <<< "$payload" - case "$status" in + case "$REGISTRY_STATUS" in 200) - echo "[SchemaRegistry]: applied - $group_id/$artifact_id" - ;; + echo "[SchemaRegistry]: applied - $group_id/$artifact_id" ;; *) - echo "Failed to apply $group_id/$artifact_id: HTTP $status" >&2 - exit 1 - ;; + registry_fail "Failed to apply $group_id/$artifact_id" ;; esac } publish_kafka_subject() { local subject="$1" local artifact_path="$SCHEMA_DIR/$2" - local status + local payload - status="$( + payload="$( jq -n \ --rawfile content "$artifact_path" \ - '{ schema: $content, schemaType: "JSON" }' \ - | curl -sS -o /dev/null -w "%{http_code}" \ - -X POST "$REGISTRY_CCOMPAT/subjects/$subject/versions" \ - -H "Content-Type: application/vnd.schemaregistry.v1+json" \ - --data-binary @- + '{ schema: $content, schemaType: "JSON" }' )" + registry_request POST "$REGISTRY_CCOMPAT/subjects/$subject/versions" \ + "application/vnd.schemaregistry.v1+json" <<< "$payload" - case "$status" in + case "$REGISTRY_STATUS" in 200) - echo "[SchemaRegistry]: applied - $subject" - ;; + echo "[SchemaRegistry]: applied - $subject" ;; 409) - echo "Incompatible schema for subject $subject" >&2 - exit 1 - ;; + registry_fail "Incompatible schema for subject $subject" ;; *) - echo "Failed to apply $subject: HTTP $status" >&2 - exit 1 - ;; + registry_fail "Failed to apply $subject" ;; esac } diff --git a/scripts/check.sh b/scripts/check.sh index 9d6e9db..1367403 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -2,8 +2,8 @@ set -euo pipefail ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -CATALOG_FILE="${CATALOG_FILE:-"$ROOT_DIR/catalog.json"}" -SCHEMA_DIR="${SCHEMA_DIR:-"$ROOT_DIR/src"}" +CATALOG_FILE="$ROOT_DIR/catalog.json" +SCHEMA_DIR="$ROOT_DIR/src" command -v jq >/dev/null 2>&1 || { echo "jq is required" >&2 @@ -27,18 +27,15 @@ while IFS=$'\t' read -r schema_type path; do case "$schema_type" in JSON) - jq empty "$artifact_path" - ;; + jq empty "$artifact_path" ;; PROTOBUF) grep -Eq '^syntax = "proto3";' "$artifact_path" || { echo "Invalid protobuf syntax header: $path" >&2 exit 1 - } - ;; + } ;; *) echo "Unsupported schema type: $schema_type" >&2 - exit 1 - ;; + exit 1 ;; esac done < <(jq -r '.artifacts[] | [.schemaType, .path] | @tsv' "$CATALOG_FILE") diff --git a/scripts/logger.sh b/scripts/logger.sh index e33f12d..282e98e 100644 --- a/scripts/logger.sh +++ b/scripts/logger.sh @@ -23,14 +23,11 @@ fi now_ms() { case "$LOG_CLOCK" in bash) - echo $((${EPOCHREALTIME/[.,]/} / 1000)) - ;; + echo $((${EPOCHREALTIME/[.,]/} / 1000)) ;; perl) - perl -MTime::HiRes -e 'printf("%d\n", Time::HiRes::time() * 1000)' - ;; + perl -MTime::HiRes -e 'printf("%d\n", Time::HiRes::time() * 1000)' ;; *) - echo $(($(date +%s) * 1000)) - ;; + echo $(($(date +%s) * 1000)) ;; esac } diff --git a/scripts/registry.sh b/scripts/registry.sh new file mode 100644 index 0000000..0aacd22 --- /dev/null +++ b/scripts/registry.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +registry_initialize() { + if [ -f "$ROOT_DIR/.env" ]; then + set -a + . "$ROOT_DIR/.env" + set +a + fi + + CATALOG_FILE="$ROOT_DIR/catalog.json" + SCHEMA_DIR="$ROOT_DIR/src" + + : "${SCHEMA_REGISTRY_URL:?SCHEMA_REGISTRY_URL is required}" + : "${SCHEMA_REGISTRY_API_PATH:?SCHEMA_REGISTRY_API_PATH is required}" + : "${SCHEMA_REGISTRY_CCOMPAT_PATH:?SCHEMA_REGISTRY_CCOMPAT_PATH is required}" + : "${SCHEMA_REGISTRY_CONNECT_TIMEOUT_SECONDS:?SCHEMA_REGISTRY_CONNECT_TIMEOUT_SECONDS is required}" + : "${SCHEMA_REGISTRY_REQUEST_TIMEOUT_SECONDS:?SCHEMA_REGISTRY_REQUEST_TIMEOUT_SECONDS is required}" + + REGISTRY_URL="$SCHEMA_REGISTRY_URL" + if [ "$#" -gt 0 ]; then + REGISTRY_URL="$1" + fi + + REGISTRY_CONNECT_TIMEOUT="$SCHEMA_REGISTRY_CONNECT_TIMEOUT_SECONDS" + REGISTRY_REQUEST_TIMEOUT="$SCHEMA_REGISTRY_REQUEST_TIMEOUT_SECONDS" + + local timeout + for timeout in "$REGISTRY_CONNECT_TIMEOUT" "$REGISTRY_REQUEST_TIMEOUT"; do + if [[ ! "$timeout" =~ ^[1-9][0-9]*$ ]]; then + echo "Schema Registry timeouts must be positive integer seconds: $timeout" >&2 + exit 1 + fi + done + + local program + for program in curl jq; do + command -v "$program" >/dev/null 2>&1 || { + echo "$program is required" >&2 + exit 1 + } + done + + REGISTRY_API="${REGISTRY_URL%/}${SCHEMA_REGISTRY_API_PATH}" + REGISTRY_CCOMPAT="${REGISTRY_URL%/}${SCHEMA_REGISTRY_CCOMPAT_PATH}" + + "$ROOT_DIR/scripts/check.sh" >/dev/null + + registry_request GET "$REGISTRY_API/system/info" application/json + if [ "$REGISTRY_STATUS" != 200 ] || ! jq -e '.version | type == "string"' <<< "$REGISTRY_BODY" >/dev/null; then + registry_fail "Apicurio Registry is unavailable or its API path is incorrect" + fi +} + +registry_request() { + local method="$1" url="$2" content_type="$3" + local response result + local arguments=( + --silent --show-error + --connect-timeout "$REGISTRY_CONNECT_TIMEOUT" + --max-time "$REGISTRY_REQUEST_TIMEOUT" + --request "$method" + --write-out $'\n%{http_code}' + ) + + if [ "$method" = POST ]; then + arguments+=(--header "Content-Type: $content_type" --data-binary @-) + fi + + if response="$(curl "${arguments[@]}" "$url")"; then + REGISTRY_STATUS="${response##*$'\n'}" + REGISTRY_BODY="${response%$'\n'*}" + else + result=$? + printf 'Registry request failed: %s %s; curl exit %s (connect timeout %ss, request timeout %ss)\n' \ + "$method" "$url" "$result" "$REGISTRY_CONNECT_TIMEOUT" "$REGISTRY_REQUEST_TIMEOUT" >&2 + exit "$result" + fi +} + +registry_fail() { + printf '%s: HTTP %s\n%s\n' "$1" "$REGISTRY_STATUS" "${REGISTRY_BODY:0:4096}" >&2 + exit 1 +} diff --git a/scripts/verify.sh b/scripts/verify.sh index 256975d..ff07bd4 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -2,47 +2,30 @@ set -euo pipefail ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" -CATALOG_FILE="${CATALOG_FILE:-"$ROOT_DIR/catalog.json"}" -SCHEMA_DIR="${SCHEMA_DIR:-"$ROOT_DIR/src"}" -ENV_FILE="${ENV_FILE:-"$ROOT_DIR/.env"}" - -if [ -f "$ENV_FILE" ]; then - set -a - . "$ENV_FILE" - set +a -fi - -REGISTRY_URL="${1:-${SCHEMA_REGISTRY_URL:-}}" -REGISTRY_API_PATH="${SCHEMA_REGISTRY_API_PATH:-/apis/registry/v3}" -REGISTRY_CCOMPAT_PATH="${SCHEMA_REGISTRY_CCOMPAT_PATH:-/apis/ccompat/v7}" - -if [ -z "$REGISTRY_URL" ]; then - echo "SCHEMA_REGISTRY_URL is required" >&2 - exit 1 -fi - -REGISTRY_API="${REGISTRY_URL%/}${REGISTRY_API_PATH}" -REGISTRY_CCOMPAT="${REGISTRY_URL%/}${REGISTRY_CCOMPAT_PATH}" - -command -v curl >/dev/null 2>&1 || { - echo "curl is required" >&2 - exit 1 -} - -command -v jq >/dev/null 2>&1 || { - echo "jq is required" >&2 - exit 1 -} - -"$ROOT_DIR/scripts/check.sh" >/dev/null +. "$ROOT_DIR/scripts/registry.sh" +registry_initialize "$@" verify_grpc_artifact() { local group_id="$1" local artifact_id="$2" local artifact_path="$SCHEMA_DIR/$3" - local status + local payload - status="$( + registry_request GET "$REGISTRY_API/groups/$group_id" application/json + case "$REGISTRY_STATUS" in + 200) ;; + 404) + if jq -e '.name == "GroupNotFoundException"' <<< "$REGISTRY_BODY" >/dev/null; then + echo "[SchemaRegistry]: new group - $group_id/$artifact_id" + return + fi + + registry_fail "Failed to find registry group $group_id" ;; + *) + registry_fail "Failed to find registry group $group_id" ;; + esac + + payload="$( jq -n \ --arg artifactId "$artifact_id" \ --rawfile content "$artifact_path" \ @@ -55,58 +38,46 @@ verify_grpc_artifact() { contentType: "text/plain" } } - }' \ - | curl -sS -o /dev/null -w "%{http_code}" \ - -X POST "$REGISTRY_API/groups/$group_id/artifacts?ifExists=FIND_OR_CREATE_VERSION&canonical=true&dryRun=true" \ - -H "Content-Type: application/json" \ - --data-binary @- + }' )" + registry_request POST "$REGISTRY_API/groups/$group_id/artifacts?ifExists=FIND_OR_CREATE_VERSION&canonical=true&dryRun=true" application/json <<< "$payload" - case "$status" in + case "$REGISTRY_STATUS" in 200) - echo "[SchemaRegistry]: verified - $group_id/$artifact_id" - ;; + echo "[SchemaRegistry]: verified - $group_id/$artifact_id" ;; *) - echo "Failed to verify $group_id/$artifact_id: HTTP $status" >&2 - exit 1 - ;; + registry_fail "Failed to verify $group_id/$artifact_id" ;; esac } verify_kafka_subject() { local subject="$1" local artifact_path="$SCHEMA_DIR/$2" - local response - local status + local payload - response="$( + payload="$( jq -n \ --rawfile content "$artifact_path" \ - '{ schema: $content, schemaType: "JSON" }' \ - | curl -sS -w "\n%{http_code}" \ - -X POST "$REGISTRY_CCOMPAT/compatibility/subjects/$subject/versions/latest" \ - -H "Content-Type: application/vnd.schemaregistry.v1+json" \ - --data-binary @- + '{ schema: $content, schemaType: "JSON" }' )" + registry_request POST "$REGISTRY_CCOMPAT/compatibility/subjects/$subject/versions/latest" \ + "application/vnd.schemaregistry.v1+json" <<< "$payload" - status="$(printf '%s' "$response" | tail -n1)" - - case "$status" in + case "$REGISTRY_STATUS" in 404) - echo "[SchemaRegistry]: new subject - $subject" - ;; + if jq -e '.error_code == 40401' <<< "$REGISTRY_BODY" >/dev/null; then + echo "[SchemaRegistry]: new subject - $subject" + else + registry_fail "Failed to find subject $subject" + fi ;; 200) - if [ "$(printf '%s' "$response" | sed '$d' | jq -r '.is_compatible')" = "true" ]; then + if jq -e '.is_compatible == true' <<< "$REGISTRY_BODY" >/dev/null; then echo "[SchemaRegistry]: verified - $subject" else - echo "Incompatible schema for subject $subject" >&2 - exit 1 - fi - ;; + registry_fail "Incompatible schema for subject $subject" + fi ;; *) - echo "Failed to verify $subject: HTTP $status" >&2 - exit 1 - ;; + registry_fail "Failed to verify $subject" ;; esac }