fix(ci): overhaul release pipeline, fix versions, add RELEASING.md

.buildkite/publish/publish-common.sh

@@ -18,6 +18,11 @@ VERSION=$(cat "$VERSION_PATH")
 IS_SNAPSHOT=$(buildkite-agent meta-data get is_snapshot)
 IS_LATEST=$(buildkite-agent meta-data get is_latest)
 
+if [[ "${IS_SNAPSHOT}" != "true" && "${IS_SNAPSHOT}" != "false" ]]; then
+  echo "!! is_snapshot has unexpected value '${IS_SNAPSHOT}'. Aborting to prevent accidental release."
+  exit 2
+fi
+
 if [[ "${IS_SNAPSHOT:-}" == "false" && "${IS_LATEST:-}" == "true" ]]; then
   # don't apply LATEST tag to SNAPSHOT builds
   export APPLY_LATEST_TAG="true"

.buildkite/release-pipeline.yml

@@ -1,5 +1,8 @@
 ## 🏠/.buildkite/pipeline-release.yml
-# Manual triggered pipeline to build and publish Docker images
+# Manual triggered pipeline to build and publish Docker images.
+#
+# Defaults to SNAPSHOT. To publish a release image, the operator must
+# explicitly choose "Release" in the input step AND trigger from an N.N branch.
 
 notify:
   - if: 'build.branch =~ /^((main)|([0-9]+\.[0-9]+))\$/ && (build.state == "failed" || pipeline.started_passing)'
@@ -11,8 +14,19 @@ notify:
 steps:
   - input: "Build information"
     fields:
+      - select: "Build type"
+        key: "build_type"
+        hint: "Snapshot is the default. Choose Release only when cutting an actual release from an N.N branch."
+        default: "snapshot"
+        options:
+          - label: "Snapshot"
+            value: "snapshot"
+          - label: "Release"
+            value: "release"
       - select: "Apply :latest tag?"
         key: "is_latest"
+        hint: "Only applies to Release builds. Ignored for Snapshots."
+        default: "false"
         options:
           - label: "Yes"
             value: "true"
@@ -21,20 +35,31 @@ steps:
   - wait: ~
 
   # ----
-  # Set is_snapshot to false initially
+  # Resolve build type into is_snapshot metadata.
+  # Release builds are only allowed from N.N branches — requesting a
+  # release from main is rejected to prevent accidents.
   # ----
-  - command: "buildkite-agent meta-data set is_snapshot false"
+  - label: ":shield: Resolve build type"
+    command: |
+      BUILD_TYPE=$$(buildkite-agent meta-data get build_type)
+        if [[ ! "$$BUILDKITE_BRANCH" =~ ^[0-9]+\.[0-9]+$$ ]]; then
+          echo "ERROR: Release builds require an N.N branch (for example, 1.0)."
+          exit 1
+        fi
+        buildkite-agent meta-data set is_snapshot false
+        echo "Build type: RELEASE (branch: $$BUILDKITE_BRANCH)"
+      else
+        buildkite-agent meta-data set is_snapshot true
+        echo "Build type: SNAPSHOT (branch: $$BUILDKITE_BRANCH)"
+      fi
+  - wait: ~
 
   # ----
   # Docker builds for amd64
   # ----
   - group: ":package: amd64 Build and Test"
     key: "build_and_test_amd64"
-    if: "(build.branch =~ /^[0-9]+\\.[0-9x]+.*/ || build.branch == \"main\")"
     steps:
-      - label: "Determine if snapshot"
-        command: "buildkite-agent meta-data set is_snapshot true"
-        if: "(build.branch == \"main\")"
       - label: "Building amd64 Docker image"
         agents:
           provider: aws
@@ -64,11 +89,7 @@ steps:
   # ----
   - group: ":package: arm64 Build and Test"
     key: "build_and_test_arm64"
-    if: "(build.branch =~ /^[0-9]+\\.[0-9x]+.*/ || build.branch == \"main\")"
     steps:
-      - label: "Determine if snapshot"
-        command: "buildkite-agent meta-data set is_snapshot true"
-        if: "(build.branch == \"main\")"
       - label: "Building arm64 Docker image"
         agents:
           provider: aws
@@ -101,7 +122,6 @@ steps:
   # Multiarch Docker image build and push
   # ----
   - group: ":truck: Publish images"
-    if: "(build.branch =~ /^[0-9]+\\.[0-9x]+.*/ || build.branch == \"main\")"
     depends_on:
       - "build_and_test_amd64"
       - "build_and_test_arm64"

RELEASING.md

@@ -0,0 +1,63 @@
+# Releasing
+
+## Version scheme
+
+`MAJOR.MINOR.PATCH` stored in the `product_version` file. Independent of the Elastic stack release schedule.
+
+**Branch model:**
+- `main` — next minor/major development line (e.g. `1.1.0`). Builds always produce `-SNAPSHOT` images.
+- `N.N` branch (e.g. `1.0`) — maintenance branch for that minor line. Patch releases (`1.0.1`, `1.0.2`, …) are cut from here.
+
+## Cutting a release
+
+### 1. Docker image (via Buildkite)
+
+Trigger the **crawler-docker-build-publish** pipeline manually:
+
+1. Select the `N.N` branch (e.g. `1.0`) — **not** `main`.
+2. Set **Build type** to **Release**. (Default is Snapshot — the pipeline will reject Release builds from `main`.)
+3. Set **Apply :latest tag** to **Yes** only for the newest GA release.
+4. The pipeline reads `product_version` from the branch to determine the image tag.
+
+### 2. GitHub release tag
+
+1. Create a GitHub release at `elastic/crawler/releases`.
+2. Tag: `v{major}.{minor}.{patch}` (e.g. `v1.0.1`).
+3. Target: the **same commit SHA** as the Buildkite build.
+4. GitHub will auto-generate release notes.
+
+Both artifacts must target the same commit.
+
+## After a release
+
+### After a patch release (e.g. released `1.0.1` from `1.0`)
+
+Bump `product_version` on the `N.N` branch to the next patch:
+```bash
+# On a PR targeting the 1.0 branch:
+echo "1.0.2" > product_version
+```
+No change needed on `main`.
+
+### After a minor release (e.g. released `1.1.0`)
+
+1. Cut the `N.N` branch from the release commit:
+   ```bash
+   git checkout -b 1.1 v1.1.0
+   git push origin 1.1
+   ```
+2. Bump `product_version` on `main` to the next minor:
+   ```bash
+   # On a PR targeting main:
+   echo "1.2.0" > product_version
+   ```
+3. Add `"1.1"` to `targetBranchChoices` in `.backportrc.json`.
+
+## Snapshot builds
+
+Triggering the pipeline with the default "Snapshot" build type produces a
+`<version>-SNAPSHOT` image. This is safe to do from any branch at any time.
+
+## Internal runbook
+
+Full step-by-step: `search-team/teams/agent-builder/agent-builder-1/crawler/releasing-new-crawler-versions` in the internal team docs.

product_version

@@ -1 +1 @@
-1.0.1
+1.1.0