Skip to content

Commit 5c0c799

Browse files
saileshwar-skyflowsaileshwar-skyflow
authored
Merge pull request #278 from skyflowapi/flowvault-release/26.9.0-unary
SK-3118: FlowVault Python SDK — unary operations
1 parent 26c0d80 commit 5c0c799

597 files changed

Lines changed: 22131 additions & 2030 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.codespellrc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
ignore-words-list = Skyflow,skyflow,skyflowapi,skyflowapis,deidentify,reidentify,detokenize,upsert,upserting,binlookup,byot,creds,fpe,devsecops,formdata,vaultid,dotenv,usecwd,runid,dateutil,Homogenous
44

55
# Skip these files and folders
6-
skip = .git,.venv,venv,env,__pycache__,*.pyc,*.egg-info,dist,build,.idea,.vscode,*.log,requirements.txt,./skyflow/generated,setup.py
6+
skip = .git,.venv,venv,env,__pycache__,*.pyc,*.egg-info,dist,build,.idea,.vscode,*.log,requirements.txt,generated,setup.py
77

88
# If you want to verify it is working, you can uncomment this line to see what files it checks
99
# count =

‎.github/workflows/beta-release.yml‎

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎.github/workflows/contract-tests.yml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
name: Contract Tests
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- skyvault-release/**
8+
- flowvault-release/**
9+
paths:
10+
- "skyvault/**"
11+
- "flowvault/**"
12+
- "common/**"
13+
- "ci-scripts/contract/**"
14+
- ".github/workflows/contract-tests.yml"
15+
16+
jobs:
17+
contract-tests:
18+
name: Contract Tests (${{ matrix.module }})
19+
runs-on: ubuntu-latest
20+
21+
strategy:
22+
fail-fast: false
23+
matrix:
24+
include:
25+
- module: skyvault
26+
pkg: skyflow
27+
- module: flowvault
28+
pkg: skyflow
29+
30+
permissions:
31+
contents: read
32+
pull-requests: write
33+
34+
env:
35+
GRIFFE_VERSION: "2.2.0"
36+
37+
steps:
38+
- name: Checkout
39+
uses: actions/checkout@v4
40+
with:
41+
fetch-depth: 0
42+
43+
- name: Setup Python
44+
uses: actions/setup-python@v2
45+
with:
46+
python-version: '3.10'
47+
48+
- name: Install griffe
49+
run: |
50+
python -m pip install --upgrade pip
51+
python -m pip install "griffe==${GRIFFE_VERSION}"
52+
53+
- name: Verify public API surface against the committed baseline
54+
run: |
55+
python ci-scripts/contract/griffe_contract.py check \
56+
"${{ matrix.module }}" \
57+
"${{ matrix.module }}/api-report/${{ matrix.pkg }}.api.json"
58+
59+
- name: How to update the baseline
60+
if: failure()
61+
run: |
62+
echo "### Public API contract drift in ${{ matrix.module }} ###"
63+
echo "If this change is intentional, run:"
64+
echo " ci-scripts/contract-snapshot-update.sh ${{ matrix.module }}"
65+
echo "review the api-report/${{ matrix.pkg }}.api.json diff, and commit it with your change."
66+
67+
- name: Detect baseline change
68+
id: baseline-diff
69+
if: always() && github.event.pull_request
70+
run: |
71+
git fetch origin "${{ github.event.pull_request.base.ref }}" --depth=1
72+
BASELINE="${{ matrix.module }}/api-report/${{ matrix.pkg }}.api.json"
73+
if ! git diff --quiet "origin/${{ github.event.pull_request.base.ref }}" HEAD -- "$BASELINE"; then
74+
echo "changed=true" >> "$GITHUB_OUTPUT"
75+
{
76+
echo 'diff<<GRIFFE_EOF'
77+
git diff "origin/${{ github.event.pull_request.base.ref }}" HEAD -- "$BASELINE" | head -300
78+
echo 'GRIFFE_EOF'
79+
} >> "$GITHUB_OUTPUT"
80+
else
81+
echo "changed=false" >> "$GITHUB_OUTPUT"
82+
fi
83+
84+
- name: Comment contract baseline change on PR
85+
if: always() && github.event.pull_request && steps.baseline-diff.outputs.changed == 'true'
86+
uses: actions/github-script@v7
87+
env:
88+
MODULE: ${{ matrix.module }}
89+
PKG: ${{ matrix.pkg }}
90+
DIFF: ${{ steps.baseline-diff.outputs.diff }}
91+
with:
92+
script: |
93+
const module = process.env.MODULE;
94+
const pkg = process.env.PKG;
95+
const marker = `<!-- contract-baseline-diff:${module} -->`;
96+
const body = `${marker}\n## Public API contract change (\`${module}\`)\n\n`
97+
+ `This PR changes \`${module}/api-report/${pkg}.api.json\` (the approved public API `
98+
+ `contract for \`${pkg}\`). Review the surface change below:\n\n`
99+
+ '```diff\n' + (process.env.DIFF || '(diff too large - see the file change)') + '\n```';
100+
const { data: comments } = await github.rest.issues.listComments({
101+
owner: context.repo.owner,
102+
repo: context.repo.repo,
103+
issue_number: context.issue.number,
104+
});
105+
const existing = comments.find(c => c.body && c.body.includes(marker));
106+
if (existing) {
107+
await github.rest.issues.updateComment({
108+
owner: context.repo.owner, repo: context.repo.repo,
109+
comment_id: existing.id, body,
110+
});
111+
} else {
112+
await github.rest.issues.createComment({
113+
owner: context.repo.owner, repo: context.repo.repo,
114+
issue_number: context.issue.number, body,
115+
});
116+
}

‎.github/workflows/internal-release.yml‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,53 @@
1-
name: Internal Release
1+
name: Publish module to the JFrog Artifactory
22

33
on:
44
push:
5+
# '**' not '*.*': Actions glob '*' does not match '/', so '*.*' let slash
6+
# tags (flowvault/v1.0.0) through and fired this branch-only workflow.
57
tags-ignore:
6-
- '*.*'
8+
- '**'
79
paths-ignore:
8-
- "setup.py"
9-
- "*.yml"
1010
- "*.md"
11-
- "skyflow/utils/_version.py"
12-
- "samples/**"
1311
branches:
12+
- flowvault-release/*
13+
- skyvault-release/*
14+
# Legacy: predates the per-module naming, still maps to skyvault.
1415
- release/*
1516

1617
jobs:
18+
resolve-module:
19+
runs-on: ubuntu-latest
20+
# Skip our own bump commit, or this loops: bump -> push -> release -> bump.
21+
# PAT-authenticated pushes DO trigger workflows; GITHUB_TOKEN pushes do not.
22+
# build-and-deploy needs this job, so skipping here skips the run.
23+
if: ${{ !contains(github.event.head_commit.message, '[AUTOMATED]') }}
24+
outputs:
25+
module: ${{ steps.set-module.outputs.module }}
26+
steps:
27+
# Explicit match, no catch-all: defaulting once published the wrong module.
28+
- name: Resolve module from branch name
29+
id: set-module
30+
env:
31+
BRANCH: ${{ github.ref_name }}
32+
run: |
33+
case "$BRANCH" in
34+
flowvault-release/*) MODULE="flowvault" ;;
35+
skyvault-release/*) MODULE="skyvault" ;;
36+
release/*) MODULE="skyvault" ;;
37+
*)
38+
echo "::error::Branch '$BRANCH' does not map to a module."
39+
exit 1
40+
;;
41+
esac
42+
echo "Branch '$BRANCH' -> module '$MODULE'"
43+
echo "module=$MODULE" >> "$GITHUB_OUTPUT"
44+
1745
build-and-deploy:
46+
needs: resolve-module
1847
uses: ./.github/workflows/shared-build-and-deploy.yml
1948
with:
2049
ref: ${{ github.ref_name }}
2150
tag: 'internal'
51+
module: ${{ needs.resolve-module.outputs.module }}
2252
secrets: inherit
2353

‎.github/workflows/pr-flowvault.yml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: PR CI Checks (flowvault)
2+
3+
# flowvault is a folder under main, alongside skyvault - not a branch.
4+
# This workflow fires for PRs targeting main or a flowvault-release/* branch
5+
# that actually touch flowvault or its common dependency, and only builds/
6+
# tests those two modules. skyvault (and the full 3-module suite) is covered
7+
# by pr.yml, not here.
8+
9+
on:
10+
pull_request:
11+
branches: [ "main", "flowvault-release/**" ]
12+
paths:
13+
- "flowvault/**"
14+
- "common/**"
15+
16+
jobs:
17+
build:
18+
name: Build
19+
runs-on: ubuntu-latest
20+
steps:
21+
- uses: actions/checkout@v4
22+
- uses: actions/setup-python@v2
23+
with:
24+
python-version: '3.9'
25+
26+
# flowvault depends on common as a local path dependency, so common
27+
# must be built and installed first or flowvault's own build/install
28+
# will fail to resolve it.
29+
- name: Build and install common
30+
run: |
31+
python -m pip install --upgrade pip setuptools wheel
32+
cd common
33+
python setup.py sdist bdist_wheel
34+
pip install dist/*.whl
35+
36+
- name: Build flowvault
37+
run: |
38+
cd flowvault
39+
python setup.py sdist bdist_wheel
40+
41+
test:
42+
name: Unit Tests
43+
runs-on: ubuntu-latest
44+
steps:
45+
- uses: actions/checkout@v4
46+
- uses: actions/setup-python@v2
47+
with:
48+
python-version: '3.9'
49+
50+
- name: create-json
51+
id: create-json
52+
uses: jsdaniell/create-json@1.1.2
53+
with:
54+
name: "credentials.json"
55+
json: ${{ secrets.VALID_SKYFLOW_CREDS_TEST }}
56+
57+
- name: Run flowvault unit tests
58+
run: |
59+
python -m pip install --upgrade pip setuptools wheel coverage
60+
cp credentials.json flowvault/credentials.json
61+
62+
# flowvault depends on common as a local path dependency.
63+
cd common
64+
python setup.py sdist bdist_wheel
65+
pip install dist/*.whl
66+
cd ..
67+
68+
cd flowvault
69+
python setup.py sdist bdist_wheel
70+
pip install dist/*.whl
71+
if [ -f requirements.txt ]; then
72+
pip install -r requirements.txt
73+
fi
74+
python -m coverage run --source=. -m unittest discover
75+
coverage xml -o test-coverage.xml
76+
77+
- name: Codecov
78+
uses: codecov/codecov-action@v2.1.0
79+
with:
80+
token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
81+
files: flowvault/test-coverage.xml
82+
flags: flowvault
83+
name: codecov-skyflow-python-flowvault
84+
verbose: true

.github/workflows/ci.yml renamed to .github/workflows/pr.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: CI Checks
1+
name: PR CI Checks
22

33
on: [pull_request]
44

‎.github/workflows/release.yml‎

Lines changed: 63 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,71 @@
1-
name: Public Release
1+
name: Public release
2+
3+
# Triggered by publishing a GitHub Release, not a raw tag push: the Release
4+
# carries both facts needed here - target_commitish (the branch picked in the
5+
# UI; a tag records only a commit) and tag_name (module prefix + version).
6+
#
7+
# Beta and final share this workflow - 'release' events cannot be filtered by
8+
# tag pattern, and both behave identically downstream. Kind comes from the tag.
29

310
on:
4-
push:
5-
tags: "*.*.*"
6-
paths-ignore:
7-
- "setup.py"
8-
- "*.yml"
9-
- "*.md"
10-
- "skyflow/utils/_version.py"
11+
release:
12+
types: [published]
1113

1214
jobs:
15+
resolve-release:
16+
runs-on: ubuntu-latest
17+
outputs:
18+
module: ${{ steps.parse.outputs.module }}
19+
version: ${{ steps.parse.outputs.version }}
20+
kind: ${{ steps.parse.outputs.kind }}
21+
steps:
22+
- name: Parse module, version and release kind from the tag
23+
id: parse
24+
env:
25+
TAG: ${{ github.event.release.tag_name }}
26+
BRANCH: ${{ github.event.release.target_commitish }}
27+
run: |
28+
# Expected: <module>/v<semver>[-beta.N] e.g. flowvault/v1.0.0,
29+
# skyvault/v2.1.2, flowvault/v1.0.0-beta.1
30+
if [[ ! "$TAG" =~ ^[a-z]+/v[0-9]+\.[0-9]+\.[0-9]+(-beta\.[0-9]+)?$ ]]; then
31+
echo "::error::Tag '$TAG' is not <module>/v<semver>[-beta.N]." \
32+
"Examples: flowvault/v1.0.0, skyvault/v2.1.2, flowvault/v1.0.0-beta.1"
33+
exit 1
34+
fi
35+
36+
PREFIX="${TAG%%/*}" # flowvault/v1.0.0 -> flowvault
37+
VERSION="${TAG#*/}" # flowvault/v1.0.0 -> v1.0.0
38+
VERSION="${VERSION#v}" # v1.0.0 -> 1.0.0
39+
40+
# Tag prefix -> module directory (both match the directory name).
41+
case "$PREFIX" in
42+
flowvault) MODULE="flowvault" ;;
43+
skyvault) MODULE="skyvault" ;;
44+
*)
45+
echo "::error::Unknown module prefix '$PREFIX' in tag '$TAG'"
46+
exit 1
47+
;;
48+
esac
49+
50+
if [[ "$VERSION" == *-beta.* ]]; then KIND="beta"; else KIND="public"; fi
51+
52+
if [ -z "$BRANCH" ]; then
53+
echo "::error::Release has no target_commitish - cannot determine the release branch."
54+
exit 1
55+
fi
56+
57+
echo "Tag '$TAG' -> module='$MODULE' version='$VERSION' kind='$KIND' branch='$BRANCH'"
58+
echo "module=$MODULE" >> "$GITHUB_OUTPUT"
59+
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
60+
echo "kind=$KIND" >> "$GITHUB_OUTPUT"
61+
1362
build-and-deploy:
63+
needs: resolve-release
1464
uses: ./.github/workflows/shared-build-and-deploy.yml
1565
with:
16-
ref: main
17-
tag: 'public'
66+
ref: ${{ github.event.release.tag_name }}
67+
tag: ${{ needs.resolve-release.outputs.kind }}
68+
module: ${{ needs.resolve-release.outputs.module }}
69+
version: ${{ needs.resolve-release.outputs.version }}
70+
release-branch: ${{ github.event.release.target_commitish }}
1871
secrets: inherit

‎.github/workflows/semgrep.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020

2121
- name: Run Semgrep
2222
run: |
23-
semgrep --config .semgreprules/customRule.yml --config auto --severity ERROR --sarif . > results.sarif
23+
semgrep --config .semgreprules/customRule.yml --config auto --severity ERROR --exclude generated --sarif . > results.sarif
2424
2525
- name: Upload SARIF file
2626
uses: github/codeql-action/upload-sarif@v3

0 commit comments

Comments
 (0)