From 85242e17260e4887de15f16ae7a20147689ed843 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 2 Oct 2026 14:09:00 +0000 Subject: [PATCH 1/3] feat: add a nightly copy pipe for repo commit contributors (CM-1824) Signed-off-by: Mouad BANI --- ...epo_commit_contributors_copy_ds.datasource | 22 +++++++++++++ .../pipes/repo_commit_contributors_copy.pipe | 31 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource create mode 100644 services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe diff --git a/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource b/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource new file mode 100644 index 0000000000..2f5fd7fcc5 --- /dev/null +++ b/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource @@ -0,0 +1,22 @@ +DESCRIPTION > + Precomputed all-time commit authorship per (channel, memberId, platform, username) over + authored-commit and co-authored-commit activities, excluding bot members. Rebuilt nightly by + repo_commit_contributors_copy.pipe. Read in sorting-key order by the packages worker + git-activity contributors sync (CM-1824), which maps `channel` to a packages-db repo and + `username` to an email or GitHub login identity. + +TAGS "Member contributors", "Repository URLs" + +SCHEMA > + `channel` String, + `memberId` String, + `platform` LowCardinality(String), + `username` String, + `commitCount` UInt64, + `firstCommitAt` DateTime64(3), + `lastCommitAt` DateTime64(3), + `computedAt` DateTime + +ENGINE ReplacingMergeTree +ENGINE_SORTING_KEY channel, memberId, platform, username +ENGINE_VER computedAt diff --git a/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe b/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe new file mode 100644 index 0000000000..90674f6ce2 --- /dev/null +++ b/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe @@ -0,0 +1,31 @@ +DESCRIPTION > + Nightly copy pipe that precomputes all-time commit authorship per + (channel, memberId, platform, username) for the packages worker git-activity + contributors sync (CM-1824). Authored and co-authored commits are summed into one + count; bot members are excluded. + +TAGS "Member contributors", "Repository URLs" + +NODE repo_commit_contributors_copy_data +SQL > + SELECT + channel, + memberId, + platform, + username, + count() AS commitCount, + min(timestamp) AS firstCommitAt, + max(timestamp) AS lastCommitAt, + now() AS computedAt + FROM activityRelations_deduplicated_cleaned_bucket_union + WHERE + type IN ('authored-commit', 'co-authored-commit') + AND channel != '' + AND username != '' + AND memberId NOT IN (SELECT id FROM members_sorted WHERE isBot = 1) + GROUP BY channel, memberId, platform, username + +TYPE COPY +TARGET_DATASOURCE repo_commit_contributors_copy_ds +COPY_MODE replace +COPY_SCHEDULE 30 3 * * * From 8435ec1e9d632025b0de29125e5a5eea9be7f6dc Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 2 Oct 2026 14:39:05 +0000 Subject: [PATCH 2/3] fix(review): drop the redundant bot filter and sentinel timestamps (CM-1824) Signed-off-by: Mouad BANI --- .../datasources/repo_commit_contributors_copy_ds.datasource | 3 ++- .../libs/tinybird/pipes/repo_commit_contributors_copy.pipe | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource b/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource index 2f5fd7fcc5..4204853326 100644 --- a/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource +++ b/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource @@ -1,6 +1,7 @@ DESCRIPTION > Precomputed all-time commit authorship per (channel, memberId, platform, username) over - authored-commit and co-authored-commit activities, excluding bot members. Rebuilt nightly by + authored-commit and co-authored-commit activities. Bots, team members and organization + profiles are excluded by the cleaned source (members_sorted). Rebuilt nightly by repo_commit_contributors_copy.pipe. Read in sorting-key order by the packages worker git-activity contributors sync (CM-1824), which maps `channel` to a packages-db repo and `username` to an email or GitHub login identity. diff --git a/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe b/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe index 90674f6ce2..f8e6e1c796 100644 --- a/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe +++ b/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe @@ -2,7 +2,9 @@ DESCRIPTION > Nightly copy pipe that precomputes all-time commit authorship per (channel, memberId, platform, username) for the packages worker git-activity contributors sync (CM-1824). Authored and co-authored commits are summed into one - count; bot members are excluded. + count. The cleaned source already restricts members to `members_sorted`, so bots, team + members and organization profiles are excluded upstream. Pre-1971 sentinel timestamps + are dropped like in repositories_populated_copy.pipe. TAGS "Member contributors", "Repository URLs" @@ -22,7 +24,7 @@ SQL > type IN ('authored-commit', 'co-authored-commit') AND channel != '' AND username != '' - AND memberId NOT IN (SELECT id FROM members_sorted WHERE isBot = 1) + AND timestamp > toDateTime('1971-01-01') GROUP BY channel, memberId, platform, username TYPE COPY From de58994eca0e9f23d2ae7ddaf996097887dc1301 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 2 Oct 2026 15:00:50 +0000 Subject: [PATCH 3/3] feat: expose the last activity update per commit contributor (CM-1824) Signed-off-by: Mouad BANI --- .../datasources/repo_commit_contributors_copy_ds.datasource | 4 +++- .../libs/tinybird/pipes/repo_commit_contributors_copy.pipe | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource b/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource index 4204853326..9ce3624872 100644 --- a/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource +++ b/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource @@ -4,7 +4,8 @@ DESCRIPTION > profiles are excluded by the cleaned source (members_sorted). Rebuilt nightly by repo_commit_contributors_copy.pipe. Read in sorting-key order by the packages worker git-activity contributors sync (CM-1824), which maps `channel` to a packages-db repo and - `username` to an email or GitHub login identity. + `username` to an email or GitHub login identity, and only pulls rows whose `lastUpdatedAt` + passed its stored watermark. TAGS "Member contributors", "Repository URLs" @@ -16,6 +17,7 @@ SCHEMA > `commitCount` UInt64, `firstCommitAt` DateTime64(3), `lastCommitAt` DateTime64(3), + `lastUpdatedAt` DateTime64(3), `computedAt` DateTime ENGINE ReplacingMergeTree diff --git a/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe b/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe index f8e6e1c796..6b25c0c35a 100644 --- a/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe +++ b/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe @@ -4,7 +4,9 @@ DESCRIPTION > contributors sync (CM-1824). Authored and co-authored commits are summed into one count. The cleaned source already restricts members to `members_sorted`, so bots, team members and organization profiles are excluded upstream. Pre-1971 sentinel timestamps - are dropped like in repositories_populated_copy.pipe. + are dropped like in repositories_populated_copy.pipe. `lastUpdatedAt` is the latest + activityRelations.updatedAt per row, which moves on ingestion and member merges; the + consumer uses it as an incremental watermark. TAGS "Member contributors", "Repository URLs" @@ -18,6 +20,7 @@ SQL > count() AS commitCount, min(timestamp) AS firstCommitAt, max(timestamp) AS lastCommitAt, + max(updatedAt) AS lastUpdatedAt, now() AS computedAt FROM activityRelations_deduplicated_cleaned_bucket_union WHERE