diff --git a/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource b/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource new file mode 100644 index 0000000000..9ce3624872 --- /dev/null +++ b/services/libs/tinybird/datasources/repo_commit_contributors_copy_ds.datasource @@ -0,0 +1,25 @@ +DESCRIPTION > + Precomputed all-time commit authorship per (channel, memberId, platform, username) over + authored-commit and co-authored-commit activities. Bots, team members and organization + profiles are excluded by the cleaned source (members_sorted). Rebuilt nightly by + repo_commit_contributors_copy.pipe. Read in sorting-key order by the packages worker + git-activity contributors sync (CM-1824), which maps `channel` to a packages-db repo and + `username` to an email or GitHub login identity, and only pulls rows whose `lastUpdatedAt` + passed its stored watermark. + +TAGS "Member contributors", "Repository URLs" + +SCHEMA > + `channel` String, + `memberId` String, + `platform` LowCardinality(String), + `username` String, + `commitCount` UInt64, + `firstCommitAt` DateTime64(3), + `lastCommitAt` DateTime64(3), + `lastUpdatedAt` DateTime64(3), + `computedAt` DateTime + +ENGINE ReplacingMergeTree +ENGINE_SORTING_KEY channel, memberId, platform, username +ENGINE_VER computedAt diff --git a/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe b/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe new file mode 100644 index 0000000000..6b25c0c35a --- /dev/null +++ b/services/libs/tinybird/pipes/repo_commit_contributors_copy.pipe @@ -0,0 +1,36 @@ +DESCRIPTION > + Nightly copy pipe that precomputes all-time commit authorship per + (channel, memberId, platform, username) for the packages worker git-activity + contributors sync (CM-1824). Authored and co-authored commits are summed into one + count. The cleaned source already restricts members to `members_sorted`, so bots, team + members and organization profiles are excluded upstream. Pre-1971 sentinel timestamps + are dropped like in repositories_populated_copy.pipe. `lastUpdatedAt` is the latest + activityRelations.updatedAt per row, which moves on ingestion and member merges; the + consumer uses it as an incremental watermark. + +TAGS "Member contributors", "Repository URLs" + +NODE repo_commit_contributors_copy_data +SQL > + SELECT + channel, + memberId, + platform, + username, + count() AS commitCount, + min(timestamp) AS firstCommitAt, + max(timestamp) AS lastCommitAt, + max(updatedAt) AS lastUpdatedAt, + now() AS computedAt + FROM activityRelations_deduplicated_cleaned_bucket_union + WHERE + type IN ('authored-commit', 'co-authored-commit') + AND channel != '' + AND username != '' + AND timestamp > toDateTime('1971-01-01') + GROUP BY channel, memberId, platform, username + +TYPE COPY +TARGET_DATASOURCE repo_commit_contributors_copy_ds +COPY_MODE replace +COPY_SCHEDULE 30 3 * * *