diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 8a2ea84fc..36e67ff27 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -342,6 +342,13 @@ jobs: pull-requests: write steps: + - name: Set Airbyte + if: (contains(github.event.pull_request.labels.*.name, 'airbyte')) + run: | + echo "TF_VAR_pg_airbyte_enabled=true" >> $GITHUB_ENV + echo "TF_VAR_airbyte_enabled=true" >> $GITHUB_ENV + echo "TF_VAR_connection_status=active" >> $GITHUB_ENV + - name: Deploy App to Review id: deploy_review uses: DFE-Digital/github-actions/deploy-to-aks@master diff --git a/.github/workflows/delete-review-app.yml b/.github/workflows/delete-review-app.yml index 3143870fa..eb816c0b2 100644 --- a/.github/workflows/delete-review-app.yml +++ b/.github/workflows/delete-review-app.yml @@ -73,6 +73,13 @@ jobs: echo "AZURE_RESOURCE_PREFIX=${AZURE_RESOURCE_PREFIX}" >> $GITHUB_ENV echo "CONFIG_SHORT=${CONFIG_SHORT}" >> $GITHUB_ENV + - name: Set Airbyte + if: (contains(github.event.pull_request.labels.*.name, 'airbyte')) + run: | + echo "TF_VAR_pg_airbyte_enabled=true" >> $GITHUB_ENV + echo "TF_VAR_airbyte_enabled=true" >> $GITHUB_ENV + echo "TF_VAR_connection_status=active" >> $GITHUB_ENV + - name: Delete Review App id: delete-review-app uses: DFE-Digital/github-actions/delete-review-app@master diff --git a/global_config/review.sh b/global_config/review.sh index 832b23bb7..deb1dfbd1 100644 --- a/global_config/review.sh +++ b/global_config/review.sh @@ -3,4 +3,4 @@ CONFIG_SHORT=rv AZURE_SUBSCRIPTION=s189-teacher-services-cloud-test AZURE_RESOURCE_PREFIX=s189t01 KV_PURGE_PROTECTION=false -TERRAFORM_MODULES_TAG=main +TERRAFORM_MODULES_TAG=main \ No newline at end of file diff --git a/src/DfE.CheckPerformanceData.Infrastructure/packages.lock.json b/src/DfE.CheckPerformanceData.Infrastructure/packages.lock.json index 3d59561a0..74402a6a0 100644 --- a/src/DfE.CheckPerformanceData.Infrastructure/packages.lock.json +++ b/src/DfE.CheckPerformanceData.Infrastructure/packages.lock.json @@ -21,9 +21,9 @@ }, "DfeAnalytics.Core": { "type": "Direct", - "requested": "[0.6.2, )", - "resolved": "0.6.2", - "contentHash": "LIW1zpoLKWPt2017anc4xtmoPrsqV/EifFTJMhwA1/ZT3XJZp/AVj7KTsYcy3FJSCIbvPyAjpdfjFV53tIPd6g==", + "requested": "[0.6.4, )", + "resolved": "0.6.4", + "contentHash": "LfmZAaetqxonAFkZSeY0M3udvzyc35QGEgDTm5ohXocTQZNOks/iqBs7tOQ7LOEw4B6cXp6aE1MGzbdsWOKIyQ==", "dependencies": { "Google.Apis.Auth": "1.71.0", "Google.Cloud.BigQuery.V2": "[3.0.0, 4.0.0)", @@ -680,6 +680,11 @@ "resolved": "7.0.0", "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" }, + "System.CommandLine": { + "type": "Transitive", + "resolved": "2.0.0", + "contentHash": "Bjklzc5NoxqAGFi7BcGlY2TWAdB06Bq3a5sfRr3ubMRU80Mf98eyq3Y2UgR6xRV0TLznZmfe5T7mUjOunRNcdA==" + }, "System.IO.Hashing": { "type": "Transitive", "resolved": "10.0.9", @@ -728,6 +733,7 @@ "dependencies": { "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Domain": "[1.0.0, )", + "DfeAnalytics.EFCore": "[0.6.4, )", "Microsoft.EntityFrameworkCore": "[10.0.13, )", "Microsoft.EntityFrameworkCore.Relational": "[10.0.13, )", "Microsoft.Extensions.Configuration.Abstractions": "[10.0.13, )", @@ -753,6 +759,20 @@ "System.IO.Hashing": "10.0.9" } }, + "DfeAnalytics.EFCore": { + "type": "CentralTransitive", + "requested": "[0.6.4, )", + "resolved": "0.6.4", + "contentHash": "t4Rw5dqWcNNsKm6hTxoTdomrfDskb2SEt70Wo7dhnKixJqkmlmcjSpoueKZcES5fogTpg87cNKLLXLkh51Cqsg==", + "dependencies": { + "DfeAnalytics.Core": "0.6.4", + "Microsoft.EntityFrameworkCore": "10.0.0", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.0", + "Microsoft.Extensions.Http": "10.0.9", + "Npgsql.EntityFrameworkCore.PostgreSQL": "10.0.0", + "System.CommandLine": "2.0.0" + } + }, "HtmlSanitizer": { "type": "CentralTransitive", "requested": "[9.2.1039, )", diff --git a/src/DfE.CheckPerformanceData.Persistence/DfE.CheckPerformanceData.Persistence.csproj b/src/DfE.CheckPerformanceData.Persistence/DfE.CheckPerformanceData.Persistence.csproj index f0a93bfb2..3bfc3a3b1 100644 --- a/src/DfE.CheckPerformanceData.Persistence/DfE.CheckPerformanceData.Persistence.csproj +++ b/src/DfE.CheckPerformanceData.Persistence/DfE.CheckPerformanceData.Persistence.csproj @@ -7,6 +7,7 @@ + diff --git a/src/DfE.CheckPerformanceData.Persistence/packages.lock.json b/src/DfE.CheckPerformanceData.Persistence/packages.lock.json index 4e2d27278..03bb8ddd9 100644 --- a/src/DfE.CheckPerformanceData.Persistence/packages.lock.json +++ b/src/DfE.CheckPerformanceData.Persistence/packages.lock.json @@ -2,6 +2,20 @@ "version": 2, "dependencies": { "net10.0": { + "DfeAnalytics.EFCore": { + "type": "Direct", + "requested": "[0.6.4, )", + "resolved": "0.6.4", + "contentHash": "t4Rw5dqWcNNsKm6hTxoTdomrfDskb2SEt70Wo7dhnKixJqkmlmcjSpoueKZcES5fogTpg87cNKLLXLkh51Cqsg==", + "dependencies": { + "DfeAnalytics.Core": "0.6.4", + "Microsoft.EntityFrameworkCore": "10.0.0", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.0", + "Microsoft.Extensions.Http": "10.0.9", + "Npgsql.EntityFrameworkCore.PostgreSQL": "10.0.0", + "System.CommandLine": "2.0.0" + } + }, "Microsoft.EntityFrameworkCore": { "type": "Direct", "requested": "[10.0.13, )", @@ -101,11 +115,79 @@ "AngleSharp": "[1.5.0, 2.0.0)" } }, + "Google.Api.Gax": { + "type": "Transitive", + "resolved": "4.0.0", + "contentHash": "2Q4RruOjUawrtjTazANpp+MNB1M2E8nj2A1kweU1zF0OoNDKUnAIlMbDXs3d+kgcuj/zUVa89/nYot5UZICBWA==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0", + "Newtonsoft.Json": "13.0.1" + } + }, + "Google.Api.Gax.Rest": { + "type": "Transitive", + "resolved": "4.0.0", + "contentHash": "YFoxv4UNP7Hn8Eask0FqwMbCAfBs5hUlzronJgFGVXcStohtKc0Mi+2YqgGN82nynx75jVWEF7siSLBwkmZsHA==", + "dependencies": { + "Google.Api.Gax": "4.0.0", + "Google.Apis.Auth": "[1.57.0, 2.0.0)", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0" + } + }, + "Google.Apis": { + "type": "Transitive", + "resolved": "1.71.0", + "contentHash": "xKVk+SLZXeBPKuEKzNTg2K7ptr5ZL40+VSR8e5+yJbzTE9uRCCDPlAzT/ij3urh4usBsn5IiW5rXQI+xAc3bRQ==", + "dependencies": { + "Google.Apis.Core": "1.71.0" + } + }, + "Google.Apis.Auth": { + "type": "Transitive", + "resolved": "1.71.0", + "contentHash": "AshxDrFfj0QQPkmGQWvWhT+iZ8Vif23hLa9mFZ6cnbwvfXmXwXOWK9bP6uWnLj+sO6otAYKm+7SMZYgwi1AdJA==", + "dependencies": { + "Google.Apis": "1.71.0", + "Google.Apis.Core": "1.71.0", + "System.Management": "7.0.2" + } + }, + "Google.Apis.Bigquery.v2": { + "type": "Transitive", + "resolved": "1.57.0.2704", + "contentHash": "soEgBHVrONkBjjFywp0rqFuBTO/Ok/jYVjc2784hY4BjW1l1C6Q30d9hgWQDFa2MWLem4PDrlFRr+sK1lYEBug==", + "dependencies": { + "Google.Apis": "1.57.0", + "Google.Apis.Auth": "1.57.0" + } + }, + "Google.Apis.Core": { + "type": "Transitive", + "resolved": "1.71.0", + "contentHash": "tCsg4USOGBVCIrSOyLht1akZe/+ioH7+LVWix6XuxP5fFiw5kEFSBXL+Of93aK/ztJa1dQI+p03dJ+AKgViRuw==", + "dependencies": { + "Newtonsoft.Json": "13.0.3" + } + }, + "Google.Cloud.BigQuery.V2": { + "type": "Transitive", + "resolved": "3.0.0", + "contentHash": "2P4+4qIRPBt8wuGhq+78m6IkIilJBuPgs7nU/NweIZzxpK9JI0BiPzWzUmci62FAH5z2Rc+q1GX/21TEzjIkcQ==", + "dependencies": { + "Google.Api.Gax.Rest": "[4.0.0, 5.0.0)", + "Google.Apis.Bigquery.v2": "[1.57.0.2704, 2.0.0)" + } + }, "Humanizer.Core": { "type": "Transitive", "resolved": "2.14.1", "contentHash": "lQKvtaTDOXnoVJ20ibTuSIOf2i0uO0MPbDhd1jm238I+U/2ZnRENj0cktKZhtchBMtCUSRQ5v4xBCUbKNmyVMw==" }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "UcSjPsst+DfAdJGVDsu346FX0ci0ah+lw3WRtn18NUwEqRt70HaOQ7lI72vy3+1LxtqI3T5GWwV39rQSrCzAeg==" + }, "Microsoft.Build.Framework": { "type": "Transitive", "resolved": "18.0.2", @@ -198,11 +280,68 @@ "Microsoft.Extensions.Primitives": "10.0.13" } }, + "Microsoft.Extensions.Configuration": { + "type": "Transitive", + "resolved": "10.0.9", + "contentHash": "woZsWLhOQsASuxbmgiZJqiGUBNo3IjRdXC92xt8rRokza+P6/nIsnzq7sm9Or6ZYcRl2kL1ufj8HVzp1QlPTXw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.9", + "Microsoft.Extensions.Primitives": "10.0.9" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "10.0.9", + "contentHash": "Tp/+LPb70RyjjtLg9m5C959eP4KrUpJHThZfAegZVpsfmGvzfuNkuYbI/ft+LvXhMSyUcAeOPaN6rzTccwnZAg==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.9", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.9" + } + }, "Microsoft.Extensions.DependencyModel": { "type": "Transitive", "resolved": "10.0.13", "contentHash": "6TyllZsToy3S/Ziu67GddljRTttuVJ8bbrbVUrsdHqH5678Oax5MpaEubxLoPoWLvgUBYo4KfcAig0mkBtktXQ==" }, + "Microsoft.Extensions.Diagnostics": { + "type": "Transitive", + "resolved": "10.0.9", + "contentHash": "NLXI3PbTe39q6/sgs7JYhmfPf7bMzReUoAJ0q9Po6yhfM+0anZa7PrEva4W2SdiLWGyB9eKZS9THGt2BP40xJg==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.9", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.9", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.9" + } + }, + "Microsoft.Extensions.Diagnostics.Abstractions": { + "type": "Transitive", + "resolved": "10.0.9", + "contentHash": "86RgyFsmVslW4Nu28IXgt8tLglynGQrwjk/xhGZaTe8j6YIeR1Ywoc42hSHsBSl920CQdfqq2dBohZiGm3AkUA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.9", + "Microsoft.Extensions.Options": "10.0.9" + } + }, + "Microsoft.Extensions.FileProviders.Abstractions": { + "type": "Transitive", + "resolved": "10.0.0", + "contentHash": "/ppSdehKk3fuXjlqCDgSOtjRK/pSHU8eWgzSHfHdwVm5BP4Dgejehkw+PtxKG2j98qTDEHDst2Y99aNsmJldmw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.0" + } + }, + "Microsoft.Extensions.Hosting.Abstractions": { + "type": "Transitive", + "resolved": "10.0.0", + "contentHash": "KrN6TGFwCwqOkLLk/idW/XtDQh+8In+CL9T4M1Dx+5ScsjTq4TlVbal8q532m82UYrMr6RiQJF2HvYCN0QwVsA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.0", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.0", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.0", + "Microsoft.Extensions.Logging.Abstractions": "10.0.0" + } + }, "Microsoft.Extensions.Logging": { "type": "Transitive", "resolved": "10.0.13", @@ -230,6 +369,18 @@ "Microsoft.Extensions.Primitives": "10.0.13" } }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "Transitive", + "resolved": "10.0.9", + "contentHash": "Y4E24zffF/aPS0igNvY6ZzAQfbxd6AYdC9L4brnH+uK0yYYHIR6FeGVQVVjAOo8wub1EQDl2B90lCcpqoTF7Yw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.9", + "Microsoft.Extensions.Configuration.Binder": "10.0.9", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.9", + "Microsoft.Extensions.Options": "10.0.9", + "Microsoft.Extensions.Primitives": "10.0.9" + } + }, "Microsoft.Extensions.Primitives": { "type": "Transitive", "resolved": "10.0.13", @@ -258,8 +409,13 @@ }, "System.CodeDom": { "type": "Transitive", - "resolved": "6.0.0", - "contentHash": "CPc6tWO1LAer3IzfZufDBRL+UZQcj5uS207NHALQzP84Vp/z6wF0Aa0YZImOQY8iStY0A2zI/e3ihKNPfUm8XA==" + "resolved": "7.0.0", + "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" + }, + "System.CommandLine": { + "type": "Transitive", + "resolved": "2.0.0", + "contentHash": "Bjklzc5NoxqAGFi7BcGlY2TWAdB06Bq3a5sfRr3ubMRU80Mf98eyq3Y2UgR6xRV0TLznZmfe5T7mUjOunRNcdA==" }, "System.Composition": { "type": "Transitive", @@ -309,6 +465,14 @@ "System.Composition.Runtime": "9.0.0" } }, + "System.Management": { + "type": "Transitive", + "resolved": "7.0.2", + "contentHash": "/qEUN91mP/MUQmJnM5y5BdT7ZoPuVrtxnFlbJ8a3kBJGhe2wCzBfnPFtK2wTtEEcf3DMGR9J00GZZfg6HRI6yA==", + "dependencies": { + "System.CodeDom": "7.0.0" + } + }, "dfe.checkperformancedata.application": { "type": "Project", "dependencies": { @@ -330,6 +494,19 @@ "resolved": "1.8.5", "contentHash": "BUgFIpmkjwMPi6QneHUHaprjUOGSXoWEF/xi/2seHCpnI8XMHACNiC09rn9qL8jbxrP8jWyJ3DnOjsGvXWNGmw==" }, + "DfeAnalytics.Core": { + "type": "CentralTransitive", + "requested": "[0.6.4, )", + "resolved": "0.6.4", + "contentHash": "LfmZAaetqxonAFkZSeY0M3udvzyc35QGEgDTm5ohXocTQZNOks/iqBs7tOQ7LOEw4B6cXp6aE1MGzbdsWOKIyQ==", + "dependencies": { + "Google.Apis.Auth": "1.71.0", + "Google.Cloud.BigQuery.V2": "[3.0.0, 4.0.0)", + "Microsoft.Extensions.Configuration": "8.0.0", + "Microsoft.Extensions.Logging": "8.0.0", + "Microsoft.Extensions.Options": "8.0.0" + } + }, "HtmlSanitizer": { "type": "CentralTransitive", "requested": "[9.2.1039, )", @@ -363,6 +540,20 @@ "dependencies": { "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13" } + }, + "Microsoft.Extensions.Http": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.9", + "contentHash": "5jdmuLRg1HO7x/v8McOf6ndhkMQWImkRBtJpXM8+xIocXNdLqCItyyFLKusdILc62TDzHuLlqaWoxtOA5FFOPg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.9", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.9", + "Microsoft.Extensions.Diagnostics": "10.0.9", + "Microsoft.Extensions.Logging": "10.0.9", + "Microsoft.Extensions.Logging.Abstractions": "10.0.9", + "Microsoft.Extensions.Options": "10.0.9" + } } } } diff --git a/src/DfE.CheckPerformanceData.RulesEngineWorker/packages.lock.json b/src/DfE.CheckPerformanceData.RulesEngineWorker/packages.lock.json index ac8a5bc9a..f1fe4fcd7 100644 --- a/src/DfE.CheckPerformanceData.RulesEngineWorker/packages.lock.json +++ b/src/DfE.CheckPerformanceData.RulesEngineWorker/packages.lock.json @@ -347,6 +347,11 @@ "resolved": "7.0.0", "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" }, + "System.CommandLine": { + "type": "Transitive", + "resolved": "2.0.0", + "contentHash": "Bjklzc5NoxqAGFi7BcGlY2TWAdB06Bq3a5sfRr3ubMRU80Mf98eyq3Y2UgR6xRV0TLznZmfe5T7mUjOunRNcdA==" + }, "System.IO.Hashing": { "type": "Transitive", "resolved": "10.0.9", @@ -391,7 +396,7 @@ "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Domain": "[1.0.0, )", "DfE.CheckPerformanceData.Persistence": "[1.0.0, )", - "DfeAnalytics.Core": "[0.6.2, )", + "DfeAnalytics.Core": "[0.6.4, )", "GovukNotify": "[8.2.0, )", "Microsoft.AspNetCore.Authentication.OpenIdConnect": "[10.0.13, )", "Microsoft.EntityFrameworkCore": "[10.0.13, )", @@ -411,6 +416,7 @@ "dependencies": { "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Domain": "[1.0.0, )", + "DfeAnalytics.EFCore": "[0.6.4, )", "Microsoft.EntityFrameworkCore": "[10.0.13, )", "Microsoft.EntityFrameworkCore.Relational": "[10.0.13, )", "Newtonsoft.Json": "[13.0.4, )", @@ -432,14 +438,26 @@ }, "DfeAnalytics.Core": { "type": "CentralTransitive", - "requested": "[0.6.2, )", - "resolved": "0.6.2", - "contentHash": "LIW1zpoLKWPt2017anc4xtmoPrsqV/EifFTJMhwA1/ZT3XJZp/AVj7KTsYcy3FJSCIbvPyAjpdfjFV53tIPd6g==", + "requested": "[0.6.4, )", + "resolved": "0.6.4", + "contentHash": "LfmZAaetqxonAFkZSeY0M3udvzyc35QGEgDTm5ohXocTQZNOks/iqBs7tOQ7LOEw4B6cXp6aE1MGzbdsWOKIyQ==", "dependencies": { "Google.Apis.Auth": "1.71.0", "Google.Cloud.BigQuery.V2": "[3.0.0, 4.0.0)" } }, + "DfeAnalytics.EFCore": { + "type": "CentralTransitive", + "requested": "[0.6.4, )", + "resolved": "0.6.4", + "contentHash": "t4Rw5dqWcNNsKm6hTxoTdomrfDskb2SEt70Wo7dhnKixJqkmlmcjSpoueKZcES5fogTpg87cNKLLXLkh51Cqsg==", + "dependencies": { + "DfeAnalytics.Core": "0.6.4", + "Microsoft.EntityFrameworkCore": "10.0.0", + "Npgsql.EntityFrameworkCore.PostgreSQL": "10.0.0", + "System.CommandLine": "2.0.0" + } + }, "GovukNotify": { "type": "CentralTransitive", "requested": "[8.2.0, )", diff --git a/src/DfE.CheckPerformanceData.Web/packages.lock.json b/src/DfE.CheckPerformanceData.Web/packages.lock.json index 7ff310fa9..3000f7995 100644 --- a/src/DfE.CheckPerformanceData.Web/packages.lock.json +++ b/src/DfE.CheckPerformanceData.Web/packages.lock.json @@ -23,11 +23,11 @@ }, "DfeAnalytics.AspNetCore": { "type": "Direct", - "requested": "[0.6.2, )", - "resolved": "0.6.2", - "contentHash": "mn6A8jAuiJ4O1HZure+HGgXReHK7sz0rDX6BhU9fqMcgQH+n68tbbbsPBg/bYLGF0KXFc1mWcKFjB3j2QyK9aA==", + "requested": "[0.6.4, )", + "resolved": "0.6.4", + "contentHash": "UlWDCeofbwz7FccSjwOxcKvOTQwTJtrrcGtL/49PZDX7QXRfB5RrCzlS9LsIOX+trZ6G133Wv3YrruPvnqw+TA==", "dependencies": { - "DfeAnalytics.Core": "0.6.2" + "DfeAnalytics.Core": "0.6.4" } }, "GovUk.Frontend.AspNetCore": { @@ -604,6 +604,11 @@ "resolved": "7.0.0", "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" }, + "System.CommandLine": { + "type": "Transitive", + "resolved": "2.0.0", + "contentHash": "Bjklzc5NoxqAGFi7BcGlY2TWAdB06Bq3a5sfRr3ubMRU80Mf98eyq3Y2UgR6xRV0TLznZmfe5T7mUjOunRNcdA==" + }, "System.Composition": { "type": "Transitive", "resolved": "9.0.0", @@ -701,7 +706,7 @@ "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Domain": "[1.0.0, )", "DfE.CheckPerformanceData.Persistence": "[1.0.0, )", - "DfeAnalytics.Core": "[0.6.2, )", + "DfeAnalytics.Core": "[0.6.4, )", "GovukNotify": "[8.2.0, )", "Microsoft.AspNetCore.Authentication.OpenIdConnect": "[10.0.13, )", "Microsoft.EntityFrameworkCore": "[10.0.13, )", @@ -721,6 +726,7 @@ "dependencies": { "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Domain": "[1.0.0, )", + "DfeAnalytics.EFCore": "[0.6.4, )", "Microsoft.EntityFrameworkCore": "[10.0.13, )", "Microsoft.EntityFrameworkCore.Relational": "[10.0.13, )", "Newtonsoft.Json": "[13.0.4, )", @@ -763,14 +769,26 @@ }, "DfeAnalytics.Core": { "type": "CentralTransitive", - "requested": "[0.6.2, )", - "resolved": "0.6.2", - "contentHash": "LIW1zpoLKWPt2017anc4xtmoPrsqV/EifFTJMhwA1/ZT3XJZp/AVj7KTsYcy3FJSCIbvPyAjpdfjFV53tIPd6g==", + "requested": "[0.6.4, )", + "resolved": "0.6.4", + "contentHash": "LfmZAaetqxonAFkZSeY0M3udvzyc35QGEgDTm5ohXocTQZNOks/iqBs7tOQ7LOEw4B6cXp6aE1MGzbdsWOKIyQ==", "dependencies": { "Google.Apis.Auth": "1.71.0", "Google.Cloud.BigQuery.V2": "[3.0.0, 4.0.0)" } }, + "DfeAnalytics.EFCore": { + "type": "CentralTransitive", + "requested": "[0.6.4, )", + "resolved": "0.6.4", + "contentHash": "t4Rw5dqWcNNsKm6hTxoTdomrfDskb2SEt70Wo7dhnKixJqkmlmcjSpoueKZcES5fogTpg87cNKLLXLkh51Cqsg==", + "dependencies": { + "DfeAnalytics.Core": "0.6.4", + "Microsoft.EntityFrameworkCore": "10.0.0", + "Npgsql.EntityFrameworkCore.PostgreSQL": "10.0.0", + "System.CommandLine": "2.0.0" + } + }, "GovukNotify": { "type": "CentralTransitive", "requested": "[8.2.0, )", diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 2d58bbe76..b77155e54 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -9,5 +9,7 @@ NU1004 instead of quietly re-resolving. build.yml and the Dockerfiles also restore in locked mode explicitly; the Dockerfiles have no CI variable. --> true + + DfE.CheckPerformanceData.Persistence.Contexts.PortalDbContext, DfE.CheckPerformanceData.Persistence diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index b82b470de..5fbf6ad7e 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -11,8 +11,9 @@ - - + + + diff --git a/terraform/application/airbyte.tf b/terraform/application/airbyte.tf new file mode 100644 index 000000000..d9ec7fb09 --- /dev/null +++ b/terraform/application/airbyte.tf @@ -0,0 +1,57 @@ +module "airbyte" { + source = "./vendor/modules/aks//aks/airbyte" + + count = var.airbyte_enabled ? 1 : 0 + + environment = var.environment + azure_resource_prefix = var.azure_resource_prefix + service_short = var.service_short + service_name = var.service_name + docker_image = var.docker_image + postgres_version = var.server_version + postgres_url = module.postgres.url + + host_name = module.postgres.host + database_name = module.postgres.name + workspace_id = var.airbyte_enabled ? module.infrastructure_secrets.map.AIRBYTE-WORKSPACE-ID : null + client_id = var.airbyte_enabled ? module.infrastructure_secrets.map.AIRBYTE-CLIENT-ID : null + client_secret = var.airbyte_enabled ? module.infrastructure_secrets.map.AIRBYTE-CLIENT-SECRET : null + repl_password = var.airbyte_enabled ? module.infrastructure_secrets.map.AIRBYTE-REPLICATION-PASSWORD : null + server_url = "https://airbyte-${var.namespace}.${module.cluster_data.ingress_domain}" + connection_status = var.connection_status + + cluster = var.cluster + namespace = var.namespace + gcp_taxonomy_id = local.gcp_taxonomy_id + gcp_policy_tag_id = local.gcp_policy_tag_id + gcp_keyring = local.gcp_keyring + gcp_key = local.gcp_key + + + config_map_ref = module.application_configuration.kubernetes_config_map_name + secret_ref = module.application_configuration.kubernetes_secret_name + cpu = module.cluster_data.configuration_map.cpu_min + + use_azure = var.deploy_azure_backing_services + gcp_bq_sa = var.airbyte_enabled ? module.infrastructure_secrets.map.AIRBYTE-BQ-SA : null + + is_rails_application = false + is_dotnet_application = true + use_legacy_dotnet_shim = false + #dotnet_application_directory = "/src/DfE.CheckPerformanceData.Web" + airbyte_sync_mode = "incremental_deduped_history" +} + +## Airbyte module variables + +variable "airbyte_enabled" { default = false } + +variable "connection_status" { + type = string + default = "inactive" + description = "Connection status, either active or inactive" +} + +locals { + gcp_dataset_name = replace("${var.service_short}_airbyte_${var.environment}", "-", "_") +} diff --git a/terraform/application/application.tf b/terraform/application/application.tf index e1acc154f..9e1819275 100644 --- a/terraform/application/application.tf +++ b/terraform/application/application.tf @@ -24,7 +24,26 @@ module "application_configuration" { AZURE_STORAGE_ACCOUNT_NAME = local.azure_storage_account_name AZURE_STORAGE_ACCESS_KEY = local.azure_storage_access_key AZURE_STORAGE_CONTAINER = local.azure_storage_container - }, local.federated_auth_secrets) + }, + { + AIRBYTE_CONFIGURATION = var.airbyte_enabled ? jsonencode({ + SOURCE_ID = module.airbyte[0].airbyte_source_id + DESTINATION_ID = module.airbyte[0].airbyte_destination_id + CONNECTION_ID = module.airbyte[0].airbyte_connection_id + }) : null + } + , local.federated_auth_secrets, local.egress_storage_secrets) +} + +# AB#294553: the LDS egress needs an account to transfer into (ConnectionStrings__EgressStorage); +# without it the app refuses to transfer with a "not configured" message. The real LDS account +# for the long-lived environments is still to be confirmed with LDS (docs/data-egress.md, gaps), +# so only review apps get one here — the E2E suite's egress facts walk the transfer end to end +# against the per-PR "lds" account, whose cypmd/extracts_input the app creates on first upload. +locals { + egress_storage_secrets = var.config == "review" ? { + ConnectionStrings__EgressStorage = module.storage_private.primary_connection_string + } : {} } diff --git a/terraform/application/config/review.tfvars.json b/terraform/application/config/review.tfvars.json index f56df07e0..ecee0d8bc 100644 --- a/terraform/application/config/review.tfvars.json +++ b/terraform/application/config/review.tfvars.json @@ -6,5 +6,8 @@ "enable_postgres_ssl": false, "blob_delete_retention_days": 60, "worker_replicas": 1, - "max_memory": "4Gi" + "max_memory": "4Gi", + "pg_airbyte_enabled": true, + "airbyte_enabled": true, + "connection_status": "active" } diff --git a/terraform/application/database.tf b/terraform/application/database.tf index aafd938ac..f309c9dbb 100644 --- a/terraform/application/database.tf +++ b/terraform/application/database.tf @@ -14,6 +14,6 @@ module "postgres" { azure_maintenance_window = var.azure_maintenance_window azure_enable_high_availability = var.enable_postgres_high_availability azure_sku_name = var.postgres_flexible_server_sku - server_version = "17" + server_version = var.server_version use_airbyte = var.pg_airbyte_enabled } diff --git a/terraform/application/dfe_analytics.tf b/terraform/application/dfe_analytics.tf index 505c97f30..a80b05172 100644 --- a/terraform/application/dfe_analytics.tf +++ b/terraform/application/dfe_analytics.tf @@ -11,10 +11,10 @@ module "dfe_analytics" { namespace = var.namespace service_short = var.service_short environment = var.environment - gcp_keyring = "cypd-key-ring" - gcp_key = "cypd-key" - gcp_taxonomy_id = "7450003253557751837" - gcp_policy_tag_id = "7087412595261736586" + gcp_taxonomy_id = local.gcp_taxonomy_id + gcp_policy_tag_id = local.gcp_policy_tag_id + gcp_keyring = local.gcp_keyring + gcp_key = local.gcp_key gcp_table_deletion_protection = var.gcp_table_deletion_protection } diff --git a/terraform/application/terraform.tf b/terraform/application/terraform.tf index 27169a0e6..d2dd84234 100644 --- a/terraform/application/terraform.tf +++ b/terraform/application/terraform.tf @@ -13,6 +13,10 @@ terraform { source = "StatusCakeDev/statuscake" version = "2.2.2" } + airbyte = { + source = "airbytehq/airbyte" + version = "0.10.0" + } } backend "azurerm" { container_name = "terraform-state" @@ -38,4 +42,11 @@ provider "kubernetes" { provider "statuscake" { api_token = module.infrastructure_secrets.map.STATUSCAKE-API-TOKEN -} \ No newline at end of file +} + +provider "airbyte" { + # Configuration options + server_url = var.airbyte_enabled ? "https://airbyte-${var.namespace}.${module.cluster_data.ingress_domain}/api/public/v1" : "" + client_id = var.airbyte_enabled ? module.infrastructure_secrets.map.AIRBYTE-CLIENT-ID : "" + client_secret = var.airbyte_enabled ? module.infrastructure_secrets.map.AIRBYTE-CLIENT-SECRET : "" +} diff --git a/terraform/application/variables.tf b/terraform/application/variables.tf index 8e5157eab..58206f601 100644 --- a/terraform/application/variables.tf +++ b/terraform/application/variables.tf @@ -88,8 +88,8 @@ variable "replicas" { } variable "max_memory" { - type = string - default = "4Gi" + type = string + default = "4Gi" } variable "worker_docker_image" { @@ -170,6 +170,12 @@ variable "pg_airbyte_enabled" { description = "Enable Write ahead logging for Postgres replication for Airbyte." default = false } + +variable "server_version" { + description = "Postgres server version" + default = "17" +} + locals { azure_storage_account_name = module.storage_private.name azure_storage_access_key = module.storage_private.primary_access_key @@ -184,4 +190,9 @@ locals { federated_auth_secrets = var.enable_dfe_analytics_federated_auth ? { DfeAnalytics__CredentialsJson = module.dfe_analytics[0].google_cloud_credentials } : {} + + gcp_keyring = "cypd-key-ring" + gcp_key = "cypd-key" + gcp_taxonomy_id = "7450003253557751837" + gcp_policy_tag_id = "7087412595261736586" } diff --git a/tests/DfE.CheckPerformanceData.IntegrationTests/packages.lock.json b/tests/DfE.CheckPerformanceData.IntegrationTests/packages.lock.json index 01bdb470f..b2bcfe074 100644 --- a/tests/DfE.CheckPerformanceData.IntegrationTests/packages.lock.json +++ b/tests/DfE.CheckPerformanceData.IntegrationTests/packages.lock.json @@ -177,21 +177,32 @@ }, "DfeAnalytics.AspNetCore": { "type": "Transitive", - "resolved": "0.6.2", - "contentHash": "mn6A8jAuiJ4O1HZure+HGgXReHK7sz0rDX6BhU9fqMcgQH+n68tbbbsPBg/bYLGF0KXFc1mWcKFjB3j2QyK9aA==", + "resolved": "0.6.4", + "contentHash": "UlWDCeofbwz7FccSjwOxcKvOTQwTJtrrcGtL/49PZDX7QXRfB5RrCzlS9LsIOX+trZ6G133Wv3YrruPvnqw+TA==", "dependencies": { - "DfeAnalytics.Core": "0.6.2" + "DfeAnalytics.Core": "0.6.4" } }, "DfeAnalytics.Core": { "type": "Transitive", - "resolved": "0.6.2", - "contentHash": "LIW1zpoLKWPt2017anc4xtmoPrsqV/EifFTJMhwA1/ZT3XJZp/AVj7KTsYcy3FJSCIbvPyAjpdfjFV53tIPd6g==", + "resolved": "0.6.4", + "contentHash": "LfmZAaetqxonAFkZSeY0M3udvzyc35QGEgDTm5ohXocTQZNOks/iqBs7tOQ7LOEw4B6cXp6aE1MGzbdsWOKIyQ==", "dependencies": { "Google.Apis.Auth": "1.71.0", "Google.Cloud.BigQuery.V2": "[3.0.0, 4.0.0)" } }, + "DfeAnalytics.EFCore": { + "type": "Transitive", + "resolved": "0.6.4", + "contentHash": "t4Rw5dqWcNNsKm6hTxoTdomrfDskb2SEt70Wo7dhnKixJqkmlmcjSpoueKZcES5fogTpg87cNKLLXLkh51Cqsg==", + "dependencies": { + "DfeAnalytics.Core": "0.6.4", + "Microsoft.EntityFrameworkCore": "10.0.0", + "Npgsql.EntityFrameworkCore.PostgreSQL": "10.0.0", + "System.CommandLine": "2.0.0" + } + }, "Docker.DotNet.Enhanced": { "type": "Transitive", "resolved": "4.3.3", @@ -814,6 +825,11 @@ "resolved": "7.0.0", "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" }, + "System.CommandLine": { + "type": "Transitive", + "resolved": "2.0.0", + "contentHash": "Bjklzc5NoxqAGFi7BcGlY2TWAdB06Bq3a5sfRr3ubMRU80Mf98eyq3Y2UgR6xRV0TLznZmfe5T7mUjOunRNcdA==" + }, "System.IdentityModel.Tokens.Jwt": { "type": "Transitive", "resolved": "8.23.0", @@ -923,7 +939,7 @@ "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Domain": "[1.0.0, )", "DfE.CheckPerformanceData.Persistence": "[1.0.0, )", - "DfeAnalytics.Core": "[0.6.2, )", + "DfeAnalytics.Core": "[0.6.4, )", "GovukNotify": "[8.2.0, )", "Microsoft.AspNetCore.Authentication.OpenIdConnect": "[10.0.13, )", "Microsoft.EntityFrameworkCore": "[10.0.13, )", @@ -943,6 +959,7 @@ "dependencies": { "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Domain": "[1.0.0, )", + "DfeAnalytics.EFCore": "[0.6.4, )", "Microsoft.EntityFrameworkCore": "[10.0.13, )", "Microsoft.EntityFrameworkCore.Relational": "[10.0.13, )", "Newtonsoft.Json": "[13.0.4, )", @@ -974,7 +991,7 @@ "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Infrastructure": "[1.0.0, )", "DfE.CheckPerformanceData.Persistence": "[1.0.0, )", - "DfeAnalytics.AspNetCore": "[0.6.2, )", + "DfeAnalytics.AspNetCore": "[0.6.4, )", "GovUk.Frontend.AspNetCore": "[5.0.1, )", "Microsoft.Identity.Web": "[4.16.1, )", "Microsoft.VisualStudio.Azure.Containers.Tools.Targets": "[1.23.0, )", diff --git a/tests/DfE.CheckPerformanceData.UnitTests/packages.lock.json b/tests/DfE.CheckPerformanceData.UnitTests/packages.lock.json index c346c22da..9871c64e8 100644 --- a/tests/DfE.CheckPerformanceData.UnitTests/packages.lock.json +++ b/tests/DfE.CheckPerformanceData.UnitTests/packages.lock.json @@ -154,21 +154,32 @@ }, "DfeAnalytics.AspNetCore": { "type": "Transitive", - "resolved": "0.6.2", - "contentHash": "mn6A8jAuiJ4O1HZure+HGgXReHK7sz0rDX6BhU9fqMcgQH+n68tbbbsPBg/bYLGF0KXFc1mWcKFjB3j2QyK9aA==", + "resolved": "0.6.4", + "contentHash": "UlWDCeofbwz7FccSjwOxcKvOTQwTJtrrcGtL/49PZDX7QXRfB5RrCzlS9LsIOX+trZ6G133Wv3YrruPvnqw+TA==", "dependencies": { - "DfeAnalytics.Core": "0.6.2" + "DfeAnalytics.Core": "0.6.4" } }, "DfeAnalytics.Core": { "type": "Transitive", - "resolved": "0.6.2", - "contentHash": "LIW1zpoLKWPt2017anc4xtmoPrsqV/EifFTJMhwA1/ZT3XJZp/AVj7KTsYcy3FJSCIbvPyAjpdfjFV53tIPd6g==", + "resolved": "0.6.4", + "contentHash": "LfmZAaetqxonAFkZSeY0M3udvzyc35QGEgDTm5ohXocTQZNOks/iqBs7tOQ7LOEw4B6cXp6aE1MGzbdsWOKIyQ==", "dependencies": { "Google.Apis.Auth": "1.71.0", "Google.Cloud.BigQuery.V2": "[3.0.0, 4.0.0)" } }, + "DfeAnalytics.EFCore": { + "type": "Transitive", + "resolved": "0.6.4", + "contentHash": "t4Rw5dqWcNNsKm6hTxoTdomrfDskb2SEt70Wo7dhnKixJqkmlmcjSpoueKZcES5fogTpg87cNKLLXLkh51Cqsg==", + "dependencies": { + "DfeAnalytics.Core": "0.6.4", + "Microsoft.EntityFrameworkCore": "10.0.0", + "Npgsql.EntityFrameworkCore.PostgreSQL": "10.0.0", + "System.CommandLine": "2.0.0" + } + }, "Google.Api.Gax": { "type": "Transitive", "resolved": "4.0.0", @@ -721,6 +732,11 @@ "resolved": "7.0.0", "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" }, + "System.CommandLine": { + "type": "Transitive", + "resolved": "2.0.0", + "contentHash": "Bjklzc5NoxqAGFi7BcGlY2TWAdB06Bq3a5sfRr3ubMRU80Mf98eyq3Y2UgR6xRV0TLznZmfe5T7mUjOunRNcdA==" + }, "System.IdentityModel.Tokens.Jwt": { "type": "Transitive", "resolved": "8.23.0", @@ -819,7 +835,7 @@ "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Domain": "[1.0.0, )", "DfE.CheckPerformanceData.Persistence": "[1.0.0, )", - "DfeAnalytics.Core": "[0.6.2, )", + "DfeAnalytics.Core": "[0.6.4, )", "GovukNotify": "[8.2.0, )", "Microsoft.AspNetCore.Authentication.OpenIdConnect": "[10.0.13, )", "Microsoft.EntityFrameworkCore": "[10.0.13, )", @@ -839,6 +855,7 @@ "dependencies": { "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Domain": "[1.0.0, )", + "DfeAnalytics.EFCore": "[0.6.4, )", "Microsoft.EntityFrameworkCore": "[10.0.13, )", "Microsoft.EntityFrameworkCore.Relational": "[10.0.13, )", "Newtonsoft.Json": "[13.0.4, )", @@ -870,7 +887,7 @@ "DfE.CheckPerformanceData.Application": "[1.0.0, )", "DfE.CheckPerformanceData.Infrastructure": "[1.0.0, )", "DfE.CheckPerformanceData.Persistence": "[1.0.0, )", - "DfeAnalytics.AspNetCore": "[0.6.2, )", + "DfeAnalytics.AspNetCore": "[0.6.4, )", "GovUk.Frontend.AspNetCore": "[5.0.1, )", "Microsoft.Identity.Web": "[4.16.1, )", "Microsoft.VisualStudio.Azure.Containers.Tools.Targets": "[1.23.0, )",