diff --git a/deployment.md b/deployment.md index 5dd27067..7c1cc5e1 100644 --- a/deployment.md +++ b/deployment.md @@ -41,6 +41,26 @@ toolforge envvars create MONTAGE_DB_URL # mysql+pymysql://:@ toolforge envvars create MONTAGE_SUPERUSERS # OPTIONAL — enables su-impersonation only, NOT admin access (see "Maintainer vs superuser" below) toolforge envvars create MONTAGE_API_LOG_PATH # e.g. /data/project/montage-beta/montage_api.log toolforge envvars create MONTAGE_REPLAY_LOG_PATH +toolforge envvars create MONTAGE_IMPORT_CHECK_PATH # the tool's own folder, e.g. /data/project/montage-beta/import_checks (see below) +``` + +`MONTAGE_IMPORT_CHECK_PATH` is required everywhere except local development: Montage refuses to +start without it. It is the folder for import checks (hatnote/montage#510): each check of a +first-round import source is written there as a JSON file (mode 0600). A check can be used to +save a round for 1 hour. The file is deleted after at most 7 days: every check, import, download +and app start deletes older files, and at most 5,000 are kept. The files hold the source link, +the rows and Commons' data per file, including Commons uploader names. + +- It must be a folder **of its own**: Montage refuses to start if it holds other files, because + the cleanup deletes old check files there. +- It must be absolute and on the tool's NFS share (`/data/project//...`, mounted with + `--mount all`), so that every pod and a restarted pod see the same files. + +Create the folder first, in the tool's home (`$HOME` is `/data/project/`), and print the +value to enter at the `envvars create` prompt: + +```bash +mkdir -m 700 "$HOME/import_checks" && ls -ld "$HOME/import_checks" && echo "$HOME/import_checks" ``` Optional env vars (all have sensible defaults): @@ -191,7 +211,25 @@ git -C ~/www/python/src checkout master git -C ~/www/python/src pull --ff-only ``` -#### 4. Run the deploy script +#### 4. Check new required settings + +A release can add a required environment variable; the app then refuses to start until it is +set. Since hatnote/montage#510 that is `MONTAGE_IMPORT_CHECK_PATH` (see Fresh install, step 3). +`deploy.sh` checks the required variables before it builds, and stops if one is missing. +Before the first deploy that includes it, create the folder and set the variable on that tool +(as the tool, after `become`): + +```bash +mkdir -m 700 "$HOME/import_checks" && ls -ld "$HOME/import_checks" && echo "$HOME/import_checks" +``` + +```bash +toolforge envvars create MONTAGE_IMPORT_CHECK_PATH +``` + +Enter the path the first command printed (e.g. `/data/project/montage-beta/import_checks`). + +#### 5. Run the deploy script Which ref to deploy depends on the tool (see [Multi-environment setup](#multi-environment-setup)): @@ -233,6 +271,24 @@ from `toolforge envvars`. A tool that worked on the legacy service can therefore buildservice if the `MONTAGE_*` env vars (Fresh install step 3) are not set — and its database may need migrating (see "Startup crash: missing column" below). +#### Import checks: troubleshooting and rollback + +- **The pod does not start and its log says `MONTAGE_IMPORT_CHECK_PATH ...`:** create the folder + and the variable (step 4), then `toolforge webservice buildservice restart --mount all`. No + rebuild and no rollback are needed. +- **A check of a large source fails with a server error after about 30 s:** the request hit gunicorn's + worker timeout. Nothing was saved; try a smaller source (split a list, or a sub-category). +- **Old check files:** every use of the folder (and every app start) deletes files older than + 7 days. To delete them by hand: + +```bash +find "$HOME/import_checks" -name '*.json' -mtime +7 -delete +``` + +- **Rolling back to a version before #510:** the variable and the folder can stay (older + versions ignore them). Delete the check files (`rm "$HOME"/import_checks/*.json`): they hold + Commons uploader names that nothing reads any more. Entries already imported stay. + --- ## Debugging diff --git a/frontend/cypress/e2e/campaign.cy.js b/frontend/cypress/e2e/campaign.cy.js index bbfcf20e..2e8e1bf3 100644 --- a/frontend/cypress/e2e/campaign.cy.js +++ b/frontend/cypress/e2e/campaign.cy.js @@ -1,3 +1,39 @@ +// #510: a clean check of one file, valid from now on +const stubCheck = () => { + const now = Date.now() + cy.intercept('POST', '/v1/admin/campaign/*/import/check', { + body: { + status: 'success', + data: { + token: 'cypress-check-token-000000000000', + checked_at: new Date(now).toISOString().replace(/\.\d+Z$/, 'Z'), + expires_at: new Date(now + 3600 * 1000).toISOString().replace(/\.\d+Z$/, 'Z'), + import_method: 'selected', + source: { file_names: ['Example.jpg'] }, + columns: { name: null, file_id: null, ignored: [] }, + counts: { + ok: 1, renamed: 0, duplicate: 0, unknown_name: 0, + unknown_file_id: 0, malformed_file_id: 0, same_name: 0 + }, + blocking: false, + total_rows: 1, + importable_count: 1, + issues: [], + issues_total: 0, + issues_truncated: false, + same_name_groups: [] + } + } + }).as('checkImport') +} + +const savedRound = () => ({ + id: 999, + name: 'My Test Round', + import: { new_round_entry_count: 1, warnings: [], disqualified: [] } +}) + + describe('Campaign Details Page', () => { beforeEach(() => { cy.setCookie('clastic_cookie', ''); @@ -84,8 +120,53 @@ describe('Campaign Details Page', () => { .find('li') .first() .click(); - cy.get('.button-group button').contains('Add Round').click().click(); - cy.log(' Round created successfully'); + // #510 / #447: the source is checked first; Save sends the round together + // with its checked import, once + stubCheck() + cy.intercept('POST', '/v1/admin/campaign/*/add_round', (req) => { + req.reply({ delay: 500, body: { status: 'success', data: savedRound() } }) + }).as('addRound') + cy.get('.form-container').contains('label', 'File List').click() + cy.get('.form-container textarea').first().type('Example.jpg') + cy.get('[data-testid="add-round-button"]').should('be.disabled') + cy.get('[data-testid="check-source-button"]').click() + cy.wait('@checkImport') + cy.get('[data-testid="import-check-result"]').should('be.visible') + // a double click must not save twice + cy.get('[data-testid="add-round-button"]').click().click({ force: true }) + cy.wait('@addRound').its('request.body').should((body) => { + expect(body.import).to.deep.equal({ + import_method: 'selected', + check_token: 'cypress-check-token-000000000000' + }) + expect(body).not.to.have.property('file_names') + }) + cy.get('@addRound.all').should('have.length', 1) + cy.get('.juror-campaign-round-card').should('not.exist') +}) + +it('keeps the form open when the server refuses the import', () => { + cy.get('.add-round-button').click() + cy.get('.form-container input[type="text"]').first().clear().type('My Test Round') + cy.get('.form-container').within(() => { + cy.get('input[placeholder="YYYY-MM-DD"]').first().clear().type('2025-08-15') + }) + cy.get('[data-testid="userlist-search"] input').type('AadarshM07') + cy.get('[data-testid="userlist-search"]').find('li').first().click() + stubCheck() + cy.intercept('POST', '/v1/admin/campaign/*/add_round', { + statusCode: 400, + body: { status: 'failure', error_type: 'import_check_expired', detail: 'check expired' } + }).as('addRound') + cy.get('.form-container').contains('label', 'File List').click() + cy.get('.form-container textarea').first().type('Example.jpg') + cy.get('[data-testid="check-source-button"]').click() + cy.wait('@checkImport') + cy.get('[data-testid="add-round-button"]').click() + cy.wait('@addRound') + cy.get('.juror-campaign-round-card').should('exist') + cy.get('[data-testid="import-check-result"]').should('contain', 'expired') + cy.get('[data-testid="add-round-button"]').should('be.disabled') }) diff --git a/frontend/src/components/Round/ImportCheckResult.vue b/frontend/src/components/Round/ImportCheckResult.vue new file mode 100644 index 00000000..2bf8d88d --- /dev/null +++ b/frontend/src/components/Round/ImportCheckResult.vue @@ -0,0 +1,316 @@ + + + + + diff --git a/frontend/src/components/Round/RoundNew.vue b/frontend/src/components/Round/RoundNew.vue index b3e4d129..ee92643d 100644 --- a/frontend/src/components/Round/RoundNew.vue +++ b/frontend/src/components/Round/RoundNew.vue @@ -1,5 +1,7 @@ + - + - + +
+ + {{ + isChecking + ? $t('montage-round-check-running') + : $t('montage-round-check-source') + }} + +

+ {{ $t('montage-round-check-required') }} +

+ +
@@ -177,17 +218,40 @@ + +
+ +

{{ saveError }}

+

+ {{ saveErrorDetail }} +

+ +
+

{{ saveHint }}

- {{ $t('montage-round-add') }} + + {{ + isLoading && roundIndex === 0 + ? $t('montage-round-saving-with-files') + : $t('montage-round-add') + }} @@ -201,7 +265,7 @@