Add durable Windows VM management, nightly shutdown, and 60-day costs

.github/workflows/deploy-vms-portal.yml

@@ -33,6 +33,15 @@ jobs:
         working-directory: vms_portal
       - run: uv run pytest
         working-directory: vms_portal
+      - run: uv run pytest ../lambda/windows_vm_shutdown/tests
+        working-directory: vms_portal
+      - name: Validate CloudFormation templates
+        run: >-
+          uvx cfn-lint
+          cloudformation/vms-portal-foundation-template.yml
+          cloudformation/vms-portal-cur-export-template.yml
+          cloudformation/vms-portal-access-template.yml
+          cloudformation/windows-a11y-instance-template.yml
       - run: docker build -t vms-portal:validate vms_portal
 
   deploy:
@@ -49,6 +58,8 @@ jobs:
       ECR_REPOSITORY: vms-portal
       IMAGE_TAG: ${{ github.sha }}
       ACCESS_STACK_NAME: vms-portal-access
+      FOUNDATION_STACK_NAME: vms-portal-foundation
+      CUR_EXPORT_STACK_NAME: vms-portal-cur-export
       RUNTIME_ROLE_NAME: coseeing-ec2-common
       AUTH_SECRET_ID: ${{ inputs.auth_secret_id }}
     steps:
@@ -70,6 +81,42 @@ jobs:
             --query ARN \
             --output text)
 
+          aws cloudformation deploy \
+            --stack-name "$FOUNDATION_STACK_NAME" \
+            --template-file cloudformation/vms-portal-foundation-template.yml \
+            --no-fail-on-empty-changeset
+
+          FOUNDATION_OUTPUTS=$(aws cloudformation describe-stacks \
+            --stack-name "$FOUNDATION_STACK_NAME" \
+            --query 'Stacks[0].Outputs' \
+            --output json)
+          ARTIFACTS_BUCKET=$(jq -r '.[] | select(.OutputKey == "DeploymentArtifactsBucketName") | .OutputValue' <<<"$FOUNDATION_OUTPUTS")
+          COST_DATA_BUCKET=$(jq -r '.[] | select(.OutputKey == "CostDataBucketName") | .OutputValue' <<<"$FOUNDATION_OUTPUTS")
+          COST_DATA_PREFIX=$(jq -r '.[] | select(.OutputKey == "CostDataPrefix") | .OutputValue' <<<"$FOUNDATION_OUTPUTS")
+          COST_QUERY_RESULTS_PREFIX=$(jq -r '.[] | select(.OutputKey == "CostQueryResultsPrefix") | .OutputValue' <<<"$FOUNDATION_OUTPUTS")
+          COST_DATABASE=$(jq -r '.[] | select(.OutputKey == "CostDatabaseName") | .OutputValue' <<<"$FOUNDATION_OUTPUTS")
+          COST_TABLE=$(jq -r '.[] | select(.OutputKey == "CostTableName") | .OutputValue' <<<"$FOUNDATION_OUTPUTS")
+          COST_WORKGROUP=$(jq -r '.[] | select(.OutputKey == "CostWorkGroupName") | .OutputValue' <<<"$FOUNDATION_OUTPUTS")
+
+          aws cloudformation deploy \
+            --region us-east-1 \
+            --stack-name "$CUR_EXPORT_STACK_NAME" \
+            --template-file cloudformation/vms-portal-cur-export-template.yml \
+            --parameter-overrides \
+              CostDataBucketName="$COST_DATA_BUCKET" \
+              CostDataBucketRegion="$AWS_REGION" \
+            --no-fail-on-empty-changeset
+
+          SHUTDOWN_CODE_KEY="lambda/windows-vm-shutdown/${GITHUB_SHA}.zip"
+          zip -X -j /tmp/windows-vm-shutdown.zip \
+            lambda/windows_vm_shutdown/lambda_function.py
+          SHUTDOWN_CODE_VERSION=$(aws s3api put-object \
+            --bucket "$ARTIFACTS_BUCKET" \
+            --key "$SHUTDOWN_CODE_KEY" \
+            --body /tmp/windows-vm-shutdown.zip \
+            --query VersionId \
+            --output text)
+
           if ! aws ecr describe-repositories \
             --repository-names "$ECR_REPOSITORY" >/dev/null 2>&1; then
             aws ecr create-repository \
@@ -86,6 +133,15 @@ jobs:
               ExistingRoleName="$RUNTIME_ROLE_NAME" \
               AuthSecretArn="$SECRET_ARN" \
               LogRetentionDays=90 \
+              ShutdownCodeS3Bucket="$ARTIFACTS_BUCKET" \
+              ShutdownCodeS3Key="$SHUTDOWN_CODE_KEY" \
+              ShutdownCodeS3Version="$SHUTDOWN_CODE_VERSION" \
+              CostDataBucketName="$COST_DATA_BUCKET" \
+              CostDataPrefix="$COST_DATA_PREFIX" \
+              CostQueryResultsPrefix="$COST_QUERY_RESULTS_PREFIX" \
+              CostDatabaseName="$COST_DATABASE" \
+              CostTableName="$COST_TABLE" \
+              CostWorkGroupName="$COST_WORKGROUP" \
             --no-fail-on-empty-changeset
 
           STACK_OUTPUTS=$(aws cloudformation describe-stacks \
@@ -112,6 +168,10 @@ jobs:
 
           echo "host_ip=$HOST_IP" >> "$GITHUB_OUTPUT"
           echo "instance_id=$INSTANCE_ID" >> "$GITHUB_OUTPUT"
+          echo "cost_data_bucket=$COST_DATA_BUCKET" >> "$GITHUB_OUTPUT"
+          echo "cost_database=$COST_DATABASE" >> "$GITHUB_OUTPUT"
+          echo "cost_table=$COST_TABLE" >> "$GITHUB_OUTPUT"
+          echo "cost_workgroup=$COST_WORKGROUP" >> "$GITHUB_OUTPUT"
       - name: Build and push image
         run: |
           set -euo pipefail
@@ -132,11 +192,14 @@ jobs:
       - name: Deploy with Ansible
         env:
           HOST_IP: ${{ steps.prepare.outputs.host_ip }}
+          COST_DATABASE: ${{ steps.prepare.outputs.cost_database }}
+          COST_TABLE: ${{ steps.prepare.outputs.cost_table }}
+          COST_WORKGROUP: ${{ steps.prepare.outputs.cost_workgroup }}
           ANSIBLE_HOST_KEY_CHECKING: "False"
         run: |
           set -euo pipefail
           printf '[portal]\n%s ansible_user=ubuntu\n' "$HOST_IP" > inventory
-          ansible-playbook -i inventory -e "deploy_tag=$IMAGE_TAG" -e "auth_secret_id=$AUTH_SECRET_ID" -e "deploy_domain=vms.coseeing.org" ansible_yaml/vms-portal-playbook.yml
+          ansible-playbook -i inventory -e "deploy_tag=$IMAGE_TAG" -e "auth_secret_id=$AUTH_SECRET_ID" -e "deploy_domain=vms.coseeing.org" -e "cost_database=$COST_DATABASE" -e "cost_table=$COST_TABLE" -e "cost_workgroup=$COST_WORKGROUP" ansible_yaml/vms-portal-playbook.yml
       - name: Verify portal health
         run: |
           set -euo pipefail

.github/workflows/launch-windows-a11y-ec2.yml

@@ -19,14 +19,10 @@ on:
         description: "Delete only: enter the full name, including prefix (example: windows-a11y-anson-test)"
         required: false
         type: string
-      ami_name:
-        description: "Launch only: AMI version label used by build-windows-a11y-ami"
-        required: false
-        type: string
-      instance_name:
-        description: "Launch only: EC2 Name tag"
+      instance_count:
+        description: "Launch only: number of VMs to create (1-20)"
         required: true
-        default: "windows-a11y"
+        default: "1"
         type: string
       instance_type:
         description: "Launch only: EC2 instance type"
@@ -59,15 +55,19 @@ jobs:
         id: stack
         env:
           ACTION: ${{ inputs.action }}
-          AMI_NAME: ${{ inputs.ami_name }}
           CONFIRM_STACK_NAME: ${{ inputs.confirm_stack_name }}
+          DISK_SIZE: ${{ inputs.disk_size }}
+          INSTANCE_COUNT: ${{ inputs.instance_count }}
+          INSTANCE_TYPE: ${{ inputs.instance_type }}
           STACK_SUFFIX: ${{ inputs.stack_suffix }}
         run: |
           STACK_NAME=$(bash scripts/windows-a11y/validate-stack-operation.sh \
             "${ACTION}" \
             "${STACK_SUFFIX}" \
             "${CONFIRM_STACK_NAME}" \
-            "${AMI_NAME}")
+            "${INSTANCE_COUNT}" \
+            "${INSTANCE_TYPE}" \
+            "${DISK_SIZE}")
 
           echo "Resolved stack name: ${STACK_NAME}"
           echo "stack_name=${STACK_NAME}" >> "$GITHUB_OUTPUT"
@@ -76,6 +76,9 @@ jobs:
             echo "- Action: \`${ACTION}\`"
             echo "- Stack prefix: \`windows-a11y-\`"
             echo "- Full stack name: \`${STACK_NAME}\`"
+            if [ "${ACTION}" = "launch" ]; then
+              echo "- Instance count: \`${INSTANCE_COUNT}\`"
+            fi
           } >> "$GITHUB_STEP_SUMMARY"
 
   launch:
@@ -100,7 +103,7 @@ jobs:
       - name: Find the built Windows A11y AMI
         id: ami
         env:
-          AMI_NAME: windows-a11y-${{ inputs.ami_name }}
+          AMI_NAME: windows-a11y-*
         run: |
           IMAGE_ID=$(aws ec2 describe-images \
             --owners self \
@@ -116,49 +119,74 @@ jobs:
           echo "Using ${AMI_NAME} (${IMAGE_ID})."
           echo "image_id=${IMAGE_ID}" >> "$GITHUB_OUTPUT"
 
-      - name: Deploy EC2 instance
+      - name: Create VM batch stack
+        id: stack
         env:
           AMI_ID: ${{ steps.ami.outputs.image_id }}
           DISK_SIZE: ${{ inputs.disk_size }}
-          INSTANCE_NAME: ${{ inputs.instance_name }}
+          INSTANCE_COUNT: ${{ inputs.instance_count }}
           INSTANCE_TYPE: ${{ inputs.instance_type }}
           STACK_NAME: ${{ needs.validate.outputs.stack_name }}
         run: |
-          aws cloudformation deploy \
+          if aws cloudformation describe-stacks --stack-name "${STACK_NAME}" >/dev/null 2>&1; then
+            echo "::error::Stack ${STACK_NAME} already exists. Choose another suffix."
+            exit 1
+          fi
+
+          STACK_ID=$(aws cloudformation create-stack \
             --stack-name "${STACK_NAME}" \
-            --template-file cloudformation/windows-a11y-instance-template.yml \
-            --parameter-overrides \
-              AmiId="${AMI_ID}" \
-              InstanceType="${INSTANCE_TYPE}" \
-              DiskSize="${DISK_SIZE}" \
-              SubnetId="${{ vars.SUBNET_ID }}" \
-              SecurityGroupId="${{ vars.SECURITY_GROUP_ID }}" \
-              InstanceProfileName="${{ vars.INSTANCE_PROFILE_NAME }}" \
-              KeyName="${{ vars.KEY_NAME }}" \
-              InstanceName="${INSTANCE_NAME}" \
-            --no-fail-on-empty-changeset
-
-      - name: Wait for instance and publish connection details
+            --template-body file://cloudformation/windows-a11y-instance-template.yml \
+            --parameters \
+              ParameterKey=AmiId,ParameterValue="${AMI_ID}" \
+              ParameterKey=InstanceCount,ParameterValue="${INSTANCE_COUNT}" \
+              ParameterKey=InstanceType,ParameterValue="${INSTANCE_TYPE}" \
+              ParameterKey=DiskSize,ParameterValue="${DISK_SIZE}" \
+              ParameterKey=SubnetId,ParameterValue="${{ vars.SUBNET_ID }}" \
+              ParameterKey=SecurityGroupId,ParameterValue="${{ vars.SECURITY_GROUP_ID }}" \
+              ParameterKey=InstanceProfileName,ParameterValue="${{ vars.INSTANCE_PROFILE_NAME }}" \
+              ParameterKey=KeyName,ParameterValue="${{ vars.KEY_NAME }}" \
+            --on-failure DELETE \
+            --query StackId \
+            --output text)
+
+          echo "stack_id=${STACK_ID}" >> "$GITHUB_OUTPUT"
+          if ! aws cloudformation wait stack-create-complete --stack-name "${STACK_ID}"; then
+            echo "::error::CloudFormation failed to create ${STACK_NAME}; the batch is being deleted."
+            aws cloudformation describe-stack-events \
+              --stack-name "${STACK_ID}" \
+              --max-items 20 \
+              --query "StackEvents[?contains(ResourceStatus, 'FAILED')].{Time:Timestamp,Status:ResourceStatus,LogicalId:LogicalResourceId,Type:ResourceType,Reason:ResourceStatusReason}" \
+              --output table || true
+            exit 1
+          fi
+
+      - name: Publish batch connection details
         env:
           AMI_ID: ${{ steps.ami.outputs.image_id }}
+          INSTANCE_COUNT: ${{ inputs.instance_count }}
+          STACK_ID: ${{ steps.stack.outputs.stack_id }}
           STACK_NAME: ${{ needs.validate.outputs.stack_name }}
         run: |
-          INSTANCE_ID=$(aws cloudformation describe-stacks \
-            --stack-name "${STACK_NAME}" \
-            --query "Stacks[0].Outputs[?OutputKey=='InstanceId'].OutputValue" \
-            --output text)
-          aws ec2 wait instance-status-ok --instance-ids "${INSTANCE_ID}"
-          PUBLIC_IP=$(aws cloudformation describe-stacks \
-            --stack-name "${STACK_NAME}" \
-            --query "Stacks[0].Outputs[?OutputKey=='PublicIp'].OutputValue" \
-            --output text)
+          OUTPUTS=$(aws cloudformation describe-stacks \
+            --stack-name "${STACK_ID}" \
+            --query 'Stacks[0].Outputs' \
+            --output json)
 
           {
-            echo "## Windows A11y EC2 launched"
+            echo "## Windows A11y VM batch launched"
             echo "- AMI: \`${AMI_ID}\`"
-            echo "- Instance: \`${INSTANCE_ID}\`"
-            echo "- Public IP: \`${PUBLIC_IP}\`"
             echo "- Stack: \`${STACK_NAME}\`"
+            echo "- Instance count: \`${INSTANCE_COUNT}\`"
+            echo
+            echo "| Name | Instance ID | Private IP | Current public IP |"
+            echo "| --- | --- | --- | --- |"
+            for INDEX in $(seq -w 1 "${INSTANCE_COUNT}"); do
+              printf -v SUFFIX '%03d' "$((10#${INDEX}))"
+              INSTANCE_ID=$(jq -r --arg key "InstanceId${SUFFIX}" '.[] | select(.OutputKey == $key).OutputValue' <<< "${OUTPUTS}")
+              PRIVATE_IP=$(jq -r --arg key "PrivateIp${SUFFIX}" '.[] | select(.OutputKey == $key).OutputValue' <<< "${OUTPUTS}")
+              PUBLIC_IP=$(jq -r --arg key "PublicIp${SUFFIX}" '.[] | select(.OutputKey == $key).OutputValue' <<< "${OUTPUTS}")
+              echo "| ${STACK_NAME}-${SUFFIX} | \`${INSTANCE_ID}\` | \`${PRIVATE_IP}\` | \`${PUBLIC_IP}\` |"
+            done
           } >> "$GITHUB_STEP_SUMMARY"
 
   delete:
@@ -185,6 +213,10 @@ jobs:
             --stack-name "${STACK_NAME}" \
             --query 'Stacks[0].StackStatus' \
             --output text)
+          INSTANCE_COUNT=$(aws cloudformation describe-stacks \
+            --stack-name "${STACK_NAME}" \
+            --query "Stacks[0].Parameters[?ParameterKey=='InstanceCount'].ParameterValue" \
+            --output text)
 
           echo "Deleting ${STACK_NAME} (current status: ${STACK_STATUS})."
           aws cloudformation delete-stack --stack-name "${STACK_NAME}"
@@ -203,4 +235,5 @@ jobs:
             echo "## Windows A11y stack deleted"
             echo "- Stack: \`${STACK_NAME}\`"
             echo "- Previous status: \`${STACK_STATUS}\`"
+            echo "- Deleted VM count: \`${INSTANCE_COUNT}\`"
           } >> "$GITHUB_STEP_SUMMARY"

ansible_yaml/vms-portal-playbook.yml

@@ -16,13 +16,25 @@
     - name: Include common preparation
       include_tasks: common/pre-common.yml
 
+    - name: Create persistent portal data directory
+      file:
+        path: /data/vms-portal/data
+        state: directory
+        owner: 10001
+        group: 10001
+        mode: "0750"
+
     - name: Write portal environment
       copy:
         dest: "{{ docker_compose_dir }}/.env"
         mode: "0600"
         content: |
           AWS_REGION=ap-northeast-1
           AUTH_SECRET_ID={{ auth_secret_id }}
+          ASSIGNMENTS_DB_PATH=/data/vms-portal/data/portal.db
+          COST_DATABASE={{ cost_database }}
+          COST_TABLE={{ cost_table }}
+          COST_WORKGROUP={{ cost_workgroup }}
           TRUSTED_PROXY_IPS=127.0.0.1
 
     - name: Write portal Compose file
@@ -38,6 +50,8 @@
               read_only: true
               tmpfs:
                 - /tmp:size=16m,mode=1777
+              volumes:
+                - /data/vms-portal/data:/data/vms-portal/data
               security_opt:
                 - no-new-privileges:true
               networks: