async_deletion_analysis.json

{
  "source": "Analysis of retained native calls, runtime SQS markers, destination records and aggregate CloudWatch metrics; no inferred handler errors",
  "capture": "testdata/aws/lambda/async_deletion_lifecycle_retry.json",
  "metrics_capture": "testdata/aws/lambda/async_deletion_metrics.json",
  "primary_sources": "testdata/aws/lambda/async_deletion_sources.json",
  "captured_at": "2026-09-30T21:46:44.251904+00:00",
  "observation_finished_at": "2026-10-01T04:10:25.258841+00:00",
  "actor": "arn:aws:iam::000000000000:user/Delegated",
  "region": "us-east-1",
  "prefix": "stackd-adel-fcb27f7b4afe",
  "applied_controls": {
    "settings": {
      "MaximumRetryAttempts": 1,
      "MaximumEventAgeInSeconds": 180
    },
    "exact_targets": {
      "alias-delete": {
        "retry": {
          "event": "alias-delete-control-retry",
          "condition": "RetriesExhausted",
          "count": 2,
          "entries": 2,
          "completed_at": "2026-09-30T21:50:41.422229+00:00"
        },
        "age": {
          "event": "alias-delete-control-age",
          "condition": "EventAgeExceeded",
          "count": 1,
          "entries": 1,
          "completed_at": "2026-09-30T21:53:08.211461+00:00"
        }
      },
      "alias-recreate": {
        "retry": {
          "event": "alias-recreate-control-retry",
          "condition": "RetriesExhausted",
          "count": 2,
          "entries": 2,
          "completed_at": "2026-09-30T21:50:56.080133+00:00"
        },
        "age": {
          "event": "alias-recreate-control-age",
          "condition": "EventAgeExceeded",
          "count": 1,
          "entries": 1,
          "completed_at": "2026-09-30T21:53:29.284795+00:00"
        }
      },
      "alias-refresh": {
        "retry": {
          "event": "alias-refresh-control-retry",
          "condition": "RetriesExhausted",
          "count": 2,
          "entries": 2,
          "completed_at": "2026-09-30T21:50:46.293189+00:00"
        },
        "age": {
          "event": "alias-refresh-control-age",
          "condition": "EventAgeExceeded",
          "count": 1,
          "entries": 1,
          "completed_at": "2026-09-30T21:53:23.635633+00:00"
        }
      },
      "function-delete": {
        "retry": {
          "event": "function-delete-control-retry",
          "condition": "RetriesExhausted",
          "count": 2,
          "entries": 2,
          "completed_at": "2026-09-30T21:50:51.056722+00:00"
        },
        "age": {
          "event": "function-delete-control-age",
          "condition": "EventAgeExceeded",
          "count": 1,
          "entries": 1,
          "completed_at": "2026-09-30T21:53:18.707408+00:00"
        }
      },
      "function-recreate": {
        "retry": {
          "event": "function-recreate-control-retry",
          "condition": "RetriesExhausted",
          "count": 2,
          "entries": 2,
          "completed_at": "2026-09-30T21:50:41.282354+00:00"
        },
        "age": {
          "event": "function-recreate-control-age",
          "condition": "EventAgeExceeded",
          "count": 1,
          "entries": 1,
          "completed_at": "2026-09-30T21:53:13.229323+00:00"
        }
      }
    },
    "proof": "On each of five unchanged targets, ordinary failures produced two real runtime entries and RetriesExhausted/count2. Independent130-second runtime failure produced one real entry and EventAgeExceeded/count1. All ten terminal controls completed before any lifecycle mutation.",
    "limit": "The120-second initial pause and API readbacks were not treated as application barriers. Completed deliveries and entries are the barriers."
  },
  "admission_barrier": {
    "observed_sequence": [
      "Synchronous blocker runtime entry",
      "Synchronous invocation returned429 while reserved slot occupied",
      "Event returned202",
      "Exact alias/function deletion returned204",
      "GetAlias/GetFunction returned404",
      "Same-name recreation/alias refresh where applicable",
      "Blocker returned200"
    ],
    "reserved_concurrency": 1,
    "blocker_sleep_seconds": 45,
    "limit": "No concurrency-zero shortcut. Poll completion does not indicate the exact service dispatch instant."
  },
  "recreation": {
    "alias-recreate": {
      "accepted_request_id": "5692fc08-9f41-46ec-9cca-f98d0a849040",
      "accepted_at": "2026-09-30T21:53:35.924786+00:00",
      "runtime_entry": {
        "case": "alias-recreate-queued",
        "event": {
          "case": "alias-recreate-queued",
          "mode": "success",
          "sleep": 0
        },
        "deployment": "two",
        "version": "2",
        "invoked_arn": "arn:aws:lambda:us-east-1:000000000000:function:stackd-adel-fcb27f7b4afe-aliases:alias-recreate",
        "request_id": "5692fc08-9f41-46ec-9cca-f98d0a849040",
        "started_at": 1790805278.362659,
        "phase": "entry",
        "at": 1790805278.362667
      },
      "destination_queue": "new",
      "terminal_at": "2026-09-30T21:54:38.794Z",
      "terminal_after_acceptance_seconds": 62.869,
      "request_context": {
        "requestId": "5692fc08-9f41-46ec-9cca-f98d0a849040",
        "functionArn": "arn:aws:lambda:us-east-1:000000000000:function:stackd-adel-fcb27f7b4afe-aliases:alias-recreate",
        "condition": "Success",
        "approximateInvokeCount": 1
      },
      "response_context": {
        "statusCode": 200,
        "executedVersion": "2"
      },
      "accepted_id_matches_runtime_and_terminal": true,
      "post_mutation_controls": [
        {
          "queue": "new",
          "timestamp": "2026-09-30T21:56:24.182Z",
          "request_context": {
            "requestId": "df206e64-47ab-40e3-a4f9-0a5ada836a8d",
            "functionArn": "arn:aws:lambda:us-east-1:000000000000:function:stackd-adel-fcb27f7b4afe-aliases:alias-recreate",
            "condition": "Success",
            "approximateInvokeCount": 1
          },
          "response_context": {
            "statusCode": 200,
            "executedVersion": "2"
          }
        }
      ]
    },
    "alias-refresh": {
      "accepted_request_id": "12f92caa-c0f8-4e24-9964-49fe022d3c35",
      "accepted_at": "2026-09-30T21:53:36.126549+00:00",
      "runtime_entry": {
        "case": "alias-refresh-queued",
        "event": {
          "case": "alias-refresh-queued",
          "mode": "success",
          "sleep": 0
        },
        "deployment": "two",
        "version": "2",
        "invoked_arn": "arn:aws:lambda:us-east-1:000000000000:function:stackd-adel-fcb27f7b4afe-aliases:alias-refresh",
        "request_id": "12f92caa-c0f8-4e24-9964-49fe022d3c35",
        "started_at": 1790805280.0515149,
        "phase": "entry",
        "at": 1790805280.051521
      },
      "destination_queue": "new",
      "terminal_at": "2026-09-30T21:54:40.186Z",
      "terminal_after_acceptance_seconds": 64.059,
      "request_context": {
        "requestId": "12f92caa-c0f8-4e24-9964-49fe022d3c35",
        "functionArn": "arn:aws:lambda:us-east-1:000000000000:function:stackd-adel-fcb27f7b4afe-aliases:alias-refresh",
        "condition": "Success",
        "approximateInvokeCount": 1
      },
      "response_context": {
        "statusCode": 200,
        "executedVersion": "2"
      },
      "accepted_id_matches_runtime_and_terminal": true,
      "post_mutation_controls": [
        {
          "queue": "new",
          "timestamp": "2026-09-30T21:56:25.327Z",
          "request_context": {
            "requestId": "edc19d5c-87de-4ad5-9fbc-cb0ef539efc6",
            "functionArn": "arn:aws:lambda:us-east-1:000000000000:function:stackd-adel-fcb27f7b4afe-aliases:alias-refresh",
            "condition": "Success",
            "approximateInvokeCount": 1
          },
          "response_context": {
            "statusCode": 200,
            "executedVersion": "2"
          }
        }
      ]
    },
    "function-recreate": {
      "accepted_request_id": "0ef0a066-b081-4bba-bc33-6f6d395d68d1",
      "accepted_at": "2026-09-30T21:55:21.820887+00:00",
      "runtime_entry": {
        "case": "function-recreate-queued",
        "event": {
          "case": "function-recreate-queued",
          "mode": "success",
          "sleep": 0
        },
        "deployment": "two",
        "version": "$LATEST",
        "invoked_arn": "arn:aws:lambda:us-east-1:000000000000:function:stackd-adel-fcb27f7b4afe-function-recreate",
        "request_id": "0ef0a066-b081-4bba-bc33-6f6d395d68d1",
        "started_at": 1790805380.7065966,
        "phase": "entry",
        "at": 1790805380.706605
      },
      "destination_queue": "old",
      "terminal_at": "2026-09-30T21:56:21.124Z",
      "terminal_after_acceptance_seconds": 59.303,
      "request_context": {
        "requestId": "0ef0a066-b081-4bba-bc33-6f6d395d68d1",
        "functionArn": "arn:aws:lambda:us-east-1:000000000000:function:stackd-adel-fcb27f7b4afe-function-recreate:$LATEST",
        "condition": "Success",
        "approximateInvokeCount": 1
      },
      "response_context": {
        "statusCode": 200,
        "executedVersion": "$LATEST"
      },
      "accepted_id_matches_runtime_and_terminal": true,
      "post_mutation_controls": [
        {
          "queue": "new",
          "timestamp": "2026-09-30T21:56:24.443Z",
          "request_context": {
            "requestId": "1b02c752-7523-4c25-9752-3ea22a450a29",
            "functionArn": "arn:aws:lambda:us-east-1:000000000000:function:stackd-adel-fcb27f7b4afe-function-recreate:$LATEST",
            "condition": "Success",
            "approximateInvokeCount": 1
          },
          "response_context": {
            "statusCode": 200,
            "executedVersion": "$LATEST"
          }
        }
      ],
      "publication_incarnations": [
        {
          "deployment": "one",
          "revision": "1daf174c-ce85-4fa8-8a80-bd5d24fb142c",
          "version": "1",
          "created_at": "2026-09-30T21:47:19.774747+00:00"
        },
        {
          "deployment": "two",
          "revision": "e48b6919-e985-4eb3-8dc5-0d29ebd74657",
          "version": "2",
          "created_at": "2026-09-30T21:55:23.616367+00:00"
        }
      ],
      "destination_boundary": "Old accepted event reached OLD destination at21:56:21.124Z; separately accepted post-mutation event reached NEW destination at21:56:24.443Z. New configuration API completed21:55:31.169831Z. Same deployment two/$LATEST, but no independently completed new-route control preceded old event completion; do not infer whether propagation changed during3.319s or old event retained settings."
    },
    "conclusion": "Both alias and whole-function same-name recreation consumed pre-deletion accepted payloads in replacement deployment two with the original request ID/reference. Refreshed qualified routing selected published version2. These positive samples contradict unconditional accepted-event incarnation fencing. Native CreateFunction Publish returned1 initially and2 after whole-function recreation, consistent with documented never-reused publication numbering."
  },
  "deleted_alias": {
    "request_id": "cc4e2ab0-f8d5-4288-823d-e5f7482fc0a2",
    "accepted_response_completed_at": "2026-09-30T21:53:35.720540+00:00",
    "last_all_queues_polled_through": "2026-10-01T04:10:22.794529+00:00",
    "conservative_seconds_after_accepted_response": 22607.074,
    "reported_observation_window_from_pre_call_epoch": 22609.669,
    "runtime_entries": 0,
    "terminal_destinations": 0,
    "metric_dimensions": [
      {
        "Name": "FunctionName",
        "Value": "stackd-adel-fcb27f7b4afe-aliases"
      },
      {
        "Name": "Resource",
        "Value": "stackd-adel-fcb27f7b4afe-aliases:alias-delete"
      }
    ],
    "metric_accounting": {
      "AsyncEventsReceived": 3,
      "AsyncEventsDropped": 3,
      "known_control_drops": 2,
      "extra_observed_drop": 1,
      "Invocations": 3,
      "known_control_entries": 3,
      "Errors": 3,
      "DestinationDeliveryFailures": "No datapoints, not measured zero"
    },
    "extra_drop_minute_utc": "2026-09-30T21:56:00+00:00",
    "extra_drop_minute_relative_to_acceptance_seconds": [
      144.279,
      204.279
    ],
    "conclusion": "The isolated alias metric has one positive additional dropped event beyond the two known terminal controls, in the minute containing the180-second age deadline. No runtime entry or destination record was observed through the six-hour-plus window. This is aggregate evidence of a drop, not a measured RetriesExhausted/EventAgeExceeded destination condition, approximateInvokeCount, status404 record, exact drop timestamp, or a universal deleted-target scheduling rule."
  },
  "deleted_whole_function": {
    "request_id": "b308dcbd-9b2d-4bf4-a1ad-bed25f5a8b72",
    "accepted_response_completed_at": "2026-09-30T21:54:24.178135+00:00",
    "last_all_queues_polled_through": "2026-10-01T04:10:22.794529+00:00",
    "conservative_seconds_after_accepted_response": 22558.616,
    "reported_observation_window_from_pre_call_epoch": 22561.215,
    "runtime_entries": 0,
    "terminal_destinations": 0,
    "metric_dimensions": [
      {
        "Name": "FunctionName",
        "Value": "stackd-adel-fcb27f7b4afe-function-delete"
      }
    ],
    "metric_accounting": {
      "AsyncEventsReceived": 3,
      "AsyncEventsDropped": 2,
      "known_control_drops": 2,
      "Invocations": 4,
      "known_control_entries": 3,
      "successful_blocker_entries": 1,
      "Errors": 3,
      "DestinationDeliveryFailures": "No datapoints, not measured zero"
    },
    "conclusion": "Two known control failures account for all positive dropped metrics; controls plus synchronous blocker account for all observed runtime invocations. No additional positive terminal/drop measurement exists for the accepted deleted-function event. No marker/destination through more than six hours is a bounded inconclusive result, not proof of pending state or permanent loss."
  },
  "metrics_window": {
    "start": "2026-09-30T21:45:44.251915+00:00",
    "end": "2026-10-01T04:15:49.500617+00:00",
    "period_seconds": 60,
    "statistic": "Sum",
    "queried_at": "2026-10-01T04:15:50.302133+00:00",
    "pagination_complete": true,
    "statuses": [
      "Complete"
    ],
    "limit": "Native timestamps in raw SDK responses include timezone offsets. Analysis UTC minutes are normalized equivalents. CloudWatch is aggregate and asynchronously published; missing datapoints are not zero and publication time is not the event time."
  },
  "cost_bounds_observed": {
    "async_events": 18,
    "queue_reads": 711,
    "owned_functions": 3,
    "owned_queues": 3,
    "owned_roles": 1,
    "provisioned_concurrency": 0,
    "cloudtrail_trails": 0
  },
  "cleanup": {
    "complete": true,
    "finished_at": "2026-10-01T04:14:59.846624+00:00",
    "resources_verified_absent": [
      "aliases function and its own versions/aliases",
      "function-delete",
      "function-recreate",
      "marker queue",
      "old destination queue",
      "new destination queue",
      "execution role"
    ],
    "initial_failure": [
      {
        "label": "verify-owned-aliases",
        "code": "ReadTimeoutError",
        "error": "Read timeout on endpoint URL: \"https://lambda.us-east-1.amazonaws.com/2015-03-31/functions/stackd-adel-fcb27f7b4afe-aliases\"",
        "started_at": "2026-10-01T04:10:25.328191+00:00",
        "finished_at": "2026-10-01T04:13:25.533478+00:00"
      }
    ],
    "recovery": "First finally-cleanup Lambda GetFunction timed out before deleting aliases function. Other owned resources were cleaned. --cleanup-only with fresh exact actor verification then verified ownership tag, deleted remaining function, and observed not-found for every resource. Original failed call and transport error remain in raw calls; no evidence overwritten."
  },
  "first_attempt": {
    "capture": "testdata/aws/lambda/async_deletion_lifecycle.json",
    "cleanup_complete": true,
    "outcome": "Stopped before deletion after five native retry1/count2 destinations because capture mistakenly assumed destination arrival drained independent unordered marker queue. Fixed control completion requires destinations plus runtime markers; first artifact preserved unchanged. Fresh attempt separately ethics approved."
  },
  "unresolved": [
    "Deleted alias terminal record condition/count/status and exact scheduling remain unmeasured despite the positive aggregate extra-drop metric.",
    "Deleted whole-function terminal fate remains unmeasured.",
    "Whole-function recreation old/new destination contrast does not distinguish propagation timing from an accepted-event retained-settings rule.",
    "One lifecycle sample does not define all possible version/alias races, backoff distributions or six-hour delivery guarantees."
  ]
}