From e8a1badecd6731ccbffa248ebfead0009048c324 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:03:35 +0000 Subject: [PATCH 01/15] Add Windows snapshots and forks --- docs/windows-images.md | 1 + docs/windows-snapshots.md | 29 +++ lib/guest/client.go | 26 +++ lib/guest/guest.pb.go | 129 ++++++++++-- lib/guest/guest.proto | 11 + lib/guest/guest_grpc.pb.go | 40 ++++ lib/images/machine.go | 6 + lib/images/machine_test.go | 34 +-- lib/instances/create.go | 3 + lib/instances/fork.go | 25 ++- lib/instances/restore.go | 14 +- lib/instances/snapshot.go | 26 ++- lib/instances/standby.go | 3 - lib/instances/start.go | 14 ++ lib/instances/types.go | 5 + lib/instances/windows.go | 70 ++++++- ...ws_snapshot_fork_integration_linux_test.go | 193 ++++++++++++++++++ lib/instances/windows_test.go | 24 ++- lib/system/guest_agent/identity_windows.go | 52 +++++ 19 files changed, 645 insertions(+), 60 deletions(-) create mode 100644 docs/windows-snapshots.md create mode 100644 lib/instances/windows_snapshot_fork_integration_linux_test.go create mode 100644 lib/system/guest_agent/identity_windows.go diff --git a/docs/windows-images.md b/docs/windows-images.md index 0b73a1478..885f97d7b 100644 --- a/docs/windows-images.md +++ b/docs/windows-images.md @@ -13,6 +13,7 @@ A machine image uses these OCI config labels: | `io.hypeman.machine-image.base` | omitted | digest-pinned base reference | | `io.hypeman.machine-image.tpm` | `2.0` | `2.0` | | `io.hypeman.machine-image.secure-boot` | `required` | `required` | +| `io.hypeman.machine-image.bitlocker` | omitted | `disabled` for forkable personas; `reseal-required` otherwise | The base must be pulled before its dependent Windows images. A base cannot be deleted while any cached image references its digest. Instance references are not tracked by the image cache, matching existing Linux behavior: do not delete a base while a dependent Windows instance exists. diff --git a/docs/windows-snapshots.md b/docs/windows-snapshots.md new file mode 100644 index 000000000..09f5c74ce --- /dev/null +++ b/docs/windows-snapshots.md @@ -0,0 +1,29 @@ +# Windows snapshots and forks + +Windows 11 QEMU instances support standby, restore, stopped snapshots, and forks. Snapshot payloads treat the writable qcow2 disk, Secure Boot NVRAM, software TPM state, saved QEMU configuration, and memory image as one machine. + +## Same-instance standby and restore + +Standby pauses QEMU, captures memory and device state, stops QEMU and swtpm, and retains the instance disk, NVRAM, and TPM directory. Restore starts swtpm from that same state before loading QEMU memory. The Windows machine identity and TPM remain unchanged. + +## Fork identity + +A fork receives independent disk and NVRAM files. Hypeman removes the copied TPM state before the child starts, so swtpm initializes a new TPM rather than cloning the parent's identity. The Windows guest agent then writes a new `MachineGuid` and records the child instance ID before the child is returned. + +Fork admission requires the persona OCI label: + +```text +io.hypeman.machine-image.bitlocker=disabled +``` + +Personas marked `reseal-required`, unlabeled personas, and unknown policies can still use same-instance snapshots, but cannot be forked. Hypeman does not expose a child whose encrypted disk was cloned without resealing it to the child's TPM. + +Stopped forks cold-boot with a unique vsock CID and can run concurrently. A standby snapshot contains the Windows VioSock driver's current CID in guest memory, so a memory-restored child initially retains that CID. The source and child must not be restored concurrently until the child has been stopped and cold-started; Hypeman reports a state error instead of allowing QEMU to fail with a CID collision. Creating a running fork directly from a running Windows source therefore requires `target_state=Stopped`. + +## Integration gates + +- `TestWindowsStandbyRestoreIntegration` verifies identity-preserving standby/restore and the stopped snapshot restore/fork APIs. +- `TestWindowsStandbyForkIntegration` verifies a captured desktop memory fork, inherited guest state, fresh machine identity and TPM state, and independent NVRAM/disk files. +- `TestWindowsForkIntegration` verifies independent guest writes and cold-booted stopped forks. + +The private Windows fixture and its license are not stored in this repository. diff --git a/lib/guest/client.go b/lib/guest/client.go index cfdee83b9..f01c3032f 100644 --- a/lib/guest/client.go +++ b/lib/guest/client.go @@ -942,6 +942,32 @@ func CopyFromInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts C return nil } +func RebindInstanceIdentity(ctx context.Context, dialer hypervisor.VsockDialer, instanceID string, waitForAgent time.Duration) (string, error) { + deadline := time.Now().Add(waitForAgent) + for { + conn, err := GetOrCreateConn(ctx, dialer) + if err == nil { + attemptCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + resp, rpcErr := NewGuestServiceClient(conn).RebindIdentity(attemptCtx, &RebindIdentityRequest{InstanceId: instanceID}) + cancel() + if rpcErr == nil { + return resp.MachineId, nil + } + err = fmt.Errorf("rebind guest identity: %w", rpcErr) + } + retryable := isRetryableConnectionError(err) || status.Code(err) == codes.DeadlineExceeded + if !retryable || waitForAgent == 0 || time.Now().After(deadline) { + return "", err + } + CloseConn(dialer.Key()) + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(guestExecSlowRetryInterval): + } + } +} + // ShutdownInstance sends a shutdown signal to the guest VM's init process (PID 1). // The guest-agent forwards the signal to init, which forwards it to the entrypoint. // sig is the signal number to send (0 = SIGTERM default). diff --git a/lib/guest/guest.pb.go b/lib/guest/guest.pb.go index d520602eb..7a8fc1542 100644 --- a/lib/guest/guest.pb.go +++ b/lib/guest/guest.pb.go @@ -1442,6 +1442,94 @@ func (*ReconfigureNetworkResponse) Descriptor() ([]byte, []int) { return file_lib_guest_guest_proto_rawDescGZIP(), []int{18} } +type RebindIdentityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RebindIdentityRequest) Reset() { + *x = RebindIdentityRequest{} + mi := &file_lib_guest_guest_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RebindIdentityRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RebindIdentityRequest) ProtoMessage() {} + +func (x *RebindIdentityRequest) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RebindIdentityRequest.ProtoReflect.Descriptor instead. +func (*RebindIdentityRequest) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{19} +} + +func (x *RebindIdentityRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +type RebindIdentityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + MachineId string `protobuf:"bytes,1,opt,name=machine_id,json=machineId,proto3" json:"machine_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RebindIdentityResponse) Reset() { + *x = RebindIdentityResponse{} + mi := &file_lib_guest_guest_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RebindIdentityResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RebindIdentityResponse) ProtoMessage() {} + +func (x *RebindIdentityResponse) ProtoReflect() protoreflect.Message { + mi := &file_lib_guest_guest_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RebindIdentityResponse.ProtoReflect.Descriptor instead. +func (*RebindIdentityResponse) Descriptor() ([]byte, []int) { + return file_lib_guest_guest_proto_rawDescGZIP(), []int{20} +} + +func (x *RebindIdentityResponse) GetMachineId() string { + if x != nil { + return x.MachineId + } + return "" +} + var File_lib_guest_guest_proto protoreflect.FileDescriptor const file_lib_guest_guest_proto_rawDesc = "" + @@ -1544,17 +1632,24 @@ const file_lib_guest_guest_proto_rawDesc = "" + "\agateway\x18\x05 \x01(\tR\agateway\x12\x1f\n" + "\vdns_servers\x18\x06 \x03(\tR\n" + "dnsServers\"\x1c\n" + - "\x1aReconfigureNetworkResponse*@\n" + + "\x1aReconfigureNetworkResponse\"8\n" + + "\x15RebindIdentityRequest\x12\x1f\n" + + "\vinstance_id\x18\x01 \x01(\tR\n" + + "instanceId\"7\n" + + "\x16RebindIdentityResponse\x12\x1d\n" + + "\n" + + "machine_id\x18\x01 \x01(\tR\tmachineId*@\n" + "\vExecSession\x12\x17\n" + "\x13EXEC_SESSION_SYSTEM\x10\x00\x12\x18\n" + - "\x14EXEC_SESSION_DESKTOP\x10\x012\xae\x03\n" + + "\x14EXEC_SESSION_DESKTOP\x10\x012\xfd\x03\n" + "\fGuestService\x123\n" + "\x04Exec\x12\x12.guest.ExecRequest\x1a\x13.guest.ExecResponse(\x010\x01\x12F\n" + "\vCopyToGuest\x12\x19.guest.CopyToGuestRequest\x1a\x1a.guest.CopyToGuestResponse(\x01\x12L\n" + "\rCopyFromGuest\x12\x1b.guest.CopyFromGuestRequest\x1a\x1c.guest.CopyFromGuestResponse0\x01\x12;\n" + "\bStatPath\x12\x16.guest.StatPathRequest\x1a\x17.guest.StatPathResponse\x12;\n" + "\bShutdown\x12\x16.guest.ShutdownRequest\x1a\x17.guest.ShutdownResponse\x12Y\n" + - "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" + "\x12ReconfigureNetwork\x12 .guest.ReconfigureNetworkRequest\x1a!.guest.ReconfigureNetworkResponse\x12M\n" + + "\x0eRebindIdentity\x12\x1c.guest.RebindIdentityRequest\x1a\x1d.guest.RebindIdentityResponseB'Z%github.com/onkernel/hypeman/lib/guestb\x06proto3" var ( file_lib_guest_guest_proto_rawDescOnce sync.Once @@ -1569,7 +1664,7 @@ func file_lib_guest_guest_proto_rawDescGZIP() []byte { } var file_lib_guest_guest_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_lib_guest_guest_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_lib_guest_guest_proto_goTypes = []any{ (ExecSession)(0), // 0: guest.ExecSession (*ExecRequest)(nil), // 1: guest.ExecRequest @@ -1591,12 +1686,14 @@ var file_lib_guest_guest_proto_goTypes = []any{ (*ShutdownResponse)(nil), // 17: guest.ShutdownResponse (*ReconfigureNetworkRequest)(nil), // 18: guest.ReconfigureNetworkRequest (*ReconfigureNetworkResponse)(nil), // 19: guest.ReconfigureNetworkResponse - nil, // 20: guest.ExecStart.EnvEntry + (*RebindIdentityRequest)(nil), // 20: guest.RebindIdentityRequest + (*RebindIdentityResponse)(nil), // 21: guest.RebindIdentityResponse + nil, // 22: guest.ExecStart.EnvEntry } var file_lib_guest_guest_proto_depIdxs = []int32{ 2, // 0: guest.ExecRequest.start:type_name -> guest.ExecStart 3, // 1: guest.ExecRequest.resize:type_name -> guest.WindowSize - 20, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry + 22, // 2: guest.ExecStart.env:type_name -> guest.ExecStart.EnvEntry 0, // 3: guest.ExecStart.session:type_name -> guest.ExecSession 6, // 4: guest.CopyToGuestRequest.start:type_name -> guest.CopyToGuestStart 7, // 5: guest.CopyToGuestRequest.end:type_name -> guest.CopyToGuestEnd @@ -1609,14 +1706,16 @@ var file_lib_guest_guest_proto_depIdxs = []int32{ 14, // 12: guest.GuestService.StatPath:input_type -> guest.StatPathRequest 16, // 13: guest.GuestService.Shutdown:input_type -> guest.ShutdownRequest 18, // 14: guest.GuestService.ReconfigureNetwork:input_type -> guest.ReconfigureNetworkRequest - 4, // 15: guest.GuestService.Exec:output_type -> guest.ExecResponse - 8, // 16: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse - 10, // 17: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse - 15, // 18: guest.GuestService.StatPath:output_type -> guest.StatPathResponse - 17, // 19: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse - 19, // 20: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse - 15, // [15:21] is the sub-list for method output_type - 9, // [9:15] is the sub-list for method input_type + 20, // 15: guest.GuestService.RebindIdentity:input_type -> guest.RebindIdentityRequest + 4, // 16: guest.GuestService.Exec:output_type -> guest.ExecResponse + 8, // 17: guest.GuestService.CopyToGuest:output_type -> guest.CopyToGuestResponse + 10, // 18: guest.GuestService.CopyFromGuest:output_type -> guest.CopyFromGuestResponse + 15, // 19: guest.GuestService.StatPath:output_type -> guest.StatPathResponse + 17, // 20: guest.GuestService.Shutdown:output_type -> guest.ShutdownResponse + 19, // 21: guest.GuestService.ReconfigureNetwork:output_type -> guest.ReconfigureNetworkResponse + 21, // 22: guest.GuestService.RebindIdentity:output_type -> guest.RebindIdentityResponse + 16, // [16:23] is the sub-list for method output_type + 9, // [9:16] is the sub-list for method input_type 9, // [9:9] is the sub-list for extension type_name 9, // [9:9] is the sub-list for extension extendee 0, // [0:9] is the sub-list for field type_name @@ -1654,7 +1753,7 @@ func file_lib_guest_guest_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_lib_guest_guest_proto_rawDesc), len(file_lib_guest_guest_proto_rawDesc)), NumEnums: 1, - NumMessages: 20, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/lib/guest/guest.proto b/lib/guest/guest.proto index 205b4c477..625259933 100644 --- a/lib/guest/guest.proto +++ b/lib/guest/guest.proto @@ -23,6 +23,9 @@ service GuestService { // ReconfigureNetwork updates the guest network identity without spawning shell commands rpc ReconfigureNetwork(ReconfigureNetworkRequest) returns (ReconfigureNetworkResponse); + + // RebindIdentity assigns a forked guest a new machine identity. + rpc RebindIdentity(RebindIdentityRequest) returns (RebindIdentityResponse); } // ExecRequest represents messages from client to server @@ -176,3 +179,11 @@ message ReconfigureNetworkRequest { // ReconfigureNetworkResponse acknowledges the network reconfiguration request message ReconfigureNetworkResponse {} + +message RebindIdentityRequest { + string instance_id = 1; +} + +message RebindIdentityResponse { + string machine_id = 1; +} diff --git a/lib/guest/guest_grpc.pb.go b/lib/guest/guest_grpc.pb.go index acad4a3d1..892b43345 100644 --- a/lib/guest/guest_grpc.pb.go +++ b/lib/guest/guest_grpc.pb.go @@ -25,6 +25,7 @@ const ( GuestService_StatPath_FullMethodName = "/guest.GuestService/StatPath" GuestService_Shutdown_FullMethodName = "/guest.GuestService/Shutdown" GuestService_ReconfigureNetwork_FullMethodName = "/guest.GuestService/ReconfigureNetwork" + GuestService_RebindIdentity_FullMethodName = "/guest.GuestService/RebindIdentity" ) // GuestServiceClient is the client API for GuestService service. @@ -45,6 +46,8 @@ type GuestServiceClient interface { Shutdown(ctx context.Context, in *ShutdownRequest, opts ...grpc.CallOption) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(ctx context.Context, in *ReconfigureNetworkRequest, opts ...grpc.CallOption) (*ReconfigureNetworkResponse, error) + // RebindIdentity assigns a forked guest a new machine identity. + RebindIdentity(ctx context.Context, in *RebindIdentityRequest, opts ...grpc.CallOption) (*RebindIdentityResponse, error) } type guestServiceClient struct { @@ -130,6 +133,16 @@ func (c *guestServiceClient) ReconfigureNetwork(ctx context.Context, in *Reconfi return out, nil } +func (c *guestServiceClient) RebindIdentity(ctx context.Context, in *RebindIdentityRequest, opts ...grpc.CallOption) (*RebindIdentityResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RebindIdentityResponse) + err := c.cc.Invoke(ctx, GuestService_RebindIdentity_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // GuestServiceServer is the server API for GuestService service. // All implementations must embed UnimplementedGuestServiceServer // for forward compatibility. @@ -148,6 +161,8 @@ type GuestServiceServer interface { Shutdown(context.Context, *ShutdownRequest) (*ShutdownResponse, error) // ReconfigureNetwork updates the guest network identity without spawning shell commands ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) + // RebindIdentity assigns a forked guest a new machine identity. + RebindIdentity(context.Context, *RebindIdentityRequest) (*RebindIdentityResponse, error) mustEmbedUnimplementedGuestServiceServer() } @@ -176,6 +191,9 @@ func (UnimplementedGuestServiceServer) Shutdown(context.Context, *ShutdownReques func (UnimplementedGuestServiceServer) ReconfigureNetwork(context.Context, *ReconfigureNetworkRequest) (*ReconfigureNetworkResponse, error) { return nil, status.Error(codes.Unimplemented, "method ReconfigureNetwork not implemented") } +func (UnimplementedGuestServiceServer) RebindIdentity(context.Context, *RebindIdentityRequest) (*RebindIdentityResponse, error) { + return nil, status.Error(codes.Unimplemented, "method RebindIdentity not implemented") +} func (UnimplementedGuestServiceServer) mustEmbedUnimplementedGuestServiceServer() {} func (UnimplementedGuestServiceServer) testEmbeddedByValue() {} @@ -276,6 +294,24 @@ func _GuestService_ReconfigureNetwork_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } +func _GuestService_RebindIdentity_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RebindIdentityRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GuestServiceServer).RebindIdentity(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: GuestService_RebindIdentity_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GuestServiceServer).RebindIdentity(ctx, req.(*RebindIdentityRequest)) + } + return interceptor(ctx, in, info, handler) +} + // GuestService_ServiceDesc is the grpc.ServiceDesc for GuestService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -295,6 +331,10 @@ var GuestService_ServiceDesc = grpc.ServiceDesc{ MethodName: "ReconfigureNetwork", Handler: _GuestService_ReconfigureNetwork_Handler, }, + { + MethodName: "RebindIdentity", + Handler: _GuestService_RebindIdentity_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/lib/images/machine.go b/lib/images/machine.go index 66fccd361..8fcab1e83 100644 --- a/lib/images/machine.go +++ b/lib/images/machine.go @@ -20,6 +20,7 @@ const ( MachineImageBaseLabel = "io.hypeman.machine-image.base" MachineImageTPMLabel = "io.hypeman.machine-image.tpm" MachineImageSecureBootLabel = "io.hypeman.machine-image.secure-boot" + MachineImageBitLockerLabel = "io.hypeman.machine-image.bitlocker" MachineImageVersion = "1" ) @@ -40,6 +41,7 @@ type MachineImage struct { Base string `json:"base,omitempty"` TPM string `json:"tpm"` SecureBoot string `json:"secure_boot"` + BitLocker string `json:"bitlocker,omitempty"` VirtualSize int64 `json:"virtual_size"` } @@ -65,6 +67,7 @@ func parseMachineImage(meta *containerMetadata) (*MachineImage, error) { Base: strings.TrimSpace(meta.Labels[MachineImageBaseLabel]), TPM: strings.TrimSpace(meta.Labels[MachineImageTPMLabel]), SecureBoot: strings.TrimSpace(meta.Labels[MachineImageSecureBootLabel]), + BitLocker: strings.TrimSpace(meta.Labels[MachineImageBitLockerLabel]), } if machine.DiskPath == "" || filepath.IsAbs(machine.DiskPath) || !filepath.IsLocal(machine.DiskPath) { return nil, fmt.Errorf("machine image disk path must be a local relative path") @@ -87,6 +90,9 @@ func parseMachineImage(meta *containerMetadata) (*MachineImage, error) { return nil, fmt.Errorf("Windows base image cannot reference another base") } case MachineImageWindowsImage: + if machine.BitLocker != "" && machine.BitLocker != "disabled" && machine.BitLocker != "reseal-required" { + return nil, fmt.Errorf("unsupported Windows image BitLocker policy %q", machine.BitLocker) + } if machine.DiskFormat != "qcow2" { return nil, fmt.Errorf("Windows image disk format must be qcow2") } diff --git a/lib/images/machine_test.go b/lib/images/machine_test.go index c01495be0..811826a29 100644 --- a/lib/images/machine_test.go +++ b/lib/images/machine_test.go @@ -28,19 +28,19 @@ func windowsMachineMetadata(kind MachineImageKind, diskPath, base string) *conta if kind == MachineImageWindowsImage { format = "qcow2" } - return &containerMetadata{ - OS: "windows", - Architecture: "amd64", - Labels: map[string]string{ - MachineImageVersionLabel: MachineImageVersion, - MachineImageKindLabel: string(kind), - MachineImageDiskPathLabel: diskPath, - MachineImageDiskFormatLabel: format, - MachineImageBaseLabel: base, - MachineImageTPMLabel: "2.0", - MachineImageSecureBootLabel: "required", - }, + labels := map[string]string{ + MachineImageVersionLabel: MachineImageVersion, + MachineImageKindLabel: string(kind), + MachineImageDiskPathLabel: diskPath, + MachineImageDiskFormatLabel: format, + MachineImageBaseLabel: base, + MachineImageTPMLabel: "2.0", + MachineImageSecureBootLabel: "required", } + if kind == MachineImageWindowsImage { + labels[MachineImageBitLockerLabel] = "disabled" + } + return &containerMetadata{OS: "windows", Architecture: "amd64", Labels: labels} } func TestParseMachineImage(t *testing.T) { @@ -55,6 +55,16 @@ func TestParseMachineImage(t *testing.T) { )) require.NoError(t, err) assert.Equal(t, MachineImageWindowsImage, image.Kind) + assert.Equal(t, "disabled", image.BitLocker) + + invalidBitLocker := windowsMachineMetadata( + MachineImageWindowsImage, + "hypeman/disk.qcow2", + "registry.example/base@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) + invalidBitLocker.Labels[MachineImageBitLockerLabel] = "unknown" + _, err = parseMachineImage(invalidBitLocker) + assert.ErrorContains(t, err, "unsupported Windows image BitLocker policy") _, err = parseMachineImage(&containerMetadata{OS: "windows", Architecture: "amd64", Labels: map[string]string{}}) assert.ErrorContains(t, err, "ordinary Windows container images are not bootable") diff --git a/lib/instances/create.go b/lib/instances/create.go index 109ea4610..070186ac6 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -392,6 +392,9 @@ func (m *manager) createInstance( HealthCheck: cloneHealthCheckPolicy(req.HealthCheck), RestartPolicy: cloneRestartPolicy(req.RestartPolicy), } + if windows { + stored.WindowsBitLockerPolicy = imageInfo.Machine.BitLocker + } // 12. Ensure directories log.DebugContext(ctx, "creating instance directories", "instance_id", id) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 260b123e5..0710d4b77 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -42,14 +42,14 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR if err != nil { return nil, "", false, err } - if err := rejectWindowsSnapshotLifecycle(meta.Platform, "fork"); err != nil { - return nil, "", false, err - } source := m.toInstance(ctx, meta) targetState, err := resolveForkTargetState(req.TargetState, source.State) if err != nil { return nil, "", false, err } + if isWindowsPlatform(source.Platform) && source.State == StateRunning && targetState != StateStopped { + return nil, "", false, fmt.Errorf("%w: Windows forks from a running source require target_state=%s", ErrNotSupported, StateStopped) + } switch source.State { case StateRunning: @@ -141,9 +141,13 @@ func ensureGuestAgentReadyForForkPhase(ctx context.Context, inst *StoredMetadata return fmt.Errorf("create vsock dialer for %s readiness check: %w", phase, err) } + command := []string{"true"} + if isWindowsPlatform(inst.Platform) { + command = []string{"cmd.exe", "/d", "/c", "exit", "0"} + } var stdout, stderr bytes.Buffer exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"true"}, + Command: command, Stdout: &stdout, Stderr: &stderr, WaitForAgent: 120 * time.Second, @@ -215,6 +219,9 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin source := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if err := validateWindowsForkPolicy(stored); err != nil { + return nil, false, err + } switch source.State { case StateStopped, StateStandby: @@ -308,16 +315,18 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.Phases.Record(phasetracking.PhaseStopped, now) } - // Keep the original CID for snapshot-based forks. Rewriting CID in restored - // memory snapshots is not reliable across hypervisors. Concurrent standby - // fork prepare is currently enabled only for Firecracker, whose host vsock - // dialer routes through the per-VM UDS path rather than this metadata CID. + // Keep the original CID for snapshot-based forks. Windows' restored VioSock + // driver retains the CID captured in guest memory until the next cold boot. if source.State == StateStandby { forkMeta.VsockCID = stored.VsockCID } else { forkMeta.VsockCID = generateVsockCID(forkID) } + if err := m.prepareWindowsForkIdentity(&forkMeta); err != nil { + return nil, false, err + } + if forkMeta.NetworkEnabled { // Clear inherited network identity. For stopped instances this is regenerated on start, // and for standby instances restore allocates if identity is empty. diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 1d6c91690..14c25e203 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -43,9 +43,6 @@ func (m *manager) restoreInstance( return nil, err } - if err := rejectWindowsSnapshotLifecycle(meta.Platform, "restore from standby"); err != nil { - return nil, err - } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) @@ -61,6 +58,9 @@ func (m *manager) restoreInstance( log.ErrorContext(ctx, "no snapshot available", "instance_id", id) return nil, fmt.Errorf("no snapshot available for instance %s", id) } + if err := m.ensureWindowsVsockCIDAvailable(ctx, stored); err != nil { + return nil, err + } // 2a. Validate the instance's image (rootfs) still exists before reserving // resources or invoking the hypervisor shim. A deleted image otherwise fails @@ -361,6 +361,14 @@ func (m *manager) restoreInstance( } reconfigureSpanEnd(nil) } + if stored.WindowsIdentityPending { + if err := rebindWindowsIdentity(ctx, stored); err != nil { + _ = hv.Shutdown(ctx) + m.rollbackAdmissionAllocationActive(stored) + releaseNetwork() + return nil, fmt.Errorf("rebind Windows fork identity: %w", err) + } + } releaseRestoreSlotOnce() // 8. Delete snapshot after successful restore unless the hypervisor is keeping it diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 41709f7b2..810a2ac0e 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -63,9 +63,6 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err != nil { return nil, err } - if err := rejectWindowsSnapshotLifecycle(meta.Platform, "snapshot creation"); err != nil { - return nil, err - } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata @@ -254,9 +251,6 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str if err != nil { return nil, err } - if err := rejectWindowsSnapshotLifecycle(rec.StoredMetadata.Platform, "snapshot restore"); err != nil { - return nil, err - } if rec.Snapshot.SourceInstanceID != id { return nil, fmt.Errorf("%w: snapshot %s belongs to instance %s", ErrInvalidRequest, snapshotID, rec.Snapshot.SourceInstanceID) } @@ -377,12 +371,12 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err != nil { return nil, err } - if err := rejectWindowsSnapshotLifecycle(rec.StoredMetadata.Platform, "snapshot fork"); err != nil { - return nil, err - } if err := validateForkVolumeSafety(rec.StoredMetadata.Volumes); err != nil { return nil, fmt.Errorf("%w: snapshot requires readonly volume attachments: %v", ErrNotSupported, err) } + if err := validateWindowsForkPolicy(&rec.StoredMetadata); err != nil { + return nil, err + } if err := m.ensureInstanceNameAvailableForSnapshotFork(ctx, req.Name, rec.StoredMetadata.NetworkEnabled); err != nil { return nil, err @@ -392,6 +386,17 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err != nil { return nil, err } + if isWindowsPlatform(rec.StoredMetadata.Platform) && rec.Snapshot.Kind == SnapshotKindStandby && targetState == StateRunning { + sourceMeta, sourceErr := m.loadMetadata(rec.Snapshot.SourceInstanceID) + if sourceErr == nil { + sourceState := m.toInstance(ctx, sourceMeta).State + if sourceState == StateRunning || sourceState == StateInitializing { + return nil, fmt.Errorf("%w: stop or standby the Windows snapshot source before restoring a running fork", ErrNotSupported) + } + } else if !errors.Is(sourceErr, ErrNotFound) { + return nil, sourceErr + } + } targetHypervisor, err := m.resolveSnapshotTargetHypervisor(rec, req.TargetHypervisor) if err != nil { return nil, err @@ -453,6 +458,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } else { forkMeta.VsockCID = generateVsockCID(forkID) } + if err := m.prepareWindowsForkIdentity(&forkMeta); err != nil { + return nil, err + } if forkMeta.NetworkEnabled { forkMeta.IP = "" forkMeta.MAC = "" diff --git a/lib/instances/standby.go b/lib/instances/standby.go index ae2a728bb..6913a9895 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -44,9 +44,6 @@ func (m *manager) standbyInstance( return nil, err } - if err := rejectWindowsSnapshotLifecycle(meta.Platform, "standby"); err != nil { - return nil, err - } inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) diff --git a/lib/instances/start.go b/lib/instances/start.go index 4c07a8048..fd4a7def7 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -54,6 +54,9 @@ func (m *manager) startInstance( stored.ExitMessage = "" stored.ProgramStartedAt = nil stored.GuestAgentReadyAt = nil + if isWindowsPlatform(stored.Platform) { + stored.VsockCID = generateVsockCID(fmt.Sprintf("%s:%d", stored.Id, time.Now().UnixNano())) + } if len(req.Entrypoint) > 0 { stored.Entrypoint = req.Entrypoint } @@ -236,6 +239,17 @@ func (m *manager) startInstance( } networkSpanEnd(nil) } + if stored.WindowsIdentityPending { + if err := rebindWindowsIdentity(ctx, stored); err != nil { + _, _ = m.stopInstance(ctx, id) + return nil, fmt.Errorf("rebind Windows fork identity: %w", err) + } + meta = &metadata{StoredMetadata: *stored} + if err := m.saveMetadata(meta); err != nil { + _, _ = m.stopInstance(ctx, id) + return nil, fmt.Errorf("save rebound Windows identity: %w", err) + } + } // Return instance state from current metadata without forcing a log scan. finalInst := m.toInstanceWithoutHydration(ctx, meta) diff --git a/lib/instances/types.go b/lib/instances/types.go index df3ab7ff8..6004c1a98 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -92,6 +92,11 @@ type StoredMetadata struct { // Read-only; echoed on the instance API. Platform string + // Windows machine identity and fork policy. These fields are internal and + // remain empty for Linux instances. + WindowsBitLockerPolicy string + WindowsIdentityPending bool + // Resources (matching Cloud Hypervisor terminology) Size int64 // Base memory in bytes HotplugSize int64 // Hotplug memory in bytes diff --git a/lib/instances/windows.go b/lib/instances/windows.go index 3a767a692..964dbc923 100644 --- a/lib/instances/windows.go +++ b/lib/instances/windows.go @@ -1,11 +1,14 @@ package instances import ( + "context" "fmt" "os" "strings" + "time" "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/network" @@ -49,16 +52,73 @@ func validateWindowsCreate(req CreateInstanceRequest, image *images.Image, caps if req.NetworkEgress != nil || len(req.Credentials) != 0 { return fmt.Errorf("%w: Windows instances do not yet support managed egress or credentials", ErrInvalidRequest) } - if req.SnapshotPolicy != nil || req.AutoStandby != nil { - return fmt.Errorf("%w: Windows snapshot policies are added in the snapshots phase", ErrInvalidRequest) + return nil +} + +func validateWindowsForkPolicy(stored *StoredMetadata) error { + if stored != nil && isWindowsPlatform(stored.Platform) && stored.WindowsBitLockerPolicy != "disabled" { + return fmt.Errorf("%w: Windows forks require a persona declared with %s=disabled", ErrNotSupported, images.MachineImageBitLockerLabel) + } + return nil +} + +func (m *manager) ensureWindowsVsockCIDAvailable(ctx context.Context, stored *StoredMetadata) error { + if stored == nil || !isWindowsPlatform(stored.Platform) { + return nil + } + instances, err := m.listInstances(ctx) + if err != nil { + return err + } + for _, instance := range instances { + if instance.Id == stored.Id || instance.VsockCID != stored.VsockCID { + continue + } + if instance.State == StateRunning || instance.State == StateInitializing { + return fmt.Errorf("%w: Windows snapshot restore requires instance %s with the same captured vsock CID to be stopped", ErrInvalidState, instance.Id) + } + } + return nil +} + +func (m *manager) prepareWindowsForkIdentity(stored *StoredMetadata) error { + if stored == nil || !isWindowsPlatform(stored.Platform) { + return nil + } + if err := validateWindowsForkPolicy(stored); err != nil { + return err + } + if err := os.RemoveAll(m.paths.InstanceTPMDir(stored.Id)); err != nil { + return fmt.Errorf("clear forked Windows TPM state: %w", err) + } + if err := os.MkdirAll(m.paths.InstanceTPMDir(stored.Id), 0700); err != nil { + return fmt.Errorf("create forked Windows TPM state: %w", err) } + if err := os.Remove(m.paths.InstanceTPMSocket(stored.Id)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove forked Windows TPM socket: %w", err) + } + stored.WindowsIdentityPending = true return nil } -func rejectWindowsSnapshotLifecycle(platform, operation string) error { - if isWindowsPlatform(platform) { - return fmt.Errorf("%w: %s is not supported for Windows until the snapshots phase", ErrNotSupported, operation) +func rebindWindowsIdentity(ctx context.Context, stored *StoredMetadata) error { + if stored == nil || !isWindowsPlatform(stored.Platform) || !stored.WindowsIdentityPending { + return nil + } + dialer, err := hypervisor.NewVsockDialer(stored.HypervisorType, stored.VsockSocket, stored.VsockCID) + if err != nil { + return fmt.Errorf("create Windows identity dialer: %w", err) + } + rebindCtx, cancel := context.WithTimeout(ctx, 120*time.Second) + defer cancel() + machineID, err := guest.RebindInstanceIdentity(rebindCtx, dialer, stored.Id, 120*time.Second) + if err != nil { + return err + } + if machineID == "" { + return fmt.Errorf("Windows guest agent returned an empty machine identity") } + stored.WindowsIdentityPending = false return nil } diff --git a/lib/instances/windows_snapshot_fork_integration_linux_test.go b/lib/instances/windows_snapshot_fork_integration_linux_test.go new file mode 100644 index 000000000..7ac64740b --- /dev/null +++ b/lib/instances/windows_snapshot_fork_integration_linux_test.go @@ -0,0 +1,193 @@ +//go:build linux && amd64 + +package instances + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWindowsStandbyRestoreIntegration(t *testing.T) { + manager, _, image := setupWindowsSnapshotIntegration(t) + ctx := context.Background() + source := createWindowsSnapshotInstance(t, ctx, manager, image, "windows-restore-source") + + sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) + standby, err := manager.StandbyInstance(ctx, source.Id, StandbyInstanceRequest{}) + require.NoError(t, err) + require.Equal(t, StateStandby, standby.State) + restored, err := manager.RestoreInstance(ctx, source.Id) + require.NoError(t, err) + assert.Equal(t, sourceMachineID, windowsMachineID(t, ctx, manager, restored.Id), "same-instance restore must preserve Windows identity") + + _, err = manager.StopInstance(ctx, source.Id) + require.NoError(t, err) + snapshot, err := manager.CreateSnapshot(ctx, source.Id, CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Name: "windows-stopped-snapshot", + }) + require.NoError(t, err) + t.Cleanup(func() { _ = manager.DeleteSnapshot(context.Background(), snapshot.Id) }) + _, err = manager.RestoreSnapshot(ctx, source.Id, snapshot.Id, RestoreSnapshotRequest{TargetState: StateStopped}) + require.NoError(t, err) + forked, err := manager.ForkSnapshot(ctx, snapshot.Id, ForkSnapshotRequest{ + Name: "windows-stopped-snapshot-child", + TargetState: StateStopped, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) + assert.True(t, forked.WindowsIdentityPending) +} + +func TestWindowsStandbyForkIntegration(t *testing.T) { + manager, p, image := setupWindowsSnapshotIntegration(t) + ctx := context.Background() + source := createWindowsSnapshotInstance(t, ctx, manager, image, "windows-standby-fork-source") + sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) + windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) + + _, err := manager.StandbyInstance(ctx, source.Id, StandbyInstanceRequest{}) + require.NoError(t, err) + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "windows-standby-child", + TargetState: StateRunning, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) + assert.Equal(t, source.VsockCID, forked.VsockCID, "memory-restored Windows forks retain the captured VioSock CID until cold boot") + assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id)) + assert.Equal(t, "inherited", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\standby.txt`)) + assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) + assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) +} + +func TestWindowsForkIntegration(t *testing.T) { + manager, p, image := setupWindowsSnapshotIntegration(t) + ctx := context.Background() + source := createWindowsSnapshotInstance(t, ctx, manager, image, "windows-fork-source") + sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) + windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) + + stopped, err := manager.StopInstance(ctx, source.Id) + require.NoError(t, err) + require.Equal(t, StateStopped, stopped.State) + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "windows-snapshot-child", + TargetState: StateRunning, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) + assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id), "fork must receive a new Windows machine identity") + assert.NotEqual(t, source.VsockCID, forked.VsockCID, "forks need unique vsock CIDs") + assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) + assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) + require.DirExists(t, p.InstanceTPMDir(forked.Id)) + + sourceDiskBefore, err := os.Stat(p.InstanceWindowsDisk(source.Id)) + require.NoError(t, err) + windowsPowerShell(t, ctx, manager, forked.Id, `Set-Content C:\ProgramData\Hypeman\identity.txt child`) + assert.Equal(t, "child", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\identity.txt`)) + sourceDiskAfter, err := os.Stat(p.InstanceWindowsDisk(source.Id)) + require.NoError(t, err) + assert.Equal(t, sourceDiskBefore.ModTime(), sourceDiskAfter.ModTime(), "child writes must not modify the stopped source disk") +} + +func setupWindowsSnapshotIntegration(t *testing.T) (*manager, *paths.Paths, *images.Image) { + t.Helper() + fixture := os.Getenv("HYPEMAN_WINDOWS_TEST_AGENT_PERSONA") + if fixture == "" { + fixture = "/ci/windows/persona-agent.qcow2" + } + if _, err := os.Stat(fixture); err != nil { + if os.Getenv("CI") == "true" { + t.Fatalf("required Windows snapshot fixture is missing: %s", fixture) + } + t.Skipf("Windows snapshot fixture is unavailable: %s", fixture) + } + acquireHeavyIO(t) + + manager, dataDir := setupTestManagerForQEMU(t) + p := paths.New(dataDir) + const digestHex = "acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac" + image := &images.Image{ + Name: "registry.example/windows/persona:snapshot-integration", + Digest: "sha256:" + digestHex, + Platform: "windows/amd64", + Status: images.StatusReady, + Machine: &images.MachineImage{ + Kind: images.MachineImageWindowsPersona, + Base: "registry.example/windows/base@sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", + TPM: "2.0", + SecureBoot: "required", + BitLocker: "disabled", + VirtualSize: 80 << 30, + }, + } + manager.imageManager = windowsFixtureImageManager{image: image} + personaPath, err := images.GetMachineDiskPath(p, image.Name, image.Digest, image.Machine) + require.NoError(t, err) + require.NoError(t, forkvm.CopyRegularFile(fixture, personaPath)) + require.NoError(t, os.Chmod(personaPath, 0444)) + return manager, p, image +} + +func createWindowsSnapshotInstance(t *testing.T, ctx context.Context, manager *manager, image *images.Image, name string) *Instance { + t.Helper() + instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: name, + Image: image.Name, + Platform: "windows/amd64", + Size: 4 << 30, + Vcpus: 4, + Hypervisor: hypervisor.TypeQEMU, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, instance.Id) }) + require.Eventually(t, func() bool { + current, err := manager.GetInstance(ctx, instance.Id) + return err == nil && current.State == StateRunning + }, 75*time.Second, 500*time.Millisecond) + return instance +} + +func windowsMachineID(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { + t.Helper() + return windowsPowerShell(t, ctx, manager, instanceID, `(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid`) +} + +func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, instanceID, command string) string { + t.Helper() + dialer, err := manager.GetVsockDialer(ctx, instanceID) + require.NoError(t, err) + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", command}, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 30 * time.Second, + Timeout: 30, + }) + require.NoError(t, err, stderr.String()) + require.Equal(t, 0, exit.Code, stderr.String()) + return string(bytes.TrimSpace(stdout.Bytes())) +} + +func assertIndependentFile(t *testing.T, source, fork string) { + t.Helper() + sourceInfo, err := os.Stat(source) + require.NoError(t, err) + forkInfo, err := os.Stat(fork) + require.NoError(t, err) + assert.False(t, os.SameFile(sourceInfo, forkInfo), "%s and %s must not share an inode", source, fork) +} diff --git a/lib/instances/windows_test.go b/lib/instances/windows_test.go index 87f71e7c0..e2814e586 100644 --- a/lib/instances/windows_test.go +++ b/lib/instances/windows_test.go @@ -2,6 +2,7 @@ package instances import ( "os" + "path/filepath" "testing" "github.com/kernel/hypeman/lib/autostandby" @@ -23,6 +24,7 @@ func windowsImageFixture() *images.Image { Base: "registry.example/windows/base@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", TPM: "2.0", SecureBoot: "required", + BitLocker: "disabled", VirtualSize: 80 << 30, }, } @@ -33,6 +35,8 @@ func TestValidateWindowsCreate(t *testing.T) { windowsCaps := hypervisor.Capabilities{SupportsUEFIBoot: true, SupportsTPM: true} require.NoError(t, validateWindowsCreate(CreateInstanceRequest{}, image, windowsCaps)) require.NoError(t, validateWindowsCreate(CreateInstanceRequest{NetworkEnabled: true}, image, windowsCaps)) + require.NoError(t, validateWindowsCreate(CreateInstanceRequest{SnapshotPolicy: &SnapshotPolicy{}}, image, windowsCaps)) + require.NoError(t, validateWindowsCreate(CreateInstanceRequest{AutoStandby: &autostandby.Policy{}}, image, windowsCaps)) tests := []struct { name string @@ -43,8 +47,6 @@ func TestValidateWindowsCreate(t *testing.T) { {name: "small memory", caps: windowsCaps, req: CreateInstanceRequest{Size: 2 << 30}}, {name: "one CPU", caps: windowsCaps, req: CreateInstanceRequest{Vcpus: 1}}, {name: "command", caps: windowsCaps, req: CreateInstanceRequest{Cmd: []string{"cmd.exe"}}}, - {name: "snapshot policy", caps: windowsCaps, req: CreateInstanceRequest{SnapshotPolicy: &SnapshotPolicy{}}}, - {name: "auto standby", caps: windowsCaps, req: CreateInstanceRequest{AutoStandby: &autostandby.Policy{}}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -53,9 +55,21 @@ func TestValidateWindowsCreate(t *testing.T) { } } -func TestRejectWindowsSnapshotLifecycle(t *testing.T) { - assert.ErrorIs(t, rejectWindowsSnapshotLifecycle("windows/amd64", "fork"), ErrNotSupported) - assert.NoError(t, rejectWindowsSnapshotLifecycle("linux/amd64", "fork")) +func TestPrepareWindowsForkIdentity(t *testing.T) { + p := paths.New(t.TempDir()) + m := &manager{paths: p} + stored := &StoredMetadata{Id: "fork", Platform: "windows/amd64", WindowsBitLockerPolicy: "disabled"} + require.NoError(t, os.MkdirAll(p.InstanceTPMDir(stored.Id), 0700)) + require.NoError(t, os.WriteFile(filepath.Join(p.InstanceTPMDir(stored.Id), "state"), []byte("source identity"), 0600)) + + require.NoError(t, m.prepareWindowsForkIdentity(stored)) + assert.True(t, stored.WindowsIdentityPending) + entries, err := os.ReadDir(p.InstanceTPMDir(stored.Id)) + require.NoError(t, err) + assert.Empty(t, entries) + + stored.WindowsBitLockerPolicy = "" + assert.ErrorIs(t, m.prepareWindowsForkIdentity(stored), ErrNotSupported) } func TestBuildWindowsHypervisorConfig(t *testing.T) { diff --git a/lib/system/guest_agent/identity_windows.go b/lib/system/guest_agent/identity_windows.go new file mode 100644 index 000000000..131a33c86 --- /dev/null +++ b/lib/system/guest_agent/identity_windows.go @@ -0,0 +1,52 @@ +//go:build windows + +package main + +import ( + "context" + "crypto/rand" + "fmt" + + pb "github.com/kernel/hypeman/lib/guest" + "golang.org/x/sys/windows/registry" +) + +func (s *guestServer) RebindIdentity(_ context.Context, req *pb.RebindIdentityRequest) (*pb.RebindIdentityResponse, error) { + if req.InstanceId == "" { + return nil, fmt.Errorf("instance id is required") + } + + machineID, err := newWindowsMachineID() + if err != nil { + return nil, fmt.Errorf("generate Windows machine id: %w", err) + } + cryptography, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Cryptography`, registry.SET_VALUE) + if err != nil { + return nil, fmt.Errorf("open Windows machine identity: %w", err) + } + defer cryptography.Close() + if err := cryptography.SetStringValue("MachineGuid", machineID); err != nil { + return nil, fmt.Errorf("set Windows machine identity: %w", err) + } + + marker, _, err := registry.CreateKey(registry.LOCAL_MACHINE, `SOFTWARE\Kernel\Hypeman`, registry.SET_VALUE) + if err != nil { + return nil, fmt.Errorf("open Hypeman identity marker: %w", err) + } + defer marker.Close() + if err := marker.SetStringValue("InstanceID", req.InstanceId); err != nil { + return nil, fmt.Errorf("set Hypeman instance identity: %w", err) + } + return &pb.RebindIdentityResponse{MachineId: machineID}, nil +} + +func newWindowsMachineID() (string, error) { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return "", err + } + value[6] = (value[6] & 0x0f) | 0x40 + value[8] = (value[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", + value[0:4], value[4:6], value[6:8], value[8:10], value[10:16]), nil +} From 716aeb6b90325fc0934c299dd720527eb04a55bc Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:37:19 +0000 Subject: [PATCH 02/15] Document TPM identity for Windows memory forks --- docs/windows-snapshots.md | 8 +++++--- lib/instances/fork.go | 5 ++++- lib/instances/snapshot.go | 5 ++++- lib/instances/windows.go | 20 ++++++++++--------- ...ws_snapshot_fork_integration_linux_test.go | 11 ++++++++++ lib/instances/windows_test.go | 10 ++++++++-- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/docs/windows-snapshots.md b/docs/windows-snapshots.md index 09f5c74ce..f3d8476b5 100644 --- a/docs/windows-snapshots.md +++ b/docs/windows-snapshots.md @@ -8,7 +8,9 @@ Standby pauses QEMU, captures memory and device state, stops QEMU and swtpm, and ## Fork identity -A fork receives independent disk and NVRAM files. Hypeman removes the copied TPM state before the child starts, so swtpm initializes a new TPM rather than cloning the parent's identity. The Windows guest agent then writes a new `MachineGuid` and records the child instance ID before the child is returned. +A fork receives independent disk and NVRAM files. A stopped fork removes the copied TPM state before cold boot, so swtpm initializes a new endorsement key and TPM identity. A memory fork retains the parent's TPM identity because QEMU includes the TPM's permanent and volatile state in its migration stream. Workloads that depend on unique TPM attestation must use stopped forks. + +The Windows guest agent writes a new `MachineGuid` and records the child instance ID before the child is returned. Memory forks retain the source SID and hostname, and services that cached `MachineGuid` before standby may observe the previous value until the next cold boot. Fork admission requires the persona OCI label: @@ -23,7 +25,7 @@ Stopped forks cold-boot with a unique vsock CID and can run concurrently. A stan ## Integration gates - `TestWindowsStandbyRestoreIntegration` verifies identity-preserving standby/restore and the stopped snapshot restore/fork APIs. -- `TestWindowsStandbyForkIntegration` verifies a captured desktop memory fork, inherited guest state, fresh machine identity and TPM state, and independent NVRAM/disk files. -- `TestWindowsForkIntegration` verifies independent guest writes and cold-booted stopped forks. +- `TestWindowsStandbyForkIntegration` verifies a captured desktop memory fork, inherited guest and TPM endorsement-key state, a fresh `MachineGuid`, and independent NVRAM/disk files. +- `TestWindowsForkIntegration` verifies independent guest writes and a fresh TPM endorsement key for cold-booted stopped forks. The private Windows fixture and its license are not stored in this repository. diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 0710d4b77..bdbb0c296 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -323,7 +323,10 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin forkMeta.VsockCID = generateVsockCID(forkID) } - if err := m.prepareWindowsForkIdentity(&forkMeta); err != nil { + // QEMU memory snapshots contain the TPM's permanent and volatile state. + // Resetting the copied directory is therefore only meaningful for cold forks. + resetWindowsTPM := source.State != StateStandby + if err := m.prepareWindowsForkIdentity(&forkMeta, resetWindowsTPM); err != nil { return nil, false, err } diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 810a2ac0e..e00c04a5c 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -458,7 +458,10 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS } else { forkMeta.VsockCID = generateVsockCID(forkID) } - if err := m.prepareWindowsForkIdentity(&forkMeta); err != nil { + // QEMU memory snapshots contain the TPM's permanent and volatile state. + // Resetting the copied directory is therefore only meaningful for cold forks. + resetWindowsTPM := rec.Snapshot.Kind != SnapshotKindStandby + if err := m.prepareWindowsForkIdentity(&forkMeta, resetWindowsTPM); err != nil { return nil, err } if forkMeta.NetworkEnabled { diff --git a/lib/instances/windows.go b/lib/instances/windows.go index 964dbc923..01cf0d73c 100644 --- a/lib/instances/windows.go +++ b/lib/instances/windows.go @@ -81,21 +81,23 @@ func (m *manager) ensureWindowsVsockCIDAvailable(ctx context.Context, stored *St return nil } -func (m *manager) prepareWindowsForkIdentity(stored *StoredMetadata) error { +func (m *manager) prepareWindowsForkIdentity(stored *StoredMetadata, resetTPM bool) error { if stored == nil || !isWindowsPlatform(stored.Platform) { return nil } if err := validateWindowsForkPolicy(stored); err != nil { return err } - if err := os.RemoveAll(m.paths.InstanceTPMDir(stored.Id)); err != nil { - return fmt.Errorf("clear forked Windows TPM state: %w", err) - } - if err := os.MkdirAll(m.paths.InstanceTPMDir(stored.Id), 0700); err != nil { - return fmt.Errorf("create forked Windows TPM state: %w", err) - } - if err := os.Remove(m.paths.InstanceTPMSocket(stored.Id)); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("remove forked Windows TPM socket: %w", err) + if resetTPM { + if err := os.RemoveAll(m.paths.InstanceTPMDir(stored.Id)); err != nil { + return fmt.Errorf("clear forked Windows TPM state: %w", err) + } + if err := os.MkdirAll(m.paths.InstanceTPMDir(stored.Id), 0700); err != nil { + return fmt.Errorf("create forked Windows TPM state: %w", err) + } + if err := os.Remove(m.paths.InstanceTPMSocket(stored.Id)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove forked Windows TPM socket: %w", err) + } } stored.WindowsIdentityPending = true return nil diff --git a/lib/instances/windows_snapshot_fork_integration_linux_test.go b/lib/instances/windows_snapshot_fork_integration_linux_test.go index 7ac64740b..7d1733cc3 100644 --- a/lib/instances/windows_snapshot_fork_integration_linux_test.go +++ b/lib/instances/windows_snapshot_fork_integration_linux_test.go @@ -55,6 +55,7 @@ func TestWindowsStandbyForkIntegration(t *testing.T) { ctx := context.Background() source := createWindowsSnapshotInstance(t, ctx, manager, image, "windows-standby-fork-source") sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) + sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) _, err := manager.StandbyInstance(ctx, source.Id, StandbyInstanceRequest{}) @@ -67,6 +68,7 @@ func TestWindowsStandbyForkIntegration(t *testing.T) { t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) assert.Equal(t, source.VsockCID, forked.VsockCID, "memory-restored Windows forks retain the captured VioSock CID until cold boot") assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id)) + assert.Equal(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "memory forks inherit TPM state from the QEMU migration stream") assert.Equal(t, "inherited", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\standby.txt`)) assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) @@ -77,6 +79,7 @@ func TestWindowsForkIntegration(t *testing.T) { ctx := context.Background() source := createWindowsSnapshotInstance(t, ctx, manager, image, "windows-fork-source") sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) + sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) stopped, err := manager.StopInstance(ctx, source.Id) @@ -89,6 +92,7 @@ func TestWindowsForkIntegration(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id), "fork must receive a new Windows machine identity") + assert.NotEqual(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "cold forks must initialize a fresh TPM") assert.NotEqual(t, source.VsockCID, forked.VsockCID, "forks need unique vsock CIDs") assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) @@ -166,6 +170,13 @@ func windowsMachineID(t *testing.T, ctx context.Context, manager *manager, insta return windowsPowerShell(t, ctx, manager, instanceID, `(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid`) } +func windowsTPMEK(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { + t.Helper() + hash := windowsPowerShell(t, ctx, manager, instanceID, `(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash`) + require.NotEmpty(t, hash, "TPM endorsement key hash") + return hash +} + func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, instanceID, command string) string { t.Helper() dialer, err := manager.GetVsockDialer(ctx, instanceID) diff --git a/lib/instances/windows_test.go b/lib/instances/windows_test.go index e2814e586..bcc5ef0f6 100644 --- a/lib/instances/windows_test.go +++ b/lib/instances/windows_test.go @@ -62,14 +62,20 @@ func TestPrepareWindowsForkIdentity(t *testing.T) { require.NoError(t, os.MkdirAll(p.InstanceTPMDir(stored.Id), 0700)) require.NoError(t, os.WriteFile(filepath.Join(p.InstanceTPMDir(stored.Id), "state"), []byte("source identity"), 0600)) - require.NoError(t, m.prepareWindowsForkIdentity(stored)) + require.NoError(t, m.prepareWindowsForkIdentity(stored, true)) assert.True(t, stored.WindowsIdentityPending) entries, err := os.ReadDir(p.InstanceTPMDir(stored.Id)) require.NoError(t, err) assert.Empty(t, entries) + require.NoError(t, os.WriteFile(filepath.Join(p.InstanceTPMDir(stored.Id), "state"), []byte("memory identity"), 0600)) + require.NoError(t, m.prepareWindowsForkIdentity(stored, false)) + state, err := os.ReadFile(filepath.Join(p.InstanceTPMDir(stored.Id), "state")) + require.NoError(t, err) + assert.Equal(t, "memory identity", string(state)) + stored.WindowsBitLockerPolicy = "" - assert.ErrorIs(t, m.prepareWindowsForkIdentity(stored), ErrNotSupported) + assert.ErrorIs(t, m.prepareWindowsForkIdentity(stored, true), ErrNotSupported) } func TestBuildWindowsHypervisorConfig(t *testing.T) { From cb7853953aff1d7cf40d835f8380350367148672 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:10:02 +0000 Subject: [PATCH 03/15] Isolate the Windows snapshots CI gates --- .github/workflows/test.yml | 24 +++++++++++++++++++ ...ws_snapshot_fork_integration_linux_test.go | 3 +++ 2 files changed, 27 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6484eab63..3d497ee4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -159,6 +159,30 @@ jobs: done exit 1 + - name: Test Windows snapshots and forks + run: | + TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" + for test_name in \ + TestWindowsStandbyRestoreIntegration \ + TestWindowsStandbyForkIntegration \ + TestWindowsForkIntegration; do + passed=false + for attempt in 1 2 3; do + if sudo env \ + "PATH=$TEST_PATH" \ + "CI=true" \ + "HYPEMAN_RUN_WINDOWS_SNAPSHOT_INTEGRATION=1" \ + "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ + "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ + go test -count=1 -run "^${test_name}$" -timeout 2m ./lib/instances; then + passed=true + break + fi + test "$attempt" = 3 || sleep 5 + done + test "$passed" = true || exit 1 + done + # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. - name: Login to Docker Hub diff --git a/lib/instances/windows_snapshot_fork_integration_linux_test.go b/lib/instances/windows_snapshot_fork_integration_linux_test.go index 7d1733cc3..82812d72c 100644 --- a/lib/instances/windows_snapshot_fork_integration_linux_test.go +++ b/lib/instances/windows_snapshot_fork_integration_linux_test.go @@ -109,6 +109,9 @@ func TestWindowsForkIntegration(t *testing.T) { func setupWindowsSnapshotIntegration(t *testing.T) (*manager, *paths.Paths, *images.Image) { t.Helper() + if os.Getenv("HYPEMAN_RUN_WINDOWS_SNAPSHOT_INTEGRATION") != "1" { + t.Skip("run by the dedicated Windows snapshots CI gate") + } fixture := os.Getenv("HYPEMAN_WINDOWS_TEST_AGENT_PERSONA") if fixture == "" { fixture = "/ci/windows/persona-agent.qcow2" From 9fe5fdc66016f0fa8cf76bf7625276cb40074301 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:24:34 +0000 Subject: [PATCH 04/15] Consolidate Windows snapshot coverage --- .github/workflows/test.yml | 33 ++- docs/windows-images.md | 2 +- docs/windows-snapshots.md | 12 +- lib/instances/README.md | 8 + lib/instances/windows.go | 2 +- ...ndows_lifecycle_integration_linux_test.go} | 148 +++++++++++-- ...ws_snapshot_fork_integration_linux_test.go | 207 ------------------ 7 files changed, 156 insertions(+), 256 deletions(-) rename lib/instances/{windows_networking_integration_linux_test.go => windows_lifecycle_integration_linux_test.go} (50%) delete mode 100644 lib/instances/windows_snapshot_fork_integration_linux_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d497ee4a..c9c80088d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -159,29 +159,22 @@ jobs: done exit 1 - - name: Test Windows snapshots and forks + - name: Test Windows stopped forks run: | TEST_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH" - for test_name in \ - TestWindowsStandbyRestoreIntegration \ - TestWindowsStandbyForkIntegration \ - TestWindowsForkIntegration; do - passed=false - for attempt in 1 2 3; do - if sudo env \ - "PATH=$TEST_PATH" \ - "CI=true" \ - "HYPEMAN_RUN_WINDOWS_SNAPSHOT_INTEGRATION=1" \ - "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ - "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ - go test -count=1 -run "^${test_name}$" -timeout 2m ./lib/instances; then - passed=true - break - fi - test "$attempt" = 3 || sleep 5 - done - test "$passed" = true || exit 1 + for attempt in 1 2 3; do + if sudo env \ + "PATH=$TEST_PATH" \ + "CI=true" \ + "HYPEMAN_RUN_WINDOWS_LIFECYCLE_INTEGRATION=1" \ + "HYPEMAN_WINDOWS_OVMF_CODE=$HYPEMAN_WINDOWS_OVMF_CODE" \ + "HYPEMAN_WINDOWS_OVMF_VARS=$HYPEMAN_WINDOWS_OVMF_VARS" \ + go test -count=1 -run '^TestWindowsStoppedForkIntegration$' -timeout 2m ./lib/instances; then + exit 0 + fi + test "$attempt" = 3 || sleep 5 done + exit 1 # Slash-command runs are maintainer-approved and need authenticated pulls # for images that are not covered by the prewarm cache. diff --git a/docs/windows-images.md b/docs/windows-images.md index 885f97d7b..607ce255a 100644 --- a/docs/windows-images.md +++ b/docs/windows-images.md @@ -13,7 +13,7 @@ A machine image uses these OCI config labels: | `io.hypeman.machine-image.base` | omitted | digest-pinned base reference | | `io.hypeman.machine-image.tpm` | `2.0` | `2.0` | | `io.hypeman.machine-image.secure-boot` | `required` | `required` | -| `io.hypeman.machine-image.bitlocker` | omitted | `disabled` for forkable personas; `reseal-required` otherwise | +| `io.hypeman.machine-image.bitlocker` | omitted | `disabled` for forkable images; `reseal-required` otherwise | The base must be pulled before its dependent Windows images. A base cannot be deleted while any cached image references its digest. Instance references are not tracked by the image cache, matching existing Linux behavior: do not delete a base while a dependent Windows instance exists. diff --git a/docs/windows-snapshots.md b/docs/windows-snapshots.md index f3d8476b5..5acd6dc7e 100644 --- a/docs/windows-snapshots.md +++ b/docs/windows-snapshots.md @@ -12,20 +12,12 @@ A fork receives independent disk and NVRAM files. A stopped fork removes the cop The Windows guest agent writes a new `MachineGuid` and records the child instance ID before the child is returned. Memory forks retain the source SID and hostname, and services that cached `MachineGuid` before standby may observe the previous value until the next cold boot. -Fork admission requires the persona OCI label: +Fork admission requires the image OCI label: ```text io.hypeman.machine-image.bitlocker=disabled ``` -Personas marked `reseal-required`, unlabeled personas, and unknown policies can still use same-instance snapshots, but cannot be forked. Hypeman does not expose a child whose encrypted disk was cloned without resealing it to the child's TPM. +Images marked `reseal-required`, unlabeled images, and unknown policies can still use same-instance snapshots, but cannot be forked. Hypeman does not expose a child whose encrypted disk was cloned without resealing it to the child's TPM. Stopped forks cold-boot with a unique vsock CID and can run concurrently. A standby snapshot contains the Windows VioSock driver's current CID in guest memory, so a memory-restored child initially retains that CID. The source and child must not be restored concurrently until the child has been stopped and cold-started; Hypeman reports a state error instead of allowing QEMU to fail with a CID collision. Creating a running fork directly from a running Windows source therefore requires `target_state=Stopped`. - -## Integration gates - -- `TestWindowsStandbyRestoreIntegration` verifies identity-preserving standby/restore and the stopped snapshot restore/fork APIs. -- `TestWindowsStandbyForkIntegration` verifies a captured desktop memory fork, inherited guest and TPM endorsement-key state, a fresh `MachineGuid`, and independent NVRAM/disk files. -- `TestWindowsForkIntegration` verifies independent guest writes and a fresh TPM endorsement key for cold-booted stopped forks. - -The private Windows fixture and its license are not stored in this repository. diff --git a/lib/instances/README.md b/lib/instances/README.md index 36278a005..851bd41b9 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -36,6 +36,14 @@ Windows uses the same host-side TAP allocation as Linux. Once the guest agent is Create treats network configuration as part of readiness and tears down a VM if it fails. Start reapplies the current allocation because a stopped instance may receive a different address or MAC before its next boot. +### Windows snapshots and forks + +A Windows machine consists of its writable qcow2 disk, Secure Boot NVRAM, software TPM state, saved QEMU configuration, and—while in standby—memory and device state. Same-instance restore keeps these components together so Windows identity and TPM state remain stable. + +Forking separates identity according to the boot path. A stopped fork receives independent disk and NVRAM files, clears copied TPM state, cold-boots with a new VioSock CID, and asks the guest agent to assign a new `MachineGuid`. A memory fork must retain the captured TPM and VioSock state from QEMU's migration stream; it receives independent files and a new `MachineGuid`, but inherits the TPM endorsement key until its next cold boot. The source and memory-restored child cannot run concurrently with the same captured CID. + +Fork admission requires the image to declare `io.hypeman.machine-image.bitlocker=disabled`. Other policies remain valid for same-instance snapshots but are rejected for forks because Hypeman does not reseal BitLocker keys to a child TPM. + ### Why Config Disk? (configdisk.go) **What:** Read-only erofs disk with instance configuration diff --git a/lib/instances/windows.go b/lib/instances/windows.go index 01cf0d73c..1f6c54a89 100644 --- a/lib/instances/windows.go +++ b/lib/instances/windows.go @@ -57,7 +57,7 @@ func validateWindowsCreate(req CreateInstanceRequest, image *images.Image, caps func validateWindowsForkPolicy(stored *StoredMetadata) error { if stored != nil && isWindowsPlatform(stored.Platform) && stored.WindowsBitLockerPolicy != "disabled" { - return fmt.Errorf("%w: Windows forks require a persona declared with %s=disabled", ErrNotSupported, images.MachineImageBitLockerLabel) + return fmt.Errorf("%w: Windows forks require an image declared with %s=disabled", ErrNotSupported, images.MachineImageBitLockerLabel) } return nil } diff --git a/lib/instances/windows_networking_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go similarity index 50% rename from lib/instances/windows_networking_integration_linux_test.go rename to lib/instances/windows_lifecycle_integration_linux_test.go index b5669ee0d..f562e3b28 100644 --- a/lib/instances/windows_networking_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -24,8 +24,79 @@ import ( ) func TestWindowsLifecycleIntegration(t *testing.T) { + manager, p, image := setupWindowsLifecycleIntegration(t) + ctx := context.Background() + source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-lifecycle-source", true) + + assertWindowsNetworkReady(t, ctx, manager, source.Id, source.IP) + assertWindowsGuestControl(t, ctx, manager, source.Id) + sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) + sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) + windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) + + standby, err := manager.StandbyInstance(ctx, source.Id, StandbyInstanceRequest{}) + require.NoError(t, err) + require.Equal(t, StateStandby, standby.State) + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "windows-standby-child", + TargetState: StateRunning, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) + assert.Equal(t, source.VsockCID, forked.VsockCID, "memory-restored Windows forks retain the captured VioSock CID until cold boot") + assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id)) + assert.Equal(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "memory forks inherit TPM state from the QEMU migration stream") + assert.Equal(t, "inherited", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\standby.txt`)) + assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) + assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) + + stoppedFork, err := manager.StopInstance(ctx, forked.Id) + require.NoError(t, err) + require.Equal(t, StateStopped, stoppedFork.State) + restored, err := manager.RestoreInstance(ctx, source.Id) + require.NoError(t, err) + waitForWindowsRunning(t, ctx, manager, restored.Id) + assert.Equal(t, sourceMachineID, windowsMachineID(t, ctx, manager, restored.Id), "same-instance restore must preserve Windows identity") + assertWindowsNetworkReady(t, ctx, manager, restored.Id, source.IP) +} + +func TestWindowsStoppedForkIntegration(t *testing.T) { + manager, p, image := setupWindowsLifecycleIntegration(t) + ctx := context.Background() + source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-stopped-fork-source", false) + sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) + sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) + windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) + + stopped, err := manager.StopInstance(ctx, source.Id) + require.NoError(t, err) + require.Equal(t, StateStopped, stopped.State) + forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ + Name: "windows-stopped-child", + TargetState: StateRunning, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) + assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id), "fork must receive a new Windows machine identity") + assert.NotEqual(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "cold forks must initialize a fresh TPM") + assert.NotEqual(t, source.VsockCID, forked.VsockCID, "forks need unique vsock CIDs") + assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) + assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) + require.DirExists(t, p.InstanceTPMDir(forked.Id)) + + sourceDiskBefore, err := os.Stat(p.InstanceWindowsDisk(source.Id)) + require.NoError(t, err) + windowsPowerShell(t, ctx, manager, forked.Id, `Set-Content C:\ProgramData\Hypeman\identity.txt child`) + assert.Equal(t, "child", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\identity.txt`)) + sourceDiskAfter, err := os.Stat(p.InstanceWindowsDisk(source.Id)) + require.NoError(t, err) + assert.Equal(t, sourceDiskBefore.ModTime(), sourceDiskAfter.ModTime(), "child writes must not modify the stopped source disk") +} + +func setupWindowsLifecycleIntegration(t *testing.T) (*manager, *paths.Paths, *images.Image) { + t.Helper() if os.Getenv("HYPEMAN_RUN_WINDOWS_LIFECYCLE_INTEGRATION") != "1" { - t.Skip("run by the dedicated Windows networking CI gate") + t.Skip("run by the dedicated Windows lifecycle CI gates") } fixture := os.Getenv("HYPEMAN_WINDOWS_TEST_AGENT_IMAGE") if fixture == "" { @@ -33,17 +104,17 @@ func TestWindowsLifecycleIntegration(t *testing.T) { } if _, err := os.Stat(fixture); err != nil { if os.Getenv("CI") == "true" { - t.Fatalf("required Windows networking fixture is missing: %s", fixture) + t.Fatalf("required Windows lifecycle fixture is missing: %s", fixture) } - t.Skipf("Windows networking fixture is unavailable: %s", fixture) + t.Skipf("Windows lifecycle fixture is unavailable: %s", fixture) } acquireHeavyIO(t) manager, dataDir := setupTestManagerForQEMU(t) p := paths.New(dataDir) - const digestHex = "abababababababababababababababababababababababababababababababab" + const digestHex = "acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac" image := &images.Image{ - Name: "registry.example/windows/image:networking-integration", + Name: "registry.example/windows/image:lifecycle-integration", Digest: "sha256:" + digestHex, Platform: "windows/amd64", Status: images.StatusReady, @@ -52,6 +123,7 @@ func TestWindowsLifecycleIntegration(t *testing.T) { Base: "registry.example/windows/base@sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", TPM: "2.0", SecureBoot: "required", + BitLocker: "disabled", VirtualSize: 80 << 30, }, } @@ -60,28 +132,32 @@ func TestWindowsLifecycleIntegration(t *testing.T) { require.NoError(t, err) require.NoError(t, forkvm.CopyRegularFile(fixture, imagePath)) require.NoError(t, os.Chmod(imagePath, 0444)) + return manager, p, image +} - ctx := context.Background() +func createWindowsLifecycleInstance(t *testing.T, ctx context.Context, manager *manager, image *images.Image, name string, networkEnabled bool) *Instance { + t.Helper() instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ - Name: "windows-networking-integration", + Name: name, Image: image.Name, Platform: "windows/amd64", - Size: 8 << 30, + Size: 4 << 30, Vcpus: 4, - NetworkEnabled: true, + NetworkEnabled: networkEnabled, Hypervisor: hypervisor.TypeQEMU, }) require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, instance.Id) }) - require.NotEmpty(t, instance.IP) - require.NotEmpty(t, instance.MAC) + waitForWindowsRunning(t, ctx, manager, instance.Id) + return instance +} + +func waitForWindowsRunning(t *testing.T, ctx context.Context, manager *manager, instanceID string) { + t.Helper() require.Eventually(t, func() bool { - current, err := manager.GetInstance(ctx, instance.Id) + current, err := manager.GetInstance(ctx, instanceID) return err == nil && current.State == StateRunning - }, 4*time.Minute, time.Second) - - assertWindowsNetworkReady(t, ctx, manager, instance.Id, instance.IP) - assertWindowsGuestControl(t, ctx, manager, instance.Id) + }, 75*time.Second, 500*time.Millisecond) } func assertWindowsGuestControl(t *testing.T, ctx context.Context, manager *manager, instanceID string) { @@ -100,7 +176,7 @@ func assertWindowsGuestControl(t *testing.T, ctx context.Context, manager *manag require.NoError(t, err, stderr.String()) assert.Less(t, time.Since(jobStart), 10*time.Second) - time.Sleep(5 * time.Second) + time.Sleep(time.Second) stdout.Reset() stderr.Reset() exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ @@ -207,3 +283,41 @@ func assertWindowsNetworkReady(t *testing.T, ctx context.Context, manager *manag ping := exec.Command("ping", "-c", "3", "-W", "2", expectedIP) require.NoError(t, ping.Run(), "allocated Windows IP did not answer ICMP") } + +func windowsMachineID(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { + t.Helper() + return windowsPowerShell(t, ctx, manager, instanceID, `(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid`) +} + +func windowsTPMEK(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { + t.Helper() + hash := windowsPowerShell(t, ctx, manager, instanceID, `(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash`) + require.NotEmpty(t, hash, "TPM endorsement key hash") + return hash +} + +func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, instanceID, command string) string { + t.Helper() + dialer, err := manager.GetVsockDialer(ctx, instanceID) + require.NoError(t, err) + var stdout, stderr bytes.Buffer + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ + Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", command}, + Stdout: &stdout, + Stderr: &stderr, + WaitForAgent: 30 * time.Second, + Timeout: 30, + }) + require.NoError(t, err, stderr.String()) + require.Equal(t, 0, exit.Code, stderr.String()) + return string(bytes.TrimSpace(stdout.Bytes())) +} + +func assertIndependentFile(t *testing.T, source, fork string) { + t.Helper() + sourceInfo, err := os.Stat(source) + require.NoError(t, err) + forkInfo, err := os.Stat(fork) + require.NoError(t, err) + assert.False(t, os.SameFile(sourceInfo, forkInfo), "%s and %s must not share an inode", source, fork) +} diff --git a/lib/instances/windows_snapshot_fork_integration_linux_test.go b/lib/instances/windows_snapshot_fork_integration_linux_test.go deleted file mode 100644 index 82812d72c..000000000 --- a/lib/instances/windows_snapshot_fork_integration_linux_test.go +++ /dev/null @@ -1,207 +0,0 @@ -//go:build linux && amd64 - -package instances - -import ( - "bytes" - "context" - "os" - "testing" - "time" - - "github.com/kernel/hypeman/lib/forkvm" - "github.com/kernel/hypeman/lib/guest" - "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/images" - "github.com/kernel/hypeman/lib/paths" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWindowsStandbyRestoreIntegration(t *testing.T) { - manager, _, image := setupWindowsSnapshotIntegration(t) - ctx := context.Background() - source := createWindowsSnapshotInstance(t, ctx, manager, image, "windows-restore-source") - - sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) - standby, err := manager.StandbyInstance(ctx, source.Id, StandbyInstanceRequest{}) - require.NoError(t, err) - require.Equal(t, StateStandby, standby.State) - restored, err := manager.RestoreInstance(ctx, source.Id) - require.NoError(t, err) - assert.Equal(t, sourceMachineID, windowsMachineID(t, ctx, manager, restored.Id), "same-instance restore must preserve Windows identity") - - _, err = manager.StopInstance(ctx, source.Id) - require.NoError(t, err) - snapshot, err := manager.CreateSnapshot(ctx, source.Id, CreateSnapshotRequest{ - Kind: SnapshotKindStopped, - Name: "windows-stopped-snapshot", - }) - require.NoError(t, err) - t.Cleanup(func() { _ = manager.DeleteSnapshot(context.Background(), snapshot.Id) }) - _, err = manager.RestoreSnapshot(ctx, source.Id, snapshot.Id, RestoreSnapshotRequest{TargetState: StateStopped}) - require.NoError(t, err) - forked, err := manager.ForkSnapshot(ctx, snapshot.Id, ForkSnapshotRequest{ - Name: "windows-stopped-snapshot-child", - TargetState: StateStopped, - }) - require.NoError(t, err) - t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) - assert.True(t, forked.WindowsIdentityPending) -} - -func TestWindowsStandbyForkIntegration(t *testing.T) { - manager, p, image := setupWindowsSnapshotIntegration(t) - ctx := context.Background() - source := createWindowsSnapshotInstance(t, ctx, manager, image, "windows-standby-fork-source") - sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) - sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) - windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) - - _, err := manager.StandbyInstance(ctx, source.Id, StandbyInstanceRequest{}) - require.NoError(t, err) - forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ - Name: "windows-standby-child", - TargetState: StateRunning, - }) - require.NoError(t, err) - t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) - assert.Equal(t, source.VsockCID, forked.VsockCID, "memory-restored Windows forks retain the captured VioSock CID until cold boot") - assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id)) - assert.Equal(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "memory forks inherit TPM state from the QEMU migration stream") - assert.Equal(t, "inherited", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\standby.txt`)) - assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) - assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) -} - -func TestWindowsForkIntegration(t *testing.T) { - manager, p, image := setupWindowsSnapshotIntegration(t) - ctx := context.Background() - source := createWindowsSnapshotInstance(t, ctx, manager, image, "windows-fork-source") - sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) - sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) - windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) - - stopped, err := manager.StopInstance(ctx, source.Id) - require.NoError(t, err) - require.Equal(t, StateStopped, stopped.State) - forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ - Name: "windows-snapshot-child", - TargetState: StateRunning, - }) - require.NoError(t, err) - t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) - assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id), "fork must receive a new Windows machine identity") - assert.NotEqual(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "cold forks must initialize a fresh TPM") - assert.NotEqual(t, source.VsockCID, forked.VsockCID, "forks need unique vsock CIDs") - assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) - assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) - require.DirExists(t, p.InstanceTPMDir(forked.Id)) - - sourceDiskBefore, err := os.Stat(p.InstanceWindowsDisk(source.Id)) - require.NoError(t, err) - windowsPowerShell(t, ctx, manager, forked.Id, `Set-Content C:\ProgramData\Hypeman\identity.txt child`) - assert.Equal(t, "child", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\identity.txt`)) - sourceDiskAfter, err := os.Stat(p.InstanceWindowsDisk(source.Id)) - require.NoError(t, err) - assert.Equal(t, sourceDiskBefore.ModTime(), sourceDiskAfter.ModTime(), "child writes must not modify the stopped source disk") -} - -func setupWindowsSnapshotIntegration(t *testing.T) (*manager, *paths.Paths, *images.Image) { - t.Helper() - if os.Getenv("HYPEMAN_RUN_WINDOWS_SNAPSHOT_INTEGRATION") != "1" { - t.Skip("run by the dedicated Windows snapshots CI gate") - } - fixture := os.Getenv("HYPEMAN_WINDOWS_TEST_AGENT_PERSONA") - if fixture == "" { - fixture = "/ci/windows/persona-agent.qcow2" - } - if _, err := os.Stat(fixture); err != nil { - if os.Getenv("CI") == "true" { - t.Fatalf("required Windows snapshot fixture is missing: %s", fixture) - } - t.Skipf("Windows snapshot fixture is unavailable: %s", fixture) - } - acquireHeavyIO(t) - - manager, dataDir := setupTestManagerForQEMU(t) - p := paths.New(dataDir) - const digestHex = "acacacacacacacacacacacacacacacacacacacacacacacacacacacacacacacac" - image := &images.Image{ - Name: "registry.example/windows/persona:snapshot-integration", - Digest: "sha256:" + digestHex, - Platform: "windows/amd64", - Status: images.StatusReady, - Machine: &images.MachineImage{ - Kind: images.MachineImageWindowsPersona, - Base: "registry.example/windows/base@sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", - TPM: "2.0", - SecureBoot: "required", - BitLocker: "disabled", - VirtualSize: 80 << 30, - }, - } - manager.imageManager = windowsFixtureImageManager{image: image} - personaPath, err := images.GetMachineDiskPath(p, image.Name, image.Digest, image.Machine) - require.NoError(t, err) - require.NoError(t, forkvm.CopyRegularFile(fixture, personaPath)) - require.NoError(t, os.Chmod(personaPath, 0444)) - return manager, p, image -} - -func createWindowsSnapshotInstance(t *testing.T, ctx context.Context, manager *manager, image *images.Image, name string) *Instance { - t.Helper() - instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ - Name: name, - Image: image.Name, - Platform: "windows/amd64", - Size: 4 << 30, - Vcpus: 4, - Hypervisor: hypervisor.TypeQEMU, - }) - require.NoError(t, err) - t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, instance.Id) }) - require.Eventually(t, func() bool { - current, err := manager.GetInstance(ctx, instance.Id) - return err == nil && current.State == StateRunning - }, 75*time.Second, 500*time.Millisecond) - return instance -} - -func windowsMachineID(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { - t.Helper() - return windowsPowerShell(t, ctx, manager, instanceID, `(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid`) -} - -func windowsTPMEK(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { - t.Helper() - hash := windowsPowerShell(t, ctx, manager, instanceID, `(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash`) - require.NotEmpty(t, hash, "TPM endorsement key hash") - return hash -} - -func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, instanceID, command string) string { - t.Helper() - dialer, err := manager.GetVsockDialer(ctx, instanceID) - require.NoError(t, err) - var stdout, stderr bytes.Buffer - exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", command}, - Stdout: &stdout, - Stderr: &stderr, - WaitForAgent: 30 * time.Second, - Timeout: 30, - }) - require.NoError(t, err, stderr.String()) - require.Equal(t, 0, exit.Code, stderr.String()) - return string(bytes.TrimSpace(stdout.Bytes())) -} - -func assertIndependentFile(t *testing.T, source, fork string) { - t.Helper() - sourceInfo, err := os.Stat(source) - require.NoError(t, err) - forkInfo, err := os.Stat(fork) - require.NoError(t, err) - assert.False(t, os.SameFile(sourceInfo, forkInfo), "%s and %s must not share an inode", source, fork) -} From 7df38579b3eb4e6c21a20e00733641305e827416 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:03:01 +0000 Subject: [PATCH 05/15] Protect Windows fork identity invariants --- lib/instances/README.md | 2 +- lib/instances/fork.go | 7 +++++-- lib/instances/manager.go | 1 + lib/instances/restore.go | 4 ++++ lib/instances/snapshot.go | 4 ++-- lib/instances/windows.go | 4 ++++ lib/instances/windows_test.go | 18 ++++++++++++++++++ 7 files changed, 35 insertions(+), 5 deletions(-) diff --git a/lib/instances/README.md b/lib/instances/README.md index 851bd41b9..d6a4e0613 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -40,7 +40,7 @@ Create treats network configuration as part of readiness and tears down a VM if A Windows machine consists of its writable qcow2 disk, Secure Boot NVRAM, software TPM state, saved QEMU configuration, and—while in standby—memory and device state. Same-instance restore keeps these components together so Windows identity and TPM state remain stable. -Forking separates identity according to the boot path. A stopped fork receives independent disk and NVRAM files, clears copied TPM state, cold-boots with a new VioSock CID, and asks the guest agent to assign a new `MachineGuid`. A memory fork must retain the captured TPM and VioSock state from QEMU's migration stream; it receives independent files and a new `MachineGuid`, but inherits the TPM endorsement key until its next cold boot. The source and memory-restored child cannot run concurrently with the same captured CID. +Forking separates identity according to the boot path. A stopped fork receives independent disk and NVRAM files, clears copied TPM state, cold-boots with a new VioSock CID, and asks the guest agent to assign a new `MachineGuid`. A memory fork must retain the captured TPM and VioSock state from QEMU's migration stream; it receives independent files and a new `MachineGuid`, but inherits the TPM endorsement key until its next cold boot. The source and memory-restored child cannot run concurrently with the same captured CID, so Windows memory restores serialize CID admission through VMM activation. Fork admission requires the image to declare `io.hypeman.machine-image.bitlocker=disabled`. Other policies remain valid for same-instance snapshots but are rejected for forks because Hypeman does not reseal BitLocker keys to a child TPM. diff --git a/lib/instances/fork.go b/lib/instances/fork.go index bdbb0c296..7e8307163 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -50,6 +50,9 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR if isWindowsPlatform(source.Platform) && source.State == StateRunning && targetState != StateStopped { return nil, "", false, fmt.Errorf("%w: Windows forks from a running source require target_state=%s", ErrNotSupported, StateStopped) } + if err := validateWindowsForkPolicy(&meta.StoredMetadata); err != nil { + return nil, "", false, err + } switch source.State { case StateRunning: @@ -324,8 +327,8 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin } // QEMU memory snapshots contain the TPM's permanent and volatile state. - // Resetting the copied directory is therefore only meaningful for cold forks. - resetWindowsTPM := source.State != StateStandby + // A stopped target discards that memory and must cold-boot with fresh TPM state. + resetWindowsTPM := windowsForkNeedsFreshTPM(source.State == StateStandby, targetState) if err := m.prepareWindowsForkIdentity(&forkMeta, resetWindowsTPM); err != nil { return nil, false, err } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 0568ff4ba..4cae31b9c 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -171,6 +171,7 @@ type manager struct { resourceValidator ResourceValidator // Optional validator for aggregate resource limits instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks forkMetadataMu sync.Mutex + windowsRestoreMu sync.Mutex bootMarkerScans sync.Map // map[string]time.Time next allowed boot-marker rescan hypervisorStateCache sync.Map // map[string]hypervisorStateCacheEntry - last observed hypervisor state per instance hostTopology *HostTopology // Cached host CPU topology diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 14c25e203..bb48e8da1 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -45,6 +45,10 @@ func (m *manager) restoreInstance( inst := m.toInstance(ctx, meta) stored := &meta.StoredMetadata + if isWindowsPlatform(stored.Platform) { + m.windowsRestoreMu.Lock() + defer m.windowsRestoreMu.Unlock() + } ctx = enrichInstancesTrace(ctx, attribute.String("hypervisor", string(stored.HypervisorType))) log.DebugContext(ctx, "loaded instance", "instance_id", id, "state", inst.State, "has_snapshot", inst.HasSnapshot) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index e00c04a5c..7cc2e35ec 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -459,8 +459,8 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS forkMeta.VsockCID = generateVsockCID(forkID) } // QEMU memory snapshots contain the TPM's permanent and volatile state. - // Resetting the copied directory is therefore only meaningful for cold forks. - resetWindowsTPM := rec.Snapshot.Kind != SnapshotKindStandby + // A stopped target discards that memory and must cold-boot with fresh TPM state. + resetWindowsTPM := windowsForkNeedsFreshTPM(rec.Snapshot.Kind == SnapshotKindStandby, targetState) if err := m.prepareWindowsForkIdentity(&forkMeta, resetWindowsTPM); err != nil { return nil, err } diff --git a/lib/instances/windows.go b/lib/instances/windows.go index 1f6c54a89..1f49aea61 100644 --- a/lib/instances/windows.go +++ b/lib/instances/windows.go @@ -81,6 +81,10 @@ func (m *manager) ensureWindowsVsockCIDAvailable(ctx context.Context, stored *St return nil } +func windowsForkNeedsFreshTPM(hasMemorySnapshot bool, targetState State) bool { + return targetState == StateStopped || !hasMemorySnapshot +} + func (m *manager) prepareWindowsForkIdentity(stored *StoredMetadata, resetTPM bool) error { if stored == nil || !isWindowsPlatform(stored.Platform) { return nil diff --git a/lib/instances/windows_test.go b/lib/instances/windows_test.go index bcc5ef0f6..919fcd043 100644 --- a/lib/instances/windows_test.go +++ b/lib/instances/windows_test.go @@ -55,6 +55,24 @@ func TestValidateWindowsCreate(t *testing.T) { } } +func TestWindowsForkNeedsFreshTPM(t *testing.T) { + tests := []struct { + name string + hasMemorySnapshot bool + targetState State + want bool + }{ + {name: "memory fork", hasMemorySnapshot: true, targetState: StateRunning}, + {name: "standby fork stopped before restore", hasMemorySnapshot: true, targetState: StateStopped, want: true}, + {name: "cold fork", targetState: StateRunning, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, windowsForkNeedsFreshTPM(tt.hasMemorySnapshot, tt.targetState)) + }) + } +} + func TestPrepareWindowsForkIdentity(t *testing.T) { p := paths.New(t.TempDir()) m := &manager{paths: p} From 4aa7063505eb1d2bbdf57d1d1563b813a77f75a6 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:17:53 +0000 Subject: [PATCH 06/15] Keep Windows copy coverage at lifecycle end --- lib/instances/windows_lifecycle_integration_linux_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/instances/windows_lifecycle_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go index f562e3b28..b86990463 100644 --- a/lib/instances/windows_lifecycle_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -58,6 +58,7 @@ func TestWindowsLifecycleIntegration(t *testing.T) { waitForWindowsRunning(t, ctx, manager, restored.Id) assert.Equal(t, sourceMachineID, windowsMachineID(t, ctx, manager, restored.Id), "same-instance restore must preserve Windows identity") assertWindowsNetworkReady(t, ctx, manager, restored.Id, source.IP) + assertWindowsCopyRoundTrip(t, ctx, manager, restored.Id) } func TestWindowsStoppedForkIntegration(t *testing.T) { @@ -236,6 +237,12 @@ func assertWindowsGuestControl(t *testing.T, ctx context.Context, manager *manag }) assert.ErrorContains(t, err, "desktop ConPTY sessions are not supported") +} + +func assertWindowsCopyRoundTrip(t *testing.T, ctx context.Context, manager *manager, instanceID string) { + t.Helper() + dialer, err := manager.GetVsockDialer(ctx, instanceID) + require.NoError(t, err) source := filepath.Join(t.TempDir(), "roundtrip.txt") require.NoError(t, os.WriteFile(source, []byte("HYPEMAN_COPY_OK"), 0644)) require.NoError(t, guest.CopyToInstance(ctx, dialer, guest.CopyToInstanceOptions{ From 87db7de0169574daf6bc8ad73ce48169b165d95c Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:25:41 +0000 Subject: [PATCH 07/15] Use default Windows memory for lifecycle coverage --- lib/instances/windows_lifecycle_integration_linux_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/instances/windows_lifecycle_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go index b86990463..8be135303 100644 --- a/lib/instances/windows_lifecycle_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -26,7 +26,7 @@ import ( func TestWindowsLifecycleIntegration(t *testing.T) { manager, p, image := setupWindowsLifecycleIntegration(t) ctx := context.Background() - source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-lifecycle-source", true) + source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-lifecycle-source", true, 8<<30) assertWindowsNetworkReady(t, ctx, manager, source.Id, source.IP) assertWindowsGuestControl(t, ctx, manager, source.Id) @@ -64,7 +64,7 @@ func TestWindowsLifecycleIntegration(t *testing.T) { func TestWindowsStoppedForkIntegration(t *testing.T) { manager, p, image := setupWindowsLifecycleIntegration(t) ctx := context.Background() - source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-stopped-fork-source", false) + source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-stopped-fork-source", false, 4<<30) sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) @@ -136,13 +136,13 @@ func setupWindowsLifecycleIntegration(t *testing.T) (*manager, *paths.Paths, *im return manager, p, image } -func createWindowsLifecycleInstance(t *testing.T, ctx context.Context, manager *manager, image *images.Image, name string, networkEnabled bool) *Instance { +func createWindowsLifecycleInstance(t *testing.T, ctx context.Context, manager *manager, image *images.Image, name string, networkEnabled bool, size int64) *Instance { t.Helper() instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ Name: name, Image: image.Name, Platform: "windows/amd64", - Size: 4 << 30, + Size: size, Vcpus: 4, NetworkEnabled: networkEnabled, Hypervisor: hypervisor.TypeQEMU, From a17d4f84b6b14ac35f5fce1981517f8f40891d31 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:37:04 +0000 Subject: [PATCH 08/15] Run guest-control checks after snapshot lifecycle --- lib/instances/windows_lifecycle_integration_linux_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/instances/windows_lifecycle_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go index 8be135303..27813665f 100644 --- a/lib/instances/windows_lifecycle_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -29,7 +29,6 @@ func TestWindowsLifecycleIntegration(t *testing.T) { source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-lifecycle-source", true, 8<<30) assertWindowsNetworkReady(t, ctx, manager, source.Id, source.IP) - assertWindowsGuestControl(t, ctx, manager, source.Id) sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) @@ -58,6 +57,7 @@ func TestWindowsLifecycleIntegration(t *testing.T) { waitForWindowsRunning(t, ctx, manager, restored.Id) assert.Equal(t, sourceMachineID, windowsMachineID(t, ctx, manager, restored.Id), "same-instance restore must preserve Windows identity") assertWindowsNetworkReady(t, ctx, manager, restored.Id, source.IP) + assertWindowsGuestControl(t, ctx, manager, restored.Id) assertWindowsCopyRoundTrip(t, ctx, manager, restored.Id) } From 612016783ecf9c407c6bfff693b0bcf4742f7b2b Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:44:14 +0000 Subject: [PATCH 09/15] Keep Windows snapshot scenarios within gate budget --- lib/instances/windows_lifecycle_integration_linux_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/instances/windows_lifecycle_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go index 27813665f..0bd3f3fc3 100644 --- a/lib/instances/windows_lifecycle_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -26,7 +26,7 @@ import ( func TestWindowsLifecycleIntegration(t *testing.T) { manager, p, image := setupWindowsLifecycleIntegration(t) ctx := context.Background() - source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-lifecycle-source", true, 8<<30) + source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-lifecycle-source", true) assertWindowsNetworkReady(t, ctx, manager, source.Id, source.IP) sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) @@ -64,7 +64,7 @@ func TestWindowsLifecycleIntegration(t *testing.T) { func TestWindowsStoppedForkIntegration(t *testing.T) { manager, p, image := setupWindowsLifecycleIntegration(t) ctx := context.Background() - source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-stopped-fork-source", false, 4<<30) + source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-stopped-fork-source", false) sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) @@ -136,13 +136,13 @@ func setupWindowsLifecycleIntegration(t *testing.T) (*manager, *paths.Paths, *im return manager, p, image } -func createWindowsLifecycleInstance(t *testing.T, ctx context.Context, manager *manager, image *images.Image, name string, networkEnabled bool, size int64) *Instance { +func createWindowsLifecycleInstance(t *testing.T, ctx context.Context, manager *manager, image *images.Image, name string, networkEnabled bool) *Instance { t.Helper() instance, err := manager.CreateInstance(ctx, CreateInstanceRequest{ Name: name, Image: image.Name, Platform: "windows/amd64", - Size: size, + Size: 4 << 30, Vcpus: 4, NetworkEnabled: networkEnabled, Hypervisor: hypervisor.TypeQEMU, From 22363a2061d9d7f025a3c930c70796932492bb7e Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:55:56 +0000 Subject: [PATCH 10/15] Preserve Windows memory-fork network identity --- docs/windows-snapshots.md | 2 +- lib/instances/README.md | 2 +- lib/instances/fork.go | 6 +++--- lib/instances/snapshot.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/windows-snapshots.md b/docs/windows-snapshots.md index 5acd6dc7e..4b87d7bec 100644 --- a/docs/windows-snapshots.md +++ b/docs/windows-snapshots.md @@ -20,4 +20,4 @@ io.hypeman.machine-image.bitlocker=disabled Images marked `reseal-required`, unlabeled images, and unknown policies can still use same-instance snapshots, but cannot be forked. Hypeman does not expose a child whose encrypted disk was cloned without resealing it to the child's TPM. -Stopped forks cold-boot with a unique vsock CID and can run concurrently. A standby snapshot contains the Windows VioSock driver's current CID in guest memory, so a memory-restored child initially retains that CID. The source and child must not be restored concurrently until the child has been stopped and cold-started; Hypeman reports a state error instead of allowing QEMU to fail with a CID collision. Creating a running fork directly from a running Windows source therefore requires `target_state=Stopped`. +Stopped forks cold-boot with a unique vsock CID and can run concurrently. A standby snapshot contains the Windows VioSock CID and NIC identity in guest memory, so a memory-restored child initially retains both. The source and child must not be restored concurrently until the child has been stopped and cold-started; Hypeman reports a state error instead of allowing a host-device collision. Creating a running fork directly from a running Windows source therefore requires `target_state=Stopped`. diff --git a/lib/instances/README.md b/lib/instances/README.md index d6a4e0613..80f46c838 100644 --- a/lib/instances/README.md +++ b/lib/instances/README.md @@ -40,7 +40,7 @@ Create treats network configuration as part of readiness and tears down a VM if A Windows machine consists of its writable qcow2 disk, Secure Boot NVRAM, software TPM state, saved QEMU configuration, and—while in standby—memory and device state. Same-instance restore keeps these components together so Windows identity and TPM state remain stable. -Forking separates identity according to the boot path. A stopped fork receives independent disk and NVRAM files, clears copied TPM state, cold-boots with a new VioSock CID, and asks the guest agent to assign a new `MachineGuid`. A memory fork must retain the captured TPM and VioSock state from QEMU's migration stream; it receives independent files and a new `MachineGuid`, but inherits the TPM endorsement key until its next cold boot. The source and memory-restored child cannot run concurrently with the same captured CID, so Windows memory restores serialize CID admission through VMM activation. +Forking separates identity according to the boot path. A stopped fork receives independent disk and NVRAM files, clears copied TPM state, cold-boots with a new VioSock CID, and asks the guest agent to assign a new `MachineGuid`. A memory fork must retain the captured TPM, VioSock, and NIC state from QEMU's migration stream; it receives independent files and a new `MachineGuid`, but inherits the TPM endorsement key and network identity until its next cold boot. The source and memory-restored child cannot run concurrently with the captured host-device identities, so Windows memory restores serialize admission through VMM activation. Fork admission requires the image to declare `io.hypeman.machine-image.bitlocker=disabled`. Other policies remain valid for same-instance snapshots but are rejected for forks because Hypeman does not reseal BitLocker keys to a child TPM. diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 7e8307163..46e9d5927 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -333,9 +333,9 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin return nil, false, err } - if forkMeta.NetworkEnabled { - // Clear inherited network identity. For stopped instances this is regenerated on start, - // and for standby instances restore allocates if identity is empty. + if forkMeta.NetworkEnabled && !(isWindowsPlatform(forkMeta.Platform) && source.State == StateStandby) { + // Clear inherited network identity. Windows memory forks retain the NIC + // identity captured by QEMU and cannot run concurrently with their source. forkMeta.IP = "" forkMeta.MAC = "" } diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 7cc2e35ec..88a65ce1f 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -464,7 +464,7 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err := m.prepareWindowsForkIdentity(&forkMeta, resetWindowsTPM); err != nil { return nil, err } - if forkMeta.NetworkEnabled { + if forkMeta.NetworkEnabled && !(isWindowsPlatform(forkMeta.Platform) && rec.Snapshot.Kind == SnapshotKindStandby) { forkMeta.IP = "" forkMeta.MAC = "" } From 511faaed92afb55d3579ad84135c33ecb3ae699c Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:08:30 +0000 Subject: [PATCH 11/15] Focus Windows lifecycle gate on snapshot invariants --- ...indows_lifecycle_integration_linux_test.go | 130 ++++-------------- 1 file changed, 23 insertions(+), 107 deletions(-) diff --git a/lib/instances/windows_lifecycle_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go index 0bd3f3fc3..2837dffe4 100644 --- a/lib/instances/windows_lifecycle_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -9,7 +9,6 @@ import ( "net" "os" "os/exec" - "path/filepath" "strings" "testing" "time" @@ -28,10 +27,7 @@ func TestWindowsLifecycleIntegration(t *testing.T) { ctx := context.Background() source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-lifecycle-source", true) - assertWindowsNetworkReady(t, ctx, manager, source.Id, source.IP) - sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) - sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) - windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) + sourceMachineID, sourceTPMEK := windowsIdentity(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) standby, err := manager.StandbyInstance(ctx, source.Id, StandbyInstanceRequest{}) require.NoError(t, err) @@ -43,8 +39,9 @@ func TestWindowsLifecycleIntegration(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) assert.Equal(t, source.VsockCID, forked.VsockCID, "memory-restored Windows forks retain the captured VioSock CID until cold boot") - assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id)) - assert.Equal(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "memory forks inherit TPM state from the QEMU migration stream") + forkedMachineID, forkedTPMEK := windowsIdentity(t, ctx, manager, forked.Id, "") + assert.NotEqual(t, sourceMachineID, forkedMachineID) + assert.Equal(t, sourceTPMEK, forkedTPMEK, "memory forks inherit TPM state from the QEMU migration stream") assert.Equal(t, "inherited", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\standby.txt`)) assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) @@ -55,19 +52,18 @@ func TestWindowsLifecycleIntegration(t *testing.T) { restored, err := manager.RestoreInstance(ctx, source.Id) require.NoError(t, err) waitForWindowsRunning(t, ctx, manager, restored.Id) - assert.Equal(t, sourceMachineID, windowsMachineID(t, ctx, manager, restored.Id), "same-instance restore must preserve Windows identity") + restoredMachineID, restoredTPMEK := windowsIdentity(t, ctx, manager, restored.Id, "") + assert.Equal(t, sourceMachineID, restoredMachineID, "same-instance restore must preserve Windows identity") + assert.Equal(t, sourceTPMEK, restoredTPMEK, "same-instance restore must preserve TPM identity") assertWindowsNetworkReady(t, ctx, manager, restored.Id, source.IP) assertWindowsGuestControl(t, ctx, manager, restored.Id) - assertWindowsCopyRoundTrip(t, ctx, manager, restored.Id) } func TestWindowsStoppedForkIntegration(t *testing.T) { manager, p, image := setupWindowsLifecycleIntegration(t) ctx := context.Background() source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-stopped-fork-source", false) - sourceMachineID := windowsMachineID(t, ctx, manager, source.Id) - sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) - windowsPowerShell(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) + sourceMachineID, sourceTPMEK := windowsIdentity(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) stopped, err := manager.StopInstance(ctx, source.Id) require.NoError(t, err) @@ -78,8 +74,9 @@ func TestWindowsStoppedForkIntegration(t *testing.T) { }) require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) - assert.NotEqual(t, sourceMachineID, windowsMachineID(t, ctx, manager, forked.Id), "fork must receive a new Windows machine identity") - assert.NotEqual(t, sourceTPMEK, windowsTPMEK(t, ctx, manager, forked.Id), "cold forks must initialize a fresh TPM") + forkedMachineID, forkedTPMEK := windowsIdentity(t, ctx, manager, forked.Id, "") + assert.NotEqual(t, sourceMachineID, forkedMachineID, "fork must receive a new Windows machine identity") + assert.NotEqual(t, sourceTPMEK, forkedTPMEK, "cold forks must initialize a fresh TPM") assert.NotEqual(t, source.VsockCID, forked.VsockCID, "forks need unique vsock CIDs") assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) @@ -165,98 +162,16 @@ func assertWindowsGuestControl(t *testing.T, ctx context.Context, manager *manag t.Helper() dialer, err := manager.GetVsockDialer(ctx, instanceID) require.NoError(t, err) - var stdout, stderr bytes.Buffer - jobStart := time.Now() exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `Copy-Item "$env:SystemRoot\System32\ping.exe" "$env:TEMP\hypeman-job-child.exe" -Force; & "$env:TEMP\hypeman-job-child.exe" -n 60 127.0.0.1`}, - Stdout: &stdout, - Stderr: &stderr, - Timeout: 2, - }) - require.NoError(t, err, stderr.String()) - assert.Less(t, time.Since(jobStart), 10*time.Second) - - time.Sleep(time.Second) - stdout.Reset() - stderr.Reset() - exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", `if (Get-Process hypeman-job-child -ErrorAction SilentlyContinue) { exit 42 }`}, + Command: []string{"cmd.exe", "/d", "/c", "echo HYPEMAN_GUEST_CONTROL_OK"}, Stdout: &stdout, Stderr: &stderr, Timeout: 15, }) require.NoError(t, err, stderr.String()) - require.Equal(t, 0, exit.Code, "job object left a child process running") - - stdout.Reset() - stderr.Reset() - exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", "[Console]::Out.Write('HYPEMAN_SYSTEM_OK')"}, - Stdout: &stdout, - Stderr: &stderr, - Timeout: 30, - }) - require.NoError(t, err, stderr.String()) - require.Equal(t, 0, exit.Code) - assert.Equal(t, "HYPEMAN_SYSTEM_OK", stdout.String()) - - stdout.Reset() - stderr.Reset() - exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"cmd.exe", "/d", "/c", "ping -n 2 127.0.0.1 >nul & echo HYPEMAN_CONPTY_OK"}, - Stdout: &stdout, - Stderr: &stderr, - TTY: true, - Rows: 31, - Cols: 97, - Timeout: 30, - }) - require.NoError(t, err, stderr.String()) - require.Equal(t, 0, exit.Code) - assert.Contains(t, stdout.String(), "HYPEMAN_CONPTY_OK") - - stdout.Reset() - stderr.Reset() - exit, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"cmd.exe", "/d", "/c", "echo HYPEMAN_DESKTOP_OK"}, - Stdout: &stdout, - Stderr: &stderr, - Session: guest.ExecSession_EXEC_SESSION_DESKTOP, - Timeout: 30, - }) - require.NoError(t, err, stderr.String()) require.Equal(t, 0, exit.Code) - assert.Contains(t, stdout.String(), "HYPEMAN_DESKTOP_OK") - - _, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"cmd.exe"}, - TTY: true, - Session: guest.ExecSession_EXEC_SESSION_DESKTOP, - Timeout: 30, - }) - assert.ErrorContains(t, err, "desktop ConPTY sessions are not supported") - -} - -func assertWindowsCopyRoundTrip(t *testing.T, ctx context.Context, manager *manager, instanceID string) { - t.Helper() - dialer, err := manager.GetVsockDialer(ctx, instanceID) - require.NoError(t, err) - source := filepath.Join(t.TempDir(), "roundtrip.txt") - require.NoError(t, os.WriteFile(source, []byte("HYPEMAN_COPY_OK"), 0644)) - require.NoError(t, guest.CopyToInstance(ctx, dialer, guest.CopyToInstanceOptions{ - SrcPath: source, - DstPath: `C:\ProgramData\Hypeman\roundtrip.txt`, - })) - destination := t.TempDir() - require.NoError(t, guest.CopyFromInstance(ctx, dialer, guest.CopyFromInstanceOptions{ - SrcPath: `C:\ProgramData\Hypeman\roundtrip.txt`, - DstPath: destination, - })) - contents, err := os.ReadFile(filepath.Join(destination, "roundtrip.txt")) - require.NoError(t, err) - assert.Equal(t, "HYPEMAN_COPY_OK", string(contents)) + assert.Contains(t, stdout.String(), "HYPEMAN_GUEST_CONTROL_OK") } func assertWindowsNetworkReady(t *testing.T, ctx context.Context, manager *manager, instanceID, expectedIP string) { @@ -291,16 +206,17 @@ func assertWindowsNetworkReady(t *testing.T, ctx context.Context, manager *manag require.NoError(t, ping.Run(), "allocated Windows IP did not answer ICMP") } -func windowsMachineID(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { +func windowsIdentity(t *testing.T, ctx context.Context, manager *manager, instanceID, prelude string) (string, string) { t.Helper() - return windowsPowerShell(t, ctx, manager, instanceID, `(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid`) -} - -func windowsTPMEK(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { - t.Helper() - hash := windowsPowerShell(t, ctx, manager, instanceID, `(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash`) - require.NotEmpty(t, hash, "TPM endorsement key hash") - return hash + command := `$machine=(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid; $tpm=(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash; [Console]::Out.Write($machine + "|" + $tpm)` + if prelude != "" { + command = prelude + "; " + command + } + parts := strings.Split(windowsPowerShell(t, ctx, manager, instanceID, command), "|") + require.Len(t, parts, 2) + require.NotEmpty(t, parts[0], "Windows machine identity") + require.NotEmpty(t, parts[1], "TPM endorsement key hash") + return parts[0], parts[1] } func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, instanceID, command string) string { From 3dda4c82dd5a4a96678c11a74b115bd086387098 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:20:04 +0000 Subject: [PATCH 12/15] Bound Windows memory-fork integration coverage --- ...indows_lifecycle_integration_linux_test.go | 76 +++---------------- 1 file changed, 12 insertions(+), 64 deletions(-) diff --git a/lib/instances/windows_lifecycle_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go index 2837dffe4..919ef8062 100644 --- a/lib/instances/windows_lifecycle_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -6,9 +6,7 @@ import ( "bytes" "context" "fmt" - "net" "os" - "os/exec" "strings" "testing" "time" @@ -39,24 +37,12 @@ func TestWindowsLifecycleIntegration(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) assert.Equal(t, source.VsockCID, forked.VsockCID, "memory-restored Windows forks retain the captured VioSock CID until cold boot") - forkedMachineID, forkedTPMEK := windowsIdentity(t, ctx, manager, forked.Id, "") + forkedMachineID, forkedTPMEK, inherited := windowsIdentityAndFile(t, ctx, manager, forked.Id, `C:\ProgramData\Hypeman\standby.txt`) assert.NotEqual(t, sourceMachineID, forkedMachineID) assert.Equal(t, sourceTPMEK, forkedTPMEK, "memory forks inherit TPM state from the QEMU migration stream") - assert.Equal(t, "inherited", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\standby.txt`)) + assert.Equal(t, "inherited", inherited) assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) - - stoppedFork, err := manager.StopInstance(ctx, forked.Id) - require.NoError(t, err) - require.Equal(t, StateStopped, stoppedFork.State) - restored, err := manager.RestoreInstance(ctx, source.Id) - require.NoError(t, err) - waitForWindowsRunning(t, ctx, manager, restored.Id) - restoredMachineID, restoredTPMEK := windowsIdentity(t, ctx, manager, restored.Id, "") - assert.Equal(t, sourceMachineID, restoredMachineID, "same-instance restore must preserve Windows identity") - assert.Equal(t, sourceTPMEK, restoredTPMEK, "same-instance restore must preserve TPM identity") - assertWindowsNetworkReady(t, ctx, manager, restored.Id, source.IP) - assertWindowsGuestControl(t, ctx, manager, restored.Id) } func TestWindowsStoppedForkIntegration(t *testing.T) { @@ -158,54 +144,6 @@ func waitForWindowsRunning(t *testing.T, ctx context.Context, manager *manager, }, 75*time.Second, 500*time.Millisecond) } -func assertWindowsGuestControl(t *testing.T, ctx context.Context, manager *manager, instanceID string) { - t.Helper() - dialer, err := manager.GetVsockDialer(ctx, instanceID) - require.NoError(t, err) - var stdout, stderr bytes.Buffer - exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"cmd.exe", "/d", "/c", "echo HYPEMAN_GUEST_CONTROL_OK"}, - Stdout: &stdout, - Stderr: &stderr, - Timeout: 15, - }) - require.NoError(t, err, stderr.String()) - require.Equal(t, 0, exit.Code) - assert.Contains(t, stdout.String(), "HYPEMAN_GUEST_CONTROL_OK") -} - -func assertWindowsNetworkReady(t *testing.T, ctx context.Context, manager *manager, instanceID, expectedIP string) { - t.Helper() - dialer, err := manager.GetVsockDialer(ctx, instanceID) - require.NoError(t, err) - allocation, err := manager.networkManager.GetAllocation(ctx, instanceID) - require.NoError(t, err) - expectedDNS := strings.Join(strings.FieldsFunc(allocation.DNS, func(r rune) bool { return r == ',' || r == ' ' }), ",") - var stdout, stderr bytes.Buffer - command := fmt.Sprintf("$a=Get-NetIPAddress -AddressFamily IPv4 | Where-Object IPAddress -eq '%s'; if (-not $a) { exit 20 }; $dns=@((Get-DnsClientServerAddress -InterfaceIndex $a.InterfaceIndex -AddressFamily IPv4).ServerAddresses); if (($dns -join ',') -ne '%s') { exit 21 }; [System.Net.Dns]::GetHostAddresses('example.com') | Out-Null; [Console]::Out.Write($a.IPAddress)", expectedIP, expectedDNS) - exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{ - Command: []string{"powershell.exe", "-NoProfile", "-NonInteractive", "-Command", command}, - Stdout: &stdout, - Stderr: &stderr, - Timeout: 30, - }) - require.NoError(t, err, stderr.String()) - require.Equal(t, 0, exit.Code, stderr.String()) - assert.Equal(t, expectedIP, stdout.String()) - - require.Eventually(t, func() bool { - conn, err := net.DialTimeout("tcp", net.JoinHostPort(expectedIP, "3389"), time.Second) - if err != nil { - return false - } - _ = conn.Close() - return true - }, 2*time.Minute, time.Second, "RDP did not become reachable over the allocated network") - - ping := exec.Command("ping", "-c", "3", "-W", "2", expectedIP) - require.NoError(t, ping.Run(), "allocated Windows IP did not answer ICMP") -} - func windowsIdentity(t *testing.T, ctx context.Context, manager *manager, instanceID, prelude string) (string, string) { t.Helper() command := `$machine=(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid; $tpm=(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash; [Console]::Out.Write($machine + "|" + $tpm)` @@ -219,6 +157,16 @@ func windowsIdentity(t *testing.T, ctx context.Context, manager *manager, instan return parts[0], parts[1] } +func windowsIdentityAndFile(t *testing.T, ctx context.Context, manager *manager, instanceID, path string) (string, string, string) { + t.Helper() + command := fmt.Sprintf(`$machine=(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid; $tpm=(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash; $content=Get-Content '%s'; [Console]::Out.Write($machine + "|" + $tpm + "|" + $content)`, path) + parts := strings.Split(windowsPowerShell(t, ctx, manager, instanceID, command), "|") + require.Len(t, parts, 3) + require.NotEmpty(t, parts[0], "Windows machine identity") + require.NotEmpty(t, parts[1], "TPM endorsement key hash") + return parts[0], parts[1], parts[2] +} + func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, instanceID, command string) string { t.Helper() dialer, err := manager.GetVsockDialer(ctx, instanceID) From 967b06d366d5d95b0bf761c770aea29f9eb6756e Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:30:29 +0000 Subject: [PATCH 13/15] Compare inherited TPM state before memory restore --- ...indows_lifecycle_integration_linux_test.go | 71 ++++++++++++++----- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/lib/instances/windows_lifecycle_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go index 919ef8062..93ab0d197 100644 --- a/lib/instances/windows_lifecycle_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "testing" "time" @@ -25,31 +26,36 @@ func TestWindowsLifecycleIntegration(t *testing.T) { ctx := context.Background() source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-lifecycle-source", true) - sourceMachineID, sourceTPMEK := windowsIdentity(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) + sourceMachineID := windowsMachineID(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\standby.txt inherited`) standby, err := manager.StandbyInstance(ctx, source.Id, StandbyInstanceRequest{}) require.NoError(t, err) require.Equal(t, StateStandby, standby.State) forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ Name: "windows-standby-child", - TargetState: StateRunning, + TargetState: StateStandby, }) require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) assert.Equal(t, source.VsockCID, forked.VsockCID, "memory-restored Windows forks retain the captured VioSock CID until cold boot") - forkedMachineID, forkedTPMEK, inherited := windowsIdentityAndFile(t, ctx, manager, forked.Id, `C:\ProgramData\Hypeman\standby.txt`) - assert.NotEqual(t, sourceMachineID, forkedMachineID) - assert.Equal(t, sourceTPMEK, forkedTPMEK, "memory forks inherit TPM state from the QEMU migration stream") - assert.Equal(t, "inherited", inherited) + assert.Equal(t, regularFileContents(t, p.InstanceTPMDir(source.Id)), regularFileContents(t, p.InstanceTPMDir(forked.Id)), "memory forks inherit TPM state") assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) + + restoredFork, err := manager.RestoreInstance(ctx, forked.Id) + require.NoError(t, err) + waitForWindowsRunning(t, ctx, manager, restoredFork.Id) + forkedMachineID, inherited := windowsMachineIDAndFile(t, ctx, manager, restoredFork.Id, `C:\ProgramData\Hypeman\standby.txt`) + assert.NotEqual(t, sourceMachineID, forkedMachineID) + assert.Equal(t, "inherited", inherited) } func TestWindowsStoppedForkIntegration(t *testing.T) { manager, p, image := setupWindowsLifecycleIntegration(t) ctx := context.Background() source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-stopped-fork-source", false) - sourceMachineID, sourceTPMEK := windowsIdentity(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) + sourceMachineID := windowsMachineID(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) + sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) stopped, err := manager.StopInstance(ctx, source.Id) require.NoError(t, err) @@ -60,7 +66,8 @@ func TestWindowsStoppedForkIntegration(t *testing.T) { }) require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) - forkedMachineID, forkedTPMEK := windowsIdentity(t, ctx, manager, forked.Id, "") + forkedMachineID := windowsMachineID(t, ctx, manager, forked.Id, "") + forkedTPMEK := windowsTPMEK(t, ctx, manager, forked.Id) assert.NotEqual(t, sourceMachineID, forkedMachineID, "fork must receive a new Windows machine identity") assert.NotEqual(t, sourceTPMEK, forkedTPMEK, "cold forks must initialize a fresh TPM") assert.NotEqual(t, source.VsockCID, forked.VsockCID, "forks need unique vsock CIDs") @@ -144,27 +151,31 @@ func waitForWindowsRunning(t *testing.T, ctx context.Context, manager *manager, }, 75*time.Second, 500*time.Millisecond) } -func windowsIdentity(t *testing.T, ctx context.Context, manager *manager, instanceID, prelude string) (string, string) { +func windowsMachineID(t *testing.T, ctx context.Context, manager *manager, instanceID, prelude string) string { t.Helper() - command := `$machine=(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid; $tpm=(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash; [Console]::Out.Write($machine + "|" + $tpm)` + command := `[Console]::Out.Write((Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid)` if prelude != "" { command = prelude + "; " + command } + machineID := windowsPowerShell(t, ctx, manager, instanceID, command) + require.NotEmpty(t, machineID, "Windows machine identity") + return machineID +} + +func windowsMachineIDAndFile(t *testing.T, ctx context.Context, manager *manager, instanceID, path string) (string, string) { + t.Helper() + command := fmt.Sprintf(`$machine=(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid; $content=Get-Content '%s'; [Console]::Out.Write($machine + "|" + $content)`, path) parts := strings.Split(windowsPowerShell(t, ctx, manager, instanceID, command), "|") require.Len(t, parts, 2) require.NotEmpty(t, parts[0], "Windows machine identity") - require.NotEmpty(t, parts[1], "TPM endorsement key hash") return parts[0], parts[1] } -func windowsIdentityAndFile(t *testing.T, ctx context.Context, manager *manager, instanceID, path string) (string, string, string) { +func windowsTPMEK(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { t.Helper() - command := fmt.Sprintf(`$machine=(Get-ItemProperty 'HKLM:\SOFTWARE\Microsoft\Cryptography').MachineGuid; $tpm=(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash; $content=Get-Content '%s'; [Console]::Out.Write($machine + "|" + $tpm + "|" + $content)`, path) - parts := strings.Split(windowsPowerShell(t, ctx, manager, instanceID, command), "|") - require.Len(t, parts, 3) - require.NotEmpty(t, parts[0], "Windows machine identity") - require.NotEmpty(t, parts[1], "TPM endorsement key hash") - return parts[0], parts[1], parts[2] + hash := windowsPowerShell(t, ctx, manager, instanceID, `(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash`) + require.NotEmpty(t, hash, "TPM endorsement key hash") + return hash } func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, instanceID, command string) string { @@ -184,6 +195,30 @@ func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, inst return string(bytes.TrimSpace(stdout.Bytes())) } +func regularFileContents(t *testing.T, root string) map[string]string { + t.Helper() + files := make(map[string]string) + require.NoError(t, filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.Type().IsRegular() { + return nil + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + contents, err := os.ReadFile(path) + if err != nil { + return err + } + files[relative] = string(contents) + return nil + })) + return files +} + func assertIndependentFile(t *testing.T, source, fork string) { t.Helper() sourceInfo, err := os.Stat(source) From 838b70ae32ceaeef3ca73a66bd00cbe6048a8711 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:40:58 +0000 Subject: [PATCH 14/15] Inspect stopped-fork TPM reset before boot --- ...indows_lifecycle_integration_linux_test.go | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/lib/instances/windows_lifecycle_integration_linux_test.go b/lib/instances/windows_lifecycle_integration_linux_test.go index 93ab0d197..3af5da090 100644 --- a/lib/instances/windows_lifecycle_integration_linux_test.go +++ b/lib/instances/windows_lifecycle_integration_linux_test.go @@ -55,30 +55,33 @@ func TestWindowsStoppedForkIntegration(t *testing.T) { ctx := context.Background() source := createWindowsLifecycleInstance(t, ctx, manager, image, "windows-stopped-fork-source", false) sourceMachineID := windowsMachineID(t, ctx, manager, source.Id, `New-Item -ItemType Directory -Force C:\ProgramData\Hypeman | Out-Null; Set-Content C:\ProgramData\Hypeman\identity.txt source`) - sourceTPMEK := windowsTPMEK(t, ctx, manager, source.Id) stopped, err := manager.StopInstance(ctx, source.Id) require.NoError(t, err) require.Equal(t, StateStopped, stopped.State) + sourceTPM := regularFileContents(t, p.InstanceTPMDir(source.Id)) forked, err := manager.ForkInstance(ctx, source.Id, ForkInstanceRequest{ Name: "windows-stopped-child", - TargetState: StateRunning, + TargetState: StateStopped, }) require.NoError(t, err) t.Cleanup(func() { _ = deleteTestInstanceNow(context.Background(), manager, forked.Id) }) - forkedMachineID := windowsMachineID(t, ctx, manager, forked.Id, "") - forkedTPMEK := windowsTPMEK(t, ctx, manager, forked.Id) - assert.NotEqual(t, sourceMachineID, forkedMachineID, "fork must receive a new Windows machine identity") - assert.NotEqual(t, sourceTPMEK, forkedTPMEK, "cold forks must initialize a fresh TPM") + forkedTPM := regularFileContents(t, p.InstanceTPMDir(forked.Id)) + assert.NotEmpty(t, sourceTPM) + assert.Empty(t, forkedTPM, "stopped forks clear copied TPM state before cold boot") assert.NotEqual(t, source.VsockCID, forked.VsockCID, "forks need unique vsock CIDs") assertIndependentFile(t, p.InstanceOVMFVars(source.Id), p.InstanceOVMFVars(forked.Id)) assertIndependentFile(t, p.InstanceWindowsDisk(source.Id), p.InstanceWindowsDisk(forked.Id)) - require.DirExists(t, p.InstanceTPMDir(forked.Id)) + + started, err := manager.StartInstance(ctx, forked.Id, StartInstanceRequest{}) + require.NoError(t, err) + waitForWindowsRunning(t, ctx, manager, started.Id) + forkedMachineID := windowsMachineID(t, ctx, manager, started.Id, "") + assert.NotEqual(t, sourceMachineID, forkedMachineID, "fork must receive a new Windows machine identity") sourceDiskBefore, err := os.Stat(p.InstanceWindowsDisk(source.Id)) require.NoError(t, err) - windowsPowerShell(t, ctx, manager, forked.Id, `Set-Content C:\ProgramData\Hypeman\identity.txt child`) - assert.Equal(t, "child", windowsPowerShell(t, ctx, manager, forked.Id, `Get-Content C:\ProgramData\Hypeman\identity.txt`)) + assert.Equal(t, "child", windowsPowerShell(t, ctx, manager, started.Id, `Set-Content C:\ProgramData\Hypeman\identity.txt child; Get-Content C:\ProgramData\Hypeman\identity.txt`)) sourceDiskAfter, err := os.Stat(p.InstanceWindowsDisk(source.Id)) require.NoError(t, err) assert.Equal(t, sourceDiskBefore.ModTime(), sourceDiskAfter.ModTime(), "child writes must not modify the stopped source disk") @@ -171,13 +174,6 @@ func windowsMachineIDAndFile(t *testing.T, ctx context.Context, manager *manager return parts[0], parts[1] } -func windowsTPMEK(t *testing.T, ctx context.Context, manager *manager, instanceID string) string { - t.Helper() - hash := windowsPowerShell(t, ctx, manager, instanceID, `(Get-TpmEndorsementKeyInfo -HashAlgorithm Sha256).PublicKeyHash`) - require.NotEmpty(t, hash, "TPM endorsement key hash") - return hash -} - func windowsPowerShell(t *testing.T, ctx context.Context, manager *manager, instanceID, command string) string { t.Helper() dialer, err := manager.GetVsockDialer(ctx, instanceID) From 6d92d23be099f6cc2c8859f3da062625e3af1201 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:59:42 +0000 Subject: [PATCH 15/15] Rotate stopped-fork network identity --- lib/instances/fork.go | 3 ++- lib/instances/snapshot.go | 3 ++- lib/instances/windows.go | 4 ++++ lib/instances/windows_test.go | 18 ++++++++++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 46e9d5927..cc1fecdf4 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -333,7 +333,8 @@ func (m *manager) forkInstanceFromStoppedOrStandby(ctx context.Context, id strin return nil, false, err } - if forkMeta.NetworkEnabled && !(isWindowsPlatform(forkMeta.Platform) && source.State == StateStandby) { + keepCapturedNetwork := isWindowsPlatform(forkMeta.Platform) && windowsForkKeepsCapturedNetwork(source.State == StateStandby, targetState) + if forkMeta.NetworkEnabled && !keepCapturedNetwork { // Clear inherited network identity. Windows memory forks retain the NIC // identity captured by QEMU and cannot run concurrently with their source. forkMeta.IP = "" diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 88a65ce1f..3f1cb9538 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -464,7 +464,8 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS if err := m.prepareWindowsForkIdentity(&forkMeta, resetWindowsTPM); err != nil { return nil, err } - if forkMeta.NetworkEnabled && !(isWindowsPlatform(forkMeta.Platform) && rec.Snapshot.Kind == SnapshotKindStandby) { + keepCapturedNetwork := isWindowsPlatform(forkMeta.Platform) && windowsForkKeepsCapturedNetwork(rec.Snapshot.Kind == SnapshotKindStandby, targetState) + if forkMeta.NetworkEnabled && !keepCapturedNetwork { forkMeta.IP = "" forkMeta.MAC = "" } diff --git a/lib/instances/windows.go b/lib/instances/windows.go index 1f49aea61..adf073880 100644 --- a/lib/instances/windows.go +++ b/lib/instances/windows.go @@ -85,6 +85,10 @@ func windowsForkNeedsFreshTPM(hasMemorySnapshot bool, targetState State) bool { return targetState == StateStopped || !hasMemorySnapshot } +func windowsForkKeepsCapturedNetwork(hasMemorySnapshot bool, targetState State) bool { + return hasMemorySnapshot && targetState != StateStopped +} + func (m *manager) prepareWindowsForkIdentity(stored *StoredMetadata, resetTPM bool) error { if stored == nil || !isWindowsPlatform(stored.Platform) { return nil diff --git a/lib/instances/windows_test.go b/lib/instances/windows_test.go index 919fcd043..0754bd96e 100644 --- a/lib/instances/windows_test.go +++ b/lib/instances/windows_test.go @@ -73,6 +73,24 @@ func TestWindowsForkNeedsFreshTPM(t *testing.T) { } } +func TestWindowsForkKeepsCapturedNetwork(t *testing.T) { + tests := []struct { + name string + hasMemorySnapshot bool + targetState State + want bool + }{ + {name: "memory fork", hasMemorySnapshot: true, targetState: StateRunning, want: true}, + {name: "standby fork stopped before restore", hasMemorySnapshot: true, targetState: StateStopped}, + {name: "cold fork", targetState: StateRunning}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, windowsForkKeepsCapturedNetwork(tt.hasMemorySnapshot, tt.targetState)) + }) + } +} + func TestPrepareWindowsForkIdentity(t *testing.T) { p := paths.New(t.TempDir()) m := &manager{paths: p}