From 810cd0697bf9432984714f78b4f150fcb1fb5575 Mon Sep 17 00:00:00 2001 From: June Rhodes Date: Mon, 29 Dec 2025 11:33:22 +1100 Subject: [PATCH] Implement experimental PXE boot provisioning server --- .gitignore | 4 +- UET/Directory.Packages.props | 222 ++--- .../rkm-initrd-builder/.dockerignore | 3 + .../rkm-initrd-builder/BuildAndCopy.ps1 | 45 + .../rkm-initrd-builder/copy.Dockerfile | 5 + .../systemd/system/display-manager.service | 9 +- .../usr/lib/systemd/system/rkm-initrd.target | 2 +- .../system/rkm-provision-client.service | 10 +- .../systemd/system/rkm-provisioning.target | 3 + .../systemd/system/rkm-rescue-shell.service | 2 +- .../usr/share/background-x11-recovery.png | Bin 96819 -> 96531 bytes .../files/usr/share/background-x11.png | Bin 8289 -> 11059 bytes UET/Lib/Container/rkm-initrd-builder/setup.sh | 5 +- .../templates/rkm.redpoint.games_rkmnode.yaml | 69 +- .../rkm.redpoint.games_rkmnodegroup.yaml | 38 +- ...rkm.redpoint.games_rkmnodeprovisioner.yaml | 36 + .../windows-client-rkmnodeprovisioner.yaml | 926 ++++++++++++++++++ .../DefaultDhcpServer.cs | 1 + .../BlockCounterWrapping.cs | 26 + .../Channel/ITransferChannel.cs | 20 + .../Channel/TftpCommandEventArgs.cs | 11 + .../Channel/TransferChannelFactory.cs | 45 + .../Channel/UdpChannel.cs | 158 +++ .../Commands/Acknowledgement.cs | 19 + .../Commands/CommandParser.cs | 170 ++++ .../Commands/CommandSerializer.cs | 105 ++ .../Commands/Data.cs | 21 + .../Commands/Error.cs | 21 + .../Commands/ITftpCommand.cs | 7 + .../Commands/ITftpCommandVisitor.cs | 12 + .../Commands/OptionAcknowledgement.cs | 25 + .../Commands/ReadOrWriteRequest.cs | 19 + .../Commands/ReadRequest.cs | 15 + .../Commands/TftpStreamReader.cs | 48 + .../Commands/TftpStreamWriter.cs | 34 + .../Commands/TransferOption.cs | 34 + .../Commands/WriteRequest.cs | 15 + .../ITftpTransfer.cs | 83 ++ .../Redpoint.ThirdParty.Tftp.Net.csproj | 24 + .../TftpClient.cs | 93 ++ .../TftpErrorHandlerArgs.cs | 9 + .../TftpProgressHandlerArgs.cs | 9 + .../TftpServer.cs | 158 +++ .../TftpServerEventHandlerArgs.cs | 12 + .../TftpTransferError.cs | 94 ++ .../TftpTransferMode.cs | 9 + .../TftpTransferProgress.cs | 34 + .../Trace/LoggingStateDecorator.cs | 66 ++ .../Trace/TftpTrace.cs | 32 + .../Transfer/LocalReadTransfer.cs | 43 + .../Transfer/LocalWriteTransfer.cs | 49 + .../Transfer/RemoteReadTransfer.cs | 31 + .../Transfer/RemoteWriteTransfer.cs | 25 + .../Transfer/SimpleTimer.cs | 32 + .../States/AcknowledgeWriteRequest.cs | 34 + .../Transfer/States/BaseState.cs | 41 + .../Transfer/States/CancelledByUser.cs | 24 + .../Transfer/States/Closed.cs | 15 + .../Transfer/States/ITransferState.cs | 28 + .../Transfer/States/ReceivedError.cs | 33 + .../Transfer/States/Receiving.cs | 62 ++ .../States/SendOptionAcknowledgementBase.cs | 28 + ...SendOptionAcknowledgementForReadRequest.cs | 24 + ...endOptionAcknowledgementForWriteRequest.cs | 21 + .../Transfer/States/SendReadRequest.cs | 69 ++ .../Transfer/States/SendWriteRequest.cs | 64 ++ .../Transfer/States/Sending.cs | 78 ++ .../Transfer/States/StartIncomingRead.cs | 46 + .../Transfer/States/StartIncomingWrite.cs | 45 + .../Transfer/States/StartOutgoingRead.cs | 21 + .../Transfer/States/StartOutgoingWrite.cs | 22 + ...eThatExpectsMessagesFromDefaultEndPoint.cs | 50 + .../States/StateWithNetworkTimeout.cs | 59 ++ .../Transfer/TftpTransfer.cs | 248 +++++ .../Transfer/TransferOptionSet.cs | 113 +++ UET/Redpoint.Concurrency/AsyncEvent.cs | 12 + UET/Redpoint.Hashing/Hash.cs | 10 + UET/Redpoint.Kestrel/DefaultKestrelFactory.cs | 8 +- .../Json/KubernetesDateTimeOffsetConverter.cs | 73 ++ .../RkmNodeProvisionerStepJsonConverter.cs | 170 ++++ ...int.KubernetesManager.Configuration.csproj | 15 + .../Sources/IRkmConfigurationSource.cs | 60 ++ .../KubernetesRkmConfigurationSource.cs | 366 +++++++ .../KubernetesRkmJsonSerializerContext.cs | 40 + .../Sources/TestRkmConfigurationSource.cs | 200 ++++ .../Step/IProvisioningStep.cs | 129 +++ .../Step/IProvisioningStepClientContext.cs | 21 + .../Step/IProvisioningStepServerContext.cs | 9 + .../Step/ProvisioningStepFlags.cs | 20 + .../Types/PatchRkmNodeForceReprovision.cs | 10 + .../Types/PatchRkmNodeFullStatus.cs | 10 + .../Types/PatchRkmNodePartialUpdate.cs | 10 + .../Types/PatchRkmNodeSpecForceReprovision.cs | 10 + .../Types/PatchRkmNodeStatusPartialUpdate.cs | 33 + .../Types/RkmConfiguration.cs | 21 + .../RkmConfigurationComponentVersions.cs | 31 + .../Types/RkmConfigurationSpec.cs | 10 + .../Types/RkmNode.cs | 27 + .../Types/RkmNodeFingerprint.cs | 27 + .../Types/RkmNodeGroup.cs | 25 + .../Types/RkmNodeGroupSpec.cs | 16 + .../Types/RkmNodePlatform.cs | 9 + .../Types/RkmNodeProvisioner.cs | 23 + .../Types/RkmNodeProvisionerSpec.cs | 14 + .../Types/RkmNodeProvisionerStep.cs | 18 + .../Types/RkmNodeRole.cs | 8 + .../Types/RkmNodeSpec.cs | 24 + .../Types/RkmNodeStatus.cs | 52 + .../Types/RkmNodeStatusBootEntry.cs | 19 + .../RkmNodeStatusLastSuccessfulProvision.cs | 13 + .../Types/RkmNodeStatusProvisioner.cs | 28 + .../Types/RkmNodeStatusRegisteredIpAddress.cs | 14 + .../ManifestJsonSerializerContext.cs | 3 - .../NodeAuthorizeRequest.cs | 24 - .../NodeAuthorizeResponse.cs | 16 - .../NodeAuthorizeResponseEncryptedBundle.cs | 22 - .../Api/ApiJsonSerializerContext.cs | 20 + .../Api/AuthorizeNodeRequest.cs | 15 + .../Api/AuthorizeNodeResponse.cs | 16 + .../Api/ForceReprovisionNodeRequest.cs | 6 + .../Api/ForceReprovisionNodeResponse.cs | 6 + .../Bootmgr/DefaultEfiBootManager.cs | 111 +++ .../Bootmgr/DefaultEfiBootManagerParser.cs | 98 ++ .../Bootmgr/EfiBootManagerConfiguration.cs | 17 + .../Bootmgr/EfiBootManagerEntry.cs | 14 + .../Bootmgr/IEfiBootManager.cs | 33 + .../Bootmgr/IEfiBootManagerParser.cs | 8 + .../DefaultProvisionContextDiscoverer.cs | 131 +++ .../Client/IProvisionContextDiscoverer.cs | 9 + .../Client/PlatformType.cs | 10 + .../Client/ProvisionContext.cs | 15 + .../Client/PxeBootProvisionClientCommand.cs | 29 + .../PxeBootProvisionClientCommandInstance.cs | 754 ++++++++++++++ .../Client/PxeBootProvisionClientOptions.cs | 9 + .../Client/WindowsRkmProvisionContext.cs | 16 + ...indowsRkmProvisionJsonSerializerContext.cs | 9 + .../Disk/DefaultParted.cs | 105 ++ .../Disk/IParted.cs | 13 + .../Disk/PartedDisk.cs | 37 + .../Disk/PartedDiskPartition.cs | 34 + .../Disk/PartedJsonSerializerContext.cs | 10 + .../Disk/PartedOutput.cs | 10 + .../FileTransfer/DefaultDurableOperation.cs | 87 ++ .../FileTransfer/DefaultFileTransferClient.cs | 307 ++++++ .../FileTransfer/DefaultFileTransferServer.cs | 153 +++ .../DownloadedFileHashInvalidException.cs | 8 + .../FileTransfer/IDurableOperation.cs | 16 + .../FileTransfer/IFileTransferClient.cs | 41 + .../FileTransfer/IFileTransferServer.cs | 18 + .../PxeBootNotifyForRebootCommand.cs | 30 + .../PxeBootNotifyForRebootCommandInstance.cs | 55 ++ .../PxeBootNotifyForRebootOptions.cs | 9 + .../Provisioning/DefaultProvisionerHasher.cs | 41 + .../Provisioning/IProvisionerHasher.cs | 14 + .../AtomicSequenceProvisioningStep.cs | 80 ++ .../AtomicSequenceProvisioningStepConfig.cs | 12 + .../DeleteBootLoaderEntryProvisioningStep.cs | 67 ++ ...teBootLoaderEntryProvisioningStepConfig.cs | 10 + .../Step/EmptyProvisioningStepConfig.cs | 6 + .../ExecuteProcessProvisioningStep.cs | 221 +++++ .../ExecuteProcessProvisioningStepConfig.cs | 50 + .../ModifyFilesProvisioningStep.cs | 101 ++ .../ModifyFilesProvisioningStepConfig.cs | 10 + .../ModifyFilesProvisioningStepConfigFile.cs | 19 + ...fyFilesProvisioningStepConfigFileAction.cs | 11 + ...isioningStepConfigJsonSerializerContext.cs | 27 + .../Step/ProvisioningStepConfigRuntimeJson.cs | 27 + .../Step/Reboot/RebootProvisioningStep.cs | 179 ++++ .../Reboot/RebootProvisioningStepConfig.cs | 16 + .../RecoveryShellProvisioningStep.cs | 67 ++ .../RegisterRemoteIpProvisioningStep.cs | 66 ++ .../Step/Test/TestProvisioningStep.cs | 60 ++ .../Step/Test/TestProvisioningStepConfig.cs | 10 + .../UploadFilesProvisioningStep.cs | 78 ++ .../UploadFilesProvisioningStepConfig.cs | 11 + .../UploadFilesProvisioningStepConfigEntry.cs | 13 + .../PxeBootCommand.cs | 23 + ...ProvisioningServiceCollectionExtensions.cs | 43 + .../Redpoint.KubernetesManager.PxeBoot.csproj | 21 + .../AuthorizeNodeProvisioningEndpoint.cs | 134 +++ .../DefaultNodeProvisioningEndpointContext.cs | 140 +++ ...ionFromRecoveryNodeProvisioningEndpoint.cs | 29 + ...ttpContextProvisioningStepServerContext.cs | 18 + .../INodeProvisioningEndpoint.cs | 13 + .../INodeProvisioningEndpointContext.cs | 50 + .../RebootToDiskNodeProvisioningEndpoint.cs | 43 + .../StepBaseNodeProvisioningEndpoint.cs | 267 +++++ .../StepCompleteNodeProvisioningEndpoint.cs | 121 +++ .../StepNodeProvisioningEndpoint.cs | 77 ++ .../SyncBootEntriesProvisioningEndpoint.cs | 42 + .../UploadFileNodeProvisioningEndpoint.cs | 45 + ...execUnauthenticatedFileTransferEndpoint.cs | 272 +++++ ...enyUnauthenticatedFileTransferException.cs | 6 + .../IUnauthenticatedFileTransferEndpoint.cs | 16 + .../IpxeProvisioningStepServerContext.cs | 17 + ...IpxeUnauthenticatedFileTransferEndpoint.cs | 49 + ...bootUnauthenticatedFileTransferEndpoint.cs | 67 ++ ...textUnauthenticatedFileTransferEndpoint.cs | 36 + ...aticUnauthenticatedFileTransferEndpoint.cs | 67 ++ .../UnauthenticatedFileTransferRequest.cs | 33 + ...adedUnauthenticatedFileTransferEndpoint.cs | 48 + .../DefaultPxeBootHttpRequestHandler.cs | 200 ++++ .../DefaultPxeBootTftpRequestHandler.cs | 115 +++ .../Handlers/IPxeBootHttpRequestHandler.cs | 12 + .../Handlers/IPxeBootTftpRequestHandler.cs | 14 + .../Server/Handlers/PxeBootServerContext.cs | 25 + .../Server/KubernetesWithDeserializeFix.cs | 116 +++ .../Server/PxeBootHostedService.cs | 300 ++++++ .../Server/PxeBootServerCommand.cs | 59 ++ .../Server/PxeBootServerCommandInstance.cs | 23 + .../Server/PxeBootServerOptions.cs | 31 + .../Server/PxeBootServerSource.cs | 9 + .../UnableToProvisionSystemException.cs | 12 + .../Variable/DefaultVariableProvider.cs | 143 +++ .../Variable/IVariableProvider.cs | 23 + .../Variable/ServerSideVariableContext.cs | 51 + .../BootManagerTests.cs | 101 ++ .../Redpoint.KubernetesManager.Tests.csproj | 1 + .../PutNodeAuthorizeControllerEndpoint.cs | 103 -- .../KubernetesManagerServiceExtensions.cs | 4 +- .../Redpoint.KubernetesManager.csproj | 1 + .../Services/DefaultTpmService.cs | 203 ---- .../Services/ITpmService.cs | 23 - .../PositionAwareStream.cs | 16 +- .../StallDetectionStream.cs | 158 +++ .../StreamStalledException.cs | 8 + .../RuntimeJsonSourceGenerator.cs | 5 + UET/Redpoint.Tpm/EnvelopingKeyMarshaller.cs | 46 + .../Internal/DefaultTpmOperationHandles.cs | 35 + .../Internal/DefaultTpmService.cs | 58 +- .../Internal/ITpmOperationHandles.cs | 2 +- UET/Redpoint.Tpm/Internal/ITpmService.cs | 2 +- .../Redpoint.YamlToJson.csproj | 15 + .../YamlToJsonConverter.cs | 187 ++++ UET/UET.sln | 27 + UET/uet/Commands/Internal/InternalCommand.cs | 3 + .../Internal/Tpm/TpmCreateAikCommand.cs | 91 +- UET/uet/uet.csproj | 1 + 238 files changed, 12520 insertions(+), 598 deletions(-) create mode 100644 UET/Lib/Container/rkm-initrd-builder/.dockerignore create mode 100644 UET/Lib/Container/rkm-initrd-builder/BuildAndCopy.ps1 create mode 100644 UET/Lib/Container/rkm-initrd-builder/copy.Dockerfile create mode 100644 UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-provisioning.target create mode 100644 UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnodeprovisioner.yaml create mode 100644 UET/Lib/Helm/rkm/templates/rkm/windows-client-rkmnodeprovisioner.yaml create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/BlockCounterWrapping.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/ITransferChannel.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/TftpCommandEventArgs.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/TransferChannelFactory.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/UdpChannel.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Acknowledgement.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/CommandParser.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/CommandSerializer.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Data.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Error.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ITftpCommand.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ITftpCommandVisitor.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/OptionAcknowledgement.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ReadOrWriteRequest.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ReadRequest.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TftpStreamReader.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TftpStreamWriter.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TransferOption.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/WriteRequest.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/ITftpTransfer.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Redpoint.ThirdParty.Tftp.Net.csproj create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpClient.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpErrorHandlerArgs.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpProgressHandlerArgs.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpServer.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpServerEventHandlerArgs.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferError.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferMode.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferProgress.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Trace/LoggingStateDecorator.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Trace/TftpTrace.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/LocalReadTransfer.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/LocalWriteTransfer.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/RemoteReadTransfer.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/RemoteWriteTransfer.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/SimpleTimer.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/AcknowledgeWriteRequest.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/BaseState.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/CancelledByUser.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Closed.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/ITransferState.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/ReceivedError.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Receiving.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementBase.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementForReadRequest.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementForWriteRequest.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendReadRequest.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendWriteRequest.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Sending.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartIncomingRead.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartIncomingWrite.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartOutgoingRead.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartOutgoingWrite.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StateThatExpectsMessagesFromDefaultEndPoint.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StateWithNetworkTimeout.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/TftpTransfer.cs create mode 100644 UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/TransferOptionSet.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Json/KubernetesDateTimeOffsetConverter.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Json/RkmNodeProvisionerStepJsonConverter.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Redpoint.KubernetesManager.Configuration.csproj create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Sources/IRkmConfigurationSource.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Sources/KubernetesRkmConfigurationSource.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Sources/KubernetesRkmJsonSerializerContext.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Sources/TestRkmConfigurationSource.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStepClientContext.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStepServerContext.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Step/ProvisioningStepFlags.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeForceReprovision.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeFullStatus.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodePartialUpdate.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeSpecForceReprovision.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeStatusPartialUpdate.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfiguration.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfigurationComponentVersions.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfigurationSpec.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNode.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeFingerprint.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeGroup.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeGroupSpec.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodePlatform.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisioner.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisionerSpec.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisionerStep.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeRole.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeSpec.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatus.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusBootEntry.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusLastSuccessfulProvision.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusProvisioner.cs create mode 100644 UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusRegisteredIpAddress.cs delete mode 100644 UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeRequest.cs delete mode 100644 UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeResponse.cs delete mode 100644 UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeResponseEncryptedBundle.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Api/ApiJsonSerializerContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Api/AuthorizeNodeRequest.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Api/AuthorizeNodeResponse.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Api/ForceReprovisionNodeRequest.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Api/ForceReprovisionNodeResponse.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/DefaultEfiBootManager.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/DefaultEfiBootManagerParser.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/EfiBootManagerConfiguration.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/EfiBootManagerEntry.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/IEfiBootManager.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/IEfiBootManagerParser.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Client/DefaultProvisionContextDiscoverer.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Client/IProvisionContextDiscoverer.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Client/PlatformType.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Client/ProvisionContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientCommand.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientCommandInstance.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientOptions.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Client/WindowsRkmProvisionContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Client/WindowsRkmProvisionJsonSerializerContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Disk/DefaultParted.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Disk/IParted.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedDisk.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedDiskPartition.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedJsonSerializerContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedOutput.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultDurableOperation.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultFileTransferClient.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultFileTransferServer.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DownloadedFileHashInvalidException.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IDurableOperation.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IFileTransferClient.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IFileTransferServer.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootCommand.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootCommandInstance.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootOptions.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/DefaultProvisionerHasher.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/IProvisionerHasher.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/AtomicSequence/AtomicSequenceProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/AtomicSequence/AtomicSequenceProvisioningStepConfig.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/DeleteBootLoaderEntry/DeleteBootLoaderEntryProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/DeleteBootLoaderEntry/DeleteBootLoaderEntryProvisioningStepConfig.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/EmptyProvisioningStepConfig.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ExecuteProcess/ExecuteProcessProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ExecuteProcess/ExecuteProcessProvisioningStepConfig.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfig.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfigFile.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfigFileAction.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ProvisioningStepConfigJsonSerializerContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ProvisioningStepConfigRuntimeJson.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Reboot/RebootProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Reboot/RebootProvisioningStepConfig.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/RecoveryShell/RecoveryShellProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/RegisterRemoteIp/RegisterRemoteIpProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Test/TestProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Test/TestProvisioningStepConfig.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStep.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStepConfig.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStepConfigEntry.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/PxeBootCommand.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/PxeBootProvisioningServiceCollectionExtensions.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Redpoint.KubernetesManager.PxeBoot.csproj create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/AuthorizeNodeProvisioningEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/DefaultNodeProvisioningEndpointContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/ForceReprovisionFromRecoveryNodeProvisioningEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/HttpContextProvisioningStepServerContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/INodeProvisioningEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/INodeProvisioningEndpointContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/RebootToDiskNodeProvisioningEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepBaseNodeProvisioningEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepCompleteNodeProvisioningEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepNodeProvisioningEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/SyncBootEntriesProvisioningEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/UploadFileNodeProvisioningEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/AutoexecUnauthenticatedFileTransferEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/DenyUnauthenticatedFileTransferException.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IUnauthenticatedFileTransferEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IpxeProvisioningStepServerContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IpxeUnauthenticatedFileTransferEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/NotifyForRebootUnauthenticatedFileTransferEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/RkmProvisionContextUnauthenticatedFileTransferEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/StaticUnauthenticatedFileTransferEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/UnauthenticatedFileTransferRequest.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/UploadedUnauthenticatedFileTransferEndpoint.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/DefaultPxeBootHttpRequestHandler.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/DefaultPxeBootTftpRequestHandler.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/IPxeBootHttpRequestHandler.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/IPxeBootTftpRequestHandler.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/PxeBootServerContext.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/KubernetesWithDeserializeFix.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootHostedService.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerCommand.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerCommandInstance.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerOptions.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerSource.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/UnableToProvisionSystemException.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Variable/DefaultVariableProvider.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Variable/IVariableProvider.cs create mode 100644 UET/Redpoint.KubernetesManager.PxeBoot/Variable/ServerSideVariableContext.cs create mode 100644 UET/Redpoint.KubernetesManager.Tests/BootManagerTests.cs delete mode 100644 UET/Redpoint.KubernetesManager/ControllerApi/PutNodeAuthorizeControllerEndpoint.cs delete mode 100644 UET/Redpoint.KubernetesManager/Services/DefaultTpmService.cs delete mode 100644 UET/Redpoint.KubernetesManager/Services/ITpmService.cs create mode 100644 UET/Redpoint.ProgressMonitor/StallDetectionStream.cs create mode 100644 UET/Redpoint.ProgressMonitor/StreamStalledException.cs create mode 100644 UET/Redpoint.Tpm/EnvelopingKeyMarshaller.cs create mode 100644 UET/Redpoint.YamlToJson/Redpoint.YamlToJson.csproj create mode 100644 UET/Redpoint.YamlToJson/YamlToJsonConverter.cs diff --git a/.gitignore b/.gitignore index 98fe97e2..4cf1a744 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ launchSettings.json *.csproj.user -TestResults/ \ No newline at end of file +TestResults/ +UET/Lib/Container/rkm-initrd-builder/static +UET/Lib/Container/rkm-initrd-builder/storage/ diff --git a/UET/Directory.Packages.props b/UET/Directory.Packages.props index 0358b563..3b6c7f72 100644 --- a/UET/Directory.Packages.props +++ b/UET/Directory.Packages.props @@ -1,113 +1,113 @@ - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/UET/Lib/Container/rkm-initrd-builder/.dockerignore b/UET/Lib/Container/rkm-initrd-builder/.dockerignore new file mode 100644 index 00000000..167a52d4 --- /dev/null +++ b/UET/Lib/Container/rkm-initrd-builder/.dockerignore @@ -0,0 +1,3 @@ +static/ +static-old/ +BuildAndCopy.ps1 \ No newline at end of file diff --git a/UET/Lib/Container/rkm-initrd-builder/BuildAndCopy.ps1 b/UET/Lib/Container/rkm-initrd-builder/BuildAndCopy.ps1 new file mode 100644 index 00000000..46b21d3a --- /dev/null +++ b/UET/Lib/Container/rkm-initrd-builder/BuildAndCopy.ps1 @@ -0,0 +1,45 @@ +param([switch] $SkipDotNet, [switch] $OnlyDotNet) + +if (!$SkipDotNet) { + Push-Location $PSScriptRoot\..\..\..\uet + try { + dotnet publish -c Release -r linux-x64 + if ($LastExitCode -ne 0) { exit $LastExitCode } + dotnet publish -c Release -r win-x64 + if ($LastExitCode -ne 0) { exit $LastExitCode } + + Copy-Item -Force ".\bin\Release\net9.0\linux-x64\publish\uet" "$PSScriptRoot\static" + Copy-Item -Force ".\bin\Release\net9.0\win-x64\publish\uet.exe" "$PSScriptRoot\static" + } finally { + Pop-Location + } +} + +if ($OnlyDotNet) { + exit 0 +} + +Push-Location $PSScriptRoot +try { + docker build . -f .\copy.Dockerfile --tag copy-buildroot + if ($LastExitCode -ne 0) { exit $LastExitCode } + + $ContainerId = $(docker run --rm --detach copy-buildroot) + $ContainerId = $ContainerId.Trim() + + docker cp "${ContainerId}:/static/vmlinuz" static/vmlinuz + if ($LastExitCode -ne 0) { exit $LastExitCode } + docker cp "${ContainerId}:/static/initrd" static/initrd + if ($LastExitCode -ne 0) { exit $LastExitCode } + docker cp "${ContainerId}:/static/ipxe.efi" static/ipxe.efi + if ($LastExitCode -ne 0) { exit $LastExitCode } + docker cp "${ContainerId}:/static/wimboot" static/wimboot + if ($LastExitCode -ne 0) { exit $LastExitCode } + docker cp "${ContainerId}:/static/background.png" static/background.png + if ($LastExitCode -ne 0) { exit $LastExitCode } + + docker stop -t 0 $ContainerId + if ($LastExitCode -ne 0) { exit $LastExitCode } +} finally { + Pop-Location +} \ No newline at end of file diff --git a/UET/Lib/Container/rkm-initrd-builder/copy.Dockerfile b/UET/Lib/Container/rkm-initrd-builder/copy.Dockerfile new file mode 100644 index 00000000..febca77b --- /dev/null +++ b/UET/Lib/Container/rkm-initrd-builder/copy.Dockerfile @@ -0,0 +1,5 @@ +FROM ghcr.io/redpointgames/uet/buildroot-prebuilt-base:latest AS source + +FROM busybox +COPY --from=source /static /static +ENTRYPOINT [ "/bin/sleep", "3600" ] \ No newline at end of file diff --git a/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/display-manager.service b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/display-manager.service index dc16e74f..12409c5f 100644 --- a/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/display-manager.service +++ b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/display-manager.service @@ -5,7 +5,8 @@ After=xorg.service [Service] Type=simple -ExecStart=/usr/bin/evilwm --bw 5 --display :0.0 -ExecStartPost=/bin/bash -c "grep rkm-in-recovery /proc/cmdline && DISPLAY=:0.0 feh --bg-scale /usr/share/background-x11-recovery.png --no-fehbg || DISPLAY=:0.0 feh --bg-scale /usr/share/background-x11.png --no-fehbg || true" -Restart=always -RestartSec=1s \ No newline at end of file +ExecStartPre=/bin/timeout 1s /bin/xset q +ExecStart=/usr/bin/evilwm --bw 5 +ExecStartPost=/bin/bash -c "grep rkm-in-recovery /proc/cmdline && feh --bg-scale /usr/share/background-x11-recovery.png --no-fehbg || feh --bg-scale /usr/share/background-x11.png --no-fehbg || true" +RestartSec=1s +Environment="DISPLAY=:0" \ No newline at end of file diff --git a/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-initrd.target b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-initrd.target index b1921c53..9b7d6040 100644 --- a/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-initrd.target +++ b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-initrd.target @@ -1,7 +1,7 @@ [Unit] Description=RKM Initrd Target Requires=basic.target network.target -Wants=dbus.service systemd-networkd.service rkm-rescue-shell.service xorg.service display-manager.service rkm-provision-client.service +Wants=dbus.service systemd-networkd.service rkm-provisioning.target Conflicts=multi-user.target rescue.service rescue.target After=multi-user.target rescue.service rescue.target systemd-networkd.service AllowIsolate=yes \ No newline at end of file diff --git a/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-provision-client.service b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-provision-client.service index d2b3fade..2a20f9b5 100644 --- a/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-provision-client.service +++ b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-provision-client.service @@ -1,13 +1,13 @@ [Unit] Description=RKM Provision Client -Requires=display-manager.service -After=display-manager.service +Requires=xorg.service display-manager.service +After=xorg.service display-manager.service [Service] Type=simple -ExecStart=/usr/bin/xterm -bg black -fg white -maximized -e /bin/bash -c '/usr/bin/uet-bootstrap internal pxeboot provision-client; systemctl reboot' -Restart=always +ExecStartPre=/bin/timeout 1s /bin/xset q +ExecStart=/usr/bin/xterm -bg black -fg white -maximized -e /bin/bash -c '/usr/bin/uet-bootstrap internal pxeboot provision-client' RestartSec=1s Environment="DOTNET_BUNDLE_EXTRACT_BASE_DIR=/tmp/dotnet-bundle" Environment="GRPC_PIPE_PATH_USER=/tmp/.grpc" -Environment="DISPLAY=:0.0" \ No newline at end of file +Environment="DISPLAY=:0" \ No newline at end of file diff --git a/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-provisioning.target b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-provisioning.target new file mode 100644 index 00000000..7235293e --- /dev/null +++ b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-provisioning.target @@ -0,0 +1,3 @@ +[Unit] +Description=RKM Provisioning Services +Upholds=xorg.service display-manager.service rkm-rescue-shell.service rkm-provision-client.service \ No newline at end of file diff --git a/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-rescue-shell.service b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-rescue-shell.service index a272e825..2ee01c83 100644 --- a/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-rescue-shell.service +++ b/UET/Lib/Container/rkm-initrd-builder/files/usr/lib/systemd/system/rkm-rescue-shell.service @@ -6,6 +6,6 @@ Type=simple ExecStart=/bin/bash StandardInput=tty StandardOutput=tty -Restart=always +RestartSec=1s TimeoutStopSec=0 KillSignal=SIGKILL \ No newline at end of file diff --git a/UET/Lib/Container/rkm-initrd-builder/files/usr/share/background-x11-recovery.png b/UET/Lib/Container/rkm-initrd-builder/files/usr/share/background-x11-recovery.png index 979a0bed3eb66edf02fa6869c1500d59ad1e59ff..3baed7a9a70f31a203286431f47d0a1b1d00aa6c 100644 GIT binary patch delta 54370 zcmcHAbx<79+b{S4A%tMT9fAaRcXxMpcXw;t-CYL@7ThJl9R_#z;O?8ezq|LoyIZw) z|Jkapp3_y`r=OXrIbG*`zEgh#opc2qp9Ii4IH{?7WS^?pI?XVAP5BwkXk}d__D9eK zeIkBCr!;NrgUtDFgwN042Pr8LW{+jkeTNq-;x1R#nzf#vC++!H)$jKd?!uH1?Er_p z+GKHcg0ICqX!rh2&fY>xm@n;})ZPa}q#Q5kVVhCE976orLoac=|JMfmkArv4UB1Kf z-sBUHUG&@u^7aEuXTJ?v{_W!0b4%#+91Zb@@kjTeEn!Ta*91D48pEqCOJ!T>-`>xPj2M=OUJlJbOMkhd2PM*f@Oz8hGo3`qF|^7 z(^LbTb|!`p1%Y?MJuV5RkWxVjZ#M?83^EmdW32{`yDYIPgtRx}RLAT}-dqH6=Yev{ zdG^+7^b^QphNAZraNk)+>>liT-QE_B2LsU?4>AAQx{11f88BeMe+v3pN`kHkqA1hbacDVZsluU zhfmgj9W^pq-e=YdpWBC(E~FB|4^D7oX9F&oTaA34yC3&b0{couwHDb79$iSplx0Sxfl35$;q4j_f%M#^IfD z_5Jhx{pMRM8@QJkfe8F@p*}@sX)TRUrF289&B-%r?-+c0D%U;!~Bpb0=k8f zZdX)d-w1SBxqLiC!}bdvdgQXIy`j9C*77Ud;t<1p>Ni5ngCvc6BB0)$@+UL!F76+G zcE=R+^eG)B%_ju(TjFOtWXvashOw=(Y#Chr`rS(k!!~L5^T^VC*}R1vJg|IN@);ZO z2?pKWK%^udYU;u zLb*VqRgP_|n`d#CqR*jmA48$0s=I@_d_!+Uz05MGM6-}UikJ$Sz4;%Kfq+2;@T)e| zOyvR8b@_*S*e~}4Z*qd6wqRzgR>nSSwc>><^pTmX4h> ztBRNt`(VON4iD^FOVV(qN13suN3qZc?EHeNW3agB4R!~lnJ8{N*NQb0|6B>GVdQrU za`X0=qI_6Fgff9wZbSbP4!8_Zq%SBa(qf!oAREDxn<)21Bp+k=XU%XYm6MGo2PA6!1|0e?#!ICrN2JMV1og) zt=m|erw=Tj;-xK>lb&QjFZ8X4dYh){0VuU-TML2PRHp6l8T1VGfSB7^SG`YBFud%b z`M25%%2<=Mp?8!iAN|k6Qxnt9p>+o$-o)hb$jz|dhRHPfVn6HaYmrFU*`yAkilm;R zc!H_nFJD2K@%aD_;8~i^~L;8&gvk}AgcF0#?p(qV;<8}7~|(a7F^B< zyFE1P10>grjM!WQ00W`h00%p(SfqI<>>+{3lp)ep$P<{UOB2jWf>~hlheTeOjn}X^ zUO5q!c2uQck4cNJXwOO646-lDFx_foaq#Bu%cci^otn>SGc+SgoMI(&q)B1F!mNE{ z;W$NIB&DH?&%ktcP*G77gMe-y#4)5~fk*We&%y*`(s^GG@Uf^qT?sM2I7rH>8{M+M zwSg(`wo*;nYHU#vDQyCn{iT4LDsZ>Wi*U~yHCHbh^N4?`&!mGcJ-l|9q8NN0=pF$5VHF$qY zb6FNjPhsQa00q;3_XX3=Lf0zgLPjhlEl;Gadit-2qdZJbSVUpSPIe+o-n*T8v=@5p zuEB!&A9>k+?J~$9_U2~;am%y6dbYcB_I;94Je%G45<1Z_oM z!?PVqk#{>JDdQsk&c4N>ii-gVtcMVz#|G`P7}6+S1vH&o-#s9t#p_>re%gt1ONlwDYYJuo@>oRpoJd#OW^9UKq_Mr3s`B zDwK)4F>srn{D%90&RM$mL_np=LQ9|-PCy99z^^AYf7f8)PmqOS3Er_y=%FhZn(BT_ z!?1_II8ETNRBI|2J+XjI=@5GmJcK;a({5r2`%@;)m+;D^Y1oit&sq1*$A(9V!;vVI z-U!R6?=!jg&9K~Uh$>rm+(imR6JER#srAH^%qDYL$p=;j<;%u|aAe29&Ys?nbEX=+ z0TqKip9r9EKN~>NpVsV1%IA{eaFC0VGxf{@r?7bq3rR#1Z#FG*-@jmkv|(T7DKKBk zzv>3RtvW75Yedlg;mfLpfr|K9S|lnil&PpfEStjtDHFMF+&>9--uro7IWr#`gwP zrR;9#tR;Uw4e_&z_4l74AUj-8o306F9!HJ6*umQx`o+k36j-Fd`>A0Ab(W?Y*@8qx zsqV(1?ZcipN0zF(@H4WKJj7LqvE;Dh6?po&`<&*OjykL z@GXioLU@kw``c`Dypxg>_XjyP@)M5AZ<3(|q(wAhu{E7m7lx63E|)f_7>mu;)i3)o zVnuQ675pUV1>IsjaSDv>*sZgC0I4Y|hkWzibEK!hv5}sPPxe_t#kkA3BkE6Eg!Y{k z%1#bhYy2YITZ=8s9#NqGK}a1a50fei4f7k-U{!)k3by>h3i|sk+jUl3bUg|teHjcQ zy&of3vY;nmiStRB&F4g++3F|9$ouJg8WZ9hL#`WPsNd$t zJDh-bGZa>yjkzd>DHd`#CC9O#^c?B2;QVa;TY(i@@bn#4h8Ddfpwtb}I3{jW!%X5j zrW21GqD?T~jVaF}9voAizO?yTx_7|t0ggHyhxPJEOOU9S!Di%j!GOnF~9zh(ysu-Hmpfrgi zf!oKIO&s{y>tS~6i^JCZCp{k6T(A)q zhY9^<4K?&vEr*WEDV@C}$JLL&O^ovwYiw1WY>vrDNtXw^(sM4NQ+XrYJ{VdB+UJ)D zr0}LA?#P6-2BV__amMv+LeSQT?xS{$w!SEy6Xv@k9Z;M7C=rib?H#a=*H4!6ts;m! z-x&MGiAfkVwMOq`fq7V+;LRDVK%n&71coTp7h{QlUyxC{BzUL&)hk7m`t)?PE#4pDr+xM2&3$~kx1(2s@{Yak#O?$Zr2b8lwUH5YBptK(y_?} zo&1*WTIvg@4{}gM5ykQltuu?H;P^gA31b{W35CH7N+CSkPoVMDDKorfZczWJUKGL6 zEr)F!61fME-ms&Ghbdw6>`$QmL*g=dXDr!Cl&Mw1VKGMP266;dWCxj~ASSJrC@(5J z3X1Uns=Z6#8Nov&Rc2wr~bW(nusq>U*P3zxy zfz?TTb*8(NY81U?9^}kzMQ+Ct`wk{1&RJID8_&z2V}Tb9*I(zKXwjF`L5vzL2dRo_ z=`sPtL5qA!1>tr>&x6?aqmmT^;2)}BDv~NTBBscTc%_X0T%koEJz4=$715FpZgA_% zWy%ZR*d7>0B_f3#U1U(`2(?xFyChLMXz=pD0pG>(Z8!gBYbO z+f@;fLn1yvT=_yDFUP$*liVQ8Crb&C*CUVGs0>%i)Z8(TFW^nySk3&W66-NAjyjlX z$Hu*MM<*fXouLkK$8yjaJR(VFG8_^S$QwIrQHjuq5wVSr&^vtTR?c>Gvjcs!_r)?? zf}TKI41U^eR7OWLf|}crwWmgotR1a~uBc=(B=c+v9ty`hf9py&(-<1C?@yjnwG1!U z7E}_xNh={&jxp>sgHI}NLM@Ix{aTGuUM=JD>X?r^+?Sd3YDl76=z{yWgATyT)+Z}< zggY*ndmJOk@)dA$U+KxvR*K-4hF~3+oRdR@9}~t3Nyw4S&)Anhdt_l(!5tq^mU%a* zkbUyNm{No|ZsTLAw;}{4P9T!2cAJ5(Tc(({Wl$d}FVG^5 z)|ANWvY+_TM)3(2uimtub*NiA>#>5rkMzIc>b|&A_AwWHl@U!K`WgOMfAnnty)E{A zlBP+@p*dx^4VJR7fRdG#m~s;U?I|otZ^8n6QtSqhVoblcr8kbZopIm7 zH&szSci{S%>we??`t8>VaoXoLOB7|k1mZqRu-cW(xLZszx232m zPHe}wy(@3UAGqnpy{c*^c(I|sEYxC8D1WydQ8gLzf@p0L>R))i`++vG8)Fw&TxU|{ zWNI$^wWw+_oUJc!E8hs;(%Kd694^z4U)C&}tE=6YYaPb{#`0c1S*wA3BdE8|LVQyV zgf4PuL94f~vQq~K!|})~^=B^6>I+r-h;P@C%T71=w)wul=6Nhx^@T_HEfkXbl8i#I zv6fn%V7l@ycY7DvXD0oMu$krW3qTKRBn~O>S?%bTH@vi@2ak|a=)eJT=gtgxbfZ*O zsKZtNSzQ*$dnXl=&;3)PJ{#cdaK zvog$$gj+oR+#aF#Vhy7)F!xsRzfGMEhjCu41-|IXDz%;^z>ZW$|M1(oq5odY#~W3d zom!ePQY1JUOK16{mPe+nXD0Q|FHqqJ_1I9ZW9$SpPKZ9nZob~fW7^Z)LR|5XfVslF z9dBZ{9)hCh!%sbjq+q4S)ay#INRrBk*|et0@eu-dkxHYkTv+U%!bnC0rG}O> z^*I9TVc@InQ03Yxc3^Mvw+?Qp(?$Ru@ZHyQFsi4Vw+p6e=*MO3gCwH>G|O*tU-cAC z`M2V$m}Px3u-b79X%}HXn*%1Yz7Le`h-4?T?YULeBWmM{$x<%Z4xu~hL8eJ&yOAX0 zn|8v7pnQX>ffSV`ln3)wO(BrZ^$M(D*m0*K03vo=c(>^v(P`C#c^HzucU6b zSUW(FQW4HLI|YXv?KaQw;E^YdE`ZxOgm#;b@ZyIx+e?fyJ@>!o@JTG&n*2D z+_gmT<4*N3HwWBVDs;d7Dzr@SN*9UspaKjUT|zRO?$PvO$O4UZBIjq(jpWLmvlpe+ zMwuO3^A>IF6_wK?II5aOq8vC2eg@JMY&DBNING)kwRaB(l{+i3o+`{OjL9}?atiXk zdsszX?d7fc5MFN+kf~LSG-&2dm~FV=IUt*7+@LxT7(jP1C*wX22lMl8({zMQ8URbC zuRKPr@g3aB`&2J1tZbu(Vg0A8bW0QUUU^xK;QqGr!n3rg9`oQHZ*WC0e`RMMzbKwz zLMTEb*DRkuQ0tDXTw{UyE{V{4_~BSx!fWvIlMfCnR(l+x7su6}r zKqx=SIzl=oB$*aVUl1446dx7p-T~Y_m$>7ukax1KryRBs?4>rC^6Tf2hJm%2m*kXd zgiS3uX#(=)ZKfR=QCm834JD!_1q;i$&B@RM_L%NEM(H~?cXsd%Ru?B^`#bZTCv^#J zrR&DPLRFeUA@X4u6w_dCwOkX&@z`e&Ba*u`6o zLRzy5SowH~o~->L>#lw6*W9gSX$1emO(;^*5$}xS16?kNcc@^1pS5$Wh7Xh(oBA|9 zsNnv%QqTToM|A4<@hETj9aw`T6kJxrEanh{!6)ox9wHfI{3Ow%<`CVg7CZdo`qcUn zDxNuY7wY9X^_HUd-7ndL2^`p{BZ$Djx^EAq24`WavWoAZ04iI(7NZV=1{=uF{ zah#f}&sA7RwNsc`&KPa2!X=fTdc(7tM|GtE!nWMC3;pF&nMb)XQWUUc0s2Z~5!C1L zqX8k&#Ddw#!h8oyOLgNv!$+&p!MO23SZ&_>8j8$A7aD)ukn}zDijmC}Xh~!^9Puf} z&j>AMK**wNngi&cL^7)N*1_b{x;#w9$ue=Fv$l~(=j}o=eH1aK)u^LSs5BR+|1uc* zW%z~O|HPA}gxF|*S${_H=zBk0AcdZRjZQ~C7E1F*RJT9QMCt);3iis|phcxYYHy%Q z2J;3Lh?>5EH&W(2+~ri7^^9P^Ts`Xf1Qu z^JXZd9M`6~#s*LIdP?p`1NQ1nyVvS!Fv$n-~cA~@^M@3wOGHVKg1`N&v z$56qCvo67cTjf>KZCSyf_haylGt)Wl1-1RX0LBeCB?+))<0cZ0OXQ)|@K;3auK zbDM(hYBp7-Q?LQjvDw0AoGgd^=(+5gO;ZR_b10lnYA~RI>o$@iB0e5TE%5mzbHi`) zjyFYlpWe6H*|hr_;al<_Pro6bXJGTt`W!X&4qz|afn(Vc4DP{@ywhQ%?y_iSW^Y+5 zKW@*(2yd0P-rJ|0g80aL%&*fkR{X|~Ry4piObo)XP>^=EXo8!76s2D+B7eaKZkn_< zJ2D5a-^Hzvm80x_oGx9<-=uiZYVKv`v@7W^LK0ecTeQHehJCjX)U7lTYHd%DH&o*8 z161XN!H~14X-1gBVE_0+Y%)(Gi;zmQTuHSmdwsvjE$oSQiuW#sF3nW$JnQz0b@MB3 zIZ5}_Y1F|%gd2<1o@EZQzaJeb=1Zm*7ahbBwO{PMkRKAOm*ktmU0yrNemA`tX3k9I zS=@Bn%I?~BBBF%f;m+0{jk^briMPr@KwY~6G~=pEcpR7HZf|mp#e&KL8e*NC`GI;r zZ>E)*J=0A0Qg7YKC&?4!-ffBw<6oN;7-4sm9VQ*)DY3S`;jR9Q+{`r;<-AF}WR?h=ko z{w^cL+SFft>E2GJxM*9AAZRNYmn;wAYD!+s=e*Fhww2fI3vQUsn`h(Vb}p4wcUcT; z%exZg;m)i(ju|-FbOcQ_{=|*bB#t8;{mx~hWu#Q=arPE4knYdfv z_2f%T#;RKF6>jJ74PGiwMFkjmth6^R$PC#d$PJri`U&LaMAhPv$<;iRrMDPu&oNw5 zsd-e~CI4vgsBiIbR`Uel&O69QJwG-kR?KhTk9cel#gI6FNWNa~d)bJ>*5u6vD9^Y1 zv1!h%-1`#(n6CgDS9KCBXKc^W)y)zxqR+8g658XpsV(&P~Wd;FN#wVdlgwu;c zF;6jEl553^YhFUf!Z_AH$U(91NJmR^(Oi4w?+ayc8#6Rn^o7+Oa!+vCD?&IeKdi(L zC9($@FeYWR{78S3e8N?RJ9uaiFG{+6Xpp`(puq{~10I*je6>>S#wzYeE>d)cHpsJi8erDe?`pNn;?eYLT9cXX!50JF**E1st5u*{rD z=7x%C^Kj|C_M^&P?NmtwQpF-{Mb=F0koUG(buJV9Dj~A^M%@q*0^~d6%*VL(S{>{z%<7Yxh(&bHa~ zFM%XC_QZR4ma{vzHfGsA|1BAx0}kfm1uh&tVY;9JN&>fKmqWsAy(|Y-j!rhR|33Rq zhX>?v)EBZZc^#%&ogY4@c)7a1I%DQu-VEn*AU<{X+~>;$`Ihw?bh~=#!il>{6u|@~ z-8u2GEO8#`={6&M(g-mN`?7Z0lLB|~dk1pratKS4753c12A)=158AlMPIL;UZ7YMs zla&Qwp>mXZgUx0pMcs9jK~`DMv#9{Lpab~P%C$>Wqa#mia~+lRw2@Sps*uySp-;M&8>_#Vrjn`f+H;(WV!wuLcS&6+#o!t$OTqLBviyYPU` z1HWT-i3%=VrBY#)_ccG5bJwRq4~xr!8S-1}JEV)o95X`Ks|$pbpEIQuoxdRbOzeAj z7K!Ql+1|PFEvs*LL9&y%cnXg&yia%?5n)O5Y`z`7kc3lYVK8@dtS>=a!eQ}9p9pVe zA!2QP2$o}+3!ogXCk#>Jf=&@l7hVBuBytS}W3UaLGT-e-`6D9WWC{534_x&Oxz4cZ`so>pr00#9np z$s!HzV4WV$ZJTL_NboqDm0&Ycd8lHL+5D+++s+dG zDOWr%>8XH{t~;Y8dCF{V*W3WKMK= z2upVph@9f%A`NJy{JXAJ)s(M^L^jT5e~K#~@H{1Q|8dJV;KzM|UzrtX?HP&-k1!GM z{z;bNa98#G>)g-#NW>U9LZqA=3{YpV*;*m-`z$$p`}2)L^TuY7g27MJ*6a#e**ej( z(fMAZS2&Mo3tPkFt#=^8l`U=CC#-Ow<763tm3op<{Ne6!>z;52LDTVz4Q2J0oyZq* zaoc5^aHNLuc+*JRP`txnN$nV|p<}CP4c$&qQyU2bXJ4y2^YN7rhm&vyCEFKC-q4d_ zf?u_rA0IvBWZe+x7SRJeXasqxm60b)kmGv$jJXp%?E;8$DV71!(B+AS4YdiR3L0rU zw+H^J(oRUgEvln|_G-j|d_Wl|ZW)C!#`)@CqA!y-nkOO$a+8L%ejBQ4$D90Rkj=wS zyWz`w_KOEL@3Z_e>oxHmO`PNiYeV({^sa;8>NSMGjxU4*%O8|V&5oxSr`r&Do=CN? zh^%j_&uI=_Nb-QqbVrr+GDKi^wDC?}pU*7Z$SaRpDv}As<&3eS_2W2P0Q3P%RG2(- zzkGSD{3arf)=NKq;f)UX2pgaTF@03@%lkY|y~Ui@PwFVfy1lGJA>}Qq7)KkUUKYkO zk)|b{lQgI+J=OBvV)aUF^&v+EKo8!aYRP9}QzTD`@yuHp1mqwd|G+ zJt`3F_nRMt1!JK8<^=x51Sl`24N$A54==XR@iW7=p=@?oy=LXvmxC3GWCwJ2;_dH# z1=a;h237qY@bBh3bS8s&;pO|2^y;-p4E4j+#K2KYoXoGp)nvNhuWQd+W%>NPU;u^# z#_O?dyqNs_br)rkncf~gOP(euyV>RQidm%16Yhi_Vl`LrE4$d0@moemT1;g!deJZ} zfAV=C*PpcH{M~Nh5nr2W#DRoQ;|@=-VCF;NOT?F((<3gwX`CS$EXHnQOh#hDB*aEy z!Yp(L+IS5qUKS^z#rhpXQZ3dEhyd)71`iSj;k||crr$gXR{JAD8=KK#hkpVt{c!%&foqzXBv3aBqp9it z_USY^>}cn|1q1$fxd0r!!B0<))tLcd%__XwcBNk2UUyYJpT*bcQVwU8s8X4@_Loc_ ztRM-UPakgD5zOBGj6-h-MLjOB=KV`*JLD~unQ?gD5`m~QXuVKu3S?Yp_ZOzaL^lOatjjdR*O?KHPs$cjhr0 z1Lq9*i~%E9Lsm*GIjx_Z3^43>5)NseokN4gTOl%pO370 zZTcp*%^2w_mLSuB75OBq*nlmn#`rxtc9J90*#U5ILH<-0#UY^kjS_7VJ@a5$)8Hxx zE8V~aLwN)u<^%Myy{MKWUDcR5Z^c|94zt^AZ}ByuE5mH)_xMf^`Ci{iqgxwAV?Xhi zGPE64ytTYlPMI96h8-@7RT}Xn?fK#mVpx%S(xD!Hd&tGRZ0_(rG};lmSWw+31PZ~&xCReMU8h(HxiYB z>BT<4wwX$i^V=dTWy1^RJYiHJz9HMsfLy7^=9wPMhkM3BNk<>c_1 zcUflenx@92x?{M=_x9-RFABpLoNH#^w5P$B%>r}8j8-|4+RB8TA`D0KH#Bw&@+6WL zvYql0d%KAe0GgD>2ui$8m8U7R)o5?uGv zaJE;&Az&1F(3zmZ=$d{5Ya2hV4_ z(q<5E+?r0iV5dcFR+sg6{e6>|eeEHM$~|;7RR(p2osYwVU0{80@h;NONnN%N_sN=i^rwz2dLP5Kuxj%nSHXu=U*+i;Z89O=1sbb-@ z(NAup?`>?Tzayz`w3t?{FYiPO!+T{& zbpYLTRGNSmj(8gEOxsu!?#|oXO$M);ggpC}*E6eRh~r@egcZ{jTDvlq7o9nQ!dFo^ z#n#i?D4oW5(D5ffIUdVi*CHn6sPd8!d#hQNd{uY*1C^IWop7YbYj+T|6 zh@_8SRYZ{^hcMZgc|~bF;+_wQIe4?TKl;Zi(>_;=96>b=CB;{J5q%}zM#Wc>Q6-oN7-KjLp12I>?eBbpS2Lz8 z2IRm8wh%*K$5Nuw(vq>G@2vXeLAtfNgL|j{U)=xyWPjj)xxbNls=QM~pHGhNdT@*9 zYeT6HahPK~*0xy!aym_w>2nGH$wxY&3rqN4-O69hqlnQ%7(qP-0sYm;v3Ex5Z8r}e zo<=yih`tDYWbF@j$=nrY-;KcYJNjabQZWQo$9TfT>(nCg1wdBX?6igYi~dA+YDwIH zkoZUCY&j1~RlR<-I!UEk<$1J_WwTaLJhW8=R>F6o69-Z{ z210%4i0Kgfvg7V??7u!avv^If`y%uJJ8xAFm%bbu{jwTcJhk5AEW`1VM(^k{-a(d6 zdushNq`q*x4 zZbAs#p=7K0I_4N_Zr4}cV(jxB{PGX)ji@Iq#yHT3{5;<|cKkRUIcexh6VJa*xv& zDo*r-nV_Lo?qY;C$1k4CM0q73{3}Z-&Wsohvmz-oQ?JH6*0+EHmm0QCAltD!yJ3K) z%JF*@(}bHl*97{~0}{yzL+*b93qq+n=@g4Q5-|?VuNV=)|6cD!`Mnr3{wRHDkIv-7 zv}zM(hPU_=5IX8^8H?V5uB@hBOjKeAEr~F#pP4Je8hV?_C{wa5m(0~9+#&^PW17Al z46q|bwU)aJi*bnn+wR7xzkFvISCYCU9?;F(0?OH09)tJ)40$s){zP%j#xN?&>dO20 z?FYc|g@`jlL%ur$W7F2GfvzmCH2-W`DFbU$Q$QTL?Br=`?1wK$4U9rl;>w7L?uU(H zzZNtBd1!%jCepWVkZS6YSv5#rx0#2gJCo0Kt@62%=ILXK)699>wO-Y@=bS_u1?IE8 z3vjtm{B<%|hyh(Rw1;?qvLpygeY3`WjPn0Y@9#7H0lE9HHf7X)W@b<11ahCR->x@Z~--j#ZNHW{+le0Dp60J#UD#cqv za7uLDn8j@YoNPL#q61qNSkY#CExE)%WmdHj?!55wZw$`SN#$+vFS&{*;hnndwnu$8 zIQo^vDAtV482XdP?^{4C?%pR%0Q=WQj@#eqV<|rF0$vi=pBa!DJQdsK2KBO{SZq{G z(Jg0}8|Tu|T}`$2IR484dgJmT!auZkt3#%}OJmOga1LLtj<6cL8_#yxkEu6v4{KQs(o-IrfQHJ@3(|2UfIRN1n@gkoe$IdWbY*G%g4%hgk`r}>|Lggjh<;t@7;4v3>?WAM2v__ZE0=#4+Ko6( zoN;_D^2mPJ_;%m(TlRliVrAWm+UenUH6h{*VYrT z-K$wOe5CW|sdw56(TbQaO3SxRK61Yc znZ7_v-C6ffxu+LY{S*0WEUzG}r!5*`YqS1o_w=IEf5=|V;T6Q8HNfH3fKphOf6l~R zYOPl5{Lj{Zs$sztbsH?Qjs~pXGA{s((8ES$?MWzZ<#7 zfd=5~#y|pcR{*(JZ zIQ~jUiyQdYT@&6V3KQ&SH|JCEtf0fsD^tZ=?Zo1I_>hW() zuZ_$ua8eOkSUZ&#>c(b=M|?N-sPe^2UVP`p(z}z3CQ&V8FfYfM2mncZO*0x@JMDga z8DRJQtF#njtQv;-Z0dX>0oj!{OFiA&9~*fSh|G!Xn(OK=WyE^8lUzJxWZU!5w*84d z8!BDt6P#tfef0ucJ*k!4FSb$laF*6L+y(Yn%pbg~H+hVy{{#BZl_}r7+|9B1bC16_ zT%{A(x6o$pCyii?8%N4VwSnZ1d2Re2G_i3{~vgG3V1+ zKV0tj^#5u4EC0jvOD6U=ZvM=QR+@t4dp!NoALteAOej#_Zo(Tc2>60b({8WB4}`AH zOy3j*R$Dd?F2J=F$&R}p$_rT6eo3xqct z55E}}X59Kh#o!uNqZIl3zr&HYe0FM&qIU%F1wjWSEL?@C^|hzlLOW)dtiJp_g)6w0 zayCDnBaRh4PYgq39QU957yMt`p8>A(ulq-!Pvt)1SJM|=6?DDWC(rQyc*b}GvL24z zmnqCmP6$8~&5~dnclm2&u3Lo7OtZ4Vd{2^}FZ{}sO-9+oz@~MU#yvck3mHl;c?xd# z6E0Y|ZV+J`1Iol|a`8BK3NDZ1^OE_;_d$ZLuEGaU+eC8>5|V(PMk3P1x%Sii>?fHx zwt>7C)g*q>T-N#Qm$RB>hMi#`EP^}?=Y2lsY3V?rf_e8_9sdSc-u6xEIz93H6&7>| zYN2_hTlxAU^N`Y;gnIHO>xT|u5Lt}mOerUsqMSCoW@|sA%H9o2sxq?9{vHQy>G3#u zWLxoG+T8~gQymp%)= z=nC>LB7b7G$yua0<&OG*p!DuIR{SasvXTKZOI1_RkE#TJ2ewi2$Vr3YuCWLYv^%c9 z@(!^oIMGV{s*MO2O@i^0Ef!kkw)3^=EnYLb%cUeoX{9s$kDH*(|I9$VJp8u|MA3BN zyVkKfp%YT`tUX`uUvhr`7k?-?MHT{}0Yb!`!#qi81+SsJ(dj!y(CPkwrFZUAjnMN3 zEU)fra{)TPHkikrj~wXquz$i@_mH0y^K<#3Ry$oSw0v*tGbdK=H~aUWPG8%uN&45t zGAk~kOdCvi{T%B`nn8-t&yDGTURyCa391=dh65WNQ^5~=^pvtf6jhbNNU2IbfXWdm z9e#l}mW0ob?!~4x)KOYQv3V;hkD*I1FsD|%VN<`M%*W{9Lo8biao^HXfnu5KgthKp ztiQ>qSMrp&eW5V(ywC|^LVk(gpFSjwi&sFhTs7n3DoY}rs?oRSFR#9&ntFF;p&MM3 zuNmF670{Bf1;>iXfDSq(-G&>0)RUfvfR4~oq_VMk=F`YRTRVp3A5Ya#_Xo>#$tTQ@ z2Bj>^E$AsWoJ}nROWX4Ul7^0_JEjlfbp|`~(zu)1PnvmRf&Ak2`dwi-out*{Z_W$E zx22@)NA(*z+lUflMY+q+B09&aLR;_4vyj4(w74u4Jq1h990#bjIB~haMWYMeqZ=t- z_r9bh5YCI8=5puJ?WYZvd7%7f_tXk`wfAPM1>`pvB;=zO&qF;J<35I;*p8kFEFojr zI6<(BzlFge$VlV-=65qnFR&j12D}yqScb_8$sBcCYb|ckeyZ3+v=! z)LnwPy7{nijruG)$iTZHtd;^M^D<#D>n0~puP0{88{A%&=XXGVEQzaBbB(a5F>biMznDP5K6d@%M6BsXp!w-Sbgfbs>^k(Hn>7z1Vz>VEv^~B!UB4n_SgTHe>hU3?T31H-G#5Nm3s(#esNc z^CYGA^y+>cP5O=4AA~zGj5%Jt!0xZ?vU$GJ$ZK#~r9g)BZSU#D5t6f16a0zK`n_u9 zDMg`e7)wbciAvq8y>d-6t1M-ey)hUs`!R;sMnI7*$EJ#q(v-r?Swn#-h{jE?*5CIu z-OHw7SZ(I591Kq*wfnm*0SK;!wp_1x^7*+D}_Fd^YR6vO^u~}DM0{@*5Twih7em(nc zJsWaP>Gdi)o3dc<6X2{Z0cBo#{uw`_MqbE1c;Qp(eIIXRb98w~J?ojG51Cy|JIlV2 zBK)WKJ?pF+OVu{Un)QmGFi}H?5Sn5KJVo}lb=cFLbA9FJC+(I@o~e;X2xWHZJ5Rei zl{T+4FF;$3kQ)Eg;bHm3?vui(vSa8;pD&JRj?_u}k;OXgXz3@Vaz(s4u}g`Uh2kzS z(TN56RYuh*73o#@Pr{jO(+0gg8jDtNzVzGDcShP=Ut-uC6d3#ORw@Hjf=KQ%ca9P} zcUWOqGq_4zd`pF_R@80jsODcr8xbk7Sarswf&gNPzR_#*I`hEvoV?h#4I8!x-8&b6!5PH_)^{-k;=R(rH;Z#7>0Y=$I|Tfs($v&Vre z1+Fz2;pE#^f+|kL>YcstL%CkrEw9nv*)82ka^GSHme=LnKSosUI}BLyO~@QI#)dA$ zkLEi@hUKu~Kf#nPkm(@E)x=1)FBFP<@yf02^uiL51tks4v6}lJW#YE)5R${Zs;e*X zyKiC3=lzM;$$y9QK{{qx4AX+Y(=J_T1RPAlSd17rEa?w~lX8;I=3Ji*{8cU7%BI#3 zby`2r^U!_3kBd%FO}dC37?`TtQ)OT~R{Q9ikDJSxs4hI?Fy7X2W!uZ9>#5odKf|(@ zVvgH}JW`QaAxv;Ja(k1oHB3D?6DF42t{g@!uD8^s#}4ml`sgJ~I7;nY#@$B$m!@j% zP84Hq6xdy<+l)*hzac_!qQ>%y{p98_@fSdd9(HT)4dcUa%JQ+8ExJ~*syjY9aHmHGuW{FwRf%Jp$r#_buuW8sK! zr=UMfE^oGGiN9yTEz@^)xR+h!Hz1c+efJ6GA)ymO97?NCbdmo>PCRpskJECdo*YKy zO)!oR#(sMX)F?}6pVtG^FuWDu>x%H6KQTqL8}ti?*VW$`H!rMdN9OQDWwY~Q-;P1M z@RBPq&|1(2+lcW7ci)CW6atikX9Z$w72N! z=B+;4RP3Dw%@DaDYMjBy0$dGul1<}?N^6LI4Ub>W&t1M{{e}B{<2e_xLB{B`)Z!D? zI6)`;EI}ullVZgh@4p$x&X~!J3&`AbRu3m!%p11YLk5c^w&u8|^1f_+(E;giJ7P6a z#pj4EH{wt6|NOlADs`>5;ar47f8bcW*wxM1HvsJ?PYy536(mf?#~c8W;VG#P2K{ zB)9OZPoU4IMZzc=S(U7M5FOB`)(nd4a%;8Z%1ZiQ7zPS**iwjSbl zTNdnv1j%c>!pcHB0d*G(mnVJE{T6zzm@&!14~s zjYNrq(`x3)hi_`cXB3V48vhXC{|5Jm&p`kGL+&N5+-&Ft`Er$$@J@wb=)(t5`gXX` zFKuuxIb?X6NHPvEu(l%#Ww(dUGwxPK9t~4WhsP1$OK%(0H~H)nvKmDv$m~)j`RC-^ z`N3<9vJ#tnksJb4O{)R*@nexl4Z)W=SQ8dO!v6x?iJFVe`QAcUx5c%XJZO8hNuZuO-N$1{QoVYrUc0_HaasM<+TBg^1*ig#5yG<`*SMnK+ zBkW~+2l%UZ%~lELOD1(R)=Osgsw%+M$a;DD@W%%C?=MQ>UiA21D)8Xty=vcajuUb4 zblExJ$Zyrd(V9g&NmFO7$;V}$QF=m-e|#l3-N#bU3mw9&k8^s`qK0gaqi%kE+UH4S zbKm^ie;$9ZVB_D>%vKZ}vvY#2#Mlp13HUc#vq(lRH9Y8VNUQQ)nx5=3<2GOQ^>%rf zl1DFGQaJ5C6p1gj7q^YjO{F#Nty>EBx>8Sd1RhOFSfe-MEOg5Omn$9O z$<1dQ#l?!D9oN^o+6A!4)ep(O2s6NCYvTBcZAP0O@7VPC>Za%ad%Kn6L|C=Fk!Ac+ z$FgSxrHS-OzW5nlF|Ip;H3l|4+5dlTFTf31FES(d30x>rw?m(iEgHm#tmpO$bKrUi z4v}jsS7%5Ob6~{lt>aDV@nWVxL(s$6?T#9)iYc9N1@!yNZ7uNj2N5xum&?L0PTuyo zV*#HW`^w2g<{^mrT?V|PKU`i8SMQ#q0wKkHK2m6~RV4oDg3|2xmzCIGVIsLN{R0MM z-v8V;sbFnO#XZ$8yW2H8BHv%pg+D^ji^mr@8;;7Xc!`T2POFtO+M~P%Xth+gTy&y1 z&XL6i9(k4CBda(8B{c~&^KsZpX^~`vVem;Uxz5@vS-o1iM{sMgKfC%^UY>g&>G>Fc zEx2w-F1wzE;CGU@HSvRA^Ni-=hFenWt!&_fYmB(;iyZj_c~M+)yF%`#l;d$4%5RfD zN69tWMIbE~kEV$=O%bPnlh2*wo0|hQtFaXn6qS6GbBB%`|3h@6kB)SSxYy$eNwMDC z_nxthnAJ{G(K7vFWwR+sqdDc@0|c5EF;@KRO1->ho0Fa=iP*-Az7PPZy+Z zL>uy@-J5dA$AA2C1dwyKNY&}d@bpSStuMmE?|DUy)j*WbCEx-(IO57KeOI(06Tk0K zt>2Z>JH0l-!>`zu8fRmJCws4C`-iIqr+MJE(VptB=!EZ(K@$gvkxCYgKHG_QO0m!cw$&4`7p37r45D1=us5}O;|^sq^f zJa`Amj~aoYhmmP=X#$;_Bg1QR`{A+$-x`wtg7J1t)7q^ES(7%8Fk=++#qiteSbB!j z_gD7ZuvUM5W3@U<9NK5xO@r0v$LSKorGVv$&&l5Nm9CWn`|mz&=Su|^Ew}jW)kh4& zJmh_g+nsA*{=8ZrmD_p!DIdtLHpU2?p0f0QIwdR$K6Rp5*QkJN@^^SjZm!W*C&0US z<#M+>Yd@2Ucr%9Uy4zJ9VfJRsN`pX^YBE&OlE_}S8f&y+!0q5koOVJOq1kc?;1Z1d z-k%5ro|NGBO|0!7`{7!Wp--%+US(8d$!dBEGgkB*YuerxB$?OU)1{El3`pSSgsj6g1?yC(j?J6I-`LkaTCt52d&GL1@7fig$9p+o(03wo$T8v)X9sPhFtFR|dGK(6h5dJr24EQHg?-Lsk4hqN*HIRu z8FY|>O_ZV5bqP9|=*oMaz@VnJ8uzR8sMS>6yQ=dORGb5{LdSc;D{Rs+yhX5S?(Hu` z-YD7HmEE0o(|gdvyi>uCq;h9tl*WkRBzYSi$ZA+n!;kEo^Zm1;9PZylI9Nti43O1( zYg=0gM^Eg1^+QK9-0UkMFZ2l}4=<8nlvNLFFh0vJ4PcH)A+tT1TkvVCl`OLGI0tl&a>q_#?it>Dng>xR4&U9EE6j(rmA5>+c`1X2L1r` zf!yJVo<=ao?YAy3{PKXbB}JjOHoWm|kPWLGryi*0fq03%zTj79+mcMk#NDr4X;?9| zxJ76wqe~d*Im=E?E!E~Hzbp%gc!@2nbIeaIptit+`3XzB;Z2?ItbNqcQ7^knl5Awb zJt&*Nz*vl7D1WC(qL$wUHn2SFvU>X6U+Gx60#A*8W{D~s zL;Lp`|5h#A7ZR=B(CCA2CUPye%vv`|A5a?F6#r|d+ySM4^=1$MZqpk{25ZK;hcJ;xzCEWs zR%|(PEd*i7eHSH~JSw^uxGL}8g_yaXZozlAsYRPJS>tK66+#5T?k8a1?#kwkljVUR zgM8iK7)sfy>p&Rx4tVQl#U$qzVGoU@OJd=g`5p{kZi0KnlB=uxa@Y{dxUsy1kN1s3 zbDR<|oi=^(V3;L0VSMSEAch}I-&617(t#eYe5XJG(D`3<^KpUbd4Fg2C30q8#~hn0gy}$k>rX%YU95ivAF^Xy=1E(DTzaul2(X~B718OFT`q^Ygaxp-LzeN zUHpGs`vSo7KN6e_5aQYM1j9i;J6&9#H-ueADw-JN?XeC$)X$ z!6(FNU6~2ZkCcsGJ#%wCCwzRE(8>$&nU6HAGdhXzq4c!z6UKdc-MpxioXalaTjLf> z`K8}4Dk^kWW0VsJX0rr#G7;R>xNON6t3*OKj+z7p4CB!y{MZV|uvqpDF76%?Bu2vB z)%)$S1wH}|cbj-DCf=W5FcgYwU=MxAKc=r>ypQ{TLrl}yO;`oI_w1L`^R&Chm^=K8 zA}eLZS>%A}4}1TR}QSjOblYB)rp3h#Pj?&DEQ%NqDCp-+#Z5W?bq~@X(7V2vZCp ztbua`U`WLcWv2f28mLFxKA8$Ijo~!XHHK(S16WKOvgs!^sI>aKX`WI?d;~*p^Zm!1 zYbMudeT?={EBh(RXJb{o`VG(-?Vrxh2p{Tbe7+p>CBK;&PG=%}{$$sb&Rk=nGgi9> zYw#U<;&PSDwX{C&NRovkz#nJQwNwwY%H~A-LB`xQ_JVfc!rq-*cm92Fm~FUZ2=phn z8a65J%4>T6?v&i!%73nUiHeJE{|{!Or2mDP$Cl$K_H4X)VS1DNJEaq6&2Fy0@UN{^ zxTE|rL$$vyYEoDH^qzP|8hl3GayO3Fl8>$5{kO9j5A?M<#uw&lS7GL@s5C{xpsI%`IQI(e6 zSd?yTP|ZWZflqq=3i@nEO4RqehV^mq<5)qhOgr@Jrvbt8|YkTuKl16dU=}J(~T6$iumoa_qCv zc;>4S!UI9Z`;}$qtc--oz|wFZD@}F>KA{lQX;X)Er24!TG!A3rWZhz|g{Ic0x1cLP zXQ%m>E%K~p_{+Q49lG&fXc*52@)SEvbA%oX12`LG8+=al#tQI@xEn?Td2E7vvB#pL z#WIzSM9R5gK#l$$eeF}m{``G~?sQH92`y8GPdDzV?Dj{EuXZgf8rKw58CKVFy;B$~ zJa*T7oI6@jGK|IglrDtv6C&(wEa>)EoM{~+N%+vlA#ULBNT)nf)V1$qzHv5~1dvf- z2wcp#<1FKK3!0f%=X8`75fJX{Wr0{}r9paZ?P`lwT~rE!=q0DHc#I!{alesc9Mv!}5H+(ppv{&vb#b$)Ri4=wyi zZ9m8-Y1hs5b65aEgp+|(uunOuv1~tyW48lSNjy^MU>+g2_$`w(I&hn7l;EG3g6Hpv z*PHTf?f7S{;buI14_(5&Ve5Rskkydwr*~zR3re;i=u{!$%h3)WnPiSFMRZfO85Tob zN1qz$6s1c%~^e=zr2;W@XtbRcGiZ)x zbzFw=|2dT(s~J0VFxrVjZw!xfYly6%{9+I<)^69x9#{BsucUfOCxWx~6V8a65$dD`)IA`K79x zY}!f_K3oSqEwelfmpx1u>($0M_EtY*cywL$v_v!)|7 zrf9yB>1MNaLMs|4=yeAW{M}ursK9u*Dek6!8@$9V0=Y4^C!6seSW}U^(`Y4fxYi}X zNZ5gitMc?0Cpr-1M~rysT)#^{+J#1h;T)=n*9a{N82haT`CP?UnicY%bmKAh(>2ih zDu~-YWE#ru|Es-u-Vd|>$b_VdH&}&iH?$C&1!Zysz}7I}-r8k8DEXeSpWXFzK&*pnx*;adO1DH>4iN>MBNfp<2#-gX05qi6BSAf;eOrjm!N#2pfpw<~R6R?ai@0vFoH*j~kwo5m{I72UW7S{%Gu?l5#`#*z$L5{5YQF4>AsI{ zDr{`-)n$itFm3jCe~u)>WpKtf`1JgFCjH=KMXc}r@eqH~D_lK3=+%ILdl9&bzaG;% zlrZ2524?3CD44I2mg0#yl2%+i)08&)18W|Nd&@7KC?nEds7=-FIsuriCBK?|1nB+B zCY8Wbmp(CRBaZg+2en=_TZE(v)JhB~d&W)s2R{cYmT+?urxZRtFvp-IrjewF%5Hi3 z=L28#?cb$wpvZVi%D_$`ll&EJN4NH&l_^)}>FIKG_25ox?3SAAo_4di&gJekGsKQ) zF(C2FFTFUUz7i=gOJvnT*CLb(aIQIJi3WruFRd0c#nPF23SGt5=EeO$gDuuZsk2v$ z&m3|+dmVh{I>p`o`yiDn{RsDA^v>LTXk)Xao_+m?ykL-2l)GRtAz%Yr z_kVeQWtJo(TTnKjQQ?yGy{A$2f&VK~h}o@T8Lv1k8fgY%;? znQ)m9)J{EcML)6)t+E%ou1%ctrsVtnWr2!8bwcg-C7J%kl+euV$BE!ScFPGNtjR zVes?DuOk_>67SPu8o-40(y-@vocHfNeZVEbPN)>cv&{jm2FGc_3!7PVH3Rsp)^bP2 zAEPHfZhT-+VQaO8-=8&^ZX7XR5|=`e12-_`r4d&ebzK3p$1m zV5UGVPTGk#)nV{{j`K255fJyi;a5k)0{kLYQa6qHMZdj4yn4L7siV`r7#$k$5>zJ#cPnSw>T4yB~; zxhqzGJmXIO>H#bTI~(*I!{8F1eKZUgdJTt5T>3ngNNVK?Q{~Do0fsKPRwnf}gU67! zY2LOIbd{xn+3vnaKZ0SgOZ=MZD`Q~VFU>Q2l2R5`k>hM;w8*xDvv>FC(^tmy1baTv zS^MXK<(L1xsxGO-U$~rnHAwQm7afG}w@sAt6yL|-BfaKa8dA;a%i+?MFN6Jcn zl|lE(?J!70lrO0a`wmpN`yNti{dvB=LxqQ2#&v?)Am#tt7r_4yeRKlqJt`2l zo49@_MSfSqJs51$&i}aOxOnnt@I_#2B3*ymKN8POHa#;r^6cyp;fu4-_V@LvUq@EWdGz#SXAFCZzJ-fSC>{i(Y5h7H=V*%~Y0Vbn|GjeyV zM4l>(Dg)APCx=}*I&##T6wlZ$%NT*y_$vWY@cimX`C_Ca-zk9+r$$7V{jjN1|3MgK zFml3)<1+O4ne3p;i*c8zBMf3ZB7SCIT8_ku`Vbs;svz;pbDOWoxHvl!ZmMR&hTD7g zG_bahyL8dd5?6D`E>FdGsE47%%&Scgsdbsx{T^C-T%NR zQ@kC1i$tCzKx`%^;q>@Rjjr>({LUf`ir{RAVJVx|Qcn&-k7ZkaimC!m%~x6BtI7i9 z?nk38a^oLR3$G0>j;d-L)Ddb|l{3EDO;5tx*({gq#V>dwVbh~FsNxBv z-j8lM5u(ez;Sn1*|#N_Tb!I7T)Oiq>aV-q=o;g)8is$sk?N( zW0FtyyLu?2vEBrvG{gV(REEbATXq)MyozMx`h~{!I0BE$vdl)muxc>^of|60^Oon| z^4eB!<1t;)%ffZ(rk_=@F=d|si=o&?!OEN2ihM!s$xKS-mTGgL;)JUpp!Cp)teX2z zlMW+6a33yrve9X(70$VIOmLq+NUKPyO#Px9Wm=TrTwL2o>T~1v76)M}?pX?2Cp>Z{ zLrrxHBB<6t^et`p&u*Zrg0YJ=Z2!`T}Gm~4F-7Ewx0Y6#YHW8(v zI`V3aOT;R|55V@IOwDx3B3)h(@_H%FK!UxV)Tr$ic?`mf6|;6Jc@V`vBi#p9CVH%6 z=!|hd6J=6T`Y+28J%*J44&oPvAj+Zto;SiM(EPl;RLKOHIxu;$ zmWYlwqZ1olXiL5CRS=OE+^uG>qYft;cBChp?V>E4Thv z@Va@gxmi}Fjg*tc@WLBb)<>EUaU zRX$L=OI;4|1cW=zr;I(gMw!O|b;SHE3niuoCUH^E?2cR?)S`nkC4YRNC_A3 zaK434aCQr$YBF(q0*pXrNh|4^l$GboL(X~xG!7Smdah(R1 zK)l{!V2~Yf1=MrtCTY95fJ1NEyc;S8JDx5WlhIXKau;_i}o=p}Xj+EG;XsWsLas z7mSNM_^JNxIn_XJ4m)(5Z&IPPG*;h_&5$E{*5(VwXu|{9M+H0&Zo-O+x0xA9cDH^} zS$D#$1I~Lv34pH2Z?F%0i9;^j4^`>8J-)YUl`|{&C#Q6x;5u5;h(P@a|94S^OGM)A zQ67y_@T0^dBdB5Q(xr+E(GwHX4V5DYono2&f^AcLze6fZaicfjygh&S49IycrrUQ& zx<_D)5wkd`sjY;-_9MxwRcGS38Un~sp&670Q&kar4zhbs(L%iAAGpoPDPFIKbJDpB z?RD136k1vhWM1IO(tYsDO9Mww_AoN@bHBC&(`(5-W^7p0hS zK$RuDXuf9RQ2J9Xz3xW0bwN}Ya8#-fI0RC~&k>h=v9$|++u71wT%KSKon7FqhS0`p zXM?q0oF$D4OXYxNwK0?$>@VA7(k(O=2PwrG4MlL5q`msGE&ixP=$ynQoN49nK8 zH0x4U##}NM1kCstbVCUZ?`ZTavk^m0+MQtM3_{NNJd>56oN@VsU@WFO&~DMug`S34 zEM3#|a8KiVg(8C=&b^*9^Iy6$V0$MVl~U5`+2iTiU*$UIJ@b8{aTVc zwo^Lt5RdEQ*1Eho)aOQ{>4LJWD1+rKF}K=kPc6`};nHrxkv^>AanOJ1{0;O8g z)LFFuujQ5;x|38OLMzRnxB4V&Z66x0P?p=cAj;QSbWZzIW6KiXjMA%nHm~13l3h8E zT`Y+!UXb$U#v+xSBgwAs0gr;IjrAnOw_~BvYmQ|%@XE8_X^7YUJSkRJ5%x%z@6WKt zX@P%aKzn4DtGeWU{~NXGyXJ2J5|R02%A#5v{llRX`wY%hJBya@UDIplGo|wYo>A9%K*3c07b{?!5lX?b&$M&;y0JFF#Lcdj6=aAQmubTE z6x&l8N;VLN*Vx6qNaV_I+8KF+_7d>{RuDX83cR_t1^-9-RQxF z-a{N%utU*+XzZRfU~ni#(-W3$YC$QO1=GJQWP!)l9)5T ztPU2~G={MZUuotT;~rdCq(zouX&_nH%Z2t4OVaZ&09E61TXeCSmD*g8il<*jZyl(0 zBf|^8p98!N53QJ8^GuGe7Q2o6bx6)KGCmrGCks$hp!?Tl&K^2{58`a8YuXc<0da`MobxDIRW%5lp*^06_t9td;tR1FC=H;T6`q02J9p z_MkiZqDBIJJZd06?K8iX?CL@8Pom0}agS7F^t6@Wv|Cu2R)vedL)vSG^oD~~I1KCi z#_v#Xy8Hd5{bjisU9-W>W_vvP*&pIZX|-B*-WrdXDOaZ0Er@zMce@1tE}gj_6^&!M z*>UFK=5Z>SrUtU2pByGjeFs_q-HPDpv6aE2l)O4^(2=ntA@G8jw`c9VQ%%LM#bpyW zm6WYW`5J3&ugFisKF^V6q0SW9_lnGQ=!V4)x3@Pm+`mRA=O!O7DfXrXV0n~4b?b`H z9L3Cm!F~BHQ-h{}ZH-RE;_vDR)*7UAYUBnD(|y3dTdWE2Vj;&d3NMli7i zq^>Xxbl?m_PsEl!R;J6aT_G~J{p6HqiC4<%X&qN5(dziG*-p8|e1~Mv=1eS>7XKRi zi*z0uC2j7Ba^t&=&*Yxho+|xQcn9vTpxy53vkjdM^fi$yifON-JTz`pI_)mb9QMpZ z<_>+dTfLqH^uY|>3xJiWmT1)Lt{Iu(5>D9!D?FO(D$hK>M5^(SXmmY<({ZJErwsc( zMSc_s5>HvXo&+{s-yuuT_Pn7aV9QjM#$UP;j1Y9EWg9%cG{v^ao6|al(@_W_TtFsq z=|CZpc1~a(eykHb(!tuP)1{j*29$kW)+}(EHQjtej_b+?h}Wu#uzJ@lw^_;%=Nj5? zUR2p1D4cg>am3e$Dp=42yZ&U3W!1OCsgL*5zYa0buf+8GE821KU*_oUv3b?7}5-O{H=~o(__msO&-a{QLdt+2GKaYg93=m`yK(#ri(^O?kzsE@HO-E@>UcO{Cjs~i>r8+6Vm5fJT0wtp3!krPEVFd zv*q#kpqgp&H;3)fnt&_T1?3*K{5NS4}R*F{IIUJ^Jo0ML@2* zb$X1mz@OFNkMQ!rSkE%jR~wS@_k;fWw+st5jB~4@B?GkG?%Cey0TN;0XqNukh{is* zHooy)iHMX5P(Qcw$Av;1Vr<#-A#Q8rewtV;7(M|IOjA zO~k?on(*E4QoQw=zQMb6atDK8X zL1&DX!n^zk^Tuy8!w+FA{Ugwn&Z!Y%?Ns8*6%p(KiWa~dtzC+Sf5>$0TSNQ9C0Z_M zfO$hI`Qc?4vDZ^K3XPFIg1zu=F%3b1p^9f1W=Lh~QJE8lQ8mYF#`Jcmto@z@{QTZC z9c!cD*(%C_)iJQUa$Tspt6=f+pOu5PRwe~iVzHc&li;T;qj$!7%Xi$q_((79Gi8mw zqD<2r0XMW8rt5G*jn3LmdAjL`IG?R<=ulT80qoNtevlu}B>5d;KW(4Mq&Q>+@YBm; zc!o&(>LM%W`iaKH$`!%68r%eWAgv+t*6yhipA0W}G3oo3EN`=n^~51HdK1mD;r+le z_@fQmkC;@9%nQGnMGuWbyg<$vMUhl)*R%m;x9G03gd0_ORNS>IkE$i~!_Y(0xUc{( z0m6Gn@!%?{{mh;@`cZYAqm0}Em4jUH0DZdJ&b=Nk6++k_6DL3BS2Qq(sqwE@0juX7 z>u!na*oNsr>DVFil~x^79)D-&E=Og6Z*2^L*MXUEJW!BH%~;2YSx;`hQ*M z$*jb35qsUl z@|$Lm7aK0-KEjyyUoKk!KcFMBMM>dPwSHEs`WGA;!NKKiCJ=mIfxyn(=GoAOAb+@` z@tSn6qpas#o-@T%@*s6X$ZwnS5~Y0yyDgkAv?CnK4^aI6R=w#PZth{oi*y?FAQL>|Sk z>h{|JoB0imQcZ|?;PW`~*Ule$)aBVK)>7p{b1|)txy6H7zaVk)0)L0+I`6ri$z+tu z$sDzCJa=7RxASH7xi_u?CEq#e%1N;RM8%$7mD%YX#V2t<_9R?K$G>wqI29e3?qMe3 zQE(^yaQI;79&0^#74-+xf9;(G(K2SML3WJd?M`% zX?dL;TnVTKa{Q|mQQNY8qpdN;Mzj^(ypz5zO9P4Qv#xGh26I?jK`vyr=9t;()U~j> z{AIv;vAQP=m&=B^><;a8AHeSS-7*ln?5qabTNIpiE3r2;026iL&@X%9k{j8$dRH;> z`LHhGx6GPlY<}mvhejuhBON&;)Uz!2M0?=%T+?7sr-1tMuX&CYUXzOBt`h>RWd}gW zzcl>)Ep^!g!e@*J_YEcHH7NP{P6r$ZzkZv>%!|Ma8WJa+yOY8Da;TQr)$W&fkAwTb zW9$#L?u2Alx8JJ9wBOF%ig)ztpB4|HneRDPzp-7eyhZZ}?+F2N{VsfX37`UY>#|~{ znM%7lHsyW-y#^_O_T$Juk?{9ibA`nSZ#ROR=Hk(cL;pQd;zeAooE(lAWaCKq|e!L)KGCN8?`4cc=} za&mUl{KF&U^3cm{6+FDJWpqS|gWH$YcLBDQFkGiYym`S+F|acu_5q^*M#W z1v}in)~sIof0FHl=_Rz4P4Y?7sbmwrVZ%W)Gjm|2`7?dsFPM?N`(}>brXKKX|Vn4NcNHhmy%i zC>~45z5-nuu^aF@pX}cln=(!=drmuEN$?f_#CHsVa}`;Ib2BvqRF^qMkYRN)HhYYA zbOCYOdpBKuQ4YZu=Z(;GYpyBco%*b%27hn?Bjefm&pt0O%xd-vl3!LQU8d6c6gZRa zKSX0RAvm5m@vSr<+lZ+34ZgCKMtuvazXg3i3^treZ=-z;Lvt7;HRDnyd376ZWtnQV ztU*{@2p^~lo41W!cKM&L zc-C*5DI6BbvuQdPC9#88nK>%9PE1e#1hi>Z7L9K?0X*CvmHqnBvAKHB z!CS$yZqJVFPp+Nu^BRn3%d{QuD1>W1)VkJwYdTSNu;{i){!`H#88Dl7@U9I)Y(od)HFD>}sNn#H{$h8t>b?iI-t z0gg0v#hVYh!0cZOKg{kKwWN%t5f);;dZRiPwlqCi5AFT4hZPX?BZHa!;>7@E@P;1} zCV3dz{pdy!27@2;2ud5?{^;(^k$O>>g`NW=+L&7;osuH+c3%Q$bW&mG&bXPj)qcjp z`E$*Aix=JwAM4NVDqW4)p3$+;@)F{ZB?l(NW=NF%rc^|KreXigC#WhIk z{wWo@(LMg-)B`YBI$2<@A8s@-xlisN zm43TVhf7ex4J@vI#AAGi`Kg=F7hdhqr(|$PC4-y|9q6)pv#dVsCUW0 zQ}*fP(DG93X~B&h;}<($y) z8ju73Fpc9uHXTsi)cUI}%*yX+Zbe~RNP%z3oX@ZI;z%;9x$P*dYHP%pNFyc=XkNXxw?i z0coK5`AVWrtq}6tu-!;qmIg{p$16Uq*5CC$H_Ce86Y!u+D*H^u$*=5LLjD;wv z58+%O)2vHMjJg6C>h-cPgwWMCp_L=Mjp-vdf^_VgV`OU%*BY)x#RL3~5iW}_8Vw=j zSAZ+|i`K@Pc{QHmSZa_l_HEtv!ccZs+?S=_u7krl=kUdp@kiblO~#;coX*2$xh%Rh z3RIpe>|I3ye8S21bzB?HS5mdIj_KN;0UY9P@Y7xMGjlKP5C-iF8=KOFT~x7xf`0tQ zk!4X^9DO1uzd|>25V)!1?`mR;KUd}^P3)h!KAzrxK)M*JEL^8-s1(5HP<%~nQ1)_v zthQ_wX~{o4PQeL04m2Ihyi~6JeG)1kL-KgRAV{(dUJRa40eC>*++hPx?BK+Qpfi}gEi!_HmEKg+UmGYRp_pc z9qV3?hXkUMEIzo&YH`t(m4vw~r8&w?*Py{yy@qh{4t%JXkshmr<6PePWrHvhfAT## z7A}mADAY^ZWo=3<5ysQ$>Z?t^%!O?--zu$(*& zXn-1SF_7MWaIDNHBAeL;!&I|E^6IccsDH2588>f1Bf5ko%hTIza2R$Z3*w1GJk!VL zMU)OUX8Bq-DTaGH`QZg|mXXZzTDW9ld>72sk0>|*kA7-x!WW=NNyM! zw7#@7fb$P-Y}NkNHaxg6+_u3HRt!OZzqHt~eqGn!Jxq)7xDeEwO*N=tt zHv*3ap&=fk&1wF~UXGa136^1r@>ZLUSN^0(S^e;TM=b% zA-p6zAL9e80nC^5X^)m~e;El$wuJ~OJqc=bXK%!cT~TFQ+4bNezH{y)O3mYX{# z4*M|6>`oNQjS>#Kum*!i2L+u!cMemcB$z99vG#^;sfGhJ6@)H>vi-{vkn@Hx(E}At z<1eHnHs=mjrB=E(F?vz=F3b415Eh;5Ybs!$lto}oG92MqDXtS+*+>GHCj5}lSa(t= zjnJSqMp>-*ygmd0UVjN;D}n)){V}hYEWJkDwYan`L*m`4Bl|4r0UIUl5Qv0ILKTsq zV4}GL2EY!@LGbav*8l}g$RI$)E&tSxd*E%;;D&D7Aj% z{ii)D!RgLaEs>Z#5jhjg7`nQup=4li%s2l>^)T%8ZU`@n-NP9PPaeHb#p@Ic^DdxG z@SKViy&q(}O0G^N7nRFNB=GAYblaN=b&nt6WK-BK7!3H+3c;^M2e*2+P?LF_Zn6$>$ka6GjS8Vntz)V zxMsQ397XaekiB}+4$5yhWE)hrE2q*v0~uoqiA zr@m)loLe(~ExD{NM^~squI_`J5M!>j{z%wf6HOmcVKrFUvoGK;tV7gB=a2pjgtXS7 zgwpj#VxW^N3r`*%0Mp0)35}mTEIR>@Gv<0$h%}W&lANK-hiBs_QY{nyYKh`MZsI~E-53GFoAMO?*BBA5$R9eQbk2^w!UGAp9 z((gWRrg~jL4z39G?CSyTJyUEUoA$JnKy14I5)=>HiJ>f6ONT!`TAtqW&#Osq(vMVO z`~OO?^SzPN12|PEo&#)&@_(R8@oC^+73BHqa(g~-mcnI#1-zRRcb+^gkYJ2>)g6ra z6z8V9ByrOc;sg5%sJ`E^75Oj$aj_pa9}m(0cD&&3Cm#{zF^s)}5AxqE)=qJq%=Qyk|L) zGRvAsy&`m9twB8B!bJ##hZBlTbwYP|LR#E*&?V%U0XfD+9q{&R_{B;^0D0+m`d9v? z$Ui-0L&&t(5qe_&TC~MT%+-}hPVw%^^)RGz$Q%{VM&gwN4<`i+iPYveEIkDZ4BD&rtfE(JIm&fyD?E=nzabqy%Ai&O?&6$R%r^omkE}h+q#9E&~w9Jt#hEuPEeO zDnwO*kly}XD4YRxES#;5{Dbjso6k~!@&XOoj^g^V=0~!I5>sF$CXiOc9N(B`S=K#v z9LADr07J0V=b?BUstsRb0^K`{&3FBc=fBj^ev5UmoI>eMr*UrlD;I74f#|+{;q0h- zyv|r|y-vr%M#Oz954$9aEzI#bGzWD2Db%LzGk`%5c22##XUx4zzJRBUp<0~GKg`U2g z9(e)PS}CG-vH!#GQz4g z|4~Cbpm^MuhWcyUe$VVM1nJx_STF${|GR;+gvCwPPGmbh0)!K(N0Q|QmB%6Ri-A;3 zX=@(1b0@O1k^w^H7%Dnj`EXSN#Eplx+ODfQ2I0mkkDQrq^ z#`YeAIYL4SmhSx00;FR7l}fL~N)bo=u9!{Fd~A03F-`m-)IM3ZBCpm74N(5^_5qP7 z^jPl~NZuXSlOKJ)J!8Cle45w?P2eSmUW%Cf%?on&l@8FMmSkqO#fXK>D^QXvTlh0q znrM{ea!KLF=2u$)VIHtN+=lIgq_WA-{QTX#!_JB?9TE#apt>wc9Pzwj2F<30v;1PQ zt0&ldb2!4R#6UIoZt!EtW%B&(1isV!%qB>vj_uX#e;$@sr^Iq}5c$<7%**saAKNjg zP;Pbc^@T!M5Vp~6q)JVkTAqH32pd{yLMwTHam(Rb$3!sU|NIgiBN6_STf1U}~6dr;>x8~=8vg_l| zwT=z3zoycXHdPZ%|HDByp9{@yZPR)aq1FLr^IYD|^W9#y8I4A5P~@_S#~{DQL+8%C z@zcklD~JJo*AFwf0yR;k2p3BJ1?GDnn%)QO9KHSw-+-cBa6Vf8YYf5%k(bGVI6D4& zd?nx{1Psw@??xB`lC({qKiI2bTQz5t&YrAt5YDx-S}@Qs8}$!!$t;kDFIDPwZ9hP?5PvvHqfDQtsvyoDRGpf(w1W?;6UcP*HD37r7ZBh z1zF#D0)(&cmiNReFNQ@^d|&$U!}=scI|S>S84@sVbd5p2j&hD2S$i-t$6+I?!#eQp zz43Mh@jy#YSYiNg6dv>O;F~(|d$G>*05kKahv~h7D`da1KQT)p#8&u>+~^PiO&F<6 z%`ME-GE#o+w`DOGwe5TRMDEF=O9LbU4Js1+>Z3yVT?F~H>CDFjmK)*E z2-x>0V0ov1*M-|{=2=i#0;hCC>j>^vxw{q5!N-bh!ZE&RDI_g9QNkZ8rZrqCWnK9@ zqGHRY1`=f0^uuOor{dlZ5Wb z&DX&Yq4E>mSN?_d!tUNkM%VM4bEy5)o?Qv7Sk%RQc?<)ZL!e9+PuNY8UD?hh5j-hj zdp@oWZ}SCB?C9a9*`#T*5kju#to!2xfx`H2)V$T^&$-TMQVu% zRAaK|4S#~-cLj;cyYk&3yN;X?VZ9ViPu^3#L<5ZMP5~YCYE0-$Tjb%DggseGTi2TF zUQ5lGL1W@|ZJ8qGoiC)`Sm%P^$-aYu+1y>}wX0N|QL&h-aO9sb;=PY+F_C+~k*xI) z`M@~=JXA%nM;gS;isqebL?>v`0mAt$=Q5YSxZXy6pkd#ef_<{`i5wuX0qPv68s*63 z4b>?`X7KtZ$D&Wf5I_l0_-k#h+__S9hE8FY0qzB?>R%GXYZ-Jh%4(|CBdt>anAH|6 z&`~a0&a?zLnIq38dzW+-m*z)?<2G(l;9IC%;EUEz5cwawfEv41ez5ICg)4~Nk6l$T zezlH&-n+T49c||y7WYyb@JMydM8crFd=x!yZ8)cLXgU3jh-!eQoQokBqb*~de_T1oXj!2h%*fEda%9p@^Y zX({fBuiG^_XQidT{IA@`E!bK)AuE_jc$^=hoj5v`I@mE0t$Dp_x1@L@WWMn9cD1Z7`nqEgRIl~|XiUJOR8&LwHV!Q- zS-@pUEjDi7mZJn-qx_@J;*y3f{Pxf0M}Hw}wTG66N1)edK)B6A$3|3ZUaF^*`hzF~ zDWtkzXyOVe+7cjx*ja@%{xD)se6MIY$Va)_zMmHIkR$JTEeQSi6CBf+4gx1(p;k#) zXG}?2{)J-a*;VCapTQ1SgT^gr{xf3dJL5}7JhwFtLy$SCqR%_;x{pBh5Fy-C76vLTYu{W0fI#7o`ahrm4J?V9c#5D7Z2lNwpf$8L znt5VdTL(*2jvSUm_ggSQ70BgRx>^JLZ4O!66Fq4%aLHCNuIl52ANH&x8#DbX z`9^6-KtC7Z>up`OEcY;~Z$Vos#{4nJ_UWSy>lTTyj|0!2$k+e^#Pu);TTmIWjMT~r zLg7ol8yXy4+m%EDBTp1Bps4)hd-euQ<3CFp+&IVqDBpi0?5{xm@;MO76iLAKBO|iF zRc;Ab0|9I0=KVno0DkF?q<^|sLh1^}-jvCgxmdILbpn((9Us7k{a04e zFFAzWP(nv7WN-uz0B%HP4t@b&raU1)3rSr52EwJ$ct>=%YpLRaL-~)()@Xs^lKbqp zF5sSjSkk`=!5oCWW!u0Y=t33W(ym*WHs@Z>gX}{)AXF z-p1)5)5UPe!S}Y`8raw;CSkx?KzpKS8SSnV(VVxfdgAJ)@K*H7 zsV?UGc;v%&yY|NwX0)?YX!$fcelQm=exl4}4zLQ96w&Aif#`^RLwJ6)-|>8zG$nw( z(f8%IFa)NNFTIfRbvi>VK@9lYYXd;5v(HcdU6h7iW1^Z*s=b?&&Axe|ALKE}+e`V- zEn;8eNY!o8@JvVn`_CgAEP7!x?rrFQ$pA=rCTWXDW&DB*OA_ub1GsvaS?KOl+adh> z6V@ASgd!zBPp(Eqvf0n^mIx$5jB+bu02~y?gkCtt1@J)SZ|%}$<3-ypKMqBOK4=4X z_F5^Q7JVLCKDCc29qH`1D1FRt3-CXczr{9_c1s^Byy=;U&kKl|e~8G>Q>N=Fqz-7d+5Q%2Nbm3?!ual<&*h z-rq)pc2<#DfvnGKGliLj3eaQ9NaE|Q?xL^=euN-G@3n+5$DUPyvrd7p~p3?Jk|A`tqtOI<8vlaLVFbQX1OqhaM9MRW)nD9}{iImX|+xfJQ`9{}E~caZ%-876$!8w8I|R zAnT)@7jf$>boO^)a-YDWSH2<0gVJt4BCqH-bne0wWV{m$1M!Mk>qkbQ3fW(e?&HYk zSL$IDpa^PS=KFo^xf!@LLpR6nrp_PAJ;K~2IQ~_<;>i&(P82GIw~{JPYbV#1Y`e^S z1J}*8#G=F{znF>8%NK#p9@a90zLOQSIgSZyf@R}<&~$kfp9P^W7h%m@V%5daJ3vEP zH_+x`draJlGay;LzK5EkPtq_%t_F&m?ZKr-{e}+ZfK{(B4BkjqD^MBfdwfodEo(-T zlfp1K#guaFX&kf3Hk)#t#YBLiED0?qf0aI8GXBHW`WQ`6c;3Hmca;gzIyiY2C|vKE zOJkH~Xb9ZPm(l!Ip0enZEW86U_MV40X`v^==(N8&{xti!8 zo<`HXP&(PPIFyDt0I~4?A7EhymN#P_1iJebt`tI$K%3dAPO~LTid?!d1sZ>~nwBl} zOn;zpf8eLB4q`x{-W zd_?3#L5IIcaIVdU(;}Ar=$%vMD<*)|i$c@zOWsV($6W|4f`wghgPz0d%jhs{$ny|N z;*8Q#Xw@qJ#@uR9D}Q%zSTedAVQ8}2#}8S#TOa)dKygCt#qdu_rQ16C;IvDugJz)_ zY_8Nl7lg6*&Frz$|1Xw?tL}y#>#?tQp6(#!9KGVIk7T@?^R?0}+_Le`oA;J~1>TQS z0|w$PL)g1ww=3NJ)Ef=4guH*(A{*pnFQOr4u-afr0Qq1eL`eD$g@D%7#-;lC)FWf# zeR_VbJJ?-@I%j-~K7T|DDY>;*m&u@~Y@Pt3$=Wq2^#43EAuU%)Y(vUj>iptFTuY?r zS0BpSLGWlgiXEGR{d;JDe~FgnP5z<*2ajzCp;;cxux8#vx8lT|H>{FhtqbBOq%e|T zRGEDdLQG+3or9!*m=(%xWA?fc&6HSZn;`l7)0V^08!Nv-9qs~LlQwdDmTyc9hu0-?N= z{t6y%fM{F`{d{UsaToc8-;;3tFo1?326Q&2~PmWpzPrLszC zC4Hjj3@2sq5@LP@I=j(}nG4)MjupbP|KLIw@6BrO&70BcPwP$yR1ylxcdG2o^mg%4 zkr%KBaue79J@9j{q-;B6>?g7i0sUrD%?F1|=Po&4K+HLVaI zy;zE33{J6SNgb&5JWv75L!UaihJZrE(T7q(YiqtjZaDISvjv#|l|vl74#IFh+&&wp zh#pfx$ulq(NUNdMCdyNd=R!Ay4@0K~eNUOL%`|-k+Ti7}zYa=aE<&1kQrONeMIk*e*wm4^boS!mMt{qJbfik1>Upth zU0!L%AllGO{&o=400PL9Lh-rMvZ*Z-7kvjWc99!xoDJ)t(h#>MX)VXR-wvBI?EAet z47AhKZ$0KxJ>}Snbh<`{L`z31)GeSVA)f!rUWd^}8v>mk@IAa}(10%ROm~@giP^?y|CV6n z-ip!DgFoOlLORQKJHy?jrQ`E3Rcm|KKH+Q*%t5Okl%5t<6=tFbKU9q1Kq3EvV=G!M zDNTlKICdBuQ+o>fpO;#fccnz1tv2h04!~09a}P6J#Bxu>o_$FNjI@Fo#pQ{=-U=@X zRzIG?t`LDBrWTh$1OwRtg@r1tbc1RHSmRSvvuEE@RgwNlj$a9luSdcr8JNonFjT|S zPlp3er;JK&uBVz>P4 z$@fZD|2|h~2!y@wEn5WLi#I#ER|40Goq75YG#EIxl#W2RwPq_ANZw|QT;_mEkQNvy zRZG`i7Rr`Nn4ik@yr*l3Iky5WkWs1xbf*kG>jV~QCH0^Q>)W!`<51O)1bsbz`7On4 z44!Qcj3AX_77eMPJ67&AoP#MISH#GV$ZbAs=MT%mgqJw4gugIXj{*OO*ABpA5Fu8< zJRtOhV29BEL;w7*{x9S|4z|>q%}XZ|^@2k}_DEm3e0lf%hzLU_#iOxD{n9hnwc+i6 zUt8L{yLVx+*qBvHCr_R{tf3*6o}O;{%?m-0P=*aH^1*`-A3ogLv1xX0t|>@L7$0$j zqvqr`cxacyTtw&|d z&hyzssm&W{ZARrSG#YUn^LpK$UrD8vx&rLc{cKzYxecWtIpMK>>ihWDlZ zJ$cBt;u2R2;V;fS+A3=z&*=htjxL<~oGh4Nv5>UJPKRzPwd^VpW_xjl(@8CKB1vMI zqitDP5%=#)@&dfQS!t}Cf&%Xrw)IV%X|hIPE?2L6ydWVZB}GzFGNrG~8==MGT8yM3 zA|i0}9M~y^dEPKrTQr0vY(36XVC#x&-5hzy$LV5&``B7EoWhK}ZXbojqBQVcRm?HF z*{AgW($}Z+B$qE=-Ui+mRmz@MJ#^?0eRejO_mHcirKN?3O}NEva1B{ZVYAtKKBn-2 zEH0Nz9b&&E!!Q3;bPOHDFIS8w4+Vd9pR2j*@}#i2qz7}k^u-E{@|^@&vgVz+)YMdc zgwS4dKzexH2^d#hcGFAm8w)BzHEnYXlOKjX?%cT(V_)|C=g*&e_0mbJN8#Uo6W_)u z&VLH8Zwd(s$wF5@Qew5gC>iskIrXGrUqk|%GarIjjZfRx^Ik1L3#6xkjor=>A9nX9 zKtkUX2SBC@-I%HR~iO-Sn%lRkSp;&j_diF8dz8Xse^aLR!Y zH!lNj98ytH!HzVijuHq2{pF9~2nspjwEcA#?A&fQ@F=bRpFeKag)T23aZA5*EF1=nVRbBS`%akxNfB6nJ>CV#!70VkFxFyMy=}DN49*27`zjoDa@*fx%Hge;} zuk%&KtG~9jnMz^g{ShevCxJRQJ8Qgg-MR*@&V8`Hj^%VoS@Ue*nnT^=3A3;O*Ih8b z2?isn`1ZBn;9w4X(Uoa#E27~)ZZ%4z+kycszE}jWMt_)nP8~=7QF6`nO8tA$V|x7- zs{6{CaiRM0ze(dP!F(%yfm7O}AbdI>BPK4Mtn8#4@TpP-Z87fe zgwlA9wiK}=Sh!}K9}^wjRIz$C7~3>H9m@8|ri4_m0}8|BXAm71J#V?*waT>XcK{QV z81$SGN+=;jh3kit*e4%g7_k%ubh8zkb_=0KUOOyt)PMZp*g}q{pDMc0ZJUzp&I@2Q z|W$7iswPe~vA%F>a|!IB`qa1vh@E8-NE% z83|Ap@k{CzkfNFzMaFDKbn@D-pmWR3#Ah^!{7AyS8S;(N%^z~rrG;FIy7CC45>r}q z6jRfdWcPwsAm9|hGuFpugOfFU()LDgXRuff9Sj1G>8*EI<81Hv!3kXGltJi_oI?Wp z67w=x?#2f-*ZiToyU3uo{DDD~T`a#lxJS*VUz-iq0f(cnQle%)A{CjlG_lt|;hP?R3s zsvH}!K_Xb?B{-Zo))C7Z7YiFX@KMaifLtK>M#kX zz(5TMQ_EX}PA!tejj->KUFQE0lw48(Uloj~W|`u`Znh72C>O-cf7*PPJI72?-b!fQ zBLYA&fa8zi`fX^Oj;*{A*be_SCF1DyGu2^J^|FT#9r|_cG`Xgxrdr=$w-2FSTxbxw zwmyS(@&<$!wao#Xr2g%Ya)F)6`tSt;=`35fU?$P-vS;dAMzQdUrm)7gc1&QGd!clA zF>B}qJ76!?;W~~#j_)$WrZ9q}TsS4kTFZ;M+`!F5Qlny5Su^()%YLG}%)6e2Z+3mx zL%%+EI2@oVX3S5UPe-P=?X%4Rk0auG`QvFXu4Bp}z=PuzAZv+9NIZrpp5G|=srUKI zmoHlpLA@;Qd2`N-1fA(%&dg|#gCBo!L7!ESl_gu1KmjKU1^N&b%b^Rng{$ypr zlvOsoJLby?Zukd8vH|{H5aqBp{@SC0=<|`am0QR%)_ZQ8etOxkde+TXP zH|tsAU+6@hl<@g{a2KUqd|zK5%0Xb?Mwm>6+}<=GFm>SZiIp+$)hJ@erqOc@&0ir6 zh(eZWK{jsKpg3Ro-~l9wmEB^JlF^U_9y}NUsGFumY2F+;m#hbT0J~Z;amOM6q)2NY z1QSYhcJ53>rn#w|GjRVH;4y$^wjHd*Z-I*egB~cQ($cCNVMrp44Y-z_9RoX-UmnV| zQ>2dN!_-DQ(L4rO$o!1MF^qpMGZbV9EN}4J3>-pelOP_`u{DM!2>!!jRs`xVD1Rhp2xObr7z?k2q z9QSP6jtZWMb&RY03%N4aPeZ;h0y>vrOh=!Y>OKV4qnm>87c>xl=;DaJUZfK#T4CW9 z>N)`kJ7BUegr!rsSYA+|L=f;;N);YcpC4{T$cc{H{^Q?^xZiTW25vmiqyF`{jSBDtNAcbA&f@30!DZ2clPT53LV{3|)+*w}dFZpABays{JR9u=( z^!{sXabda*wl0d`m6{+(a0gd~WE0%?ZG|$mq2cSy%uFoL$kFA)S`77)oqORj*k-CNk0nnc9Mif`d7Woph%QtVTm?UeoJTHzVThxV|`AHCBxWcZ5Wxd-(S@q6d zMCk5qut}{fF7ybCrr?VFlG;3(=Z%$v-N7$$J5sbn6xgX@feU^(KUYSS03l3LhvhO{Qw|5y`k|)nZ~PSOuuQ)3s~+0bq%_x38E^>%{vBfb~n2 zn8U5nym2B{nS6i^UXP*|f$sz3B(hauHZgsI0YOdjff}`GeEe9O8RgT7mGYu;UJHRq z#W->T|J3K(tdK%Bf1aJ-h7@w~$=pM@1p3${-8~h)0%(H%;uG1EIKhZl8z zi6y$|G7`A>W(p^i@_7J9J3(0-VBdsS6YEh1Kdf{}ceUI?Z*e>R(ioOvH6gh8eBqG- zo3FsGuwt@4Bna}X#`+XzJ#tBcy2il8=~oMVWS*TaZF+K^n>0!~%VM-GHZ9+Nk#huD z5uamGfW{LR3VkC~1UQ3!*sNDteEh!;^ok3vJ$_A+$*RR((=xRcdp#166W4UW!B5@Rn*F=PaQWlNfZ#;>8q} z=oiZ2^m*K|Cs`@JoJjJ2f6wvzQTu(&6lKMeo^p{N$HWqkY!9AbI3Y#JOk>O~+zjQP z8U8n{S6QcuEN9M<_HxWWhS#ftWHsqv9sT6LC(AWGby6E}O%gR9KHgR+L4ItWJaB)m zVAQsc&G8p_kZwKHjM@F5Pa|acn*PSWxM{bDw17TdRYi}L&@tTN?cL`HEV+#5k!k<% ztN7}?`+2PT<#vpAS!8_on&)PS=)tu8M}n8<;53@qnYQ}H*x-cWd!N0^j*i4P9$`kM zr&31>zZQ+ZMmW+pt_xkhmXtX_^+>W;T(bimT=GMm@r{PB?Xmlze} zjeWUM56;G(I*{?4#jW!kBd_kw&rcSs67wu54(b_MW@)+k(|d$vhh>mgw)q^YPe|o? zw@zda6kX7z>=5qlD3Z5`mC<)BvYlQ=%fy_s_|~F3@>QZO`OvH7o`P<*jUM!r{CA`5 zcE8$e-vU^CW{d$`DstM!>d)fS@Rvf`}NeRTA$qKW~&Rn;pU>w@bWO&*piCO z&$(wQwh=UT1{+g%*-Kz?Uh8HlS795n^cuyw1;G`mQ4^`oGrJat!>FE~+eJOP+SNB; zox0;?_c^_Jt~FwLlxEEJdM2DXa5+}liHqd+nvu-wg=*OrHEm@|!Y!_|(3FS4I3(>dju;X+^@e#-fB`(QjA?^Cj5_fdZ^t}Oe}Tt5Ra94 z%Ji3W6zdT1W@fI=B=zOrQfvuGvVAA9+MJ1?HqWPS;C1)-m*lB#cV}o3x(BqJ!>q11 zlf2&UHBNO zF@zAtJLn^Bn3#ztXSy9czUd2(ho!H_&O8oypSFK+bkv|@gJW8>pn zNz>Z#+LXPK(gtns2@f}!&=(de*a;M^)o&2W6#v1$x9fro*#}o_%_@I}yttv6JgDm;XF{{tG0my#}!@rnO*35=lP@ih$sq z;rS!#8EywzefVb@(R;~7cZ6TeN!&9Y9V(W8ref?|)q4pa*09(1Dm_TCn>jbKYYRv5 z6a9k7X)MVFD|?lf9m(R(&Tl*2(z$wbw$k;8xQA(H>>V5!+AXwY0NGEFq@<)f-f1sO z>+kXh_mhtV1qCJCDx3N<6I)h)Z!gKwtBS%DaVor@ufCh-rQ%@2(^ zqp2$05s6JdeR_e}nt!KII}CDIF9R76)a?ov7GG7r&d* z4b!&v_V!ncJH@e8$CMEF=|9fLIh!p8%0CUOwxy*DM~Iw0kL>xt#!hM%OjOlV@zgly zglYoeYzlo=j6pi$sg_h+YRuBdo|l>8^ICT z2zRR}FH-=l?uWT(n_5D%XcX)sgI7V`V#MuhxHmkIUFnNB*89Bdp767|B7c+E<;ZQ@ zy9-p3Eu_N06xkHIfjDWFlBJTAY(;JjIww*);_ku0w^v;Gk}O^eUbGu@Tw+2$lGy_(qK ztwbIFL?#=)atKaPF&-kvE3GD1BLe#=ZN2aQ)^SraGdG{(GChx2h}^sDSl*Q-kRHZ* zD`rjGzd0%zCmEs1)Ggav*AGAd8deM+!S9d!MwsIQT%3W$vZ79((hXY-d@`-3Ss1r> zx`vJI^eRiIOC*IU1?7vmQ=X#>Pg+{|AiD5TJ~QcmEZonPxb@HDn_L`yMP0gQ?=25- zDSNnk&2}4VEU%3s(}67s3jTPYyVJwQAuuKD<00{e<@g4qxwvwce<{uQWC5#+F?Y$a zD5s~HEu-bhxwJ={KKEQBw)h?RTBPmvfvHei zGF*G03GrzdTH%BJpo78K=E;+n+|u)~u{pq(=(H-seSphvVOB|&1#`ud|X4;$^I zxH_NOnS(xn&Hc2<>9Yqiz7}0Lef}TFR~8nbW~EoIT4%m}+s}CQxqjf1-ss-pT2*>& zB3)zH_jY*`z%(X5kznW}c7HbOYETM-`d`TfHU%u39U$a#X2LF@w-x1*prQsE*cS`f+NQBGn<>*5um5QT5ayQHA2xaN7KQ09isvKwbML#Aq*s&S zrd7Qrp|k>SpRGxZsmSSAXKT8ZyVJn;ahs5&eO=ttg%EaXp?I4|Knnd1Fq$mL1qsT+v)QHfr>H1dBzTT$mZhB&St-T`mw|w69uvk5O_o`29tcAr3VBsTg+O*G))@xGy_DbK`T4Z0U*~#)h zn$b@4j>FOlly1x%Iligg)2$#UNSX{+OIzXO6UloG(w>{m!m}p3JfaAPb_Gw z1#!}?5aT*%NC2rO9f(YOCb=5%Bh^i7GeWEGYPPBs#J^0B(;yn{2zX|z>}ivjsZTw? zN&L*lGHge*_}spcm~C#J(RxXKAGT1{P6nOWt-da-pp17qqRcDUSA{FLO`(>3R2 z6mr^WxH<;2)FDUD!n+`cvHm7k8u0zjR#WaNt+E2<+3I?O>>QpQBDzYeYa6d{S@@_% zmq*(hsZ1x$_xbsg?!=CpM@zYL;wv%F20oPX_AyeX5;LZpvpv5LHauV4c{I$3lTJ_7 zro`wnCn8np_DNVztyP)a3cEn2mcPdwuUVsW9A~&=SJHo4T9TxCFUiHn@vI^;BO!32 zuCsIWTg)pNf3E!}`ZXr2Hi;@NPA*Q(vD#0);VWE%TtU30Ky|&=d1#$;apwcGvE8~@ zj*K)S*!Rv5QvUqB746tQgDpY9!Lb0J!0oju&-O>6$*kw9OE!TJ!ZrcU70ePo^p~J1 z&z+Y^+|gk~@~QV6LzrEUoa6IhqueWB?d>WqEgWz(;d(|JKBLONP%`xX1m=w^c@=S= zV;+}*(u~`?p8!uWh;zOZqiW+f;c4TpD0xX>#tplc>`|&Yq-q%Rn8{IejDFw9&{gBQ zJX{FAI17fQ~#*0y_CTn7&e^UaH#7f~pO zBVK9+^eO468RoH``@J!?S8{wj9aUehbjxV0bgzMQSy>r;M8?4Prg70J#jDo)3}Vm3 z$^cFDD6^@ks!|SC2*34fJ;t^=ZU5D(Uf|sNNYoP1cI`cYy;jBXDyogC`n>&dRe$uW z9&*uhkV!Z@iZ~{Ph;l1JuQ<3{STIs_PKE&glC% zH`V7-)uDGY-@oUx%!bb-Roj1TCEXuIZDQO#Q-)xm5gDCt5Z9oZt@Q1+e3fEI^r6Zgf65X)$}wNH6-pwP;%sULwW`wt z3art8ZWFFLM5R3eB5UCEe2KeDYX2p>X4!$r6KYA1zg7PByxB$xGM9^lfC;rJTPf3H zxEm_$tWp2WQhhzMdKG$yZ~nlIe{{4QW|B4uNPAvo4wApC8TqxM4rzBq^7>rX{A9Tt zi{$hfrnfqtco$0;6nk0J$!_$yK71w4Ifs~S73|?b4fwT=@BH2Fr9m4*(kHxESi57JLwjx44<2;W`zak`qLn0LuK zZne78;flhB??ud{3Kgl0^jp2w0oSTdpBL|axu0CrH$gXp9*7Xc)Oo&Ki%xJS-E4}#OgrTN{#3x4?v;C;XU6sCdF1pua@+4(-*f@^H zOD(n11^?eGKCIt4<5mxursRO9T$pyOUwuWEgmqkYk-R?r!2uGkvEp*QJCfv|GO|l9 zvEkkdpJ7xy`lwMYMX8jkx@M;pO0#qnGYRf+=Qa`_h?NLzv35l)b85+;TD%x}_nJpt zPL&efb1RPDI8WI@xZ5*>CH93Kbue0sF)vkrzTEC!5@E6} zto$HBY*)n$r%6X&tvh54@p0gMKP0gP_}$-W^x-&K@R8Qofy1-5@9J#Rgs)^Z9l!l@ zldCrded4jzY1*E*rv^tf2eTgRCH#GA(+Z!*j@(hAv2AO`Yk9K4=?%9vZr#`jM|_f= zIVX6iC8Z}BjD~m^VOQ+KT-Jo=PW7#J{v+O3@fX(NYf-P?$Yv80 z6N5j09{(ssc|V7B00adG=6HR>giixlLCtI#;uLy-ej6BI(xzqc6S50@3!vV8;sVk% zVKz%CCT)orz3)B4c1uX?OX{^Xf-wXVUgx$}^7toI$YP zGHi&64~GxG2l5ZKt9L%;#+aJ^d6|tHH}(gm1f~QOF_)dp9L_NGj69;)ZES(euDLEE~zWR@`7h zf}m_46csF0lZfu*=^=U{M{M7iXys8E+3RFbasd+|F;ML_Wt*H*|hC z@OEHjiw6t5a=i3uRIG8lJDWVM4N8lol+@$8>Ad&W^lubBfjl&aT=BF-gh8$-ycLuZ z@H}Q8qM7SgSYfnj#njJF5g$rlMn4->BXv-c7lxm3_mkYA+|Dno9=FMd z8UOlu=Rqj)%&Ci&I>{=U6YsGvWofjkW8RM!C8~Lcxj*v63+HSgos~fGzZMx;SY*sCaea`6?dd^%AEtsUS;Vru~sx-7RGe?L+rDw-h`(G}A_%jeWg5?96?wVqTe@-D}|cs|z?dQH|oq@Y&hz z5S*0Bkc&){^u`l&-a@r-OYvxX8NsK;X)LI(#R}y5vJ%aXs@{Cbg12uk0A-oJ%Os5M zdNx;S9JBIOfqArg5^{L_E4k3g@hmZ-P9W}gBP5vX({OG0a>PL-1C%Z?E{?(CazUH=uTQXi~#m`eW@j@Mq)Uh~N3&fG|A>H`IY9P(C&5XpW%EwQ zF&I?Uk_w(nuMFL&!#U|wqwS1#0pYthv2?-Z%gtn?DPm`S;p`~Y#rh&Ub%B%boYZ}I zhbZLEsF~p-jp}jC!<1=TCY!6Z@qXX#?-NO>)f2&f*Mj9$%DkUd`3>toTYS|DE}&lj zL3Q}@vw>?&EaDLoYEsXu@Jp@N$O~#%H(L*;SHQe%2+HiX@-z8!P5&F8xmBC)Zf%(z z4dh&mP!Dnrv}Ta&oFn z_9W(KH=6r4U9~IK?0FG&vb%VPY0{#)wF9m8I#r`{2Fc@b=I#|Y$?}G?S?fVIO<(a>{}0($Z15x>I3AWFq44pWUSNm`k*(= znb!JYkzTycUeqY^(B@vuj1vL8-yC>ic)-6SlgtY zb4Zit*yPnDuAlr1)A&Pu^pEoxZ&4ZsV{!)3O3BP#2xtK1)X2Ox>i{LZ?m)PdSDyw}$&dv-m(V_JLj zVJTU2q?93i`?Xh;8?lblG0XYgqJS0ug*)|=OMv9{Yt+wDQ6695+1S|YS4NB(ou@b| z`|l&QT#D1sHTFXqLt6)KjwRb%50k3gU^OM~n@y4l_Yr}uGmJgehYyhjk{9dT`GMB2 z5!c;q1F1ZeCi;{Wqx*H*7KkIl7y|Jh`V&6lQ4iX2@btJMWCO);XbvMxzcm7xz35v$IJu_L~r4xDzfq+-3P!-Jt@CA4r<@ zRNDni@`vn>pcFEL!5Go#d=$l(xo~O~mBA&1a_5Yp0YgP(rSUOoHo*?vtrgupM74^L ziMg`QXo0vI319E?KCXh4OD2M$&%YOBCdY?MB zN%017qke}mHO5^l-g>(r8llj|#u;0#6A{>TlSjwaJ2pCg_H^_&^^Zt?`1 zn@-x>v?G4l=X*)?TxIuw>YP|jHn(nYm5+JM%C??YNj6SSni;f{1Kx>unBopsHb@?g zHo?ajN1Jx{lu10xKa}9zq2(;@~``H_pxLrMP~4a}$#9kGpP@ zIdC&FIm!1?L&XLw-wm&h>ZZ^qhjD-;DDTwRT8fJ#e!G!@yL;4;lYOK`>g+8&X%hrW zFz?|%`fWcPKC$WZ!+z*3u5mq~N1yLuzQ(O^6cxb?N2G`*;-(e|kbSOr{Bc6#FGv~I zzf;DDX>B)Ask#(FZ)0H|9Ug?EhfY z|2Wd`b^fmd!T3vr`}b`A^ELmKm;7t}|7mOfuS^C(jqrbEr2jn7zt;cXW&`BE&-Iqv z>i7Ep`)q*x|A>|Uia7Ya{{J;9|0_mXa`OLepg;eo@aId7OndxrG6?yBUTbjLV4TaZAo;4JQL1HlRI?i$=3f(KhPxVyVAus6T+ zoVw4cEAKt|Bk$JK&UdF~rh9t2XL|ZGwXx@jHJ6A{u>gy$y_&jf+L78(bn3g;@n7F5 z1!LeiDp8nIbDr`aH74UkHGL`f=ME_H3MG3!+w}ao{j8pioxGcMbZ(etw9m-jmy(DWzm*OLpq?NLJZJ3q_XPas z4;W|9^uU>Su<@Z;pml%EuZ>FH4yyzNH#g%0n~_FNz@IK!nfP?`*^X~GwvR05`N_MDBzniZ+uG z_IS@?N_AzY`#oFqA>?ZfR>xJQrqT~Qq`u4Sm?!N6IeeAXGqE{8H#oaXs z-Ldso+H>@^2<;5|d?55b_w=EQc3`h2SI5256UmYDidC?Hx{E6wgEA<>cB<~O;XGNq zC2fI}QztaB@8OM_G+;QS71&Jrf#)amC+EbA+|!v|*zFeQfR?i;`#N)S@-(joeg$a5 z<(9ke-~lco^);z(_%+&t;M+?$Z_}(g%9h1!4K~qHDSeV49iU_Qfp9?nOaRd}PP ztN2~Mv3%5)y>VVf8Dp|%vTjB#sDgMsum~Fzi$qb---`{(H-0^MalDElD{-f(jDBX@ zrG%Px`x9l&O}*#!nwpZ)QQIAWtk1jD{8WCsrT8-V`Q&&FeM5pnhl0MdQCfmlTB-%< zQcvU-rND!uh@hR>)utmGtDbd+ClT<8?!Qz`Mi^|Q;v*dnN`Pqgzh|KUwh-JCI|*C3 zkXLBz0D>9-3bV3MRr%e7bhKO+r0gY3W(>6wY}k2IgIagfsAyR=SdX8Ri7Ur(Ob5s+ zle&HJ6pq4Yq||*okmBTe+HjHfec$@EDVNl0SN-ZN@rm2RG&vq)LmLgX7`;F(Ue7&=t zaoyWjx`6^lNUp%0f?M~P^BL8Z6Z4b{YM752HCD(nHkf=T(%kqURvu6YqwN%uRkIK2 zxjnkVcA$M!-P<%xBP%~4CWY-TJ9nU~7 zE-cEcJ^EUL*(FFurHG|4V)wTUj2eLl;aD)Ncbj}{gr^PFqg&qw70n0<542bQu;Gf4 zAxcBbQ`-5M;-}GBgKC_8+QgA$B1o1r%9AAju~Am0^MSCCs9i>FA)o}|7MkZJ=c_^V zo}Bq>H7aXXKc-6a8)Xlu_ zeIPcJ$s%!9!;`$_frH;}P=up&QKU9|p;vx#oWOrdi864lv~=1)m>h_d zE3I>L00qkic}W{CA^kug@4IgY7Kfz#G~LMro}jF7CF>RH7vre$offgpVHKN)1#VTZ zc7W7akqdlHr$F30eB}J86T-%MRK^!*^_?5bD7{oKaU+Z=G`9iNT4K`nuN@FkC|~!` zmQNEgTjBvtjp-tR|3omB``0Ce{k;oc4W=>q{w5rocsbZk)u|bWLs5qDch%7?DSPr7 z+b}Eb)g&!?vflcz?rz^}ElfrTq7jH&Gn|Tm-I>oXz-9VFf>Ag8PwN;?f=Cle2K;Jy zY)sp%{7A8vF~tD8st}8_9s_t;Sa;a}IcM7ORFsEb?yQ9zM%CyG#$>S$c$3BPh`VK} zmci~A@!D})3Ab}~b28K9)vC*1j_=YiS`bi_moN0T1n&gZ2M3U+ies1$ia~)+y((T| z@0loIv^MW{Sl_ffd<;Q_DxoUX=M3;B3MVhIgP+^R;^zbY6>S`Xvd&G|l3Gb+YvIbS z-lJy%^1~r_N=MK$j(x?(uD51ia~IwPyF*=%K%`n0oj`_Lh8cvoK9MfMeWfiBrniM)*U z(vggV^}s-03w-99r9IyJclnL(KbNdf0C7N=qv&M=g zoGbQ)zMOfhtc3W-japV!@WMzLL~%WD3`KR8a9bJBr#A@WbTw&aqkLZN_R@%*s?3K% zTI*ZAKh@-i{Q4LK9~qflXM%b0Pqj#uIo~4LbILDn72YN?_C28jjx$w4Tk9W3kp=4wt4Ox% zzqX7nsx4C7{QQ-?BBP_IUc_*)ccIVFSG9moQ1rUDkBZNwGgaLECfNe*rP!wkl_i+1 z9P^t3-$G}F1A)r#7q!|;#>bdV(r=BYl3ZbL1E|EWFWy`*2Nc#oGi#N02-|-K7;3U3 z0fPhz%J1?Ifoi_9QYt%l^5AZXC0FZP|B8?UE|56|r09pZC<8uPP)Tm8nq-GC>oG%< zlDZwG?90StBfboWVB)KZq~CnxBSc-Xzu&0l*AUj4)^Nqr;7o|vyR9qb%26QJ7RNT z%w@wUJ}0`B?=H?FBIBnZ!61*RX`!f_?w$c*?n6YgTwFn61Z{EmUf`IBM|OYYU7DJo=Z2+d zaXA@njt=yTL~Y@c72$}CwQ?pQb~@4 zz2VEel2A=)p|}=8W0xCefDm3HQ~vHT{M~MP5_QBn`2QwqjjXlDtjdda&}NPWiJjUU-rl zjlQq2{jg9fZS_Jll{3Fh5-^`RuHQ|tPSoMoOLbWMaAaiK&(MSSVDOUm6TRCO(}heE zLvpE1+wY(D@c8O89?}v$uT#Cp$~NU zk?EV&D3c%x11+kP_E0}mq^DKAK6lf=1LBq+MUN7 zR_1x%JY!UC|I=Y=9eB{_yR%=nPijn)(KOEcMM3*ss#F}qEk-R(XoyEwL6eiFOq06o z{e(I8+irXeiui%Mny`zlye&2}xA@vnsRR1NX;R(b`{Y~Zy)=QXbEPFauA2z^>QZ)% zX+{m)=q)jpp%>xhU-}In^;@I%80z`%EO~hmE<&2Rok9w!Ld%)%MW;*sn5W7qCy`&UBSx;N`;2xbPQ zS&U*{sk~W;_+^@hR*M_5E;vH-TC7>rYV!c>P@Y(l*U8@2XhnehVS$<5$tXYKa7u}6 zg-{$C_k$%DC}3?&IUXnHLc<-SO7Kdddag(v_v@2caIuz%rr4!=0c|;<_pjcKD9fC^@6^#9bcm6Z?KDaJMgmoB^<{5ozQw+#y>AdpE{C>)*=6I%h7h82OKGO*- zsF`|JL@`5$2eom$n#q4X+9djpA(X_|Z8%dN3AudC=>agQ`}9@o_ikzOk^*U_I3lB- zYYl(@1dxgj-efo(kSV-=_;A5sQ>)T`el~MgYBlY&QVg!NlAqgIS`$2g-(y26meiGM z&F3*fSDz{4yWG$)0;k?4iaR5K_0Ymls~z?_n5u~)P8E^A7H;S&lrQh@SIDB(D{Bjm zj1g>S>X04pZskP_oM_;2C>p%F8wTsG7WxPc12tHmMh$3K2ffEAhS?Ks z0WZ`6%z>3Q=L6@~8HF&52q_YDGu-zyZ-`D$n7ztki#sv$$fZWe){}o8dfOY__QP)Y z+7|3enu1`hG+GfWxmR@+rXxxvcQ48xVqbicUK*g?@{0|(IA47vaIvOdreA@ko5(FK z`K7FTC{n0C*-V$6wvd@pAdb#_EZJmN1a=8BrwyM#Y{*YjVn-_+Ugxy&Tz6@|Snk>r z?Y_aDJJoPVGMHiPzdnZ?C$%#<7S-Ki!?MHc2z8v+G)$9C%z=Q%XOLCW45A5}Q#af4 z7wVtWiJNO)zzA&-Bi|NJA=ry((B+!z^4!JdP33w-#~Zt)C=;M*rrL{5Hn?*n1Lrg< z_b_D5E25u;@>hgPMNYj3E-0hAq8f*k>SS5NiXSx9ZCOH=2^N7aHRm(s45~{v+U)Ux z7~MYi{{7&T#r4cL=Vl-|we(EdN}dAg9PS78Z!ATlSc(M2h4p)s;F&Sav(%5DJ1nxM zN|k2`GvnkLjJf<9iXG4CrJ;cWfPC%R+SFv^Bo-1gT0S3@9ms#aIzSKap&@8i>uGqY zE?V`TH13vhZ^)xQH2LrhYxZqPugAm1jt`2dzv6xQ_nM5S*un54R%|m__qX9El21n& z2T82$2&wjYV4Cal=@g;prGkU&Fp%9D|LCaE5Y-%C3^JAr9K`!4Z_5ZSu&#FK89Ap> zG23LY6PnH)mEOW}ay=88e!|8-#9Z>Pr85JN$ifSfH)1P>fm{rM<@n78yJ$wcxBxWy#W;6a?-Q5r0-C{mydN?%qgiUhc zjifrUj@{)TuVywGAe-t2R!w`9x4Wi+w5IMl^v=BjuW&=!hlX21(>s`*mwS?3-tem* zYF9ZV{}TaPhpi#(Ck)Qzvv*I`7dqhOPpFSpY?r^cF8L>4q1i+KME45@yYgvn>i^1C zV*2msvY~e@mFJ@TpLI@H#a6twb$Z8v6`{aZ!3LDM1s9xvsVf#)fCYuT_k-GcqK# z%)fN=`6D~HZ(+{GleTp0&2;^a_5w|Z`zedl%u?-WW(xIZvlT~-Pf!}i(F8}}gkYx4 zsDm7u8@V3lt7yfP!1|7w6^z+l^e{b$-PXZAouP24+@0AhV|pIwk!x?G=9U}J{rz0x zn=MEAqp&O;SH=-nmEuL1;XTo;) zZUu4_Mr7(_>h%m}aKe@IWt`>- zn=o(=ZmJ^cGGdG9)OxDtHHX8?n+nnH?K(jXKOmr9p7W1D*-98*Xkj z*j#1DyT3>C?EZE6{*>;nwk92Y5LFEda?yN?=hMp9o!`gQnw#?F6qc3}64%aJbH1PAxUDVk=Sv@;Mgc^n|}alHlwnb}GHf z*=Uiu1CNP*15!4Xz6g4M>;+eL|3UlpN1~Mrd}Z%r#i*`( zl^a3?l_qJa3zQN7!|2k982s-t?g@0oFIj42(xz0dHdVwIjP3>q-h3k2sgn!WSI*sJ z^3|NEu>}g~3^PoA{#-)yoBJ%J4vLpVs2PuX&9v)B<@&uABgVv(-NY2Ufv@>%IW(Q| zcN6`6Q5iAZDeD=99oy2hGWCv%g-&=u>XV2#%(#^Ja{cUp+)P68+^W|pHXJdxcXn@G zA`iNP7)h6u4Pzk?VAxDmQ1r!M2l$ z%g1)|;0wgQzO7fG6jcvj9;j<=n?WZ<5K}1GM%7m{^jCR&V9+Ni^)Kc~aRIlf)469` zte4)FcnS3YJ{mzEp@DT;;HDXj;l1368;fZ>O;#qm&ZK~1dzMn%dUFGsqV`1F*>2#} z_a3IGxNE^@1FH@4k|9q;@^_S68{fjs69;zvIdsL4!aJ7{lWFtDiw9o%DV)Jy^Bsia zO)-CoQ5X?{A%a0?@pB{@X7d5uKp&egR?Iu;7c?3`nIbOfkLiN5ZllfZv0Vt?4$i2} zBSYT#j%!cAfRVj+9|~$CZcM*vzngq?_ex%o+r*%g-b2`S$~qm(`BX%sdypmKy7l;M ziaejq1XO9q>fE3BJPK1ME&^d@e~K}ON>mi|8Tq(ItNkJ42 z#X}NscM1D3J;S1saqUvH;lQzG7xn{P`dWsKvAt+1naI-R`*nFge3j!`+2RsBiSStN zQ#azU_C8DuWNsPDBw8)($Mk!Ak%i|KPOfd_@oC}uxUt4+X#(vK3|t)?BFQ~|eRxNE zFO!o{>MlR3Q#@Mt!|DhRtpn~xA5Qb*4r4O#@z|PJCCMnLanMXy)5m>X`Pe|1@U=2m zcOKpIwy0mj>;vXk=UX@M$NI*2TPul!v_tkMudK*2prUsDhvpNaZ= zF0?YYzv||<#A3Z}TTgUl*~A#jQOChBhBYzjKLvR@kG~n&f5NY5k9Vk3nB z5yqHgsWn0*XTPQy%$mv$#Ju>ut`$I$BJ3h!gW5H2nojG>^Ktnhyj}-gLM^`zVtX8l zb|h08M#~R-98!ng>&_$9anB;9&s-a!J&6iVY%aTSEW64x7TVaCdw*77z268+Vg4Ek zZqxmx&ad)#gz)1rwomOI1-e453kKT*`L*rgQVxzOnfzG-TEcwB(G_n_L3l7H)AWST zRX(KC!4Di^0qTY@jj}HR3Ad`({6%eaP?D>ST%#w|cwTuo%kK@FlV5`fwVaYNh*;V^ z?1Z(V^XcDybatz)_8G!GU>pE+ud^>8W|eGNb>~^yTTx#VLw&*4W11yo(+WI z2nFDV_cS*l9KHsytJ^!SN1r47(_Z&W3MY z)pvsngVWMLfLE$4g8HW;I}+)0{Oxsu*aWhW)8I}C zqctX>qw!s%_d-V#0{Gt~$j@}0En`7tGTbJ09@2bDni8Y=O|0_t&O8 zxWQ}E=qjnTo^kuAR0-s2&FR2cu+V~%{2;|X&BDP|Kkk>Hb1HaQ!e$~Rvo&rsy%uAv zj37N_Dxn%~BC*!Nautv6CRq|S;Ao*v5l!p=7;N%n=*TpU6?}I~a=v8~Y_?)?SW>Wy zu@a+0t)<$>*vXcO*}Du-h?n!c=gy(^cl10Lw=g2(gGGED5S-!D?wrFeNsiQg9%AJ6 zB=;m275r|qR6K<9&A1)L2mD7-n6opqVMF3#*MK;X?-8$V4in01?qc93lw;T663aT- zgNm=PE`)Q z79$q=y(~p9f*F>0zf$6@1}yMqX{v~H?LSgwI5 z*={vkz!%p3$#L@?=A z>`~eop*u?6^hUEySP5E3lkOK-_e#noKMTedhWJ}0Qq1mUzVi`A3o~tS=;0EaKjfM# z5Kkr5ey?HoyRedZ)|MVR+V zl0>->TNU=WjG4i|pT1Vm%OxqO++lKpPqtuj{gGC2Ss5+T-O+JOs4tQDk^rMVSa$sP zPMAmPA|MNnCwFYWI#%*$I%6AoZH^{}7IseInfZ9@ZTe3!_ED`w>)B={61gg^HXil( z6+RHHZnpukG}h1-NmcC~@M4;wczDqqPp*9m+62VJFcDJkyjSGp$qzEW>8^Sj47o2P zshzy6l%Q6Vr?=C>E)xZljYiGce! z2$Uo63U@EG*rIaH2=$WaeLG%b@y^M!lS%bXWh(Y+mi=WCphnUO&-YcL!LRP*QsYPw z`WbqRyl50AN3QeggMCD1K$ZTE)yqEbc|j_6E9}fm7nixdMtr13PS!pyOGRI)R?~(K z$=X?3-#SIbqOO+BZ1SuTE+W5>QJjMsmen~%*3ry>OC{lrpg4=eSk!9eh zJ6T{i)jcx)(vCj33}KDzB#&er3}}mDxwY z!CoKFX47MF*UqN25d>v}E{b($RPR6Ku#g4E&Mr|do1Z}}A`ltP2r)8gjK6#&%IqT8 z^ zXaxSUn%BEfn)i;qPd_>KtK{JOo3wZ{%8BZHU03uzlZ8I&nf7mmrTI2lt4_P)9JpB1 z?yvP|P}(T3fd~*u>PT{AD7e++r@qtzTMK58f$ew??#K>nPP*&xp?iuv7M`tGL)CX35TH;m;VLgV=~<*-YOP5nP;*7#`rVN zGL>&!RtS&fY70h0ueg<1jGn{;P?;kirP_vc-R8#&_kz1W8JH|$vKwk739cov>E5wl zP4NK6hPYf@-6k9eL^>hzuk$L$!Ft%Ps6<0dVdx)JpGS_ao)HL;`t=*0x++y5ELpw_ zb{A~#J@nhVwf%asiEv7dWP47HyUjy=BKH-Um4-tejGN+(&Ot4o8c>jD_GC~NOa43| zJ+N}dtddc%US9M*SZto$qjl`7>yI!v1{w$8(C1{-@T5ttq-WygQSW@&o48u7j-n+K z{n`;WzN}f}pDJTyiN+0x_rCD6yF;DZcQ{-wQ<;c2aR-v79)n#n7Lk@(;+No&dTP2h z@!eM1rPo6O1i#gtL)EdD_GzSVVDTe*wVJ=mw^FOqpxyy(9);039y@P_i1B%Xg(P3# zA8^MPRZ#HH0F(E;&})C9fO=9Mh4Z1(1a)q*Cwabzou3(gHR-<$3*P3EO&&~M*uc+M zvDyB^ZTgh_+Riy)LMQZm-lR7ILmzQbGG+bDrZ0T&!u;m)qN%RYgrco)88oaC zw_8?y`ucH_>L|n}U%7rQXwPc5OBheMmXK$~2o-0$&oK1wUgef?G=J-rPg4hIK=?o@ zUH#hppg2$#v3zcv0&)(NKZ!PVG@)nM8Q-K&7+#+0(J@f--4IAjC{hg*gT~guP1i?} z6{%6T@?%CyRoGe&XlUNpZ@Xtqc;m2#9%;PN9%($#8jo9E68u_OG8{j!rdsnGTvh2) zF~4T=xWqSVj=CC~^02Xto@YQ6hUjySvy1m%AenA0gb#fr&J_xmN<;twAh;fo;cy|@n8eYkb+^6Sp< zZL&8goP6E>+uLyi8^iocuQg5csED?IFqKzSc^oF8f=y2Ee7Gcl2tQFqz0L2YVU&} z+iIj2f@z2ue$?MgbJxPoH!G$XWgzmbPDJs&*G;HC6Q?V+RgJj&nGs2)n^neVbh zvx)1v@QBzkGf)@?BF$GK3zg;?*tgLzre)B3ch>oj=l$E`#8&m@{o-^DOU4}U8qu@O ztE^I?{2cIoDi@XVw6Yo}QISd(x^7)ctVeS1)`c^j z!hryt*mHi$r8l>z;MkTZU;5r(V#n4sQsxR^w51rVeV`C57COP_-Vgz_2C5w$9_hDPR9{l;m%@pC(KD``o8Zt!4m zhl~$&;_)v{`nT`P>E>(H-myXOT2nLLBT6bw!eioAwNRhiD0ut)@-EF+R!ebDIKIC~t4X2IJB+DlzpzxdBek5SjaQ7~ZpYom`M7n;^)*g;mW&Z9)0z0K>%f1X*C)_*&%J9rr3wStn5YY!Skp<7<9mwH$MfN`h2_(}FH3$o<&qAw zeUzuk#lIi>k4;jti znE9KwI|3Qonei9?&$9mCgLS~=2gt}f*-$Mo)f-!@U;J0byW z+Mh~9z1QU+5$z9H{lBg1z<;v3uUE^i0|5b(XBiq;Q;~P*jfxiX73Vjj*qW)t-b_JK zG4Z3^=_A*T_VQ_kK<83X&ra4rTjlr|_1{m}aV0SW-=C*GHcGH&gG3(SLMcv~Y_L{T z8YALaRw*qLA8I^>`ZG&RPdeaM&q_jW&&6g}`W4B#y!Q_{fmqqV#-5IjuQSW*oR7yR zY%=Yl+Fh{u0fJW)nTbv!drPcS-Wt{+y3&c839ZTs*0`~$p9ac;7c=adobK?v=4|8( z^Zze6K1X}Fc-f$IGb}@m#MfpX4^`o1*on#By42xS221Q zHes20M=N+TK7E3?wz6;Xn%g}wZFut9@ycl{{6paTQP>1j(>Xx`(u|Ny<1|**hmrpYBP);N8j z`pZCdFcXCjYE|c@RhSa`-ix)_+Yfr0{kkicljrd=WAV!U#>$3)flXq_a zdMV&l@@AK6Z@nRB7sg406wZkmmoNUr8*@rsAt2SEO`94mW3Sh0NQ!*kI=PsV(aESV zVt0o*$21db)_Cd?QGHU7u&1CCKBcExG1h;z*uT1&0gcUxQOY9tY4Eb=li|k-0>8^o z3n3K|hp!`))phK~uOxk~^H`_T4)KWW!;OgUHa-Dx1sXE+UuYT+UM*fFY;#xs9(>rr zG6(eY=Xr{Igt@(h(1aJCZ1KCcE_<6F@|b=YKpvD1EgGox(_j-z7kLS%Df1voY;0wA_~YE#pJo_a%EOGb7EqLph2c>D9fb z)2~de`f>xcbF~jU#((K1iVEzYbQaM$KL@ti04@@$MtGpjP~!h6uTU>)^=`BRK}H)fKIKORUAa3C7Rjngt8w4i!!Y$?Kk`n;}(0zq*FL69w^!8K#g_4%A=1r z=QS3P%W^f0?cP^-mDHgq_f_cXBa~Y{^Fcaa9t5tFZ9g$yTo)2qVj78-nfK%KZ>`}k zV0}em&ivE^m>QNzsOz}m0uUnLeBsyWjK2JRwhh95a7X7(XA|M$EcX+yBN#w?;@qX} zbYhN?5hTZp3wZziIv`d^T1K!LPeUW2kUoAqtB#(V!>dFxmfq%lJ}fiqQ-`nTJRuZK z7qzb3^-*E%PL}SQ*XBM#++B#&D*(*>XI2jU36}o_D+m4r%m0Fv1Al_$f5FOuKf&_9 zVCBG{VEJFLa^O#}{4ZEJ@F!UQ7pxrk6Dwd%Ne)rv7cCFHMmiJ^tJp~+J#M##3C*?@PEV|vo+z;8`< zQ-wT2`Q35nWrVg5p7oATL}1_8cL3Qnh|*ARZk-OFz=<&sG=?@x9GdHE)@Y(bd{?iZ zTF&!bE6^-u{bvo@(#u|@kzd)UcAM_^W2ZgQ8+!k^UXuj{>sV3!K(CP8qnapw{_&;>WiCBywG5Hi{uo@MX`WynJY^)TYu(VNkW^0 zc6zofjot9LM9ENo!EOzy%8{6|OjF_=d0_^wsgeb2wKz`kwUdz5LDxP5TOS@L(HHhe z&)LIO+OauDk-K(ppN9&8%*Oe1Ay3G?StKKc-e4s6pl(Afsh!hc!0L1wpjGI5_#o^s zZaFBT=@@8?AsR}__N51PTh}aMC`xVO`HxYG5eGkfth{HO5fRH@Uq(8w!27l>_ZvW( zzbBrRzs1^O^-c%`P}IR`|af-0yfrMCeZ; zf+CLF&Y2p}xjwx61vqIvLoB!&Ecc85T3ze1h9B1fg!1dvFLmoYfTT9!xE{HR9ifhi z-$zi&@ZI3T7I&R1uhT&#k7i^RD}uhMVbREkU|%W7St=9PVJlkazCufo5v$ev!>41G z1$?}XXARrEvw=tSpKI}su5W@f@2M)U_mDriCN3g8ubneN0Lae>Y&GvcCX>bmchTna zA5Lx``bP=wa*A~3%q(H6h8_P#$q4sG-g%48I&~Arshq*bsa!QD^t}RwicT)={2wj` zoSM#eW@V}~1>fG>36+(!1u`5)aaW5iBYE3}c^76tyz^=(2OTTx=<)T>-c(iM#UKJ% zR}3djnNu$?0C136>X+yahMXQ)PDQs4?Xhu_OO;w!yc-&|b}Z|Sh{yM;tBRznHnG>pg_LsCH@tI7W8S(>|Is+MDMqd6|uH%9Z2;N<$m zddn3Q3(O;rK+eRGv#3Cxf1EI~7CiOGe@`U4WhS04 z%A$ygaIfKaQpbnobABdlc2I2}129a{mlXc5|L0efq54{9j&K-2|QpE0A=Jc`9$O<)xnslQflCPh!@cd-%_Gkwn#t^>&K2m8>7M1CY z&&5|6vD`$Uo&B<(E=Wr+A={6R?_inHI%4jntgb?HlyErt=xB**xrAX{e)r93nrUX% z_0Bu1zyEDUSqV%%%MVH&tnooLf>rjy5#2Ied1d!4Y(7GM3=@<|Vjw|j6#Wru2| zF)F-3zVFTitAA7;g+7P@c%&C zE$C)o{?ux|o>FTWlD0y$P;_8MK@-`v!^}>3@Ay*1M^6xhXUuW{C+9T{@iPRy#El)MJ=g z)`lHwY|;ts_!P-s&VGCFS4{nID8NCqgu5`f7JjaK(R(ocdFbk)*zVd3e*+^UG>0>*!*?ua^G<#J*V-oY)s8e6I1)Guk%eA+=&T2dSpitvJ|XwJY?u!MTJ z21hkm4!Ao=cdG!rxzRoxil38C8&H0d%xz(;Qcd02eCpiIgkKQ6u68{fymM$O;K~8d zs}FUId@3U|avI&ftQ&x8uvly*JTNjgS#w{fe?@9KJpppue;1KY#W>-_{e}6Rx_ci>HTBku96A2HQ$>gH-A{e<}{>>(X*P|KyYAx#_g zI-X~E;t3k8*0PL2Oj*!K#`jjJL;qXI0+khXDZb*v$)TrNod#54v*Y4wiv5C{5o6{3 zgUf1g_~xw_rABMcqmrQWRT|6$h`YUc(Fl~UF7XbMuyGi6F2;+!46Pal<36tsAH3wa{KG9V6LX@Am;uo0T(z6H|j z=sn3Ao-Stt0Hr6d$F;&4Bo9M=EdDR|XFP_|qmiBc!Yezug&CZH{=&YARZc+F=4%VA zRN5(Ca~1rcoaISepgLEkQQ`|vs*4J(64+mOk?lmIOE*MH7(aLU9-)~BlYO=oWMgx z5kT9(H)t=Onyb+8IFxu=N&#+9GFcBf7lRBt)M^eEsZ)Pm^rh=2SLXnpms7vDynTON z-6)ymojWjKJ&rN9pxdI*AL({UGyu+QXP}72lE3AL6;Xm8-%9;ST(+fzAE1pmW;~E9@GP| z5o~2?9qYo}=n_@mVRfo-`*cao=4~V6$Qs;k7`oZPIV**JjX2!N32mR6LG%cG>-j|q z!ZvpFhz>lM&o<5p@&}L8JwoZL9~fObA}myR0sH&dhX+!_g&#`$K*cj}I%iakwS}EWYzZJh)^Gn0KaCNPSA+rP{%|ww~h#t;<*Ev*63%uAySvLlDQQ` zz*&iPg~k26L*7w0{^!kf+n%5H^e*}2h(YP)p;cjIcNQGJb-2nLiA&q|}f$;2q`3TrOJNCd; zipHN%{`~A2(#L-nO%}p`6is?~bglquV8wHptbclo^c2*2{}EPFTO!l@V$~yf9o#*j z^OZdyZN^ldu;(_x8EI4-Scc6(FM|HvSNxCYflC6j$#cl24AyY5ba{T)a}78@cBHSh z94RVV!oV|%Kb7{{p}+7?Dl$XtrXxCd%IS%Kmyb8NPS#bd7*&?K;}s3lnbicIJVYIi zE!y!%^!pz5_pT((GrOsUi8Udv_D4MBmy_SPiC4C0e`8m_ZpfDT;1^<&xJ0?s<4*u& zJBa?3t|-)-`k}r_x)*QoV8yv?gm>Q!)U7*fb;kbYL}!jDPVfO~I# z?i>B{4>MD`)dY>j=l_GVH;;$1{ojYDRl7>1vP>Hil3U0+O64vg6xl;l>s%q6QV+c%sWxMXYID)7DSn-H-`-oi+GO(RGUj~3 zT*Z!_y9eq$vQjCNNIA7_O}KA*rm2Dulu+s^fuF?qkwqJum*lb|VU)|^CAr$cFY$&S z669wz^OQQKV~4{9?MDTtCN3o=%vIv=1cEndIFuB!voFxh^O|+cW#Am^9{tDl9wJR_ zNZ`~>yN;P1*AD1WkN!&F=`m(yw7~H%@V@4+TbQP#!ynaCMp1FBSl0;q?hL)TISOez zyFDx9A-lfy$$+oS;a+1x z9jUyA^u@}mWt}GPrwcP5$3Gwf2Qu6a5(`y}c0>;tavXzyic^UCoB3WFs8*hqaZVeG}Es2d#Lte;1VxO0V20^0i+gfTJxt7!lj(4gIOThQGx5j4+ zmQ_?57!Tx_Hxg7sU^y$R+u_)6+4^!v&FzNcKforjMqFc8$$tV;CBxB=m_!Eg$G?Gx z)`p)cyQu!R@y!#l$h7oXmWlc)TVj^9ZmEQnRHXn~A)(KU|`8CNdZQ;;*) zt_A>5R+w0f`R4U_p$^I>`U&K#gmcF?FS$a5z^r1CACp;yLs0NxE_=qCrT!3pr(dbIxka{qv0Y9nZ}xF@ z^Yh%8r^{OZU9A6F6$h^!xZR5hJJib?_VCY2K(M>iZ$IIZZh>&4()tL>!aBg5Yc zMPpZJY^M6b?N|TzYxT<8zu&mQ8k%v#Qw%FZyT+oe=%#dU@=np3_8R>pNNOW0x0(JK z>+ga)z*^1?pMe~9Se7)rsl;i_%*xh5Z!u_%Z^Qbii)2HaCRSS){%FgUNwc~|VVUBu z!xo)-^7kj#2iC9?xx>Fm0a96~^&q*rApY3;*vndI4@TXjuf4W&m}h232r+zW{lV>$ ztENrs_v4*i+-2tRi6Wmkql*qKT=M+;*Ddw#``rA}S3+5;)#-J%#WpH>(VOb1^;tiKRe;EaUiR z{tcpTGEg1iVewv{HqXXYwqd*A8(9O)}K)Zox;(HM_abeF^} zHOJg3m|LX;!<=H-pK)+ySbZ$tBHB5M>2Lu4Qy1dzM-is*{#jS|(fiqVeL%=NyYo{u zm9w3W8q3(M;K+4E^;0=whSR(s77RiiJBo1zHGie^XSuD3aNyd#;KkCUW5SY=yCnqXY=PpF1k8!^`3mhVk4{s8mKsim0vQONzj z3%A#)DDrl{gQIrm;<*$4pxbWwD|P3Wyt>%kSA5F1li^EQ3-FElruPq*V+Ph!I0&K( zO)iAEklLKYnhS$$_UIhAJu5c3kS)JVRUpABR~n=*Gv+=<{dVr|Uu=6^r1j0LRSqLO z&bn|zWO2Bu8^w`QnXfBxX{+mv`Sd&Hn(@bKUFC+|QnBonr5DIkq01KRrUA7wOtWzaJYmH9pnWzI@|IQ`fYN)w=S+wpb&+ z{#KPRvGGG*%G=AHo>hS0%Mn~tH(61gL4LUvno~nxIQFIL!SGkslutlN{R>a@Sx?{F ztdc7nWusdV`cg!Xyw__tyP2CG-?qcVo$!Li9(i=+DB4POPY&k4f9yRB54{-odGH+S z_2b<7P3E$H3B`)Oo4ni{lwu=jHzu~KOi509gn6_Z;y9Dl=T=)eqRHvYFI(F_@^z1+ zy%yUpzCHc3?<9-jQ-jr0k@$hhV#j9n$l0Lh*4@7mE`DuM`@*~c?^(!@dU&V0zlY*; z5kAvlGw4dJt}kz9%3AE^%t?P{Y-uz0Fa?}bzbH2qF{qa?YW{4o(ek}@Yozf2$1tys zk=zWy$iW10S?x?E2>t6_tnuJB^wXt3?sl$MUTO`y7F9${5Jjer707V)_FQNQ>w;fE z|C9Dy6PdTzujrEAUa>C&>8#s#d6v7E^a49o#^jQ4_$)ojpL-Yu%srmv$R%m6GMwR3 z{3@Rxf!WEl3^{_6iFJfTF(iR@d<=i&A9&zUCH{MC*UyzcXAG&U^9IwOrQ}M!H3W|( zdvkR}LJ5P7ZP38n6)A?A`S>c0$H(_1g?e|9lO{v_Baqa>H>B;7df*cqD%9-l&=!;C10Hsw{E|j~ zeTeZ;q$yspm}2@pY^Eo6$o9)D-;N5KLiPn4z6~&vUVIFCg>lu_H~nB&;6r9gq14Bv z{7yppuTS18#Lm)eTk(v{y<}Ez$4{%+_))ZMQm)clyM}d0%h!o4U{a~&>q`oKJ{MM0 zaiWR#Z2yWEWT)n8+4Eg)omDD5NYQCdON`r<^T9}Dtt4_BylIx5Uh&npC4b9|I$L={ z?^d%Jg@;99z3mrW#LqAa*A4j3siaTVzu2X*7$Sw57{}Bpt{F^y>-EoOWk0tg!zgzhhhwr64 zE3kr2c;2<6?d{e0Hck|sVX#)*_Cn;FViV~yyuejKp2(FmXe3q9&&HUe4J6?1EJ!8Z zE+(N;iQ!t#6i^XQ&$wg%ZHV`~k0~5{iGHDa>4oYptnHkHe=nx z<7OmZai2R4*BYDY@I<_&E5V_V*@pQaieyuz_yo5%8+ov-%grMvnppD<^~6*@f7e4? zw{cIuEKf{piV0`-Mu2ZS_|-qV z1K+{7>SbXLXL^J`ev_I~ktubfreH&=r0X(PdZW02q7=13JNQMu^zuxHxkd=bO0#;o zvP~{cnewa-d$lZGqa&S^0OPvgOe4__i>t;b9ZqihkkBL5ZnTYh%(3V1^7Nj&@#kSk ztoxG21|}+KdDXVf>r$P1|1k{XBvWXESy1unp%L`}jW)BC#C|k!(p2F~r9q6?-rw|a ztVD_9uvTHbPU-uQ!UQqJrSOD9+-7?GRQZ)k@zzh+FB9_HIo$VJr@Vy4s4th)8J2&s z55-udit{JXs)`$S1t+&L<~a^Qxh5p2ctbiwl;Z7(ZJXv6V6mRdR$hlo@lq0Owz6bU z!E&J~_n_-JwZ36KZx`m3t_NqekC|ry&%1fKHhEGKgGt6L9h;Yhi59Kqw8A^EO^2_L zVT{^7a#(LJ{Yt=k-cdOsFLiF9I)53C(X={zo)Op`tIYAva<~O=Z9LvWg>;CfLAgTC zbgxUTvkoUQ!D7=POW|}ccZZSA5%3@?uI?wv8etP-xji9#D%{<8o0#tU3NvN~$CD8o zky%f%SpnN2Bts~kYo|fCIC_SjL*F>o_UuikL<$$H z^xEMFQ$rcfArI_``b-&ib$-24?_)MY-caG~YC6yUXcFQR<(j!pD$e@Ki{`y3`{YAc zJ>}B?E||8mCfmx)_QX`Trd}aa+ZF;{eV`SQ{1B_^zV+)~B^Ut-vIbA>4=Oobd4@*a zL+TCNaGL1V+t7z7W5ZX@l0z$H5B-G3sE{*5q0|Z;!$=hOy`z@!qE$_~F1At1ySISC zi^s5fpOviCJzF(TQg+?zgPvh0ub(~XYP2EwM!Np|(tui|joXBuhn1zW_0^$7>7GJA zP8qJAIOOR)_I>Q|gEI2j-(=}d1q+T@5*&Jkndg(jDe2*tAobS2q2oU*$w4vQs41Fy zAa!~=XxoZhOt-VfSVY7QE7hCX9F0-d9yYhMEG4Rmm3a9hO@l1T4r4C&Z&)>}Gx2t7 z@#VbCT7rh@&6xPB&5=|xk4>j_*06;4YJf_gbSPR~nlv?VoOy`1W_OR53U-Lk>b5rj zf%*7~?itf<*Ef>M>aK6HA)9+DDrSc%@+9Sz85;9IT5Vy5R=P(CgcU^edfzuAuEVjMZG1Rlh*#_6tMN2}w)zy7l`H1xD+Y9#8$f z+*}!@R19%>suw)&^BxudA3NH>7FxXR(anLqheK(uy&4AM8>uEgS%Tu6p>1ssJBV>{ zns{iv38yq$uDXt0RkYR*sVsze|42Lc?%`&ZkoG>r?to1Kh2-HPTzsCMfl^U+8+@7TsX<8@Tj}{m~VRd=~6g z`OE!X3hphhUCR@7JGhelT4se}b1j_LYHR;U-}*E%m9UhBH-k;ffN68;8HF6e>K;!GHg^;*l3?m1~f zJx<`f|IrJfXPl=b(0{qFUGkj@mnS-9LsaTr_;b8Jw~kMDNNEQ*wIe!6XI-J&vuAXE zb-zbQXl5^3VIdxUVZVN=xzQAjn-NUuE7k0)>oBSE<%~9^O zqIF5maj{JKxOQ3_khRs_eRMsP$y$venZzISNh^6Wh6+}}=X)YlJl(R?XG^r(McQuS z{#qH~BAp)DZzMi|m!+4buJg_z#t{p#U7jLZcz3%wb6FzTNv3K&cm?1goW2Hw@v{Eu>3VUcn zMZRf_D}h!j@l?gm7enYQ}n+gZ=L*UnZ(?J*1xE^OJWVa4?$C%vZ0SnyL1lYi&R}axN1#!Qfrb`j{+# zwzm38aZKe5M^|;9p&B1?O%=vdLsaZ2RZHuFH8%C$irhuL#!zRqO4N+29XrR`N=k}~ zZ|@#^8f-%d^l9m=Nr(A+SHGVu>aD%v&k4vCXd{1hwfI(+ucRVfc+fiE1`=NxRSOyx zNWioG_6x|xXS%WP%GJCsq0Qvm6xSnXs96C#bnLOgp6J8%4+hFbjn$7TeqVMhl3L z#%n@x+0Mzi%@y(vLWb2eD+JU=js`E~(dvoe3S5=^FCm97h z6>HZ&Ixco(ndif+x7!P8vLkBQgiiAO?m(Z0wV`B1dCJNIH`i9)BwUGE@QSjD3i%5g z|57G_-tG9FjUPFrVWHDwGe>>UBww+tB+BQXYkWlm4->L44o4&r*u$$|MmwbbY{cWz zZ?tuj>!7C&MLvz=r$#$}`LpBQMJ)TQ(9Sao+h%R#d&bsgGmoxZ3GKQe?vYh{+()*) zX|>Eed*Ww0snWyQWpX_{sA){IZ(;l*6+dzo&ZCbkkM^F>RLW6QDBIy6m&{jdS)V$J z_mH4%^QRVUjeLMrlnBKu%Y|Hm+;6o$IJ6)spm+6TAEEJ&5m7NPqu3T+{ z>W*$1`YXOA&q2~BJz=F^UFkuw`0rwE&c!)4!_ah1MvQ#dX3hD71If5o52EVHW*y^Q z?2%P=F0^du?OBE`-E0;e+LnGEt{F;CUtJ*(!U-hf9Qn5{lU)` zz%EKxH~46?#LGXKuxoH>#%mJ3k4w7C)M;bIOT4rQ($B6z#z(G3MWM=odwRoi%}-kw zR_?+stR#Ey<+M11MMHDELvy9S{Q&I%^TW_B;yiWPja!&Ev*O0eIK<`WiWhG)}$ zOD?UKZw(^Y*i-i29R@x5-4gobR6nkGO3XM*`6rP*k9)(oZmGR>G|$(2eQPDHAHV$R zO}+>e=BjlEUm{57S}(()>lYl&NCDkms{CPV%&{gLV>sR}6XXB8km-Nsu3AjB&ibEq zudftZ>V8S=A@JEdFU+on>{nl8#H@^CzxGTi^_hGQF_~G+U;0s}>(j+Z{aREooYcR( z6kSRm@69VM#Ge)an%||CQTD38?bZgI;Cmgb(zYLO+f>f*cEM6$ol$r%w!ZpUQR<)3 zwLe`XiqV?u?k_S$c23+=B~BSKT2G9;0;gIi(eXjjVR&N*a#rQnFRmr&Kx_GR(ZH5> zLR;DwTRmUhYo%Ia#Ktd|rc%0`?ASJMU`bYbH)Qld#ysFD#mt?u(HA_|FZr(gN_V4i zjaaw2KXa?|DM>DM(x9TX?~miF*iOtpJPWLM-v?j27s+95NyFj9u6$&28!@ewA=|ir z8ahc(0bb5Iv2DOvC)=ucEvJyPP>=7fDzzmR@kbC}!!c?O+}<8q^rd6%F>U&>fgRoP z8|LRYqy;1RVhDb?cmupcfrIPg81G0-$gafeGGEH94?XE=G6rGcO$ngq+$?0D3BzFL z&p({?ipsWXI+ZYX$TO;*7{B;q=5bc7uCuA)xTqepDkE#P)tF>2xEURrbY(^1M@u$T zj6D4eDQ&+zN9#|BKp2T9E+D>Qwb!tNa1;N(i$4P7@eo~lc-TVE74+Tn;ph+WmQPyU z`=!By-B76AC;uxwo-4WhcfaDwzRJV8*Z$qDM>g#L&#uMYtpZcs(!Tx&^Ca@Oc)&!t zG92&>>*a83-)C&x+1eZ?*XXamrG)N? z)fa~0`F(YU4>OXh4P4AQ(DpQgk^8F{>+%8fkJ-9$hVYfUu#5w7Z{|kN!D2I4eT4l=sm$e(PB zUSFQQJ&EBXe}&3C+-HMIGv#o$cMP!&9+$J^U*#)R(bI!E1alVqq4nd$;B~S>t6+Ib zuejTRkjaxi>g=ot(k|T4&AcHs;$uotWp&|%NbRLZ>>F{~@>S9N9+HjoiI!wBK4W)5 zub1=podqX|2jb_o`r__s*^PaFpKWJ~=RWP&I~QaV#Of;1n8Y-&T@Tq~@9aZa3ZC>W z;HH|pRCMpOA3HAP1rbitr4B@Fq_oPrluOsg``YY z)jU7YPLPD0L0ir8^hUC|u*vhHo^jH7a)3QIp?z}kt6NySV>&f^&Md-|W|;eG^E5!2 zP$EM3C3|#)V-3BZCFS^3e94o0zY8Awd3Bb>-MqBsIB__G5QNR-w0oMFLM>M@RPQ3= zHVN9+ZtF}Ow`cvK2}Spr?3cDm*;U714`gY)COx|~Pe~P^!f81&t{rLX2Wjdjwzk5500tY`@1l-cj2=KDbMxeoc1wu#G5<7n8mmHmdw{>_Pj0$9pDwW7_05Z7BQDYJjpD zuNCfW2)TNN{dsQwh|Pe(nKIeO4%O7<_FgBnL|tkZ&lB3-U%nd5PZgy}#dE{s{Ksp^ z1A5mw$>COH1zDYH!+AT%+kN&qgR|STvmtOVp}Je-@PIvQs>BUb{*!G+Hey#U=Py#H zZw`#AFzC1V`7uH(PF=^BWtu;kv3b+xK4uFf{8GMviN8*@vdXu*TQ;v8cDgl^jq4=4 zJXsh093WQbci5ioY_i;!j1N5?|3+ff3QLX=-YR-I*K9!eVM~Dr+nE-0Q%=BqkBnxHjQpqNwy7B#&F|g+h{&#pm2=dB z+vPIN7gR1kHG-C?)AWKJ*n-lZ%H0{5jjkJluq?_-C_X~O)%OFF{OUnCNpR@JjcH@S z%)v-YzWew!%z3R}`2;NrX33<)gRR3mK69bCFei=rxjrG-fMVB7l#)wI*p`hYlijv%-p?Vp=dM0Zd>(dIcWC$7IL(0F-&L{p~8aUlNs8S&{atClFg0o|3LdXzezI3 zzWEu$o&3bW8-0tl?+fK`8iAJHe+aX;jabayPs6`Sc&Rgr|JAXc84fWxiL49|go61So_ZzUQMJThx5F;4Zt*)@ik8Ez$B4Lww!!`MCqsj4 zU&PmA|MuVy{NXO&dd4ImGjSng9;$p7y*$j7b0Wd)0M9SdpL%0|Pxk^Vf?b^M*kk$B z#i>)PWO8;F<;!fd?l6f9GI&DslKfnE_9>+6{q=|VMCHaIDB%PDR&N%)PI(S{uk6pa z<#gA_UXdkF%(=X3Mi;Us9=@+9PmgT zxl^%Xxyo!o?T{D>4>kEPW!KI(*<*=vzSZP>x(>9`*0LwpE`eOAx7G7z&u!W`J?0@* znudDTW~Mp#(y=;%k0NM z!~A_#YQNv5BR6!wt)`-Xs~$f5{>65FLVuz3muf|kQhZp?H?^o8$jzlnYfbm>=eK=V za^M&ZJ$pcahd8;j2`!wy|9o*6B$2oV=hfah!$d+KCYtT);J~csAEjde1X< zcF6QyxPhezE1xZOH2U*D<@@s=Ec}*uFD~?=#76S85$>YquxUOnR|XvygUdO!r}WZr zuEeLmd*3X+{IK)UW9+LF#vb#L_D6g^#SeY&x?R0bLqG3P;Nz>p{`-e&Q3JqL=PmJ$ zPR0CFroF1?-bxcWqZ~SPFHT`qDQ)H(1&$4Yc&>fi4muYQPcf?Wo}1JxA;Zy(*$-wEBkTxPRhv2oZz%|J_0?~ z%&gm57nTNY@GQAvD3NFMvVUlZhvTMZ-VSxy{)D|fF|m+S+SJpG+bPM?tUocvfmt{2 ziToq?HfyWIq$kaN4jGKQm)|jT^(SKY^*#& zCb2oRk5ygFIdxA#0gO`3VYf7mG*&Zj7$jR@)+W0bo?7GEyIyPUQf=YvCyYQTLB?;h zcB&3-N1;yq^9rYr^!F(5(rpjsg-^e@J8etq zSPjn74dsRJ<|n5g(ORW$a2G_;nK!nmc7O=&#+TtI=WT;DHDB!w>sn~=R~)H&By*?v z4zGz?f9-&|_Up*d+;6vV@F!?Fzoo6f;p?t(4G_=Jon^Nz678ByPHnyijUL=hK@T3= zne(MG#RYtf*QC+c`jLSIUeg^$FPvf3^1~?9-C~LyXnHM*rG1wEXst~RXBNdJ22aX& zk@BZ5T*q1DMb|2w$Y>tCtf~l>EayVurH7V}6@1)i`fHM;(ofJ}?3T;(e^N8Sw4Y9# z9|$jpUo6-Pe&CkF=()+D^fQyE_Mm#&B-tR&VE6%=m047l57m_(~4ep1)Q@QyzLT691 z`{Qzx(pC3f8CHBhUK6Pr50*P4CQ8;3+D-D@7#K+uDhSn?p4wN>1CDL4G>&ASttfHU z2vsxo^8p#~z#t;HbwQt|id+vnr`oizh~bY-EuJMOzkwZ(nC{##cNEQtG`LI=p)U}k$>@;bxZI5I~;}b zi@Xa?9$v^K%biQqtcd%Q2O*GU`77Qq@ZX9g={Jt)4TEioPr$?l;UJT%LN;*6O!r4Z zi=%ZCw>&~PpRM6M#xnq_QRI+V5Srb90?B~@m1*&bI4gt zBU5B34=G!$K)_ZjKkF_>T3Ya33xg{8FP3+dUr;Q&eaq}z+*h+6#dB6j=zmPwfGqu9 z21wx7Z$ZriYh3Nu{N8+;UZ~23Z|ut1lXf^ZW@M?FInr#}#F?0NgR?Fj%uPF_Dh4qB zYv~25cEI?8h`h>%3=eEQR~uwpwamA%IUoek_v;uq;P&D?EW$niA~38gDgEkIKzQfm zKMQ?#K~8o5X;rD<$YP9P@i zW_%4Gl7Dc;4baSd+#Bjh^wHCK?iILGvu}=^+giYbyeF;b`ooNae;TVH<_LzhVg3RY zh(L4D^!L9o__!u%z?55g871c+QFapb61nb3x*OxVX&Due*@&XK@5#)HBS?-&It=Q% z$t2;6z14cdz5z0K`Uf&-??RmiwJ-d>R(s%?0B#f_P7J5g9pn%2NRFF=TC zu@W){2zeemL;hqEa?J1ME)>ev4vcu%?42jr{+}nDP;q&7+-$H!sKFZcwe2I>3rJu! z@-(+yz6L7FfMOR(y1x;1`o#ioF{aJi+V8fyjVHL10KlCAvS{RXUA>GA!0f-Ii#m_M z@${+)mXUD!j1^M}R9Z^@FKd9H8J^DzdLfTo`$ZPJ;-1R87 zqE4j_;E3-?Rr`0zRhvL8--jEB4J5a6IVC~fXJgh;^vK3rX5s=wwpaW8L==M5_B>}8 zwTukiI-NF}wM>{wSBl)}qu6c@U)0hM%-TuK0L!k0DFpu=P#k3^1PZ=9ApN8RJ4yhO zm(;c#aO^)xzdkG9eqoT^)*&`AcdN+8uMiOS`!8Vb>fqNrP~6wyU67StSbL+L5CHkP z^4lbE>Lwa1A9D);ZB;eJ6oC}=i#!JgH_VOV++KBD0s}R;N5p^pbO6I?X-P`|p)k;X ze^1)R*nuru|9%6)`)E!qqIy_Y-pGqrBz@$XVTL8Q2=X)t0)Mix&(@z4t|SCOtd}I45i*QP^DUP|FF>y0Exdw_N^)o zk(x>%KLxNBKdS=ylMB|9I+zR*%YC8B9Ag3^z$qfTFVPW~WEOR~;rraLfeaQI7 z%Z8cI)5o7E00to>?#?+m2}1R=5Y&@Jkb?+Scnq&4jI1u^UW#o0I1O^Nm-HE9cH_NX?s$CODAh-K$kRg(YaWUjArqWM`9X@vUj3eMbAEl{ zR?MJRmYbWfyx9i3Tuze}^09 zhxE^TRy_-5${ID6fO70lv`kz_wkd3s{%@z~KMU zZu8eyy^UW;);Jebv;bttfTUXZh!lf3NzDFi5vBBv;oqrWCrlrtbqVTI=eCz~%zc5v zEXC(fJ&Rer@w?x~m5oJ2&J$>K<%;`GjYN?BlO zR0dgh0G|GS4$?q~;6u|7J^fwPKy-HM!VNK6f9Inntt3y7RPFa)&xdj0s%nqX+^;I3 zk2AOr27z~2r<~kcGr!~4+iDcT2s_X06v#vbSViQ*!Vy876V%atE-GvLdC=SV9Ysb* z6kJmJp|1I#VQ);mJTXtb?CyiwuHG~q^hw2)vA2n1Z3&U8F2|T z9nY#o6V5uK7ShQ@-9)dH3guGA4!|xW^(3i9^8<*2+cw78k_58zZZVCp{^YNwtl9)< zSC0ED)Mg#Q_8a|MN6vnt*DoQ8P0nXW;!#i%z77B|IVE{l0$%6AA6HFZdA}AzTq)Bu z|CsSAgop1{)hJRTDhix6kVt`TZogMZxP?S2jDv2g8+0!HBdw>?WE_NuvDnLV0g>#c zs^F|pK(UY}V7EjINYObN$koOA6iuBbCIvS3x#XhsBQPywe&}iGtz%Du6AYKQ6Tg9C ze0dg7{3kP%sJzi|>IOnPGR`KCLzT$K8 z&i!`69F5`m`S|}-&YuWA?w8BTJQ4Im^jw&UG#f~Zm#e7{2?(Kb)v>o_D1xN3TOtV# ziR1A-2Lmoc9g0{`RHig#QlK^i0k|C&#{mmy9YH$ zKQokOjz)K+pL0gIKA!x`0Q3NPV1%~D*}Y^j_BDHz4B1LqOHHQz~cRStXsi^filpEVl(rDVyc}XT(oiH z;_$O`1{o+@5rnc)U&)|ahs7v4AZc>QxsQa1igMf~jf>US$|0iz&vk0720*WqVi`64BEphM#INe-M_ZAFBP2= z$=Ggd07_~1mq81KPpb=ZglSD%-07t?R%OakD7A@X)F((UJ7{VEL{PoA73!|i<{GJ! zFQwxU&UJ-CaXQ>&j?z;vvCLo)+O(`G>UetzyrQ3uvc-XsZJk9u*bqG5;^gr3FC(wj zjNnVZK*86%S$t88LA&i(xX14Z2>XMuBeXjmCJ0v#&8*r^$mA&?8n^0dQ@czTh-Vc4 z+bycOU;nsfQAjN_YzEYe54J***1_!~&_rmoA9>d&JsR3{`+c zDLt?Fo@|qxdkoMz*+KQ?Omn0^!yDij5K)g^UUgv3Q`5sOqDnL!ykQ6OhNaQEmKz2Z zEf+Br=t3o)CW$!&2p5^4;ABkuzl0GcK@=PWTnMPW>p}iZ^S#cc z3j(G(O&YbmosZTQv5q&zu<8b)h>|Z@LMky(L@5R2H`EF$vHQ_cK__f z8@$(cRqmb*d+d33DG$QRm4FG&GVQtXraxsLAvHhqX1mv-qR0`T0CRW zhW8mjXS51DojSTxv!!9yE27gc7mp@HwRCt}pnjQv>1IMl?Wd)BzHp!ZxX&RTVOz&l z?*b61I1mp)T@0y9@!^dCq0Mgn#sl9&ewN@8zcz8jB;1-gP%MOsp@M4N`35!<#hQaB z1*awat|fd18GVHve)=$K4!Q$4@~v+tNxHOrAnL3(s)WL~)h`-!K~9s7gVwN_8O3oO z26|nF%#`_LTd&T)WQJDO1dsq{0z90IRmpgc$1xo9;&uLH!D~O~tDK~0`q=AcbgXfcyCQrk zIcp&z7;0Uxm{dcfbk70JI0#C>8i3u1|DB7DNRgU(IRkRiaJG2D1gZ42p||?O!@8wH z_|M4sP2Ul&Zkk|j2G;M+9H^}zK>J9Z-sjV$X5r(dOh}9AC<;pZ*{5D`>F=_!xdh-@ z91vEJf7}8nX|Ji*7C&fm`a;=OR7L_g`?ay(sX3%HOkfsguV(Wd^-4X{=_&gL#V+b} zGQJo|WE(!YVpc81xN_SF#3DcKj5udhixf%yA7YZIUrJk1e!^bm*i-jxQls^K-9hC8 z85YT6tW$r(7L1!iUe(EgT_4SwS)%jRcVbhi~JC_ix!zniAXzSwWEtF^xYzg*D?lA}ZF2GF;;$XNW7vB*c>&k)Cn zUTuhS0bRhUz4Ds(iUkj-6PF2~;C3!`H-W3RP>Cgr8PKP9;qQm`})Yu_rje~&y3%c!yyA5Lp;Gy@wC7S zsE3AGySBa;18H@uoVDQFnF52F{zzGW1BeVAiY_R5ku@fJHi^2(Q1TVxM#?W=+Pj%j zq4fSZ&!*0o2dl^{g_OAA2b_6M6i{*bFRH@12fm6C5QDCFQjHg7i}e4GPNWNx2R@XS zk)#CW5+PAtjWfmgCs+lOZac9k0! zWFkBTQU;hQ5tQu# z5S<1uf{!@xaX~`|*0;&D`KW_7P8(4K|2D=r;VgBrY};*sHh?|R-sRa0c=%RE3F`X{ zP^Eu2zHE5fX$=E9Hk4!*#{we&O7?Ka=u{rMsLo%cQvj^om%PSb<`5)!{@ceUK12c{ z1)S3ZKqeS@fn`M$SizF+k@9pVgUGu4lopg!Smm>6TNr;Zz@ieBk-6&L%&a;|j6QP7 z4lkzuRr&l`mabB*+EiNj7T8ex@nKUL|C?YL!2LR3pzDN1^&%Zd*8>VlTTg0 zg0Na0+39d^MC zG?Ep9&cEyRL8+m~AO5eym4Vb>Rb!;-{3^|X`|4UAv&^|RSMFx)2k>d+F^DiR^16|Z z=nCyQqg5Z29YF=>@o7s@mIGU5w!dY1Q5+)(@)NLgNM6 zACm5{BWDk421vlfloQKjj-|FI4k%iPpBUHe^DkE9uDpWaSHuOEFY?=s+8@;1S3!9q z@zXuk_f8RB!uPi=&y73v+GN zp4Y0CPymR)-bQlI42|9kFZRHS-=v0>e*fIzs4VO9n{eu*Xb~M%x159k|I~?CYEdogCKAXsIyjBaS!j z-gHWZhJ;{**0t_o!N!8$BajHu=Tfp7S}p?{YBS>X<}l5vMWv~#9SCd1QKxsf^q%U8 zpVc?s7CqpFL0Ta-d`H*Ub>tiYLljIsMV$k7$FFms{!i5r-ytflwPtle1U?Djk}yW* z4s#n1kb^&G|28`MFgJn+X)SyPVVi5Je!{>oxZJQ=vMSBoWQ?@ts^9^0)|uFYp^xme zacys{hWMN(M%W9j*Nqa8&8U|BQCAQjPG$7Mp3F&vnb&Ld-BGqTHVrvzoB82=z}gm4 zwlJuyONE51%e(l%PQ}2*joqh}PEt8-VYp z1jGj0fbnxr;sS*kk$c^|6^L(H4ty@o5N}jfJ;TwuQqog*c4AhSRZ(jq|M{rE#!UWr z@QKYf`*L65zJRTO!vhcj3;|H+u7!{||WcP%5HW&++E4YQ>nnR=c^=Kayrm_idI_s^u``GEBk)51$)# z8SryX>j6Rp-LwHTfYp1Pn`H5U4Yuouq(8a|e)>I(ict>RO zA=J$|FM7pv4uKwU@{%ItK*|uA6KU;@R;!yJN{9`Xl)lj+wIv zs=zuU`_8k-!J*y;^9KbrXO6iNaK0h|<^5=TU*V^cp|nq4#Dl;D1A?gl><_9wbNQGk zPT!bkwSwm?I-MTrgF3m&ZjXQt)po)(CVHR9KTiULGpEX*^ZG@eMxoB%J&<1pq(QP- z&0+~)HSLYj8DtxD(_aTbz5`;ind}DpDp9s4z)C%zxOC(c{RTgH3dz&ID&jpEGu&sn zcQ~ng6A%RrRt4;T`FQ~pNSq4!j8fbW?XV3+{7%pjgZUu%QDIPe366|dP*D&BLJ^SOq=X(e9t8#|u~1Y>MiGz_B483B1RMk+ zU$;!41Jvsm{ISG& z@584)6b>)vsm%ETj#48Ez5RaOoHysOF&|e1uthrK)eeeF`Tw)NKrn$u^4xc znVCb&13YwkWj{>uu1Pz*Cnq}>J*iG{k?e;M;y9=mou@+SmUe8}V2Myy~pMNa7>9;#S@s7-@O}BybIG-(O#lqOOZFXncD_0l$UJ7rkc-TV)8N*h!*D|5BIw|l zTZ2{>7BVa|!di$cgrM01!`jzhI)Ea)=2h`6)lP^1a34t7r#4%NlhCvF!%>d15kyM? zKh<#8PXdOc|F=A!a1apyYHw#_#idIJAsM(``_Hh*Ty^3B*rh+L7Ua_k5=yIFL-Hhy zu3nBD24a-eSru6RDuJQ)nb&KG=NQ-FGBL3PL1so0L$m7QJ8hy}?P z;e8*}&#f;;j@JNHK%k+4ZFjY51!~CijzgfBdAa(-8c~S>09m)_5*cqJ;E}p;%=Jfr zeg88NM0duaQQ4&~WNpOJT+27dWnRGbzw;nG=dV?P>c+JxeuqyjzKFYSBnxlHS9A0au|TPpbPY(a3m3U7*w0@2L4n{*jsTJ!tsRe!--E_7~7Q zpnt)ve{l|6wg}YT{U!x5K=T_`h&cULZA2?(B^KO^DiHniu6+Tje`LU#@9Qsve;Ms$ zzH7eA+IT7uxMRRE!Jgr?$-sKxhYdgs2Xca@LlhLjZ-=*Y0B+#{t7zZ$wfunC4qMU= zl08`qi-c%8+`oZ>~B_sCi`AL z%Y-+!=xy$`-x`=_9g$rZzDjE zf1}esgOc#5A+%UIVzCfX-EVL2<^y->jvjpf-MOa#$E(BDE&2p`n<{!d-Jn23>E+Hv zc03w$;w6+HTX+E)*SE|Sf4C1s!c%TmxDVgt-6a}o_fJWc{Imp4oH!)={ik5y?7u@m=tyGjlm<&-y$a}r z___wllE<50?cvkD#U-*8eK{Gd->uJOeS|}ZVBj}9z>CLF=UwUm=Pr@tUFCmL1vrQ` z^p}%n31KU&<7#0IHU0TV5nbxcug5qK1cw`-UaZ$ zdy=Y!ly50ISdgjE4ex@$qKyAU{AmE2tsR&R)Vu=O@6SQ?G;Flz^9&P zMtmz3-gs#KxIpzc#4$_IGfwUSuU-Y|J9;4;r2Z^Q2#_f0up`IgOL@d;FvT zyx!7U_-R}GBrwK_U{GpvYeZ*0_Ij6QXpT5EgDwHggPxatyaq~Io%W~e&6-;y0dIvd zwZjI)Js?=WKPRj3x)=xDBfmgx{L9KG1tyBq;qYFTnWl#zL^Q)x(^aAa5m&(Wqf?$> zU3GTdz`Phqw|Z*#1pv54;$2Q4;?FoR;SXTKF}J$AuV#mdfNXrwWM$ZQqB9+|96+7; z^N%Qf7-HpGMhbpFD1k21`zteMG9mU|cyBNt=%KTd8&HleFVU2(V3eogOgpd)HexdZ9!=RI4u>eqOU`v=0w;B#V2uf((Ame=;+|tF# z&IFNnR7}q^1mBjR{Nt;)aAE;z7Pb*wFGvLdO!!3`LD4|aa6JDeTMkhM-<#cxJL@PY z{JfFd-jlFz>g{OgcS-{g#eqy2JmXDP|2qYL#;<$(GjNYR?XLbYMiHZdhmrz|hllH& z5;I_EZN}ASAP~GWU{z~WT1elrneI}LX885-tB*5;39y9c3fh>jp)Y`rry%!mBI?(! zvA(-;9#%WlSkBX*Qw?)sWECi7!C-b1?_P9UHv64bZ^}zugtyeVHI|(~yb1>gSMB)` zwD8hxM*tOk=)5k7XeIXu?mPf_6?mr39?m3;1ZL7U>a6?80~~*PW6hTD`}d#^pPI4w zgl~ahrVlW@yi;0HG_lIW_i@GE zJB&s)9XLn;DU{hlfOOaDK$G)F4*r77sXQ>0@xdgIIk$>w)zjzVBzJoazuF9zee10D z95^VQ8_C{^5rQ~$DaK5CytBxr?6!nhS*Jk{f5N_&TquadU}uo^;WynjL8KUZF&Le=veqO zXQ_?XZyrM;y$qTbr*3NzV@6M|8B!_3+#1w9yVH+0pl)HIZDa{b`=w?z%h$O?pqt z+}8xhpj*K)tbgY1a9a4Lpj7Mu>dmpg%L5F)K5YX2Gm{m)!~>yshWGs~$zOY^h^+go z50hcVgm_%$)D{qX2j2i*2zCKE!~4GgROo6S0=jIX-*(OZrK~pB8)*u+OGt%1*%#L) z{>oE-8F1t*$ToiM`9FgVGB#nm5m{e@PH(I}1Gg1Hr^6)s*>0If(5dSmcmJeDOW%tU zvNfG*cbT6(`)+x8`8fkMvtnjuHv0DVdeG_U$TL#VaH{><@9qboKFrVGQdCuyRlj>n zFw=WWI6IIzVy+)PYd4I&^xH>1`7^WFqzBpV#mL`!o#342AbkJ+oZ_B6;E~&34Gn9@ zBJ9ghe8x4Z$S*8R8%j-2?-S>)@Lg}-+`mOuR(En)4@ITDix};i%~_tcKF2DjlT=YA zEi{_fS#xuF%!r$Z?AERHbS#$90Y3RAG2tJXhEZQ89-?c*rn`p5!34N*Y67Hg0{-(7 z>MNHu=x!7~+dm6`z>W8WfXmECkUqqKn>TL?p`n^k%QXzu)LLK&W?;s6B2@)f$O&0X z(O1QX3^@c>2hDar7belOx{5X+xO0}03CsLtR8UY5JSEM#`P2GQUvBD0%nj`{k&#zY zQu?u=px~KR$z7b}YjJ4A=c$~>k2h|RRoQqpUB&*Z4h|z6_*F}5`!AUn%mgkafzXj7 zt73Wa>I7a${eqKpLqBpl@{W>DY+Y@&jN1_>Sp<7>(+pBDOAK;mdogKitW?By35CSgHeW@@R|V>KJHr*EcyArtqzY0hsHdq2zsH-!+zH!+ z@IFFB{7EK9w9?#tkq8?(n)6x?@ ze*D|H#X4A{>#mMUS65+7&BWF%4vMC@9fl+E~FnLoZZQAp##SX zny4Yx_O)>OMqu*}j}w`zs}`9xJxICHR`=-XuELsl-)DdQ_1CS5_YZ#6*4}kGGVEC- z@V-NHxxjN0*Kd$LW!V#G&z^BK@$3sP4l~q|R)5Wb@wqf%?B>O6@*icpM?b`LJnL43{=Bd zSze9=zvzM)tFE`Bz}_;nC~u@np@fn=xy6@4Wc_$U}-`V~`$k(-HRQmKS88+E2>wkw*MbSgBC28Zi zrgCrBILCFSpc3W`kLMJk!U7RJTWFE(2@IpG#3jvFE^@X%CX7@zLe<~{d;XqZ#nj0A(8dod42+XChMF@c65UUb>!2hPeY+uEG&`HF;MM=d)W@(PG@K5 zbD9q%vEwU>V!>xlOb@b7%b@hcpbmQ~-y=$AN<@td0zq72UzHrEVPp-%Oy#8N{<`69 zhzL`;WFx9~`T$QUCMG63Kq18h8eISe#&h4$r5gTUjc(xCITitFv$)=TO%(nTZ8=p_ z7_C*$XD@CO}#Yb$R2r3E_LFrrL|qk!fmdYTCU1~K~tj&tIC z-V9DcYjR6w5q7~fjLF7wb(qM3DW(>&HH7Aq?Oc&V^_znwViTTpU zMm!{3A0=_;4iu-=;S@zdHMPt^klyFrZ(pP0o)PEXB@iNJ`zt-j&H2^+D{iBrO|EE} zw~d%zPn+S@z|8dNavf$Z+V$#`hFm z7wvH=!8flu6P6xpca8K_VY5Z~&X8{z8Q&8lTUJ~*DLlK6R%FlS>Y^s_^FQ{V7?{zL zwug)^`nvR_h~L=H5Kr5!xfO+XDJm&BSR<}bZWsZpxMNEal+qkiE514`k|)>2ClKt7 zrpqTQQ^XtXrfV#meis6BI#Z+3&F*jdm@afAL43}xJatemelP*U6GO+1g+QG^xlqck z#0 zPAo}aZ%&&lC?pLgcu)ggK3Ll6F4g%m;Pdt5Eeb*AzASa${&!({Sxnqer-|AuxCo;llAfFB_O^91*cS%%$oFhjolaKn1@9-n^ zo-^^2k7)p@=f{S%NlmpferM;ENa29jJ4{~Govf}o^^i{v_Nw_HTsY}ilIh0}F0Wwx2YyyR4FgHa-MQEd`c-oS=YTNkvv-fgY!)n?h97eZp zWh%$j-~U)H2=Rf|R;%kd?}ICMqRR4TebI4?bUkjIzsb%$$CuueT_+@R@?!$$q?Y-{ zd*Z%4!hN{pXfM)QT!N%ap(6Z)=!yyW7O6`kn_)Bpr~IG&y-5&=PrwdG7v1Ly%(bKI zQ1-K!H%~WHCl5YzENf1`$_pYcsWuiY(m_^f+^kvG`MOrSG;`3NDk|RHk*ziO)mMly z2{=B*!N1p49v@<#LzQq;NPVR1JuBZmJ%czk3a`0~RN#^XyHqYfYk@6;2~)*97@Kqz zT0q}`3XF{w+GD9S=sydvmbZIZ&MwfT$$8_ z_T&TLRy(&0Zh+yW2&@=mk@$lSapzy}-@hMPbPF?50mu(7`$kbJ`nP@p@z#;I-_JL#_x^?#Sh7OGxR5AAXq13U*j_FM^kHrnhXAV=7&7H~ph;ajfBixO&K4K{e^I z=;ih_Rlcol57JRH1ryHElv8djV1#T`j}@`l`IIt4XodLv^#vopo6=qA`dm$OD94#z zPbda?xo-Ecr+cnQ+2^Pr_g&wh&`PSY-*g?q!JPNP2E~p!K+Inc66IT$xh7kYn<@w- z22wQpe7xV*9pnhuPeaj!5;PCR^}BfS;$1Lm0+CSEA4 znrL8JGopbD>0bLKvu3rN)0qjJ*9yB=Nh$hyncpnt8lhK(Z)*e39pLiDojJSjQ2y1u zSMVhAzYNB>lp2KwyA4K`0ZCFlP@LbSqe0i8Gh;a)s<-YN5 zmD}{5V0Iv#*qKq6%CXlDe)?Uo1+0=j^KwK24l*3iz^JL<$C3TvlamdYOG|3(`GirTiMK|UZcERitG`+F-Piw z8U@y)4}e!XF#%E#gPQiz=q1Yn0{b(Pkf|eG7aJ~Y)yeduc}e8*R`|iL zPd2OR8Bvz2vNx+W41FI(6_giam~rnvNL}A_$uA~drUHzQaTfhXhWZx>5o*VY_=AX- zOJZ$N(%72eu`TD6SGM5MYJGl{DW9TQW4`O+!JLu8_4Pt>GkfI7`qMT7wsk`yBUZiq$;I!wn({8kZG8I);)mIGKFV=^PrFI^jzS2te08Ug_3z{0{pBiCCrto0-> z4>MbFpPn^C8ro~fDo9ade;-UzVL5E)#BqMj!B^5x#GkUxg+MreOCDv~AY61{oC5gL zm6B^y2XDR52WZI^r2jyxRzdEXocASAG!ib~Iv0atgf@gKmOGCgxl8x72u1mRaJn2} z<`88BSm{u(z5bp|FmBeklRZicRS^(C?GSgk>au1?(4qiBXww1p80s>}YV$e%4gw

C-OB%`Tzcrz!=nz!@lnJ(n##7=PFznRs6X6`lQm; zWvlOWS#B&)bzldKb?)tVNnYs9De=eDI+?L)--CU5N@AGPiWuUG_+ zjOEY|mfMyoJZg?b%5epTyUuL)tcL9S1IezHb}H0-ZzL{>7p`nuX7iZt`RrIbIHP^? z#Ifvlg?5EAl@0Q>c#VRO-qN(s-r9rABX^++3m1)SM#c7H?I$uk_K;Y-6kc;}Gn0|_ zP8|%HN*foi4mO3SnHPw4XQ0GwjgiK(-P79OdvIaOutqtP`^r%R!g-)`c)p>(bfCu< z^gno3|EiRBnMA~2THtF13If8?R*nRq{khpkSVlzuY++u31=@r)9`ME0twL zl3Ft%Y^b36+VeSwU}X^JtGWt_Y_3E3fK!hraYtl1# zdEE-4^yVoO#AQJ;3R{C@m6c{=s<|<&Rkv8k22D{r(|tvU^_d1b?a*VAa@ebL&S_>3CS6*V?5yw_X1cJ;B5UShEjVhY$I zM`$jxwDc<`Y2aBCK#;Z)j~)6;mf45(0kqms z##Tzl_ZPw^r*>b_u)FuX-!ip&z~A&L@M|$wkbKZVBim-^ZE@`V8BIwoIvL)e^Vdk^AKOy^L@{B%%-&LM&imym^!2M+H0o zjFy5?MA3Sp>)AZClh97In&G+blO0Ymt%`jS|K;XkR}I!!~DYfC}X)< zqadLX;54a02bB*Lw4EweQ2S%3S@}QZ0Z8^vF2=O6*?Kngr1s&(^Sz30; zs7iZj@Sv&H^s>XkPlMrC?(KA|)}T|K=$>p?xUtRDZg1O9a=VkF*Znm44x%>I>M|(~ zcxRQ^r!3=LIjarUW3it<*_vL7FiG~FE9-eo&V1u2o*>jn2K=+?s7Ud%fqG=&6NG-> zp+rR})Qnio70L?e8MP!q^WzB%gJ&j0YKB&Jmz;GL@8qFyACz%aI;(_S7WXjrr6c!E z@MRZ^G$&LE^v1Vd+KEg(G%P>^KnR?TjP8|!b!wu3fQ@1r}wdx&PDZ-Nn2xF>< zAV+)3NU;yqIZ4elZZxgIS$Py4%5JLOM&CT`vgtVQAtG}8 z-t&#>Dyp+Ao^I^g@ahKXsRr`9%B@FO34raLb*{I zR&5jv%D8;4<7$=%s7<5Y#g<<9LkBv72S|5VH5>j=cON08FN2p1wF&2Z938v-d^wf% zk%9l<-S5$27J_tT)Rf!r36@d1#xZpTWe$9RM6pd3Xp!%*YNm~DmHJVS7|hNfjolPv zt+2p4m`bvsz516KoEi&h-nhj-ti^`Jj+ACMsGe+hOsMu=x;%~*Yq`Eh);Vgt>+ZIG zeDRgLQZj6*bW zUZ_+e;-{F{@7v}dH& zBDJPoL(5hn<&2$1wulmt?XhR10aDA)&uu#KQ5{AUq^%&2~PbtiGHv~D!XdFfUT*xavedg+2 zV4p-jyTOI1apvkQ)OSrY*vja@{n5_{I%s1%)bzTU_qBq`EX2X4B@l~J92|U}P?9HDU}Z$_ zxu}3?8LEuy&!yBJGwR9hw`Huv)=R-1UIdj?^k^dTXUu!Q&wAPSoJup|c5#RL^gOHW zVFR4~hx5Ba_kJ(&l0Fk>l&>+`<^Gxx)l?Ff=fD9dp*~Km1kBR_ZE;>)zj7ZFNf}NI zV%M8M7V}}^9Y;J=&l&V>jrz7+L5{g~gUCNyg32`o$D=MdwN-5a(Rkfz-A|WO>EE4- zrH)isG^-z!8KK5R^ywjz;&#a6#5rJYNm+ZbG%x9p|NF<#>euOk)-zXQ4hA`%{kKj(_CPX=Ioau6|Ge34VeE3D!0Txr7y2xCHg<(Hgx0w$9d4%bF5~G{ zfi>eX;sq=EnlaRA}OHqyyRJ_wcXr&@p^ zY5e?Umc(c`({q;6QityucY<^yY)ni)3x=JU9&$uY!5z`@9SF16Cp}{59@bpL9J=t3 z8H{ikobwnhj~FMK<7`q&^VmSLM=T!C2!}0H^g7_glREv9a?^-fi<*gG_dn{hc2Gzs zeqZ-fp;|&GlHC#qjYPQUWm7%3P-dkX+=itZ$P>Bm z5?yI9x@I+Xdt0pm-xXp_z#(+Is*>#5d}_!QK+E*XS*r!oeS( z;Eic8X$LzM!G@vRQiaox*|j?m0U)J;{0*qlEAUHAflhfNb0vGh7supNe$Bi5&RaWV z{F#w1NH^Zv7wV7Lu$KyJ`@$1X{LZ%YS@cV>bP3WrNGQxN8Ekor^5LR@eRJ*6Y4!9@Dr!>>u3sf|~0bIeVFQ3BXZGo;@ zi6qa0A9qbLcj5d5cbGdKF|8OGA>Lc+NF-S;-Q5S4zCbeZl!eLm?0FuGoKhYGlH|72 z%ahqIjlRU%_qvK5aCPM>?8C4l2&_G)ec5VmD)mLnj_rQQt$rn5_srZs5s8A%{@4Is zJPN!*;eD!{lwihgKUsmcr$K&ou@g@*Qk@n~m$Su1XA2XNclvE|S0A}c_6^volKrAp5mYRN}bO1x}U0~RD;>q3Cn>$W#bgdq^+v_VBiw8*>#YDlY3p3|e!+~;r z0oqClfa&*f(f$6qF#1@CR3Whf?2gHUv!Ge7OO{7JFB|{|BFWuYH3=NrG9ClZS@4y`bZmveyMWF7j~GIbk{Q0@feN;@eRxg8h90$?cJI%8x8MSOI9RkfIy-9wp#^adtgsqfK$@2nlX=N*l)*tYcc4Q2 z;A4>;9p)shTzEPttY8^ydzx(&p=owDy^AX^@j=&n+a<%XE;ACIVfs_n`6&Lg`WLzD@7pA_-QNbPm z^6U0lqU9#tCcC+CimzcFH0Pq({-rtjH@1hnk$J(#>YWJ{o8p;6>8qPM{-(?1eMB`i zhX`s1J-RscL4Sk$u|c=pE-4t(g9TP@gB}pBlvXKV()_4w5bk2bnM1;2(!hblU=5#i z`s3Fue_j0IRpTQB#?b&VWu?AewA7>To1-a8{|o6}??-{s1>&CI$jh--LKjqxu(FUF zS#N`zJ{k0NZ_JTk@Kd*x4s7u%`oh}fwA;}rYno>$r0YIB+5b_e-l=6uXUgPE)IL}h z$e7q6r?_xxhlR0SQ>$VAcRCH`dM!F;KLz5sa-rL+uctVI53u zhRel5ylJH*_XEBf%9ayad99BtsIKpe zNY!Rubz%7H#wY9>ZIsjr2K z``^7YTV54#CDFB2;d`%l15RZcx7M$D^2OApa5s!@4n4(B{8So9f#zaBdRmBFE2joy zoW^dok%QRI$5?d|-N1r1J`r; z$`z5>M#iewg^QLxWoK~$0x+%E6VTz};^He=lhq=96~CBlMSv#%O&tY9Kpka=lLD>C zkTc=kf3VccOytsVLvhPTb*GUS-||?=+R8+Dpbj~yQxl0>bmq|Xvf4N|!{h-Y%=!A5~YfCYVH znX0OdV2aBN<7Yw6-w>o5g$?~wS=puI6NTkLhDCgdvIEoIg1HcT+>EY_X>s1@Lz~bo zSscf&3bwgdV?)PTVO;%2>LbLb-h1rMX1RJGBCLs**aXRTFCYk*O4m-_owqbMjH}+s z&lidK4W=l`VDK_vj+#}oL~3Suoo#mdvJOI5PM0_J5W99=Co`O~6>#*J@Sl6v zI+fKlyM-sKK%E3k{oN9uZ5enJiR37OVL%m=Q$x;Hj|pfTZ8Wg%4EB18=|EX_7FAqUp>co@F~WmI9q!#Doo zw^HzC{v<$!ktks(096A(n3O*spipD}e&OCSbaV9wn*66!4B%Wu6G9O}5*x zK+v=M%ys~afHolF98()~K3RnMYA-Fe#5w0aragkXdPoszu4_vG^#X)wRFenlKhPzX z1oy@)9p%KvOAj%{gw>OuxN5tAK|wR<^Xv2&f2I~@iGAM+j)g_y7K5M{=~N_b!i;6? z6<258aCvWzS?E0^i%!mXY&3$7qE7{onw|M@^siO9%999-982n1|j zfS;DGFHS2tL8K>D&{%j`X=%&&!bjZ}qsL6HJvJz6qGQNxmTeXxaMqjqJR+bjY%f|c z(V9Af`Kiz;X*F&}e^wcnA!gbJ2K!GN<`a6o5+CcWrou)bJv20xHF1_pyFF!S)#BM^ z(EY5os723EM^|$+XfvCU1c5xo`NiDs#Nl)WRU^`C;LPR?r4H zWctH=?xWZ4g1u?xqC0*#{nwcp>s?~0abI@Hy$YzDuJymVOLI`g@*$;3e4JPykxY%3 z)(XeXCi|2Atl%Pdam;u?_{NW`ACMgRd)6BADPC&($NWlI7l%!QSIo{+ri1|0r#yld zRu^lry^BL)qpwLdphzuXdCB7a9_24yAJvh$<*kyx6IUT3;D^TgyU@li)r?#txHMwY zTj>u^m^!=FgN#7mmrd@?)E_L7HyIBSr!cC0ITsn4-?>g zere2l_s~B=i!zpr(&;gemp+YrBV!ngwdt7bSrJNO?9ao~ll|nRiw$a$i3|YwB{DX4i>0Nd$fa!; z1jA?_Ad@&P^T_N#r!(&#R&H8(6D5E8s#W{V6H(zNOgwj=?7c6m_goM|6)Lr=BlTA@ zSn}@HS4ZN4#K$9$J`%aobDzU5$@$3Hn2oG$_SdOLX@=3u9O z_zy-Vvk8K~eIE}VBLFSeQ}B_96F+@x!0rK0B*J}H@Z8-0$9eFV|2Iy9{%42#uhG8E z|G%vAe`|LCH5`25{}>HI+?V;AH2?3c0rWq=K;Q`;X!NYc%NZF!^6w1K@xE z8)5fvqd@;7QU34Y{%bTa|9{Ct|FdgwTPpYY;D6!BKhs%;n!JPT%;){{bjH Ba&iCw diff --git a/UET/Lib/Container/rkm-initrd-builder/files/usr/share/background-x11.png b/UET/Lib/Container/rkm-initrd-builder/files/usr/share/background-x11.png index 9382e91ffd30d3febb02b9216a387a44b70597a7..d7f60ed49649c0b63ce07e844e170cae8ad0e3ee 100644 GIT binary patch literal 11059 zcmeHtS6Ecr((UY~fkr|T6ci*WAQ@C7NNPk>K!O4iB(#zxO3o}rML_|PARvf}faIJ* zBO6JQa|RKRoFu1z(QfwL_x(Q)_q*qTuUV^B)f_cyRL$Apg}Ul>%43Yj005MVH?H0W z00sX`1kgnA%PW%!*~2TxD~g(EGgwwA%woe#BF(3Su~;nrq(BZmPr`97VPWAnv}c}Bv${|*Sx}y2 zV`I}HKdwuDLWzv#GAR``H8mgcF%}dl26+fT!=NY!ej&5Jq3Z|$mDAxL!ipoe8@_qk zNkPX+!_Lgf^`3(%kTbEnZ_0Mf(!tc#&cVu&OiwrrY~ zHl{%MypZq(p$nq*>*K!wU;~O* zC%{3pjNdvD$5JuB>C6QfOo$RQDE9UJbpc!Gqylx|!odNq7wPf}-ODmIX+rID- z?0KMWmv8WnvvN2o-e_q0X@l6fo#;oQr2B~(CW2`2K&Kt_ajqQNKCt0DG_r5w*YOE}7b8!E>^ ztR$j@vmQAbJbirhHv__}g5P4sf#Ayva$=$GI56a3g1}uS-fa1q1S$=*M4Y7wt_;BL zeZQ!|(EY0j+|JvEVv--F(?#z};tz|(MFABSMpT2LD_0Rq#l8}ST$SiimtQ|RPc$9E zwgh=jIkOS$WDx6KAX%%*rBM8Q-P138(TBC@2W~s3FE@0r^^hTPi3|vos{T!b_Cu~< zh6bkaFA7FdG}a%3tXSl`5>jCiC-)zrut-CqFN0wjVwb#w2ULAKM0in*&Ia!D@5g5& zxiTz8;TqKw;p~yWN%2xbNOZHIsPipC7OX&?E?x{wN?>$~MjU>8Jd9N02Tr~P<#$u2 z&p^H!yE!L({C5P-Wjr{H;)#atk+m{_L>7Gg6vArjH=Xt+Cb+}~DD09ouF-Y$RU84? z_8DlF9iRWAF`5#IosmJ}ZXkc7@ccqZi`Y_Jf6nCLGH(Eysx-W-n;GQ3BTd_Dud$qs z#=uV-BH6@wH+@6JTu33fDxD07JO=59_j2Ka4XFTBr8DQwVk|UpqMdE@O5Khm!6T+L&2sY>I2;3Zc4h`ux z6Lq^2jaw?m94;M)WGk9p^LgQl!k@My8TLN-YH39G(u z0$|dO2D>=l$dyhoc#?y?D5wDZ+POd(y?z{bpc}}O8bb#o8io-`y9Q0!kbwayT!$g5 zRo2$MeZ}kvOkgQc-ykYL;VFcW)D(B4^D0iUgM5l(d&P7Kr!3PI!0OI@k>%fl?e~mSx%2BgVQNDYB+?%y zen*Ow!q7H?5>uTF;bNXg;vH~;m1nYa+Xn@a>Knl~wPy}veL_L^s>50+``tdg+C*YhCF!c;j zcU;0tQbJL&$h;9r%Rjf6tYQ=yK^y$o$jL%dVHG`f#t$X_SnzzBanzdxf|*>RtZN@( zZ*^(9&*T0O^Oz8&lmc>t_k&=2GyEMJ&WAP_zoXk042a`Xw6|R;qqq62URZgaL_u#e z5W~ixm!#{d%UYvGxaD=yK6ogEMyzAe;Jg7*9w}5XrOX1_R1w{P4ZCxU@mF=jbJgUj^hrXNi^AY$$)aS)0;Ak4^{8IXR~8BcTcQ=T z4q8Rm5)iEp$Cfy{MC87CevacznH{4B_hHVc0KEOSgI}eqy-6Y$i3j{)EMaR90ZUmm znHaO;Ja?hIu) zNl5h@Xf5<#?ra@kT+meuaYQY*}@1+la^fm;;>u5VA}5_Q(j8OSR884AOK)DP5b zsfJH{Fl1Vg$T#{dlCjg32A?l3|M+Zdh1*$>@F!msG!ZD-rHgOX_UHlS+_04bk)e?M zjhGOa5A@+Ff9s}nOp ztg$y-B12&{#`E$&5NdbFsGQ1&);A)`0s zAuA%@hffJyH5Dv%_$)D9-V)tZ@0TjF$A^sS!1l>2?+6-IX92z>UdHE&X)W+;PuJP2 zmM6~(Mv*@xbR52^$y{ zB_boL!Tvd+pIVH*?(Z2k>`Uw>Yr9qjy<9&7@g>F(Q1`JI zDTXWxvLJC(ky-LrP9&D!=PWaCjWupN5M(^v<}CSQ_mnMO@GCqV_~rPRxob2*b>tTo z+SATx`=%vQmluFqxmC$u@gb-E%h+S{A*0D~de3CD;K`&DV0QUNfFpFvDLzHt3waH9%1d7pR~VBBWtX4=2Aj}vEP%Z#8@~VD*UbFS zeg1k%x%-p?5hMKE>(rysTP<^N!$u^3*83bq#?g(EZ)dN1n2E4Gx#aHxGq2pGX+zsY z%w5Ww3|Zf>--YFQ&(}&84rOA~b$=8Za>*{CSCD^9cUZ|iD^EG_oYN|mzJ7u9`1~E? zP(DB7ArfxfDC+=x3gb60?oxJD9(9W)-;;l=apJ8QGSv&HBKk4R0usiYQ#_L(1?p-T?{`Xbh&RD4 z`L|2^ddlnHAYMlp(Y`#X_~K)3EUSU+D5$H!hD>uGqQq8g>oC9IoIEv}!k5hGU;g(0j89n`@jbgJCp zawo;|=*b#y>f=6IK}_+vOsM z`1Ik_BUysyVK$APP4HDmkoV#e+tRA37_?{ji%~HePXv#&Zx#iy*9H%S{wm@9^}y)N zJnR%Ly5t5s-RigT&n`T8F2;Hw;e9jJ6y`k3Y?5WM1k|{#d!{DvW{%6~ zVIJbq8o|8S-<9})I2dlhi=sp5io7A)i+d@bRHSdm-Oi#y?5Fuy<(Mtu2qWjESnt~V z_4>7s>7TTcXRDl8GQ~gR%1Oj!kF$eNw!?9^bUmR>@O_2r_e^H<6R{sD(cjg0bo{|o z>}wR3mN0zw11?<2bX;YfO6KU+lIv@GvZj5T0y=Z#ulN7UDzSPPOJ{@{bv8e)4S8-O-QsQ_7*LR&zObyQzc> z;vnSBU()HipE|Xs9nw4a~mYheD;0=Q7_jdbq?b#~Ou*GvmGXd*>O8$)TFy0b4t+0;KpJwD>F8-c4JzLq>La7A%4bDoTgqxd9*{d>vL zFB|=X+FU2eaXj_g`O^CkzVQ+*P$cZ=ZpVP5QPe0KUNh9$tv%T>bV_hT;%GrBqbtTT zXtd2k8l?-qHrGpeZ%n52JUSxxTlf9HjX8$lVha#{mhQt598pojRXOa^w`dxr=xPWb zlXs+stgZRjIX1jb=$4h|Uw7#SogV#zBzwoU!b#9+;V^~N$Jdys-I%HLXu10uQz-^xDj9mUt|*&VK{%psJ8T`B^7dxb7%v?x7PUo z6Ob5!iC)dYBRF*{3uH-~Z|(+p2?~oJi#|$k)^T&v56UQ(dlqK0*RLweChWET=ExW$ zC)Iu4(ZJDqo8*g2$IqqV>N2=yoAyiE-8_Q_$x;; z6b9eN7xgMml17^B_p6GqWf>2iJ>qQaP2c3b)v+2%qi^0Sbi!rzWj07PoQy zIiU_Z3EkH7Zmd3?L$?p<=sdqAJD%e(q`^uJED7rDWdxH!8&%hb1UREA3&yF?b5Xm` z!Hqu$=Jh?@)vc8eNyGh0C4rV-J?);W9wp}q-d-P}kVabwOtX-n)`uyhjuHrB5rBIh zzwDY$h)9XHW<^@>rmqQN{v5DMOY+i55kx{?Erl#sQXhNRJ%Sm#LTFrV@D_u7Ht*;f zCb92b$Q`9< zljx;ZZ>--xAt}M?y4iw@a)AadRt)5R^lT)!=Dz( zC>JfW#aAY@WgerYK->{nyji9{MHrE=0AVQNpb$2aqLh4;PaL^%stkBh;|7LNoUf!* z+*&xW@=6-~eU}J_Nre%cNiw9$KuiM1s+Qa%@2%n2#2?x5myUc*B@4U9Toe7kEyc62 z9%50ykVmr30m4Rp#u8}iaFHJ(V>w4o)|>0pIB9g&JNGx?BJiO8dSrm#>V4iD=r*ve zjuN}4c$N}9SA?;kK8n^3gcluVr$}fbMA}8h+W+NLV_sA%i4KV{iR04Ocx^%(PkzLH z01;kl^O`qhP*DO6397*#Z15+)^4L-(NQ9ep+>FKRQ#rL!oML?vS)>Y5TtvcGvk6ea?ne2&(8P%-|U6j4$o zW^58_0fjoz#i;+C^E>78d4!jkiVH|B)S21$*9wjgl|dMpKoP z#M#uvP(|1NZk+gxn8v}`L8 zg>VE;2Cc47pUltGCNkFlFS_#0OWwTq=!rBb=S_CR#(SWbnrKbZQkVan@YmECX5}C+ zTWT9c9uiav7oy~7+ls#o+SRC&1$p*G#-9Bv-kP65Q|(}T;L>kSg%Q5<@M-i&-xd}# zib2Js;m19(@eZ`{!)l%P@X`8&w;8R}0RH4~Ou=nB6E8vTq7wZKFe#tP;Z-9@HE(W? zO9q`4cHA*!Mfix^=pdI4)mn32o=K50KkC2R(xSh=|J%~qF#|)#Tyh+5y_Z(VbijM6 zKRzjYmQ!WvCxxnLoQZ<&hWa#TviNH4ocTshwaX4bEgIm@BZ2)(_x>XRgD1*^dy>v17E}@<38HIM?*ZPdm@%<@zCw3|6B>~&DEwwzb4*X-pmxjtMyPBr4Pa#m z#Pr2gd#+^+P2Lx(d9=}uC&)?7U^7+6dJqayXXHSePSkn zwxzKNd0%VY%tTE}D7Dt1kEL-YM98w(&d5D~DbkC*F>VwH;TU5R$e;|0s}!^ zMQCoi_mH?#b3w>n?Ss;?W6&&vwl|0w&mPXuUJF3xkhV? zjMuXrdVk9|OhdM2IixGkU4ZItzqX-JvOYSAruI5`nYrAwm($=$v~s;Wv-F$o-ZWqF zzF|v2!rF3qf7wc;)*?mu<{1#@qHE2+-qpBzvRdMq_D!NJAABBThatgd^agWB*fWDJ*;(z*e6dts7lff|%hNlCyM_8B2x0iE46m(- zki~hH0#Zw0t;VEQv5KRDHhz>AZ0WznVMb5(5*1NEGt{GbjkJ0a2;oae9CC|iZ^A5B z-)SCECRW>Res|Qr?2WB~HD@8he)-1?HO`X>Y_-)kE_>&@&rm`tZJfPxM0J>-fkX$@ zT@GT-tawvk^4=k~TuOGlufzd~djJca$~o9w8;F#5#?EHe$l4N(es$*_Y&KQ62skK=mqpoKQq*H{UZ6$)u%@bV#!Sll4-MYdd=t z9&A=$UVYg!8W<_%TX;Ebab*|5+TFD{%|Z<>#B9fj)~Y-d8;sng(yiT%bNf7`vFW`< ziM@Ud^3r<9(2?`k@QGNNw2B0T-BV%&TdwY+OH0jOaRA^hJQ#PmwTQG99ow)8WSScy z!QW%B7^xyvSaG!-AJj|DejCB*lEIz%^Y^_qPD(dEel&=>PvonoeUMUcFzz^{UP7qlv=7Tzb5mIfk@-xA}_5Z(uPkjKE7Gm8(%NGWg2WQyBPdR zA+j-LW_Yo|XY*rSit@gPDFa`Fk5^wsx4|b<&OU$X${f8rcnQ7R`T{(=}V_f_k*XpvMYQy?rpK}97H-V|K8d> zI1OeeJc?Tu17qhE`6FW{Jj&|__AU=ZakKExX$LkvSpMMi(Z@bYx~S62C0{oUQ}d&t zOA`||+c-XLsw;E5-!v-UP)O9@qirpxp#;-tv)r6hXW+cEm>O=7#y!BCB75~iT7s@Z zaoU6V)UF5FO)NU}_4eDlNiE$4p@EoNLMxhj@i*#VQnz%WHLJDE%fKaYZ|4W+2%?uI z()oRtV@0{`N<^i>X;qb&k{Bubd3psL`l+w>yQI(B4TwLp{d1G?2cw<2|Z?vjHK}Op4hh_GZ6Z%@W1X4X#^1s*o=;?Bsoz8u#?XdQL z!I03jk+LdbO(j{sr)7O>E5pa}w4=GQq|K03oL59&-*thHGtr(Vq$@3N$OkQ7cdJMOhxGvKxy^lh|j z)IY?4I*f&wG-63p?3eerw_1$xX5RwPSv=OHlI3Q%)dFW+qTQ0D-KpS`b`zdrIn7oW zV!VbL(D!-%;Pq5HQsK%o5W?fCyU|sdg5ut8GGyrTRb}3cnYb__YtZT07KzBpbkTC{ z3%c57S6)U5twilg7Sn((KmQnWN#v+;zH3=3Cct%~;Pr+9y^w?u-+}s?Ory%{s_+xC zB8PRJuATw=~{jSfQE{`@w(3PXA7Dp7nyruR&-5KuM zKUnIj>6jnEpLBgs926~ZhF5Cz*x3C%M{DghYrS&StwOw4#%`c*d2X8Gu8;K&yFJ?e zd%;l6{WT8>VJ_d<`)fX{*1}*t)mnN>G~{61i8_w8_NT0!nw6vgUuLP`M_rShhGsrq z*}t;49;s!fML2t`oCdF2e7CI)_$0iad%T|Nw+kE;(#vSZ6C-g&G3TKgqMZSTJ1q;* z2ey*N-o3>1_Q^uB6f&NsMwv;)p)%LgJuC>iL)x~?xN5DLr~#V!2k9!_&;~R{%_>}&^wp^4~X5z%stg_QU3m|5dIx0 OP*hO8nsEj9_bRZJw3J4#q-wLc3VFNY3(-9} z+vM&D{e4|fCXiq9BHwmL-fgFNMa;OjYN7ZQIC!LyC;>P#pzq(#DbNSE@3B+<0PqJe z|6pFbO4^}^q5(#Z0Z07Ky9an;ePr!Cz0U)c{mLo_lnCwY>LHDP7x}v&_i#S>Xr*<_Ts=m)) z{20vGfzvR8yQRwWc_4$`fo*}RANVC2Eo(eddo==;%t(jOKQ@}lUP>P?e{;b(yB zYkqM}U;JXn`U)eAUS5)8IV-%`^kzBzvlnKHzH)1>;|_C8exl{Us^FCs{iW?ech;OA z*pN1==UZOKB55n3XcLDkq7ou%y8%9Ey(6|Qc(NN~%F#61ra#0Frz|Ae=~Ub*Y0c07 z2S#m@iP%_2JKN4E=fEi~Z8d+b3*4nlC)y+^4enFjdnpivEsVyxa1OcZ3WL&!>0W;c z1bw3$I_0|O-%Q{VAQ2UZqq-V zW^$uoRu$Zorv7xGuV3SLB|wu24aGs8NF;PCyn|ExA@4a2SXH=d>z83BF;XG`6M=Nm z6;2AxbOb3%u+1wpG`R+27Z9M!^nK4k{+i(>)6@OwI-I!Na(qTjrMFNx-9#(^s_IoY zprd5B}whu`S378wZ$pp^NM^M=P!=JR3n;_7oTq0Zcy%9;;EtXI; z2Gw(-69k1+2Bc(aW5l=!cY2Y>$BB`x7cX zKmtk{0Ce7*qCMO|`!`lgfzohtdu-~6^L-bs^2c8xCDkL`aPZ40vL+Vp*!^Z|>=BDY z0m2*ZVsS9qjk>&AHC$_y83T#pFqk?{XtKOCE=SP!P}T1LIv@P$TA>ynB&S%=nGT#` zR$7Q4nx31(<`T-`Dl@qltT;%G`VmJ_kc3%PB7+t!fU*fenO|vA721;?+|3ex8A;xQ zg_l?0YM-CajD^yef_(`+4VRn+2(~kbz>W5IXD77;pd9JI@jlNcW=P*ffY62}XG2b| zi?z`#Xck}}e$p6UL*irxbJX+>zU)2<)ni2%#K}~Wa|xy3(6FAH7xu&RR8|seu#c|~ z)!a|wAd5o|e9w+3Xm5p*WCQsuE*EZQ>Q`%E@T|EuGj$$tq>p*yZ;+5=aT(6)O?ur~ zFP5PY$Y*kvO9PK8Znv|v{??x-j%GiaQ2=Q^wOtjrBQ_ABb>ZY!oJ0Y|nWNB#tKmdH zL0>h{OjDtH?IerEisJr;Sn_}s(^j2p(r}8PIY^j1GWd!A8EbtnTs8ttaRQga|54If z7NRVljN4hLEes(lgAnPEnipKT2|xAR-a~rOEepFCy!(;0HBJnSFAyBylPM41ztts+Fqfhcln03&^!3e+zFD&p9BGI% zDjiOICD1wLO5&bz2{?Dd0|NB{bn2KdkJtBA+RBz`wi=p_hHeiE5Ew$3>xjhPl#l{_ zVVhPJe6#7@4S*M&IV;O5tQ|98$`S;D+L4D8_gvArucF$$x-k^IaFBjx2Od~|~ zz5Lvg^B`CU5fnzXT?@y#1ej}DKy_^7ZlOu?US`zCoCZr-^lw)Q>F89$(Z<+FkKQQ6 zCnnf?JHR_(zkjY!#L$^OGR5Ea#0`Gb5rdvf0fglTjOG*C{zZlpZm6DY+p$mJ=>ii) z2>2!>dqs#6Y?^yEVJ8;eC=%w9ISK(-KoC{aD+8=&KAH@lGR>>xmR+T`4f5zh@hT1j zyTDP4Zo$Q;Ic<7wD7`H@>r*f4d341gn-Q>+ZY^2V>OyDExKn;$Ox~XiQG%ot2l98^ zeLu@Zf0I+vgG|!X_j7r|%%j0gs6!pB))>PwT9|^--J-+zI#z6BX$&=7VXzvvFt__U z-M;O|GVIfX9hyb${!qQUkOrH(4bY%DP}rdV*06hN)-BfE(1xm<`vhvt3Us&7%QL{< zgK@DUp$DMs*`Pc`xlb(`9QvW>mYW=1U%&!;5}eb|oU4Q>p);rD2yG&9|32UHno{ay zllPm1dQJS=A`E~Z6AE#jR}Bs zHM#>@eF2uFJ9IiEKK1sAKZS@Fh3;U5&qA>dqkSkN}=I%Dpf0D>e%(%~k^ z$Mbw*S!y2n!+eV}fLRaE!xD+9Cp!KEWt&GnZeTng@M zOW8@ns_#1eM6fVDMHwuAcF+Mar{PsD=68dy5(D21q~A!@-NxEVmjyfqK@0%JYeUI5 zDv=IQdwndj?Di|H3NUMbpsg}1T|2==efP3X`j&C@u&A3ZcwruX3AnsQ(mp-)j=I@m zw$YMPA)UZ7YJxV8iN~De_B-!;KmXWMt_Kg>pZgS*W9|N$BPdofYTG`j?7@a_GOtKb z%)^niGlnMxOj(3naE`LzBalgP;yY`#jS0$fc1uaBdoMFgcJB&b4vl!`pqM!rs`fw| zzbmK(fSxMXBM3tf%wF28$sssMh zYxMGD${VO<*Ykw57Yg;Y~HZt-2=@k z`$h|`;>!kViYy6@Ac%nzdg~T%vgz?M^6Ms1?_J)|-EsRZSk}h~BJD}j{ldh;En-vB z+|mc&Vb4p_)LC&$U{>^{^$Y#Ew=MU(ynuvz(mZeHfrd;FBm}HkJNf|fr8k;2IjnXl z{nQZBxd4^cVNd+8z;GgdSzE6tL$I2glB>J3_8P=-pk3QdR{cgwsMrk(ZhW9?=E(TG zZ+}8E#Tue&*zg`pz_(E3QK`O8+Zp7~D-rJDyMns)8uU|mJ&l>7QIP--aHrR^{C1MF z`UfVCQH!@y;?77-v(m=EDTC|b9@Tj+h)$gAb9EsdA0+LOo$(PCUTmM@`oe|)hB3OC z-P_Hlz5hoBaFy5HcX9SCL3CTeMM#t}q%Rw0>602$Js9EvAst`H+sepXLPQ;(Jfn72 zqnHlOBZ`^nj_LSqh6WY?^B7WV-QFJ0279oMIjAe76973-Im;mJ3)_$D9h^Z*&t?Cu z{pa(8Y2!)E)r#V&WGjePyr3<9RY(+7ymVArD-- z_HgVNwKJ3uQ8F0sW0YNYLGk-N(@q#oBf&;NK0{8^>%7Z z8PVGPV7gcHX9NLj9KZcpBC>^Pm`~Gy47MF!4c`!i*$B69=$O)2j`*-Gtx@jS^=r@% zr`Npd?jP+bEw?1NT*SGv)SweF%1&XF|wr~A?uC@Euj?0br zLkX^uYJ%m5gmkVzKGX~b)Jra}-HX4g0jIa+=bX}dc<2wq=A)j#$`(esKOZSCEk>X7 zVlB0daoE$Ylh1xS&DjNP-+b>15g-Ni>|!8&+c>(%?Yqz2mngDYHYdDKPoK{!LNufy z1x+PD7mDbo!0&t!j0la1BBEFgdL#j0KA%D)nq=4cyzp`8UR%(o_75^;HKTg=p zP$)?OrF}OAoQoq>%bqr~=joUulG*8XQ-Q?vuAkLLR782 zKdivyVJgwg210ocb9SZAxety@R5A?T7RM&p<6w;art;u%EAv*SJ5~>b&-r+pyfHI1 zs^^3k;_t@@0$Ub5d^awmIirS@xk^g6+i2~jiC#~~yZ+C==JkFTJ+7e2jbV)Zc;6-X z=f3EGcju)~v2iH^z0u7(43|E0e+Sd=`{|yRe>c&+*|=70J%*ntla6mrD?V8()txFL zR$=mFImp`};!c%+ue_QTLq(<7^~PamN*e9yxZTgeUFe`evw;anS(!|bLgttkBn{P5{W$x2=kMKC?Wjx>fU9316rfK|imMCW zoeSmRA?#r_ab=hl9@)@#qYQ>lt$})Y($EhO#m?FBV{-oW3-UgLTxQg16#N0hh`kdM zkbj`Sy91eGR*(Cy8zRL z6)QR(CK0CSdxgh|t$}$7JhXdnY?3beWX-El0i+UUW83diRZ9#!Y5x3dn(sv(lbXdy z^6=_z))uE--VJkir(PqRW_o$_i3AK+<*`e#)Zw^_o6q+^RPSR72PaE)>gE1jLoxh& zh_a#+$XSIaVQ2hC)cYM0gOE&3LEJ#M|7K>`Rculwx@XXU^8K_R+hXqV;v)*yp(he* zUV!&*k>qj-;jtcHI)dE;msKhQVwUFAMK zp=}0~kKNh7VfwC~2@JlBUmaeqX$bcgu2x^%#n#}949g{FzhEsN1Jjk47PJZDMs_>} z2w|E_x8ykk)8VKF0c6o^)LypNs|S6p*9hGxpDF7Pv`F%jq9CgZ>KD}F0<@$qX58f+kDzBu17ZM!7F^ zeA>wr_uPUug80I&Z3lv=6JD|+OSxpaq9Rzns(OyCzK`P1<`0vbS1Svd0(=iOJ{g!& zp!dQO?F93D`c9_=F3Gx$S{&*y*P<5@n@osY1XEVJ4whbKHCD__O$3lWIt9m+4Kt&EG2*2HiP^zjaiu_uWQ!`qkvybtltUAK~hXg}G=p(YoMc zUeEI7_oYs4k%-7i=yqq*8E+DrE&vZa-uE;;pc|5c1}wyw9dIi!*@1kTZU^O=HK4Tb zOp0K45FZYMRav55RE@RG)%q8ih)Cx@LOuiX0+eL} z<=f=OZhA!%Yt-dr@@2Jy+@mx;f{093+F)zw*1qZa!v@m}qVnSnUDvUf;V<)EzEvoD z%)BwSulXoo1s&I_jLi1-iM*sb&(K4fr)c0%hKhs!R zWgawdOb;?0-njV)VRmE2h3j`)5H_dVqhK11ITV5oGJ7Itrj3e0|7mf5VyQXOd$MgTk~}15kimE) zqP=OOz-pY)Oms+WT}A||IYQLZBF&f(mpip;o5`9I?D114^tkdei1hl z7?yk67&@$RzqMYx=vk>=Uk1@~&p{t%hEAR3+1oQ$@~aw8meP9`mh%P_4?X-!C{x^1 zGujcO{G{*icar>TTmO6Q_TNj$|K7X)|9K0&9uX_q+Nr!qON8Fl15=|DM~e*IqW%}! C<#1#G diff --git a/UET/Lib/Container/rkm-initrd-builder/setup.sh b/UET/Lib/Container/rkm-initrd-builder/setup.sh index d3b1e56b..97ad43ad 100644 --- a/UET/Lib/Container/rkm-initrd-builder/setup.sh +++ b/UET/Lib/Container/rkm-initrd-builder/setup.sh @@ -6,4 +6,7 @@ set -x if [ "$TARGET_DIR" == "" ]; then echo "TARGET_DIR not set!" exit 1 -fi \ No newline at end of file +fi + +chmod a-x $TARGET_DIR/usr/lib/systemd/system/*.service +chmod a-x $TARGET_DIR/usr/lib/systemd/system/*.target diff --git a/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnode.yaml b/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnode.yaml index 01d41197..4b3743a3 100644 --- a/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnode.yaml +++ b/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnode.yaml @@ -33,6 +33,12 @@ spec: description: If true, the node will forcibly reprovision the operating system even if it's already provisioned. nullable: true type: boolean + inactiveBootEntries: + description: A list of boot entries that RKM should set to inactive during provisioning. This can be used to disable network boot entries from network adapters that are not attached, speeding up the boot process. Entries not in this list will always be set to active. + nullable: true + type: array + items: + type: string status: type: object properties: @@ -68,15 +74,70 @@ spec: architecture: type: string description: Specifies the CPU architecture of this machine. + provisioner: + type: object + nullable: true + description: The current provisioner state for this node. + properties: + name: + type: string + hash: + type: string + lastStepCommittedIndex: + type: number + rebootStepIndex: + type: number + rebootNotificationForOnceViaNotifyOccurred: + type: boolean + currentStepIndex: + type: number + currentStepStarted: + type: boolean + lastSuccessfulProvision: + type: object + nullable: true + description: Information about the last time this node was successfully provisioned. This is used to automatically detect when the node is out-of-date with the provisioner and trigger a reprovision. + properties: + name: + type: string + hash: + type: string + registeredIpAddresses: + type: array + items: + type: object + properties: + address: + type: string + expiresAt: + type: string + format: date-time + bootToDisk: + type: boolean + bootEntries: + description: The list of boot entries in the EFI firmware, automatically synchronised whenever the initrd starts up. This can be used to then set inactiveBootEntries if needed. + nullable: true + type: array + items: + type: object + properties: + bootId: + type: string + description: The boot ID; this should be the value set into inactiveBootEntries. + name: + type: string + description: The boot entry name. + path: + type: string + description: The boot entry path. + active: + type: boolean + description: If the entry was active last time the initrd environment ran. selectableFields: - jsonPath: .spec.nodeName - jsonPath: .spec.nodeGroup - jsonPath: .spec.authorized additionalPrinterColumns: - - jsonPath: .status.attestationIdentityKeyFingerprint - name: "Fingerprint" - description: The fingerprint of the attestation identity key. - type: string - jsonPath: .spec.nodeName name: "Node Name" description: The name given to this node if it is authorized. You can't authorize a node without giving it a name. diff --git a/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnodegroup.yaml b/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnodegroup.yaml index 66c18f0d..bd1f0db2 100644 --- a/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnodegroup.yaml +++ b/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnodegroup.yaml @@ -20,37 +20,21 @@ spec: spec: type: object properties: - platform: + provisioner: type: string nullable: true - description: Specifies the platform that this machine should be provisioned as. If not set, nodes in this group will provision as the first platform they are capable of running. - enum: [Windows, Linux, Mac] - activeDirectory: + description: Specifies the provisioner to use to provision this machine. + provisionerArguments: type: object + description: An arbitrary map of parameter values that should be available when provisioning. These will override the default parameter values that the provisioner has set. Each entry here will be cause '{{`{{param:}}`}}' to be substituted, and RKM_PARAM_KEY_NAME_TRANSFORMED as an environment variable. KEY_NAME_TRANSFORMED is the key transformed such that 'keyNameTransformed' is 'KEY_NAME_TRANSFORMED'. + x-kubernetes-preserve-unknown-fields: true nullable: true - properties: - domain: - type: string - description: The Active Directory domain that this machine will be joined to. - join: - type: boolean - description: If true, this machine will be joined to Active Directory. - computerGroups: - description: The list of groups that the machine's Active Directory account will be a member of. - type: array - items: - type: string + clusterControllerIpAddress: + type: string + nullable: true + description: If set, overrides the Kubernetes controller IP address that provisioned machines should join. selectableFields: [] additionalPrinterColumns: - - jsonPath: .spec.platform - name: "Platform" - type: string - - jsonPath: .spec.activeDirectory.domain - name: "AD Domain" - type: string - - jsonPath: .spec.activeDirectory.join - name: "AD Join" - type: string - - jsonPath: .spec.activeDirectory.computerGroups[*] - name: "AD Computer Groups" + - jsonPath: .spec.provisioner + name: "Provisioner" type: string diff --git a/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnodeprovisioner.yaml b/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnodeprovisioner.yaml new file mode 100644 index 00000000..5931b62e --- /dev/null +++ b/UET/Lib/Helm/rkm-crds/templates/rkm.redpoint.games_rkmnodeprovisioner.yaml @@ -0,0 +1,36 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: rkmnodeprovisioners.rkm.redpoint.games +spec: + group: rkm.redpoint.games + scope: Cluster + names: + plural: rkmnodeprovisioners + singular: rkmnodeprovisioner + kind: RkmNodeProvisioner + shortNames: + - rkmprovisioner + - rkmprovisioners + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + parameters: + type: object + description: An arbitrary map of parameters that should be available when provisioning. Each entry here will be cause '{{`{{param:}}`}}' to be substituted, and RKM_PARAM_KEY_NAME_TRANSFORMED as an environment variable. KEY_NAME_TRANSFORMED is the key transformed such that 'keyNameTransformed' is 'KEY_NAME_TRANSFORMED'. The values in this map are the default values for the parameters, which can be overridden by setting an entry inside 'provisionerArguments' on the node group. + x-kubernetes-preserve-unknown-fields: true + nullable: true + steps: + type: array + items: + type: object + x-kubernetes-preserve-unknown-fields: true + selectableFields: [] diff --git a/UET/Lib/Helm/rkm/templates/rkm/windows-client-rkmnodeprovisioner.yaml b/UET/Lib/Helm/rkm/templates/rkm/windows-client-rkmnodeprovisioner.yaml new file mode 100644 index 00000000..38cbbb37 --- /dev/null +++ b/UET/Lib/Helm/rkm/templates/rkm/windows-client-rkmnodeprovisioner.yaml @@ -0,0 +1,926 @@ +apiVersion: rkm.redpoint.games/v1 +kind: RkmNodeProvisioner +metadata: + name: windows-client +spec: + parameters: + esdFileDownloadUrl: "http://dl.delivery.mp.microsoft.com/filestreamingservice/files/009d9a0d-8e1a-45ce-9540-21377534803e/26100.4349.250607-1500.ge_release_svc_refresh_CLIENTCONSUMER_RET_x64FRE_en-us.esd" + esdFileExpectedHash: "8ceab2838f8e90180ac7490e8752157fb05588cb" + esdFileImageIndex: "9" # Matches "Windows 11 Pro" in the default ESD. + clusterControllerIpAddress: "{{`{{provision:apiAddressIp}}`}}" + steps: + - type: atomicSequence + atomicSequence: + steps: + - type: deleteBootLoaderEntry + deleteBootLoaderEntry: + name: Windows Boot Manager + - type: executeProcess + executeProcess: + executable: bash + arguments: + - "{{script-path}}" + script: | + #!/bin/bash + + set -e + set -x + + if [ -e /var/mount/images/Windows.vhdx ]; then + echo "Deleting existing Windows VHDX so we have enough space to work..." + rm /var/mount/images/Windows.vhdx + fi + if [ -e /var/mount/boot/EFI/Boot ]; then + echo "Cleaning up installed bootloader..." + rm -rf /var/mount/boot/EFI/Boot + fi + if [ -e /var/mount/boot/EFI/Microsoft ]; then + echo "Cleaning up installed Microsoft bootloader..." + rm -rf /var/mount/boot/EFI/Microsoft + fi + + mkdir -p /var/mount/pe + + if [ -e /var/mount/images/reinitialize_wine ] || [ ! -e /var/mount/images/LinuxTmp.ext4 ]; then + echo "Initializing from start..." + truncate -s 8G /var/mount/images/LinuxTmp.ext4 + mkfs.ext4 -F -F /var/mount/images/LinuxTmp.ext4 + fi + + echo "Mounting temporary disk..." + mount -o loop,exec,defaults /var/mount/images/LinuxTmp.ext4 /var/mount/pe + if [ ! -e /var/mount/pe/adk_installed ]; then + echo "Creating initial directories..." + mkdir -pv /var/mount/pe/wineprefix + mkdir -pv /var/mount/pe/.local/share + mkdir -pv /var/mount/pe/.cache + fi + + export WINEPREFIX=/var/mount/pe/wineprefix + export XDG_DATA_HOME=/var/mount/pe/.local/share + export XDG_CACHE_HOME=/var/mount/pe/.cache + + if [ ! -e /var/mount/pe/adk_installed ]; then + echo "Installing ADK and dependencies..." + + if [ ! -e /var/mount/pe/winetricks ]; then + echo "Downloading winetricks..." + curl -o /var/mount/pe/winetricks -L https://raw.githubusercontent.com/Winetricks/winetricks/master/src/winetricks + fi + + echo "Installing .NET 4.8 framework (this might take a while)..." + WINEDLLOVERRIDES=mscoree=d bash /var/mount/pe/winetricks -q dotnet48 + + if [ ! -e /var/mount/pe/adksetup.exe ]; then + echo "Downloading adksetup.exe..." + curl -o /var/mount/pe/adksetup.exe -L https://download.microsoft.com/download/615540bc-be0b-433a-b91b-1f2b0642bb24/adk/adksetup.exe + fi + + echo "Installing ADK..." + wine /var/mount/pe/adksetup.exe /quiet + + if [ ! -e /var/mount/pe/adkwinpesetup.exe ]; then + echo "Downloading adkwinpesetup.exe..." + curl -o /var/mount/pe/adkwinpesetup.exe -L https://download.microsoft.com/download/5/5/6/556e01ec-9d78-417d-b1e1-d83a2eff20bc/ADKWinPEAddons/adkwinpesetup.exe + fi + + echo "Installing WinPE add-ons for ADK..." + wine /var/mount/pe/adkwinpesetup.exe /quiet + sleep 10 + kill $(pidof wineserver wine) || true + + echo "ADK and dependencies has now been set up." + touch /var/mount/pe/adk_installed + else + echo "ADK and dependencies already set up." + fi + + if [ ! -e /var/mount/images/install-$RKM_PARAM_ESD_FILE_EXPECTED_HASH.esd ]; then + echo "Downloading Windows 11 24H2 ESD file from Microsoft..." + curl --retry 10 --retry-all-errors -o /var/mount/images/install.esd.tmp "$RKM_PARAM_ESD_FILE_DOWNLOAD_URL" + + echo "Hashing downloaded file to check integrity..." + sha1sum /var/mount/images/install.esd.tmp > /var/mount/images/install.esd.hash + SHA1SUM_RESULT=$(cat /var/mount/images/install.esd.hash) + DOWNLOADED_FILE_HASH=${SHA1SUM_RESULT%% *} + if [ "$DOWNLOADED_FILE_HASH" != "$RKM_PARAM_ESD_FILE_EXPECTED_HASH" ]; then + echo "Error! Downloaded ESD should match hash $RKM_PARAM_ESD_FILE_EXPECTED_HASH, but was $DOWNLOADED_FILE_HASH!" + exit 1 + fi + + mv /var/mount/images/install.esd.tmp /var/mount/images/install-$RKM_PARAM_ESD_FILE_EXPECTED_HASH.esd + rm /var/mount/images/install.esd.hash + echo "Windows 11 Pro 24H2 ESD file ready to go." + else + echo "Windows 11 Pro 24H2 ESD file already prepared." + fi + + echo "Setting up boot files for WinPE Stage 0..." + echo '[LaunchApps]' >/var/mount/pe/winpeshl.ini + echo '"boot.bat"' >>/var/mount/pe/winpeshl.ini + echo '@echo off' >/var/mount/pe/boot.bat + echo 'wpeinit' >>/var/mount/pe/boot.bat + echo 'chkdsk /F C:' >>/var/mount/pe/boot.bat + echo 'rmdir /Q /S C:\WinPEStage1\mount' >>/var/mount/pe/boot.bat + echo 'rmdir /Q /S C:\WinPEStage1\scratch' >>/var/mount/pe/boot.bat + echo 'mkdir C:\WinPEStage1\mount' >>/var/mount/pe/boot.bat + echo 'mkdir C:\WinPEStage1\scratch' >>/var/mount/pe/boot.bat + + echo "Copy files for WinPE Stage 1 Preparation..." + rm -rf /var/mount/images/WinPEStage1/data || true; + mkdir -p /var/mount/images/WinPEStage1/data + cp "/var/mount/pe/wineprefix/drive_c/Program Files (x86)/Windows Kits/10/Assessment and Deployment Kit/Windows Preinstallation Environment/amd64/en-us/winpe.wim" "/var/mount/images/WinPEStage1/data/winpe.wim" + echo 'copy /Y C:\WinPEStage1\data\winpe.wim C:\WinPEStage1\data\winpe-prep.wim' >>/var/mount/pe/boot.bat + echo 'if %errorlevel% neq 0 exit 1' >>/var/mount/pe/boot.bat + echo 'dism /Mount-Image /ImageFile:C:\WinPEStage1\data\winpe-prep.wim /MountDir:C:\WinPEStage1\mount' >>/var/mount/pe/boot.bat + echo 'if %errorlevel% neq 0 exit 1' >>/var/mount/pe/boot.bat + for OC in WinPE-WMI WinPE-NetFx WinPE-Scripting WinPE-PowerShell WinPE-DismCmdlets WinPE-SecureBootCmdlets WinPE-StorageWMI WinPE-Setup WinPE-Setup-Client WinPE-SecureStartup; do + echo " Copying $OC package..." + cp "/var/mount/pe/wineprefix/drive_c/Program Files (x86)/Windows Kits/10/Assessment and Deployment Kit/Windows Preinstallation Environment/amd64/WinPE_OCs/$OC.cab" "/var/mount/images/WinPEStage1/data/$OC.cab" + echo "dism /Add-Package /Image:C:\\WinPEStage1\\mount /PackagePath:C:\\WinPEStage1\\data\\$OC.cab /ScratchDir:C:\\WinPEStage1\\scratch" >>/var/mount/pe/boot.bat + echo 'if %errorlevel% neq 0 exit 1' >>/var/mount/pe/boot.bat + done + + echo "Finalising files..." + echo 'copy /Y C:\WinPEStage1\mount\Windows\System32\vcruntime140_clr0400.dll C:\WinPEStage1\mount\Windows\System32\vcruntime140.dll' >>/var/mount/pe/boot.bat + echo 'if %errorlevel% neq 0 exit 1' >>/var/mount/pe/boot.bat + echo 'dism /Unmount-Image /MountDir:C:\WinPEStage1\mount /Commit' >>/var/mount/pe/boot.bat + echo 'if %errorlevel% neq 0 exit 1' >>/var/mount/pe/boot.bat + echo 'copy /Y C:\WinPEStage1\data\winpe-prep.wim C:\WinPEStage1\data\winpe-ready.wim' >>/var/mount/pe/boot.bat + echo "uet internal pxeboot notify-for-reboot --fingerprint $RKM_PROVISION_AIK_FINGERPRINT" >>/var/mount/pe/boot.bat + unix2dos /var/mount/pe/winpeshl.ini + unix2dos /var/mount/pe/boot.bat + - type: uploadFiles + uploadFiles: + files: + - source: "/var/mount/pe/wineprefix/drive_c/Program Files (x86)/Windows Kits/10/Assessment and Deployment Kit/Windows Preinstallation Environment/amd64/en-us/winpe.wim" + target: winpe.wim + - source: "/var/mount/pe/wineprefix/drive_c/Program Files (x86)/Windows Kits/10/Assessment and Deployment Kit/Windows Preinstallation Environment/amd64/Media/Boot/BCD" + target: BCD + - source: "/var/mount/pe/wineprefix/drive_c/Program Files (x86)/Windows Kits/10/Assessment and Deployment Kit/Windows Preinstallation Environment/amd64/Media/Boot/boot.sdi" + target: boot.sdi + - source: "/var/mount/pe/winpeshl.ini" + target: winpeshl.ini + - source: "/var/mount/pe/boot.bat" + target: boot.bat + - type: reboot + reboot: + onceViaNotify: true + ipxeScriptTemplate: | + #!ipxe + {{dhcp}} + kernel static/wimboot + imgfetch --name BCD uploaded/BCD BCD + imgfetch --name boot.sdi uploaded/boot.sdi boot.sdi + imgfetch --name boot.wim uploaded/winpe.wim boot.wim + imgfetch --name winpeshl.ini uploaded/winpeshl.ini winpeshl.ini + imgfetch --name boot.bat uploaded/boot.bat boot.bat + imgfetch --name uet.exe static/uet.exe uet.exe + imgfetch --name rkm-provision-context.json rkm-provision-context.json rkm-provision-context.json + boot + - type: atomicSequence + atomicSequence: + steps: + - type: executeProcess + executeProcess: + executable: bash + arguments: + - "{{script-path}}" + script: | + #!/bin/bash + + set -e + set -x + + if [ ! -e /var/mount/images/WinPEStage1/data/winpe-ready.wim ]; then + echo "WinPE Stage 0 failed to produce winpe-ready.wim!" + exit 1 + fi + + echo "Setting up boot files for WinPE Stage 1..." + echo '[LaunchApps]' >/var/mount/images/WinPEStage1/data/winpeshl.ini + echo '"boot.bat"' >>/var/mount/images/WinPEStage1/data/winpeshl.ini + echo '@echo off' >/var/mount/images/WinPEStage1/data/boot.bat + echo 'wpeinit' >>/var/mount/images/WinPEStage1/data/boot.bat + echo 'chkdsk /F C:' >>/var/mount/images/WinPEStage1/data/boot.bat + echo 'uet internal pxeboot provision-client' >>/var/mount/images/WinPEStage1/data/boot.bat + echo 'cmd' >>/var/mount/images/WinPEStage1/data/boot.bat + unix2dos /var/mount/images/WinPEStage1/data/winpeshl.ini + unix2dos /var/mount/images/WinPEStage1/data/boot.bat + - type: uploadFiles + uploadFiles: + files: + - source: "/var/mount/images/WinPEStage1/data/winpe-ready.wim" + target: winpe.wim + - source: "/var/mount/images/WinPEStage1/data/winpeshl.ini" + target: winpeshl.ini + - source: "/var/mount/images/WinPEStage1/data/boot.bat" + target: boot.bat + - type: reboot + reboot: + ipxeScriptTemplate: | + #!ipxe + {{dhcp}} + kernel static/wimboot + imgfetch --name BCD uploaded/BCD BCD + imgfetch --name boot.sdi uploaded/boot.sdi boot.sdi + imgfetch --name boot.wim uploaded/winpe.wim boot.wim + imgfetch --name winpeshl.ini uploaded/winpeshl.ini winpeshl.ini + imgfetch --name boot.bat uploaded/boot.bat boot.bat + imgfetch --name uet.ppkg uploaded/uet.ppkg uet.ppkg + imgfetch --name uet.exe static/uet.exe uet.exe + imgfetch --name rkm-provision-context.json rkm-provision-context.json rkm-provision-context.json + boot + - type: atomicSequence + atomicSequence: + steps: + - type: executeProcess + executeProcess: + executable: powershell + arguments: + - "-ExecutionPolicy" + - "Bypass" + - "{{script-path}}" + script: | + param() + + $ErrorActionPreference = 'Stop' + + if (Test-Path C:\Windows.vhdx) { + Write-Host "Deleting existing Windows installation..." + Remove-Item C:\Windows.vhdx + } + + Write-Host "Creating new virtual disk for Windows..." + $TargetSize = [Math]::Round(((Get-Volume -DriveLetter C).SizeRemaining / 1024 / 1024) - (10 * 1024)) + $Script = @" + create vdisk file="C:\Windows.vhdx" type=fixed maximum=$TargetSize + attach vdisk + clean + create partition primary + format quick fs=ntfs + assign letter=w + exit + "@ + Set-Content -Path "$env:TEMP\vdisksetup.txt" -Value $Script + diskpart /s "$env:TEMP\vdisksetup.txt" + if ($LastExitCode -ne 0) { + exit 1 + } + + Write-Host "Applying image to virtual disk..." + dism /Apply-Image /ImageFile:C:\install-${env:RKM_PARAM_ESD_FILE_EXPECTED_HASH}.esd /Index:${env:RKM_PARAM_ESD_FILE_IMAGE_INDEX} /ApplyDir:W:\ + + Write-Host "Enabling Hyper-V and Containers features..." + dism /Image:W:\ /Enable-Feature /FeatureName:Containers /FeatureName:Microsoft-Hyper-V /All /ScratchDir:C:\ + + Write-Host "Install the bootloader from template..." + bcdboot W:\Windows /addlast /p + bcdedit /set "{fwbootmgr}" displayorder "{bootmgr}" /addlast + + Write-Host "Copying UET to installed version of Windows..." + New-Item -ItemType Directory -Path W:\ProgramData\UET\RKM-Provisioned + Copy-Item -Force X:\Windows\System32\uet.exe W:\ProgramData\UET\RKM-Provisioned\uet.exe + - type: modifyFiles + modifyFiles: + files: + - path: "W:\\Windows\\Panther\\unattend.xml" + enableReplacements: true + content: | + + + + + {{provision:nodeName}} + + + + + 1 + powershell.exe -WindowStyle "Normal" -ExecutionPolicy "Unrestricted" -NoProfile -File "C:\Windows\Setup\Scripts\Specialize.ps1" + + + 2 + reg.exe load "HKU\DefaultUser" "C:\Users\Default\NTUSER.DAT" + + + 3 + powershell.exe -WindowStyle "Normal" -ExecutionPolicy "Unrestricted" -NoProfile -File "C:\Windows\Setup\Scripts\DefaultUser.ps1" + + + 4 + reg.exe unload "HKU\DefaultUser" + + + + + + + 0409:00000409 + en-US + en-US + en-US + + + + + + kubernetes + + Administrators + + kubernetes +

true</PlainText> + </Password> + </LocalAccount> + </LocalAccounts> + </UserAccounts> + <AutoLogon> + <Username>kubernetes</Username> + <Enabled>true</Enabled> + <LogonCount>1</LogonCount> + <Password> + <Value>kubernetes</Value> + <PlainText>true</PlainText> + </Password> + </AutoLogon> + <OOBE> + <ProtectYourPC>3</ProtectYourPC> + <HideEULAPage>true</HideEULAPage> + <HideWirelessSetupInOOBE>false</HideWirelessSetupInOOBE> + <HideOnlineAccountScreens>false</HideOnlineAccountScreens> + </OOBE> + <FirstLogonCommands> + <SynchronousCommand wcm:action="add"> + <Order>1</Order> + <CommandLine>powershell.exe -WindowStyle "Normal" -ExecutionPolicy "Unrestricted" -NoProfile -File "C:\Windows\Setup\Scripts\FirstLogon.ps1"</CommandLine> + </SynchronousCommand> + </FirstLogonCommands> + </component> + </settings> + </unattend> + - path: "W:\\Windows\\Setup\\Scripts\\RemovePackages.ps1" + content: | + $selectors = @( + 'Microsoft.Microsoft3DViewer'; + 'Microsoft.BingSearch'; + 'Microsoft.WindowsCamera'; + 'Clipchamp.Clipchamp'; + 'Microsoft.Copilot'; + 'Microsoft.549981C3F5F10'; + 'MicrosoftCorporationII.MicrosoftFamily'; + 'Microsoft.WindowsFeedbackHub'; + 'Microsoft.Edge.GameAssist'; + 'Microsoft.Getstarted'; + 'microsoft.windowscommunicationsapps'; + 'Microsoft.WindowsMaps'; + 'Microsoft.MixedReality.Portal'; + 'Microsoft.BingNews'; + 'Microsoft.MicrosoftOfficeHub'; + 'Microsoft.Office.OneNote'; + 'Microsoft.OutlookForWindows'; + 'Microsoft.MSPaint'; + 'Microsoft.People'; + 'Microsoft.PowerAutomateDesktop'; + 'MicrosoftCorporationII.QuickAssist'; + 'Microsoft.SkypeApp'; + 'Microsoft.MicrosoftSolitaireCollection'; + 'Microsoft.MicrosoftStickyNotes'; + 'MicrosoftTeams'; + 'MSTeams'; + 'Microsoft.Todos'; + 'Microsoft.WindowsSoundRecorder'; + 'Microsoft.Wallet'; + 'Microsoft.BingWeather'; + 'Microsoft.Xbox.TCUI'; + 'Microsoft.XboxApp'; + 'Microsoft.XboxGameOverlay'; + 'Microsoft.XboxGamingOverlay'; + 'Microsoft.XboxIdentityProvider'; + 'Microsoft.XboxSpeechToTextOverlay'; + 'Microsoft.GamingApp'; + 'Microsoft.YourPhone'; + 'Microsoft.ZuneMusic'; + 'Microsoft.ZuneVideo'; + ); + $getCommand = { + Get-AppxProvisionedPackage -Online; + }; + $filterCommand = { + $_.DisplayName -eq $selector; + }; + $removeCommand = { + [CmdletBinding()] + param( + [Parameter( Mandatory, ValueFromPipeline )] + $InputObject + ); + process { + $InputObject | Remove-AppxProvisionedPackage -AllUsers -Online -ErrorAction 'Continue'; + } + }; + $type = 'Package'; + $logfile = 'C:\Windows\Setup\Scripts\RemovePackages.log'; + & { + $installed = & $getCommand; + foreach( $selector in $selectors ) { + $result = [ordered] @{ + Selector = $selector; + }; + $found = $installed | Where-Object -FilterScript $filterCommand; + if( $found ) { + $result.Output = $found | & $removeCommand; + if( $? ) { + $result.Message = "$type removed."; + } else { + $result.Message = "$type not removed."; + $result.Error = $Error[0]; + } + } else { + $result.Message = "$type not installed."; + } + $result | ConvertTo-Json -Depth 3 -Compress; + } + } *>&1 | Out-String -Width 1KB -Stream >> $logfile; + - path: "W:\\Windows\\Setup\\Scripts\\RemoveCapabilities.ps1" + content: | + $selectors = @( + 'Print.Fax.Scan'; + 'Language.Handwriting'; + 'MathRecognizer'; + 'OneCoreUAP.OneSync'; + 'App.Support.QuickAssist'; + 'Language.Speech'; + 'Language.TextToSpeech'; + 'App.StepsRecorder'; + 'Hello.Face.18967'; + 'Hello.Face.Migration.18967'; + 'Hello.Face.20134'; + ); + $getCommand = { + Get-WindowsCapability -Online | Where-Object -Property 'State' -NotIn -Value @( + 'NotPresent'; + 'Removed'; + ); + }; + $filterCommand = { + ($_.Name -split '~')[0] -eq $selector; + }; + $removeCommand = { + [CmdletBinding()] + param( + [Parameter( Mandatory, ValueFromPipeline )] + $InputObject + ); + process { + $InputObject | Remove-WindowsCapability -Online -ErrorAction 'Continue'; + } + }; + $type = 'Capability'; + $logfile = 'C:\Windows\Setup\Scripts\RemoveCapabilities.log'; + & { + $installed = & $getCommand; + foreach( $selector in $selectors ) { + $result = [ordered] @{ + Selector = $selector; + }; + $found = $installed | Where-Object -FilterScript $filterCommand; + if( $found ) { + $result.Output = $found | & $removeCommand; + if( $? ) { + $result.Message = "$type removed."; + } else { + $result.Message = "$type not removed."; + $result.Error = $Error[0]; + } + } else { + $result.Message = "$type not installed."; + } + $result | ConvertTo-Json -Depth 3 -Compress; + } + } *>&1 | Out-String -Width 1KB -Stream >> $logfile; + - path: "W:\\Windows\\Setup\\Scripts\\RemoveFeatures.ps1" + content: | + $selectors = @( + 'MediaPlayback'; + 'Recall'; + ); + $getCommand = { + Get-WindowsOptionalFeature -Online | Where-Object -Property 'State' -NotIn -Value @( + 'Disabled'; + 'DisabledWithPayloadRemoved'; + ); + }; + $filterCommand = { + $_.FeatureName -eq $selector; + }; + $removeCommand = { + [CmdletBinding()] + param( + [Parameter( Mandatory, ValueFromPipeline )] + $InputObject + ); + process { + $InputObject | Disable-WindowsOptionalFeature -Online -Remove -NoRestart -ErrorAction 'Continue'; + } + }; + $type = 'Feature'; + $logfile = 'C:\Windows\Setup\Scripts\RemoveFeatures.log'; + & { + $installed = & $getCommand; + foreach( $selector in $selectors ) { + $result = [ordered] @{ + Selector = $selector; + }; + $found = $installed | Where-Object -FilterScript $filterCommand; + if( $found ) { + $result.Output = $found | & $removeCommand; + if( $? ) { + $result.Message = "$type removed."; + } else { + $result.Message = "$type not removed."; + $result.Error = $Error[0]; + } + } else { + $result.Message = "$type not installed."; + } + $result | ConvertTo-Json -Depth 3 -Compress; + } + } *>&1 | Out-String -Width 1KB -Stream >> $logfile; + - path: "W:\\Windows\\Setup\\Scripts\\PauseWindowsUpdate.ps1" + content: | + $formatter = { + $args[0].ToString( "yyyy'-'MM'-'dd'T'HH':'mm':'ssK" ); + }; + $now = [datetime]::UtcNow; + $start = & $formatter $now; + $end = & $formatter $now.AddDays( 7 ); + + $params = @{ + LiteralPath = 'Registry::HKLM\SOFTWARE\Microsoft\WindowsUpdate\UX\Settings'; + Type = 'String'; + Force = $true; + }; + + Set-ItemProperty @params -Name 'PauseFeatureUpdatesStartTime' -Value $start; + Set-ItemProperty @params -Name 'PauseFeatureUpdatesEndTime' -Value $end; + Set-ItemProperty @params -Name 'PauseQualityUpdatesStartTime' -Value $start; + Set-ItemProperty @params -Name 'PauseQualityUpdatesEndTime' -Value $end; + Set-ItemProperty @params -Name 'PauseUpdatesStartTime' -Value $start; + Set-ItemProperty @params -Name 'PauseUpdatesExpiryTime' -Value $end; + - path: "W:\\Windows\\Setup\\Scripts\\PauseWindowsUpdate.xml" + content: | + <Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task"> + <Triggers> + <BootTrigger> + <Repetition> + <Interval>P1D</Interval> + <StopAtDurationEnd>false</StopAtDurationEnd> + </Repetition> + <Enabled>true</Enabled> + </BootTrigger> + </Triggers> + <Principals> + <Principal id="Author"> + <UserId>S-1-5-19</UserId> + <RunLevel>LeastPrivilege</RunLevel> + </Principal> + </Principals> + <Settings> + <MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy> + <DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries> + <StopIfGoingOnBatteries>false</StopIfGoingOnBatteries> + <AllowHardTerminate>true</AllowHardTerminate> + <StartWhenAvailable>false</StartWhenAvailable> + <RunOnlyIfNetworkAvailable>false</RunOnlyIfNetworkAvailable> + <IdleSettings> + <StopOnIdleEnd>true</StopOnIdleEnd> + <RestartOnIdle>false</RestartOnIdle> + </IdleSettings> + <AllowStartOnDemand>true</AllowStartOnDemand> + <Enabled>true</Enabled> + <Hidden>false</Hidden> + <RunOnlyIfIdle>false</RunOnlyIfIdle> + <WakeToRun>false</WakeToRun> + <ExecutionTimeLimit>PT72H</ExecutionTimeLimit> + <Priority>7</Priority> + </Settings> + <Actions Context="Author"> + <Exec> + <Command>C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe</Command> + <Arguments>-ExecutionPolicy "Unrestricted" -NoProfile -File "C:\Windows\Setup\Scripts\PauseWindowsUpdate.ps1"</Arguments> + </Exec> + </Actions> + </Task> + - path: "W:\\Windows\\Setup\\Scripts\\MoveActiveHours.vbs" + content: | + HKLM = &H80000002 + key = "SOFTWARE\Microsoft\WindowsUpdate\UX\Settings" + Set reg = GetObject("winmgmts://./root/default:StdRegProv") + current = Hour(Now) + reg.SetDWORDValue HKLM, key, "ActiveHoursStart", ( current + 23 ) Mod 24 + reg.SetDWORDValue HKLM, key, "ActiveHoursEnd", ( current + 11 ) Mod 24 + reg.SetDWORDValue HKLM, key, "SmartActiveHoursState", 2 + - path: "W:\\Windows\\Setup\\Scripts\\MoveActiveHours.xml" + content: | + <Task version="1.2" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task"> + <Triggers> + <BootTrigger> + <Repetition> + <Interval>PT4H</Interval> + <StopAtDurationEnd>false</StopAtDurationEnd> + </Repetition> + <Enabled>true</Enabled> + </BootTrigger> + <RegistrationTrigger> + <Repetition> + <Interval>PT4H</Interval> + <StopAtDurationEnd>false</StopAtDurationEnd> + </Repetition> + <Enabled>true</Enabled> + </RegistrationTrigger> + </Triggers> + <Principals> + <Principal id="Author"> + <UserId>S-1-5-19</UserId> + <RunLevel>LeastPrivilege</RunLevel> + </Principal> + </Principals> + <Settings> + <MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy> + <DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries> + <StopIfGoingOnBatteries>false</StopIfGoingOnBatteries> + <AllowHardTerminate>true</AllowHardTerminate> + <StartWhenAvailable>false</StartWhenAvailable> + <RunOnlyIfNetworkAvailable>false</RunOnlyIfNetworkAvailable> + <IdleSettings> + <StopOnIdleEnd>true</StopOnIdleEnd> + <RestartOnIdle>false</RestartOnIdle> + </IdleSettings> + <AllowStartOnDemand>true</AllowStartOnDemand> + <Enabled>true</Enabled> + <Hidden>false</Hidden> + <RunOnlyIfIdle>false</RunOnlyIfIdle> + <WakeToRun>false</WakeToRun> + <ExecutionTimeLimit>PT72H</ExecutionTimeLimit> + <Priority>7</Priority> + </Settings> + <Actions Context="Author"> + <Exec> + <Command>C:\Windows\System32\wscript.exe</Command> + <Arguments>C:\Windows\Setup\Scripts\MoveActiveHours.vbs</Arguments> + </Exec> + </Actions> + </Task> + - path: "W:\\Windows\\Setup\\Scripts\\SetStartPins.ps1" + content: | + $json = '{"pinnedList":[]}'; + if( [System.Environment]::OSVersion.Version.Build -lt 20000 ) { + return; + } + $key = 'Registry::HKLM\SOFTWARE\Microsoft\PolicyManager\current\device\Start'; + New-Item -Path $key -ItemType 'Directory' -ErrorAction 'SilentlyContinue'; + Set-ItemProperty -LiteralPath $key -Name 'ConfigureStartPins' -Value $json -Type 'String'; + - path: "W:\\Windows\\Setup\\Scripts\\Specialize.ps1" + content: | + $scripts = @( + { + Remove-Item -LiteralPath 'C:\Users\Default\AppData\Roaming\Microsoft\Windows\Start Menu\Programs\OneDrive.lnk', 'C:\Windows\System32\OneDriveSetup.exe', 'C:\Windows\SysWOW64\OneDriveSetup.exe' -ErrorAction 'Continue'; + }; + { + Remove-Item -LiteralPath 'Registry::HKLM\Software\Microsoft\WindowsUpdate\Orchestrator\UScheduler_Oobe\OutlookUpdate' -Force -ErrorAction 'SilentlyContinue'; + }; + { + reg.exe add "HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Communications" /v ConfigureChatAutoInstall /t REG_DWORD /d 0 /f; + }; + { + & 'C:\Windows\Setup\Scripts\RemovePackages.ps1'; + }; + { + & 'C:\Windows\Setup\Scripts\RemoveCapabilities.ps1'; + }; + { + & 'C:\Windows\Setup\Scripts\RemoveFeatures.ps1'; + }; + { + net.exe accounts /maxpwage:UNLIMITED; + }; + { + Register-ScheduledTask -TaskName 'PauseWindowsUpdate' -Xml $( Get-Content -LiteralPath 'C:\Windows\Setup\Scripts\PauseWindowsUpdate.xml' -Raw ); + }; + { + reg.exe add "HKLM\SYSTEM\CurrentControlSet\Control\FileSystem" /v LongPathsEnabled /t REG_DWORD /d 1 /f + }; + { + netsh.exe advfirewall firewall set rule group="@FirewallAPI.dll,-28752" new enable=Yes; + reg.exe add "HKLM\SYSTEM\CurrentControlSet\Control\Terminal Server" /v fDenyTSConnections /t REG_DWORD /d 0 /f; + }; + { + Set-ExecutionPolicy -Scope 'LocalMachine' -ExecutionPolicy 'RemoteSigned' -Force; + }; + { + reg.exe add "HKLM\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate\AU" /v AUOptions /t REG_DWORD /d 4 /f; + reg.exe add "HKLM\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate\AU" /v NoAutoRebootWithLoggedOnUsers /t REG_DWORD /d 1 /f; + }; + { + Register-ScheduledTask -TaskName 'MoveActiveHours' -Xml $( Get-Content -LiteralPath 'C:\Windows\Setup\Scripts\MoveActiveHours.xml' -Raw ); + }; + { + reg.exe add "HKLM\SOFTWARE\Policies\Microsoft\Dsh" /v AllowNewsAndInterests /t REG_DWORD /d 0 /f; + }; + { + reg.exe add "HKLM\Software\Policies\Microsoft\Edge" /v HideFirstRunExperience /t REG_DWORD /d 1 /f; + }; + { + & 'C:\Windows\Setup\Scripts\SetStartPins.ps1'; + }; + ); + + & { + [float] $complete = 0; + [float] $increment = 100 / $scripts.Count; + foreach( $script in $scripts ) { + Write-Progress -Activity 'Running scripts to customize your Windows installation. Do not close this window.' -PercentComplete $complete; + '*** Will now execute command «{0}».' -f $( + $str = $script.ToString().Trim() -replace '\s+', ' '; + $max = 100; + if( $str.Length -le $max ) { + $str; + } else { + $str.Substring( 0, $max - 1 ) + '…'; + } + ); + $start = [datetime]::Now; + & $script; + '*** Finished executing command after {0:0} ms.' -f [datetime]::Now.Subtract( $start ).TotalMilliseconds; + "`r`n" * 3; + $complete += $increment; + } + } *>&1 | Out-String -Width 1KB -Stream >> "C:\Windows\Setup\Scripts\Specialize.log"; + - path: "W:\\Windows\\Setup\\Scripts\\UserOnce.ps1" + content: | + $scripts = @( + { + Get-AppxPackage -Name 'Microsoft.Windows.Ai.Copilot.Provider' | Remove-AppxPackage; + }; + { + Set-ItemProperty -LiteralPath 'Registry::HKCU\Software\Microsoft\Windows\CurrentVersion\Search' -Name 'SearchboxTaskbarMode' -Type 'DWord' -Value 0; + }; + { + Get-Process -Name 'explorer' -ErrorAction 'SilentlyContinue' | Where-Object -FilterScript { + $_.SessionId -eq ( Get-Process -Id $PID ).SessionId; + } | Stop-Process -Force; + }; + ); + + & { + [float] $complete = 0; + [float] $increment = 100 / $scripts.Count; + foreach( $script in $scripts ) { + Write-Progress -Activity 'Running scripts to configure this user account. Do not close this window.' -PercentComplete $complete; + '*** Will now execute command «{0}».' -f $( + $str = $script.ToString().Trim() -replace '\s+', ' '; + $max = 100; + if( $str.Length -le $max ) { + $str; + } else { + $str.Substring( 0, $max - 1 ) + '…'; + } + ); + $start = [datetime]::Now; + & $script; + '*** Finished executing command after {0:0} ms.' -f [datetime]::Now.Subtract( $start ).TotalMilliseconds; + "`r`n" * 3; + $complete += $increment; + } + } *>&1 | Out-String -Width 1KB -Stream >> "$env:TEMP\UserOnce.log"; + - path: "W:\\Windows\\Setup\\Scripts\\DefaultUser.ps1" + content: | + $scripts = @( + { + reg.exe add "HKU\DefaultUser\Software\Policies\Microsoft\Windows\WindowsCopilot" /v TurnOffWindowsCopilot /t REG_DWORD /d 1 /f; + }; + { + Remove-ItemProperty -LiteralPath 'Registry::HKU\DefaultUser\Software\Microsoft\Windows\CurrentVersion\Run' -Name 'OneDriveSetup' -Force -ErrorAction 'Continue'; + }; + { + reg.exe add "HKU\DefaultUser\Software\Microsoft\Windows\CurrentVersion\GameDVR" /v AppCaptureEnabled /t REG_DWORD /d 0 /f; + }; + { + $names = @( + 'ContentDeliveryAllowed'; + 'FeatureManagementEnabled'; + 'OEMPreInstalledAppsEnabled'; + 'PreInstalledAppsEnabled'; + 'PreInstalledAppsEverEnabled'; + 'SilentInstalledAppsEnabled'; + 'SoftLandingEnabled'; + 'SubscribedContentEnabled'; + 'SubscribedContent-310093Enabled'; + 'SubscribedContent-338387Enabled'; + 'SubscribedContent-338388Enabled'; + 'SubscribedContent-338389Enabled'; + 'SubscribedContent-338393Enabled'; + 'SubscribedContent-353694Enabled'; + 'SubscribedContent-353696Enabled'; + 'SubscribedContent-353698Enabled'; + 'SystemPaneSuggestionsEnabled'; + ); + + foreach( $name in $names ) { + reg.exe add "HKU\DefaultUser\Software\Microsoft\Windows\CurrentVersion\ContentDeliveryManager" /v $name /t REG_DWORD /d 0 /f; + } + }; + { + reg.exe add "HKU\DefaultUser\Software\Microsoft\Windows\CurrentVersion\Explorer\Advanced" /v TaskbarAl /t REG_DWORD /d 0 /f; + }; + { + reg.exe add "HKU\DefaultUser\Software\Microsoft\Windows\CurrentVersion\RunOnce" /v "UnattendedSetup" /t REG_SZ /d "powershell.exe -WindowStyle \""Normal\"" -ExecutionPolicy \""Unrestricted\"" -NoProfile -File \""C:\Windows\Setup\Scripts\UserOnce.ps1\""" /f; + }; + ); + + & { + [float] $complete = 0; + [float] $increment = 100 / $scripts.Count; + foreach( $script in $scripts ) { + Write-Progress -Activity 'Running scripts to modify the default user’’s registry hive. Do not close this window.' -PercentComplete $complete; + '*** Will now execute command «{0}».' -f $( + $str = $script.ToString().Trim() -replace '\s+', ' '; + $max = 100; + if( $str.Length -le $max ) { + $str; + } else { + $str.Substring( 0, $max - 1 ) + '…'; + } + ); + $start = [datetime]::Now; + & $script; + '*** Finished executing command after {0:0} ms.' -f [datetime]::Now.Subtract( $start ).TotalMilliseconds; + "`r`n" * 3; + $complete += $increment; + } + } *>&1 | Out-String -Width 1KB -Stream >> "C:\Windows\Setup\Scripts\DefaultUser.log"; + - path: "W:\\Windows\\Setup\\Scripts\\FirstLogon.ps1" + enableReplacements: true + content: | + $scripts = @( + { + Set-ItemProperty -LiteralPath 'Registry::HKLM\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Winlogon' -Name 'AutoLogonCount' -Type 'DWord' -Force -Value 0; + }; + { + Disable-ComputerRestore -Drive 'C:\'; + }; + { + & 'C:\ProgramData\UET\RKM-Provisioned\uet.exe' cluster start --no-stream-logs --node "{{param:clusterControllerIpAddress}}" + }; + { + Remove-Item -LiteralPath @( + 'C:\Windows\Panther\unattend.xml'; + 'C:\Windows\Panther\unattend-original.xml'; + 'C:\Windows\Setup\Scripts\Wifi.xml'; + ) -Force -ErrorAction 'SilentlyContinue' -Verbose; + }; + ); + + & { + [float] $complete = 0; + [float] $increment = 100 / $scripts.Count; + foreach( $script in $scripts ) { + Write-Progress -Activity 'Running scripts to finalize your Windows installation. Do not close this window.' -PercentComplete $complete; + '*** Will now execute command «{0}».' -f $( + $str = $script.ToString().Trim() -replace '\s+', ' '; + $max = 100; + if( $str.Length -le $max ) { + $str; + } else { + $str.Substring( 0, $max - 1 ) + '…'; + } + ); + $start = [datetime]::Now; + & $script; + '*** Finished executing command after {0:0} ms.' -f [datetime]::Now.Subtract( $start ).TotalMilliseconds; + "`r`n" * 3; + $complete += $increment; + } + } *>&1 | Out-String -Width 1KB -Stream >> "C:\Windows\Setup\Scripts\FirstLogon.log"; + - type: reboot + reboot: + defaultInitrd: true + - type: executeProcess + executeProcess: + executable: bash + arguments: + - "{{script-path}}" + script: | + #!/bin/bash + + set -e + set -x + + echo "Tidying up files we no longer need..." + rm -rf /var/mount/images/WinPEStage1 || true; + + # note: we keep LinuxTmp.ext4 and install.esd as these take a long to prepare + - type: test + test: + value: Running in Linux initrd once more to fix up boot manager order, and then we're done. \ No newline at end of file diff --git a/UET/Lib/Redpoint.ThirdParty.GitHub.JPMikkers.Dhcp/DefaultDhcpServer.cs b/UET/Lib/Redpoint.ThirdParty.GitHub.JPMikkers.Dhcp/DefaultDhcpServer.cs index 405a851f..16e0252d 100644 --- a/UET/Lib/Redpoint.ThirdParty.GitHub.JPMikkers.Dhcp/DefaultDhcpServer.cs +++ b/UET/Lib/Redpoint.ThirdParty.GitHub.JPMikkers.Dhcp/DefaultDhcpServer.cs @@ -937,6 +937,7 @@ private async Task OnReceive(IPEndPoint endPoint, ReadOnlyMemory<byte> data) { DhcpClient client = DhcpClient.CreateFromMessage(dhcpMessage); _logger.LogTrace($"Client {client} sent {dhcpMessage.MessageType}"); + _logger.LogInformation($"DHCP client is identified as '{Utils.BytesToHexString(client.Identifier, "-")}'."); // is it a known client? DhcpClient? knownClient = GetKnownClient(client); diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/BlockCounterWrapping.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/BlockCounterWrapping.cs new file mode 100644 index 00000000..388701f7 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/BlockCounterWrapping.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tftp.Net +{ + public enum BlockCounterWrapAround + { + ToZero, + ToOne + } + + static class BlockCounterWrappingHelpers + { + private const ushort LAST_AVAILABLE_BLOCK_NUMBER = 65535; + + public static ushort CalculateNextBlockNumber(this BlockCounterWrapAround wrapping, ushort previousBlockNumber) + { + if (previousBlockNumber == LAST_AVAILABLE_BLOCK_NUMBER) + return wrapping == BlockCounterWrapAround.ToZero ? (ushort)0 : (ushort)1; + + return (ushort)(previousBlockNumber + 1); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/ITransferChannel.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/ITransferChannel.cs new file mode 100644 index 00000000..ff825d26 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/ITransferChannel.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Net; +using Redpoint.Concurrency; + +namespace Tftp.Net.Channel +{ + interface ITransferChannel : IAsyncDisposable + { + IAsyncEvent<TftpCommandEventArgs> OnCommandReceived { get; } + IAsyncEvent<TftpTransferError> OnError { get; } + + EndPoint RemoteEndpoint { get; set; } + + void Open(); + void Send(ITftpCommand command); + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/TftpCommandEventArgs.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/TftpCommandEventArgs.cs new file mode 100644 index 00000000..53ca7447 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/TftpCommandEventArgs.cs @@ -0,0 +1,11 @@ +using System.Net; + +namespace Tftp.Net.Channel +{ + internal class TftpCommandEventArgs + { + public required ITftpCommand Command { get; init; } + + public required EndPoint Endpoint { get; init; } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/TransferChannelFactory.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/TransferChannelFactory.cs new file mode 100644 index 00000000..18c86228 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/TransferChannelFactory.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Net; +using System.Net.Sockets; + +namespace Tftp.Net.Channel +{ + static class TransferChannelFactory + { + public static ITransferChannel CreateServer(EndPoint localAddress) + { + if (localAddress is IPEndPoint) + return CreateServerUdp((IPEndPoint)localAddress); + + throw new NotSupportedException("Unsupported endpoint type."); + } + + public static ITransferChannel CreateConnection(EndPoint remoteAddress) + { + if (remoteAddress is IPEndPoint) + return CreateConnectionUdp((IPEndPoint)remoteAddress); + + throw new NotSupportedException("Unsupported endpoint type."); + } + + #region UDP connections + + private static ITransferChannel CreateServerUdp(IPEndPoint localAddress) + { + UdpClient socket = new UdpClient(localAddress); + return new UdpChannel(socket); + } + + private static ITransferChannel CreateConnectionUdp(IPEndPoint remoteAddress) + { + IPEndPoint localAddress = new IPEndPoint(IPAddress.Any, 0); + UdpChannel channel = new UdpChannel(new UdpClient(localAddress)); + channel.RemoteEndpoint = remoteAddress; + return channel; + } + #endregion + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/UdpChannel.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/UdpChannel.cs new file mode 100644 index 00000000..80889e8f --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Channel/UdpChannel.cs @@ -0,0 +1,158 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Net.Sockets; +using System.Net; +using System.IO; +using System.Diagnostics; +using System.Threading.Tasks; +using Redpoint.Concurrency; + +namespace Tftp.Net.Channel +{ + class UdpChannel : ITransferChannel + { + private AsyncEvent<TftpCommandEventArgs> _onCommandReceived = new(); + private AsyncEvent<TftpTransferError> _onError = new(); + + public IAsyncEvent<TftpCommandEventArgs> OnCommandReceived => _onCommandReceived; + public IAsyncEvent<TftpTransferError> OnError => _onError; + + private IPEndPoint endpoint; + private UdpClient client; + private readonly CommandSerializer serializer = new CommandSerializer(); + private readonly CommandParser parser = new CommandParser(); + + private CancellationTokenSource _cancelReceiving = new(); + private Task _receivingTask; + + public UdpChannel(UdpClient client) + { + this.client = client; + this.endpoint = null; + } + + public void Open() + { + if (client == null) + throw new ObjectDisposedException("UdpChannel"); + + _receivingTask = Task.Run( + async () => + { + while (!_cancelReceiving.IsCancellationRequested) + { + UdpReceiveResult result = await client.ReceiveAsync(_cancelReceiving.Token); + + IPEndPoint endpoint = new IPEndPoint(0, 0); + ITftpCommand command = null; + + try + { + byte[] data = result.Buffer; + endpoint = result.RemoteEndPoint; + + command = parser.Parse(data); + } + catch (SocketException e) + { + //Handle receive error + await RaiseOnError(new NetworkError(e)); + } + catch (TftpParserException e2) + { + //Handle parser error + await RaiseOnError(new NetworkError(e2)); + } + + if (command != null) + { + await RaiseOnCommand(command, endpoint); + } + } + }, + CancellationToken.None); + } + + private async Task RaiseOnCommand(ITftpCommand command, IPEndPoint endpoint) + { + await _onCommandReceived.BroadcastAsync( + new TftpCommandEventArgs + { + Command = command, + Endpoint = endpoint, + }, + CancellationToken.None); + } + + private async Task RaiseOnError(TftpTransferError error) + { + await _onError.BroadcastAsync( + error, + CancellationToken.None); + } + + public void Send(ITftpCommand command) + { + if (client == null) + throw new ObjectDisposedException("UdpChannel"); + + if (endpoint == null) + throw new InvalidOperationException("RemoteEndpoint needs to be set before you can send TFTP commands."); + + using (MemoryStream stream = new MemoryStream()) + { + serializer.Serialize(command, stream); + byte[] data = stream.GetBuffer(); + client.Send(data, (int)stream.Length, endpoint); + } + } + + public async ValueTask DisposeAsync() + { + if (_receivingTask != null) + { + _cancelReceiving.Cancel(); + + try + { + await _receivingTask; + } + catch + { + } + + _cancelReceiving.Dispose(); + + _receivingTask = null; + _cancelReceiving = null; + } + + if (this.client != null) + { + client.Close(); + this.client = null; + } + } + + public EndPoint RemoteEndpoint + { + get + { + return endpoint; + } + + set + { + if (!(value is IPEndPoint)) + throw new NotSupportedException("UdpChannel can only connect to IPEndPoints."); + + if (client == null) + throw new ObjectDisposedException("UdpChannel"); + + this.endpoint = (IPEndPoint)value; + } + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Acknowledgement.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Acknowledgement.cs new file mode 100644 index 00000000..58e7ca88 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Acknowledgement.cs @@ -0,0 +1,19 @@ +namespace Tftp.Net +{ + class Acknowledgement : ITftpCommand + { + public const ushort OpCode = 4; + + public ushort BlockNumber { get; private set; } + + public Acknowledgement(ushort blockNumber) + { + this.BlockNumber = blockNumber; + } + + public Task Visit(ITftpCommandVisitor visitor) + { + return visitor.OnAcknowledgement(this); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/CommandParser.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/CommandParser.cs new file mode 100644 index 00000000..65385c06 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/CommandParser.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Tftp.Net.Transfer; + +namespace Tftp.Net +{ + /// <summary> + /// Parses a ITftpCommand. + /// </summary> + class CommandParser + { + /// <summary> + /// Parses an ITftpCommand from the given byte array. If the byte array cannot be parsed for some reason, a TftpParserException is thrown. + /// </summary> + public ITftpCommand Parse(byte[] message) + { + try + { + return ParseInternal(message); + } + catch (TftpParserException) + { + throw; + } + catch (Exception e) + { + throw new TftpParserException(e); + } + } + + private ITftpCommand ParseInternal(byte[] message) + { + TftpStreamReader reader = new TftpStreamReader(new MemoryStream(message)); + + ushort opcode = reader.ReadUInt16(); + switch (opcode) + { + case ReadRequest.OpCode: + return ParseReadRequest(reader); + + case WriteRequest.OpCode: + return ParseWriteRequest(reader); + + case Data.OpCode: + return ParseData(reader); + + case Acknowledgement.OpCode: + return ParseAcknowledgement(reader); + + case Error.OpCode: + return ParseError(reader); + + case OptionAcknowledgement.OpCode: + return ParseOptionAcknowledgement(reader); + + default: + throw new TftpParserException("Invalid opcode"); + } + } + + private OptionAcknowledgement ParseOptionAcknowledgement(TftpStreamReader reader) + { + IEnumerable<TransferOption> options = ParseTransferOptions(reader); + return new OptionAcknowledgement(options); + } + + private Error ParseError(TftpStreamReader reader) + { + ushort errorCode = reader.ReadUInt16(); + String message = ParseNullTerminatedString(reader); + return new Error(errorCode, message); + } + + private Acknowledgement ParseAcknowledgement(TftpStreamReader reader) + { + ushort blockNumber = reader.ReadUInt16(); + return new Acknowledgement(blockNumber); + } + + private Data ParseData(TftpStreamReader reader) + { + ushort blockNumber = reader.ReadUInt16(); + byte[] data = reader.ReadBytes(10000); + return new Data(blockNumber, data); + } + + private WriteRequest ParseWriteRequest(TftpStreamReader reader) + { + String filename = ParseNullTerminatedString(reader); + TftpTransferMode mode = ParseModeType(ParseNullTerminatedString(reader)); + IEnumerable<TransferOption> options = ParseTransferOptions(reader); + return new WriteRequest(filename, mode, options); + } + + private ReadRequest ParseReadRequest(TftpStreamReader reader) + { + String filename = ParseNullTerminatedString(reader); + TftpTransferMode mode = ParseModeType(ParseNullTerminatedString(reader)); + IEnumerable<TransferOption> options = ParseTransferOptions(reader); + return new ReadRequest(filename, mode, options); + } + + private List<TransferOption> ParseTransferOptions(TftpStreamReader reader) + { + List<TransferOption> options = new List<TransferOption>(); + + while (true) + { + String name; + + try + { + name = ParseNullTerminatedString(reader); + } + catch (IOException) + { + name = ""; + } + + if (name.Length == 0) + break; + + string value = ParseNullTerminatedString(reader); + options.Add(new TransferOption(name, value)); + } + return options; + } + + private String ParseNullTerminatedString(TftpStreamReader reader) + { + byte b; + StringBuilder str = new StringBuilder(); + while ((b = reader.ReadByte()) > 0) + { + str.Append((char)b); + } + + return str.ToString(); + } + + private TftpTransferMode ParseModeType(String mode) + { + mode = mode.ToLowerInvariant(); + + if (mode == "netascii") + return TftpTransferMode.netascii; + + if (mode == "mail") + return TftpTransferMode.mail; + + if (mode == "octet") + return TftpTransferMode.octet; + + throw new TftpParserException("Unknown mode type: " + mode); + } + } + + [Serializable] + class TftpParserException : Exception + { + public TftpParserException(String message) + : base(message) { } + + public TftpParserException(Exception e) + : base("Error while parsing message.", e) { } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/CommandSerializer.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/CommandSerializer.cs new file mode 100644 index 00000000..494c5601 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/CommandSerializer.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace Tftp.Net +{ + /// <summary> + /// Serializes an ITftpCommand into a stream of bytes. + /// </summary> + class CommandSerializer + { + /// <summary> + /// Call this method to serialize the given <code>command</code> using the given <code>writer</code>. + /// </summary> + public void Serialize(ITftpCommand command, Stream stream) + { + CommandComposerVisitor visitor = new CommandComposerVisitor(stream); + command.Visit(visitor); + } + + private class CommandComposerVisitor : ITftpCommandVisitor + { + private readonly TftpStreamWriter writer; + + public CommandComposerVisitor(Stream stream) + { + this.writer = new TftpStreamWriter(stream); + } + + private void OnReadOrWriteRequest(ReadOrWriteRequest command) + { + writer.WriteBytes(Encoding.ASCII.GetBytes(command.Filename)); + writer.WriteByte(0); + writer.WriteBytes(Encoding.ASCII.GetBytes(command.Mode.ToString())); + writer.WriteByte(0); + + if (command.Options != null) + { + foreach (var option in command.Options) + { + writer.WriteBytes(Encoding.ASCII.GetBytes(option.Name)); + writer.WriteByte(0); + writer.WriteBytes(Encoding.ASCII.GetBytes(option.Value)); + writer.WriteByte(0); + } + } + } + + public Task OnReadRequest(ReadRequest command) + { + writer.WriteUInt16(ReadRequest.OpCode); + OnReadOrWriteRequest(command); + return Task.CompletedTask; + } + + public Task OnWriteRequest(WriteRequest command) + { + writer.WriteUInt16(WriteRequest.OpCode); + OnReadOrWriteRequest(command); + return Task.CompletedTask; + } + + public Task OnData(Data command) + { + writer.WriteUInt16(Data.OpCode); + writer.WriteUInt16(command.BlockNumber); + writer.WriteBytes(command.Bytes); + return Task.CompletedTask; + } + + public Task OnAcknowledgement(Acknowledgement command) + { + writer.WriteUInt16(Acknowledgement.OpCode); + writer.WriteUInt16(command.BlockNumber); + return Task.CompletedTask; + } + + public Task OnError(Error command) + { + writer.WriteUInt16(Error.OpCode); + writer.WriteUInt16(command.ErrorCode); + writer.WriteBytes(Encoding.ASCII.GetBytes(command.Message)); + writer.WriteByte(0); + return Task.CompletedTask; + } + + public Task OnOptionAcknowledgement(OptionAcknowledgement command) + { + writer.WriteUInt16(OptionAcknowledgement.OpCode); + + foreach (var option in command.Options) + { + writer.WriteBytes(Encoding.ASCII.GetBytes(option.Name)); + writer.WriteByte(0); + writer.WriteBytes(Encoding.ASCII.GetBytes(option.Value)); + writer.WriteByte(0); + } + + return Task.CompletedTask; + } + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Data.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Data.cs new file mode 100644 index 00000000..5e242aa7 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Data.cs @@ -0,0 +1,21 @@ +namespace Tftp.Net +{ + class Data : ITftpCommand + { + public const ushort OpCode = 3; + + public ushort BlockNumber { get; private set; } + public byte[] Bytes { get; private set; } + + public Data(ushort blockNumber, byte[] data) + { + this.BlockNumber = blockNumber; + this.Bytes = data; + } + + public Task Visit(ITftpCommandVisitor visitor) + { + return visitor.OnData(this); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Error.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Error.cs new file mode 100644 index 00000000..b475d148 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/Error.cs @@ -0,0 +1,21 @@ +namespace Tftp.Net +{ + class Error : ITftpCommand + { + public const ushort OpCode = 5; + + public ushort ErrorCode { get; private set; } + public String Message { get; private set; } + + public Error(ushort errorCode, String message) + { + this.ErrorCode = errorCode; + this.Message = message; + } + + public Task Visit(ITftpCommandVisitor visitor) + { + return visitor.OnError(this); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ITftpCommand.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ITftpCommand.cs new file mode 100644 index 00000000..2df479ba --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ITftpCommand.cs @@ -0,0 +1,7 @@ +namespace Tftp.Net +{ + interface ITftpCommand + { + Task Visit(ITftpCommandVisitor visitor); + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ITftpCommandVisitor.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ITftpCommandVisitor.cs new file mode 100644 index 00000000..5fe3f2dd --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ITftpCommandVisitor.cs @@ -0,0 +1,12 @@ +namespace Tftp.Net +{ + interface ITftpCommandVisitor + { + Task OnReadRequest(ReadRequest command); + Task OnWriteRequest(WriteRequest command); + Task OnData(Data command); + Task OnAcknowledgement(Acknowledgement command); + Task OnError(Error command); + Task OnOptionAcknowledgement(OptionAcknowledgement command); + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/OptionAcknowledgement.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/OptionAcknowledgement.cs new file mode 100644 index 00000000..99b15c58 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/OptionAcknowledgement.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Tftp.Net.Transfer; + +namespace Tftp.Net +{ + class OptionAcknowledgement : ITftpCommand + { + public const ushort OpCode = 6; + public IEnumerable<TransferOption> Options { get; private set; } + + public OptionAcknowledgement(IEnumerable<TransferOption> options) + { + this.Options = options; + } + + public Task Visit(ITftpCommandVisitor visitor) + { + return visitor.OnOptionAcknowledgement(this); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ReadOrWriteRequest.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ReadOrWriteRequest.cs new file mode 100644 index 00000000..1f2ba811 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ReadOrWriteRequest.cs @@ -0,0 +1,19 @@ +namespace Tftp.Net +{ + abstract class ReadOrWriteRequest + { + private readonly ushort opCode; + + public String Filename { get; private set; } + public TftpTransferMode Mode { get; private set; } + public IEnumerable<TransferOption> Options { get; private set; } + + protected ReadOrWriteRequest(ushort opCode, String filename, TftpTransferMode mode, IEnumerable<TransferOption> options) + { + this.opCode = opCode; + this.Filename = filename; + this.Mode = mode; + this.Options = options; + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ReadRequest.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ReadRequest.cs new file mode 100644 index 00000000..33c23442 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/ReadRequest.cs @@ -0,0 +1,15 @@ +namespace Tftp.Net +{ + class ReadRequest : ReadOrWriteRequest, ITftpCommand + { + public const ushort OpCode = 1; + + public ReadRequest(String filename, TftpTransferMode mode, IEnumerable<TransferOption> options) + : base(OpCode, filename, mode, options) { } + + public Task Visit(ITftpCommandVisitor visitor) + { + return visitor.OnReadRequest(this); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TftpStreamReader.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TftpStreamReader.cs new file mode 100644 index 00000000..91deee98 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TftpStreamReader.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Net.Sockets; + +namespace Tftp.Net +{ + class TftpStreamReader + { + private readonly Stream stream; + + public TftpStreamReader(Stream stream) + { + this.stream = stream; + } + + public ushort ReadUInt16() + { + int byte1 = stream.ReadByte(); + int byte2 = stream.ReadByte(); + return (ushort)((byte)byte1 << 8 | (byte)byte2); + } + + public byte ReadByte() + { + int nextByte = stream.ReadByte(); + + if (nextByte == -1) + throw new IOException(); + + return (byte)nextByte; + } + + public byte[] ReadBytes(int maxBytes) + { + byte[] buffer = new byte[maxBytes]; + int bytesRead = stream.Read(buffer, 0, buffer.Length); + + if (bytesRead == -1) + throw new IOException(); + + Array.Resize(ref buffer, bytesRead); + return buffer; + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TftpStreamWriter.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TftpStreamWriter.cs new file mode 100644 index 00000000..bc9d6214 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TftpStreamWriter.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace Tftp.Net +{ + class TftpStreamWriter + { + private readonly Stream stream; + + public TftpStreamWriter(Stream stream) + { + this.stream = stream; + } + + public void WriteUInt16(ushort value) + { + stream.WriteByte((byte)(value >> 8)); + stream.WriteByte((byte)(value & 0xFF)); + } + + public void WriteByte(byte b) + { + stream.WriteByte(b); + } + + public void WriteBytes(byte[] data) + { + stream.Write(data, 0, data.Length); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TransferOption.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TransferOption.cs new file mode 100644 index 00000000..45ff5c9b --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/TransferOption.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tftp.Net +{ + /// <summary> + /// A single transfer options according to RFC2347. + /// </summary> + public class TransferOption + { + public String Name { get; private set; } + public String Value { get; set; } + public bool IsAcknowledged { get; internal set; } + + internal TransferOption(string name, string value) + { + if (String.IsNullOrEmpty(name)) + throw new ArgumentException("name must not be null or empty."); + + if (value == null) + throw new ArgumentNullException("value must not be null."); + + this.Name = name; + this.Value = value; + } + + public override string ToString() + { + return Name + "=" + Value; + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/WriteRequest.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/WriteRequest.cs new file mode 100644 index 00000000..b3120dca --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Commands/WriteRequest.cs @@ -0,0 +1,15 @@ +namespace Tftp.Net +{ + class WriteRequest : ReadOrWriteRequest, ITftpCommand + { + public const ushort OpCode = 2; + + public WriteRequest(String filename, TftpTransferMode mode, IEnumerable<TransferOption> options) + : base(OpCode, filename, mode, options) { } + + public Task Visit(ITftpCommandVisitor visitor) + { + return visitor.OnWriteRequest(this); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/ITftpTransfer.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/ITftpTransfer.cs new file mode 100644 index 00000000..bf0203f5 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/ITftpTransfer.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Redpoint.Concurrency; + +namespace Tftp.Net +{ + /// <summary> + /// Represents a single data transfer between a TFTP server and client. + /// </summary> + public interface ITftpTransfer : IAsyncDisposable + { + /// <summary> + /// Event that is being called while data is being transferred. + /// </summary> + IAsyncEvent<TftpProgressHandlerArgs> OnProgress { get; } + + /// <summary> + /// Event that will be called once the data transfer is finished. + /// </summary> + IAsyncEvent<ITftpTransfer> OnFinished { get; } + + /// <summary> + /// Event that will be called if there is an error during the data transfer. + /// Currently, this will return instances of ErrorFromRemoteEndpoint or NetworkError. + /// </summary> + IAsyncEvent<TftpErrorHandlerArgs> OnError { get; } + + /// <summary> + /// Requested TFTP transfer mode. For outgoing transfers, this member may be used to set the transfer mode. + /// </summary> + TftpTransferMode TransferMode { get; set; } + + /// <summary> + /// Transfer blocksize. Set this member to control the TFTP blocksize option (RFC 2349). + /// </summary> + int BlockSize { get; set; } + + /// <summary> + /// Timeout after which commands are sent again. + /// This member is also transmitted as the TFTP timeout interval option (RFC 2349). + /// </summary> + TimeSpan RetryTimeout { get; set; } + + /// <summary> + /// Number of times that a RetryTimeout may occour before the transfer is cancelled with a TimeoutError. + /// </summary> + int RetryCount { get; set; } + + /// <summary> + /// Tftp can transfer up to 65535 blocks. After that, the block counter wraps to either zero or one, depending on the expectations of the client. + /// </summary> + BlockCounterWrapAround BlockCounterWrapping { get; set; } + + /// <summary> + /// Expected transfer size in bytes. 0 if size is unknown. + /// </summary> + long ExpectedSize { get; set; } + + /// <summary> + /// Filename for the transferred file. + /// </summary> + String Filename { get; } + + /// <summary> + /// You can set your own object here to associate custom data with this transfer. + /// </summary> + object UserContext { get; set; } + + /// <summary> + /// Call this function to start the transfer. + /// </summary> + /// <param name="data">The stream from which data is either read (when sending) or written to (when receiving).</param> + void Start(Stream data); + + /// <summary> + /// Cancel the currently running transfer, possibly sending the provided reason to the remote endpoint. + /// </summary> + void Cancel(TftpErrorPacket reason); + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Redpoint.ThirdParty.Tftp.Net.csproj b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Redpoint.ThirdParty.Tftp.Net.csproj new file mode 100644 index 00000000..24974df1 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Redpoint.ThirdParty.Tftp.Net.csproj @@ -0,0 +1,24 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <Import Project="$(MSBuildThisFileDirectory)../ThirdPartyCommon.Build.props" /> + + <PropertyGroup> + <RootNamespace>Tftp.Net</RootNamespace> + <Nullable>disable</Nullable> + <NoWarn>$(NoWarn);CS1591</NoWarn> + </PropertyGroup> + + <Import Project="$(MSBuildThisFileDirectory)../LibraryPackaging.Build.props" /> + <PropertyGroup> + <Description>A fork of Tftp.Net that has been updated to provide async compatible APIs.</Description> + <PackageId>Redpoint.ThirdParty.Tftp.Net</PackageId> + <PackageTags>tftp</PackageTags> + <PackageLicenseExpression>MS-PL</PackageLicenseExpression> + <Authors>June Rhodes, Callisto, linnet</Authors> + <Company></Company> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="..\..\Redpoint.Concurrency\Redpoint.Concurrency.csproj" /> + </ItemGroup> + +</Project> diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpClient.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpClient.cs new file mode 100644 index 00000000..4fe7dd77 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpClient.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Net; +using System.Net.Sockets; +using Tftp.Net.Channel; +using Tftp.Net.Transfer.States; +using Tftp.Net.Transfer; + +namespace Tftp.Net +{ + /// <summary> + /// A TFTP client that can connect to a TFTP server. + /// </summary> + public class TftpClient + { + private const int DEFAULT_SERVER_PORT = 69; + private readonly IPEndPoint remoteAddress; + + /// <summary> + /// Default constructor. + /// </summary> + /// <param name="remoteAddress">Address of the server that you would like to connect to.</param> + public TftpClient(IPEndPoint remoteAddress) + { + this.remoteAddress = remoteAddress; + } + + /// <summary> + /// Connects to a server + /// </summary> + /// <param name="ip">Address of the server that you want connect to.</param> + /// <param name="port">Port on the server that you want connect to (default: 69)</param> + public TftpClient(IPAddress ip, int port) + : this(new IPEndPoint(ip, port)) + { + } + + /// <summary> + /// Connects to a server on port 69. + /// </summary> + /// <param name="ip">Address of the server that you want connect to.</param> + public TftpClient(IPAddress ip) + : this(new IPEndPoint(ip, DEFAULT_SERVER_PORT)) + { + } + + /// <summary> + /// Connect to a server by hostname. + /// </summary> + /// <param name="host">Hostname or ip to connect to</param> + public TftpClient(String host) + : this(host, DEFAULT_SERVER_PORT) + { + } + + /// <summary> + /// Connect to a server by hostname and port . + /// </summary> + /// <param name="host">Hostname or ip to connect to</param> + /// <param name="port">Port to connect to</param> + public TftpClient(String host, int port) + { + IPAddress ip = Dns.GetHostAddresses(host).FirstOrDefault(x => x.AddressFamily == AddressFamily.InterNetwork); + + if (ip == null) + throw new ArgumentException("Could not convert '" + host + "' to an IPv4 address.", "host"); + + this.remoteAddress = new IPEndPoint(ip, port); + } + + /// <summary> + /// GET a file from the server. + /// You have to call Start() on the returned ITftpTransfer to start the transfer. + /// </summary> + public async Task<ITftpTransfer> Download(String filename) + { + ITransferChannel channel = TransferChannelFactory.CreateConnection(remoteAddress); + return await RemoteReadTransfer.CreateRemoteReadTransferAsync(channel, filename); + } + + /// <summary> + /// PUT a file from the server. + /// You have to call Start() on the returned ITftpTransfer to start the transfer. + /// </summary> + public async Task<ITftpTransfer> Upload(String filename) + { + ITransferChannel channel = TransferChannelFactory.CreateConnection(remoteAddress); + return await RemoteWriteTransfer.CreateRemoteWriteTransferAsync(channel, filename); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpErrorHandlerArgs.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpErrorHandlerArgs.cs new file mode 100644 index 00000000..e016ff52 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpErrorHandlerArgs.cs @@ -0,0 +1,9 @@ +namespace Tftp.Net +{ + public class TftpErrorHandlerArgs + { + public required ITftpTransfer Transfer { get; init; } + + public required TftpTransferError Error { get; init; } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpProgressHandlerArgs.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpProgressHandlerArgs.cs new file mode 100644 index 00000000..e33f5d32 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpProgressHandlerArgs.cs @@ -0,0 +1,9 @@ +namespace Tftp.Net +{ + public class TftpProgressHandlerArgs + { + public required ITftpTransfer Transfer { get; init; } + + public required TftpTransferProgress Progress { get; init; } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpServer.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpServer.cs new file mode 100644 index 00000000..62423b9e --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpServer.cs @@ -0,0 +1,158 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Net.Sockets; +using System.Net; +using Tftp.Net.Channel; +using Tftp.Net.Transfer; +using Redpoint.Concurrency; + +namespace Tftp.Net +{ + /// <summary> + /// A simple TFTP server class. <code>Dispose()</code> the server to close the socket that it listens on. + /// </summary> + public class TftpServer : IAsyncDisposable + { + private AsyncEvent<TftpServerEventHandlerArgs> _onReadRequest = new(); + private AsyncEvent<TftpServerEventHandlerArgs> _onWriteRequest = new(); + private AsyncEvent<TftpTransferError> _onError = new(); + + public const int DEFAULT_SERVER_PORT = 69; + + /// <summary> + /// Fired when the server receives a new read request. + /// </summary> + public IAsyncEvent<TftpServerEventHandlerArgs> OnReadRequest => _onReadRequest; + + /// <summary> + /// Fired when the server receives a new write request. + /// </summary> + public IAsyncEvent<TftpServerEventHandlerArgs> OnWriteRequest => _onWriteRequest; + + /// <summary> + /// Fired when the server encounters an error (for example, a non-parseable request) + /// </summary> + public IAsyncEvent<TftpTransferError> OnError => _onError; + + /// <summary> + /// Server port that we're listening on. + /// </summary> + private readonly ITransferChannel serverSocket; + + public TftpServer(IPEndPoint localAddress) + { + if (localAddress == null) + throw new ArgumentNullException("localAddress"); + + serverSocket = TransferChannelFactory.CreateServer(localAddress); + serverSocket.OnCommandReceived.Add(serverSocket_OnCommandReceived); + serverSocket.OnError.Add(serverSocket_OnError); + } + + public TftpServer(IPAddress localAddress) + : this(localAddress, DEFAULT_SERVER_PORT) + { + } + + public TftpServer(IPAddress localAddress, int port) + : this(new IPEndPoint(localAddress, port)) + { + } + + public TftpServer(int port) + : this(new IPEndPoint(IPAddress.Any, port)) + { + } + + public TftpServer() + : this(DEFAULT_SERVER_PORT) + { + } + + + /// <summary> + /// Start accepting incoming connections. + /// </summary> + public void Start() + { + serverSocket.Open(); + } + + async Task serverSocket_OnError(TftpTransferError error, CancellationToken cancellationToken) + { + await RaiseOnError(error, cancellationToken); + } + + private async Task serverSocket_OnCommandReceived(TftpCommandEventArgs args, CancellationToken cancellationToken) + { + //Ignore all other commands + if (!(args.Command is ReadOrWriteRequest)) + return; + + //Open a connection to the client + ITransferChannel channel = TransferChannelFactory.CreateConnection(args.Endpoint); + + //Create a wrapper for the transfer request + ReadOrWriteRequest request = (ReadOrWriteRequest)args.Command; + ITftpTransfer transfer = request is ReadRequest + ? (ITftpTransfer)(await LocalReadTransfer.CreateLocalReadTransferAsync(channel, request.Filename, request.Options)) + : (await LocalWriteTransfer.CreateLocalWriteTransferAsync(channel, request.Filename, request.Options)); + + if (args.Command is ReadRequest) + await RaiseOnReadRequest(transfer, args.Endpoint, cancellationToken); + else if (args.Command is WriteRequest) + await RaiseOnWriteRequest(transfer, args.Endpoint, cancellationToken); + else + throw new Exception("Unexpected tftp transfer request: " + args.Command); + } + + public async ValueTask DisposeAsync() + { + await serverSocket.DisposeAsync(); + } + + private async Task RaiseOnError(TftpTransferError error, CancellationToken cancellationToken) + { + await _onError.BroadcastAsync(error, cancellationToken); + } + + private async Task RaiseOnWriteRequest(ITftpTransfer transfer, EndPoint client, CancellationToken cancellationToken) + { + if (_onWriteRequest.HasAnyBindings) + { + await _onWriteRequest.BroadcastAsync( + new TftpServerEventHandlerArgs + { + Transfer = transfer, + EndPoint = client, + }, + cancellationToken); + } + else + { + transfer.Cancel(new TftpErrorPacket(0, "Server did not provide a OnWriteRequest handler.")); + } + } + + private async Task RaiseOnReadRequest(ITftpTransfer transfer, EndPoint client, CancellationToken cancellationToken) + { + if (_onReadRequest.HasAnyBindings) + { + await _onReadRequest.BroadcastAsync( + new TftpServerEventHandlerArgs + { + Transfer = transfer, + EndPoint = client, + }, + cancellationToken); + } + else + { + transfer.Cancel(new TftpErrorPacket(0, "Server did not provide a OnReadRequest handler.")); + } + } + } +} + diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpServerEventHandlerArgs.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpServerEventHandlerArgs.cs new file mode 100644 index 00000000..af6edf40 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpServerEventHandlerArgs.cs @@ -0,0 +1,12 @@ +using System.Net; + +namespace Tftp.Net +{ + public class TftpServerEventHandlerArgs + { + public required ITftpTransfer Transfer { get; init; } + + public required EndPoint EndPoint { get; init; } + } +} + diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferError.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferError.cs new file mode 100644 index 00000000..a6c7a327 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferError.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tftp.Net +{ + /// <summary> + /// Base class for all errors that may be passed to <code>ITftpTransfer.OnError</code>. + /// </summary> + public abstract class TftpTransferError + { + public abstract override String ToString(); + } + + /// <summary> + /// Errors that are sent from the remote party using the TFTP Error Packet are represented + /// by this class. + /// </summary> + public class TftpErrorPacket : TftpTransferError + { + /// <summary> + /// Error code that was sent from the other party. + /// </summary> + public ushort ErrorCode { get; private set; } + + /// <summary> + /// Error description that was sent by the other party. + /// </summary> + public string ErrorMessage { get; private set; } + + public TftpErrorPacket(ushort errorCode, string errorMessage) + { + if (String.IsNullOrEmpty(errorMessage)) + throw new ArgumentException("You must provide an errorMessage."); + + this.ErrorCode = errorCode; + this.ErrorMessage = errorMessage; + } + + public override string ToString() + { + return ErrorCode + " - " + ErrorMessage; + } + + #region Predefined error packets from RFC 1350 + public static readonly TftpErrorPacket FileNotFound = new TftpErrorPacket(1, "File not found"); + public static readonly TftpErrorPacket AccessViolation = new TftpErrorPacket(2, "Access violation"); + public static readonly TftpErrorPacket DiskFull = new TftpErrorPacket(3, "Disk full or allocation exceeded"); + public static readonly TftpErrorPacket IllegalOperation = new TftpErrorPacket(4, "Illegal TFTP operation"); + public static readonly TftpErrorPacket UnknownTransferId = new TftpErrorPacket(5, "Unknown transfer ID"); + public static readonly TftpErrorPacket FileAlreadyExists = new TftpErrorPacket(6, "File already exists"); + public static readonly TftpErrorPacket NoSuchUser = new TftpErrorPacket(7, "No such user"); + #endregion + } + + /// <summary> + /// Network errors (i.e. socket exceptions) are represented by this class. + /// </summary> + public class NetworkError : TftpTransferError + { + public Exception Exception { get; private set; } + + public NetworkError(Exception exception) + { + this.Exception = exception; + } + + public override string ToString() + { + return Exception.ToString(); + } + } + + /// <summary> + /// $(ITftpTransfer.RetryTimeout) has been exceeded more than $(ITftpTransfer.RetryCount) times in a row. + /// </summary> + public class TimeoutError : TftpTransferError + { + private readonly TimeSpan RetryTimeout; + private readonly int RetryCount; + + public TimeoutError(TimeSpan retryTimeout, int retryCount) + { + this.RetryTimeout = retryTimeout; + this.RetryCount = retryCount; + } + + public override string ToString() + { + return "Timeout error. RetryTimeout (" + RetryTimeout + ") violated more than " + RetryCount + " times in a row"; + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferMode.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferMode.cs new file mode 100644 index 00000000..f8d3a40e --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferMode.cs @@ -0,0 +1,9 @@ +namespace Tftp.Net +{ + public enum TftpTransferMode + { + netascii, + octet, + mail + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferProgress.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferProgress.cs new file mode 100644 index 00000000..0ca84945 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/TftpTransferProgress.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tftp.Net +{ + public class TftpTransferProgress + { + /// <summary> + /// Number of bytes that have already been transferred. + /// </summary> + public long TransferredBytes { get; private set; } + + /// <summary> + /// Total number of bytes being transferred. May be 0 if unknown. + /// </summary> + public long TotalBytes { get; private set; } + + public TftpTransferProgress(long transferred, long total) + { + TransferredBytes = transferred; + TotalBytes = total; + } + + public override string ToString() + { + if (TotalBytes > 0) + return (TransferredBytes * 100L) / TotalBytes + "% completed"; + else + return TransferredBytes + " bytes transferred"; + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Trace/LoggingStateDecorator.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Trace/LoggingStateDecorator.cs new file mode 100644 index 00000000..a40206b0 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Trace/LoggingStateDecorator.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Diagnostics; +using Tftp.Net.Transfer.States; +using System.IO; +using Tftp.Net.Channel; +using System.Net; +using Tftp.Net.Trace; +using Tftp.Net.Transfer; + +namespace Tftp.Net.Trace +{ + class LoggingStateDecorator : ITransferState + { + public TftpTransfer Context + { + get { return decoratee.Context; } + set { decoratee.Context = value; } + } + + private readonly ITransferState decoratee; + private readonly TftpTransfer transfer; + + public LoggingStateDecorator(ITransferState decoratee, TftpTransfer transfer) + { + this.decoratee = decoratee; + this.transfer = transfer; + } + + public String GetStateName() + { + return "[" + decoratee.GetType().Name + "]"; + } + + public Task OnStateEnter() + { + TftpTrace.Trace(GetStateName() + " OnStateEnter", transfer); + return decoratee.OnStateEnter(); + } + + public Task OnStart() + { + TftpTrace.Trace(GetStateName() + " OnStart", transfer); + return decoratee.OnStart(); + } + + public Task OnCancel(TftpErrorPacket reason) + { + TftpTrace.Trace(GetStateName() + " OnCancel: " + reason, transfer); + return decoratee.OnCancel(reason); + } + + public Task OnCommand(ITftpCommand command, EndPoint endpoint) + { + TftpTrace.Trace(GetStateName() + " OnCommand: " + command + " from " + endpoint, transfer); + return decoratee.OnCommand(command, endpoint); + } + + public Task OnTimer() + { + return decoratee.OnTimer(); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Trace/TftpTrace.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Trace/TftpTrace.cs new file mode 100644 index 00000000..017e262d --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Trace/TftpTrace.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Transfer; + +namespace Tftp.Net.Trace +{ + /// <summary> + /// Class that controls all tracing in the TFTP module. + /// </summary> + public static class TftpTrace + { + /// <summary> + /// Set this property to <code>false</code> to disable tracing. + /// </summary> + public static bool Enabled { get; set; } + + static TftpTrace() + { + Enabled = false; + } + + internal static void Trace(String message, ITftpTransfer transfer) + { + if (!Enabled) + return; + + System.Diagnostics.Trace.WriteLine(message, transfer.ToString()); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/LocalReadTransfer.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/LocalReadTransfer.cs new file mode 100644 index 00000000..092d359c --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/LocalReadTransfer.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Transfer.States; +using Tftp.Net.Channel; +using Tftp.Net.Transfer; + +namespace Tftp.Net.Transfer +{ + class LocalReadTransfer : TftpTransfer + { + public static async Task<LocalReadTransfer> CreateLocalReadTransferAsync(ITransferChannel connection, String filename, IEnumerable<TransferOption> options) + { + var transfer = new LocalReadTransfer(connection, filename); + await transfer.InitAsync(new StartIncomingRead(options)); + return transfer; + } + + private LocalReadTransfer(ITransferChannel connection, string filename) + : base(connection, filename) + { + } + + public override TftpTransferMode TransferMode + { + get { return base.TransferMode; } + set { throw new NotSupportedException("Cannot change the transfer mode for incoming transfers. The transfer mode is determined by the client."); } + } + + public override int BlockSize + { + get { return base.BlockSize; } + set { throw new NotSupportedException("For incoming transfers, the blocksize is determined by the client."); } + } + + public override TimeSpan RetryTimeout + { + get { return base.RetryTimeout; } + set { throw new NotSupportedException("For incoming transfers, the retry timeout is determined by the client."); } + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/LocalWriteTransfer.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/LocalWriteTransfer.cs new file mode 100644 index 00000000..80daa6d8 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/LocalWriteTransfer.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Transfer.States; +using Tftp.Net.Channel; +using Tftp.Net.Transfer; + +namespace Tftp.Net.Transfer +{ + class LocalWriteTransfer : TftpTransfer + { + public static async Task<LocalWriteTransfer> CreateLocalWriteTransferAsync(ITransferChannel connection, string filename, IEnumerable<TransferOption> options) + { + var transfer = new LocalWriteTransfer(connection, filename); + await transfer.InitAsync(new StartIncomingWrite(options)); + return transfer; + } + + private LocalWriteTransfer(ITransferChannel connection, string filename) + : base(connection, filename) + { + } + + public override TftpTransferMode TransferMode + { + get { return base.TransferMode; } + set { throw new NotSupportedException("Cannot change the transfer mode for incoming transfers. The transfer mode is determined by the client."); } + } + + public override int BlockSize + { + get { return base.BlockSize; } + set { throw new NotSupportedException("For incoming transfers, the blocksize is determined by the client."); } + } + + public override TimeSpan RetryTimeout + { + get { return base.RetryTimeout; } + set { throw new NotSupportedException("For incoming transfers, the retry timeout is determined by the client."); } + } + + public override long ExpectedSize + { + get { return base.ExpectedSize; } + set { throw new NotSupportedException("You cannot set the expected size of a file that is remotely transferred to this system."); } + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/RemoteReadTransfer.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/RemoteReadTransfer.cs new file mode 100644 index 00000000..9df84511 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/RemoteReadTransfer.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Transfer.States; +using Tftp.Net.Channel; +using Tftp.Net.Transfer; + +namespace Tftp.Net.Transfer +{ + class RemoteReadTransfer : TftpTransfer + { + public static async Task<RemoteReadTransfer> CreateRemoteReadTransferAsync(ITransferChannel connection, string filename) + { + var transfer = new RemoteReadTransfer(connection, filename); + await transfer.InitAsync(new StartOutgoingRead()); + return transfer; + } + + private RemoteReadTransfer(ITransferChannel connection, string filename) + : base(connection, filename) + { + } + + public override long ExpectedSize + { + get { return base.ExpectedSize; } + set { throw new NotSupportedException("You cannot set the expected size of a file that is remotely transferred to this system."); } + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/RemoteWriteTransfer.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/RemoteWriteTransfer.cs new file mode 100644 index 00000000..0288c850 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/RemoteWriteTransfer.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Channel; +using Tftp.Net.Transfer.States; +using Tftp.Net.Transfer; + +namespace Tftp.Net.Transfer +{ + class RemoteWriteTransfer : TftpTransfer + { + public static async Task<RemoteWriteTransfer> CreateRemoteWriteTransferAsync(ITransferChannel connection, string filename) + { + var transfer = new RemoteWriteTransfer(connection, filename); + await transfer.InitAsync(new StartOutgoingWrite()); + return transfer; + } + + private RemoteWriteTransfer(ITransferChannel connection, string filename) + : base(connection, filename) + { + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/SimpleTimer.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/SimpleTimer.cs new file mode 100644 index 00000000..96b3a761 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/SimpleTimer.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tftp.Net.Transfer +{ + /// <summary> + /// Simple implementation of a timer. + /// </summary> + class SimpleTimer + { + private DateTime nextTimeout; + private readonly TimeSpan timeout; + + public SimpleTimer(TimeSpan timeout) + { + this.timeout = timeout; + Restart(); + } + + public void Restart() + { + this.nextTimeout = DateTime.Now.Add(timeout); + } + + public bool IsTimeout() + { + return DateTime.Now >= nextTimeout; + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/AcknowledgeWriteRequest.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/AcknowledgeWriteRequest.cs new file mode 100644 index 00000000..48488869 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/AcknowledgeWriteRequest.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Trace; + +namespace Tftp.Net.Transfer.States +{ + class AcknowledgeWriteRequest : StateThatExpectsMessagesFromDefaultEndPoint + { + public override async Task OnStateEnter() + { + await base.OnStateEnter(); + SendAndRepeat(new Acknowledgement(0)); + } + + public override async Task OnData(Data command) + { + var nextState = new Receiving(); + await Context.SetState(nextState); + await nextState.OnCommand(command, Context.GetConnection().RemoteEndpoint); + } + + public override async Task OnCancel(TftpErrorPacket reason) + { + await Context.SetState(new CancelledByUser(reason)); + } + + public override async Task OnError(Error command) + { + await Context.SetState(new ReceivedError(command)); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/BaseState.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/BaseState.cs new file mode 100644 index 00000000..41593045 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/BaseState.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using System.Net; + +namespace Tftp.Net.Transfer.States +{ + class BaseState : ITransferState + { + public TftpTransfer Context { get; set; } + + public virtual Task OnStateEnter() + { + //no-op + return Task.CompletedTask; + } + + public virtual Task OnStart() + { + return Task.CompletedTask; + } + + public virtual Task OnCancel(TftpErrorPacket reason) + { + return Task.CompletedTask; + } + + public virtual Task OnCommand(ITftpCommand command, EndPoint endpoint) + { + return Task.CompletedTask; + } + + public virtual Task OnTimer() + { + //Ignore timer events + return Task.CompletedTask; + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/CancelledByUser.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/CancelledByUser.cs new file mode 100644 index 00000000..554f4dcf --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/CancelledByUser.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tftp.Net.Transfer.States +{ + class CancelledByUser : BaseState + { + private readonly TftpErrorPacket reason; + + public CancelledByUser(TftpErrorPacket reason) + { + this.reason = reason; + } + + public override async Task OnStateEnter() + { + Error command = new Error(reason.ErrorCode, reason.ErrorMessage); + Context.GetConnection().Send(command); + await Context.SetState(new Closed()); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Closed.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Closed.cs new file mode 100644 index 00000000..666eef01 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Closed.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tftp.Net.Transfer.States +{ + class Closed : BaseState + { + public override async Task OnStateEnter() + { + await Context.DisposeAsync(); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/ITransferState.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/ITransferState.cs new file mode 100644 index 00000000..f83fe73f --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/ITransferState.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Tftp.Net.Channel; +using System.Net; + +namespace Tftp.Net.Transfer.States +{ + interface ITransferState + { + TftpTransfer Context { get; set; } + + //Called by TftpTransfer + Task OnStateEnter(); + + //Called if the user calls TftpTransfer.Start() or TftpTransfer.Cancel() + Task OnStart(); + Task OnCancel(TftpErrorPacket reason); + + //Called regularely by the context + Task OnTimer(); + + //Called when a command is received + Task OnCommand(ITftpCommand command, EndPoint endpoint); + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/ReceivedError.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/ReceivedError.cs new file mode 100644 index 00000000..d51570a1 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/ReceivedError.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Trace; + +namespace Tftp.Net.Transfer.States +{ + class ReceivedError : BaseState + { + private readonly TftpTransferError error; + + public ReceivedError(Error error) + : this(new TftpErrorPacket(error.ErrorCode, GetNonEmptyErrorMessage(error))) { } + + private static string GetNonEmptyErrorMessage(Error error) + { + return string.IsNullOrEmpty(error.Message) ? "(No error message provided)" : error.Message; + } + + public ReceivedError(TftpTransferError error) + { + this.error = error; + } + + public override async Task OnStateEnter() + { + TftpTrace.Trace("Received error: " + error, Context); + await Context.RaiseOnError(error, CancellationToken.None); + await Context.SetState(new Closed()); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Receiving.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Receiving.cs new file mode 100644 index 00000000..8551cfe9 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Receiving.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace Tftp.Net.Transfer.States +{ + class Receiving : StateThatExpectsMessagesFromDefaultEndPoint + { + private ushort lastBlockNumber = 0; + private ushort nextBlockNumber = 1; + private long bytesReceived = 0; + + public override async Task OnData(Data command) + { + if (command.BlockNumber == nextBlockNumber) + { + //We received a new block of data + Context.InputOutputStream.Write(command.Bytes, 0, command.Bytes.Length); + SendAcknowledgement(command.BlockNumber); + + //Was that the last block of data? + if (command.Bytes.Length < Context.BlockSize) + { + await Context.RaiseOnFinished(CancellationToken.None); + await Context.SetState(new Closed()); + } + else + { + lastBlockNumber = command.BlockNumber; + nextBlockNumber = Context.BlockCounterWrapping.CalculateNextBlockNumber(command.BlockNumber); + bytesReceived += command.Bytes.Length; + await Context.RaiseOnProgress(bytesReceived, CancellationToken.None); + } + } + else + if (command.BlockNumber == lastBlockNumber) + { + //We received the previous block again. Re-sent the acknowledgement + SendAcknowledgement(command.BlockNumber); + } + } + + public override Task OnCancel(TftpErrorPacket reason) + { + return Context.SetState(new CancelledByUser(reason)); + } + + public override Task OnError(Error command) + { + return Context.SetState(new ReceivedError(command)); + } + + private void SendAcknowledgement(ushort blockNumber) + { + Acknowledgement ack = new Acknowledgement(blockNumber); + Context.GetConnection().Send(ack); + ResetTimeout(); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementBase.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementBase.cs new file mode 100644 index 00000000..2058fce0 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementBase.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Transfer.States; +using Tftp.Net.Trace; + +namespace Tftp.Net.Transfer +{ + class SendOptionAcknowledgementBase : StateThatExpectsMessagesFromDefaultEndPoint + { + public override async Task OnStateEnter() + { + await base.OnStateEnter(); + SendAndRepeat(new OptionAcknowledgement(Context.NegotiatedOptions.ToOptionList())); + } + + public override Task OnError(Error command) + { + return Context.SetState(new ReceivedError(command)); + } + + public override Task OnCancel(TftpErrorPacket reason) + { + return Context.SetState(new CancelledByUser(reason)); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementForReadRequest.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementForReadRequest.cs new file mode 100644 index 00000000..a40dbf31 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementForReadRequest.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Transfer.States; + +namespace Tftp.Net.Transfer.States +{ + class SendOptionAcknowledgementForReadRequest : SendOptionAcknowledgementBase + { + public override Task OnAcknowledgement(Acknowledgement command) + { + if (command.BlockNumber == 0) + { + //We received an OACK, so let's get going ;) + return Context.SetState(new Sending()); + } + else + { + return Task.CompletedTask; + } + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementForWriteRequest.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementForWriteRequest.cs new file mode 100644 index 00000000..3367bab7 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendOptionAcknowledgementForWriteRequest.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Tftp.Net.Transfer.States +{ + class SendOptionAcknowledgementForWriteRequest : SendOptionAcknowledgementBase + { + public override async Task OnData(Data command) + { + if (command.BlockNumber == 1) + { + //The client confirmed the options, so let's start receiving + ITransferState nextState = new Receiving(); + await Context.SetState(nextState); + await nextState.OnCommand(command, Context.GetConnection().RemoteEndpoint); + } + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendReadRequest.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendReadRequest.cs new file mode 100644 index 00000000..117c700a --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendReadRequest.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Tftp.Net.Channel; +using System.Net; +using Tftp.Net.Transfer; +using Tftp.Net.Trace; + +namespace Tftp.Net.Transfer.States +{ + class SendReadRequest : StateWithNetworkTimeout + { + public override async Task OnStateEnter() + { + await base.OnStateEnter(); + SendRequest(); //Send a read request to the server + } + + private void SendRequest() + { + ReadRequest request = new ReadRequest(Context.Filename, Context.TransferMode, Context.ProposedOptions.ToOptionList()); + SendAndRepeat(request); + } + + public override async Task OnCommand(ITftpCommand command, EndPoint endpoint) + { + if (command is Data || command is OptionAcknowledgement) + { + //The server acknowledged our read request. + //Fix out remote endpoint + Context.GetConnection().RemoteEndpoint = endpoint; + } + + if (command is Data) + { + if (Context.NegotiatedOptions == null) + Context.FinishOptionNegotiation(TransferOptionSet.NewEmptySet()); + + //Switch to the receiving state... + ITransferState nextState = new Receiving(); + await Context.SetState(nextState); + + //...and let it handle the data packet + await nextState.OnCommand(command, endpoint); + } + else if (command is OptionAcknowledgement) + { + //Check which options were acknowledged + Context.FinishOptionNegotiation(new TransferOptionSet((command as OptionAcknowledgement).Options)); + + //the server acknowledged our options. Confirm the final options + SendAndRepeat(new Acknowledgement(0)); + } + else if (command is Error) + { + await Context.SetState(new ReceivedError((Error)command)); + } + else + await base.OnCommand(command, endpoint); + } + + public override Task OnCancel(TftpErrorPacket reason) + { + return Context.SetState(new CancelledByUser(reason)); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendWriteRequest.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendWriteRequest.cs new file mode 100644 index 00000000..3489e6c4 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/SendWriteRequest.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Tftp.Net.Transfer; +using Tftp.Net.Trace; + +namespace Tftp.Net.Transfer.States +{ + class SendWriteRequest : StateWithNetworkTimeout + { + public override async Task OnStateEnter() + { + await base.OnStateEnter(); + SendRequest(); + } + + private void SendRequest() + { + WriteRequest request = new WriteRequest(Context.Filename, Context.TransferMode, Context.ProposedOptions.ToOptionList()); + SendAndRepeat(request); + } + + public override async Task OnCommand(ITftpCommand command, System.Net.EndPoint endpoint) + { + if (command is OptionAcknowledgement) + { + TransferOptionSet acknowledged = new TransferOptionSet((command as OptionAcknowledgement).Options); + Context.FinishOptionNegotiation(acknowledged); + await BeginSendingTo(endpoint); + } + else + if (command is Acknowledgement && (command as Acknowledgement).BlockNumber == 0) + { + Context.FinishOptionNegotiation(TransferOptionSet.NewEmptySet()); + await BeginSendingTo(endpoint); + } + else + if (command is Error) + { + //The server denied our request + Error error = (Error)command; + await Context.SetState(new ReceivedError(error)); + } + else + await base.OnCommand(command, endpoint); + } + + private Task BeginSendingTo(System.Net.EndPoint endpoint) + { + //Switch to the endpoint that we received from the server + Context.GetConnection().RemoteEndpoint = endpoint; + + //Start sending packets + return Context.SetState(new Sending()); + } + + public override Task OnCancel(TftpErrorPacket reason) + { + return Context.SetState(new CancelledByUser(reason)); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Sending.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Sending.cs new file mode 100644 index 00000000..a2d42e61 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/Sending.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Tftp.Net.Trace; + +namespace Tftp.Net.Transfer.States +{ + class Sending : StateThatExpectsMessagesFromDefaultEndPoint + { + private byte[] lastData; + private ushort lastBlockNumber; + private long bytesSent = 0; + private bool lastPacketWasSent = false; + + public override async Task OnStateEnter() + { + await base.OnStateEnter(); + lastData = new byte[Context.BlockSize]; + SendNextPacket(1); + } + + public override async Task OnAcknowledgement(Acknowledgement command) + { + //Drop acknowledgments for other packets than the previous one + if (command.BlockNumber != lastBlockNumber) + return; + + //Notify our observers about our progress + bytesSent += lastData.Length; + await Context.RaiseOnProgress(bytesSent, CancellationToken.None); + + if (lastPacketWasSent) + { + //We're done here + await Context.RaiseOnFinished(CancellationToken.None); + await Context.SetState(new Closed()); + } + else + { + SendNextPacket(Context.BlockCounterWrapping.CalculateNextBlockNumber(lastBlockNumber)); + } + } + + public override Task OnError(Error command) + { + return Context.SetState(new ReceivedError(command)); + } + + public override Task OnCancel(TftpErrorPacket reason) + { + return Context.SetState(new CancelledByUser(reason)); + } + + #region Helper Methods + private void SendNextPacket(ushort blockNumber) + { + if (Context.InputOutputStream == null) + return; + + int packetLength = Context.InputOutputStream.Read(lastData, 0, lastData.Length); + lastBlockNumber = blockNumber; + + if (packetLength != lastData.Length) + { + //This means we just sent the last packet + lastPacketWasSent = true; + Array.Resize(ref lastData, packetLength); + } + + ITftpCommand dataCommand = new Data(blockNumber, lastData); + SendAndRepeat(dataCommand); + } + + #endregion + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartIncomingRead.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartIncomingRead.cs new file mode 100644 index 00000000..614e5ed8 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartIncomingRead.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Tftp.Net.Transfer; + +namespace Tftp.Net.Transfer.States +{ + class StartIncomingRead : BaseState + { + private readonly IEnumerable<TransferOption> optionsRequestedByClient; + + public StartIncomingRead(IEnumerable<TransferOption> optionsRequestedByClient) + { + this.optionsRequestedByClient = optionsRequestedByClient; + } + + public override Task OnStateEnter() + { + Context.ProposedOptions = new TransferOptionSet(optionsRequestedByClient); + return Task.CompletedTask; + } + + public override async Task OnStart() + { + Context.FillOrDisableTransferSizeOption(); + Context.FinishOptionNegotiation(Context.ProposedOptions); + List<TransferOption> options = Context.NegotiatedOptions.ToOptionList(); + if (options.Count > 0) + { + await Context.SetState(new SendOptionAcknowledgementForReadRequest()); + } + else + { + //Otherwise just start sending + await Context.SetState(new Sending()); + } + } + + public override Task OnCancel(TftpErrorPacket reason) + { + return Context.SetState(new CancelledByUser(reason)); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartIncomingWrite.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartIncomingWrite.cs new file mode 100644 index 00000000..36e2a2b3 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartIncomingWrite.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Tftp.Net.Transfer; + +namespace Tftp.Net.Transfer.States +{ + class StartIncomingWrite : BaseState + { + private readonly IEnumerable<TransferOption> optionsRequestedByClient; + public StartIncomingWrite(IEnumerable<TransferOption> optionsRequestedByClient) + { + this.optionsRequestedByClient = optionsRequestedByClient; + } + + public override Task OnStateEnter() + { + Context.ProposedOptions = new TransferOptionSet(optionsRequestedByClient); + return Task.CompletedTask; + } + + public override Task OnStart() + { + //Do we have any acknowledged options? + Context.FinishOptionNegotiation(Context.ProposedOptions); + List<TransferOption> options = Context.NegotiatedOptions.ToOptionList(); + if (options.Count > 0) + { + return Context.SetState(new SendOptionAcknowledgementForWriteRequest()); + } + else + { + //Start receiving + return Context.SetState(new AcknowledgeWriteRequest()); + } + } + + public override Task OnCancel(TftpErrorPacket reason) + { + return Context.SetState(new CancelledByUser(reason)); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartOutgoingRead.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartOutgoingRead.cs new file mode 100644 index 00000000..8c716ccb --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartOutgoingRead.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace Tftp.Net.Transfer.States +{ + class StartOutgoingRead : BaseState + { + public override Task OnStart() + { + return Context.SetState(new SendReadRequest()); + } + + public override Task OnCancel(TftpErrorPacket reason) + { + return Context.SetState(new Closed()); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartOutgoingWrite.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartOutgoingWrite.cs new file mode 100644 index 00000000..b82d659c --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StartOutgoingWrite.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace Tftp.Net.Transfer.States +{ + class StartOutgoingWrite : BaseState + { + public override Task OnStart() + { + Context.FillOrDisableTransferSizeOption(); + return Context.SetState(new SendWriteRequest()); + } + + public override Task OnCancel(TftpErrorPacket reason) + { + return Context.SetState(new Closed()); + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StateThatExpectsMessagesFromDefaultEndPoint.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StateThatExpectsMessagesFromDefaultEndPoint.cs new file mode 100644 index 00000000..70f07db3 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StateThatExpectsMessagesFromDefaultEndPoint.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Channel; +using System.Net; + +namespace Tftp.Net.Transfer.States +{ + class StateThatExpectsMessagesFromDefaultEndPoint : StateWithNetworkTimeout, ITftpCommandVisitor + { + public override async Task OnCommand(ITftpCommand command, EndPoint endpoint) + { + if (!endpoint.Equals(Context.GetConnection().RemoteEndpoint)) + throw new Exception("Received message from illegal endpoint. Actual: " + endpoint + ". Expected: " + Context.GetConnection().RemoteEndpoint); + + await command.Visit(this); + } + + public virtual Task OnReadRequest(ReadRequest command) + { + return Task.CompletedTask; + } + + public virtual Task OnWriteRequest(WriteRequest command) + { + return Task.CompletedTask; + } + + public virtual Task OnData(Data command) + { + return Task.CompletedTask; + } + + public virtual Task OnAcknowledgement(Acknowledgement command) + { + return Task.CompletedTask; + } + + public virtual Task OnError(Error command) + { + return Task.CompletedTask; + } + + public virtual Task OnOptionAcknowledgement(OptionAcknowledgement command) + { + return Task.CompletedTask; + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StateWithNetworkTimeout.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StateWithNetworkTimeout.cs new file mode 100644 index 00000000..6ed5026d --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/States/StateWithNetworkTimeout.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Trace; + +namespace Tftp.Net.Transfer.States +{ + class StateWithNetworkTimeout : BaseState + { + private SimpleTimer timer; + private ITftpCommand lastCommand; + private int retriesUsed = 0; + + public override Task OnStateEnter() + { + timer = new SimpleTimer(Context.RetryTimeout); + return Task.CompletedTask; + } + + public override async Task OnTimer() + { + if (timer.IsTimeout()) + { + TftpTrace.Trace("Network timeout.", Context); + timer.Restart(); + + if (retriesUsed++ >= Context.RetryCount) + { + TftpTransferError error = new TimeoutError(Context.RetryTimeout, Context.RetryCount); + await Context.SetState(new ReceivedError(error)); + } + else + HandleTimeout(); + } + } + + private void HandleTimeout() + { + if (lastCommand != null) + { + Context.GetConnection().Send(lastCommand); + } + } + + protected void SendAndRepeat(ITftpCommand command) + { + Context.GetConnection().Send(command); + lastCommand = command; + ResetTimeout(); + } + + protected void ResetTimeout() + { + timer.Restart(); + retriesUsed = 0; + } + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/TftpTransfer.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/TftpTransfer.cs new file mode 100644 index 00000000..4b9ba6c5 --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/TftpTransfer.cs @@ -0,0 +1,248 @@ +using Redpoint.Concurrency; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Threading; +using Tftp.Net.Channel; +using Tftp.Net.Trace; +using Tftp.Net.Transfer; +using Tftp.Net.Transfer.States; + +namespace Tftp.Net.Transfer +{ + class TftpTransfer : ITftpTransfer + { + protected ITransferState state; + protected readonly ITransferChannel connection; + protected Timer timer; + + public TransferOptionSet ProposedOptions { get; set; } + public TransferOptionSet NegotiatedOptions { get; private set; } + public bool WasStarted { get; private set; } + public Stream InputOutputStream { get; protected set; } + + public static async Task<TftpTransfer> CreateTftpTransferAsync(ITransferChannel connection, String filename, ITransferState initialState) + { + var transfer = new TftpTransfer(connection, filename); + await transfer.InitAsync(initialState); + return transfer; + } + + protected async Task InitAsync(ITransferState initialState) + { + this.ProposedOptions = TransferOptionSet.NewDefaultSet(); + this.RetryCount = 5; + await this.SetState(initialState); + this.connection.OnCommandReceived.Add(connection_OnCommandReceived); + this.connection.OnError.Add(connection_OnError); + this.connection.Open(); + this.timer = new Timer(timer_OnTimer, null, 500, 500); + } + + protected TftpTransfer(ITransferChannel connection, String filename) + { + this.Filename = filename; + this.connection = connection; + } + + private void timer_OnTimer(object context) + { + try + { + lock (this) + { + state.OnTimer(); + } + } + catch (Exception e) + { + TftpTrace.Trace("Ignoring unhandled exception: " + e, this); + } + } + + private Task connection_OnCommandReceived(TftpCommandEventArgs args, CancellationToken cancellationToken) + { + lock (this) + { + state.OnCommand(args.Command, args.Endpoint); + return Task.CompletedTask; + } + } + + private async Task connection_OnError(TftpTransferError error, CancellationToken cancellationToken) + { + await RaiseOnError(error, cancellationToken); + } + + internal virtual async Task SetState(ITransferState newState) + { + state = DecorateForLogging(newState); + state.Context = this; + await state.OnStateEnter(); + } + + protected virtual ITransferState DecorateForLogging(ITransferState state) + { + return TftpTrace.Enabled ? new LoggingStateDecorator(state, this) : state; + } + + internal ITransferChannel GetConnection() + { + return connection; + } + + internal async Task RaiseOnProgress(long bytesTransferred, CancellationToken cancellationToken) + { + await _onProgress.BroadcastAsync( + new TftpProgressHandlerArgs + { + Progress = new TftpTransferProgress(bytesTransferred, ExpectedSize), + Transfer = this, + }, + cancellationToken); + } + + internal async Task RaiseOnError(TftpTransferError error, CancellationToken cancellationToken) + { + await _onError.BroadcastAsync( + new TftpErrorHandlerArgs + { + Error = error, + Transfer = this, + }, + cancellationToken); + } + + internal async Task RaiseOnFinished(CancellationToken cancellationToken) + { + await _onFinished.BroadcastAsync( + this, + cancellationToken); + } + + internal void FinishOptionNegotiation(TransferOptionSet negotiated) + { + NegotiatedOptions = negotiated; + if (!NegotiatedOptions.IncludesBlockSizeOption) + NegotiatedOptions.BlockSize = TransferOptionSet.DEFAULT_BLOCKSIZE; + + if (!NegotiatedOptions.IncludesTimeoutOption) + NegotiatedOptions.Timeout = TransferOptionSet.DEFAULT_TIMEOUT_SECS; + } + + public override string ToString() + { + return GetHashCode() + " (" + Filename + ")"; + } + + internal void FillOrDisableTransferSizeOption() + { + try + { + ProposedOptions.TransferSize = InputOutputStream.Length; + } + catch (NotSupportedException) { } + finally + { + if (ProposedOptions.TransferSize <= 0) + ProposedOptions.IncludesTransferSizeOption = false; + } + } + + #region ITftpTransfer + + private readonly AsyncEvent<TftpProgressHandlerArgs> _onProgress = new(); + private readonly AsyncEvent<ITftpTransfer> _onFinished = new(); + private readonly AsyncEvent<TftpErrorHandlerArgs> _onError = new(); + + public IAsyncEvent<TftpProgressHandlerArgs> OnProgress => _onProgress; + public IAsyncEvent<ITftpTransfer> OnFinished => _onFinished; + public IAsyncEvent<TftpErrorHandlerArgs> OnError => _onError; + + public string Filename { get; private set; } + public int RetryCount { get; set; } + public virtual TftpTransferMode TransferMode { get; set; } + public object UserContext { get; set; } + public virtual TimeSpan RetryTimeout + { + get { return TimeSpan.FromSeconds(NegotiatedOptions != null ? NegotiatedOptions.Timeout : ProposedOptions.Timeout); } + set { ThrowExceptionIfTransferAlreadyStarted(); ProposedOptions.Timeout = value.Seconds; } + } + + public virtual long ExpectedSize + { + get { return NegotiatedOptions != null ? NegotiatedOptions.TransferSize : ProposedOptions.TransferSize; } + set { ThrowExceptionIfTransferAlreadyStarted(); ProposedOptions.TransferSize = value; } + } + + public virtual int BlockSize + { + get { return NegotiatedOptions != null ? NegotiatedOptions.BlockSize : ProposedOptions.BlockSize; } + set { ThrowExceptionIfTransferAlreadyStarted(); ProposedOptions.BlockSize = value; } + } + + private BlockCounterWrapAround wrapping = BlockCounterWrapAround.ToZero; + public virtual BlockCounterWrapAround BlockCounterWrapping + { + get { return wrapping; } + set { ThrowExceptionIfTransferAlreadyStarted(); wrapping = value; } + } + + private void ThrowExceptionIfTransferAlreadyStarted() + { + if (WasStarted) + throw new InvalidOperationException("You cannot change tftp transfer options after the transfer has been started."); + } + + public void Start(Stream data) + { + if (data == null) + throw new ArgumentNullException("data"); + + if (WasStarted) + throw new InvalidOperationException("This transfer has already been started."); + + this.WasStarted = true; + this.InputOutputStream = data; + + lock (this) + { + state.OnStart(); + } + } + + public void Cancel(TftpErrorPacket reason) + { + if (reason == null) + throw new ArgumentNullException("reason"); + + lock (this) + { + state.OnCancel(reason); + } + } + + public virtual async ValueTask DisposeAsync() + { + lock (this) + { + timer.Dispose(); + Cancel(new TftpErrorPacket(0, "ITftpTransfer has been disposed.")); + + if (InputOutputStream != null) + { + InputOutputStream.Close(); + InputOutputStream = null; + } + } + + await connection.DisposeAsync(); + } + + #endregion + } +} diff --git a/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/TransferOptionSet.cs b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/TransferOptionSet.cs new file mode 100644 index 00000000..721bb0ac --- /dev/null +++ b/UET/Lib/Redpoint.ThirdParty.Tftp.Net/Transfer/TransferOptionSet.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tftp.Net.Transfer; + +namespace Tftp.Net.Transfer +{ + class TransferOptionSet + { + public const int DEFAULT_BLOCKSIZE = 512; + public const int DEFAULT_TIMEOUT_SECS = 5; + + public bool IncludesBlockSizeOption = false; + public int BlockSize = DEFAULT_BLOCKSIZE; + + public bool IncludesTimeoutOption = false; + public int Timeout = DEFAULT_TIMEOUT_SECS; + + public bool IncludesTransferSizeOption = false; + public long TransferSize = 0; + + public static TransferOptionSet NewDefaultSet() + { + return new TransferOptionSet() { IncludesBlockSizeOption = true, IncludesTimeoutOption = true, IncludesTransferSizeOption = true }; + } + + public static TransferOptionSet NewEmptySet() + { + return new TransferOptionSet() { IncludesBlockSizeOption = false, IncludesTimeoutOption = false, IncludesTransferSizeOption = false }; + } + + private TransferOptionSet() + { + } + + public TransferOptionSet(IEnumerable<TransferOption> options) + { + IncludesBlockSizeOption = IncludesTimeoutOption = IncludesTransferSizeOption = false; + + foreach (TransferOption option in options) + { + Parse(option); + } + } + + private void Parse(TransferOption option) + { + switch (option.Name) + { + case "blksize": + IncludesBlockSizeOption = ParseBlockSizeOption(option.Value); + break; + + case "timeout": + IncludesTimeoutOption = ParseTimeoutOption(option.Value); + break; + + case "tsize": + IncludesTransferSizeOption = ParseTransferSizeOption(option.Value); + break; + } + } + + public List<TransferOption> ToOptionList() + { + List<TransferOption> result = new List<TransferOption>(); + if (IncludesBlockSizeOption) + result.Add(new TransferOption("blksize", BlockSize.ToString())); + + if (IncludesTimeoutOption) + result.Add(new TransferOption("timeout", Timeout.ToString())); + + if (IncludesTransferSizeOption) + result.Add(new TransferOption("tsize", TransferSize.ToString())); + + return result; + } + + private bool ParseTransferSizeOption(string value) + { + return long.TryParse(value, out TransferSize) && TransferSize >= 0; + } + + private bool ParseTimeoutOption(string value) + { + int timeout; + if (!int.TryParse(value, out timeout)) + return false; + + //Only accept timeouts in the range [1, 255] + if (timeout < 1 || timeout > 255) + return false; + + Timeout = timeout; + return true; + } + + private bool ParseBlockSizeOption(string value) + { + int blockSize; + if (!int.TryParse(value, out blockSize)) + return false; + + //Only accept block sizes in the range [8, 65464] + if (blockSize < 8 || blockSize > 65464) + return false; + + BlockSize = blockSize; + return true; + } + } +} diff --git a/UET/Redpoint.Concurrency/AsyncEvent.cs b/UET/Redpoint.Concurrency/AsyncEvent.cs index 0fcfaa9c..e38294dd 100644 --- a/UET/Redpoint.Concurrency/AsyncEvent.cs +++ b/UET/Redpoint.Concurrency/AsyncEvent.cs @@ -163,5 +163,17 @@ await Parallel.ForEachAsync( }).ConfigureAwait(false); } } + + /// <summary> + /// Returns true if there are any handlers bound to this event. + /// </summary> + public bool HasAnyBindings + { + get + { + using var _ = _handlersLock.Wait(CancellationToken.None); + return _handlers.Count > 0; + } + } } } diff --git a/UET/Redpoint.Hashing/Hash.cs b/UET/Redpoint.Hashing/Hash.cs index 91d4bfb3..8c724d4b 100644 --- a/UET/Redpoint.Hashing/Hash.cs +++ b/UET/Redpoint.Hashing/Hash.cs @@ -39,6 +39,16 @@ public static string Sha256AsHexString(ReadOnlySpan<byte> value) return HexString(SHA256.HashData(value)); } + /// <summary> + /// Computes the SHA256 hash for the stream and returns it as a lowercase hexadecimal string. + /// </summary> + /// <param name="stream">The stream to compute the hash for.</param> + /// <returns>The lowercase hexadecimal string.</returns> + public static string Sha256AsHexString(Stream stream) + { + return HexString(SHA256.HashData(stream)); + } + /// <summary> /// Computes the SHA256 hash for the string value using the specified encoding and returns it as a lowercase hexadecimal string. /// </summary> diff --git a/UET/Redpoint.Kestrel/DefaultKestrelFactory.cs b/UET/Redpoint.Kestrel/DefaultKestrelFactory.cs index 7f7c3a21..04ebeaff 100644 --- a/UET/Redpoint.Kestrel/DefaultKestrelFactory.cs +++ b/UET/Redpoint.Kestrel/DefaultKestrelFactory.cs @@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets; using Microsoft.AspNetCore.WebSockets; using Microsoft.Extensions.Hosting; + using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using System; @@ -17,11 +18,14 @@ internal class DefaultKestrelFactory : IKestrelFactory, Microsoft.AspNetCore.Hosting.IApplicationLifetime { private readonly IHostApplicationLifetime _hostApplicationLifetime; + private readonly ILoggerFactory? _loggerFactory; public DefaultKestrelFactory( - IHostApplicationLifetime hostApplicationLifetime) + IHostApplicationLifetime hostApplicationLifetime, + ILoggerFactory? loggerFactory = null) { _hostApplicationLifetime = hostApplicationLifetime; + _loggerFactory = loggerFactory; } public CancellationToken ApplicationStarted => _hostApplicationLifetime.ApplicationStarted; @@ -76,7 +80,7 @@ public async Task<KestrelServer> CreateAndStartServerAsync( CancellationToken cancellationToken) { var transportOptions = new SocketTransportOptions(); - var loggerFactory = new NullLoggerFactory(); + var loggerFactory = _loggerFactory ?? new NullLoggerFactory(); var transportFactory = new SocketTransportFactory( new OptionsWrapper<SocketTransportOptions>(transportOptions), diff --git a/UET/Redpoint.KubernetesManager.Configuration/Json/KubernetesDateTimeOffsetConverter.cs b/UET/Redpoint.KubernetesManager.Configuration/Json/KubernetesDateTimeOffsetConverter.cs new file mode 100644 index 00000000..a41aa87a --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Json/KubernetesDateTimeOffsetConverter.cs @@ -0,0 +1,73 @@ +namespace Redpoint.KubernetesManager.Configuration.Json +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Text.Json.Serialization; + using System.Text.RegularExpressions; + using System.Threading.Tasks; + + /// <summary> + /// From Kubernetes C# client library. This converter is not exposed by the library, but we need it so we can store dates in a compatible format. + /// </summary> + public sealed class KubernetesDateTimeOffsetConverter : JsonConverter<DateTimeOffset> + { + private const string _rfc3339MicroFormat = "yyyy'-'MM'-'dd'T'HH':'mm':'ss.ffffffZ"; + private const string _rfc3339NanoFormat = "yyyy-MM-dd'T'HH':'mm':'ss.fffffffZ"; + private const string _rfc3339Format = "yyyy'-'MM'-'dd'T'HH':'mm':'ssZ"; + + private const string _rfc3339MicroWithOffsetFormat = "yyyy'-'MM'-'dd'T'HH':'mm':'ss.ffffffK"; + private const string _rfc3339NanoWithOffsetFormat = "yyyy-MM-dd'T'HH':'mm':'ss.fffffffK"; + private const string _rfc3339WithOffsetFormat = "yyyy'-'MM'-'dd'T'HH':'mm':'ssK"; + + private static readonly string[] _standardFormats = { _rfc3339Format, _rfc3339MicroFormat, _rfc3339WithOffsetFormat, _rfc3339MicroWithOffsetFormat }; + private static readonly string[] _nanoFormats = { _rfc3339NanoFormat, _rfc3339NanoWithOffsetFormat }; + + public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var str = reader.GetString() ?? string.Empty; + + // Try standard formats first + if (DateTimeOffset.TryParseExact(str, _standardFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out var result)) + { + return result; + } + + // Try RFC3339NanoLenient by trimming 1-9 digits to 7 digits + var originalstr = str; + str = Regex.Replace(str, @"\.\d+", m => (m.Value + "000000000").Substring(0, 7 + 1)); // 7 digits + 1 for the dot + if (DateTimeOffset.TryParseExact(str, _nanoFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out result)) + { + return result; + } + + throw new FormatException($"Unable to parse {originalstr} as RFC3339 RFC3339Micro or RFC3339Nano"); + } + + public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options) + { + ArgumentNullException.ThrowIfNull(writer); + + // Output as RFC3339Micro + var date = value.ToUniversalTime(); + + // Check if there are any fractional seconds + var ticks = date.Ticks % TimeSpan.TicksPerSecond; + if (ticks == 0) + { + // No fractional seconds - use format without fractional part + var basePart = date.ToString("yyyy-MM-dd'T'HH:mm:ss", CultureInfo.InvariantCulture); + writer.WriteStringValue(basePart + "Z"); + } + else + { + // Has fractional seconds - always use exactly 6 decimal places + var formatted = date.ToString(_rfc3339MicroFormat, CultureInfo.InvariantCulture); + writer.WriteStringValue(formatted); + } + } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Json/RkmNodeProvisionerStepJsonConverter.cs b/UET/Redpoint.KubernetesManager.Configuration/Json/RkmNodeProvisionerStepJsonConverter.cs new file mode 100644 index 00000000..2074d105 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Json/RkmNodeProvisionerStepJsonConverter.cs @@ -0,0 +1,170 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step +{ + using Redpoint.KubernetesManager.Configuration.Types; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Text.Json.Serialization; + using System.Threading.Tasks; + + public class RkmNodeProvisionerStepJsonConverter : JsonConverter<RkmNodeProvisionerStep> + { + private readonly IProvisioningStep[] _provisioningSteps; + + public RkmNodeProvisionerStepJsonConverter( + IEnumerable<IProvisioningStep> provisioningSteps) + { + _provisioningSteps = provisioningSteps.ToArray(); + } + + public override RkmNodeProvisionerStep? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + IProvisioningStep? provisioningStep; + { + var readerClone = reader; + provisioningStep = ReadAndGetProvisioningStep( + _provisioningSteps, + ref readerClone, + options); + } + + var result = new RkmNodeProvisionerStep(); + ReadInto( + result, + provisioningStep!, + ref reader, + typeToConvert, + options); + return result; + } + + public override void Write(Utf8JsonWriter writer, RkmNodeProvisionerStep value, JsonSerializerOptions options) + { + ArgumentNullException.ThrowIfNull(writer); + ArgumentNullException.ThrowIfNull(value); + + writer.WriteStartObject(); + + writer.WriteString("type", value.Type); + + if (value.DynamicSettings != null) + { + writer.WritePropertyName(value.Type[0].ToString().ToLowerInvariant() + value.Type[1..]); + var provider = _provisioningSteps.First(x => x.Type == value.Type); + provider.GetJsonType(options).Serialize(writer, value.DynamicSettings); + } + + writer.WriteEndObject(); + } + + private static IProvisioningStep? ReadAndGetProvisioningStep( + IProvisioningStep[] provisioningSteps, + ref Utf8JsonReader readerClone, + JsonSerializerOptions options) + { + if (readerClone.TokenType != JsonTokenType.StartObject) + { + throw new JsonException($"Expected provisioning step entry to be a JSON object."); + } + while (readerClone.Read()) + { + if (readerClone.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (readerClone.TokenType == JsonTokenType.PropertyName) + { + var propertyName = readerClone.GetString(); + readerClone.Read(); + if (propertyName == "type") + { + var propertyValue = readerClone.GetString(); + if (provisioningSteps.Length > 0) + { + foreach (var provider in provisioningSteps) + { + if (string.Equals(provider.Type, propertyValue, StringComparison.OrdinalIgnoreCase)) + { + return provider; + } + } + throw new JsonException($"Provisioning step of type '{propertyValue}' is not recognised as a provisioning step provider. Supported provisioning step types: {string.Join(", ", provisioningSteps.Select(x => $"'{x.Type}'"))}"); + } + else + { + throw new JsonException($"Provisioning step of type '{propertyValue}' is not recognised as a provisioning step provider. There are no supported provisioning step types for this type of BuildConfig.json."); + } + } + else + { + readerClone.TrySkip(); + } + } + } + if (provisioningSteps.Length > 0) + { + throw new JsonException($"Provisioning step entry was missing the 'Type' property. It must be set to one of the supported provisioning step types: {string.Join(", ", provisioningSteps.Select(x => $"'{x.Type}'"))}"); + } + else + { + throw new JsonException($"Provisioning step entry was missing the 'Type' property. There are no supported provisioning step types for this type of BuildConfig.json."); + } + } + + internal static void ReadInto( + RkmNodeProvisionerStep result, + IProvisioningStep provider, + ref Utf8JsonReader reader, + Type typeToConvert, + JsonSerializerOptions options) + { + var gotType = false; + + var providerType = provider.Type; + + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException($"Expected provisioning step entry to be a JSON object."); + } + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); + switch (propertyName) + { + case "type": + var type = reader.GetString(); + if (type == null) + { + throw new JsonException($"Expected provisioning step entry to have a non-null type."); + } + result.Type = type; + gotType = true; + break; + default: + if (string.Equals(propertyName, providerType, StringComparison.OrdinalIgnoreCase)) + { + result.DynamicSettings = provider.GetJsonType(options).Deserialize(ref reader); + } + break; + } + } + } + + if (!gotType) + { + throw new JsonException($"Expected property 'Type' to be found on provisioning step entry."); + } + } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Redpoint.KubernetesManager.Configuration.csproj b/UET/Redpoint.KubernetesManager.Configuration/Redpoint.KubernetesManager.Configuration.csproj new file mode 100644 index 00000000..ce38a639 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Redpoint.KubernetesManager.Configuration.csproj @@ -0,0 +1,15 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <Import Project="$(MSBuildThisFileDirectory)../Lib/Common.Build.props" /> + + <ItemGroup> + <ProjectReference Include="..\Redpoint.RuntimeJson\Redpoint.RuntimeJson.csproj" /> + <ProjectReference Include="..\Redpoint.YamlToJson\Redpoint.YamlToJson.csproj" /> + </ItemGroup> + + <ItemGroup> + <PackageReference Include="KubernetesClient.Aot" /> + <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" /> + </ItemGroup> + +</Project> diff --git a/UET/Redpoint.KubernetesManager.Configuration/Sources/IRkmConfigurationSource.cs b/UET/Redpoint.KubernetesManager.Configuration/Sources/IRkmConfigurationSource.cs new file mode 100644 index 00000000..46568324 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Sources/IRkmConfigurationSource.cs @@ -0,0 +1,60 @@ +namespace Redpoint.KubernetesManager.Configuration.Sources +{ + using Redpoint.KubernetesManager.Configuration.Types; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.Json.Serialization; + using System.Text.Json.Serialization.Metadata; + using System.Threading.Tasks; + + public interface IRkmConfigurationSource + { + Task<RkmNode> CreateOrUpdateRkmNodeByAttestationIdentityKeyPemAsync( + string attestationIdentityKeyPem, + RkmNodeRole[] roles, + bool immutable, + IList<RkmNodePlatform> capablePlatforms, + string architecture, + CancellationToken cancellationToken); + + Task<RkmNode?> GetRkmNodeByAttestationIdentityKeyFingerprintAsync( + string attestationIdentityKeyFingerprint, + CancellationToken cancellationToken); + + Task<RkmNode?> GetRkmNodeByAttestationIdentityKeyPemAsync( + string attestationIdentityKeyPem, + CancellationToken cancellationToken); + + Task<RkmNode?> GetRkmNodeByRegisteredIpAddressAsync( + string registeredIpAddress, + CancellationToken cancellationToken); + + Task<RkmNodeGroup?> GetRkmNodeGroupAsync( + string name, + CancellationToken cancellationToken); + + Task UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + string attestationIdentityKeyFingerprint, + RkmNodeStatus status, + CancellationToken cancellationToken); + + Task UpdateRkmNodeForceReprovisionByAttestationIdentityKeyFingerprintAsync( + string attestationIdentityKeyFingerprint, + bool forceReprovision, + CancellationToken cancellationToken); + + Task<RkmNodeProvisioner?> GetRkmNodeProvisionerAsync( + string name, + JsonTypeInfo<RkmNodeProvisioner> jsonTypeInfoWithSerializerForSteps, + CancellationToken cancellationToken); + + Task<RkmConfiguration> GetRkmConfigurationAsync( + CancellationToken cancellationToken); + + Task ReplaceRkmConfigurationAsync( + RkmConfiguration configuration, + CancellationToken cancellationToken); + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Sources/KubernetesRkmConfigurationSource.cs b/UET/Redpoint.KubernetesManager.Configuration/Sources/KubernetesRkmConfigurationSource.cs new file mode 100644 index 00000000..5cc63ac5 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Sources/KubernetesRkmConfigurationSource.cs @@ -0,0 +1,366 @@ +namespace Redpoint.KubernetesManager.Configuration.Sources +{ + using k8s; + using k8s.Autorest; + using k8s.Models; + using Redpoint.KubernetesManager.Configuration.Types; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Net; + using System.Net.NetworkInformation; + using System.Text; + using System.Text.Json; + using System.Text.Json.Serialization.Metadata; + using System.Threading; + using System.Threading.Tasks; + using YamlDotNet.Core.Tokens; + + public class KubernetesRkmConfigurationSource : IRkmConfigurationSource + { + private readonly IKubernetes _kubernetes; + + public KubernetesRkmConfigurationSource( + IKubernetes kubernetes) + { + _kubernetes = kubernetes; + } + + public async Task<RkmNode> CreateOrUpdateRkmNodeByAttestationIdentityKeyPemAsync( + string attestationIdentityKeyPem, + RkmNodeRole[] roles, + bool immutable, + IList<RkmNodePlatform> capablePlatforms, + string architecture, + CancellationToken cancellationToken) + { + var fingerprint = RkmNodeFingerprint.CreateFromPem(attestationIdentityKeyPem); + + object? nodeJson; + var node = await GetRkmNodeByAttestationIdentityKeyPemAsync( + attestationIdentityKeyPem, + cancellationToken); + if (node == null) + { + node = new RkmNode + { + ApiVersion = "rkm.redpoint.games/v1", + Kind = "RkmNode", + Metadata = new V1ObjectMeta + { + Name = fingerprint.Substring(0, 8), + }, + Spec = new RkmNodeSpec + { + NodeName = string.Empty, + NodeGroup = string.Empty, + Authorized = false, + ForceReprovision = false, + }, + Status = new RkmNodeStatus + { + Roles = null, + Immutable = immutable, + AttestationIdentityKeyFingerprint = fingerprint, + AttestationIdentityKeyPem = attestationIdentityKeyPem, + FirstSeen = DateTimeOffset.UtcNow, + MostRecentJoinRequest = null, + CapablePlatforms = null, + Architecture = null, + LastSuccessfulProvision = null, + Provisioner = null, + RegisteredIpAddresses = new(), + BootToDisk = false, + }, + }; + nodeJson = await _kubernetes.CustomObjects.CreateClusterCustomObjectAsync<JsonElement>( + JsonSerializer.SerializeToElement( + node, + KubernetesRkmJsonSerializerContext.WithStringEnum.RkmNode), + "rkm.redpoint.games", + "v1", + "rkmnodes", + cancellationToken: cancellationToken); + } + else + { + nodeJson = await _kubernetes.CustomObjects.PatchClusterCustomObjectAsync<JsonElement>( + JsonSerializer.SerializeToElement( + new PatchRkmNodePartialUpdate + { + Status = new PatchRkmNodeStatusPartialUpdate + { + Roles = roles, + Immutable = immutable, + AttestationIdentityKeyFingerprint = fingerprint, + AttestationIdentityKeyPem = attestationIdentityKeyPem, + FirstSeen = node?.Status?.FirstSeen ?? DateTimeOffset.UtcNow, + MostRecentJoinRequest = DateTimeOffset.UtcNow, + CapablePlatforms = [.. capablePlatforms], + Architecture = architecture, + } + }, + KubernetesRkmJsonSerializerContext.WithStringEnum.PatchRkmNodePartialUpdate), + "rkm.redpoint.games", + "v1", + "rkmnodes", + node!.Metadata.Name, + cancellationToken: cancellationToken); + } + + if (nodeJson == null) + { + throw new InvalidOperationException("Unable to create RkmNode!"); + } + return JsonSerializer.Deserialize( + (JsonElement)nodeJson, + KubernetesRkmJsonSerializerContext.WithStringEnum.RkmNode)!; + } + + public async Task<RkmNode?> GetRkmNodeByAttestationIdentityKeyPemAsync( + string attestationIdentityKeyPem, + CancellationToken cancellationToken) + { + var fingerprint = RkmNodeFingerprint.CreateFromPem(attestationIdentityKeyPem); + + try + { + var nodeJson = await _kubernetes.CustomObjects.GetClusterCustomObjectAsync<JsonElement>( + "rkm.redpoint.games", + "v1", + "rkmnodes", + fingerprint.Substring(0, 8), + cancellationToken); + var node = JsonSerializer.Deserialize( + (JsonElement)nodeJson, + KubernetesRkmJsonSerializerContext.WithStringEnum.RkmNode); + if (node == null || + node?.Status?.AttestationIdentityKeyFingerprint != fingerprint || + node?.Status?.AttestationIdentityKeyPem != attestationIdentityKeyPem) + { + return null; + } + return node; + } + catch (HttpOperationException ex) when (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + + public async Task<RkmNode?> GetRkmNodeByAttestationIdentityKeyFingerprintAsync( + string attestationIdentityKeyFingerprint, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNullOrWhiteSpace(attestationIdentityKeyFingerprint); + + try + { + var nodeJson = await _kubernetes.CustomObjects.GetClusterCustomObjectAsync<JsonElement>( + "rkm.redpoint.games", + "v1", + "rkmnodes", + attestationIdentityKeyFingerprint.Substring(0, 8), + cancellationToken); + var node = JsonSerializer.Deserialize( + (JsonElement)nodeJson, + KubernetesRkmJsonSerializerContext.WithStringEnum.RkmNode); + if (node == null || + node?.Status?.AttestationIdentityKeyFingerprint != attestationIdentityKeyFingerprint) + { + return null; + } + return node; + } + catch (HttpOperationException ex) when (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + + public async Task<RkmNode?> GetRkmNodeByRegisteredIpAddressAsync(string registeredIpAddress, CancellationToken cancellationToken) + { + // We can't use a field selector here, since field selectors for CRDs can't + // access arrays. + var nodesJson = await _kubernetes.CustomObjects.ListClusterCustomObjectAsync<JsonElement>( + "rkm.redpoint.games", + "v1", + "rkmnodes", + cancellationToken: cancellationToken); + var nodes = JsonSerializer.Deserialize( + (JsonElement)nodesJson, + KubernetesRkmJsonSerializerContext.WithStringEnum.KubernetesListRkmNode); + if (nodes == null) + { + return null; + } + var eligible = new List<(RkmNode? node, DateTimeOffset expiresAt)>(); + foreach (var node in nodes) + { + var ipAddresses = node?.Status?.RegisteredIpAddresses ?? []; + foreach (var ipAddress in ipAddresses) + { + if (ipAddress?.Address == registeredIpAddress && + ipAddress?.ExpiresAt > DateTimeOffset.UtcNow) + { + eligible.Add((node, ipAddress.ExpiresAt ?? DateTimeOffset.MaxValue)); + return node; + } + } + } + return eligible + .OrderByDescending(x => x.expiresAt) + .FirstOrDefault().node; + } + + public async Task UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + string attestationIdentityKeyFingerprint, + RkmNodeStatus status, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNullOrWhiteSpace(attestationIdentityKeyFingerprint); + + await _kubernetes.CustomObjects.PatchClusterCustomObjectAsync<JsonElement>( + JsonSerializer.Serialize( + new PatchRkmNodeFullStatus + { + Status = status, + }, + KubernetesRkmJsonSerializerContext.WithStringEnum.PatchRkmNodeFullStatus), + "rkm.redpoint.games", + "v1", + "rkmnodes", + attestationIdentityKeyFingerprint.Substring(0, 8), + cancellationToken: cancellationToken); + } + + public async Task UpdateRkmNodeForceReprovisionByAttestationIdentityKeyFingerprintAsync( + string attestationIdentityKeyFingerprint, + bool forceReprovision, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNullOrWhiteSpace(attestationIdentityKeyFingerprint); + + await _kubernetes.CustomObjects.PatchClusterCustomObjectAsync<JsonElement>( + JsonSerializer.Serialize( + new PatchRkmNodeForceReprovision + { + Spec = new PatchRkmNodeSpecForceReprovision + { + ForceReprovision = forceReprovision, + } + }, + KubernetesRkmJsonSerializerContext.WithStringEnum.PatchRkmNodeForceReprovision), + "rkm.redpoint.games", + "v1", + "rkmnodes", + attestationIdentityKeyFingerprint.Substring(0, 8), + cancellationToken: cancellationToken); + } + + public async Task<RkmNodeProvisioner?> GetRkmNodeProvisionerAsync( + string name, + JsonTypeInfo<RkmNodeProvisioner> jsonTypeInfoWithSerializerForSteps, + CancellationToken cancellationToken) + { + try + { + var nodeProvisioner = await _kubernetes.CustomObjects.GetClusterCustomObjectAsync<JsonElement>( + "rkm.redpoint.games", + "v1", + "rkmnodeprovisioners", + name, + cancellationToken); + return JsonSerializer.Deserialize( + (JsonElement)nodeProvisioner, + jsonTypeInfoWithSerializerForSteps)!; + } + catch (HttpOperationException ex) when (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + + public async Task<RkmConfiguration> GetRkmConfigurationAsync(CancellationToken cancellationToken) + { + object? configuration = null; + try + { + configuration = await _kubernetes.CustomObjects.GetClusterCustomObjectAsync<JsonElement>( + "rkm.redpoint.games", + "v1", + "rkmconfigurations", + "default", + cancellationToken); + } + catch (HttpOperationException ex) when (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + } + if (configuration == null) + { + // Provide a default configuration in case one hasn't been deployed by + // RKM itself, or it's been deleted. + return new RkmConfiguration + { + ApiVersion = "rkm.redpoint.games/v1", + Kind = "RkmConfiguration", + Metadata = new V1ObjectMeta + { + Name = "default" + }, + Spec = new RkmConfigurationSpec + { + ComponentVersions = new RkmConfigurationComponentVersions + { + Rkm = "2025.1364.398", + Containerd = "2.2.1", + Runc = "1.3.4", + Kubernetes = "1.35.0", + Etcd = "3.6.7", + CniPlugins = "1.9.0", + Flannel = "0.27.4", + FlannelCniSuffix = "-flannel1", + } + } + }; + } + + return JsonSerializer.Deserialize( + (JsonElement)configuration, + KubernetesRkmJsonSerializerContext.WithStringEnum.RkmConfiguration)!; + } + + public async Task ReplaceRkmConfigurationAsync(RkmConfiguration configuration, CancellationToken cancellationToken) + { + await _kubernetes.CustomObjects.ReplaceClusterCustomObjectAsync<JsonElement>( + JsonSerializer.SerializeToElement( + configuration, + KubernetesRkmJsonSerializerContext.WithStringEnum.RkmConfiguration), + "rkm.redpoint.games", + "v1", + "rkmconfigurations", + "default", + cancellationToken: cancellationToken); + } + + public async Task<RkmNodeGroup?> GetRkmNodeGroupAsync(string name, CancellationToken cancellationToken) + { + try + { + var nodeProvisioner = await _kubernetes.CustomObjects.GetClusterCustomObjectAsync<JsonElement>( + "rkm.redpoint.games", + "v1", + "rkmnodegroups", + name, + cancellationToken); + return JsonSerializer.Deserialize( + (JsonElement)nodeProvisioner, + KubernetesRkmJsonSerializerContext.WithStringEnum.RkmNodeGroup)!; + } + catch (HttpOperationException ex) when (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + return null; + } + } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Sources/KubernetesRkmJsonSerializerContext.cs b/UET/Redpoint.KubernetesManager.Configuration/Sources/KubernetesRkmJsonSerializerContext.cs new file mode 100644 index 00000000..14ba40f2 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Sources/KubernetesRkmJsonSerializerContext.cs @@ -0,0 +1,40 @@ +namespace Redpoint.KubernetesManager.Configuration.Sources +{ + using k8s.Models; + using Redpoint.KubernetesManager.Configuration.Json; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using System.Text.Json; + using System.Text.Json.Serialization; + + [JsonSerializable(typeof(KubernetesList<RkmNode>))] + [JsonSerializable(typeof(KubernetesList<RkmNodeGroup>))] + [JsonSerializable(typeof(KubernetesList<RkmNodeProvisioner>))] + [JsonSerializable(typeof(KubernetesList<RkmConfiguration>))] + [JsonSerializable(typeof(PatchRkmNodeFullStatus))] + [JsonSerializable(typeof(PatchRkmNodePartialUpdate))] + [JsonSerializable(typeof(PatchRkmNodeForceReprovision))] + public partial class KubernetesRkmJsonSerializerContext : JsonSerializerContext + { + public static readonly KubernetesRkmJsonSerializerContext WithStringEnum = CreateStringEnumWithAdditionalConverters(); + + public static KubernetesRkmJsonSerializerContext CreateStringEnumWithAdditionalConverters(params JsonConverter[] additionalConverters) + { + ArgumentNullException.ThrowIfNull(additionalConverters); + + var options = new JsonSerializerOptions + { + Converters = + { + new JsonStringEnumConverter(), + new KubernetesDateTimeOffsetConverter(), + } + }; + foreach (var converter in additionalConverters) + { + options.Converters.Add(converter); + } + return new KubernetesRkmJsonSerializerContext(options); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Sources/TestRkmConfigurationSource.cs b/UET/Redpoint.KubernetesManager.Configuration/Sources/TestRkmConfigurationSource.cs new file mode 100644 index 00000000..40de9095 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Sources/TestRkmConfigurationSource.cs @@ -0,0 +1,200 @@ +namespace Redpoint.KubernetesManager.Configuration.Sources +{ + using k8s.Models; + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Json; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.YamlToJson; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Text.Json.Serialization; + using System.Text.Json.Serialization.Metadata; + using System.Threading; + using System.Threading.Tasks; + using System.Xml.Linq; + using YamlDotNet.Core; + using YamlDotNet.Core.Events; + using YamlDotNet.RepresentationModel; + using YamlDotNet.Serialization; + + public class TestRkmConfigurationSource : IRkmConfigurationSource + { + private readonly ILogger<TestRkmConfigurationSource> _logger; + + private Dictionary<string, RkmNode> _nodes = new(); + + public TestRkmConfigurationSource(ILogger<TestRkmConfigurationSource> logger) + { + _logger = logger; + } + + public Task<RkmNode> CreateOrUpdateRkmNodeByAttestationIdentityKeyPemAsync( + string attestationIdentityKeyPem, + RkmNodeRole[] roles, + bool immutable, + IList<RkmNodePlatform> capablePlatforms, + string architecture, + CancellationToken cancellationToken) + { + var fingerprint = RkmNodeFingerprint.CreateFromPem(attestationIdentityKeyPem); + + if (!_nodes.TryGetValue(fingerprint, out var value)) + { + value = new RkmNode + { + ApiVersion = "rkm.redpoint.games/v1", + Kind = "RkmNode", + Metadata = new V1ObjectMeta + { + Name = fingerprint.Substring(0, 8), + }, + Spec = new RkmNodeSpec + { + NodeGroup = string.Empty, + NodeName = string.Empty, + Authorized = false, + ForceReprovision = true, + }, + Status = new RkmNodeStatus + { + Roles = null, + Immutable = false, + AttestationIdentityKeyFingerprint = fingerprint, + AttestationIdentityKeyPem = attestationIdentityKeyPem, + FirstSeen = DateTimeOffset.UtcNow, + MostRecentJoinRequest = null, + CapablePlatforms = null, + Architecture = null, + }, + }; + _nodes.Add(fingerprint, value); + } + + value.Spec ??= new(); + value.Status ??= new(); + + value.Status.Roles = roles; + value.Status.Immutable = immutable; + value.Status.MostRecentJoinRequest = DateTimeOffset.UtcNow; + value.Status.CapablePlatforms = [.. capablePlatforms]; + value.Status.Architecture = architecture; + + // @note: The test configuration immediately authorizes nodes and sets them + // up for provisioning. + if (value.Spec.ForceReprovision) + { + value.Spec.ForceReprovision = false; + value.Spec.Authorized = true; + value.Spec.NodeName = "test-node"; + value.Status.Provisioner = new RkmNodeStatusProvisioner + { + Name = "default", + Hash = string.Empty, + CurrentStepIndex = 0, + }; + } + + return Task.FromResult(value); + } + + public Task<RkmNode?> GetRkmNodeByAttestationIdentityKeyPemAsync(string attestationIdentityKeyPem, CancellationToken cancellationToken) + { + var fingerprint = RkmNodeFingerprint.CreateFromPem(attestationIdentityKeyPem); + + if (_nodes.TryGetValue(fingerprint, out var value)) + { + return Task.FromResult<RkmNode?>(value); + } + + return Task.FromResult<RkmNode?>(null); + } + + public Task<RkmNode?> GetRkmNodeByAttestationIdentityKeyFingerprintAsync(string attestationIdentityKeyFingerprint, CancellationToken cancellationToken) + { + if (_nodes.TryGetValue(attestationIdentityKeyFingerprint, out var value)) + { + return Task.FromResult<RkmNode?>(value); + } + + return Task.FromResult<RkmNode?>(null); + } + + public Task<RkmNode?> GetRkmNodeByRegisteredIpAddressAsync(string registeredIpAddress, CancellationToken cancellationToken) + { + foreach (var node in _nodes.Values) + { + foreach (var ipAddress in (node.Status?.RegisteredIpAddresses ?? [])) + { + if (ipAddress.Address == registeredIpAddress && + ipAddress.ExpiresAt.HasValue && + ipAddress.ExpiresAt.Value > DateTimeOffset.UtcNow) + { + return Task.FromResult<RkmNode?>(node); + } + } + } + + return Task.FromResult<RkmNode?>(null); + } + + public Task UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync(string attestationIdentityKeyFingerprint, RkmNodeStatus status, CancellationToken cancellationToken) + { + if (_nodes.TryGetValue(attestationIdentityKeyFingerprint, out var value)) + { + value.Status = status; + } + + _logger.LogTrace(JsonSerializer.Serialize(status, KubernetesRkmJsonSerializerContext.WithStringEnum.RkmNodeStatus)); + + return Task.CompletedTask; + } + + public Task UpdateRkmNodeForceReprovisionByAttestationIdentityKeyFingerprintAsync(string attestationIdentityKeyFingerprint, bool forceReprovision, CancellationToken cancellationToken) + { + if (_nodes.TryGetValue(attestationIdentityKeyFingerprint, out var value)) + { + value.Spec ??= new(); + value.Spec.ForceReprovision = forceReprovision; + } + + return Task.CompletedTask; + } + + public async Task<RkmNodeProvisioner?> GetRkmNodeProvisionerAsync( + string name, + JsonTypeInfo<RkmNodeProvisioner> jsonTypeInfoWithSerializerForSteps, + CancellationToken cancellationToken) + { + using (var stream = new FileStream("provisioner.yaml", FileMode.Open, FileAccess.Read, FileShare.Read)) + { + using var targetStream = new MemoryStream(); + YamlToJsonConverter.Convert(stream, targetStream); + targetStream.Seek(0, SeekOrigin.Begin); + + return await JsonSerializer.DeserializeAsync( + targetStream, + jsonTypeInfoWithSerializerForSteps, + cancellationToken); + } + } + + public Task<RkmConfiguration> GetRkmConfigurationAsync(CancellationToken cancellationToken) + { + throw new NotSupportedException(); + } + + public Task ReplaceRkmConfigurationAsync(RkmConfiguration configuration, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public Task<RkmNodeGroup?> GetRkmNodeGroupAsync(string name, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStep.cs b/UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStep.cs new file mode 100644 index 00000000..60f665ab --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStep.cs @@ -0,0 +1,129 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step +{ + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.RuntimeJson; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading.Tasks; + + public interface IProvisioningStep + { + string Type { get; } + + IRuntimeJson GetJsonType(JsonSerializerOptions options); + + /// <summary> + /// Flags that change the behaviour of how the server handles this provisioning step. + /// </summary> + ProvisioningStepFlags Flags { get; } + + Task ExecuteOnServerUncastedBeforeAsync( + object? configUncasted, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken); + + Task ExecuteOnClientUncastedAsync( + object? configUncasted, + IProvisioningStepClientContext context, + CancellationToken cancellationToken); + + Task ExecuteOnServerUncastedAfterAsync( + object? configUncasted, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken); + + Task<string?> GetIpxeAutoexecScriptOverrideOnServerUncastedAsync( + object? configUncasted, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken); + } + + public interface IProvisioningStep<TConfig> : IProvisioningStep where TConfig : new() + { + Task ExecuteOnServerBeforeAsync( + TConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken); + + Task ExecuteOnClientAsync( + TConfig config, + IProvisioningStepClientContext context, + CancellationToken cancellationToken); + + Task ExecuteOnServerAfterAsync( + TConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken); + + Task<string?> GetIpxeAutoexecScriptOverrideOnServerAsync( + TConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + return Task.FromResult<string?>(null); + } + + [SuppressMessage("Design", "CA1033:Interface methods should be callable by child types", Justification = "This can't be sealed.")] + Task IProvisioningStep.ExecuteOnServerUncastedBeforeAsync( + object? configUncasted, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + return ExecuteOnServerBeforeAsync( + configUncasted == null ? new() : (TConfig)configUncasted, + nodeStatus, + serverContext, + cancellationToken); + } + + [SuppressMessage("Design", "CA1033:Interface methods should be callable by child types", Justification = "This can't be sealed.")] + Task IProvisioningStep.ExecuteOnClientUncastedAsync( + object? configUncasted, + IProvisioningStepClientContext context, + CancellationToken cancellationToken) + { + return ExecuteOnClientAsync( + configUncasted == null ? new() : (TConfig)configUncasted, + context, + cancellationToken); + } + + [SuppressMessage("Design", "CA1033:Interface methods should be callable by child types", Justification = "This can't be sealed.")] + Task IProvisioningStep.ExecuteOnServerUncastedAfterAsync( + object? configUncasted, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + return ExecuteOnServerAfterAsync( + configUncasted == null ? new() : (TConfig)configUncasted, + nodeStatus, + serverContext, + cancellationToken); + } + + [SuppressMessage("Design", "CA1033:Interface methods should be callable by child types", Justification = "This can't be sealed.")] + Task<string?> IProvisioningStep.GetIpxeAutoexecScriptOverrideOnServerUncastedAsync( + object? configUncasted, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + return GetIpxeAutoexecScriptOverrideOnServerAsync( + configUncasted == null ? new() : (TConfig)configUncasted, + nodeStatus, + serverContext, + cancellationToken); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStepClientContext.cs b/UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStepClientContext.cs new file mode 100644 index 00000000..8af4e808 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStepClientContext.cs @@ -0,0 +1,21 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step +{ + public interface IProvisioningStepClientContext + { + bool IsLocalTesting { get; } + + HttpClient ProvisioningApiClient { get; } + + string ProvisioningApiEndpointHttps { get; } + + string ProvisioningApiEndpointHttp { get; } + + string ProvisioningApiAddress { get; } + + string AuthorizedNodeName { get; } + + string AikFingerprint { get; } + + Dictionary<string, string> ParameterValues { get; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStepServerContext.cs b/UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStepServerContext.cs new file mode 100644 index 00000000..4fa2442e --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Step/IProvisioningStepServerContext.cs @@ -0,0 +1,9 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step +{ + using System.Net; + + public interface IProvisioningStepServerContext + { + IPAddress RemoteIpAddress { get; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Step/ProvisioningStepFlags.cs b/UET/Redpoint.KubernetesManager.Configuration/Step/ProvisioningStepFlags.cs new file mode 100644 index 00000000..327608a8 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Step/ProvisioningStepFlags.cs @@ -0,0 +1,20 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step +{ + using System; + using System.Diagnostics.CodeAnalysis; + + [Flags] + [SuppressMessage("Naming", "CA1711:Identifiers should not have incorrect suffix", Justification = "This is a flags enumeration.")] + public enum ProvisioningStepFlags + { + None = 0, + + DoNotStartAutomaticallyNextStepOnCompletion = 0x1, + + AssumeCompleteWhenIpxeScriptFetched = 0x2, + + CommitOnCompletion = 0x4, + + SetAsRebootStepIndex = 0x8, + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeForceReprovision.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeForceReprovision.cs new file mode 100644 index 00000000..e4076be7 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeForceReprovision.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class PatchRkmNodeForceReprovision + { + [JsonPropertyName("spec")] + public PatchRkmNodeSpecForceReprovision? Spec { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeFullStatus.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeFullStatus.cs new file mode 100644 index 00000000..db611f93 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeFullStatus.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class PatchRkmNodeFullStatus + { + [JsonPropertyName("status")] + public RkmNodeStatus? Status { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodePartialUpdate.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodePartialUpdate.cs new file mode 100644 index 00000000..83f1a5c0 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodePartialUpdate.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class PatchRkmNodePartialUpdate + { + [JsonPropertyName("status")] + public PatchRkmNodeStatusPartialUpdate? Status { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeSpecForceReprovision.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeSpecForceReprovision.cs new file mode 100644 index 00000000..5766a116 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeSpecForceReprovision.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class PatchRkmNodeSpecForceReprovision + { + [JsonPropertyName("forceReprovision")] + public bool ForceReprovision { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeStatusPartialUpdate.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeStatusPartialUpdate.cs new file mode 100644 index 00000000..5afd428b --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/PatchRkmNodeStatusPartialUpdate.cs @@ -0,0 +1,33 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Diagnostics.CodeAnalysis; + using System.Text.Json.Serialization; + + public class PatchRkmNodeStatusPartialUpdate + { + [JsonPropertyName("roles")] + public IList<RkmNodeRole>? Roles { get; set; } + + [JsonPropertyName("immutable")] + public bool Immutable { get; set; } + + [JsonPropertyName("attestationIdentityKeyFingerprint")] + public string? AttestationIdentityKeyFingerprint { get; set; } + + [JsonPropertyName("attestationIdentityKeyPem")] + public string? AttestationIdentityKeyPem { get; set; } + + [JsonPropertyName("firstSeen")] + public DateTimeOffset? FirstSeen { get; set; } + + [JsonPropertyName("mostRecentJoinRequest")] + public DateTimeOffset? MostRecentJoinRequest { get; set; } + + [JsonPropertyName("capablePlatforms")] + [SuppressMessage("Design", "CA1002:Do not expose generic lists", Justification = "Used for JSON serialization.")] + public List<RkmNodePlatform>? CapablePlatforms { get; set; } + + [JsonPropertyName("architecture")] + public string? Architecture { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfiguration.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfiguration.cs new file mode 100644 index 00000000..b03f4287 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfiguration.cs @@ -0,0 +1,21 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using k8s; + using k8s.Models; + using System.Text.Json.Serialization; + + public class RkmConfiguration : IKubernetesObject<V1ObjectMeta> + { + [JsonPropertyName("apiVersion")] + public required string ApiVersion { get; set; } + + [JsonPropertyName("kind")] + public required string Kind { get; set; } + + [JsonPropertyName("metadata")] + public required V1ObjectMeta Metadata { get; set; } + + [JsonPropertyName("spec")] + public RkmConfigurationSpec? Spec { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfigurationComponentVersions.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfigurationComponentVersions.cs new file mode 100644 index 00000000..3bdd3ac9 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfigurationComponentVersions.cs @@ -0,0 +1,31 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class RkmConfigurationComponentVersions + { + [JsonPropertyName("rkm")] + public string? Rkm { get; set; } + + [JsonPropertyName("containerd")] + public string? Containerd { get; set; } + + [JsonPropertyName("runc")] + public string? Runc { get; set; } + + [JsonPropertyName("kubernetes")] + public string? Kubernetes { get; set; } + + [JsonPropertyName("etcd")] + public string? Etcd { get; set; } + + [JsonPropertyName("cniPlugins")] + public string? CniPlugins { get; set; } + + [JsonPropertyName("flannel")] + public string? Flannel { get; set; } + + [JsonPropertyName("flannelCniSuffix")] + public string? FlannelCniSuffix { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfigurationSpec.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfigurationSpec.cs new file mode 100644 index 00000000..e7c61217 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmConfigurationSpec.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class RkmConfigurationSpec + { + [JsonPropertyName("componentVersions")] + public RkmConfigurationComponentVersions? ComponentVersions { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNode.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNode.cs new file mode 100644 index 00000000..7c0794b6 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNode.cs @@ -0,0 +1,27 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using k8s; + using k8s.Models; + using System.Linq; + using System.Text; + using System.Text.Json.Serialization; + using System.Threading.Tasks; + + public class RkmNode : IKubernetesObject<V1ObjectMeta> + { + [JsonPropertyName("apiVersion")] + public required string ApiVersion { get; set; } + + [JsonPropertyName("kind")] + public required string Kind { get; set; } + + [JsonPropertyName("metadata")] + public required V1ObjectMeta Metadata { get; set; } + + [JsonPropertyName("spec")] + public RkmNodeSpec? Spec { get; set; } + + [JsonPropertyName("status")] + public RkmNodeStatus? Status { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeFingerprint.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeFingerprint.cs new file mode 100644 index 00000000..c414134f --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeFingerprint.cs @@ -0,0 +1,27 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + using System.Reflection.Metadata; + using System.Security.Cryptography; + using System.Security.Cryptography.X509Certificates; + using System.Text; + using System.Threading.Tasks; + + public static class RkmNodeFingerprint + { + public static string CreateFromPem(string pem) + { + var publicKey = new RSACryptoServiceProvider(); + publicKey.ImportFromPem(pem); + + var parameters = publicKey.ExportParameters(false); + + return Convert.ToHexStringLower(SHA256.HashData([ + .. parameters.Exponent ?? [], + .. parameters.Modulus ?? []])); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeGroup.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeGroup.cs new file mode 100644 index 00000000..55b309a0 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeGroup.cs @@ -0,0 +1,25 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using k8s; + using k8s.Models; + using System; + using System.Linq; + using System.Text; + using System.Text.Json.Serialization; + using System.Threading.Tasks; + + public class RkmNodeGroup : IKubernetesObject<V1ObjectMeta> + { + [JsonPropertyName("apiVersion")] + public required string ApiVersion { get; set; } + + [JsonPropertyName("kind")] + public required string Kind { get; set; } + + [JsonPropertyName("metadata")] + public required V1ObjectMeta Metadata { get; set; } + + [JsonPropertyName("spec")] + public RkmNodeGroupSpec? Spec { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeGroupSpec.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeGroupSpec.cs new file mode 100644 index 00000000..1f7e7e99 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeGroupSpec.cs @@ -0,0 +1,16 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class RkmNodeGroupSpec + { + [JsonPropertyName("provisioner")] + public string? Provisioner { get; set; } + + [JsonPropertyName("provisionerArguments")] + public Dictionary<string, string?>? ProvisionerArguments { get; set; } + + [JsonPropertyName("clusterControllerIpAddress")] + public string? ClusterControllerIpAddress { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodePlatform.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodePlatform.cs new file mode 100644 index 00000000..a4d56c26 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodePlatform.cs @@ -0,0 +1,9 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + public enum RkmNodePlatform + { + Windows, + Linux, + Mac + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisioner.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisioner.cs new file mode 100644 index 00000000..6b858e9d --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisioner.cs @@ -0,0 +1,23 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using k8s; + using k8s.Models; + using System.Linq; + using System.Text.Json.Serialization; + using System.Threading.Tasks; + + public class RkmNodeProvisioner : IKubernetesObject<V1ObjectMeta> + { + [JsonPropertyName("apiVersion")] + public required string ApiVersion { get; set; } + + [JsonPropertyName("kind")] + public required string Kind { get; set; } + + [JsonPropertyName("metadata")] + public required V1ObjectMeta Metadata { get; set; } + + [JsonPropertyName("spec")] + public RkmNodeProvisionerSpec? Spec { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisionerSpec.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisionerSpec.cs new file mode 100644 index 00000000..416a90db --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisionerSpec.cs @@ -0,0 +1,14 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Collections.Generic; + using System.Text.Json.Serialization; + + public class RkmNodeProvisionerSpec + { + [JsonPropertyName("parameters")] + public Dictionary<string, string?>? Parameters { get; set; } + + [JsonPropertyName("steps")] + public IList<RkmNodeProvisionerStep?>? Steps { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisionerStep.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisionerStep.cs new file mode 100644 index 00000000..f903f17c --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeProvisionerStep.cs @@ -0,0 +1,18 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class RkmNodeProvisionerStep + { + /// <summary> + /// Specifies the provisioning step type. + /// </summary> + [JsonPropertyName("type"), JsonRequired] + public string Type { get; set; } = string.Empty; + + /// <summary> + /// The dynamic settings associated with this provisioning step. + /// </summary> + public object? DynamicSettings { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeRole.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeRole.cs new file mode 100644 index 00000000..0c859906 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeRole.cs @@ -0,0 +1,8 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + public enum RkmNodeRole + { + Controller, + Worker, + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeSpec.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeSpec.cs new file mode 100644 index 00000000..6a9ee39f --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeSpec.cs @@ -0,0 +1,24 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Diagnostics.CodeAnalysis; + using System.Text.Json.Serialization; + + public class RkmNodeSpec + { + [JsonPropertyName("nodeName")] + public string? NodeName { get; set; } + + [JsonPropertyName("nodeGroup")] + public string? NodeGroup { get; set; } + + [JsonPropertyName("authorized")] + public bool Authorized { get; set; } + + [JsonPropertyName("forceReprovision")] + public bool ForceReprovision { get; set; } + + [JsonPropertyName("inactiveBootEntries")] + [SuppressMessage("Design", "CA1819", Justification = "Used for JSON serialization.")] + public string?[]? InactiveBootEntries { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatus.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatus.cs new file mode 100644 index 00000000..1570d03c --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatus.cs @@ -0,0 +1,52 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.Text.Json.Serialization; + + public class RkmNodeStatus + { + [JsonPropertyName("roles")] + public IList<RkmNodeRole>? Roles { get; set; } + + [JsonPropertyName("immutable")] + public bool Immutable { get; set; } + + [JsonPropertyName("attestationIdentityKeyFingerprint")] + public string? AttestationIdentityKeyFingerprint { get; set; } + + [JsonPropertyName("attestationIdentityKeyPem")] + public string? AttestationIdentityKeyPem { get; set; } + + [JsonPropertyName("firstSeen")] + public DateTimeOffset? FirstSeen { get; set; } + + [JsonPropertyName("mostRecentJoinRequest")] + public DateTimeOffset? MostRecentJoinRequest { get; set; } + + [JsonPropertyName("capablePlatforms")] + [SuppressMessage("Design", "CA1002:Do not expose generic lists", Justification = "Used for JSON serialization.")] + public List<RkmNodePlatform>? CapablePlatforms { get; set; } + + [JsonPropertyName("architecture")] + public string? Architecture { get; set; } + + [JsonPropertyName("provisioner")] + public RkmNodeStatusProvisioner? Provisioner { get; set; } + + [JsonPropertyName("lastSuccessfulProvision")] + public RkmNodeStatusLastSuccessfulProvision? LastSuccessfulProvision { get; set; } + + [JsonPropertyName("registeredIpAddresses")] + [SuppressMessage("Design", "CA1002:Do not expose generic lists", Justification = "We need RemoveAll on this property.")] + public List<RkmNodeStatusRegisteredIpAddress>? RegisteredIpAddresses { get; set; } + + [JsonPropertyName("bootToDisk")] + public bool? BootToDisk { get; set; } + + [JsonPropertyName("bootEntries")] + [SuppressMessage("Design", "CA1002:Do not expose generic lists", Justification = "We need RemoveAll on this property.")] + public List<RkmNodeStatusBootEntry>? BootEntries { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusBootEntry.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusBootEntry.cs new file mode 100644 index 00000000..5f14ce87 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusBootEntry.cs @@ -0,0 +1,19 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class RkmNodeStatusBootEntry + { + [JsonPropertyName("bootId")] + public string BootId { get; set; } = string.Empty; + + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("path")] + public string Path { get; set; } = string.Empty; + + [JsonPropertyName("active")] + public bool Active { get; set; } = true; + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusLastSuccessfulProvision.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusLastSuccessfulProvision.cs new file mode 100644 index 00000000..91343dbd --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusLastSuccessfulProvision.cs @@ -0,0 +1,13 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class RkmNodeStatusLastSuccessfulProvision + { + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("hash")] + public string? Hash { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusProvisioner.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusProvisioner.cs new file mode 100644 index 00000000..78c717d2 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusProvisioner.cs @@ -0,0 +1,28 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System.Text.Json.Serialization; + + public class RkmNodeStatusProvisioner + { + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("hash")] + public string? Hash { get; set; } + + [JsonPropertyName("lastStepCommittedIndex")] + public int? LastStepCommittedIndex { get; set; } + + [JsonPropertyName("rebootStepIndex")] + public int? RebootStepIndex { get; set; } + + [JsonPropertyName("rebootNotificationForOnceViaNotifyOccurred")] + public bool? RebootNotificationForOnceViaNotifyOccurred { get; set; } + + [JsonPropertyName("currentStepIndex")] + public int? CurrentStepIndex { get; set; } + + [JsonPropertyName("currentStepStarted")] + public bool? CurrentStepStarted { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusRegisteredIpAddress.cs b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusRegisteredIpAddress.cs new file mode 100644 index 00000000..be69f2e1 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Configuration/Types/RkmNodeStatusRegisteredIpAddress.cs @@ -0,0 +1,14 @@ +namespace Redpoint.KubernetesManager.Configuration.Types +{ + using System; + using System.Text.Json.Serialization; + + public class RkmNodeStatusRegisteredIpAddress + { + [JsonPropertyName("address")] + public string? Address { get; set; } + + [JsonPropertyName("expiresAt")] + public DateTimeOffset? ExpiresAt { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.Manifest/ManifestJsonSerializerContext.cs b/UET/Redpoint.KubernetesManager.Manifest/ManifestJsonSerializerContext.cs index e91336e3..8e18bfc9 100644 --- a/UET/Redpoint.KubernetesManager.Manifest/ManifestJsonSerializerContext.cs +++ b/UET/Redpoint.KubernetesManager.Manifest/ManifestJsonSerializerContext.cs @@ -8,9 +8,6 @@ [JsonSerializable(typeof(PxeBootManifest))] [JsonSerializable(typeof(ActiveDirectoryManifest))] [JsonSerializable(typeof(NodeManifest))] - [JsonSerializable(typeof(NodeAuthorizeRequest))] - [JsonSerializable(typeof(NodeAuthorizeResponse))] - [JsonSerializable(typeof(NodeAuthorizeResponseEncryptedBundle))] public partial class ManifestJsonSerializerContext : JsonSerializerContext { } diff --git a/UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeRequest.cs b/UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeRequest.cs deleted file mode 100644 index c10b3a60..00000000 --- a/UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeRequest.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace Redpoint.KubernetesManager.Manifest -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Text; - using System.Text.Json.Serialization; - using System.Threading.Tasks; - - /// <summary> - /// Used by the node to send it's public EK and AIK from the TPM to the controller. - /// </summary> - public class NodeAuthorizeRequest - { - [JsonPropertyName("ekPublicTpmRepresentationBase64")] - public required string EkPublicTpmRepresentationBase64 { get; set; } - - [JsonPropertyName("aikPublicTpmRepresentationBase64")] - public required string AikPublicTpmRepresentationBase64 { get; set; } - - [JsonPropertyName("suggestedNodeName")] - public required string SuggestedNodeName { get; set; } - } -} diff --git a/UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeResponse.cs b/UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeResponse.cs deleted file mode 100644 index 7a170f23..00000000 --- a/UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeResponse.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace Redpoint.KubernetesManager.Manifest -{ - using System.Text.Json.Serialization; - - /// <summary> - /// Used by the controller to send an authorized node's private key to the node. - /// </summary> - public class NodeAuthorizeResponse - { - [JsonPropertyName("envelopingKeyBase64")] - public required string EnvelopingKeyBase64 { get; set; } - - [JsonPropertyName("encryptedBundleBase64")] - public required string EncryptedBundleBase64 { get; set; } - } -} diff --git a/UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeResponseEncryptedBundle.cs b/UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeResponseEncryptedBundle.cs deleted file mode 100644 index dc66d2c2..00000000 --- a/UET/Redpoint.KubernetesManager.Manifest/NodeAuthorizeResponseEncryptedBundle.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Redpoint.KubernetesManager.Manifest -{ - using System.Text.Json.Serialization; - - /// <summary> - /// The bundle which is encrypted in the <see cref="NodeAuthorizeResponse"/>. - /// </summary> - public class NodeAuthorizeResponseEncryptedBundle - { - [JsonPropertyName("nodePrivateKeyPem")] - public required string NodePrivateKeyPem { get; set; } - - [JsonPropertyName("nodeCertificatePem")] - public required string NodeCertificatePem { get; set; } - - [JsonPropertyName("certificateAuthorityPem")] - public required string CertificateAuthorityPem { get; set; } - - [JsonPropertyName("nodeName")] - public required string NodeName { get; set; } - } -} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Api/ApiJsonSerializerContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Api/ApiJsonSerializerContext.cs new file mode 100644 index 00000000..bc192f39 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Api/ApiJsonSerializerContext.cs @@ -0,0 +1,20 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Api +{ + using System.Text.Json; + using System.Text.Json.Serialization; + + [JsonSerializable(typeof(AuthorizeNodeRequest))] + [JsonSerializable(typeof(AuthorizeNodeResponse))] + [JsonSerializable(typeof(ForceReprovisionNodeRequest))] + [JsonSerializable(typeof(ForceReprovisionNodeResponse))] + internal partial class ApiJsonSerializerContext : JsonSerializerContext + { + public static ApiJsonSerializerContext WithStringEnum = new ApiJsonSerializerContext(new JsonSerializerOptions + { + Converters = + { + new JsonStringEnumConverter() + } + }); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Api/AuthorizeNodeRequest.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Api/AuthorizeNodeRequest.cs new file mode 100644 index 00000000..f97db66c --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Api/AuthorizeNodeRequest.cs @@ -0,0 +1,15 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Api +{ + using Redpoint.KubernetesManager.Configuration.Types; + using System.Collections.Generic; + using System.Text.Json.Serialization; + + internal class AuthorizeNodeRequest + { + [JsonPropertyName("capablePlatforms")] + public required IList<RkmNodePlatform> CapablePlatforms { get; set; } + + [JsonPropertyName("architecture")] + public required string Architecture { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Api/AuthorizeNodeResponse.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Api/AuthorizeNodeResponse.cs new file mode 100644 index 00000000..c34e6047 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Api/AuthorizeNodeResponse.cs @@ -0,0 +1,16 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Api +{ + using System.Text.Json.Serialization; + + internal class AuthorizeNodeResponse + { + [JsonPropertyName("nodeName")] + public required string NodeName { get; set; } + + [JsonPropertyName("aikFingerprint")] + public required string AikFingerprint { get; set; } + + [JsonPropertyName("parameterValues")] + public required Dictionary<string, string> ParameterValues { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Api/ForceReprovisionNodeRequest.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Api/ForceReprovisionNodeRequest.cs new file mode 100644 index 00000000..e3512dde --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Api/ForceReprovisionNodeRequest.cs @@ -0,0 +1,6 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Api +{ + internal class ForceReprovisionNodeRequest + { + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Api/ForceReprovisionNodeResponse.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Api/ForceReprovisionNodeResponse.cs new file mode 100644 index 00000000..84ef79cb --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Api/ForceReprovisionNodeResponse.cs @@ -0,0 +1,6 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Api +{ + internal class ForceReprovisionNodeResponse + { + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/DefaultEfiBootManager.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/DefaultEfiBootManager.cs new file mode 100644 index 00000000..6d4b0b63 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/DefaultEfiBootManager.cs @@ -0,0 +1,111 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Bootmgr +{ + using Redpoint.PathResolution; + using Redpoint.ProcessExecution; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + using System.Reflection.Emit; + using System.Text; + using System.Threading.Tasks; + using Tpm2Lib; + + internal class DefaultEfiBootManager : IEfiBootManager + { + private readonly IEfiBootManagerParser _parser; + private readonly IProcessExecutor _processExecutor; + private readonly IPathResolver _pathResolver; + + public DefaultEfiBootManager( + IEfiBootManagerParser parser, + IProcessExecutor processExecutor, + IPathResolver pathResolver) + { + _parser = parser; + _processExecutor = processExecutor; + _pathResolver = pathResolver; + } + + public async Task<EfiBootManagerConfiguration> GetBootManagerConfigurationAsync( + CancellationToken cancellationToken) + { + var output = new StringBuilder(); + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("efibootmgr"), + Arguments = [], + }, + CaptureSpecification.CreateFromStdoutStringBuilder(output), + cancellationToken); + return _parser.ParseBootManagerConfiguration(output.ToString()); + } + + public async Task RemoveBootManagerEntryAsync(int bootEntry, CancellationToken cancellationToken) + { + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("efibootmgr"), + Arguments = ["-b", bootEntry.ToString("X4", CultureInfo.InvariantCulture), "-B"], + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + + public async Task AddBootManagerDiskEntryAsync(string disk, int partition, string label, string path, CancellationToken cancellationToken) + { + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("efibootmgr"), + Arguments = [ + "-C", + "-d", + disk, + "-p", + partition.ToString(CultureInfo.InvariantCulture), + "-L", + label, + "-l", + path + ], + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + + public async Task SetBootManagerBootOrderAsync(IEnumerable<int> bootOrder, CancellationToken cancellationToken) + { + var bootOrderString = string.Join(",", bootOrder.Select(x => x.ToString("X4", CultureInfo.InvariantCulture))); + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("efibootmgr"), + Arguments = [ + "-o", + bootOrderString, + ], + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + + public async Task SetBootManagerEntryActiveAsync(int bootEntry, bool active, CancellationToken cancellationToken) + { + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("efibootmgr"), + Arguments = [ + "-b", + bootEntry.ToString("X4", CultureInfo.InvariantCulture), + active ? "-a" : "-A", + ], + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/DefaultEfiBootManagerParser.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/DefaultEfiBootManagerParser.cs new file mode 100644 index 00000000..ccabc83a --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/DefaultEfiBootManagerParser.cs @@ -0,0 +1,98 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Redpoint.KubernetesManager.Tests")] + +namespace Redpoint.KubernetesManager.PxeBoot.Bootmgr +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.Globalization; + using System.Linq; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading.Tasks; + + internal class DefaultEfiBootManagerParser : IEfiBootManagerParser + { + private readonly Regex _bootCurrentRegex = new Regex("^BootCurrent: (?<bootid>[0-9A-F]+)$"); + private readonly Regex _timeoutRegex = new Regex("^Timeout: (?<seconds>[0-9]+) seconds$"); + private readonly Regex _bootOrderRegex = new Regex("^BootOrder: (?<order>[0-9A-F,]+)$"); + private readonly Regex _bootEntryRegex = new Regex("^Boot(?<bootid>[0-9A-F]+)(?<activeflag>( |\\*)) (?<name>[^\\t]+)\\t(?<path>.+)$"); + + public EfiBootManagerConfiguration ParseBootManagerConfiguration(string efibootmgrOutput) + { + int bootCurrentId = 0; + int timeout = 0; + List<int> bootOrder = new(); + Dictionary<int, EfiBootManagerEntry> bootEntries = new(); + + var lines = efibootmgrOutput.Split('\n'); + for (int i = 0; i < lines.Length; i++) + { + var bootCurrentMatch = _bootCurrentRegex.Match(lines[i]); + var timeoutMatch = _timeoutRegex.Match(lines[i]); + var bootOrderMatch = _bootOrderRegex.Match(lines[i]); + var bootEntryMatch = _bootEntryRegex.Match(lines[i]); + + if (bootCurrentMatch.Success) + { + bootCurrentId = int.Parse( + bootCurrentMatch.Groups["bootid"].Value, + NumberStyles.HexNumber, + CultureInfo.InvariantCulture); + continue; + } + + if (timeoutMatch.Success) + { + timeout = int.Parse( + timeoutMatch.Groups["seconds"].Value, + CultureInfo.InvariantCulture); + continue; + } + + if (bootOrderMatch.Success) + { + foreach (var entry in bootOrderMatch.Groups["order"].Value.Split(',')) + { + bootOrder.Add( + int.Parse( + entry, + NumberStyles.HexNumber, + CultureInfo.InvariantCulture)); + } + continue; + } + + if (bootEntryMatch.Success) + { + var bootId = int.Parse( + bootEntryMatch.Groups["bootid"].Value, + NumberStyles.HexNumber, + CultureInfo.InvariantCulture); + var isActive = bootEntryMatch.Groups["activeflag"].Value == "*"; + var name = bootEntryMatch.Groups["name"].Value; + var path = bootEntryMatch.Groups["path"].Value; + bootEntries.Add( + bootId, + new EfiBootManagerEntry + { + BootId = bootId, + Name = name, + Path = path, + Active = isActive, + }); + } + } + + return new EfiBootManagerConfiguration + { + BootCurrentId = bootCurrentId, + Timeout = timeout, + BootOrder = bootOrder, + BootEntries = bootEntries, + }; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/EfiBootManagerConfiguration.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/EfiBootManagerConfiguration.cs new file mode 100644 index 00000000..e3bc9b45 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/EfiBootManagerConfiguration.cs @@ -0,0 +1,17 @@ + +namespace Redpoint.KubernetesManager.PxeBoot.Bootmgr +{ + using System.Collections.Generic; + + public record EfiBootManagerConfiguration + { + public required int BootCurrentId { get; set; } + + public required int Timeout { get; set; } + + public required IList<int> BootOrder { get; set; } + + public required Dictionary<int, EfiBootManagerEntry> BootEntries { get; set; } + + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/EfiBootManagerEntry.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/EfiBootManagerEntry.cs new file mode 100644 index 00000000..18a8a4ce --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/EfiBootManagerEntry.cs @@ -0,0 +1,14 @@ + +namespace Redpoint.KubernetesManager.PxeBoot.Bootmgr +{ + public record EfiBootManagerEntry + { + public required int BootId { get; set; } + + public required string Name { get; set; } + + public required string Path { get; set; } + + public required bool Active { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/IEfiBootManager.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/IEfiBootManager.cs new file mode 100644 index 00000000..4838cbb1 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/IEfiBootManager.cs @@ -0,0 +1,33 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Bootmgr +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + + internal interface IEfiBootManager + { + Task<EfiBootManagerConfiguration> GetBootManagerConfigurationAsync( + CancellationToken cancellationToken); + + Task RemoveBootManagerEntryAsync( + int bootEntry, + CancellationToken cancellationToken); + + Task AddBootManagerDiskEntryAsync( + string disk, + int partition, + string label, + string path, + CancellationToken cancellationToken); + + Task SetBootManagerBootOrderAsync( + IEnumerable<int> bootOrder, + CancellationToken cancellationToken); + + Task SetBootManagerEntryActiveAsync( + int bootEntry, + bool active, + CancellationToken cancellationToken); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/IEfiBootManagerParser.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/IEfiBootManagerParser.cs new file mode 100644 index 00000000..6b0d501e --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Bootmgr/IEfiBootManagerParser.cs @@ -0,0 +1,8 @@ + +namespace Redpoint.KubernetesManager.PxeBoot.Bootmgr +{ + public interface IEfiBootManagerParser + { + EfiBootManagerConfiguration ParseBootManagerConfiguration(string efibootmgrOutput); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Client/DefaultProvisionContextDiscoverer.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Client/DefaultProvisionContextDiscoverer.cs new file mode 100644 index 00000000..ee669808 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Client/DefaultProvisionContextDiscoverer.cs @@ -0,0 +1,131 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Client +{ + using Microsoft.Extensions.Logging; + using System.Globalization; + using System.Text.Json; + using System.Text.RegularExpressions; + + internal class DefaultProvisionContextDiscoverer : IProvisionContextDiscoverer + { + private readonly ILogger<DefaultProvisionContextDiscoverer> _logger; + + public DefaultProvisionContextDiscoverer( + ILogger<DefaultProvisionContextDiscoverer> logger) + { + _logger = logger; + } + + public async Task<ProvisionContext> GetProvisionContextAsync( + bool isLocal, + CancellationToken cancellationToken) + { + bool allowRecoveryShell = false; + + // Figure out the environment we're running in. + PlatformType platformType; + if (OperatingSystem.IsLinux()) + { + if (File.Exists("/rkm-initrd")) + { + _logger.LogInformation("Running on Linux initrd platform."); + platformType = PlatformType.LinuxInitrd; + allowRecoveryShell = true; + } + else + { + _logger.LogInformation("Running on Linux platform."); + platformType = PlatformType.Linux; + } + } + else if (OperatingSystem.IsMacOS()) + { + _logger.LogInformation("Running on macOS platform."); + platformType = PlatformType.Mac; + } + else if (OperatingSystem.IsWindows()) + { + _logger.LogInformation("Running on Windows platform."); + platformType = PlatformType.Windows; + } + else + { + throw new PlatformNotSupportedException(); + } + + // Determine the API address. + string apiAddress; + int bootedFromStepIndex = -1; + var isInRecovery = false; + if (isLocal) + { + apiAddress = "127.0.0.1"; + } + else + { + if (platformType == PlatformType.LinuxInitrd || platformType == PlatformType.Linux) + { + var kernelCmdline = await File.ReadAllTextAsync("/proc/cmdline", cancellationToken); + var kernelCmdlineAddressRegex = new Regex("rkm-api-address=(?<address>[0-9a-f:\\.]+)"); + var kernelCmdlineAddressRegexMatch = kernelCmdlineAddressRegex.Match(kernelCmdline); + var kernelCmdlineBootStepIndexRegex = new Regex("rkm-booted-from-step-index=(?<index>[0-9-]+)"); + var kernelCmdlineBootStepIndexRegexMatch = kernelCmdlineBootStepIndexRegex.Match(kernelCmdline); + if (!kernelCmdlineAddressRegexMatch.Success) + { + throw new UnableToProvisionSystemException("/proc/cmdline is missing the rkm-api-address= option."); + } + apiAddress = kernelCmdlineAddressRegexMatch.Groups["address"].Value; + if (kernelCmdline.Contains("rkm-in-recovery", StringComparison.Ordinal)) + { + _logger.LogInformation("RKM is running in recovery mode."); + isInRecovery = true; + } + else + { + if (!kernelCmdlineBootStepIndexRegexMatch.Success) + { + throw new UnableToProvisionSystemException("/proc/cmdline is missing the rkm-booted-from-step-index= option."); + } + bootedFromStepIndex = int.Parse(kernelCmdlineBootStepIndexRegexMatch.Groups["index"].Value, CultureInfo.InvariantCulture); + } + } + else if (platformType == PlatformType.Mac) + { + // @todo: Probably need to use UDP auto-discovery... + throw new PlatformNotSupportedException(); + } + else + { + WindowsRkmProvisionContext jsonContext; + using (var stream = new FileStream( + Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.System), + "rkm-provision-context.json"), + FileMode.Open, + FileAccess.Read, + FileShare.Read)) + { + jsonContext = (await JsonSerializer.DeserializeAsync( + stream, + WindowsRkmProvisionJsonSerializerContext.Default.WindowsRkmProvisionContext, + cancellationToken))!; + } + + allowRecoveryShell = false; + apiAddress = jsonContext.ApiAddress; + isInRecovery = jsonContext.IsInRecovery; + bootedFromStepIndex = jsonContext.BootedFromStepIndex; + } + } + _logger.LogInformation($"Using provisioner API address: {apiAddress}"); + + return new ProvisionContext + { + AllowRecoveryShell = allowRecoveryShell, + Platform = platformType, + ApiAddress = apiAddress, + IsInRecovery = isInRecovery, + BootedFromStepIndex = bootedFromStepIndex, + }; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Client/IProvisionContextDiscoverer.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Client/IProvisionContextDiscoverer.cs new file mode 100644 index 00000000..d767b7a7 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Client/IProvisionContextDiscoverer.cs @@ -0,0 +1,9 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Client +{ + internal interface IProvisionContextDiscoverer + { + Task<ProvisionContext> GetProvisionContextAsync( + bool isLocal, + CancellationToken cancellationToken); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Client/PlatformType.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Client/PlatformType.cs new file mode 100644 index 00000000..44a7839e --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Client/PlatformType.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Client +{ + internal enum PlatformType + { + LinuxInitrd, + Linux, + Mac, + Windows, + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Client/ProvisionContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Client/ProvisionContext.cs new file mode 100644 index 00000000..b52babc5 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Client/ProvisionContext.cs @@ -0,0 +1,15 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Client +{ + internal class ProvisionContext + { + public required bool AllowRecoveryShell { get; init; } + + public required PlatformType Platform { get; init; } + + public required bool IsInRecovery { get; init; } + + public required string ApiAddress { get; init; } + + public required int BootedFromStepIndex { get; init; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientCommand.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientCommand.cs new file mode 100644 index 00000000..46d8dfd2 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientCommand.cs @@ -0,0 +1,29 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Client +{ + using Microsoft.Extensions.DependencyInjection; + using Redpoint.CommandLine; + using Redpoint.KubernetesManager.PxeBoot.Bootmgr; + using Redpoint.KubernetesManager.PxeBoot.Disk; + using System.CommandLine; + + internal class PxeBootProvisionClientCommand : ICommandDescriptorProvider + { + public static CommandDescriptor Descriptor => CommandDescriptor.NewBuilder() + .WithOptions<PxeBootProvisionClientOptions>() + .WithInstance<PxeBootProvisionClientCommandInstance>() + .WithCommand( + builder => + { + return new Command("provision-client", "Provision this client via PXE boot."); + }) + .WithRuntimeServices( + (_, services, _) => + { + services.AddSingleton<IParted, DefaultParted>(); + services.AddSingleton<IEfiBootManagerParser, DefaultEfiBootManagerParser>(); + services.AddSingleton<IEfiBootManager, DefaultEfiBootManager>(); + services.AddPxeBootProvisioning(); + }) + .Build(); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientCommandInstance.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientCommandInstance.cs new file mode 100644 index 00000000..d1773f05 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientCommandInstance.cs @@ -0,0 +1,754 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Client +{ + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.Logging; + using Redpoint.CommandLine; + using Redpoint.Hashing; + using Redpoint.KubernetesManager.Configuration.Json; + using Redpoint.KubernetesManager.Configuration.Sources; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Api; + using Redpoint.KubernetesManager.PxeBoot.Bootmgr; + using Redpoint.KubernetesManager.PxeBoot.Disk; + using Redpoint.KubernetesManager.PxeBoot.FileTransfer; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.PathResolution; + using Redpoint.ProcessExecution; + using Redpoint.ProgressMonitor; + using Redpoint.Tpm; + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Net; + using System.Net.Http.Headers; + using System.Net.Http.Json; + using System.Net.Sockets; + using System.Runtime.InteropServices; + using System.Text.Json; + using System.Text.Json.Serialization; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + + internal class PxeBootProvisionClientCommandInstance : ICommandInstance + { + private readonly ILogger<PxeBootProvisionClientCommandInstance> _logger; + private readonly IPathResolver _pathResolver; + private readonly IProcessExecutor _processExecutor; + private readonly IParted _parted; + private readonly ITpmSecuredHttp _tpmSecuredHttp; + private readonly IEfiBootManager _efiBootManager; + private readonly IProgressFactory _progressFactory; + private readonly IMonitorFactory _monitorFactory; + private readonly IFileTransferClient _fileTransferClient; + private readonly IDurableOperation _durableOperation; + private readonly IProvisionContextDiscoverer _provisionContextDiscoverer; + private readonly PxeBootProvisionClientOptions _options; + private readonly List<IProvisioningStep> _provisioningSteps; + private readonly KubernetesRkmJsonSerializerContext _jsonSerializerContext; + + public PxeBootProvisionClientCommandInstance( + ILogger<PxeBootProvisionClientCommandInstance> logger, + IPathResolver pathResolver, + IProcessExecutor processExecutor, + IParted parted, + ITpmSecuredHttp tpmSecuredHttp, + IEnumerable<IProvisioningStep> provisioningSteps, + IEfiBootManager efiBootManager, + IProgressFactory progressFactory, + IMonitorFactory monitorFactory, + IFileTransferClient fileTransferClient, + IDurableOperation durableOperation, + IProvisionContextDiscoverer provisionContextDiscoverer, + PxeBootProvisionClientOptions options) + { + _logger = logger; + _pathResolver = pathResolver; + _processExecutor = processExecutor; + _parted = parted; + _tpmSecuredHttp = tpmSecuredHttp; + _efiBootManager = efiBootManager; + _progressFactory = progressFactory; + _monitorFactory = monitorFactory; + _fileTransferClient = fileTransferClient; + _durableOperation = durableOperation; + _provisionContextDiscoverer = provisionContextDiscoverer; + _options = options; + _provisioningSteps = provisioningSteps.ToList(); + + _jsonSerializerContext = KubernetesRkmJsonSerializerContext.CreateStringEnumWithAdditionalConverters( + new RkmNodeProvisionerStepJsonConverter(provisioningSteps)); + } + + private async Task ProvisionAndMountDisksAsync( + string apiAddress, + HttpClient client, + CancellationToken cancellationToken) + { + var diskPaths = await _parted.GetDiskPathsAsync(cancellationToken); + + if (diskPaths.Length == 0) + { + throw new UnableToProvisionSystemException("There are zero disks present under /dev/disk/by-diskseq. This system can not be provisioned by PXE boot."); + } + else if (diskPaths.Length >= 2) + { + throw new UnableToProvisionSystemException("There is more than one disk present under /dev/disk/by-diskseq. Expected exactly one disk attached for provisioning via PXE boot to work."); + } + + var diskPath = diskPaths[0]; + _logger.LogInformation($"Using disk {diskPath}"); + + var disk = await _parted.GetDiskAsync(diskPath, cancellationToken); + if (disk.Label == "unknown") + { + _logger.LogInformation("Initializing disk as it is neither MBR nor GPT..."); + await _parted.RunCommandAsync(diskPath, ["mktable", "gpt"], cancellationToken); + disk = await _parted.GetDiskAsync(diskPath, cancellationToken); + } + else if (disk.Label != "gpt") + { + throw new UnableToProvisionSystemException("Disk is already initialized and it is not a GPT-based disk!"); + } + + int exitCode; + if (disk.Partitions != null && + disk.Partitions.Length == 0) + { + _logger.LogInformation("The primary disk has no partitions and this machine can be provisioned for PXE boot."); + + var mkfsFat = await _pathResolver.ResolveBinaryPath("mkfs.fat"); + var mkfsNtfs = await _pathResolver.ResolveBinaryPath("mkfs.ntfs"); + + await _parted.RunCommandAsync(diskPath, ["mkpart", "primary", "fat32", "1MiB", "2048MiB"], cancellationToken); + await _parted.RunCommandAsync(diskPath, ["set", "1", "esp", "on"], cancellationToken); + await _parted.RunCommandAsync(diskPath, ["mkpart", "primary", "2049MiB", "2081MiB"], cancellationToken); + await _parted.RunCommandAsync(diskPath, ["type", "2", "e3c9e316-0b5c-4db8-817d-f92df00215ae"], cancellationToken); + await _parted.RunCommandAsync(diskPath, ["mkpart", "primary", "ntfs", "2082MiB", "100%"], cancellationToken); + await _parted.RunCommandAsync(diskPath, ["name", "3", "UetBootDisk"], cancellationToken); + + exitCode = await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = mkfsFat, + Arguments = [$"{diskPath}-part1"] + }, + CaptureSpecification.Passthrough, + cancellationToken); + if (exitCode != 0) + { + throw new UnableToProvisionSystemException("mkfs.fat on partition 1 failed!"); + } + exitCode = await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = mkfsNtfs, + Arguments = ["-Q", "-L", "UetBootDisk", $"{diskPath}-part3"] + }, + CaptureSpecification.Passthrough, + cancellationToken); + if (exitCode != 0) + { + throw new UnableToProvisionSystemException("mkfs.ntfs on partition 3 failed!"); + } + } + else if ( + disk.Partitions == null || + disk.Partitions.Length != 3 || + disk.Partitions[2] == null || + disk.Partitions[2]?.Name != "UetBootDisk") + { + throw new UnableToProvisionSystemException("Disk is already initialized with partitions that are not recognised as a provisioned PXE boot setup. If you would like to provision this machine, you will need to manually remove the partitions on the disk (destroying all data) and then reboot."); + } + else + { + _logger.LogInformation("Machine disks are already provisioned."); + } + + Directory.CreateDirectory("/var/mount/boot"); + Directory.CreateDirectory("/var/mount/images"); + + var mount = await _pathResolver.ResolveBinaryPath("mount"); + var mtab = File.ReadAllText("/etc/mtab"); + + if (!mtab.Contains("/var/mount/boot", StringComparison.Ordinal)) + { + for (int retryAttempt = 0; retryAttempt < 30; retryAttempt++) + { + exitCode = await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("fsck.vfat"), + Arguments = ["-a", $"{diskPath}-part1"], + }, + CaptureSpecification.Passthrough, + cancellationToken); + exitCode = await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = mount, + Arguments = [$"{diskPath}-part1", "/var/mount/boot"], + }, + CaptureSpecification.Passthrough, + cancellationToken); + if (exitCode != 0) + { + if (retryAttempt == 29) + { + throw new UnableToProvisionSystemException("mount partition 1 to /var/mount/boot failed!"); + } + else + { + await Task.Delay(1000, cancellationToken); + } + } + else + { + break; + } + } + } + if (!mtab.Contains("/var/mount/images", StringComparison.Ordinal)) + { + exitCode = await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("ntfsfix"), + Arguments = [$"{diskPath}-part3", "-d"], + }, + CaptureSpecification.Passthrough, + cancellationToken); + if (exitCode != 0) + { + throw new UnableToProvisionSystemException("fsck partition 3 failed!"); + } + exitCode = await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = mount, + Arguments = [$"{diskPath}-part3", "-t", "ntfs3", "/var/mount/images"], + }, + CaptureSpecification.Passthrough, + cancellationToken); + if (exitCode != 0) + { + throw new UnableToProvisionSystemException("mount partition 3 to /var/mount/images failed!"); + } + } + + _logger.LogInformation("Reading EFI boot manager configuration..."); + var configuration = await _efiBootManager.GetBootManagerConfigurationAsync( + cancellationToken); + + _logger.LogInformation($"EFI current boot: {configuration.BootCurrentId}"); + _logger.LogInformation($"EFI timeout: {configuration.Timeout} seconds"); + _logger.LogInformation($"EFI boot order: {string.Join(", ", configuration.BootOrder)}"); + foreach (var kv in configuration.BootEntries) + { + _logger.LogInformation($"EFI boot entry {kv.Key}, name '{kv.Value.Name}', {(kv.Value.Active ? "active" : "inactive")}, path '{kv.Value.Path}'"); + } + + Directory.CreateDirectory("/var/mount/boot/EFI/RKM"); + await _fileTransferClient.DownloadFilesAsync( + new Uri($"https://{apiAddress}:8791/static"), + "/var/mount/boot/EFI/RKM", + new Dictionary<string, string> + { + { "ipxe.efi", "ipxe.efi" }, + { "vmlinuz", "vmlinuz" }, + { "initrd", "initrd" }, + { "uet", "uet" }, + }, + client: client, + cancellationToken: cancellationToken); + + _logger.LogInformation($"Setting autoexec.ipxe for recovery..."); + // @note: 'dhcp' is required here, iPXE will not boot even from file without a configured net interface. + await File.WriteAllTextAsync( + $"/var/mount/boot/EFI/RKM/autoexec.ipxe", + $""" + #!ipxe + dhcp + kernel file:/EFI/RKM/vmlinuz rkm-api-address={apiAddress} rkm-in-recovery + initrd file:/EFI/RKM/initrd + initrd file:/EFI/RKM/uet /usr/bin/uet-bootstrap mode=555 + boot + """, + CancellationToken.None); + + var entriesToRemove = configuration.BootEntries + .Where(x => x.Value.Name == "RKM Recovery") + .Select(k => k.Key) + .ToList(); + if (entriesToRemove.Count > 0) + { + _logger.LogInformation("Removing existing EFI recovery entries..."); + foreach (var entry in entriesToRemove) + { + await _efiBootManager.RemoveBootManagerEntryAsync( + entry, + CancellationToken.None); + } + } + + _logger.LogInformation("Adding EFI boot entry for recovery..."); + await _efiBootManager.AddBootManagerDiskEntryAsync( + diskPath, + 1, + "RKM Recovery", + @"\EFI\RKM\ipxe.efi", + CancellationToken.None); + + _logger.LogInformation("Reading EFI boot manager configuration..."); + configuration = await _efiBootManager.GetBootManagerConfigurationAsync( + CancellationToken.None); + + async Task<List<int>> UpdateBootOrderAsync() + { + _logger.LogInformation("Updating EFI boot order..."); + var desiredBootOrder = configuration.BootEntries + .Values + .OrderByDescending(kv => + { + if (!kv.Active) + { + // Inactive entries should be right before the UEFI fallback. + return -15; + } + else if (kv.Path.Contains("/MAC(", StringComparison.Ordinal) || + kv.Path.Contains(",DHCP,", StringComparison.Ordinal)) + { + // Always network boot first. + return 10; + } + else if (kv.Name == "RKM Recovery") + { + // Recovery should always be last. + return -10; + } + else if ( + kv.Name == "FrontPage" || + kv.Path.Contains("MemoryMapped(", StringComparison.Ordinal)) + { + // UEFI fallback entries that should be absolutely last because + // they halt the boot sequence and leave the machine idling. + // These are present on Hyper-V. + return -20; + } + else + { + // Other entries should be in the middle, since they will + // have been installed via provisioning. + return 0; + } + }) + .ThenBy(x => configuration.BootOrder.IndexOf(x.BootId)) // Preserve boot order within priorities. + .Select(x => x.BootId) + .ToList(); + await _efiBootManager.SetBootManagerBootOrderAsync( + desiredBootOrder, + CancellationToken.None); + return desiredBootOrder; + } + var desiredBootOrder = await UpdateBootOrderAsync(); + + // Notify the API of our boot entries, and ask for the inactive list. + _logger.LogInformation($"Synchronising our boot loader entries..."); + var bootEntriesJson = new List<RkmNodeStatusBootEntry>(); + foreach (var bootId in desiredBootOrder) + { + var bootEntry = configuration.BootEntries[bootId]; + bootEntriesJson.Add(new RkmNodeStatusBootEntry + { + BootId = $"Boot{bootId:X4}", + Name = bootEntry.Name, + Path = bootEntry.Path, + Active = bootEntry.Active, + }); + } + var bootEntryResponse = await client.PutAsJsonAsync( + new Uri($"https://{apiAddress}:8791/api/node-provisioning/sync-boot-entries"), + bootEntriesJson, + _jsonSerializerContext.ListRkmNodeStatusBootEntry, + cancellationToken); + bootEntryResponse.EnsureSuccessStatusCode(); + var inactiveBootEntries = await bootEntryResponse.Content.ReadFromJsonAsync( + _jsonSerializerContext.IListString, + cancellationToken) ?? []; + _logger.LogInformation($"The following boot entries should be inactive: {string.Join(", ", inactiveBootEntries)}"); + var anyBootActiveChanges = false; + foreach (var bootKv in configuration.BootEntries) + { + _logger.LogInformation($"Checking 'Boot{bootKv.Key:X4}'..."); + + // Make sure our recovery entry is always active, even if the provisioner things it should not be. This prevents + // unrecoverable boot states. + var isRecoveryEntry = bootKv.Value.Name == "RKM Recovery"; + + if (bootKv.Value.Active && inactiveBootEntries.Contains($"Boot{bootKv.Key:X4}", StringComparer.Ordinal) && !isRecoveryEntry) + { + _logger.LogInformation($"Need to mark boot entry {bootKv.Key} as inactive..."); + await _efiBootManager.SetBootManagerEntryActiveAsync( + bootKv.Key, + false, + cancellationToken); + anyBootActiveChanges = true; + } + else if (!bootKv.Value.Active && (!inactiveBootEntries.Contains($"Boot{bootKv.Key:X4}", StringComparer.Ordinal) || isRecoveryEntry)) + { + _logger.LogInformation($"Need to mark boot entry {bootKv.Key} as active..."); + await _efiBootManager.SetBootManagerEntryActiveAsync( + bootKv.Key, + false, + cancellationToken); + anyBootActiveChanges = true; + } + } + if (anyBootActiveChanges) + { + _logger.LogInformation($"Changes were made to the boot loader active states. Synchronising again..."); + + // Update boot order to set the inactive entries after everything else. + desiredBootOrder = await UpdateBootOrderAsync(); + + // Notify API again with new active states. + configuration = await _efiBootManager.GetBootManagerConfigurationAsync( + CancellationToken.None); + bootEntriesJson = new List<RkmNodeStatusBootEntry>(); + foreach (var bootId in desiredBootOrder) + { + var bootEntry = configuration.BootEntries[bootId]; + bootEntriesJson.Add(new RkmNodeStatusBootEntry + { + BootId = $"Boot{bootId:X4}", + Name = bootEntry.Name, + Path = bootEntry.Path, + Active = bootEntry.Active, + }); + } + bootEntryResponse = await client.PutAsJsonAsync( + new Uri($"https://{apiAddress}:8791/api/node-provisioning/sync-boot-entries"), + bootEntriesJson, + _jsonSerializerContext.ListRkmNodeStatusBootEntry, + cancellationToken); + bootEntryResponse.EnsureSuccessStatusCode(); + } + } + + private class DefaultProvisioningStepClientContext( + bool isLocalTesting, + HttpClient provisioningApiClient, + string provisioningApiEndpointHttps, + string provisioningApiEndpointHttp, + string provisioningApiAddress, + string authorizedNodeName, + string aikFingerprint, + Dictionary<string, string> parameterValues) + : IProvisioningStepClientContext + { + public bool IsLocalTesting => isLocalTesting; + + public HttpClient ProvisioningApiClient => provisioningApiClient; + + public string ProvisioningApiEndpointHttps => provisioningApiEndpointHttps; + + public string ProvisioningApiEndpointHttp => provisioningApiEndpointHttp; + + public string ProvisioningApiAddress => provisioningApiAddress; + + public string AuthorizedNodeName => authorizedNodeName; + + public string AikFingerprint => aikFingerprint; + + public Dictionary<string, string> ParameterValues => parameterValues; + } + + public async Task<int> ExecuteAsync(ICommandInvocationContext context) + { + var allowRecoveryShell = false; + try + { + // Get the provisioning context. + var provisionContext = await _provisionContextDiscoverer.GetProvisionContextAsync( + context.ParseResult.GetValueForOption(_options.Local), + context.GetCancellationToken()); + allowRecoveryShell = provisionContext.AllowRecoveryShell; + + // Create our TPM-secured HTTP client, and negotiate the client certificate. + using var client = await _durableOperation.DurableOperationAsync( + async cancellationToken => + { + var client = await _tpmSecuredHttp.CreateHttpClientAsync( + new Uri($"http://{provisionContext.ApiAddress}:8790/api/node-provisioning/negotiate-certificate"), + cancellationToken); + client.Timeout = TimeSpan.FromSeconds(5); + return client; + }, + context.GetCancellationToken()); + + // Attempt to authorize ourselves with the cluster. + var authorizeRequest = new AuthorizeNodeRequest + { + CapablePlatforms = OperatingSystem.IsMacOS() + ? [RkmNodePlatform.Mac] + : [RkmNodePlatform.Windows, RkmNodePlatform.Linux], + Architecture = RuntimeInformation.OSArchitecture == Architecture.Arm64 ? "arm64" : "amd64", + }; + var secureEndpoint = $"https://{provisionContext.ApiAddress}:8791/api/node-provisioning"; + AuthorizeNodeResponse authorizeResponse; + retryNegotiate: + var authorizeResponseRaw = await _durableOperation.DurableOperationAsync( + async cancellationToken => + { + return await client.PutAsJsonAsync( + new Uri($"{secureEndpoint}/authorize"), + authorizeRequest, + ApiJsonSerializerContext.WithStringEnum.AuthorizeNodeRequest, + cancellationToken); + }, + context.GetCancellationToken()); + if (authorizeResponseRaw.StatusCode == HttpStatusCode.Unauthorized) + { + var fingerprint = await authorizeResponseRaw.Content.ReadAsStringAsync(); + + _logger.LogError($"This node '{fingerprint}' is not yet authorized to join the cluster. To authorize it, update the RkmNode object in the cluster. Waiting 1 minute and then checking again..."); + await Task.Delay(60000, context.GetCancellationToken()); + goto retryNegotiate; + } + else if (authorizeResponseRaw.StatusCode == HttpStatusCode.OK) + { + authorizeResponse = (await authorizeResponseRaw.Content.ReadFromJsonAsync( + ApiJsonSerializerContext.WithStringEnum.AuthorizeNodeResponse, + context.GetCancellationToken()))!; + } + else + { + throw new UnableToProvisionSystemException($"Certificate negotiation endpoint returned unexpected response status code {authorizeResponseRaw.StatusCode}."); + } + _logger.LogInformation($"Authorized to join the cluster with node name '{authorizeResponse.NodeName}'."); + + // If we are running in the Linux initrd environment, make sure that we have provisioned the disks. + if (provisionContext.Platform == PlatformType.LinuxInitrd) + { + // This function will throw if provisioning disks fails. + await ProvisionAndMountDisksAsync( + provisionContext.ApiAddress, + client, + context.GetCancellationToken()); + + // If we are in recovery, we need to tell the cluster to forcibly reprovision us and then reboot. + if (provisionContext.IsInRecovery) + { + _logger.LogInformation("Requesting reprovisioning from cluster..."); + var forceReprovisionResponseRaw = await _durableOperation.DurableOperationAsync( + async cancellationToken => + { + return await client.PutAsJsonAsync( + new Uri($"{secureEndpoint}/force-reprovision"), + new ForceReprovisionNodeRequest(), + ApiJsonSerializerContext.WithStringEnum.ForceReprovisionNodeRequest, + cancellationToken); + }, + context.GetCancellationToken()); + forceReprovisionResponseRaw.EnsureSuccessStatusCode(); + + _logger.LogInformation("Rebooting from recovery..."); + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("systemctl"), + Arguments = ["--message=\"RKM Reboot from Recovery\"", "reboot"] + }, + CaptureSpecification.Passthrough, + context.GetCancellationToken()); + return 0; + } + } + + // Now process provisioning steps. + var clientContext = new DefaultProvisioningStepClientContext( + context.ParseResult.GetValueForOption(_options.Local), + client, + $"https://{provisionContext.ApiAddress}:8791", + $"http://{provisionContext.ApiAddress}:8790", + provisionContext.ApiAddress, + authorizeResponse.NodeName, + authorizeResponse.AikFingerprint, + authorizeResponse.ParameterValues); + var initial = true; + do + { + var @params = string.Empty; + if (initial) + { + @params = $"?initial=true&bootedFromStepIndex={provisionContext.BootedFromStepIndex}"; + } + + var stepResponseRaw = await _durableOperation.DurableOperationAsync( + async cancellationToken => + { + return await client.GetAsync( + new Uri($"{secureEndpoint}/step{@params}"), + cancellationToken); + }, + context.GetCancellationToken()); + if (await HandleStepStatusCodeAsync( + stepResponseRaw, + client, + clientContext, + secureEndpoint, + context.GetCancellationToken())) + { + return 0; + } + stepResponseRaw.EnsureSuccessStatusCode(); + + var currentStep = await stepResponseRaw.Content.ReadFromJsonAsync( + _jsonSerializerContext.RkmNodeProvisionerStep, + context.GetCancellationToken()); + var provisioningStep = _provisioningSteps.FirstOrDefault(x => string.Equals(x.Type, currentStep?.Type, StringComparison.OrdinalIgnoreCase)); + if (provisioningStep == null) + { + throw new UnableToProvisionSystemException($"The provisioning step type '{currentStep?.Type}' does not exist on the client."); + } + + initial = false; + + immediatelyStartNextStep: + _logger.LogInformation($"Now executing step '{currentStep?.Type}'..."); + await provisioningStep.ExecuteOnClientUncastedAsync( + currentStep?.DynamicSettings, + clientContext, + context.GetCancellationToken()); + + if (provisioningStep.Flags.HasFlag(ProvisioningStepFlags.AssumeCompleteWhenIpxeScriptFetched)) + { + _logger.LogInformation("Provisioning step completes on next iPXE script fetch. Exiting now."); + return 0; + } + else + { + var stepCompleteResponseRaw = await _durableOperation.DurableOperationAsync( + async cancellationToken => + { + return await client.GetAsync( + new Uri($"{secureEndpoint}/step-complete"), + cancellationToken); + }, + context.GetCancellationToken()); + if (await HandleStepStatusCodeAsync( + stepCompleteResponseRaw, + client, + clientContext, + secureEndpoint, + context.GetCancellationToken())) + { + return 0; + } + else if (stepCompleteResponseRaw.StatusCode == HttpStatusCode.PartialContent) + { + // We didn't implicitly get the next step (there might be none). Loop + // again and exit if /step also returns 204 No Content. + continue; + } + stepCompleteResponseRaw.EnsureSuccessStatusCode(); + + currentStep = await stepCompleteResponseRaw.Content.ReadFromJsonAsync( + _jsonSerializerContext.RkmNodeProvisionerStep, + context.GetCancellationToken()); + provisioningStep = _provisioningSteps.FirstOrDefault(x => string.Equals(x.Type, currentStep?.Type, StringComparison.OrdinalIgnoreCase)); + if (provisioningStep == null) + { + throw new UnableToProvisionSystemException($"The provisioning step type '{currentStep?.Type}' does not exist on the client."); + } + goto immediatelyStartNextStep; + } + } + while (!context.GetCancellationToken().IsCancellationRequested); + + return 0; + } + catch (Exception ex) + { + // @todo: Report error to API address if we have that. + + if (ex is UnableToProvisionSystemException) + { + _logger.LogError(ex.Message); + } + else + { + _logger.LogError(ex, "Unexpected exception!"); + } + + if (allowRecoveryShell) + { + _logger.LogInformation("Starting recovery shell..."); + return await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath(OperatingSystem.IsWindows() ? "powershell" : "bash"), + Arguments = [] + }, + CaptureSpecification.Passthrough, + context.GetCancellationToken()); + } + else + { + return 1; + } + } + } + + private async Task<bool> HandleStepStatusCodeAsync( + HttpResponseMessage stepResponse, + HttpClient client, + IProvisioningStepClientContext clientContext, + string secureEndpoint, + CancellationToken cancellationToken) + { + if (stepResponse.StatusCode == HttpStatusCode.Conflict) + { + _logger.LogError("Provisioning configuration changed during provisioning. Restarting to fresh environment."); + + _logger.LogInformation("Scheduling reboot on client..."); + var rebootProvisioningStep = _provisioningSteps.First(x => x.Type == "reboot"); + await rebootProvisioningStep.ExecuteOnClientUncastedAsync( + null, + clientContext, + cancellationToken); + return true; + } + else if (stepResponse.StatusCode == HttpStatusCode.UnprocessableEntity) + { + _logger.LogError("Provisioning configuration on server is no longer valid. Restarting to go through authorization checks again."); + + _logger.LogInformation("Scheduling reboot on client..."); + var rebootProvisioningStep = _provisioningSteps.First(x => x.Type == "reboot"); + await rebootProvisioningStep.ExecuteOnClientUncastedAsync( + null, + clientContext, + cancellationToken); + return true; + } + else if (stepResponse.StatusCode == HttpStatusCode.NoContent) + { + _logger.LogInformation("No further provisioning steps to run, requesting reboot to disk on server..."); + var rebootToDiskResponse = await client.GetAsync(new Uri($"{secureEndpoint}/reboot-to-disk"), cancellationToken); + rebootToDiskResponse.EnsureSuccessStatusCode(); + + _logger.LogInformation("Scheduling reboot on client..."); + var rebootProvisioningStep = _provisioningSteps.First(x => x.Type == "reboot"); + await rebootProvisioningStep.ExecuteOnClientUncastedAsync( + null, + clientContext, + cancellationToken); + return true; + } + + return false; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientOptions.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientOptions.cs new file mode 100644 index 00000000..d2baf362 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Client/PxeBootProvisionClientOptions.cs @@ -0,0 +1,9 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Client +{ + using System.CommandLine; + + internal class PxeBootProvisionClientOptions + { + public Option<bool> Local = new Option<bool>("--local", "Assume the provisioning API is running locally for testing."); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Client/WindowsRkmProvisionContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Client/WindowsRkmProvisionContext.cs new file mode 100644 index 00000000..362a88dc --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Client/WindowsRkmProvisionContext.cs @@ -0,0 +1,16 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Client +{ + using System.Text.Json.Serialization; + + internal class WindowsRkmProvisionContext + { + [JsonPropertyName("isInRecovery")] + public required bool IsInRecovery { get; init; } + + [JsonPropertyName("apiAddress")] + public required string ApiAddress { get; set; } + + [JsonPropertyName("bootedFromStepIndex")] + public required int BootedFromStepIndex { get; init; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Client/WindowsRkmProvisionJsonSerializerContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Client/WindowsRkmProvisionJsonSerializerContext.cs new file mode 100644 index 00000000..85049aa4 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Client/WindowsRkmProvisionJsonSerializerContext.cs @@ -0,0 +1,9 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Client +{ + using System.Text.Json.Serialization; + + [JsonSerializable(typeof(WindowsRkmProvisionContext))] + internal partial class WindowsRkmProvisionJsonSerializerContext : JsonSerializerContext + { + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Disk/DefaultParted.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/DefaultParted.cs new file mode 100644 index 00000000..d9c80686 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/DefaultParted.cs @@ -0,0 +1,105 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Disk +{ + using Microsoft.Extensions.Logging; + using Redpoint.PathResolution; + using Redpoint.ProcessExecution; + using System; + using System.IO; + using System.Text; + using System.Text.Json; + using System.Threading.Tasks; + using UET.Commands.Internal.PxeBoot; + + internal class DefaultParted : IParted + { + private readonly ILogger<DefaultParted> _logger; + private readonly IPathResolver _pathResolver; + private readonly IProcessExecutor _processExecutor; + + public DefaultParted( + ILogger<DefaultParted> logger, + IPathResolver pathResolver, + IProcessExecutor processExecutor) + { + _logger = logger; + _pathResolver = pathResolver; + _processExecutor = processExecutor; + } + + public Task<string[]> GetDiskPathsAsync(CancellationToken cancellationToken) + { + var dev = new DirectoryInfo("/dev/disk/by-diskseq"); + var results = new List<string>(); + if (dev.Exists) + { + foreach (var fileInfo in dev.GetFiles()) + { + if (fileInfo.LinkTarget == null) + { + // Ignore anything that isn't a link. + continue; + } + if (fileInfo.LinkTarget.Contains("loop", StringComparison.Ordinal)) + { + // Ignore loopback devices. + continue; + } + if (fileInfo.Name.Contains("part", StringComparison.Ordinal)) + { + // Ignore partitions. + continue; + } + + _logger.LogInformation($"Discovered disk {fileInfo.FullName} ({fileInfo.LinkTarget})"); + results.Add(fileInfo.FullName); + } + } + return Task.FromResult(results.ToArray()); + } + + public async Task<PartedDisk> GetDiskAsync(string path, CancellationToken cancellationToken) + { + var parted = await _pathResolver.ResolveBinaryPath("parted"); + + var diskInfoOutput = new StringBuilder(); + var exitCode = await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = parted, + Arguments = ["-s", "-j", path, "print"], + }, + CaptureSpecification.CreateFromStdoutStringBuilder(diskInfoOutput), + cancellationToken); + + var diskInfo = JsonSerializer.Deserialize( + diskInfoOutput.ToString(), + PartedJsonSerializerContext.Default.PartedOutput); + + if (exitCode != 0 && diskInfo?.Disk?.Label != "unknown") + { + throw new InvalidOperationException($"'parted print' exited with non-zero exit code {exitCode}"); + } + + _logger.LogInformation($"Queried disk information: {JsonSerializer.Serialize(diskInfo, PartedJsonSerializerContext.Default.PartedOutput)}"); + + return diskInfo!.Disk!; + } + + public async Task RunCommandAsync(string diskPath, string[] args, CancellationToken cancellationToken) + { + var parted = await _pathResolver.ResolveBinaryPath("parted"); + var exitCode = await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = parted, + Arguments = ["-s", "-j", diskPath, .. args], + }, + CaptureSpecification.Passthrough, + cancellationToken); + if (exitCode != 0) + { + throw new InvalidOperationException($"'parted {string.Join(" ", args)}' exited with non-zero exit code {exitCode}"); + } + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Disk/IParted.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/IParted.cs new file mode 100644 index 00000000..9ea4d929 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/IParted.cs @@ -0,0 +1,13 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Disk +{ + using System.Threading.Tasks; + + internal interface IParted + { + Task<string[]> GetDiskPathsAsync(CancellationToken cancellationToken); + + Task<PartedDisk> GetDiskAsync(string path, CancellationToken cancellationToken); + + Task RunCommandAsync(string diskPath, string[] args, CancellationToken cancellationToken); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedDisk.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedDisk.cs new file mode 100644 index 00000000..7953e457 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedDisk.cs @@ -0,0 +1,37 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Disk +{ + using System.Text.Json.Serialization; + + internal class PartedDisk + { + [JsonPropertyName("path")] + public string? Path { get; set; } + + [JsonPropertyName("size")] + public string? Size { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } + + [JsonPropertyName("transport")] + public string? Transport { get; set; } + + [JsonPropertyName("logical-sector-size")] + public int? LogicalSectorSize { get; set; } + + [JsonPropertyName("physical-sector-size")] + public int? PhysicalSectorSize { get; set; } + + [JsonPropertyName("label")] + public string? Label { get; set; } + + [JsonPropertyName("uuid")] + public string? Uuid { get; set; } + + [JsonPropertyName("max-partitions")] + public int? MaxPartitions { get; set; } + + [JsonPropertyName("partitions")] + public PartedDiskPartition?[]? Partitions { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedDiskPartition.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedDiskPartition.cs new file mode 100644 index 00000000..94fdad56 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedDiskPartition.cs @@ -0,0 +1,34 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Disk +{ + using System.Text.Json.Serialization; + + internal class PartedDiskPartition + { + [JsonPropertyName("number")] + public int? Number { get; set; } + + [JsonPropertyName("start")] + public string? Start { get; set; } + + [JsonPropertyName("end")] + public string? End { get; set; } + + [JsonPropertyName("size")] + public string? Size { get; set; } + + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonPropertyName("type-uuid")] + public string? TypeUuid { get; set; } + + [JsonPropertyName("uuid")] + public string? Uuid { get; set; } + + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonPropertyName("flags")] + public string?[]? Flags { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedJsonSerializerContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedJsonSerializerContext.cs new file mode 100644 index 00000000..8a7185d8 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedJsonSerializerContext.cs @@ -0,0 +1,10 @@ +namespace UET.Commands.Internal.PxeBoot +{ + using Redpoint.KubernetesManager.PxeBoot.Disk; + using System.Text.Json.Serialization; + + [JsonSerializable(typeof(PartedOutput))] + internal partial class PartedJsonSerializerContext : JsonSerializerContext + { + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedOutput.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedOutput.cs new file mode 100644 index 00000000..fc8d74a2 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Disk/PartedOutput.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Disk +{ + using System.Text.Json.Serialization; + + internal class PartedOutput + { + [JsonPropertyName("disk")] + public PartedDisk? Disk { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultDurableOperation.cs b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultDurableOperation.cs new file mode 100644 index 00000000..2720415d --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultDurableOperation.cs @@ -0,0 +1,87 @@ +namespace Redpoint.KubernetesManager.PxeBoot.FileTransfer +{ + using Microsoft.Extensions.Logging; + using Redpoint.ProgressMonitor; + using System; + using System.Net.Sockets; + using System.Threading.Tasks; + using static Redpoint.KubernetesManager.PxeBoot.Client.PxeBootProvisionClientCommandInstance; + + internal class DefaultDurableOperation : IDurableOperation + { + private readonly ILogger<DefaultDurableOperation> _logger; + + public DefaultDurableOperation( + ILogger<DefaultDurableOperation> logger) + { + _logger = logger; + } + + public async Task DurableOperationAsync(Func<CancellationToken, Task> operation, CancellationToken cancellationToken) + { + retry: + try + { + await operation(cancellationToken); + return; + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogWarning("Timeout while running operation, retrying in 1 seconds..."); + await Task.Delay(1000, cancellationToken); + goto retry; + } + catch (HttpRequestException ex) when (ex.InnerException is SocketException socketEx) + { + _logger.LogWarning($"Socket exception '{socketEx.Message}' while running durable operation, retrying in 1 seconds..."); + await Task.Delay(1000, cancellationToken); + goto retry; + } + catch (StreamStalledException) + { + _logger.LogWarning($"Stream stalled while running durable operation, retrying in 1 seconds..."); + await Task.Delay(1000, cancellationToken); + goto retry; + } + catch (DownloadedFileHashInvalidException) + { + _logger.LogWarning($"Downloaded file does not match hash in header, retrying in 1 seconds..."); + await Task.Delay(1000, cancellationToken); + goto retry; + } + } + + public async Task<TOut> DurableOperationAsync<TOut>(Func<CancellationToken, Task<TOut>> operation, CancellationToken cancellationToken) + { + retry: + try + { + return await operation(cancellationToken); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + _logger.LogWarning("Timeout while running operation, retrying in 1 seconds..."); + await Task.Delay(1000, cancellationToken); + goto retry; + } + catch (HttpRequestException ex) when (ex.InnerException is SocketException socketEx) + { + _logger.LogWarning($"Socket exception '{socketEx.Message}' while running durable operation, retrying in 1 seconds..."); + await Task.Delay(1000, cancellationToken); + goto retry; + } + catch (StreamStalledException) + { + _logger.LogWarning($"Stream stalled while running durable operation, retrying in 1 seconds..."); + await Task.Delay(1000, cancellationToken); + goto retry; + } + catch (DownloadedFileHashInvalidException) + { + _logger.LogWarning($"Downloaded file does not match hash in header, retrying in 1 seconds..."); + await Task.Delay(1000, cancellationToken); + goto retry; + } + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultFileTransferClient.cs b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultFileTransferClient.cs new file mode 100644 index 00000000..342d4d2e --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultFileTransferClient.cs @@ -0,0 +1,307 @@ +namespace Redpoint.KubernetesManager.PxeBoot.FileTransfer +{ + using k8s.Autorest; + using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; + using Microsoft.Extensions.Logging; + using Redpoint.Hashing; + using Redpoint.ProgressMonitor; + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + using System.IO.Pipes; + using System.Linq; + using System.Net; + using System.Threading.Tasks; + + internal class DefaultFileTransferClient : IFileTransferClient + { + private readonly IDurableOperation _durableOperation; + private readonly ILogger<DefaultFileTransferClient> _logger; + private readonly IProgressFactory _progressFactory; + private readonly IMonitorFactory _monitorFactory; + + public DefaultFileTransferClient( + IDurableOperation durableOperation, + ILogger<DefaultFileTransferClient> logger, + IProgressFactory progressFactory, + IMonitorFactory monitorFactory) + { + _durableOperation = durableOperation; + _logger = logger; + _progressFactory = progressFactory; + _monitorFactory = monitorFactory; + } + + public async Task DownloadFilesAsync( + Dictionary<Uri, string> sourceToTargetMappings, + HttpClient? client = null, + CancellationToken cancellationToken = default) + { + var disposeClient = false; + if (client == null) + { + client = new HttpClient(); + disposeClient = true; + } + try + { + var filesToReplace = new HashSet<string>(); + + foreach (var mapping in sourceToTargetMappings) + { + var fetched = await _durableOperation.DurableOperationAsync( + async cancellationToken => + { + var existingHash = string.Empty; + var newHash = string.Empty; + if (File.Exists(mapping.Value)) + { + using (var fileReadStream = new FileStream(mapping.Value, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + existingHash = $"sha256:{Hash.Sha256AsHexString(fileReadStream)}"; + } + } + if (!string.IsNullOrWhiteSpace(existingHash)) + { + _logger.LogInformation($"Checking if {mapping.Value} needs to be downloaded..."); + using var headTimerCts = new CancellationTokenSource(5000); + using var headCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, headTimerCts.Token); + using (var headResponse = await client.SendAsync( + new HttpRequestMessage + { + RequestUri = mapping.Key, + Method = HttpMethod.Head, + }, + HttpCompletionOption.ResponseHeadersRead, + headCts.Token)) + { + headResponse.EnsureSuccessStatusCode(); + if (headResponse.Headers.TryGetValues("Content-Hash", out var newHashes)) + { + newHash = newHashes.FirstOrDefault() ?? string.Empty; + } + } + } + + if (newHash != existingHash || string.IsNullOrWhiteSpace(existingHash)) + { + _logger.LogInformation($"Downloading {mapping.Value}..."); + using (var fileStream = new FileStream($"{mapping.Value}.tmp", FileMode.Create, FileAccess.ReadWrite, FileShare.None)) + { + using var response = await client.GetAsync( + mapping.Key, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken); + var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); + using (var positionAwareStream = new PositionAwareStream( + responseStream, + response.Content.Headers.ContentLength!.Value)) + { + using (var stream = new StallDetectionStream( + positionAwareStream, + TimeSpan.FromSeconds(5))) + { + var cts = new CancellationTokenSource(); + var progress = _progressFactory.CreateProgressForStream(stream); + var monitorTask = Task.Run( + async () => + { + var monitor = _monitorFactory.CreateByteBasedMonitor(); + await monitor.MonitorAsync( + progress, + SystemConsole.ConsoleInformation, + SystemConsole.WriteProgressToConsole, + cts.Token); + }, cts.Token); + + await stream.CopyToAsync(fileStream, cancellationToken); + + await SystemConsole.CancelAndWaitForConsoleMonitoringTaskAsync(monitorTask, cts); + } + } + + if (response.Content.Headers.TryGetValues("Content-Hash", out var contentHashes)) + { + fileStream.Seek(0, SeekOrigin.Begin); + + var hash = Hash.Sha256AsHexString(fileStream); + if (contentHashes.FirstOrDefault() != $"sha256:{hash}") + { + throw new DownloadedFileHashInvalidException(); + } + } + } + + return true; + } + else + { + return false; + } + }, + cancellationToken); + if (fetched) + { + filesToReplace.Add(mapping.Value); + } + } + foreach (var file in filesToReplace) + { + _logger.LogInformation($"Finalising download of {file}..."); + File.Move( + $"{file}.tmp", + file, + true); + } + } + finally + { + if (disposeClient) + { + client.Dispose(); + } + } + } + + public Task DownloadFilesAsync( + Uri sourceBaseUri, + Dictionary<string, string> sourceToTargetMappings, + HttpClient? client = null, + CancellationToken cancellationToken = default) + { + var newMappings = new Dictionary<Uri, string>(); + foreach (var kv in sourceToTargetMappings) + { + newMappings[new(sourceBaseUri.ToString().TrimEnd('/') + "/" + kv.Key)] = kv.Value; + } + return DownloadFilesAsync( + newMappings, + client, + cancellationToken); + } + + public Task DownloadFilesAsync( + Uri sourceBaseUri, + string targetBasePath, + Dictionary<string, string> sourceToTargetMappings, + HttpClient? client = null, + CancellationToken cancellationToken = default) + { + var newMappings = new Dictionary<Uri, string>(); + foreach (var kv in sourceToTargetMappings) + { + newMappings[new(sourceBaseUri.ToString().TrimEnd('/') + "/" + kv.Key)] = Path.Combine(targetBasePath, kv.Value); + } + return DownloadFilesAsync( + newMappings, + client, + cancellationToken); + } + + public Task DownloadFileAsync( + Uri source, + string target, + HttpClient? client = null, + CancellationToken cancellationToken = default) + { + return DownloadFilesAsync( + new Dictionary<Uri, string> { { source, target } }, + client, + cancellationToken); + } + + public async Task UploadFileAsync( + string source, + Uri target, + HttpClient client, + CancellationToken cancellationToken = default) + { + var localHash = string.Empty; + using (var fileReadStream = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + await _durableOperation.DurableOperationAsync( + async cancellationToken => + { + fileReadStream.Seek(0, SeekOrigin.Begin); + var localHash = $"sha256:{Hash.Sha256AsHexString(fileReadStream)}"; + + _logger.LogInformation($"Checking if {source} needs to be uploaded..."); + { + using var headTimerCts = new CancellationTokenSource(5000); + using var headCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, headTimerCts.Token); + using (var headResponse = await client.SendAsync( + new HttpRequestMessage + { + RequestUri = target, + Headers = + { + { "Intent", "upload" }, + { "Content-Hash", localHash }, + }, + Method = HttpMethod.Head, + }, + HttpCompletionOption.ResponseHeadersRead, + headCts.Token)) + { + if (headResponse.StatusCode == HttpStatusCode.NotModified) + { + return; + } + } + } + + _logger.LogInformation($"Uploading {source}..."); + fileReadStream.Seek(0, SeekOrigin.Begin); + + using (var positionAwareStream = new PositionAwareStream( + fileReadStream, + fileReadStream.Length)) + { + using (var stream = new StallDetectionStream( + positionAwareStream, + TimeSpan.FromSeconds(5))) + { + var cts = new CancellationTokenSource(); + var progress = _progressFactory.CreateProgressForStream(stream); + var monitorTask = Task.Run( + async () => + { + var monitor = _monitorFactory.CreateByteBasedMonitor(); + await monitor.MonitorAsync( + progress, + SystemConsole.ConsoleInformation, + SystemConsole.WriteProgressToConsole, + cts.Token); + }, cts.Token); + + try + { + using var uploadResponse = await client.SendAsync( + new HttpRequestMessage + { + RequestUri = target, + Headers = + { + { "Intent", "upload" }, + { "Content-Hash", localHash }, + }, + Method = HttpMethod.Put, + Content = new StreamContent(stream), + }, + HttpCompletionOption.ResponseContentRead, + cancellationToken); + uploadResponse.EnsureSuccessStatusCode(); + } + finally + { + await SystemConsole.CancelAndWaitForConsoleMonitoringTaskAsync(monitorTask, cts); + } + } + } + }, + cancellationToken); + } + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultFileTransferServer.cs b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultFileTransferServer.cs new file mode 100644 index 00000000..2e0c8489 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DefaultFileTransferServer.cs @@ -0,0 +1,153 @@ +namespace Redpoint.KubernetesManager.PxeBoot.FileTransfer +{ + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.Logging; + using Redpoint.Hashing; + using System.Globalization; + using System.IO; + using System.Net; + using System.Threading.Tasks; + + internal class DefaultFileTransferServer : IFileTransferServer + { + private readonly ILogger<DefaultFileTransferServer> _logger; + + public DefaultFileTransferServer( + ILogger<DefaultFileTransferServer> logger) + { + _logger = logger; + } + + public async Task HandleDownloadFileAsync( + HttpContext httpContext, + Stream contentStream, + bool leaveOpen) + { + try + { + MemoryStream? streamToDispose = null; + long contentLength = -1; + if (contentStream is FileStream fileStream) + { + contentLength = new FileInfo(fileStream.Name).Length; + } + if (contentLength == -1) + { + try + { + contentLength = contentStream.Length; + } + catch + { + } + } + if (!contentStream.CanSeek || contentLength == -1) + { + streamToDispose = new MemoryStream(); + } + try + { + if (streamToDispose != null) + { + await contentStream.CopyToAsync( + streamToDispose, + httpContext.RequestAborted); + streamToDispose.Seek(0, SeekOrigin.Begin); + contentStream = streamToDispose; + contentLength = contentStream.Length; + } + + httpContext.Response.StatusCode = 200; + httpContext.Response.Headers.Add( + "Content-Type", + "application/octet-stream"); + httpContext.Response.Headers.Add( + "Content-Length", + contentLength.ToString(CultureInfo.InvariantCulture)); + + var hash = Hash.Sha256AsHexString(contentStream); + contentStream.Seek(0, SeekOrigin.Begin); + httpContext.Response.Headers.Add("Content-Hash", $"sha256:{hash}"); + + if (!HttpMethods.IsHead(httpContext.Request.Method)) + { + _logger.LogInformation($"Sending data for {httpContext.Request.Path} ({contentLength} bytes)..."); + await contentStream.CopyToAsync( + httpContext.Response.Body, + httpContext.RequestAborted); + } + } + finally + { + if (streamToDispose != null) + { + streamToDispose.Dispose(); + } + } + } + finally + { + if (!leaveOpen) + { + contentStream.Dispose(); + } + } + } + + public async Task HandleUploadFileAsync( + HttpContext httpContext, + string targetPath) + { + if (!httpContext.Request.Headers.TryGetValue("Intent", out var intent) || + intent.Count != 1 || + intent != "upload") + { + httpContext.Response.StatusCode = (int)HttpStatusCode.BadRequest; + return; + } + + if (HttpMethods.IsHead(httpContext.Request.Method)) + { + var existingHash = string.Empty; + if (File.Exists(targetPath)) + { + using (var fileStream = new FileStream(targetPath, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + existingHash = $"sha256:{Hash.Sha256AsHexString(fileStream)}"; + } + } + + if (!httpContext.Response.Headers.TryGetValue("Content-Hash", out var clientHash) || + clientHash.Count != 1 || + string.IsNullOrWhiteSpace(clientHash[0]) || + clientHash[0] != existingHash) + { + httpContext.Response.StatusCode = (int)HttpStatusCode.NoContent; + } + else + { + httpContext.Response.StatusCode = (int)HttpStatusCode.NotModified; + } + return; + } + + var targetDirectoryPath = Path.GetDirectoryName(targetPath); + if (!string.IsNullOrWhiteSpace(targetDirectoryPath)) + { + Directory.CreateDirectory(targetDirectoryPath); + } + using (var fileStream = new FileStream( + targetPath, + FileMode.Create, + FileAccess.Write, + FileShare.None)) + { + await httpContext.Request.Body.CopyToAsync( + fileStream, + httpContext.RequestAborted); + } + + httpContext.Response.StatusCode = (int)HttpStatusCode.OK; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DownloadedFileHashInvalidException.cs b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DownloadedFileHashInvalidException.cs new file mode 100644 index 00000000..e281c0c3 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/DownloadedFileHashInvalidException.cs @@ -0,0 +1,8 @@ +namespace Redpoint.KubernetesManager.PxeBoot.FileTransfer +{ + using System; + + public class DownloadedFileHashInvalidException : Exception + { + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IDurableOperation.cs b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IDurableOperation.cs new file mode 100644 index 00000000..8d55cecf --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IDurableOperation.cs @@ -0,0 +1,16 @@ +namespace Redpoint.KubernetesManager.PxeBoot.FileTransfer +{ + using System; + using System.Threading.Tasks; + + internal interface IDurableOperation + { + Task DurableOperationAsync( + Func<CancellationToken, Task> operation, + CancellationToken cancellationToken); + + Task<TOut> DurableOperationAsync<TOut>( + Func<CancellationToken, Task<TOut>> operation, + CancellationToken cancellationToken); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IFileTransferClient.cs b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IFileTransferClient.cs new file mode 100644 index 00000000..b5456e99 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IFileTransferClient.cs @@ -0,0 +1,41 @@ +namespace Redpoint.KubernetesManager.PxeBoot.FileTransfer +{ + using System; + using System.Collections.Generic; + using System.Net; + using System.Text; + using System.Threading.Tasks; + + internal interface IFileTransferClient + { + Task DownloadFilesAsync( + Dictionary<Uri, string> sourceToTargetMappings, + HttpClient? client = null, + CancellationToken cancellationToken = default); + + Task DownloadFilesAsync( + Uri sourceBaseUri, + Dictionary<string, string> sourceToTargetMappings, + HttpClient? client = null, + CancellationToken cancellationToken = default); + + Task DownloadFilesAsync( + Uri sourceBaseUri, + string targetBasePath, + Dictionary<string, string> sourceToTargetMappings, + HttpClient? client = null, + CancellationToken cancellationToken = default); + + Task DownloadFileAsync( + Uri source, + string target, + HttpClient? client = null, + CancellationToken cancellationToken = default); + + Task UploadFileAsync( + string source, + Uri target, + HttpClient client, + CancellationToken cancellationToken = default); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IFileTransferServer.cs b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IFileTransferServer.cs new file mode 100644 index 00000000..8c53bd6e --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/FileTransfer/IFileTransferServer.cs @@ -0,0 +1,18 @@ +namespace Redpoint.KubernetesManager.PxeBoot.FileTransfer +{ + using Microsoft.AspNetCore.Http; + using System.IO; + using System.Threading.Tasks; + + internal interface IFileTransferServer + { + Task HandleDownloadFileAsync( + HttpContext httpContext, + Stream contentStream, + bool leaveOpen = false); + + Task HandleUploadFileAsync( + HttpContext httpContext, + string targetPath); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootCommand.cs b/UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootCommand.cs new file mode 100644 index 00000000..4eaa9c7a --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootCommand.cs @@ -0,0 +1,30 @@ +namespace Redpoint.KubernetesManager.PxeBoot.NotifyForReboot +{ + using Microsoft.Extensions.DependencyInjection; + using Redpoint.CommandLine; + using Redpoint.KubernetesManager.PxeBoot.Client; + using Redpoint.KubernetesManager.PxeBoot.FileTransfer; + using System.Collections.Generic; + using System.CommandLine; + using System.Linq; + using System.Text; + + internal class PxeBootNotifyForRebootCommand : ICommandDescriptorProvider + { + public static CommandDescriptor Descriptor => CommandDescriptor.NewBuilder() + .WithOptions<PxeBootNotifyForRebootOptions>() + .WithInstance<PxeBootNotifyForRebootCommandInstance>() + .WithCommand( + builder => + { + return new Command("notify-for-reboot", "Call when a reboot step should no longer boot into the custom iPXE script."); + }) + .WithRuntimeServices( + (_, services, _) => + { + services.AddSingleton<IProvisionContextDiscoverer, DefaultProvisionContextDiscoverer>(); + services.AddSingleton<IDurableOperation, DefaultDurableOperation>(); + }) + .Build(); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootCommandInstance.cs b/UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootCommandInstance.cs new file mode 100644 index 00000000..c2e5558f --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootCommandInstance.cs @@ -0,0 +1,55 @@ +namespace Redpoint.KubernetesManager.PxeBoot.NotifyForReboot +{ + using Microsoft.Extensions.Logging; + using Redpoint.CommandLine; + using Redpoint.KubernetesManager.PxeBoot.Client; + using Redpoint.KubernetesManager.PxeBoot.FileTransfer; + using System; + using System.Threading.Tasks; + using System.Web; + + internal class PxeBootNotifyForRebootCommandInstance : ICommandInstance + { + private readonly IProvisionContextDiscoverer _provisionContextDiscoverer; + private readonly IDurableOperation _durableOperation; + private readonly ILogger<PxeBootNotifyForRebootCommandInstance> _logger; + private readonly PxeBootNotifyForRebootOptions _options; + + public PxeBootNotifyForRebootCommandInstance( + IProvisionContextDiscoverer provisionContextDiscoverer, + IDurableOperation durableOperation, + ILogger<PxeBootNotifyForRebootCommandInstance> logger, + PxeBootNotifyForRebootOptions options) + { + _provisionContextDiscoverer = provisionContextDiscoverer; + _durableOperation = durableOperation; + _logger = logger; + _options = options; + } + + public async Task<int> ExecuteAsync(ICommandInvocationContext context) + { + // Get the provisioning context, just for the API address server. + var provisionContext = await _provisionContextDiscoverer.GetProvisionContextAsync( + false, + context.GetCancellationToken()); + + // Notify on an unauthenticated endpoint. This command needs to run in very limited WinPE + // environments that can not use the TPM yet. + await _durableOperation.DurableOperationAsync( + async cancellationToken => + { + using var client = new HttpClient(); + var response = await client.GetAsync( + new Uri($"http://{provisionContext.ApiAddress}:8790/notify-for-reboot?fingerprint={HttpUtility.UrlEncode(context.ParseResult.GetValueForOption(_options.Fingerprint))}"), + cancellationToken); + response.EnsureSuccessStatusCode(); + }, + context.GetCancellationToken()); + _logger.LogInformation("Notified server that we have completed our once-only reboot."); + + // We're done. + return 0; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootOptions.cs b/UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootOptions.cs new file mode 100644 index 00000000..badf6c86 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/NotifyForReboot/PxeBootNotifyForRebootOptions.cs @@ -0,0 +1,9 @@ +namespace Redpoint.KubernetesManager.PxeBoot.NotifyForReboot +{ + using System.CommandLine; + + internal class PxeBootNotifyForRebootOptions + { + public Option<string> Fingerprint = new Option<string>("--fingerprint"); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/DefaultProvisionerHasher.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/DefaultProvisionerHasher.cs new file mode 100644 index 00000000..98420677 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/DefaultProvisionerHasher.cs @@ -0,0 +1,41 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning +{ + using Redpoint.Hashing; + using Redpoint.KubernetesManager.Configuration.Sources; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using System.Collections.Generic; + using System.Text; + using System.Text.Json; + + internal class DefaultProvisionerHasher : IProvisionerHasher + { + private readonly IVariableProvider _variableProvider; + private readonly KubernetesRkmJsonSerializerContext _jsonSerializerContext; + + public DefaultProvisionerHasher( + IVariableProvider variableProvider, + IEnumerable<IProvisioningStep> provisioningSteps) + { + _variableProvider = variableProvider; + _jsonSerializerContext = KubernetesRkmJsonSerializerContext.CreateStringEnumWithAdditionalConverters( + new RkmNodeProvisionerStepJsonConverter(provisioningSteps)); + } + + public string GetProvisionerHash( + ServerSideVariableContext context) + { + var provisionerSpecJson = JsonSerializer.Serialize( + context.RkmNodeProvisioner.Spec, + _jsonSerializerContext.RkmNodeProvisionerSpec); + var effectiveArgumentsJson = JsonSerializer.Serialize( + _variableProvider.ComputeParameterValuesNodeProvisioningEndpoint( + context), + _jsonSerializerContext.DictionaryStringString); + + return Hash.Sha1AsHexString( + provisionerSpecJson + "\n" + effectiveArgumentsJson, + Encoding.UTF8); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/IProvisionerHasher.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/IProvisionerHasher.cs new file mode 100644 index 00000000..91435c80 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/IProvisionerHasher.cs @@ -0,0 +1,14 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning +{ + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using System; + using System.Linq; + using System.Threading.Tasks; + + internal interface IProvisionerHasher + { + string GetProvisionerHash( + ServerSideVariableContext context); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/AtomicSequence/AtomicSequenceProvisioningStep.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/AtomicSequence/AtomicSequenceProvisioningStep.cs new file mode 100644 index 00000000..0c626780 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/AtomicSequence/AtomicSequenceProvisioningStep.cs @@ -0,0 +1,80 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Sequence +{ + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.ExecuteProcess; + using Redpoint.KubernetesManager.PxeBoot.ProvisioningStep; + using Redpoint.RuntimeJson; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading.Tasks; + + internal class AtomicSequenceProvisioningStep : IProvisioningStep<AtomicSequenceProvisioningStepConfig> + { + private readonly ILogger<AtomicSequenceProvisioningStep> _logger; + private readonly IServiceProvider _serviceProvider; + + public AtomicSequenceProvisioningStep( + ILogger<AtomicSequenceProvisioningStep> logger, + IServiceProvider serviceProvider) + { + _logger = logger; + _serviceProvider = serviceProvider; + } + + public string Type => "atomicSequence"; + + public IRuntimeJson GetJsonType(JsonSerializerOptions options) + { + return new ProvisioningStepConfigRuntimeJson(options).AtomicSequenceProvisioningStepConfig; + } + + public ProvisioningStepFlags Flags => ProvisioningStepFlags.None; + + public Task ExecuteOnServerBeforeAsync(AtomicSequenceProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public async Task ExecuteOnClientAsync(AtomicSequenceProvisioningStepConfig config, IProvisioningStepClientContext context, CancellationToken cancellationToken) + { + _logger.LogInformation("Starting atomic sequence of steps..."); + + // @note: We need to get services here and not the constructor, otherwise it's + // a recursive dependency. + var provisioningSteps = _serviceProvider + .GetServices<IProvisioningStep>() + .ToList(); + + foreach (var currentStep in config.Steps ?? []) + { + if (currentStep == null) + { + continue; + } + + _logger.LogInformation($"Starting step '{currentStep.Type}'..."); + var provisioningStep = provisioningSteps.FirstOrDefault(x => string.Equals(x.Type, currentStep?.Type, StringComparison.OrdinalIgnoreCase)); + if (provisioningStep == null) + { + throw new UnableToProvisionSystemException($"Step inside atomic sequence with type '{currentStep.Type}' is not known."); + } + await provisioningStep.ExecuteOnClientUncastedAsync( + currentStep?.DynamicSettings, + context, + cancellationToken); + } + + _logger.LogInformation("Atomic sequence completed successfully."); + } + + public Task ExecuteOnServerAfterAsync(AtomicSequenceProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/AtomicSequence/AtomicSequenceProvisioningStepConfig.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/AtomicSequence/AtomicSequenceProvisioningStepConfig.cs new file mode 100644 index 00000000..a80ad9ca --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/AtomicSequence/AtomicSequenceProvisioningStepConfig.cs @@ -0,0 +1,12 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Sequence +{ + using Redpoint.KubernetesManager.Configuration.Types; + using System.Collections.Generic; + using System.Text.Json.Serialization; + + internal class AtomicSequenceProvisioningStepConfig + { + [JsonPropertyName("steps")] + public IList<RkmNodeProvisionerStep?>? Steps { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/DeleteBootLoaderEntry/DeleteBootLoaderEntryProvisioningStep.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/DeleteBootLoaderEntry/DeleteBootLoaderEntryProvisioningStep.cs new file mode 100644 index 00000000..bea86bfb --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/DeleteBootLoaderEntry/DeleteBootLoaderEntryProvisioningStep.cs @@ -0,0 +1,67 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.DeleteBootLoaderEntry +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Bootmgr; + using Redpoint.RuntimeJson; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal class DeleteBootLoaderEntryProvisioningStep : IProvisioningStep<DeleteBootLoaderEntryProvisioningStepConfig> + { + private readonly ILogger<DeleteBootLoaderEntryProvisioningStep> _logger; + private readonly IEfiBootManager? _efiBootManager; + + public DeleteBootLoaderEntryProvisioningStep( + ILogger<DeleteBootLoaderEntryProvisioningStep> logger, + IEfiBootManager? efiBootManager = null) + { + _logger = logger; + // @note: This is optional because it's not available on the server. + _efiBootManager = efiBootManager; + } + + public string Type => "deleteBootLoaderEntry"; + + public IRuntimeJson GetJsonType(JsonSerializerOptions options) + { + return new ProvisioningStepConfigRuntimeJson(options).DeleteBootLoaderEntryProvisioningStepConfig; + } + + public ProvisioningStepFlags Flags => throw new NotImplementedException(); + + public Task ExecuteOnServerBeforeAsync(DeleteBootLoaderEntryProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public async Task ExecuteOnClientAsync(DeleteBootLoaderEntryProvisioningStepConfig config, IProvisioningStepClientContext context, CancellationToken cancellationToken) + { + if (_efiBootManager == null) + { + throw new UnableToProvisionSystemException("EFI boot manager was not available (unexpected bug)."); + } + + var configuration = await _efiBootManager.GetBootManagerConfigurationAsync(cancellationToken); + + foreach (var entry in configuration.BootEntries) + { + if (string.Equals(entry.Value.Name, config.Name, StringComparison.Ordinal)) + { + _logger.LogInformation($"Removing boot loader entry {entry.Key} '{entry.Value.Name}'..."); + await _efiBootManager.RemoveBootManagerEntryAsync(entry.Key, cancellationToken); + } + } + } + + public Task ExecuteOnServerAfterAsync(DeleteBootLoaderEntryProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/DeleteBootLoaderEntry/DeleteBootLoaderEntryProvisioningStepConfig.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/DeleteBootLoaderEntry/DeleteBootLoaderEntryProvisioningStepConfig.cs new file mode 100644 index 00000000..a1f5774d --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/DeleteBootLoaderEntry/DeleteBootLoaderEntryProvisioningStepConfig.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.DeleteBootLoaderEntry +{ + using System.Text.Json.Serialization; + + internal class DeleteBootLoaderEntryProvisioningStepConfig + { + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/EmptyProvisioningStepConfig.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/EmptyProvisioningStepConfig.cs new file mode 100644 index 00000000..90b24030 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/EmptyProvisioningStepConfig.cs @@ -0,0 +1,6 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step +{ + internal class EmptyProvisioningStepConfig + { + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ExecuteProcess/ExecuteProcessProvisioningStep.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ExecuteProcess/ExecuteProcessProvisioningStep.cs new file mode 100644 index 00000000..12325b18 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ExecuteProcess/ExecuteProcessProvisioningStep.cs @@ -0,0 +1,221 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.ExecuteProcess +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.ProvisioningStep; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using Redpoint.PathResolution; + using Redpoint.ProcessExecution; + using Redpoint.RuntimeJson; + using System; + using System.Collections; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + using YamlDotNet.Core.Tokens; + + internal class ExecuteProcessProvisioningStep : IProvisioningStep<ExecuteProcessProvisioningStepConfig> + { + private readonly IProcessExecutor _processExecutor; + private readonly IPathResolver _pathResolver; + private readonly IVariableProvider _variableProvider; + private readonly ILogger<ExecuteProcessProvisioningStep> _logger; + + public ExecuteProcessProvisioningStep( + IProcessExecutor processExecutor, + IPathResolver pathResolver, + IVariableProvider variableProvider, + ILogger<ExecuteProcessProvisioningStep> logger) + { + _processExecutor = processExecutor; + _pathResolver = pathResolver; + _variableProvider = variableProvider; + _logger = logger; + } + + public string Type => "executeProcess"; + + public IRuntimeJson GetJsonType(JsonSerializerOptions options) + { + return new ProvisioningStepConfigRuntimeJson(options).ExecuteProcessProvisioningStepConfig; + } + + public ProvisioningStepFlags Flags => ProvisioningStepFlags.None; + + public Task ExecuteOnServerBeforeAsync(ExecuteProcessProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + private async Task<int> ExecuteProcessAsync( + ExecuteProcessProvisioningStepConfig config, + CancellationToken cancellationToken) + { + var envVars = new Dictionary<string, string>(); + if (config.InheritEnvironmentVariables) + { + foreach (DictionaryEntry kv in Environment.GetEnvironmentVariables()) + { + if (kv.Key is string key && + kv.Value is string value && + !string.IsNullOrWhiteSpace(key) && + !string.IsNullOrWhiteSpace(value)) + { + envVars[key] = value; + } + } + } + foreach (var kv in config.EnvironmentVariables) + { + envVars[kv.Key] = kv.Value; + } + + IEnumerable<string> arguments = config.Arguments; + string? scriptPath = null; + if (config.Script != null) + { + scriptPath = Path.GetTempFileName() + (OperatingSystem.IsWindows() ? ".ps1" : string.Empty); + await File.WriteAllTextAsync( + scriptPath, + config.Script.Trim(), + cancellationToken); + arguments = config.Arguments.Select(x => x.Replace("{{script-path}}", scriptPath, StringComparison.Ordinal)); + _logger.LogInformation($"Wrote temporary script to '{scriptPath}'."); + } + + try + { + var exitCode = await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = config.Search + ? await _pathResolver.ResolveBinaryPath(config.Executable) + : config.Executable, + Arguments = [.. arguments], + WorkingDirectory = config.WorkingDirectory, + EnvironmentVariables = envVars, + }, + CaptureSpecification.Passthrough, + cancellationToken); + if (exitCode != 0 && !config.IgnoreExitCode) + { + throw new UnableToProvisionSystemException($"Process '{config.Executable}' exited with non-zero exit code {exitCode}."); + } + return exitCode; + } + finally + { + if (scriptPath != null && !OperatingSystem.IsLinux()) + { + File.Delete(scriptPath); + } + } + } + + private async Task<int> ExecuteProcessWithLogMonitoringAsync( + ExecuteProcessProvisioningStepConfig config, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(config.MonitorLogDirectory)) + { + return await ExecuteProcessAsync(config, cancellationToken); + } + + Directory.CreateDirectory(config.MonitorLogDirectory); + + using var stopWatching = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var watchers = new ConcurrentDictionary<string, Task>(); + void HandleWatcherEvent(object sender, FileSystemEventArgs e) + { + if (watchers!.ContainsKey(e.FullPath)) + { + return; + } + + watchers.TryAdd( + e.FullPath, + Task.Run( + async () => + { + try + { + using (var stream = new FileStream(e.FullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete)) + { + using (var reader = new StreamReader(stream)) + { + while (!reader.EndOfStream) + { + var line = await reader.ReadLineAsync(stopWatching.Token); + if (!string.IsNullOrWhiteSpace(line)) + { + line = line.TrimEnd(); + Console.WriteLine($"{Path.GetFileName(e.FullPath)}: {line}"); + } + } + } + } + } + finally + { + watchers.TryRemove(e.FullPath, out _); + } + }, + stopWatching.Token)); + } + + using var watcher = new FileSystemWatcher(config.MonitorLogDirectory); + watcher.NotifyFilter = + NotifyFilters.Attributes | + NotifyFilters.CreationTime | + NotifyFilters.LastWrite | + NotifyFilters.Size; + watcher.Changed += HandleWatcherEvent; + watcher.Created += HandleWatcherEvent; + watcher.Renamed += HandleWatcherEvent; + watcher.Filter = "*"; + watcher.IncludeSubdirectories = false; + watcher.EnableRaisingEvents = true; + + try + { + return await ExecuteProcessAsync(config, cancellationToken); + } + finally + { + stopWatching.Cancel(); + foreach (var task in watchers.Values.ToList()) + { + try + { + await task; + } + catch + { + } + } + } + } + + public async Task ExecuteOnClientAsync(ExecuteProcessProvisioningStepConfig config, IProvisioningStepClientContext context, CancellationToken cancellationToken) + { + foreach (var kv in _variableProvider.GetEnvironmentVariables(context)) + { + config.EnvironmentVariables[kv.Key] = kv.Value; + } + + await ExecuteProcessWithLogMonitoringAsync( + config, + cancellationToken); + } + + public Task ExecuteOnServerAfterAsync(ExecuteProcessProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ExecuteProcess/ExecuteProcessProvisioningStepConfig.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ExecuteProcess/ExecuteProcessProvisioningStepConfig.cs new file mode 100644 index 00000000..c85d68f6 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ExecuteProcess/ExecuteProcessProvisioningStepConfig.cs @@ -0,0 +1,50 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.ExecuteProcess +{ + using System.Text.Json.Serialization; + + internal class ExecuteProcessProvisioningStepConfig + { + [JsonPropertyName("executable")] + public string Executable { get; set; } = string.Empty; + + [JsonPropertyName("search")] + public bool Search { get; set; } = true; + + [JsonPropertyName("arguments")] + public string[] Arguments { get; set; } = []; + + [JsonPropertyName("workingDirectory")] + public string? WorkingDirectory { get; set; } + + [JsonPropertyName("ignoreExitCode")] + public bool IgnoreExitCode { get; set; } = false; + + [JsonPropertyName("environmentVariables")] + public Dictionary<string, string> EnvironmentVariables { get; set; } = new(); + + [JsonPropertyName("inheritEnvironmentVariables")] + public bool InheritEnvironmentVariables { get; set; } = true; + + [JsonPropertyName("monitorLogDirectory")] + public string? MonitorLogDirectory { get; set; } + + [JsonPropertyName("script")] + public string? Script { get; set; } + + public ExecuteProcessProvisioningStepConfig Clone() + { + return new ExecuteProcessProvisioningStepConfig + { + Executable = Executable, + Search = Search, + Arguments = Arguments, + WorkingDirectory = WorkingDirectory, + IgnoreExitCode = IgnoreExitCode, + EnvironmentVariables = new(EnvironmentVariables), + InheritEnvironmentVariables = InheritEnvironmentVariables, + MonitorLogDirectory = MonitorLogDirectory, + Script = Script, + }; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStep.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStep.cs new file mode 100644 index 00000000..4a213f61 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStep.cs @@ -0,0 +1,101 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.SetFileContent +{ + using Microsoft.Extensions.Logging; + using Redpoint.IO; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using Redpoint.RuntimeJson; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal class ModifyFilesProvisioningStep : IProvisioningStep<ModifyFilesProvisioningStepConfig> + { + private readonly IVariableProvider _variableProvider; + private readonly ILogger<ModifyFilesProvisioningStep> _logger; + + public ModifyFilesProvisioningStep( + IVariableProvider variableProvider, + ILogger<ModifyFilesProvisioningStep> logger) + { + _variableProvider = variableProvider; + _logger = logger; + } + + public string Type => "modifyFiles"; + + public IRuntimeJson GetJsonType(JsonSerializerOptions options) + { + return new ProvisioningStepConfigRuntimeJson(options).ModifyFilesProvisioningStepConfig; + } + + public ProvisioningStepFlags Flags => ProvisioningStepFlags.None; + + public Task ExecuteOnServerBeforeAsync(ModifyFilesProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public async Task ExecuteOnClientAsync(ModifyFilesProvisioningStepConfig config, IProvisioningStepClientContext context, CancellationToken cancellationToken) + { + foreach (var file in config.Files) + { + if (string.IsNullOrWhiteSpace(file.Path) || + !Path.IsPathFullyQualified(file.Path)) + { + throw new UnableToProvisionSystemException($"Path '{file.Path}' is not fully qualified. Refusing to set file content."); + } + + _logger.LogInformation($"Applying action '{file.Action}' to '{file.Path}'..."); + + switch (file.Action) + { + case ModifyFilesProvisioningStepConfigFileAction.CreateDirectory: + Directory.CreateDirectory(file.Path); + break; + case ModifyFilesProvisioningStepConfigFileAction.Delete: + if (Directory.Exists(file.Path)) + { + await DirectoryAsync.DeleteAsync(file.Path, true); + } + else + { + File.Delete(file.Path); + } + break; + case ModifyFilesProvisioningStepConfigFileAction.SetContents: + { + var directoryName = Path.GetDirectoryName(file.Path); + if (!string.IsNullOrWhiteSpace(directoryName)) + { + Directory.CreateDirectory(directoryName); + } + + var content = file.Content; + if (file.EnableReplacements) + { + content = _variableProvider.SubstituteVariables( + context, + content); + } + + await File.WriteAllTextAsync( + file.Path, + content, + cancellationToken); + break; + } + } + } + } + + public Task ExecuteOnServerAfterAsync(ModifyFilesProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfig.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfig.cs new file mode 100644 index 00000000..69f3afe5 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfig.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.SetFileContent +{ + using System.Text.Json.Serialization; + + internal class ModifyFilesProvisioningStepConfig + { + [JsonPropertyName("files")] + public IList<ModifyFilesProvisioningStepConfigFile> Files { get; set; } = new List<ModifyFilesProvisioningStepConfigFile>(); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfigFile.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfigFile.cs new file mode 100644 index 00000000..e6590ab3 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfigFile.cs @@ -0,0 +1,19 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.SetFileContent +{ + using System.Text.Json.Serialization; + + internal class ModifyFilesProvisioningStepConfigFile + { + [JsonPropertyName("path")] + public string Path { get; set; } = string.Empty; + + [JsonPropertyName("action")] + public ModifyFilesProvisioningStepConfigFileAction Action { get; set; } = ModifyFilesProvisioningStepConfigFileAction.SetContents; + + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; + + [JsonPropertyName("enableReplacements")] + public bool EnableReplacements { get; set; } = false; + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfigFileAction.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfigFileAction.cs new file mode 100644 index 00000000..adec6d35 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ModifyFiles/ModifyFilesProvisioningStepConfigFileAction.cs @@ -0,0 +1,11 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.SetFileContent +{ + internal enum ModifyFilesProvisioningStepConfigFileAction + { + SetContents, + + Delete, + + CreateDirectory, + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ProvisioningStepConfigJsonSerializerContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ProvisioningStepConfigJsonSerializerContext.cs new file mode 100644 index 00000000..7b06104f --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ProvisioningStepConfigJsonSerializerContext.cs @@ -0,0 +1,27 @@ +namespace Redpoint.KubernetesManager.PxeBoot.ProvisioningStep +{ + using Redpoint.KubernetesManager.Configuration.Json; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.DeleteBootLoaderEntry; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.ExecuteProcess; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Reboot; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Sequence; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.SetFileContent; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Test; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.UploadFiles; + using System.Text.Json; + using System.Text.Json.Serialization; + + [JsonSourceGenerationOptions(WriteIndented = true)] + [JsonSerializable(typeof(TestProvisioningStepConfig))] + [JsonSerializable(typeof(EmptyProvisioningStepConfig))] + [JsonSerializable(typeof(RebootProvisioningStepConfig))] + [JsonSerializable(typeof(ExecuteProcessProvisioningStepConfig))] + [JsonSerializable(typeof(AtomicSequenceProvisioningStepConfig))] + [JsonSerializable(typeof(UploadFilesProvisioningStepConfig))] + [JsonSerializable(typeof(ModifyFilesProvisioningStepConfig))] + [JsonSerializable(typeof(DeleteBootLoaderEntryProvisioningStepConfig))] + internal partial class ProvisioningStepConfigJsonSerializerContext : JsonSerializerContext + { + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ProvisioningStepConfigRuntimeJson.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ProvisioningStepConfigRuntimeJson.cs new file mode 100644 index 00000000..7247c13f --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/ProvisioningStepConfigRuntimeJson.cs @@ -0,0 +1,27 @@ +using Redpoint.KubernetesManager.PxeBoot.ProvisioningStep; + +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step +{ + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.DeleteBootLoaderEntry; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.ExecuteProcess; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Reboot; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Sequence; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.SetFileContent; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Test; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.UploadFiles; + using Redpoint.RuntimeJson; + using System.Text.Json.Serialization; + + [RuntimeJsonProvider(typeof(ProvisioningStepConfigJsonSerializerContext))] + [JsonSerializable(typeof(TestProvisioningStepConfig))] + [JsonSerializable(typeof(EmptyProvisioningStepConfig))] + [JsonSerializable(typeof(RebootProvisioningStepConfig))] + [JsonSerializable(typeof(ExecuteProcessProvisioningStepConfig))] + [JsonSerializable(typeof(AtomicSequenceProvisioningStepConfig))] + [JsonSerializable(typeof(UploadFilesProvisioningStepConfig))] + [JsonSerializable(typeof(ModifyFilesProvisioningStepConfig))] + [JsonSerializable(typeof(DeleteBootLoaderEntryProvisioningStepConfig))] + internal partial class ProvisioningStepConfigRuntimeJson + { + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Reboot/RebootProvisioningStep.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Reboot/RebootProvisioningStep.cs new file mode 100644 index 00000000..06983baf --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Reboot/RebootProvisioningStep.cs @@ -0,0 +1,179 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Reboot +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.ProvisioningStep; + using Redpoint.PathResolution; + using Redpoint.ProcessExecution; + using Redpoint.RuntimeJson; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal class RebootProvisioningStep : IProvisioningStep<RebootProvisioningStepConfig> + { + private readonly ILogger<RebootProvisioningStep> _logger; + private readonly IPathResolver _pathResolver; + private readonly IProcessExecutor _processExecutor; + + public RebootProvisioningStep( + ILogger<RebootProvisioningStep> logger, + IPathResolver pathResolver, + IProcessExecutor processExecutor) + { + _logger = logger; + _pathResolver = pathResolver; + _processExecutor = processExecutor; + } + + public string Type => "reboot"; + + public IRuntimeJson GetJsonType(JsonSerializerOptions options) + { + return new ProvisioningStepConfigRuntimeJson(options).RebootProvisioningStepConfig; + } + + public ProvisioningStepFlags Flags => + ProvisioningStepFlags.DoNotStartAutomaticallyNextStepOnCompletion | + ProvisioningStepFlags.AssumeCompleteWhenIpxeScriptFetched | + ProvisioningStepFlags.SetAsRebootStepIndex; + + public Task ExecuteOnServerBeforeAsync( + RebootProvisioningStepConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + // Nothing to do before this step runs. + return Task.CompletedTask; + } + + public async Task ExecuteOnClientAsync( + RebootProvisioningStepConfig config, + IProvisioningStepClientContext context, + CancellationToken cancellationToken) + { + if (context.IsLocalTesting) + { + // Fetch autoexec.ipxe from server to trigger completion. + var autoexecScript = await context.ProvisioningApiClient.GetStringAsync( + new Uri($"{context.ProvisioningApiEndpointHttps}/autoexec.ipxe"), + cancellationToken); + _logger.LogInformation($"Fetched ipxe script instead of rebooting: {autoexecScript}"); + } + else + { + // Reboot the machine. + _logger.LogInformation("Rebooting machine..."); + if (OperatingSystem.IsWindows()) + { + string? shutdown = null; + try + { + shutdown = await _pathResolver.ResolveBinaryPath("shutdown.exe"); + } + catch (FileNotFoundException) + { + } + if (shutdown != null) + { + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = shutdown, + Arguments = ["/g", "/t", "0", "/c", "RKM Provisioning", "/f", "/d", "p:4:1"] + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + else if (File.Exists(@"X:\windows\System32\WindowsPowerShell\v1.0\powershell.exe")) + { + // For some reason, ResolveBinaryPath doesn't find powershell.exe on Windows PE, + // but it's always at the same path. + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = @"X:\windows\System32\WindowsPowerShell\v1.0\powershell.exe", + Arguments = ["-Command", "Restart-Computer -Force"] + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + else + { + // Try to search for powershell.exe. + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("powershell.exe"), + Arguments = ["-Command", "Restart-Computer -Force"] + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + } + else if (OperatingSystem.IsMacOS()) + { + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("shutdown"), + Arguments = ["-r", "now"] + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + else if (OperatingSystem.IsLinux()) + { + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath("systemctl"), + Arguments = ["--message=\"RKM Provisioning\"", "reboot"] + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + else + { + throw new PlatformNotSupportedException(); + } + + // Sleep indefinitely until the machine reboots. + await Task.Delay(-1, cancellationToken); + } + } + + public Task ExecuteOnServerAfterAsync( + RebootProvisioningStepConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + // Nothing to do after this step runs. + return Task.CompletedTask; + } + + public Task<string?> GetIpxeAutoexecScriptOverrideOnServerAsync( + RebootProvisioningStepConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + if (config.OnceViaNotify && (nodeStatus?.Provisioner?.RebootNotificationForOnceViaNotifyOccurred ?? false)) + { + // We have run, no longer override the script. + return Task.FromResult<string?>(null); + } + + if (config.DefaultInitrd) + { + // Script intentionally wants to reboot into the default initrd. + return Task.FromResult<string?>(null); + } + + return Task.FromResult<string?>(config.IpxeScriptTemplate); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Reboot/RebootProvisioningStepConfig.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Reboot/RebootProvisioningStepConfig.cs new file mode 100644 index 00000000..341b8456 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Reboot/RebootProvisioningStepConfig.cs @@ -0,0 +1,16 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Reboot +{ + using System.Text.Json.Serialization; + + internal class RebootProvisioningStepConfig + { + [JsonPropertyName("ipxeScriptTemplate")] + public string IpxeScriptTemplate { get; set; } = string.Empty; + + [JsonPropertyName("defaultInitrd")] + public bool DefaultInitrd { get; set; } = false; + + [JsonPropertyName("onceViaNotify")] + public bool OnceViaNotify { get; set; } = false; + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/RecoveryShell/RecoveryShellProvisioningStep.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/RecoveryShell/RecoveryShellProvisioningStep.cs new file mode 100644 index 00000000..ae9d2d78 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/RecoveryShell/RecoveryShellProvisioningStep.cs @@ -0,0 +1,67 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.RecoveryShell +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.ProvisioningStep; + using Redpoint.PathResolution; + using Redpoint.ProcessExecution; + using Redpoint.RuntimeJson; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal class RecoveryShellProvisioningStep : IProvisioningStep<EmptyProvisioningStepConfig> + { + private readonly ILogger<RecoveryShellProvisioningStep> _logger; + private readonly IProcessExecutor _processExecutor; + private readonly IPathResolver _pathResolver; + + public RecoveryShellProvisioningStep( + ILogger<RecoveryShellProvisioningStep> logger, + IProcessExecutor processExecutor, + IPathResolver pathResolver) + { + _logger = logger; + _processExecutor = processExecutor; + _pathResolver = pathResolver; + } + + public string Type => "recoveryShell"; + + public IRuntimeJson GetJsonType(JsonSerializerOptions options) + { + return new ProvisioningStepConfigRuntimeJson(options).EmptyProvisioningStepConfig; + } + + public ProvisioningStepFlags Flags => ProvisioningStepFlags.None; + + public Task ExecuteOnServerBeforeAsync(EmptyProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + // Nothing to do before this step runs. + return Task.CompletedTask; + } + + public async Task ExecuteOnClientAsync(EmptyProvisioningStepConfig config, IProvisioningStepClientContext context, CancellationToken cancellationToken) + { + _logger.LogWarning("Starting recovery shell as requested by the provisioner..."); + await _processExecutor.ExecuteAsync( + new ProcessSpecification + { + FilePath = await _pathResolver.ResolveBinaryPath(OperatingSystem.IsWindows() ? "powershell.exe" : "bash"), + Arguments = [] + }, + CaptureSpecification.Passthrough, + cancellationToken); + } + + public Task ExecuteOnServerAfterAsync(EmptyProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + // Nothing to do after this step runs. + return Task.CompletedTask; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/RegisterRemoteIp/RegisterRemoteIpProvisioningStep.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/RegisterRemoteIp/RegisterRemoteIpProvisioningStep.cs new file mode 100644 index 00000000..80fd5962 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/RegisterRemoteIp/RegisterRemoteIpProvisioningStep.cs @@ -0,0 +1,66 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.RegisterRemoteIp +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.ProvisioningStep; + using Redpoint.RuntimeJson; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Net; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal class RegisterRemoteIpProvisioningStep : IProvisioningStep<EmptyProvisioningStepConfig> + { + private readonly ILogger<RegisterRemoteIpProvisioningStep> _logger; + + public RegisterRemoteIpProvisioningStep( + ILogger<RegisterRemoteIpProvisioningStep> logger) + { + _logger = logger; + } + + public string Type => "registerRemoteIp"; + + public IRuntimeJson GetJsonType(JsonSerializerOptions options) + { + return new ProvisioningStepConfigRuntimeJson(options).EmptyProvisioningStepConfig; + } + + public ProvisioningStepFlags Flags => ProvisioningStepFlags.None; + + public Task ExecuteOnServerBeforeAsync( + EmptyProvisioningStepConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + // This step is now deprecated as registered IP addresses are automatically + // updated whenever the node requests a reboot or calls /step with initial=true. + _logger.LogWarning($"The '{Type}' step is deprecated and no longer does anything. Please remove it from your provisioner steps."); + return Task.CompletedTask; + } + + public Task ExecuteOnClientAsync( + EmptyProvisioningStepConfig config, + IProvisioningStepClientContext context, + CancellationToken cancellationToken) + { + // Nothing to do on the client. + return Task.CompletedTask; + } + + public Task ExecuteOnServerAfterAsync( + EmptyProvisioningStepConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + // Nothing to do after this step runs. + return Task.CompletedTask; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Test/TestProvisioningStep.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Test/TestProvisioningStep.cs new file mode 100644 index 00000000..6b9bc5da --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Test/TestProvisioningStep.cs @@ -0,0 +1,60 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Test +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.ProvisioningStep; + using Redpoint.RuntimeJson; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + internal class TestProvisioningStep : IProvisioningStep<TestProvisioningStepConfig> + { + private readonly ILogger<TestProvisioningStep> _logger; + + public TestProvisioningStep( + ILogger<TestProvisioningStep> logger) + { + _logger = logger; + } + + public string Type => "test"; + + public IRuntimeJson GetJsonType(JsonSerializerOptions options) + { + return new ProvisioningStepConfigRuntimeJson(options).TestProvisioningStepConfig; + } + + public ProvisioningStepFlags Flags => ProvisioningStepFlags.None; + + public Task ExecuteOnServerBeforeAsync( + TestProvisioningStepConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + _logger.LogInformation($"ServerBefore '{config.Value}'"); + return Task.CompletedTask; + } + + public Task ExecuteOnClientAsync( + TestProvisioningStepConfig config, + IProvisioningStepClientContext context, + CancellationToken cancellationToken) + { + _logger.LogInformation($"Client '{config.Value}'"); + return Task.CompletedTask; + } + + public Task ExecuteOnServerAfterAsync( + TestProvisioningStepConfig config, + RkmNodeStatus nodeStatus, + IProvisioningStepServerContext serverContext, + CancellationToken cancellationToken) + { + _logger.LogInformation($"ServerAfter '{config.Value}'"); + return Task.CompletedTask; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Test/TestProvisioningStepConfig.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Test/TestProvisioningStepConfig.cs new file mode 100644 index 00000000..32463bb9 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/Test/TestProvisioningStepConfig.cs @@ -0,0 +1,10 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Test +{ + using System.Text.Json.Serialization; + + internal class TestProvisioningStepConfig + { + [JsonPropertyName("value")] + public string Value { get; set; } = string.Empty; + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStep.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStep.cs new file mode 100644 index 00000000..8a81ea96 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStep.cs @@ -0,0 +1,78 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.UploadFiles +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.FileTransfer; + using Redpoint.RuntimeJson; + using System; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + using System.Web; + + internal class UploadFilesProvisioningStep : IProvisioningStep<UploadFilesProvisioningStepConfig> + { + private readonly IFileTransferClient _fileTransferClient; + private readonly ILogger<UploadFilesProvisioningStep> _logger; + private readonly IDurableOperation _durableOperation; + + public UploadFilesProvisioningStep( + IFileTransferClient fileTransferClient, + ILogger<UploadFilesProvisioningStep> logger, + IDurableOperation durableOperation) + { + _fileTransferClient = fileTransferClient; + _logger = logger; + _durableOperation = durableOperation; + } + + public string Type => "uploadFiles"; + + public IRuntimeJson GetJsonType(JsonSerializerOptions options) + { + return new ProvisioningStepConfigRuntimeJson(options).UploadFilesProvisioningStepConfig; + } + + public ProvisioningStepFlags Flags => ProvisioningStepFlags.None; + + public Task ExecuteOnServerBeforeAsync(UploadFilesProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public async Task ExecuteOnClientAsync(UploadFilesProvisioningStepConfig config, IProvisioningStepClientContext context, CancellationToken cancellationToken) + { + _logger.LogInformation($"Upload files step with {config.Files?.Count ?? 0} to consider."); + foreach (var file in config.Files ?? []) + { + if (file == null || + string.IsNullOrWhiteSpace(file.Source) || + !File.Exists(file.Source) || + string.IsNullOrWhiteSpace(file.Target)) + { + throw new UnableToProvisionSystemException($"Skipping upload of '{file?.Source}', it may not exist on disk."); + } + + _logger.LogInformation($"Uploading '{file.Source}' as '{file.Target}'..."); + await _durableOperation.DurableOperationAsync( + async cancellationToken => + { + await _fileTransferClient.UploadFileAsync( + file.Source, + new Uri($"{context.ProvisioningApiEndpointHttps}/api/node-provisioning/upload-file?name={HttpUtility.UrlEncode(file.Target)}"), + context.ProvisioningApiClient, + cancellationToken); + }, + cancellationToken); + } + } + + public Task ExecuteOnServerAfterAsync(UploadFilesProvisioningStepConfig config, RkmNodeStatus nodeStatus, IProvisioningStepServerContext serverContext, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStepConfig.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStepConfig.cs new file mode 100644 index 00000000..ad0a2343 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStepConfig.cs @@ -0,0 +1,11 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.UploadFiles +{ + using System.Collections.Generic; + using System.Text.Json.Serialization; + + internal class UploadFilesProvisioningStepConfig + { + [JsonPropertyName("files")] + public IList<UploadFilesProvisioningStepConfigEntry?>? Files { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStepConfigEntry.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStepConfigEntry.cs new file mode 100644 index 00000000..41962ad3 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Provisioning/Step/UploadFiles/UploadFilesProvisioningStepConfigEntry.cs @@ -0,0 +1,13 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.UploadFiles +{ + using System.Text.Json.Serialization; + + internal class UploadFilesProvisioningStepConfigEntry + { + [JsonPropertyName("source")] + public string? Source { get; set; } + + [JsonPropertyName("target")] + public string? Target { get; set; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/PxeBootCommand.cs b/UET/Redpoint.KubernetesManager.PxeBoot/PxeBootCommand.cs new file mode 100644 index 00000000..34fe5eed --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/PxeBootCommand.cs @@ -0,0 +1,23 @@ +namespace Redpoint.KubernetesManager.PxeBoot +{ + using Redpoint.CommandLine; + using Redpoint.KubernetesManager.PxeBoot.Client; + using Redpoint.KubernetesManager.PxeBoot.NotifyForReboot; + using Redpoint.KubernetesManager.PxeBoot.Server; + using System.CommandLine; + + public class PxeBootCommand : ICommandDescriptorProvider + { + public static CommandDescriptor Descriptor => CommandDescriptor.NewBuilder() + .WithCommand( + builder => + { + builder.AddCommand<PxeBootProvisionClientCommand>(); + builder.AddCommand<PxeBootServerCommand>(); + builder.AddCommand<PxeBootNotifyForRebootCommand>(); + + return new Command("pxeboot", "Internal commands for PXE Boot."); + }) + .Build(); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/PxeBootProvisioningServiceCollectionExtensions.cs b/UET/Redpoint.KubernetesManager.PxeBoot/PxeBootProvisioningServiceCollectionExtensions.cs new file mode 100644 index 00000000..f681f124 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/PxeBootProvisioningServiceCollectionExtensions.cs @@ -0,0 +1,43 @@ +namespace Redpoint.KubernetesManager.PxeBoot +{ + using Microsoft.Extensions.DependencyInjection; + using Redpoint.KubernetesManager.PxeBoot.Client; + using Redpoint.KubernetesManager.PxeBoot.FileTransfer; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.DeleteBootLoaderEntry; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.ExecuteProcess; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Reboot; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.RecoveryShell; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.RegisterRemoteIp; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Sequence; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.SetFileContent; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.Test; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step.UploadFiles; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using Redpoint.Tpm; + + internal static class PxeBootProvisioningServiceCollectionExtensions + { + public static void AddPxeBootProvisioning(this IServiceCollection services) + { + services.AddTpm(); + + services.AddSingleton<IProvisioningStep, TestProvisioningStep>(); + services.AddSingleton<IProvisioningStep, RegisterRemoteIpProvisioningStep>(); + services.AddSingleton<IProvisioningStep, RebootProvisioningStep>(); + services.AddSingleton<IProvisioningStep, RecoveryShellProvisioningStep>(); + services.AddSingleton<IProvisioningStep, ExecuteProcessProvisioningStep>(); + services.AddSingleton<IProvisioningStep, AtomicSequenceProvisioningStep>(); + services.AddSingleton<IProvisioningStep, UploadFilesProvisioningStep>(); + services.AddSingleton<IProvisioningStep, ModifyFilesProvisioningStep>(); + services.AddSingleton<IProvisioningStep, DeleteBootLoaderEntryProvisioningStep>(); + + services.AddSingleton<IDurableOperation, DefaultDurableOperation>(); + services.AddSingleton<IFileTransferClient, DefaultFileTransferClient>(); + + services.AddSingleton<IVariableProvider, DefaultVariableProvider>(); + + services.AddSingleton<IProvisionContextDiscoverer, DefaultProvisionContextDiscoverer>(); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Redpoint.KubernetesManager.PxeBoot.csproj b/UET/Redpoint.KubernetesManager.PxeBoot/Redpoint.KubernetesManager.PxeBoot.csproj new file mode 100644 index 00000000..7254d368 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Redpoint.KubernetesManager.PxeBoot.csproj @@ -0,0 +1,21 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <Import Project="$(MSBuildThisFileDirectory)../Lib/Common.Build.props" /> + + <ItemGroup> + <ProjectReference Include="..\Lib\Redpoint.ThirdParty.GitHub.JPMikkers.Dhcp\Redpoint.ThirdParty.GitHub.JPMikkers.Dhcp.csproj" /> + <ProjectReference Include="..\Lib\Redpoint.ThirdParty.Tftp.Net\Redpoint.ThirdParty.Tftp.Net.csproj" /> + <ProjectReference Include="..\Redpoint.Collections\Redpoint.Collections.csproj" /> + <ProjectReference Include="..\Redpoint.CommandLine\Redpoint.CommandLine.csproj" /> + <ProjectReference Include="..\Redpoint.Kestrel\Redpoint.Kestrel.csproj" /> + <ProjectReference Include="..\Redpoint.KubernetesManager.Configuration\Redpoint.KubernetesManager.Configuration.csproj" /> + <ProjectReference Include="..\Redpoint.KubernetesManager.HostedService\Redpoint.KubernetesManager.HostedService.csproj" /> + <ProjectReference Include="..\Redpoint.PathResolution\Redpoint.PathResolution.csproj" /> + <ProjectReference Include="..\Redpoint.ProcessExecution\Redpoint.ProcessExecution.csproj" /> + <ProjectReference Include="..\Redpoint.ProgressMonitor\Redpoint.ProgressMonitor.csproj" /> + <ProjectReference Include="..\Redpoint.RuntimeJson\Redpoint.RuntimeJson.csproj" /> + <ProjectReference Include="..\Redpoint.RuntimeJson.SourceGenerator\Redpoint.RuntimeJson.SourceGenerator.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" /> + <ProjectReference Include="..\Redpoint.Tpm\Redpoint.Tpm.csproj" /> + </ItemGroup> + +</Project> diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/AuthorizeNodeProvisioningEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/AuthorizeNodeProvisioningEndpoint.cs new file mode 100644 index 00000000..17f2ef28 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/AuthorizeNodeProvisioningEndpoint.cs @@ -0,0 +1,134 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Api; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using System; + using System.Collections.Generic; + using System.Diagnostics.CodeAnalysis; + using System.Linq; + using System.Net; + using System.Text; + using System.Text.Json; + using System.Threading.Tasks; + + internal class AuthorizeNodeProvisioningEndpoint : INodeProvisioningEndpoint + { + private readonly IVariableProvider _variableProvider; + private readonly ILogger<AuthorizeNodeProvisioningEndpoint> _logger; + + public AuthorizeNodeProvisioningEndpoint( + IVariableProvider variableProvider, + ILogger<AuthorizeNodeProvisioningEndpoint> logger) + { + _variableProvider = variableProvider; + _logger = logger; + } + + public string Path => "/authorize"; + + public bool RequireNodeObjects => false; + + private static async Task RespondWithUnauthorizedAsync(INodeProvisioningEndpointContext context) + { + context.Response.StatusCode = (int)HttpStatusCode.Unauthorized; + await context.Response.WriteAsync(context.AikFingerprintShort, context.CancellationToken); + return; + } + + public class AuthorizedNodeInfo + { + public required RkmNodeGroup RkmNodeGroup { get; set; } + + public required RkmNodeProvisioner RkmNodeProvisioner { get; set; } + } + + public static async Task<AuthorizedNodeInfo?> CheckIfNodeAuthorizedAsync( + INodeProvisioningEndpointContext context, + RkmNode? node) + { + if (!(node?.Spec?.Authorized ?? false) || + string.IsNullOrWhiteSpace(node?.Spec?.NodeName)) + { + await RespondWithUnauthorizedAsync(context); + return null; + } + + if (string.IsNullOrWhiteSpace(node?.Spec?.NodeGroup)) + { + // Not yet authorized, or not configured properly. + await RespondWithUnauthorizedAsync(context); + return null; + } + + var nodeGroup = await context.ConfigurationSource.GetRkmNodeGroupAsync( + node.Spec.NodeGroup, + context.CancellationToken); + if (nodeGroup?.Spec?.Provisioner == null) + { + // Not yet authorized, or not configured properly. + await RespondWithUnauthorizedAsync(context); + return null; + } + + var nodeProvisioner = await context.ConfigurationSource.GetRkmNodeProvisionerAsync( + nodeGroup.Spec.Provisioner, + context.JsonSerializerContext.RkmNodeProvisioner, + context.CancellationToken); + if (nodeProvisioner == null) + { + // Not yet authorized, or not configured properly. + await RespondWithUnauthorizedAsync(context); + return null; + } + + return new AuthorizedNodeInfo + { + RkmNodeGroup = nodeGroup, + RkmNodeProvisioner = nodeProvisioner, + }; + } + + public async Task HandleRequestAsync(INodeProvisioningEndpointContext context) + { + var request = (await JsonSerializer.DeserializeAsync( + context.Request.Body, + ApiJsonSerializerContext.WithStringEnum.AuthorizeNodeRequest, + context.CancellationToken))!; + + var candidateNode = await context.ConfigurationSource.CreateOrUpdateRkmNodeByAttestationIdentityKeyPemAsync( + context.AikPem, + [RkmNodeRole.Worker], + false, + request.CapablePlatforms, + request.Architecture, + context.CancellationToken); + var authorizedNodeInfo = await CheckIfNodeAuthorizedAsync(context, candidateNode); + if (authorizedNodeInfo == null) + { + return; + } + + var parameterValues = _variableProvider.ComputeParameterValuesNodeProvisioningEndpoint( + ServerSideVariableContext.FromNodeProvisionerWithoutContextLoadedObjects( + context, + candidateNode, + authorizedNodeInfo.RkmNodeGroup, + authorizedNodeInfo.RkmNodeProvisioner)); + + context.Response.StatusCode = (int)HttpStatusCode.OK; + await JsonSerializer.SerializeAsync( + context.Response.Body, + new AuthorizeNodeResponse + { + NodeName = candidateNode.Spec!.NodeName!, + AikFingerprint = context.AikFingerprint, + ParameterValues = parameterValues, + }, + ApiJsonSerializerContext.WithStringEnum.AuthorizeNodeResponse, + context.CancellationToken); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/DefaultNodeProvisioningEndpointContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/DefaultNodeProvisioningEndpointContext.cs new file mode 100644 index 00000000..cd95122b --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/DefaultNodeProvisioningEndpointContext.cs @@ -0,0 +1,140 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using k8s.Models; + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Sources; + using Redpoint.KubernetesManager.Configuration.Types; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Net; + using System.Text; + using System.Threading.Tasks; + + internal class DefaultNodeProvisioningEndpointContext : INodeProvisioningEndpointContext + { + private readonly ILogger _logger; + private readonly DirectoryInfo _storageDirectoryForAllNodes; + + public DefaultNodeProvisioningEndpointContext( + ILogger logger, + HttpContext httpContext, + string aikPem, + string aikFingerprint, + IRkmConfigurationSource configurationSource, + KubernetesRkmJsonSerializerContext jsonSerializerContext, + DirectoryInfo storageDirectoryForAllNodes, + string hostAddress, + int hostHttpPort, + int hostHttpsPort) + { + _logger = logger; + HttpContext = httpContext; + AikPem = aikPem; + AikFingerprint = aikFingerprint; + ConfigurationSource = configurationSource; + JsonSerializerContext = jsonSerializerContext; + _storageDirectoryForAllNodes = storageDirectoryForAllNodes; + HostAddress = hostAddress; + HostHttpPort = hostHttpPort; + HostHttpsPort = hostHttpsPort; + } + + public HttpContext HttpContext { get; } + + public string AikPem { get; } + + public string AikFingerprint { get; } + + public IRkmConfigurationSource ConfigurationSource { get; } + + public RkmNode? RkmNode { get; set; } + + public RkmNodeGroup? RkmNodeGroup { get; set; } + + public RkmNodeProvisioner? RkmNodeGroupProvisioner { get; set; } + + public RkmNodeProvisioner? RkmNodeProvisioner { get; set; } + + public KubernetesRkmJsonSerializerContext JsonSerializerContext { get; } + + public DirectoryInfo NodeFileStorageDirectory + { + get + { + if (string.IsNullOrWhiteSpace(AikFingerprint)) + { + throw new InvalidOperationException(); + } + + return _storageDirectoryForAllNodes.CreateSubdirectory(AikFingerprint); + } + } + + public string HostAddress { get; } + + public int HostHttpPort { get; } + + public int HostHttpsPort { get; } + + public void MarkProvisioningCompleteForNode() + { + if (RkmNode?.Status?.Provisioner == null) + { + return; + } + + RkmNode.Status.LastSuccessfulProvision = new RkmNodeStatusLastSuccessfulProvision + { + Name = RkmNode.Status.Provisioner.Name, + Hash = RkmNode.Status.Provisioner.Hash, + }; + RkmNode.Status.Provisioner = null; + } + + public void UpdateRegisteredIpAddressesForNode() + { + if (RkmNode?.Status == null) + { + return; + } + + var threshold = DateTimeOffset.UtcNow; + var newExpiry = DateTimeOffset.UtcNow.AddDays(1); + + var addresses = new List<IPAddress> + { + HttpContext.Connection.RemoteIpAddress + }; + if (HttpContext.Connection.RemoteIpAddress.IsIPv4MappedToIPv6) + { + addresses.Add(HttpContext.Connection.RemoteIpAddress.MapToIPv4()); + } + + RkmNode.Status.RegisteredIpAddresses ??= new List<RkmNodeStatusRegisteredIpAddress>(); + RkmNode.Status.RegisteredIpAddresses.RemoveAll(x => !x.ExpiresAt.HasValue || x.ExpiresAt.Value < threshold); + + foreach (var addressRaw in addresses) + { + var address = addressRaw.ToString(); + + var existingEntry = RkmNode.Status.RegisteredIpAddresses.FirstOrDefault(x => x.Address == address); + if (existingEntry != null) + { + _logger.LogInformation($"Updating existing expiry of registered IP address '{address}' to {newExpiry}..."); + existingEntry.ExpiresAt = DateTimeOffset.UtcNow.AddDays(1); + } + else + { + _logger.LogInformation($"Adding new entry for registered IP address '{address}' with expiry {newExpiry}..."); + RkmNode.Status.RegisteredIpAddresses.Add(new RkmNodeStatusRegisteredIpAddress + { + Address = address, + ExpiresAt = DateTimeOffset.UtcNow.AddDays(1), + }); + } + } + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/ForceReprovisionFromRecoveryNodeProvisioningEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/ForceReprovisionFromRecoveryNodeProvisioningEndpoint.cs new file mode 100644 index 00000000..f34667ac --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/ForceReprovisionFromRecoveryNodeProvisioningEndpoint.cs @@ -0,0 +1,29 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using Redpoint.KubernetesManager.PxeBoot.Api; + using System.Net; + using System.Text.Json; + using System.Threading.Tasks; + + internal class ForceReprovisionFromRecoveryNodeProvisioningEndpoint : INodeProvisioningEndpoint + { + public string Path => "/force-reprovision"; + + public bool RequireNodeObjects => true; + + public async Task HandleRequestAsync(INodeProvisioningEndpointContext context) + { + await context.ConfigurationSource.UpdateRkmNodeForceReprovisionByAttestationIdentityKeyFingerprintAsync( + context.AikFingerprint, + true, + context.CancellationToken); + + context.Response.StatusCode = (int)HttpStatusCode.OK; + await JsonSerializer.SerializeAsync( + context.Response.Body, + new ForceReprovisionNodeResponse(), + ApiJsonSerializerContext.WithStringEnum.ForceReprovisionNodeResponse, + context.CancellationToken); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/HttpContextProvisioningStepServerContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/HttpContextProvisioningStepServerContext.cs new file mode 100644 index 00000000..ddbc61b1 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/HttpContextProvisioningStepServerContext.cs @@ -0,0 +1,18 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using Microsoft.AspNetCore.Http; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using System.Net; + + internal class HttpContextProvisioningStepServerContext : IProvisioningStepServerContext + { + private readonly HttpContext _httpContext; + + public HttpContextProvisioningStepServerContext(HttpContext httpContext) + { + _httpContext = httpContext; + } + + public IPAddress RemoteIpAddress => _httpContext.Connection.RemoteIpAddress; + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/INodeProvisioningEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/INodeProvisioningEndpoint.cs new file mode 100644 index 00000000..464c0c09 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/INodeProvisioningEndpoint.cs @@ -0,0 +1,13 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using System.Threading.Tasks; + + internal interface INodeProvisioningEndpoint + { + string Path { get; } + + bool RequireNodeObjects { get; } + + Task HandleRequestAsync(INodeProvisioningEndpointContext context); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/INodeProvisioningEndpointContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/INodeProvisioningEndpointContext.cs new file mode 100644 index 00000000..d3517a4a --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/INodeProvisioningEndpointContext.cs @@ -0,0 +1,50 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using k8s.Models; + using Microsoft.AspNetCore.Http; + using Redpoint.KubernetesManager.Configuration.Sources; + using Redpoint.KubernetesManager.Configuration.Types; + using System.Net; + using System.Text.Json.Serialization; + + internal interface INodeProvisioningEndpointContext + { + HttpContext HttpContext { get; } + + HttpRequest Request => HttpContext.Request; + + HttpResponse Response => HttpContext.Response; + + CancellationToken CancellationToken => HttpContext.RequestAborted; + + string AikPem { get; } + + string AikFingerprint { get; } + + string AikFingerprintShort => AikFingerprint.Substring(0, 8); + + IRkmConfigurationSource ConfigurationSource { get; } + + RkmNode? RkmNode { get; } + + RkmNodeGroup? RkmNodeGroup { get; } + + RkmNodeProvisioner? RkmNodeGroupProvisioner { get; } + + RkmNodeProvisioner? RkmNodeProvisioner { get; set; } + + KubernetesRkmJsonSerializerContext JsonSerializerContext { get; } + + DirectoryInfo NodeFileStorageDirectory { get; } + + string HostAddress { get; } + + int HostHttpPort { get; } + + int HostHttpsPort { get; } + + void UpdateRegisteredIpAddressesForNode(); + + void MarkProvisioningCompleteForNode(); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/RebootToDiskNodeProvisioningEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/RebootToDiskNodeProvisioningEndpoint.cs new file mode 100644 index 00000000..c970bd8a --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/RebootToDiskNodeProvisioningEndpoint.cs @@ -0,0 +1,43 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Sources; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using System.Xml.Linq; + + internal class RebootToDiskNodeProvisioningEndpoint : INodeProvisioningEndpoint + { + private readonly ILogger<RebootToDiskNodeProvisioningEndpoint> _logger; + + public RebootToDiskNodeProvisioningEndpoint( + ILogger<RebootToDiskNodeProvisioningEndpoint> logger) + { + _logger = logger; + } + + public string Path => "/reboot-to-disk"; + + public bool RequireNodeObjects => true; + + public async Task HandleRequestAsync(INodeProvisioningEndpointContext context) + { + _logger.LogInformation($"Node {context.RkmNode!.Metadata.Name} is requesting to boot to disk on next PXE boot."); + + context.RkmNode.Status ??= new(); + context.RkmNode.Status.BootToDisk = true; + context.UpdateRegisteredIpAddressesForNode(); + + await context.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + context.AikFingerprint, + context.RkmNode.Status, + context.CancellationToken); + + context.Response.StatusCode = 200; + return; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepBaseNodeProvisioningEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepBaseNodeProvisioningEndpoint.cs new file mode 100644 index 00000000..ac513db4 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepBaseNodeProvisioningEndpoint.cs @@ -0,0 +1,267 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Redpoint.Hashing; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using System.Collections.Generic; + using System.Globalization; + using System.Net; + using System.Text; + using System.Text.Json; + using System.Text.Json.Serialization.Metadata; + using System.Threading.Tasks; + + internal abstract class StepBaseNodeProvisioningEndpoint : INodeProvisioningEndpoint + { + private readonly ILogger _logger; + private readonly Dictionary<string, IProvisioningStep> _provisioningSteps; + private readonly IProvisionerHasher _provisionerHasher; + + public StepBaseNodeProvisioningEndpoint( + IServiceProvider serviceProvider) + { + _logger = serviceProvider.GetRequiredService<ILogger<StepBaseNodeProvisioningEndpoint>>(); + _provisioningSteps = serviceProvider.GetServices<IProvisioningStep>().ToDictionary(k => k.Type, v => v); + _provisionerHasher = serviceProvider.GetRequiredService<IProvisionerHasher>(); + } + + public abstract string Path { get; } + + public bool RequireNodeObjects => true; + + private bool ShouldResetProvisionState(INodeProvisioningEndpointContext context) + { + if (context.RkmNode!.Spec?.ForceReprovision ?? false) + { + _logger.LogInformation($"{context.RkmNode.Metadata.Name} has force reprovision requested."); + return true; + } + + if (context.RkmNode!.Status?.LastSuccessfulProvision == null && + context.RkmNode!.Status?.Provisioner == null) + { + _logger.LogInformation($"{context.RkmNode.Metadata.Name} has no last successful provision."); + return true; + } + + if (context.RkmNodeGroup != null && + context.RkmNodeGroupProvisioner != null && + context.RkmNodeProvisioner != null) + { + if (context.RkmNodeGroupProvisioner.Metadata.Name != context.RkmNodeProvisioner.Metadata.Name) + { + _logger.LogInformation($"{context.RkmNode.Metadata.Name} group changed from provisioner '{context.RkmNodeProvisioner.Metadata.Name}' to provisioner '{context.RkmNodeGroupProvisioner.Metadata.Name}' during provision."); + return true; + } + + var groupProvisionerStepsHash = _provisionerHasher.GetProvisionerHash( + ServerSideVariableContext.FromNodeGroupProvisioner(context)); + if (groupProvisionerStepsHash != context.RkmNode.Status.Provisioner?.Hash) + { + _logger.LogInformation($"{context.RkmNode.Metadata.Name} group assigned provisioner '{context.RkmNodeGroupProvisioner.Metadata.Name}' now has hash '{groupProvisionerStepsHash}', changed from hash '{context.RkmNode.Status.Provisioner?.Hash}' during provision."); + return true; + } + } + + if (context.RkmNodeGroup != null && + context.RkmNodeGroupProvisioner != null && + context.RkmNode.Status.LastSuccessfulProvision != null) + { + if (context.RkmNodeGroupProvisioner.Metadata.Name != context.RkmNode.Status.LastSuccessfulProvision.Name) + { + _logger.LogInformation($"{context.RkmNode.Metadata.Name} group changed from provisioner '{context.RkmNode.Status.LastSuccessfulProvision.Name}' to provisioner '{context.RkmNodeGroupProvisioner.Metadata.Name}' since last successful provision."); + return true; + } + + var groupProvisionerStepsHash = _provisionerHasher.GetProvisionerHash( + ServerSideVariableContext.FromNodeGroupProvisioner(context)); + if (groupProvisionerStepsHash != context.RkmNode.Status.LastSuccessfulProvision.Hash) + { + _logger.LogInformation($"{context.RkmNode.Metadata.Name} group assigned provisioner '{context.RkmNodeGroupProvisioner.Metadata.Name}' now has hash '{groupProvisionerStepsHash}', changed from hash '{context.RkmNode.Status.LastSuccessfulProvision.Hash}' since last successful provision."); + return true; + } + } + + return false; + } + + private static async Task RespondWithRebootAsync(INodeProvisioningEndpointContext context) + { + context.Response.StatusCode = (int)HttpStatusCode.Conflict; + await context.Response.WriteAsync(context.AikFingerprintShort, context.CancellationToken); + return; + } + + private static async Task RespondWithMisconfiguredAsync(INodeProvisioningEndpointContext context) + { + context.Response.StatusCode = (int)HttpStatusCode.UnprocessableEntity; + await context.Response.WriteAsync(context.AikFingerprintShort, context.CancellationToken); + return; + } + + private static Task RespondWithProvisionCompleteAsync(INodeProvisioningEndpointContext context) + { + context.Response.StatusCode = (int)HttpStatusCode.NoContent; + return Task.CompletedTask; + } + + private async Task<bool> ResetProvisioningStateAndReturnTrueIfRebootRequiredAsync(INodeProvisioningEndpointContext context, bool forceReboot) + { + _logger.LogInformation($"Provisioning state is resetting for node {context.RkmNode!.Metadata.Name}."); + context.RkmNode.Status ??= new(); + context.RkmNode.Status.BootToDisk = false; + context.RkmNode.Status.LastSuccessfulProvision = null; + context.RkmNode.Spec ??= new(); + var requireReboot = context.RkmNode.Status.Provisioner != null; + if (context.RkmNodeGroupProvisioner != null && context.RkmNodeGroup != null) + { + context.RkmNode.Status.Provisioner = new RkmNodeStatusProvisioner + { + Name = context.RkmNodeGroupProvisioner.Metadata.Name, + Hash = _provisionerHasher.GetProvisionerHash( + ServerSideVariableContext.FromNodeGroupProvisioner(context)), + CurrentStepIndex = null, + CurrentStepStarted = false, + LastStepCommittedIndex = null, + RebootStepIndex = null, + RebootNotificationForOnceViaNotifyOccurred = null, + }; + context.RkmNodeProvisioner = context.RkmNodeGroupProvisioner; + } + if (context.RkmNode.Spec.ForceReprovision) + { + _logger.LogInformation("Turning off 'force provision' flag..."); + context.RkmNode.Spec.ForceReprovision = false; + await context.ConfigurationSource.UpdateRkmNodeForceReprovisionByAttestationIdentityKeyFingerprintAsync( + context.AikFingerprint, + context.RkmNode.Spec.ForceReprovision, + context.CancellationToken); + } + + // If the node is in an unknown state (i.e. it hasn't just booted into the default initrd environment), then + // it's possible we're hitting this state mid-provision or while the node is in a different environment. Update + // the node state in configuration, and then tell the node it needs to immediately reboot. + if (requireReboot || forceReboot) + { + _logger.LogWarning($"Provisioning state changed while provisioning was already happening for {context.RkmNode!.Metadata.Name}. Telling the node it needs to reboot!"); + context.UpdateRegisteredIpAddressesForNode(); + await context.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + context.AikFingerprint, + context.RkmNode.Status, + context.CancellationToken); + await RespondWithRebootAsync(context); + return true; + } + + return false; + } + + public async Task HandleRequestAsync(INodeProvisioningEndpointContext context) + { + // Check if we should restart provisioning. + if (ShouldResetProvisionState(context)) + { + if (context.RkmNodeGroup == null) + { + _logger.LogError($"Provisioning state needs to reset for node {context.RkmNode!.Metadata.Name}, but this node has no valid node group assigned!"); + await RespondWithMisconfiguredAsync(context); + return; + } + + if (context.RkmNodeGroupProvisioner == null) + { + _logger.LogError($"Provisioning state needs to reset for node {context.RkmNode!.Metadata.Name}, but this node's group has no valid provisioner assigned!"); + await RespondWithMisconfiguredAsync(context); + return; + } + + if (await ResetProvisioningStateAndReturnTrueIfRebootRequiredAsync(context, false)) + { + return; + } + } + + // Check if there is nothing to provision. + if (string.IsNullOrWhiteSpace(context.RkmNode!.Status?.Provisioner?.Name) || + context.RkmNodeProvisioner == null) + { + await RespondWithProvisionCompleteAsync(context); + return; + } + + // Check if we've completed provisioning. + var currentProvisionerStepCount = context.RkmNodeProvisioner.Spec?.Steps?.Count ?? 0; + var currentStepIndex = context.RkmNode.Status.Provisioner.CurrentStepIndex ?? 0; + if (currentProvisionerStepCount <= currentStepIndex) + { + _logger.LogInformation($"Node {context.RkmNode!.Metadata.Name} completed provisioning."); + + context.MarkProvisioningCompleteForNode(); + + await context.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + context.AikFingerprint, + context.RkmNode.Status, + context.CancellationToken); + + await RespondWithProvisionCompleteAsync(context); + return; + } + + // If this is the initial request after boot, check that the node booted into the correct environment. + var isInitial = context.Request.Query.TryGetValue("initial", out var initial) && initial.FirstOrDefault() == "true"; + if (isInitial) + { + if (!context.Request.Query.TryGetValue("bootedFromStepIndex", out var bootedFromStepIndex) || + string.IsNullOrWhiteSpace(bootedFromStepIndex) || + int.Parse(bootedFromStepIndex!, CultureInfo.InvariantCulture) != (context.RkmNode.Status.Provisioner.RebootStepIndex ?? -1)) + { + // The machine didn't boot with the expected autoexec.ipxe script, usually because + // the IP address of the machine during PXE boot and the IP address of the machine + // during provisioning inside initrd is different. When this happens, autoexec.ipxe isn't + // serving the desired script (which could result in the following provisioning steps + // running in the complete wrong environment). + // + // In this case, reset provisioning and force the machine to reboot. + _logger.LogError($"Machine should have booted from step index {(context.RkmNode.Status.Provisioner.RebootStepIndex ?? -1)}, but booted from {bootedFromStepIndex} instead."); + await ResetProvisioningStateAndReturnTrueIfRebootRequiredAsync(context, true); + return; + } + + // Reset the "currentStepIndex" to the index after "lastStepCommittedIndex". This ensures + // that step completion status don't carry over reboots unless explicitly committed. + var lastStepCommittedIndex = context.RkmNode.Status.Provisioner.LastStepCommittedIndex ?? -1; + var rebootStepIndex = context.RkmNode.Status.Provisioner.RebootStepIndex ?? -1; + if (lastStepCommittedIndex < rebootStepIndex) + { + // Last committed step must always at least be the reboot step index. + lastStepCommittedIndex = rebootStepIndex; + } + _logger.LogInformation($"Setting current step to {lastStepCommittedIndex + 1} during initial step fetch. Last committed step index is {lastStepCommittedIndex}, reboot step index is {rebootStepIndex}."); + context.RkmNode.Status.Provisioner.CurrentStepIndex = lastStepCommittedIndex + 1; + context.UpdateRegisteredIpAddressesForNode(); + } + + var currentStep = context.RkmNodeProvisioner.Spec!.Steps![currentStepIndex]; + var provisioningStepImpl = _provisioningSteps[currentStep!.Type]; + + var serverContext = new HttpContextProvisioningStepServerContext(context.HttpContext); + + await HandleStepRequestAsync( + context, + serverContext, + currentStep, + provisioningStepImpl); + } + + protected abstract Task HandleStepRequestAsync( + INodeProvisioningEndpointContext context, + IProvisioningStepServerContext serverContext, + RkmNodeProvisionerStep currentStep, + IProvisioningStep provisioningStepImpl); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepCompleteNodeProvisioningEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepCompleteNodeProvisioningEndpoint.cs new file mode 100644 index 00000000..a6981060 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepCompleteNodeProvisioningEndpoint.cs @@ -0,0 +1,121 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using System.Collections.Generic; + using System.Net; + using System.Text.Json; + using System.Threading.Tasks; + + internal class StepCompleteNodeProvisioningEndpoint : StepBaseNodeProvisioningEndpoint + { + private readonly ILogger<StepCompleteNodeProvisioningEndpoint> _logger; + private readonly Dictionary<string, IProvisioningStep> _provisioningSteps; + + public StepCompleteNodeProvisioningEndpoint( + ILogger<StepCompleteNodeProvisioningEndpoint> logger, + IServiceProvider serviceProvider, + IEnumerable<IProvisioningStep> provisioningSteps) + : base(serviceProvider) + { + _logger = logger; + _provisioningSteps = provisioningSteps.ToDictionary(k => k.Type, v => v); + } + + public override string Path => "/step-complete"; + + protected override async Task HandleStepRequestAsync( + INodeProvisioningEndpointContext context, + IProvisioningStepServerContext serverContext, + RkmNodeProvisionerStep currentStep, + IProvisioningStep provisioningStep) + { + if (!(context.RkmNode!.Status!.Provisioner!.CurrentStepStarted ?? false)) + { + // The /step endpoint must be called first because this step hasn't started. + _logger.LogInformation($"Step {context.RkmNode.Status.Provisioner.CurrentStepIndex} can't be completed, because it hasn't been started yet."); + context.Response.StatusCode = (int)HttpStatusCode.BadRequest; + return; + } + + _logger.LogInformation($"Provisioning: '{context.AikFingerprintShort}' is completing step '{currentStep!.Type}' at index {context.RkmNode.Status.Provisioner!.CurrentStepIndex!.Value}."); + + await provisioningStep.ExecuteOnServerUncastedAfterAsync( + currentStep!.DynamicSettings, + context.RkmNode.Status, + serverContext, + context.CancellationToken); + + // Increment current step index and then update node status. + if (!context.RkmNode.Status.Provisioner.CurrentStepIndex.HasValue) + { + context.RkmNode.Status.Provisioner.CurrentStepIndex = 0; + } + if (provisioningStep.Flags.HasFlag(ProvisioningStepFlags.CommitOnCompletion)) + { + context.RkmNode.Status.Provisioner.LastStepCommittedIndex = context.RkmNode.Status.Provisioner.CurrentStepIndex; + } + context.RkmNode.Status.Provisioner.CurrentStepIndex += 1; + if (context.RkmNode.Status.Provisioner.CurrentStepIndex >= context.RkmNodeProvisioner!.Spec!.Steps!.Count) + { + context.MarkProvisioningCompleteForNode(); + } + else + { + context.RkmNode.Status.Provisioner.CurrentStepStarted = !provisioningStep.Flags.HasFlag(ProvisioningStepFlags.DoNotStartAutomaticallyNextStepOnCompletion); + } + + // If we completed the last step, return 204. Otherwise, serialize the + // next step as if /step had been called. + if (context.RkmNode.Status.Provisioner == null || provisioningStep.Flags.HasFlag(ProvisioningStepFlags.DoNotStartAutomaticallyNextStepOnCompletion)) + { + await context.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + context.AikFingerprint, + context.RkmNode.Status, + context.CancellationToken); + + // Client needs to call /step to get content. + context.Response.StatusCode = (int)HttpStatusCode.PartialContent; + } + else + { + var nextStep = context.RkmNodeProvisioner.Spec!.Steps[context.RkmNode.Status.Provisioner!.CurrentStepIndex!.Value]; + + var nextProvisioningStep = _provisioningSteps[nextStep!.Type]; + + _logger.LogInformation($"Provisioning: '{context.AikFingerprintShort}' is starting step '{nextStep!.Type}' at index {context.RkmNode.Status.Provisioner!.CurrentStepIndex!.Value}."); + + await nextProvisioningStep.ExecuteOnServerUncastedBeforeAsync( + nextStep!.DynamicSettings, + context.RkmNode.Status, + serverContext, + context.CancellationToken); + + if (nextProvisioningStep.Flags.HasFlag(ProvisioningStepFlags.SetAsRebootStepIndex)) + { + // Set the reboot step index if this is a reboot step. This must be done when a reboot step is + // started as part of /step-complete's next handling, since the reboot steps don't "complete" like other steps. + _logger.LogInformation($"Setting reboot step index to {context.RkmNode.Status.Provisioner.CurrentStepIndex}."); + context.RkmNode.Status.Provisioner.RebootStepIndex = context.RkmNode.Status.Provisioner.CurrentStepIndex; + context.RkmNode.Status.Provisioner.RebootNotificationForOnceViaNotifyOccurred = null; + } + + await context.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + context.AikFingerprint, + context.RkmNode.Status, + context.CancellationToken); + + var nextStepSerialized = JsonSerializer.Serialize(nextStep, context.JsonSerializerContext.RkmNodeProvisionerStep); + context.Response.StatusCode = (int)HttpStatusCode.OK; + context.Response.Headers.Add("Content-Type", "application/json"); + using (var writer = new StreamWriter(context.Response.Body)) + { + await writer.WriteAsync(nextStepSerialized); + } + } + return; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepNodeProvisioningEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepNodeProvisioningEndpoint.cs new file mode 100644 index 00000000..7da8d2ba --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/StepNodeProvisioningEndpoint.cs @@ -0,0 +1,77 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Sources; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using System; + using System.Collections.Generic; + using System.Net; + using System.Security.Cryptography; + using System.Text.Json; + using System.Text.Json.Serialization; + using System.Threading.Tasks; + using System.Xml.Linq; + + internal class StepNodeProvisioningEndpoint : StepBaseNodeProvisioningEndpoint + { + private readonly ILogger<StepNodeProvisioningEndpoint> _logger; + + public StepNodeProvisioningEndpoint( + ILogger<StepNodeProvisioningEndpoint> logger, + IServiceProvider serviceProvider) + : base(serviceProvider) + { + _logger = logger; + } + + public override string Path => "/step"; + + protected override async Task HandleStepRequestAsync( + INodeProvisioningEndpointContext context, + IProvisioningStepServerContext serverContext, + RkmNodeProvisionerStep currentStep, + IProvisioningStep provisioningStep) + { + if (!(context.RkmNode!.Status!.Provisioner!.CurrentStepStarted ?? false)) + { + _logger.LogInformation($"Provisioning: '{context.AikFingerprintShort}' is starting step '{currentStep!.Type}' at index {context.RkmNode.Status.Provisioner!.CurrentStepIndex!.Value}."); + + await provisioningStep.ExecuteOnServerUncastedBeforeAsync( + currentStep!.DynamicSettings, + context.RkmNode.Status, + serverContext, + context.CancellationToken); + + context.RkmNode.Status.Provisioner.CurrentStepStarted = true; + + if (provisioningStep.Flags.HasFlag(ProvisioningStepFlags.SetAsRebootStepIndex)) + { + // Set the reboot step index if this is a reboot step. This must be done during /step, since + // the reboot steps don't "complete" like other steps. + _logger.LogInformation($"Setting reboot step index to {context.RkmNode.Status.Provisioner.CurrentStepIndex}."); + context.RkmNode.Status.Provisioner.RebootStepIndex = context.RkmNode.Status.Provisioner.CurrentStepIndex; + context.RkmNode.Status.Provisioner.RebootNotificationForOnceViaNotifyOccurred = null; + } + + await context.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + context.AikFingerprint, + context.RkmNode.Status, + context.CancellationToken); + } + + // Serialize the current step to the client. + // @todo: Replace variables in step config... + var currentStepSerialized = JsonSerializer.Serialize(currentStep, context.JsonSerializerContext.RkmNodeProvisionerStep); + context.Response.StatusCode = (int)HttpStatusCode.OK; + context.Response.Headers.Add("Content-Type", "application/json"); + using (var writer = new StreamWriter(context.Response.Body)) + { + await writer.WriteAsync(currentStepSerialized); + } + return; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/SyncBootEntriesProvisioningEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/SyncBootEntriesProvisioningEndpoint.cs new file mode 100644 index 00000000..452be34e --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/SyncBootEntriesProvisioningEndpoint.cs @@ -0,0 +1,42 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using Redpoint.Collections; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Net; + using System.Text; + using System.Text.Json; + using System.Threading.Tasks; + + internal class SyncBootEntriesProvisioningEndpoint : INodeProvisioningEndpoint + { + public string Path => "/sync-boot-entries"; + + public bool RequireNodeObjects => true; + + public async Task HandleRequestAsync(INodeProvisioningEndpointContext context) + { + var entries = (await JsonSerializer.DeserializeAsync( + context.Request.Body, + context.JsonSerializerContext.ListRkmNodeStatusBootEntry, + context.CancellationToken))!; + + context.RkmNode!.Status ??= new(); + context.RkmNode!.Status.BootEntries = entries; + + await context.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + context.AikFingerprint, + context.RkmNode.Status, + context.CancellationToken); + + context.Response.StatusCode = (int)HttpStatusCode.OK; + context.Response.ContentType = "application/json"; + await JsonSerializer.SerializeAsync( + context.Response.Body, + (context.RkmNode!.Spec?.InactiveBootEntries ?? []).WhereNotNull().ToList(), + context.JsonSerializerContext.IListString, + context.CancellationToken); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/UploadFileNodeProvisioningEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/UploadFileNodeProvisioningEndpoint.cs new file mode 100644 index 00000000..d43d6681 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/NodeProvisioning/UploadFileNodeProvisioningEndpoint.cs @@ -0,0 +1,45 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning +{ + using Microsoft.AspNetCore.Http; + using Redpoint.Hashing; + using Redpoint.KubernetesManager.PxeBoot.FileTransfer; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Net; + using System.Text; + using System.Threading.Tasks; + + internal class UploadFileNodeProvisioningEndpoint : INodeProvisioningEndpoint + { + private readonly IFileTransferServer _fileTransferServer; + + public UploadFileNodeProvisioningEndpoint( + IFileTransferServer fileTransferServer) + { + _fileTransferServer = fileTransferServer; + } + + public string Path => "/upload-file"; + + public bool RequireNodeObjects => true; + + public async Task HandleRequestAsync( + INodeProvisioningEndpointContext context) + { + if (!context.HttpContext.Request.Query.TryGetValue("name", out var names) || + names.Count != 1 || + string.IsNullOrWhiteSpace(names[0])) + { + context.Response.StatusCode = (int)HttpStatusCode.BadRequest; + return; + } + + await _fileTransferServer.HandleUploadFileAsync( + context.HttpContext, + System.IO.Path.Combine( + context.NodeFileStorageDirectory.FullName, + Hash.Sha256AsHexString(names[0]!, Encoding.UTF8))); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/AutoexecUnauthenticatedFileTransferEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/AutoexecUnauthenticatedFileTransferEndpoint.cs new file mode 100644 index 00000000..50c8ef63 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/AutoexecUnauthenticatedFileTransferEndpoint.cs @@ -0,0 +1,272 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.PxeBoot.Provisioning; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning; + using Redpoint.KubernetesManager.PxeBoot.Variable; + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Net; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading.Tasks; + + internal class AutoexecUnauthenticatedFileTransferEndpoint : IUnauthenticatedFileTransferEndpoint + { + private readonly ILogger<AutoexecUnauthenticatedFileTransferEndpoint> _logger; + private readonly IProvisionerHasher _provisionerHasher; + private readonly Dictionary<string, IProvisioningStep> _provisioningSteps; + + public AutoexecUnauthenticatedFileTransferEndpoint( + ILogger<AutoexecUnauthenticatedFileTransferEndpoint> logger, + IEnumerable<IProvisioningStep> provisioningSteps, + IProvisionerHasher provisionerHasher) + { + _logger = logger; + _provisionerHasher = provisionerHasher; + _provisioningSteps = provisioningSteps.ToDictionary( + k => k.Type, + v => v, + StringComparer.OrdinalIgnoreCase); + } + + public string[] Prefixes => ["/autoexec.ipxe", "/autoexec-nodhcp.ipxe"]; + + public async Task<Stream?> GetDownloadStreamAsync(UnauthenticatedFileTransferRequest request, CancellationToken cancellationToken) + { + if (request.IsTftp) + { + if (request.PathPrefix == "/autoexec.ipxe") + { + // Chain the client into HTTP so that we have faster file transfers. + var stream = new MemoryStream(); + using (var writer = new StreamWriter(stream, Encoding.ASCII, leaveOpen: true)) + { + writer.Write( + $""" + #!ipxe + dhcp + chain --replace http://{request.HostAddress}:{request.HostHttpPort}/autoexec-nodhcp.ipxe + """); + } + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + else + { + // autoexec-nodhcp.ipxe should not be accessed over TFTP. + return null; + } + } + + // Return the autoexec.ipxe script. + string script; + if (request.PathPrefix == "/autoexec.ipxe") + { + script = await GetAutoexecScript( + request, + false, + cancellationToken); + } + else + { + script = await GetAutoexecScript( + request, + true, + cancellationToken); + } + { + var stream = new MemoryStream(); + using (var writer = new StreamWriter(stream, Encoding.ASCII, leaveOpen: true)) + { + writer.Write(script); + } + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + } + + private async Task<string> GetAutoexecScript( + UnauthenticatedFileTransferRequest request, + bool skipDhcp, + CancellationToken cancellationToken) + { + var defaultScript = + $$$""" + #!ipxe + {{dhcp}} + kernel static/vmlinuz rkm-api-address={{provisioner-api-address}} rkm-booted-from-step-index={{booted-from-step-index}} + initrd static/initrd + initrd static/uet /usr/bin/uet-bootstrap mode=555 + boot + """; + + var node = await request.ConfigurationSource.GetRkmNodeByRegisteredIpAddressAsync( + request.RemoteAddress.ToString(), + cancellationToken); + + async Task<string> GetSelectedScript() + { + if (node == null) + { + return defaultScript; + } + if (string.IsNullOrWhiteSpace(node?.Status?.Provisioner?.Name) || + string.IsNullOrWhiteSpace(node?.Spec?.NodeGroup)) + { + _logger.LogInformation("Returning default initrd script as node is not currently provisioning or does not have a node group set."); + return defaultScript; + } + var group = await request.ConfigurationSource.GetRkmNodeGroupAsync( + node.Spec.NodeGroup, + cancellationToken); + if (group == null) + { + _logger.LogInformation("Returning default initrd script as node's group could not be found."); + return defaultScript; + } + var provisioner = await request.ConfigurationSource.GetRkmNodeProvisionerAsync( + node.Status.Provisioner.Name, + request.JsonSerializerContext.RkmNodeProvisioner, + cancellationToken); + if (provisioner == null) + { + _logger.LogInformation("Returning default initrd script as node's provisioner could not be found."); + return defaultScript; + } + var groupProvisioner = provisioner; + if (group.Spec?.Provisioner != null && + node.Status.Provisioner.Name != group.Spec.Provisioner) + { + groupProvisioner = await request.ConfigurationSource.GetRkmNodeProvisionerAsync( + group.Spec.Provisioner, + request.JsonSerializerContext.RkmNodeProvisioner, + cancellationToken); + } + var provisionerHash = _provisionerHasher.GetProvisionerHash( + new ServerSideVariableContext + { + RkmNode = node, + RkmNodeGroup = group, + RkmNodeProvisioner = groupProvisioner ?? provisioner, + ApiHostAddress = request.HostAddress.ToString(), + ApiHostHttpPort = request.HostHttpPort, + ApiHostHttpsPort = request.HostHttpsPort, + }); + if (!string.IsNullOrWhiteSpace(node.Status.Provisioner.Hash) && + provisionerHash != node.Status.Provisioner.Hash) + { + _logger.LogInformation($"Returning default initrd script as node's cached provisioner hash of {node.Status.Provisioner.Hash} does not match actual hash {provisionerHash}."); + return defaultScript; + } + if (!node.Status.Provisioner.RebootStepIndex.HasValue) + { + _logger.LogInformation($"Returning default initrd script as node has not yet hit a reboot step index."); + return defaultScript; + } + var provisionerStepCount = provisioner.Spec?.Steps?.Count ?? 0; + var rebootStepIndex = node.Status.Provisioner.RebootStepIndex ?? 0; + if (provisionerStepCount <= rebootStepIndex) + { + _logger.LogInformation("Returning default initrd script as node's reboot step index exceeds provision step count."); + return defaultScript; + } + + var serverContext = new IpxeProvisioningStepServerContext(request.RemoteAddress); + + var rebootStep = provisioner.Spec!.Steps![rebootStepIndex]; + var provisioningRebootStep = _provisioningSteps[rebootStep!.Type]; + + var overrideScript = await provisioningRebootStep.GetIpxeAutoexecScriptOverrideOnServerUncastedAsync( + rebootStep!.DynamicSettings, + node.Status, + serverContext, + cancellationToken); + if (overrideScript != null) + { + if (string.IsNullOrWhiteSpace(overrideScript) || !overrideScript.StartsWith("#!ipxe", StringComparison.Ordinal)) + { + _logger.LogWarning("Reboot script is not valid, ignoring!"); + return + $$$""" + #!ipxe + {{dhcp}} + echo + echo REBOOT SCRIPT IS INVALID, PLEASE FIX YOUR PROVISIONER STEPS + echo + echo (waiting 30 seconds, then continuing with default initrd environment) + echo + sleep 30 + kernel static/vmlinuz rkm-api-address={{provisioner-api-address}} rkm-booted-from-step-index=-1 + initrd static/initrd + initrd static/uet /usr/bin/uet-bootstrap mode=555 + boot + """; + } + } + + if (provisioningRebootStep.Flags.HasFlag(ProvisioningStepFlags.AssumeCompleteWhenIpxeScriptFetched) && + rebootStepIndex == node.Status.Provisioner.CurrentStepIndex) + { + _logger.LogInformation($"Provisioning: '{node.Status.AttestationIdentityKeyFingerprint?.Substring(0, 8)}' is completing step '{rebootStep!.Type}' at index {rebootStepIndex}."); + + await provisioningRebootStep.ExecuteOnServerUncastedAfterAsync( + rebootStep!.DynamicSettings, + node.Status, + serverContext, + cancellationToken); + + // Increment current step index and then update node status. + if (!node.Status.Provisioner.CurrentStepIndex.HasValue) + { + node.Status.Provisioner.CurrentStepIndex = 0; + } + node.Status.Provisioner.CurrentStepIndex += 1; + if (node.Status.Provisioner.CurrentStepIndex <= node.Status.Provisioner.RebootStepIndex) + { + // Make sure when the client grabs the next step, it's always continuing from the reboot point + // if a later step hasn't committed. + node.Status.Provisioner.CurrentStepIndex = node.Status.Provisioner.RebootStepIndex + 1; + } + if (node.Status.Provisioner.CurrentStepIndex >= provisioner.Spec.Steps.Count) + { + node.Status.Provisioner = null; + } + else + { + node.Status.Provisioner.CurrentStepStarted = false; + } + + // We never automatically start the next step in this context, because we aren't returning + // the next step data to UET; instead we're returning the IPXE script used for booting. + await request.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + node.Status.AttestationIdentityKeyFingerprint!, + node.Status, + cancellationToken); + } + + return overrideScript ?? defaultScript; + } + + var bootedFromStepIndex = (node?.Status?.Provisioner?.RebootStepIndex ?? -1).ToString(CultureInfo.InvariantCulture); + _logger.LogInformation($"Informing machine that they are booting from step index {bootedFromStepIndex}."); + + var replacements = new Dictionary<string, string> + { + { "booted-from-step-index", bootedFromStepIndex }, + { "dhcp", !skipDhcp ? "dhcp" : string.Empty }, + { "provisioner-api-address", request.HostAddress.ToString() }, + }; + var selectedScript = await GetSelectedScript(); + foreach (var kv in replacements) + { + selectedScript = selectedScript.Replace("{{" + kv.Key + "}}", kv.Value, StringComparison.Ordinal); + } + return selectedScript; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/DenyUnauthenticatedFileTransferException.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/DenyUnauthenticatedFileTransferException.cs new file mode 100644 index 00000000..3f5a21fc --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/DenyUnauthenticatedFileTransferException.cs @@ -0,0 +1,6 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + public class DenyUnauthenticatedFileTransferException : Exception + { + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IUnauthenticatedFileTransferEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IUnauthenticatedFileTransferEndpoint.cs new file mode 100644 index 00000000..663d011a --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IUnauthenticatedFileTransferEndpoint.cs @@ -0,0 +1,16 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + using Redpoint.KubernetesManager.Configuration.Sources; + using System.Text.Json.Serialization; + using System.Threading.Tasks; + using Tftp.Net; + + internal interface IUnauthenticatedFileTransferEndpoint + { + string[] Prefixes { get; } + + Task<Stream?> GetDownloadStreamAsync( + UnauthenticatedFileTransferRequest request, + CancellationToken cancellationToken); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IpxeProvisioningStepServerContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IpxeProvisioningStepServerContext.cs new file mode 100644 index 00000000..0f5452e9 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IpxeProvisioningStepServerContext.cs @@ -0,0 +1,17 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using System.Net; + + internal class IpxeProvisioningStepServerContext : IProvisioningStepServerContext + { + private readonly IPAddress _remoteIpAddress; + + public IpxeProvisioningStepServerContext(IPAddress remoteIpAddress) + { + _remoteIpAddress = remoteIpAddress; + } + + public IPAddress RemoteIpAddress => _remoteIpAddress; + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IpxeUnauthenticatedFileTransferEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IpxeUnauthenticatedFileTransferEndpoint.cs new file mode 100644 index 00000000..c563cc38 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/IpxeUnauthenticatedFileTransferEndpoint.cs @@ -0,0 +1,49 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + using Microsoft.Extensions.Logging; + using System.Threading.Tasks; + + internal class IpxeUnauthenticatedFileTransferEndpoint : IUnauthenticatedFileTransferEndpoint + { + private readonly ILogger<IpxeUnauthenticatedFileTransferEndpoint> _logger; + + public IpxeUnauthenticatedFileTransferEndpoint( + ILogger<IpxeUnauthenticatedFileTransferEndpoint> logger) + { + _logger = logger; + } + + public string[] Prefixes => ["/ipxe.efi"]; + + public async Task<Stream?> GetDownloadStreamAsync(UnauthenticatedFileTransferRequest request, CancellationToken cancellationToken) + { + if (request.PathRemaining.HasValue) + { + // Only matches "/ipxe.efi" exactly. + return null; + } + + var node = await request.ConfigurationSource.GetRkmNodeByRegisteredIpAddressAsync( + request.RemoteAddress.ToString(), + CancellationToken.None); + + if (node != null && (node.Status?.BootToDisk ?? false)) + { + _logger.LogInformation("Denying transfer of ipxe.efi because this machine should boot to disk."); + node.Status.BootToDisk = false; + await request.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + node.Status.AttestationIdentityKeyFingerprint!, + node.Status, + CancellationToken.None); + + throw new DenyUnauthenticatedFileTransferException(); + } + + return new FileStream( + Path.Combine(request.StaticFilesDirectory.FullName, "ipxe.efi"), + FileMode.Open, + FileAccess.Read, + FileShare.Read); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/NotifyForRebootUnauthenticatedFileTransferEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/NotifyForRebootUnauthenticatedFileTransferEndpoint.cs new file mode 100644 index 00000000..abdfc4fb --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/NotifyForRebootUnauthenticatedFileTransferEndpoint.cs @@ -0,0 +1,67 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.PxeBoot.Client; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Text.Json; + using System.Threading.Tasks; + + internal class NotifyForRebootUnauthenticatedFileTransferEndpoint : IUnauthenticatedFileTransferEndpoint + { + private readonly ILogger<NotifyForRebootUnauthenticatedFileTransferEndpoint> _logger; + + public NotifyForRebootUnauthenticatedFileTransferEndpoint( + ILogger<NotifyForRebootUnauthenticatedFileTransferEndpoint> logger) + { + _logger = logger; + } + + public string[] Prefixes => ["/notify-for-reboot"]; + + public async Task<Stream?> GetDownloadStreamAsync(UnauthenticatedFileTransferRequest request, CancellationToken cancellationToken) + { + if (request.PathRemaining.HasValue || + request.IsTftp || + request.HttpContext == null) + { + // Only matches "/notify-for-reboot" exactly. + return null; + } + + string aikFingerprint; + if (!request.HttpContext.Request.Query.TryGetValue("fingerprint", out var fingerprints) || + fingerprints.Count != 1 || + string.IsNullOrWhiteSpace(fingerprints[0])) + { + // 'fingerprint' query string isn't valid. + return null; + } + aikFingerprint = fingerprints[0]!; + + var node = await request.ConfigurationSource.GetRkmNodeByAttestationIdentityKeyFingerprintAsync( + aikFingerprint, + CancellationToken.None); + + if (node?.Status?.Provisioner != null) + { + _logger.LogInformation("Node has notified that it has completed a once-only reboot."); + node.Status.Provisioner.RebootNotificationForOnceViaNotifyOccurred = true; + await request.ConfigurationSource.UpdateRkmNodeStatusByAttestationIdentityKeyFingerprintAsync( + node.Status.AttestationIdentityKeyFingerprint!, + node.Status, + CancellationToken.None); + } + + var stream = new MemoryStream(); + using (var writer = new StreamWriter(stream, leaveOpen: true)) + { + writer.Write("ok"); + } + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/RkmProvisionContextUnauthenticatedFileTransferEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/RkmProvisionContextUnauthenticatedFileTransferEndpoint.cs new file mode 100644 index 00000000..bb605844 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/RkmProvisionContextUnauthenticatedFileTransferEndpoint.cs @@ -0,0 +1,36 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + using Redpoint.KubernetesManager.PxeBoot.Client; + using System.Globalization; + using System.Text; + using System.Text.Json; + using System.Threading.Tasks; + + internal class RkmProvisionContextUnauthenticatedFileTransferEndpoint : IUnauthenticatedFileTransferEndpoint + { + public string[] Prefixes => ["/rkm-provision-context.json"]; + + public async Task<Stream?> GetDownloadStreamAsync(UnauthenticatedFileTransferRequest request, CancellationToken cancellationToken) + { + var node = await request.ConfigurationSource.GetRkmNodeByRegisteredIpAddressAsync( + request.RemoteAddress.ToString(), + cancellationToken); + + var bootedFromStepIndex = node?.Status?.Provisioner?.RebootStepIndex ?? -1; + + var stream = new MemoryStream(); + await JsonSerializer.SerializeAsync( + stream, + new WindowsRkmProvisionContext + { + ApiAddress = request.HostAddress.ToString(), + BootedFromStepIndex = bootedFromStepIndex, + IsInRecovery = false, + }, + WindowsRkmProvisionJsonSerializerContext.Default.WindowsRkmProvisionContext, + cancellationToken); + stream.Seek(0, SeekOrigin.Begin); + return stream; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/StaticUnauthenticatedFileTransferEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/StaticUnauthenticatedFileTransferEndpoint.cs new file mode 100644 index 00000000..66c8986a --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/StaticUnauthenticatedFileTransferEndpoint.cs @@ -0,0 +1,67 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + using Microsoft.Extensions.Logging; + using System; + using System.Linq; + using System.Threading.Tasks; + + internal class StaticUnauthenticatedFileTransferEndpoint : IUnauthenticatedFileTransferEndpoint + { + public StaticUnauthenticatedFileTransferEndpoint( + ILogger<StaticUnauthenticatedFileTransferEndpoint> logger) + { + _logger = logger; + } + + public string[] Prefixes => ["/static"]; + + private static readonly string[] _allowedFilenames = + [ + // @note: ipxe.efi is served as both /ipxe.efi and /static/ipxe.efi, but + // only /ipxe.efi denies serving the file if the machine should boot from + // disk. This ensures that the "recovery setup" can always download ipxe.efi + // via the static endpoint. + "ipxe.efi", + "wimboot", + "background.png", + "vmlinuz", + "initrd", + "uet", + "uet.exe", + ]; + private readonly ILogger<StaticUnauthenticatedFileTransferEndpoint> _logger; + + public Task<Stream?> GetDownloadStreamAsync(UnauthenticatedFileTransferRequest request, CancellationToken cancellationToken) + { + if (!request.PathRemaining.HasValue) + { + // No filename specified. + _logger.LogWarning("/static endpoint had no path specified."); + return Task.FromResult<Stream?>(null); + } + + var targetName = request.PathRemaining.Value.TrimStart('/'); + + if (!_allowedFilenames.Contains(targetName)) + { + // Not a permitted filename. + _logger.LogWarning($"/static endpoint received filename '{targetName}', but this is not a permitted filename."); + return Task.FromResult<Stream?>(null); + } + + var targetPath = Path.Combine(request.StaticFilesDirectory.FullName, targetName); + if (!File.Exists(targetPath)) + { + // File does not exist. + _logger.LogWarning($"/static endpoint received filename '{targetName}', but this file does not exist on disk."); + return Task.FromResult<Stream?>(null); + } + + return Task.FromResult<Stream?>(new FileStream( + targetPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read)); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/UnauthenticatedFileTransferRequest.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/UnauthenticatedFileTransferRequest.cs new file mode 100644 index 00000000..daa75c8a --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/UnauthenticatedFileTransferRequest.cs @@ -0,0 +1,33 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + using Microsoft.AspNetCore.Http; + using Redpoint.KubernetesManager.Configuration.Sources; + using System.Net; + + internal class UnauthenticatedFileTransferRequest + { + public required string PathPrefix { get; init; } + + public required PathString PathRemaining { get; init; } + + public required IPAddress RemoteAddress { get; init; } + + public required bool IsTftp { get; init; } + + public required IRkmConfigurationSource ConfigurationSource { get; init; } + + public required DirectoryInfo StaticFilesDirectory { get; init; } + + public required DirectoryInfo StorageFilesDirectory { get; init; } + + public required IPAddress HostAddress { get; init; } + + public required int HostHttpPort { get; init; } + + public required int HostHttpsPort { get; init; } + + public required KubernetesRkmJsonSerializerContext JsonSerializerContext { get; init; } + + public required HttpContext? HttpContext { get; init; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/UploadedUnauthenticatedFileTransferEndpoint.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/UploadedUnauthenticatedFileTransferEndpoint.cs new file mode 100644 index 00000000..f462e039 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Endpoints/UnauthenticatedFileTransfer/UploadedUnauthenticatedFileTransferEndpoint.cs @@ -0,0 +1,48 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer +{ + using Redpoint.Hashing; + using System.Text; + using System.Threading.Tasks; + + internal class UploadedUnauthenticatedFileTransferEndpoint : IUnauthenticatedFileTransferEndpoint + { + public string[] Prefixes => ["/uploaded"]; + + public async Task<Stream?> GetDownloadStreamAsync(UnauthenticatedFileTransferRequest request, CancellationToken cancellationToken) + { + if (!request.PathRemaining.HasValue) + { + // No filename specified. + return null; + } + + var targetName = request.PathRemaining.Value.TrimStart('/'); + + var node = await request.ConfigurationSource.GetRkmNodeByRegisteredIpAddressAsync( + request.RemoteAddress.ToString(), + cancellationToken); + if (node == null || + string.IsNullOrWhiteSpace(node?.Status?.AttestationIdentityKeyFingerprint)) + { + // No node recognised by this IP address, so can't serve any uploaded files. + return null; + } + + var targetPath = Path.Combine( + request.StorageFilesDirectory.FullName, + node.Status.AttestationIdentityKeyFingerprint, + Hash.Sha256AsHexString(targetName, Encoding.UTF8)); + if (!File.Exists(targetPath)) + { + // File does not exist. + return null; + } + + return new FileStream( + targetPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read); + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/DefaultPxeBootHttpRequestHandler.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/DefaultPxeBootHttpRequestHandler.cs new file mode 100644 index 00000000..bb089c3c --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/DefaultPxeBootHttpRequestHandler.cs @@ -0,0 +1,200 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Handlers +{ + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.FileTransfer; + using Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning; + using Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Threading.Tasks; + + internal class DefaultPxeBootHttpRequestHandler : IPxeBootHttpRequestHandler + { + private readonly ILogger<DefaultPxeBootHttpRequestHandler> _logger; + private readonly IFileTransferServer _fileTransferServer; + private readonly List<IUnauthenticatedFileTransferEndpoint> _unauthenticatedFileTransferEndpoints; + private readonly List<INodeProvisioningEndpoint> _nodeProvisioningEndpoints; + + public DefaultPxeBootHttpRequestHandler( + ILogger<DefaultPxeBootHttpRequestHandler> logger, + IFileTransferServer fileTransferServer, + IEnumerable<IUnauthenticatedFileTransferEndpoint> unauthenticatedFileTransferEndpoints, + IEnumerable<INodeProvisioningEndpoint> nodeProvisioningEndpoints) + { + _logger = logger; + _fileTransferServer = fileTransferServer; + _unauthenticatedFileTransferEndpoints = unauthenticatedFileTransferEndpoints.ToList(); + _nodeProvisioningEndpoints = nodeProvisioningEndpoints.ToList(); + } + + public async Task<bool> TryHandleAsUnauthenticatedFileTransfer( + PxeBootServerContext serverContext, + HttpContext httpContext) + { + foreach (var endpoint in _unauthenticatedFileTransferEndpoints) + { + foreach (var prefix in endpoint.Prefixes) + { + _logger.LogTrace($"HTTP: Checking request path '{httpContext.Request.Path}' against '{prefix}'..."); + if (httpContext.Request.Path.StartsWithSegments(prefix, out var remaining)) + { + _logger.LogTrace($"HTTP: Matched path against prefix, with remaining '{remaining}'."); + var request = new UnauthenticatedFileTransferRequest + { + PathPrefix = prefix, + PathRemaining = remaining, + RemoteAddress = httpContext.Connection.RemoteIpAddress, + IsTftp = false, + ConfigurationSource = serverContext.ConfigurationSource, + StaticFilesDirectory = serverContext.StaticFilesDirectory, + StorageFilesDirectory = serverContext.StorageFilesDirectory, + HostAddress = serverContext.HostAddress, + HostHttpPort = serverContext.HostHttpPort, + HostHttpsPort = serverContext.HostHttpsPort, + JsonSerializerContext = serverContext.JsonSerializerContext, + HttpContext = httpContext, + }; + try + { + var stream = await endpoint.GetDownloadStreamAsync( + request, + CancellationToken.None); + if (stream != null) + { + _logger.LogInformation($"HTTP: Successfully returning stream for '{httpContext.Request.Path}'."); + await _fileTransferServer.HandleDownloadFileAsync( + httpContext, + stream); + return true; + } + } + catch (DenyUnauthenticatedFileTransferException) + { + _logger.LogInformation($"HTTP: Explicitly denied access to path '{httpContext.Request.Path}'."); + httpContext.Response.StatusCode = (int)HttpStatusCode.Forbidden; + return true; + } + } + } + } + + return false; + } + + private async Task<bool> TryHandleAsNodeProvisioning(PxeBootServerContext serverContext, HttpContext httpContext) + { + if (!httpContext.Request.Path.StartsWithSegments("/api/node-provisioning", out var remaining)) + { + // Not a node provisioning endpoint. + return false; + } + + var tpmSecuredHttpServer = serverContext.GetTpmSecuredHttpServer(); + + if (remaining == "/negotiate-certificate") + { + // Handle negotiation. + await tpmSecuredHttpServer.HandleNegotiationRequestAsync(httpContext); + return true; + } + + var pem = await tpmSecuredHttpServer.GetAikPemVerifiedByClientCertificateAsync(httpContext); + var fingerprint = RkmNodeFingerprint.CreateFromPem(pem); + + var nodeProvisioningEndpoint = _nodeProvisioningEndpoints.FirstOrDefault(x => x.Path == remaining); + if (nodeProvisioningEndpoint == null) + { + // Not a node provisioning endpoint. + return false; + } + + var endpointContext = new DefaultNodeProvisioningEndpointContext( + _logger, + httpContext, + pem, + fingerprint, + serverContext.ConfigurationSource, + serverContext.JsonSerializerContext, + serverContext.StorageFilesDirectory, + serverContext.HostAddress.ToString(), + serverContext.HostHttpPort, + serverContext.HostHttpsPort); + + if (nodeProvisioningEndpoint.RequireNodeObjects) + { + endpointContext.RkmNode = await serverContext.ConfigurationSource.GetRkmNodeByAttestationIdentityKeyPemAsync( + pem, + httpContext.RequestAborted); + var authorizedNodeInfo = await AuthorizeNodeProvisioningEndpoint.CheckIfNodeAuthorizedAsync(endpointContext, endpointContext.RkmNode); + if (authorizedNodeInfo == null) + { + // CheckIfNodeAuthorizedAsync responded with "Unauthorized". + return true; + } + + if (!string.IsNullOrWhiteSpace(endpointContext.RkmNode?.Spec?.NodeGroup)) + { + endpointContext.RkmNodeGroup = await serverContext.ConfigurationSource.GetRkmNodeGroupAsync( + endpointContext.RkmNode.Spec.NodeGroup, + httpContext.RequestAborted); + } + if (!string.IsNullOrWhiteSpace(endpointContext.RkmNodeGroup?.Spec?.Provisioner)) + { + endpointContext.RkmNodeGroupProvisioner = await serverContext.ConfigurationSource.GetRkmNodeProvisionerAsync( + endpointContext.RkmNodeGroup.Spec.Provisioner, + serverContext.JsonSerializerContext.RkmNodeProvisioner, + httpContext.RequestAborted); + } + if (!string.IsNullOrWhiteSpace(endpointContext.RkmNode?.Status?.Provisioner?.Name)) + { + if (endpointContext.RkmNodeGroupProvisioner != null && + endpointContext.RkmNode.Status.Provisioner.Name == endpointContext.RkmNodeGroupProvisioner.Metadata.Name) + { + endpointContext.RkmNodeProvisioner = endpointContext.RkmNodeGroupProvisioner; + } + else + { + endpointContext.RkmNodeProvisioner = await serverContext.ConfigurationSource.GetRkmNodeProvisionerAsync( + endpointContext.RkmNode.Status.Provisioner.Name, + serverContext.JsonSerializerContext.RkmNodeProvisioner, + httpContext.RequestAborted); + } + } + } + + await nodeProvisioningEndpoint.HandleRequestAsync(endpointContext); + return true; + } + + public async Task HandleRequestAsync(PxeBootServerContext serverContext, HttpContext httpContext) + { + _logger.LogInformation($"HTTP: Incoming request from {httpContext.Connection.RemoteIpAddress} for '{httpContext.Request.Path}'."); + + // @note: There's some weird bug in Hyper-V where the source IP address of connections from the VM to the host + // can appear as the host's IP address. This is only dangerous when fetching the /autoexec.ipxe file during + // boot, and can be mitigated by ensuring that the default switch network adapter is after the internal + // network adapter. This scenario should never happen on real bare metal machines. + if (!httpContext.Connection.RemoteIpAddress.Equals(httpContext.Connection.LocalIpAddress)) + { + if (await TryHandleAsUnauthenticatedFileTransfer(serverContext, httpContext)) + { + return; + } + } + + if (await TryHandleAsNodeProvisioning(serverContext, httpContext)) + { + return; + } + + _logger.LogWarning($"HTTP: No handler found for '{httpContext.Request.Path}'."); + httpContext.Response.StatusCode = 404; + return; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/DefaultPxeBootTftpRequestHandler.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/DefaultPxeBootTftpRequestHandler.cs new file mode 100644 index 00000000..81c0ff5a --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/DefaultPxeBootTftpRequestHandler.cs @@ -0,0 +1,115 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Handlers +{ + using k8s.Models; + using Microsoft.AspNetCore.Http; + using Microsoft.Extensions.Logging; + using Redpoint.CommandLine; + using Redpoint.KubernetesManager.Configuration.Sources; + using Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer; + using Redpoint.Tpm; + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Text; + using System.Text.Json.Serialization; + using System.Threading.Tasks; + using Tftp.Net; + + internal class DefaultPxeBootTftpRequestHandler : IPxeBootTftpRequestHandler + { + private readonly ILogger<DefaultPxeBootTftpRequestHandler> _logger; + private readonly List<IUnauthenticatedFileTransferEndpoint> _endpoints; + + public DefaultPxeBootTftpRequestHandler( + ILogger<DefaultPxeBootTftpRequestHandler> logger, + IEnumerable<IUnauthenticatedFileTransferEndpoint> endpoints) + { + _logger = logger; + _endpoints = endpoints.ToList(); + } + + public async Task HandleRequestAsync( + PxeBootServerContext serverContext, + ITftpTransfer transfer, + EndPoint client) + { + try + { + var path = new PathString('/' + transfer.Filename.TrimStart('/')); + var remoteAddress = ((IPEndPoint)client).Address; + + _logger.LogInformation($"TFTP: Incoming request from {remoteAddress} for '{path}'."); + transfer.OnProgress.Add((args, _) => + { + _logger.LogTrace($"TFTP: Transfer progress: {args.Progress.TransferredBytes} / {args.Progress.TotalBytes}"); + return Task.CompletedTask; + }); + transfer.OnFinished.Add((args, _) => + { + _logger.LogInformation($"TFTP: Transfer finished."); + return Task.CompletedTask; + }); + transfer.OnError.Add((args, _) => + { + _logger.LogInformation($"TFTP: Transfer error: {args.Error}"); + return Task.CompletedTask; + }); + + foreach (var endpoint in _endpoints) + { + foreach (var prefix in endpoint.Prefixes) + { + _logger.LogTrace($"TFTP: Checking request path '{path}' against '{prefix}'..."); + if (path.StartsWithSegments(prefix, out var remaining)) + { + _logger.LogTrace($"TFTP: Matched path against prefix, with remaining '{remaining}'."); + var request = new UnauthenticatedFileTransferRequest + { + PathPrefix = prefix, + PathRemaining = remaining, + RemoteAddress = remoteAddress, + IsTftp = true, + ConfigurationSource = serverContext.ConfigurationSource, + StaticFilesDirectory = serverContext.StaticFilesDirectory, + StorageFilesDirectory = serverContext.StorageFilesDirectory, + HostAddress = serverContext.HostAddress, + HostHttpPort = serverContext.HostHttpPort, + HostHttpsPort = serverContext.HostHttpsPort, + JsonSerializerContext = serverContext.JsonSerializerContext, + HttpContext = null, + }; + try + { + var stream = await endpoint.GetDownloadStreamAsync( + request, + CancellationToken.None); + if (stream != null) + { + _logger.LogInformation($"TFTP: Successfully returning stream for '{path}'."); + transfer.Start(stream); + return; + } + } + catch (DenyUnauthenticatedFileTransferException) + { + _logger.LogInformation($"TFTP: Explicitly denied access to path '{path}'."); + transfer.Cancel(TftpErrorPacket.AccessViolation); + return; + } + } + } + } + + _logger.LogWarning($"TFTP: No result stream found for '{path}'."); + transfer.Cancel(TftpErrorPacket.FileNotFound); + } + catch (Exception ex) + { + _logger.LogError(ex, ex.Message); + transfer.Cancel(TftpErrorPacket.IllegalOperation); + } + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/IPxeBootHttpRequestHandler.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/IPxeBootHttpRequestHandler.cs new file mode 100644 index 00000000..5ee7af05 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/IPxeBootHttpRequestHandler.cs @@ -0,0 +1,12 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Handlers +{ + using Microsoft.AspNetCore.Http; + using System.Threading.Tasks; + + internal interface IPxeBootHttpRequestHandler + { + Task HandleRequestAsync( + PxeBootServerContext serverContext, + HttpContext httpContext); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/IPxeBootTftpRequestHandler.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/IPxeBootTftpRequestHandler.cs new file mode 100644 index 00000000..24301fad --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/IPxeBootTftpRequestHandler.cs @@ -0,0 +1,14 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Handlers +{ + using System.Net; + using System.Threading.Tasks; + using Tftp.Net; + + internal interface IPxeBootTftpRequestHandler + { + Task HandleRequestAsync( + PxeBootServerContext serverContext, + ITftpTransfer transfer, + EndPoint client); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/PxeBootServerContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/PxeBootServerContext.cs new file mode 100644 index 00000000..7a15b600 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/Handlers/PxeBootServerContext.cs @@ -0,0 +1,25 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server.Handlers +{ + using Redpoint.KubernetesManager.Configuration.Sources; + using Redpoint.Tpm; + using System.Net; + + internal class PxeBootServerContext + { + public required IRkmConfigurationSource ConfigurationSource { get; init; } + + public required DirectoryInfo StaticFilesDirectory { get; init; } + + public required DirectoryInfo StorageFilesDirectory { get; init; } + + public required IPAddress HostAddress { get; init; } + + public required int HostHttpPort { get; init; } + + public required int HostHttpsPort { get; init; } + + public required KubernetesRkmJsonSerializerContext JsonSerializerContext { get; init; } + + public required Func<ITpmSecuredHttpServer> GetTpmSecuredHttpServer { get; init; } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/KubernetesWithDeserializeFix.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/KubernetesWithDeserializeFix.cs new file mode 100644 index 00000000..7ce256a0 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/KubernetesWithDeserializeFix.cs @@ -0,0 +1,116 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server +{ + using k8s; + using k8s.Autorest; + using Microsoft.AspNetCore.Http; + using System; + using System.Collections.Generic; + using System.IO; + using System.Net; + using System.Net.Http; + using System.Net.Http.Headers; + using System.Text.Json; + using System.Threading.Tasks; + + internal class KubernetesWithDeserializeFix : Kubernetes + { + public KubernetesWithDeserializeFix(KubernetesClientConfiguration config, params DelegatingHandler[] handlers) : base(config, handlers) + { + } + + protected override Task<HttpResponseMessage> SendRequest<T>(string relativeUri, HttpMethod method, IReadOnlyDictionary<string, IReadOnlyList<string>> customHeaders, T body, CancellationToken cancellationToken) + { + if (body == null) + { + return base.SendRequest<T>(relativeUri, method, customHeaders, body, cancellationToken); + } + + var httpRequest = new HttpRequestMessage + { + Method = method, + RequestUri = new Uri(BaseUri, relativeUri), + }; + httpRequest.Version = HttpVersion.Version20; + + // Set Headers + if (customHeaders != null) + { + foreach (var header in customHeaders) + { + httpRequest.Headers.Remove(header.Key); + httpRequest.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + } + + string requestContent; + if (body is string str) + { + requestContent = str; + } + else if (body is JsonElement jsonElement) + { + using (var jsonStream = new MemoryStream()) + { + using (var jsonWriter = new Utf8JsonWriter(jsonStream)) + { + jsonElement.WriteTo(jsonWriter); + } + jsonStream.Seek(0, SeekOrigin.Begin); + using (var reader = new StreamReader(jsonStream, leaveOpen: true)) + { + requestContent = reader.ReadToEnd(); + } + } + } + else + { + throw new NotSupportedException($"Type ({body.GetType().FullName}) <{typeof(T).FullName}> used for SendRequest, but this Kubernetes implementation expects all calls to use JsonElement or string as the type."); + } + + httpRequest.Content = new StringContent(requestContent, System.Text.Encoding.UTF8); + if (method.Method == HttpMethods.Patch) + { + httpRequest.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/merge-patch+json; charset=utf-8"); + } + else + { + httpRequest.Content.Headers.ContentType = GetHeader(body); + } + return SendRequestRaw(requestContent, httpRequest, cancellationToken); + } + + protected override async Task<HttpOperationResponse<T>> CreateResultAsync<T>(HttpRequestMessage httpRequest, HttpResponseMessage httpResponse, bool? watch, CancellationToken cancellationToken) + { + if (typeof(T) != typeof(JsonElement)) + { + throw new NotSupportedException($"Type {typeof(T).FullName} used for CreateResultAsync, but this Kubernetes implementation expects all calls to use JsonElement as the type."); + } + + ArgumentNullException.ThrowIfNull(httpRequest); + ArgumentNullException.ThrowIfNull(httpResponse); + + var result = new HttpOperationResponse<T>() { Request = httpRequest, Response = httpResponse }; + + if (watch == true) + { + throw new NotSupportedException(); + } + + try + { + using (Stream stream = await httpResponse.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false)) + { + result.Body = (T)(object)JsonDocument.Parse(stream).RootElement; + } + } + catch (JsonException) + { + httpRequest.Dispose(); + httpResponse.Dispose(); + throw; + } + + return result; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootHostedService.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootHostedService.cs new file mode 100644 index 00000000..109b8ebe --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootHostedService.cs @@ -0,0 +1,300 @@ +using k8s.Models; + +namespace Redpoint.KubernetesManager.PxeBoot.Server +{ + using GitHub.JPMikkers.Dhcp; + using k8s; + using Microsoft.AspNetCore.Hosting; + using Microsoft.AspNetCore.Http; + using Microsoft.AspNetCore.Server.Kestrel.Core; + using Microsoft.Extensions.Hosting; + using Microsoft.Extensions.Logging; + using Redpoint.CommandLine; + using Redpoint.Hashing; + using Redpoint.Kestrel; + using Redpoint.KubernetesManager.Configuration.Json; + using Redpoint.KubernetesManager.Configuration.Sources; + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Api; + using Redpoint.KubernetesManager.PxeBoot.FileTransfer; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning; + using Redpoint.KubernetesManager.PxeBoot.Server.Handlers; + using Redpoint.Tpm; + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + using System.Net; + using System.Net.NetworkInformation; + using System.Security.Cryptography; + using System.Security.Cryptography.X509Certificates; + using System.Text; + using System.Text.Json; + using System.Text.Json.Serialization; + using System.Threading; + using System.Threading.Tasks; + using Tftp.Net; + + internal class PxeBootHostedService : IHostedService, IAsyncDisposable, IDhcpMessageInterceptor, IKestrelRequestHandler + { + private readonly ILogger<PxeBootHostedService> _logger; + private readonly IDhcpServerFactory _dhcpServerFactory; + private readonly PxeBootServerOptions _options; + private readonly ICommandInvocationContext _commandInvocationContext; + private readonly IKestrelFactory _kestrelFactory; + private readonly ILoggerFactory _loggerFactory; + private readonly IHostApplicationLifetime _hostApplicationLifetime; + private readonly ITpmSecuredHttp _tpmSecuredHttp; + private readonly IServiceProvider _serviceProvider; + private readonly IPxeBootTftpRequestHandler _pxeBootTftpRequestHandler; + private readonly IPxeBootHttpRequestHandler _pxeBootHttpRequestHandler; + private readonly PxeBootServerContext _serverContext; + private TftpServer? _tftpServer; + private IDhcpServer? _dhcpServer; + private KestrelServer? _kestrelServer; + private ITpmSecuredHttpServer? _tpmSecuredHttpServer; + + private const int _httpPort = 8790; + private const int _httpsPort = 8791; + + public PxeBootHostedService( + ILogger<PxeBootHostedService> logger, + IDhcpServerFactory dhcpServerFactory, + PxeBootServerOptions options, + ICommandInvocationContext commandInvocationContext, + IKestrelFactory kestrelFactory, + IEnumerable<IProvisioningStep> provisioningSteps, + ILoggerFactory loggerFactory, + IHostApplicationLifetime hostApplicationLifetime, + ITpmSecuredHttp tpmSecuredHttp, + IServiceProvider serviceProvider, + IPxeBootTftpRequestHandler pxeBootTftpRequestHandler, + IPxeBootHttpRequestHandler pxeBootHttpRequestHandler) + { + _logger = logger; + _dhcpServerFactory = dhcpServerFactory; + _options = options; + _commandInvocationContext = commandInvocationContext; + _kestrelFactory = kestrelFactory; + _loggerFactory = loggerFactory; + _hostApplicationLifetime = hostApplicationLifetime; + _tpmSecuredHttp = tpmSecuredHttp; + _serviceProvider = serviceProvider; + _pxeBootTftpRequestHandler = pxeBootTftpRequestHandler; + _pxeBootHttpRequestHandler = pxeBootHttpRequestHandler; + + _serverContext = new PxeBootServerContext + { + ConfigurationSource = _commandInvocationContext.ParseResult.GetValueForOption(_options.Source) switch + { + PxeBootServerSource.Test => new TestRkmConfigurationSource( + _loggerFactory.CreateLogger<TestRkmConfigurationSource>()), + PxeBootServerSource.KubernetesDefault => new KubernetesRkmConfigurationSource( + new KubernetesWithDeserializeFix(KubernetesClientConfiguration.BuildDefaultConfig())), + PxeBootServerSource.KubernetesInCluster => new KubernetesRkmConfigurationSource( + new KubernetesWithDeserializeFix(KubernetesClientConfiguration.InClusterConfig())), + _ => throw new InvalidOperationException("Invalid configuration source selected."), + }, + StaticFilesDirectory = _commandInvocationContext.ParseResult.GetValueForOption(_options.StaticFiles)!, + StorageFilesDirectory = _commandInvocationContext.ParseResult.GetValueForOption(_options.StorageFiles)!, + HostAddress = _commandInvocationContext.ParseResult.GetValueForOption(_options.HostAddress) ?? IPAddress.Parse("192.168.0.1"), + HostHttpPort = _httpPort, + HostHttpsPort = _httpsPort, + JsonSerializerContext = KubernetesRkmJsonSerializerContext.CreateStringEnumWithAdditionalConverters( + new RkmNodeProvisionerStepJsonConverter(provisioningSteps)), + GetTpmSecuredHttpServer = () => _tpmSecuredHttpServer!, + }; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + if (_tftpServer != null) + { + await _tftpServer.DisposeAsync(); + } + + _tftpServer = new TftpServer(IPAddress.Any); + _tftpServer.OnReadRequest.Add(OnTftpReadRequest); + _tftpServer.OnWriteRequest.Add(OnTftpWriteRequest); + _tftpServer.OnError.Add(OnTftpError); + _tftpServer.Start(); + _logger.LogInformation("TFTP server started."); + + _dhcpServer?.Dispose(); + + var dhcpServerInterfaceName = _commandInvocationContext.ParseResult.GetValueForOption(_options.DhcpOnInterface); + if (!string.IsNullOrWhiteSpace(dhcpServerInterfaceName)) + { + string? foundPhysicalNetworkInterface = null; + var networkInterfaces = NetworkInterface.GetAllNetworkInterfaces(); + foreach (var networkInterface in networkInterfaces) + { + if (networkInterface.Name.Contains("RKM-DHCP", StringComparison.Ordinal)) + { + _logger.LogInformation($"Discovered RKM-DHCP network interface with physical address '{networkInterface.GetPhysicalAddress().ToString()}'."); + foundPhysicalNetworkInterface = networkInterface.GetPhysicalAddress().ToString(); + break; + } + } + if (!string.IsNullOrWhiteSpace(foundPhysicalNetworkInterface)) + { + _dhcpServer = _dhcpServerFactory.Create(); + _dhcpServer.EndPoint = new IPEndPoint(IPAddress.Any, 67); + _dhcpServer.NetworkPrefix = IPAddress.Parse("192.168.0.0"); + _dhcpServer.ServerAddress = IPAddress.Parse("192.168.0.1"); + _dhcpServer.SubnetMask = IPAddress.Parse("255.255.255.0"); + _dhcpServer.PoolStart = IPAddress.Parse("192.168.0.100"); + _dhcpServer.PoolEnd = IPAddress.Parse("192.168.0.200"); + _dhcpServer.LeaseTime = Utils.InfiniteTimeSpan; + _dhcpServer.OfferExpirationTime = TimeSpan.FromSeconds(3600); + _dhcpServer.Interceptors.Add(this); + _dhcpServer.Reservations.Add(new ReservationItem + { + MacTaste = foundPhysicalNetworkInterface, + PoolStart = IPAddress.Parse("192.168.0.1"), + PoolEnd = IPAddress.Parse("192.168.0.1"), + Preempt = true, + }); + _dhcpServer.Start(); + } + else + { + _logger.LogWarning("Can't reserve 192.168.0.1 for this server on DHCP network interface because it can't be found."); + } + } + + if (_kestrelServer != null) + { + await _kestrelServer.StopAsync(cancellationToken); + _kestrelServer.Dispose(); + } + + // @todo: Source certificate authority from somewhere rather than generating it here. + using var certificateAuthorityPrivateKey = RSA.Create(); + var certificateAuthorityCertificateRequest = new CertificateRequest( + "CN=Test Issuing Authority", + certificateAuthorityPrivateKey, + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1); + certificateAuthorityCertificateRequest.CertificateExtensions.Add( + new X509BasicConstraintsExtension( + certificateAuthority: true, + hasPathLengthConstraint: false, + pathLengthConstraint: 0, + critical: true)); + var certificateAuthority = certificateAuthorityCertificateRequest.CreateSelfSigned( + DateTimeOffset.UtcNow.AddDays(-1), + DateTimeOffset.UtcNow.AddDays(1)); + + // @todo: If the certificate authority doesn't change, this also doesn't need to be recreated. + _tpmSecuredHttpServer = _tpmSecuredHttp.CreateHttpServer(certificateAuthority); + + var kestrelOptions = new KestrelServerOptions(); + kestrelOptions.ApplicationServices = _serviceProvider; + kestrelOptions.Limits.MaxRequestBodySize = null; + kestrelOptions.Listen(IPAddress.Any, _httpPort); + kestrelOptions.Listen(IPAddress.Any, _httpsPort, options => + { + options.UseHttps(https => + { + _tpmSecuredHttpServer.ConfigureHttps(https); + }); + }); + + _kestrelServer = await _kestrelFactory.CreateAndStartServerAsync( + kestrelOptions, + this, + cancellationToken); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + if (_tftpServer != null) + { + await _tftpServer.DisposeAsync(); + } + _tftpServer = null; + } + + public async ValueTask DisposeAsync() + { + if (_tftpServer != null) + { + await _tftpServer.DisposeAsync(); + } + _tftpServer = null; + + _dhcpServer?.Dispose(); + _dhcpServer = null; + + _kestrelServer?.Dispose(); + _kestrelServer = null; + } + + #region TFTP Request Handling + + private Task OnTftpError(TftpTransferError error, CancellationToken cancellationToken) + { + _logger.LogError($"TFTP error: {error}"); + return Task.CompletedTask; + } + + private Task OnTftpWriteRequest(TftpServerEventHandlerArgs args, CancellationToken cancellationToken) + { + args.Transfer.Cancel(TftpErrorPacket.AccessViolation); + return Task.CompletedTask; + } + + private Task OnTftpReadRequest(TftpServerEventHandlerArgs args, CancellationToken cancellationToken) + { + return OnTftpReadRequestAsync(args.Transfer, args.EndPoint); + } + + private async Task OnTftpReadRequestAsync(ITftpTransfer transfer, EndPoint client) + { + await _pxeBootTftpRequestHandler.HandleRequestAsync( + _serverContext, + transfer, + client); + } + + #endregion + + #region DHCP Handling + + void IDhcpMessageInterceptor.Apply(DhcpMessage sourceMsg, DhcpMessage targetMsg) + { + var vendorClientMatch = sourceMsg.FindOption<DhcpOptionVendorClassIdentifier>(); + if (vendorClientMatch != null) + { + var vendorClient = Encoding.ASCII.GetString(vendorClientMatch?.Data ?? []).Trim(); + if (vendorClient.StartsWith("HTTPClient", StringComparison.Ordinal)) + { + _logger.LogInformation("DHCP boot from HTTPClient..."); + targetMsg.Options.Add(new DhcpOptionBootFileName($"http://192.168.0.1:{_httpPort}/static/ipxe.efi")); + } + else if (vendorClient.StartsWith("PXEClient", StringComparison.Ordinal)) + { + _logger.LogInformation("DHCP boot from PXEClient..."); + targetMsg.Options.Add(new DhcpOptionTftpServerName("192.168.0.1")); + targetMsg.Options.Add(new DhcpOptionBootFileName("ipxe.efi")); + targetMsg.NextServerIPAddress = IPAddress.Parse("192.168.0.1"); + } + } + } + + #endregion + + #region HTTP Handling + + async Task IKestrelRequestHandler.HandleRequestAsync(HttpContext httpContext) + { + await _pxeBootHttpRequestHandler.HandleRequestAsync( + _serverContext, + httpContext); + } + + #endregion + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerCommand.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerCommand.cs new file mode 100644 index 00000000..2682e5c7 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerCommand.cs @@ -0,0 +1,59 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server +{ + using GitHub.JPMikkers.Dhcp; + using Microsoft.Extensions.DependencyInjection; + using Redpoint.CommandLine; + using Redpoint.Kestrel; + using Redpoint.KubernetesManager.HostedService; + using Redpoint.KubernetesManager.PxeBoot.FileTransfer; + using Redpoint.KubernetesManager.PxeBoot.Provisioning; + using Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning; + using Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.UnauthenticatedFileTransfer; + using Redpoint.KubernetesManager.PxeBoot.Server.Handlers; + using System.CommandLine; + + internal class PxeBootServerCommand : ICommandDescriptorProvider + { + public static CommandDescriptor Descriptor => CommandDescriptor.NewBuilder() + .WithInstance<PxeBootServerCommandInstance>() + .WithOptions<PxeBootServerOptions>() + .WithCommand( + builder => + { + return new Command("server", "Runs the server that serves PXE boot requests and files for provisioning machines."); + }) + .WithRuntimeServices( + (_, services, _) => + { + services.AddRkmHostedServiceEnvironment("rkm-pxeboot"); + services.AddHostedService<PxeBootHostedService>(); + services.AddDhcpServer(); + services.AddKestrelFactory(); + services.AddPxeBootProvisioning(); + + services.AddSingleton<IFileTransferServer, DefaultFileTransferServer>(); + + services.AddSingleton<IPxeBootHttpRequestHandler, DefaultPxeBootHttpRequestHandler>(); + services.AddSingleton<IPxeBootTftpRequestHandler, DefaultPxeBootTftpRequestHandler>(); + + services.AddSingleton<IUnauthenticatedFileTransferEndpoint, IpxeUnauthenticatedFileTransferEndpoint>(); + services.AddSingleton<IUnauthenticatedFileTransferEndpoint, AutoexecUnauthenticatedFileTransferEndpoint>(); + services.AddSingleton<IUnauthenticatedFileTransferEndpoint, StaticUnauthenticatedFileTransferEndpoint>(); + services.AddSingleton<IUnauthenticatedFileTransferEndpoint, IpxeUnauthenticatedFileTransferEndpoint>(); + services.AddSingleton<IUnauthenticatedFileTransferEndpoint, UploadedUnauthenticatedFileTransferEndpoint>(); + services.AddSingleton<IUnauthenticatedFileTransferEndpoint, RkmProvisionContextUnauthenticatedFileTransferEndpoint>(); + services.AddSingleton<IUnauthenticatedFileTransferEndpoint, NotifyForRebootUnauthenticatedFileTransferEndpoint>(); + + services.AddSingleton<INodeProvisioningEndpoint, AuthorizeNodeProvisioningEndpoint>(); + services.AddSingleton<INodeProvisioningEndpoint, RebootToDiskNodeProvisioningEndpoint>(); + services.AddSingleton<INodeProvisioningEndpoint, StepNodeProvisioningEndpoint>(); + services.AddSingleton<INodeProvisioningEndpoint, StepCompleteNodeProvisioningEndpoint>(); + services.AddSingleton<INodeProvisioningEndpoint, ForceReprovisionFromRecoveryNodeProvisioningEndpoint>(); + services.AddSingleton<INodeProvisioningEndpoint, UploadFileNodeProvisioningEndpoint>(); + services.AddSingleton<INodeProvisioningEndpoint, SyncBootEntriesProvisioningEndpoint>(); + + services.AddSingleton<IProvisionerHasher, DefaultProvisionerHasher>(); + }) + .Build(); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerCommandInstance.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerCommandInstance.cs new file mode 100644 index 00000000..769641e7 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerCommandInstance.cs @@ -0,0 +1,23 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server +{ + using Redpoint.CommandLine; + using Redpoint.KubernetesManager.HostedService; + using System.Threading.Tasks; + + internal class PxeBootServerCommandInstance : ICommandInstance + { + private readonly IHostedServiceFromExecutable _hostedServiceFromExecutable; + + public PxeBootServerCommandInstance( + IHostedServiceFromExecutable hostedServiceFromExecutable) + { + _hostedServiceFromExecutable = hostedServiceFromExecutable; + } + + public async Task<int> ExecuteAsync(ICommandInvocationContext context) + { + await _hostedServiceFromExecutable.RunHostedServicesAsync(context.GetCancellationToken()); + return 0; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerOptions.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerOptions.cs new file mode 100644 index 00000000..50ffb858 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerOptions.cs @@ -0,0 +1,31 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server +{ + using System.CommandLine; + using System.Net; + + internal class PxeBootServerOptions + { + public Option<string> DhcpOnInterface = new Option<string>( + "--dhcp", + "If specified, this command will run a DHCP server that will provide addresses to the network on the specified interface. This can be used to simplify testing of PXE boot with virtual machines."); + + public Option<PxeBootServerSource> Source = new Option<PxeBootServerSource>( + "--source", + () => PxeBootServerSource.Test, + "The configuration source."); + + public Option<IPAddress> HostAddress = new Option<IPAddress>( + "--host-address", + "The address that the PXE boot server is listening on; used to notify machines of the API address."); + + public Option<DirectoryInfo> StaticFiles = new Option<DirectoryInfo>( + "--static-files", + () => new DirectoryInfo("/static"), + "The directory that contains the static delivery files."); + + public Option<DirectoryInfo> StorageFiles = new Option<DirectoryInfo>( + "--storage-files", + () => new DirectoryInfo("/storage"), + "The directory under which to store dynamic, temporary data from provisioning nodes."); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerSource.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerSource.cs new file mode 100644 index 00000000..290d18d7 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Server/PxeBootServerSource.cs @@ -0,0 +1,9 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Server +{ + internal enum PxeBootServerSource + { + Test, + KubernetesInCluster, + KubernetesDefault, + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/UnableToProvisionSystemException.cs b/UET/Redpoint.KubernetesManager.PxeBoot/UnableToProvisionSystemException.cs new file mode 100644 index 00000000..76290a40 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/UnableToProvisionSystemException.cs @@ -0,0 +1,12 @@ +namespace Redpoint.KubernetesManager.PxeBoot +{ + using System; + + public class UnableToProvisionSystemException : Exception + { + public UnableToProvisionSystemException(string message) + : base(message) + { + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Variable/DefaultVariableProvider.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Variable/DefaultVariableProvider.cs new file mode 100644 index 00000000..28244555 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Variable/DefaultVariableProvider.cs @@ -0,0 +1,143 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Variable +{ + using Microsoft.Extensions.Logging; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using System; + using System.Collections.Generic; + using static System.CommandLine.Help.HelpBuilder; + + internal class DefaultVariableProvider : IVariableProvider + { + private readonly ILogger<DefaultVariableProvider> _logger; + + public DefaultVariableProvider( + ILogger<DefaultVariableProvider> logger) + { + _logger = logger; + } + + public Dictionary<string, string> ComputeParameterValuesNodeProvisioningEndpoint( + ServerSideVariableContext context) + { + var serverOnlySubstitutions = new Dictionary<string, string> + { + { "provision:nodeName", context.RkmNode.Spec?.NodeName ?? string.Empty }, + { "provision:apiAddressIp", context.ApiHostAddress }, + { "provision:apiAddressHttp", $"http://{context.ApiHostAddress}:{context.ApiHostHttpPort}" }, + { "provision:aikFingerprint", context.RkmNode.Status?.AttestationIdentityKeyFingerprint ?? string.Empty }, + }; + string PerformServerSideSubstitutions(string content) + { + foreach (var substitution in serverOnlySubstitutions) + { + content = content.Replace("{{" + substitution.Key + "}}", substitution.Value, StringComparison.Ordinal); + } + return content; + } + + var parameterValues = new Dictionary<string, string>(); + if (context.RkmNodeProvisioner.Spec?.Parameters != null) + { + foreach (var defaultKv in context.RkmNodeProvisioner.Spec.Parameters) + { + if (defaultKv.Value != null) + { + parameterValues[defaultKv.Key] = PerformServerSideSubstitutions(defaultKv.Value); + } + else + { + _logger.LogWarning($"Provisioner parameter '{defaultKv.Key}' is being ignored because it's value is null (you probably should have set it to an empty string instead)."); + } + } + } + if (context.RkmNodeGroup.Spec?.ProvisionerArguments != null) + { + foreach (var kv in context.RkmNodeGroup.Spec.ProvisionerArguments) + { + // @note: You can only provide arguments for parameters that are actually defined. + // Parameters can have an empty string as a default value though. + if (parameterValues.ContainsKey(kv.Key)) + { + if (kv.Value != null) + { + parameterValues[kv.Key] = PerformServerSideSubstitutions(kv.Value); + } + else + { + _logger.LogWarning($"Provisioner argument '{kv.Key}' is being ignored because it's value is null."); + } + } + else + { + _logger.LogWarning($"Provisioner argument '{kv.Key}' is being ignored because the provisioner does not define it as a parameter."); + } + } + } + + foreach (var parameterKv in parameterValues) + { + _logger.LogInformation($"Node '{context.RkmNode.Spec?.NodeName}' provisioner parameter '{parameterKv.Key}' evaluated as '{parameterKv.Value}'."); + } + + return parameterValues; + } + + private static Dictionary<string, string> GetSubstitutions(IProvisioningStepClientContext context) + { + var substitutions = new Dictionary<string, string> + { + { "provision:nodeName", context.AuthorizedNodeName }, + { "provision:apiAddressIp", context.ProvisioningApiAddress }, + { "provision:apiAddressHttp", context.ProvisioningApiEndpointHttp }, + { "provision:aikFingerprint", context.AikFingerprint }, + }; + foreach (var kv in context.ParameterValues) + { + substitutions.Add($"param:{kv.Key}", kv.Value); + } + return substitutions; + } + + public Dictionary<string, string> GetEnvironmentVariables(IProvisioningStepClientContext context) + { + var environmentVariables = new Dictionary<string, string>(); + + foreach (var substitution in GetSubstitutions(context)) + { + var transformedKey = "RKM_"; + for (int i = 0; i < substitution.Key.Length; i++) + { + var s = substitution.Key[i]; + if (s >= 'A' && s <= 'Z' && i != 0) + { + transformedKey += "_"; + transformedKey += s; + } + else if ((s >= '0' && s <= '9') || (s >= 'a' && s <= 'z')) + { + transformedKey += s; + } + else + { + transformedKey += "_"; + } + } + transformedKey = transformedKey.ToUpperInvariant(); + _logger.LogInformation($"Parameter key transformed from '{substitution.Key}' to '{transformedKey}'."); + environmentVariables.Add(transformedKey, substitution.Value); + } + + return environmentVariables; + } + + public string SubstituteVariables(IProvisioningStepClientContext context, string content) + { + foreach (var substitution in GetSubstitutions(context)) + { + content = content.Replace("{{" + substitution.Key + "}}", substitution.Value, StringComparison.Ordinal); + } + + return content; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Variable/IVariableProvider.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Variable/IVariableProvider.cs new file mode 100644 index 00000000..ebc890ca --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Variable/IVariableProvider.cs @@ -0,0 +1,23 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Variable +{ + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Provisioning.Step; + using Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using static Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning.AuthorizeNodeProvisioningEndpoint; + + internal interface IVariableProvider + { + string SubstituteVariables( + IProvisioningStepClientContext context, + string content); + + Dictionary<string, string> GetEnvironmentVariables( + IProvisioningStepClientContext context); + + Dictionary<string, string> ComputeParameterValuesNodeProvisioningEndpoint(ServerSideVariableContext context); + } +} diff --git a/UET/Redpoint.KubernetesManager.PxeBoot/Variable/ServerSideVariableContext.cs b/UET/Redpoint.KubernetesManager.PxeBoot/Variable/ServerSideVariableContext.cs new file mode 100644 index 00000000..799234f2 --- /dev/null +++ b/UET/Redpoint.KubernetesManager.PxeBoot/Variable/ServerSideVariableContext.cs @@ -0,0 +1,51 @@ +namespace Redpoint.KubernetesManager.PxeBoot.Variable +{ + using Redpoint.KubernetesManager.Configuration.Types; + using Redpoint.KubernetesManager.PxeBoot.Server.Endpoints.NodeProvisioning; + + internal class ServerSideVariableContext + { + public required RkmNode RkmNode { get; init; } + + public required RkmNodeGroup RkmNodeGroup { get; init; } + + public required RkmNodeProvisioner RkmNodeProvisioner { get; init; } + + public required string ApiHostAddress { get; init; } + + public required int ApiHostHttpPort { get; init; } + + public required int ApiHostHttpsPort { get; init; } + + public static ServerSideVariableContext FromNodeGroupProvisioner( + INodeProvisioningEndpointContext context) + { + return new ServerSideVariableContext + { + RkmNode = context.RkmNode!, + RkmNodeGroup = context.RkmNodeGroup!, + RkmNodeProvisioner = context.RkmNodeGroupProvisioner!, + ApiHostAddress = context.HostAddress, + ApiHostHttpPort = context.HostHttpPort, + ApiHostHttpsPort = context.HostHttpsPort, + }; + } + + public static ServerSideVariableContext FromNodeProvisionerWithoutContextLoadedObjects( + INodeProvisioningEndpointContext context, + RkmNode rkmNode, + RkmNodeGroup rkmNodeGroup, + RkmNodeProvisioner rkmNodeProvisioner) + { + return new ServerSideVariableContext + { + RkmNode = rkmNode, + RkmNodeGroup = rkmNodeGroup, + RkmNodeProvisioner = rkmNodeProvisioner, + ApiHostAddress = context.HostAddress, + ApiHostHttpPort = context.HostHttpPort, + ApiHostHttpsPort = context.HostHttpsPort, + }; + } + } +} diff --git a/UET/Redpoint.KubernetesManager.Tests/BootManagerTests.cs b/UET/Redpoint.KubernetesManager.Tests/BootManagerTests.cs new file mode 100644 index 00000000..a5082f0b --- /dev/null +++ b/UET/Redpoint.KubernetesManager.Tests/BootManagerTests.cs @@ -0,0 +1,101 @@ +namespace Redpoint.KubernetesManager.Tests +{ + using Redpoint.KubernetesManager.PxeBoot.Bootmgr; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using Xunit; + + public class BootManagerTests + { + [Fact] + public void TestEfibootmgrParse() + { + var tab = '\t'; + var output = + $""" + BootCurrent: 0000 + Timeout: 0 seconds + BootOrder: 0000,0004,0003,0005 + Boot0000* EFI Network{tab}AcpiEx(VMBus,,)/VenHw(9b17e5a2-0891-42dd-b653-80b5c22809ba,635161f83edfc546913ff2d2f965ed0e7c43d7a29f048f469cfd765e69eec77b)/MAC(000000000000,0)/IPv4(0.0.0.0,0,DHCP,0.0.0.0,0.0.0.0,0.0.0.0) + Boot0001* FrontPage{tab}MemoryMapped(11,0x100000,0x5dffff)/FvFile(4042708a-0f2d-4823-ac60-0d77b3111889) + Boot0002 FrontPage{tab}MemoryMapped(11,0x100000,0x5dffff)/FvFile(4042708a-0f2d-4823-ac60-0d77b3111889) + Boot0003* Windows Boot Manager{tab}HD(1,GPT,65604577-96f4-4851-872f-df5886722450,0x800,0x82000)/\EFI\Microsoft\Boot\bootmgfw.efi57494e444f5753000100000088000000780000004200430044004f0042004a004500430054003d007b00390064006500610038003600320063002d0035006300640064002d0034006500370030002d0061006300630031002d006600330032006200330034003400640034003700390035007d00000000000100000010000000040000007fff0400 + Boot0004* EFI Network{tab}AcpiEx(VMBus,,)/VenHw(9b17e5a2-0891-42dd-b653-80b5c22809ba,635161f83edfc546913ff2d2f965ed0e1ed81a093682d44286cce8d7aa339ee8)/MAC(000000000000,0)/IPv4(0.0.0.0,0,DHCP,0.0.0.0,0.0.0.0,0.0.0.0) + Boot0005* EFI SCSI Device{tab}AcpiEx(VMBus,,)/VenHw(9b17e5a2-0891-42dd-b653-80b5c22809ba,d96361baa104294db60572e2ffb1dc7f7a8a3fdfec51db4b8203351a459be4d5)/SCSI(0,0) + """; + + var bootManager = new DefaultEfiBootManagerParser(); + var configuration = bootManager.ParseBootManagerConfiguration(output); + + Assert.Equal(0, configuration.BootCurrentId); + Assert.Equal(0, configuration.Timeout); + Assert.Equal( + new int[] { 0, 4, 3, 5 }, + configuration.BootOrder); + Assert.Equal( + new EfiBootManagerEntry + { + BootId = 0, + Name = "EFI Network", + Path = "AcpiEx(VMBus,,)/VenHw(9b17e5a2-0891-42dd-b653-80b5c22809ba,635161f83edfc546913ff2d2f965ed0e7c43d7a29f048f469cfd765e69eec77b)/MAC(000000000000,0)/IPv4(0.0.0.0,0,DHCP,0.0.0.0,0.0.0.0,0.0.0.0)", + Active = true, + }, + Assert.Contains(0, configuration.BootEntries)); + Assert.Equal( + new EfiBootManagerEntry + { + BootId = 1, + Name = "FrontPage", + Path = "MemoryMapped(11,0x100000,0x5dffff)/FvFile(4042708a-0f2d-4823-ac60-0d77b3111889)", + Active = true, + }, + Assert.Contains(1, configuration.BootEntries)); + Assert.Equal( + new EfiBootManagerEntry + { + BootId = 2, + Name = "FrontPage", + Path = "MemoryMapped(11,0x100000,0x5dffff)/FvFile(4042708a-0f2d-4823-ac60-0d77b3111889)", + Active = false, + }, + Assert.Contains(2, configuration.BootEntries)); + Assert.Equal( + new EfiBootManagerEntry + { + BootId = 3, + Name = "Windows Boot Manager", + Path = "HD(1,GPT,65604577-96f4-4851-872f-df5886722450,0x800,0x82000)/\\EFI\\Microsoft\\Boot\\bootmgfw.efi57494e444f5753000100000088000000780000004200430044004f0042004a004500430054003d007b00390064006500610038003600320063002d0035006300640064002d0034006500370030002d0061006300630031002d006600330032006200330034003400640034003700390035007d00000000000100000010000000040000007fff0400", + Active = true, + }, + Assert.Contains(3, configuration.BootEntries)); + Assert.Equal( + new EfiBootManagerEntry + { + BootId = 4, + Name = "EFI Network", + Path = "AcpiEx(VMBus,,)/VenHw(9b17e5a2-0891-42dd-b653-80b5c22809ba,635161f83edfc546913ff2d2f965ed0e1ed81a093682d44286cce8d7aa339ee8)/MAC(000000000000,0)/IPv4(0.0.0.0,0,DHCP,0.0.0.0,0.0.0.0,0.0.0.0)", + Active = true, + }, + Assert.Contains(4, configuration.BootEntries)); + Assert.Equal( + new EfiBootManagerEntry + { + BootId = 5, + Name = "EFI SCSI Device", + Path = "AcpiEx(VMBus,,)/VenHw(9b17e5a2-0891-42dd-b653-80b5c22809ba,d96361baa104294db60572e2ffb1dc7f7a8a3fdfec51db4b8203351a459be4d5)/SCSI(0,0)", + Active = true, + }, + Assert.Contains(5, configuration.BootEntries)); + + + // Remove entry 0: + // efibootmgr -b 0 -B + + // create + // efibootmgr -c -d /dev/sda -L ipxe-recovery -l '\EFI\ipxe-recovery\ipxe.efi' + } + } +} diff --git a/UET/Redpoint.KubernetesManager.Tests/Redpoint.KubernetesManager.Tests.csproj b/UET/Redpoint.KubernetesManager.Tests/Redpoint.KubernetesManager.Tests.csproj index a343e6ce..496c9f29 100644 --- a/UET/Redpoint.KubernetesManager.Tests/Redpoint.KubernetesManager.Tests.csproj +++ b/UET/Redpoint.KubernetesManager.Tests/Redpoint.KubernetesManager.Tests.csproj @@ -6,6 +6,7 @@ <ItemGroup> <ProjectReference Include="..\Redpoint.KubernetesManager.HostedService\Redpoint.KubernetesManager.HostedService.csproj" /> + <ProjectReference Include="..\Redpoint.KubernetesManager.PxeBoot\Redpoint.KubernetesManager.PxeBoot.csproj" /> <ProjectReference Include="..\Redpoint.KubernetesManager\Redpoint.KubernetesManager.csproj" /> </ItemGroup> diff --git a/UET/Redpoint.KubernetesManager/ControllerApi/PutNodeAuthorizeControllerEndpoint.cs b/UET/Redpoint.KubernetesManager/ControllerApi/PutNodeAuthorizeControllerEndpoint.cs deleted file mode 100644 index 09ae12b6..00000000 --- a/UET/Redpoint.KubernetesManager/ControllerApi/PutNodeAuthorizeControllerEndpoint.cs +++ /dev/null @@ -1,103 +0,0 @@ -namespace Redpoint.KubernetesManager.ControllerApi -{ - using Microsoft.AspNetCore.Http; - using Redpoint.KubernetesManager.Abstractions; - using Redpoint.KubernetesManager.Manifest; - using Redpoint.KubernetesManager.Manifests; - using Redpoint.KubernetesManager.Services; - using System; - using System.Collections.Generic; - using System.Linq; - using System.Net; - using System.Security.Cryptography; - using System.Text; - using System.Text.Json; - using System.Threading; - using System.Threading.Tasks; - - internal class PutNodeAuthorizeControllerEndpoint : IControllerEndpoint - { - private readonly ITpmService _tpmService; - private readonly ICertificateManager _certificateManager; - private readonly IPathProvider _pathProvider; - - public PutNodeAuthorizeControllerEndpoint( - ITpmService tpmService, - ICertificateManager certificateManager, - IPathProvider pathProvider) - { - _tpmService = tpmService; - _certificateManager = certificateManager; - _pathProvider = pathProvider; - } - - public string Path => "/node-authorize"; - - public async Task HandleAsync(HttpContext context, CancellationToken cancellationToken) - { - // This should be a PUT request. - if (context.Request.Method != "PUT") - { - context.Response.StatusCode = (int)HttpStatusCode.BadRequest; - return; - } - - // Deserialize the request. - var nodeAuthorizeRequest = JsonSerializer.Deserialize( - context.Request.Body, - ManifestJsonSerializerContext.Default.NodeAuthorizeRequest)!; - var ekPublicKeyTpmRepresentation = Convert.FromBase64String(nodeAuthorizeRequest.EkPublicTpmRepresentationBase64); - var aikPublicKeyTpmRepresentation = Convert.FromBase64String(nodeAuthorizeRequest.AikPublicTpmRepresentationBase64); - - // @todo: Authorize either: - // - If the EK and AIK match our own, then this request is coming from the controller itself. We must authorize here since the API server won't necessarily be running at this point. - // - If they don't match our own, check RkmNode resources on the API server. - var authorized = true; - var nodeName = nodeAuthorizeRequest.SuggestedNodeName; - - // If we are not authorized, return appropriate HTTP response. - if (!authorized) - { - context.Response.StatusCode = (int)HttpStatusCode.Unauthorized; - return; - } - - // Generate node certificate. - var nodeCertificate = await _certificateManager.GenerateCertificateForAuthorizedNodeAsync( - nodeName, - context.Connection.RemoteIpAddress); - var certificateAuthorityPem = await File.ReadAllTextAsync( - System.IO.Path.Combine(_pathProvider.RKMRoot, "certs", "ca", "ca.pem"), - cancellationToken); - var bundle = new NodeAuthorizeResponseEncryptedBundle - { - NodePrivateKeyPem = nodeCertificate.PrivateKeyPem, - NodeCertificatePem = nodeCertificate.CertificatePem, - CertificateAuthorityPem = certificateAuthorityPem, - NodeName = nodeName, - }; - - // Authorize and encrypt bundle. - var (envelopingKey, encryptedBundle) = _tpmService.Authorize( - ekPublicKeyTpmRepresentation, - aikPublicKeyTpmRepresentation, - Encoding.UTF8.GetBytes(JsonSerializer.Serialize( - bundle, - ManifestJsonSerializerContext.Default.NodeAuthorizeResponseEncryptedBundle))); - - // Send response payload. - var response = new NodeAuthorizeResponse - { - EnvelopingKeyBase64 = Convert.ToBase64String(envelopingKey), - EncryptedBundleBase64 = Convert.ToBase64String(encryptedBundle), - }; - context.Response.StatusCode = (int)HttpStatusCode.OK; - context.Response.ContentType = "application/json"; - await JsonSerializer.SerializeAsync( - context.Response.Body, - response, - ManifestJsonSerializerContext.Default.NodeAuthorizeResponse, - cancellationToken); - } - } -} diff --git a/UET/Redpoint.KubernetesManager/KubernetesManagerServiceExtensions.cs b/UET/Redpoint.KubernetesManager/KubernetesManagerServiceExtensions.cs index 3a491b19..0de2f8cc 100644 --- a/UET/Redpoint.KubernetesManager/KubernetesManagerServiceExtensions.cs +++ b/UET/Redpoint.KubernetesManager/KubernetesManagerServiceExtensions.cs @@ -16,6 +16,7 @@ using Redpoint.KubernetesManager.Services.Linux; using Redpoint.KubernetesManager.Services.Windows; using Redpoint.KubernetesManager.Services.Wsl; + using Redpoint.Tpm; using Redpoint.Windows.Firewall; using Redpoint.Windows.HostNetworkingService; @@ -43,15 +44,14 @@ public static void AddKubernetesManager(this IServiceCollection services, bool w services.AddSingleton<IWslTranslation, DefaultWslTranslation>(); services.AddSingleton<IRkmGlobalRootProvider, DefaultRkmGlobalRootProvider>(); services.AddSingleton<IHelmDeployment, DefaultHelmDeployment>(); - services.AddSingleton<ITpmService, DefaultTpmService>(); services.AddRkmManifest(); services.AddRkmPerpetualProcess(); + services.AddTpm(); services.AddKestrelFactory(); services.AddSingleton<IControllerEndpoint, GetLegacyManifestControllerEndpoint>(); services.AddSingleton<IControllerEndpoint, GetNodeManifestControllerEndpoint>(); - services.AddSingleton<IControllerEndpoint, PutNodeAuthorizeControllerEndpoint>(); // Register controller-only components. if (withPathProvider) diff --git a/UET/Redpoint.KubernetesManager/Redpoint.KubernetesManager.csproj b/UET/Redpoint.KubernetesManager/Redpoint.KubernetesManager.csproj index 68e8a8e4..35519925 100644 --- a/UET/Redpoint.KubernetesManager/Redpoint.KubernetesManager.csproj +++ b/UET/Redpoint.KubernetesManager/Redpoint.KubernetesManager.csproj @@ -34,6 +34,7 @@ <ProjectReference Include="..\Redpoint.PackageManagement\Redpoint.PackageManagement.csproj" /> <ProjectReference Include="..\Redpoint.Registry\Redpoint.Registry.csproj" /> <ProjectReference Include="..\Redpoint.ServiceControl\Redpoint.ServiceControl.csproj" /> + <ProjectReference Include="..\Redpoint.Tpm\Redpoint.Tpm.csproj" /> <ProjectReference Include="..\Redpoint.Uet.Configuration\Redpoint.Uet.Configuration.csproj" /> <ProjectReference Include="..\Redpoint.Windows.Firewall\Redpoint.Windows.Firewall.csproj" /> <ProjectReference Include="..\Redpoint.Windows.HostNetworkingService\Redpoint.Windows.HostNetworkingService.csproj" /> diff --git a/UET/Redpoint.KubernetesManager/Services/DefaultTpmService.cs b/UET/Redpoint.KubernetesManager/Services/DefaultTpmService.cs deleted file mode 100644 index 8a770114..00000000 --- a/UET/Redpoint.KubernetesManager/Services/DefaultTpmService.cs +++ /dev/null @@ -1,203 +0,0 @@ -namespace Redpoint.KubernetesManager.Services -{ - using Microsoft.Extensions.Logging; - using System; - using System.Security.Cryptography; - using System.Threading.Tasks; - using Tpm2Lib; - - internal class DefaultTpmService : ITpmService - { - private readonly ILogger<DefaultTpmService> _logger; - - public DefaultTpmService( - ILogger<DefaultTpmService> logger) - { - _logger = logger; - } - - private const TpmAlgId _tpmAlgId = TpmAlgId.Sha256; - - private static PolicyTree GetAikPolicyTree() - { - var aikPolicyTree = new PolicyTree(_tpmAlgId); - var aikPolicyOr = new TpmPolicyOr(); - aikPolicyTree.SetPolicyRoot(aikPolicyOr); - aikPolicyOr.AddPolicyBranch(new TpmPolicyCommand(TpmCc.ActivateCredential, "Activate")); - aikPolicyOr.AddPolicyBranch(new TpmPolicyCommand(TpmCc.Certify, "Certify")); - aikPolicyOr.AddPolicyBranch(new TpmPolicyCommand(TpmCc.CertifyCreation, "CertifyCreation")); - aikPolicyOr.AddPolicyBranch(new TpmPolicyCommand(TpmCc.Quote, "Quote")); - return aikPolicyTree; - } - - public async Task<(byte[] ekPublicBytes, byte[] aikPublicBytes, byte[] aikContextBytes)> CreateRequestAsync() - { - using Tpm2Device tpmDevice = OperatingSystem.IsWindows() ? new TbsDevice() : new LinuxTpmDevice(); - tpmDevice.Connect(); - - using var tpm = new Tpm2(tpmDevice); - - // Get EK. - _logger.LogInformation("Getting EK handle..."); - var ekHandle = new TpmHandle(0x81010001); - var ekPublicKey = tpm._AllowErrors().ReadPublic(ekHandle, out var ekName, out var ekQName); - if (!tpm._LastCommandSucceeded()) - { - throw new InvalidOperationException("Failed to read public EK from TPM!"); - } - - // Create AIK. - _logger.LogInformation("Creating AIK..."); - var aikPublicKey = new TpmPublic( - _tpmAlgId, - ObjectAttr.Restricted | - ObjectAttr.Sign | - ObjectAttr.FixedParent | - ObjectAttr.FixedTPM | - ObjectAttr.AdminWithPolicy | - ObjectAttr.SensitiveDataOrigin, - GetAikPolicyTree().GetPolicyDigest(), - new RsaParms(new SymDefObject(), new SchemeRsassa(_tpmAlgId), 2048, 0), - new Tpm2bPublicKeyRsa()); - - var aikCreateResponse = await tpm.CreatePrimaryAsync( - TpmRh.Endorsement, - new SensitiveCreate(tpm.GetRandom(TpmHash.DigestSize(_tpmAlgId)), null), - aikPublicKey, - null, - null); - - aikPublicKey = aikCreateResponse.outPublic; - var aikContext = tpm.ContextSave(aikCreateResponse.handle); - - return ( - ekPublicKey.GetTpmRepresentation(), - aikPublicKey.GetTpmRepresentation(), - aikContext.GetTpmRepresentation()); - } - - public (string pem, string hash) GetPemAndHash(byte[] publicKeyBytes) - { - var tpmPublicKey = new Marshaller(publicKeyBytes).Get<TpmPublic>(); - - var rsaParams = (RsaParms)tpmPublicKey.parameters; - var exponent = rsaParams.exponent != 0 ? Globs.HostToNet(rsaParams.exponent) : RsaParms.DefaultExponent; - var modulus = (tpmPublicKey.unique as Tpm2bPublicKeyRsa)!.buffer; - - var publicKey = new RSACryptoServiceProvider(); - publicKey.ImportParameters(new RSAParameters - { - Exponent = exponent, - Modulus = modulus, - }); - - return (publicKey.ExportRSAPublicKeyPem(), Convert.ToHexStringLower(SHA256.HashData([.. exponent, .. modulus]))); - } - - public (byte[] envelopingKey, byte[] encryptedData) Authorize(byte[] ekPublicBytes, byte[] aikPublicBytes, byte[] data) - { - using Tpm2Device tpmDevice = OperatingSystem.IsWindows() ? new TbsDevice() : new LinuxTpmDevice(); - tpmDevice.Connect(); - - using var tpm = new Tpm2(tpmDevice); - - var ekPublicKey = new Marshaller(ekPublicBytes).Get<TpmPublic>(); - var aikPublicKey = new Marshaller(aikPublicBytes).Get<TpmPublic>(); - - _logger.LogInformation("Creating activation credentials for AIK..."); - var certInfo = ekPublicKey.CreateActivationCredentials( - data, - aikPublicKey.GetName(), - out var encryptedData); - - var envelopingKeyMarshaller = new Marshaller(); - envelopingKeyMarshaller.Put(certInfo.integrityHMAC.Length, "integrityHMAC.Length"); - envelopingKeyMarshaller.Put(certInfo.integrityHMAC, "integrityHMAC"); - envelopingKeyMarshaller.Put(certInfo.encIdentity.Length, "encIdentity"); - envelopingKeyMarshaller.Put(certInfo.encIdentity, "encIdentity.Length"); - byte[] envelopingKey = envelopingKeyMarshaller.GetBytes(); - - return (envelopingKey, encryptedData); - } - - public byte[] DecryptSecretKey(byte[] aikContextBytes, byte[] envelopingKeyBytes, byte[] encryptedSecret) - { - using Tpm2Device tpmDevice = OperatingSystem.IsWindows() ? new TbsDevice() : new LinuxTpmDevice(); - tpmDevice.Connect(); - - using var tpm = new Tpm2(tpmDevice); - - // Get enveloping key. - IdObject envelopingKey = new IdObject(); - { - var envelopingKeyMarshaller = new Marshaller(envelopingKeyBytes); - int len = envelopingKeyMarshaller.Get<int>(); - envelopingKey.integrityHMAC = envelopingKeyMarshaller.GetArray<byte>(len); - len = envelopingKeyMarshaller.Get<int>(); - envelopingKey.encIdentity = envelopingKeyMarshaller.GetArray<byte>(len); - } - - // Get EK. - _logger.LogInformation("Getting EK handle..."); - var ekHandle = new TpmHandle(0x81010001); - var ekPublicKey = tpm._AllowErrors().ReadPublic(ekHandle, out var ekName, out var ekQName); - if (!tpm._LastCommandSucceeded()) - { - throw new InvalidOperationException("Failed to read public EK from TPM!"); - } - - // Load AIK context. - var aikContext = new Marshaller(aikContextBytes).Get<Context>(); - var aikHandle = tpm.ContextLoad(aikContext); - - // Create sessions for AIK usage. - _logger.LogInformation("Activating AIK..."); - var aikSession = tpm.StartAuthSessionEx( - TpmSe.Policy, - _tpmAlgId); - using var aikSessionAutoFlush = new TpmAutoFlush(tpm, aikSession); - aikSession.RunPolicy(tpm, GetAikPolicyTree(), "Activate"); - - // Determine policy tree for EK usage. - var ekPolicyTree = new PolicyTree(ekPublicKey.nameAlg); - ekPolicyTree.SetPolicyRoot(new TpmPolicySecret( - TpmRh.Endorsement, - false, - 0, - null, - null)); - - // Create session for EK usage. - var ekSession = tpm.StartAuthSessionEx( - TpmSe.Policy, - ekPublicKey.nameAlg); - using var ekSessionAutoFlush = new TpmAutoFlush(tpm, ekSession); - ekSession.RunPolicy(tpm, ekPolicyTree); - - // Activate the AIK credential. - return tpm[aikSession, ekSession] - .ActivateCredential( - aikHandle, - ekHandle, - envelopingKey, - encryptedSecret); - } - - private class TpmAutoFlush : IDisposable - { - private readonly Tpm2 _tpm; - private readonly TpmHandle _handle; - - public TpmAutoFlush(Tpm2 tpm, TpmHandle handle) - { - _tpm = tpm; - _handle = handle; - } - - public void Dispose() - { - _tpm.FlushContext(_handle); - } - } - } -} diff --git a/UET/Redpoint.KubernetesManager/Services/ITpmService.cs b/UET/Redpoint.KubernetesManager/Services/ITpmService.cs deleted file mode 100644 index e4e1099f..00000000 --- a/UET/Redpoint.KubernetesManager/Services/ITpmService.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace Redpoint.KubernetesManager.Services -{ - using Redpoint.KubernetesManager.Manifest; - using System.Collections.Generic; - using System.Globalization; - using System.Linq; - using System.Security.Cryptography.X509Certificates; - using System.Text; - using System.Threading; - using System.Threading.Tasks; - using YamlDotNet.Core.Tokens; - - internal interface ITpmService - { - Task<(byte[] ekPublicBytes, byte[] aikPublicBytes, byte[] aikContextBytes)> CreateRequestAsync(); - - (string pem, string hash) GetPemAndHash(byte[] publicKeyBytes); - - (byte[] envelopingKey, byte[] encryptedData) Authorize(byte[] ekPublicBytes, byte[] aikPublicBytes, byte[] data); - - byte[] DecryptSecretKey(byte[] aikContextBytes, byte[] envelopingKey, byte[] encryptedSecret); - } -} diff --git a/UET/Redpoint.ProgressMonitor/PositionAwareStream.cs b/UET/Redpoint.ProgressMonitor/PositionAwareStream.cs index d641105d..67033b03 100644 --- a/UET/Redpoint.ProgressMonitor/PositionAwareStream.cs +++ b/UET/Redpoint.ProgressMonitor/PositionAwareStream.cs @@ -1,6 +1,4 @@ -using System; - -namespace Redpoint.ProgressMonitor +namespace Redpoint.ProgressMonitor { using System; using System.IO; @@ -59,6 +57,12 @@ public override long Position } } + /// <inheritdoc/> + public override int ReadTimeout { get => _underlyingStream.ReadTimeout; set => _underlyingStream.ReadTimeout = value; } + + /// <inheritdoc/> + public override int WriteTimeout { get => _underlyingStream.WriteTimeout; set => _underlyingStream.WriteTimeout = value; } + /// <inheritdoc/> public override void Flush() { @@ -76,16 +80,11 @@ public override int Read(byte[] buffer, int offset, int count) /// <inheritdoc/> public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { -#if NETCOREAPP2_1_OR_GREATER var bytesRead = await _underlyingStream.ReadAsync(buffer.AsMemory(offset, count), cancellationToken).ConfigureAwait(false); -#else - var bytesRead = await _underlyingStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); -#endif _position += bytesRead; return bytesRead; } -#if NETCOREAPP2_1_OR_GREATER /// <inheritdoc/> public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) { @@ -93,7 +92,6 @@ public override async ValueTask<int> ReadAsync(Memory<byte> buffer, Cancellation _position += bytesRead; return bytesRead; } -#endif /// <inheritdoc/> public override long Seek(long offset, SeekOrigin origin) diff --git a/UET/Redpoint.ProgressMonitor/StallDetectionStream.cs b/UET/Redpoint.ProgressMonitor/StallDetectionStream.cs new file mode 100644 index 00000000..7bab1ddc --- /dev/null +++ b/UET/Redpoint.ProgressMonitor/StallDetectionStream.cs @@ -0,0 +1,158 @@ +namespace Redpoint.ProgressMonitor +{ + using System; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + + /// <summary> + /// Wraps another stream (such as a HTTP response stream) and throws an exception if <see cref="Read"/> operations ever cease returning data for a given timeout without hitting EOF. + /// </summary> + public class StallDetectionStream : Stream + { + private readonly Stream _underlyingStream; + private readonly TimeSpan _stallPeriod; + private DateTimeOffset? _lastTimeDataReceived; + + /// <summary> + /// Creates a <see cref="StallDetectionStream"/> which wraps the target system. + /// </summary> + /// <param name="underlyingStream">The stream to wrap.</param> + /// <param name="stallPeriod">The maximum amount of time that can elapse without reading data.</param> + public StallDetectionStream(Stream underlyingStream, TimeSpan stallPeriod) + { + _underlyingStream = underlyingStream; + _stallPeriod = stallPeriod; + _lastTimeDataReceived = null; + } + + /// <inheritdoc/> + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + _underlyingStream.Dispose(); + } + + /// <inheritdoc/> + public override bool CanRead => true; + + /// <inheritdoc/> + public override bool CanSeek => false; + + /// <inheritdoc/> + public override bool CanWrite => false; + + /// <inheritdoc/> + public override long Length => _underlyingStream.Length; + + /// <inheritdoc/> + public override long Position { get => _underlyingStream.Position; set => _underlyingStream.Position = value; } + + /// <inheritdoc/> + public override int ReadTimeout { get => (int)_stallPeriod.TotalMilliseconds; set { } } + + /// <inheritdoc/> + public override int WriteTimeout { get => _underlyingStream.WriteTimeout; set => _underlyingStream.WriteTimeout = value; } + + /// <inheritdoc/> + public override void Flush() + { + _underlyingStream.Flush(); + } + + /// <inheritdoc/> + public override int Read(byte[] buffer, int offset, int count) + { + if (_lastTimeDataReceived.HasValue && + _lastTimeDataReceived.Value < DateTimeOffset.UtcNow - _stallPeriod) + { + throw new StreamStalledException(); + } + + try + { + _underlyingStream.ReadTimeout = (int)_stallPeriod.TotalMilliseconds; + } + catch (InvalidOperationException) + { + } + + var bytesRead = _underlyingStream.Read(buffer, offset, count); + if (bytesRead > 0) + { + _lastTimeDataReceived = DateTimeOffset.UtcNow; + } + return bytesRead; + } + + /// <inheritdoc/> + public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + if (_lastTimeDataReceived.HasValue && + _lastTimeDataReceived.Value < DateTimeOffset.UtcNow - _stallPeriod) + { + throw new StreamStalledException(); + } + + using var ctsTimer = new CancellationTokenSource((int)_stallPeriod.TotalMilliseconds); + try + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, ctsTimer.Token); + var bytesRead = await _underlyingStream.ReadAsync(buffer.AsMemory(offset, count), cts.Token).ConfigureAwait(false); + if (bytesRead > 0) + { + _lastTimeDataReceived = DateTime.UtcNow; + } + return bytesRead; + } + catch (InvalidOperationException) when (ctsTimer.IsCancellationRequested) + { + throw new StreamStalledException(); + } + } + + /// <inheritdoc/> + public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) + { + if (_lastTimeDataReceived.HasValue && + _lastTimeDataReceived.Value < DateTimeOffset.UtcNow - _stallPeriod) + { + throw new StreamStalledException(); + } + + using var ctsTimer = new CancellationTokenSource((int)_stallPeriod.TotalMilliseconds); + try + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, ctsTimer.Token); + var bytesRead = await _underlyingStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (bytesRead > 0) + { + _lastTimeDataReceived = DateTime.UtcNow; + } + return bytesRead; + } + catch (InvalidOperationException) when (ctsTimer.IsCancellationRequested) + { + throw new StreamStalledException(); + } + } + + /// <inheritdoc/> + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + /// <inheritdoc/> + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + /// <inheritdoc/> + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + } +} diff --git a/UET/Redpoint.ProgressMonitor/StreamStalledException.cs b/UET/Redpoint.ProgressMonitor/StreamStalledException.cs new file mode 100644 index 00000000..593af93d --- /dev/null +++ b/UET/Redpoint.ProgressMonitor/StreamStalledException.cs @@ -0,0 +1,8 @@ +namespace Redpoint.ProgressMonitor +{ + using System; + + public class StreamStalledException : Exception + { + } +} diff --git a/UET/Redpoint.RuntimeJson.SourceGenerator/RuntimeJsonSourceGenerator.cs b/UET/Redpoint.RuntimeJson.SourceGenerator/RuntimeJsonSourceGenerator.cs index 865ed18b..c04cbd75 100644 --- a/UET/Redpoint.RuntimeJson.SourceGenerator/RuntimeJsonSourceGenerator.cs +++ b/UET/Redpoint.RuntimeJson.SourceGenerator/RuntimeJsonSourceGenerator.cs @@ -35,6 +35,11 @@ namespace {entry.Namespace} {{ partial class {entry.Class} {{ + public {entry.Class}(JsonSerializerOptions options) + : this(new {entry.JsonSerializerContextType}(new JsonSerializerOptions(options))) + {{ + }} + public {entry.Class}({entry.JsonSerializerContextType} context) {{"); foreach (var type in entry.SerializableClassNames) diff --git a/UET/Redpoint.Tpm/EnvelopingKeyMarshaller.cs b/UET/Redpoint.Tpm/EnvelopingKeyMarshaller.cs new file mode 100644 index 00000000..98e70f0e --- /dev/null +++ b/UET/Redpoint.Tpm/EnvelopingKeyMarshaller.cs @@ -0,0 +1,46 @@ +namespace Redpoint.Tpm +{ + using Microsoft.Extensions.Logging; + using System; + using System.Collections.Generic; + using System.Globalization; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using Tpm2Lib; + + public static class EnvelopingKeyMarshaller + { + public static byte[] EnvelopingKeyToBytes(this IdObject certInfo, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(certInfo); + + var envelopingKeyMarshaller = new Marshaller(); + envelopingKeyMarshaller.Put(certInfo.integrityHMAC.Length, "integrityHMAC.Length"); + envelopingKeyMarshaller.Put(certInfo.integrityHMAC, "integrityHMAC"); + envelopingKeyMarshaller.Put(certInfo.encIdentity.Length, "encIdentity"); + envelopingKeyMarshaller.Put(certInfo.encIdentity, "encIdentity.Length"); + var bytes = envelopingKeyMarshaller.GetBytes(); + + logger?.LogTrace($"Converted enveloping key to byte array (identity HMAC length: {certInfo.integrityHMAC.Length}, encrypted identity length: {certInfo.encIdentity.Length}, result byte array length: {bytes.Length})."); + + return bytes; + } + + public static IdObject BytesToEnvelopingKey(this byte[] bytes, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(bytes); + + IdObject envelopingKey = new IdObject(); + var envelopingKeyMarshaller = new Marshaller(bytes); + int identityHmacLength = envelopingKeyMarshaller.Get<int>(); + envelopingKey.integrityHMAC = envelopingKeyMarshaller.GetArray<byte>(identityHmacLength); + int encIdentityLength = envelopingKeyMarshaller.Get<int>(); + envelopingKey.encIdentity = envelopingKeyMarshaller.GetArray<byte>(encIdentityLength); + + logger?.LogTrace($"Converted byte array to enveloping key (identity HMAC length: {identityHmacLength}, encrypted identity length: {encIdentityLength}, result byte array length: {bytes.Length})."); + + return envelopingKey; + } + } +} diff --git a/UET/Redpoint.Tpm/Internal/DefaultTpmOperationHandles.cs b/UET/Redpoint.Tpm/Internal/DefaultTpmOperationHandles.cs index 711148e2..a6fee928 100644 --- a/UET/Redpoint.Tpm/Internal/DefaultTpmOperationHandles.cs +++ b/UET/Redpoint.Tpm/Internal/DefaultTpmOperationHandles.cs @@ -1,9 +1,12 @@ namespace Redpoint.Tpm.Internal { + using Microsoft.Extensions.Logging; using Tpm2Lib; internal class DefaultTpmOperationHandles : ITpmOperationHandles { + private readonly ILogger _logger; + internal Tpm2Device? _tpmDevice; internal Tpm2? _tpm; internal TpmHandle? _ekHandle; @@ -19,13 +22,45 @@ internal class DefaultTpmOperationHandles : ITpmOperationHandles public TpmHandle AikHandle => _aikHandle!; public TpmPublic AikPublic => _aikPublic!; + public DefaultTpmOperationHandles(ILogger logger) + { + _logger = logger; + } + protected virtual void Dispose(bool disposing) { if (!_disposedValue) { if (disposing) { + if (_aikHandle != null && _tpm != null) + { + _logger.LogTrace("Disposing AIK handle..."); + _tpm.FlushContext(_aikHandle); + _aikHandle = null; + } + + if (_ekHandle != null && _tpm != null) + { + _logger.LogTrace("Disposing EK handle..."); + try + { + _tpm.FlushContext(_ekHandle); + } + catch { } + _ekHandle = null; + } + + if (_tpm != null) + { + _logger.LogTrace("Disposing TPM..."); + } _tpm?.Dispose(); + + if (_tpmDevice != null) + { + _logger.LogTrace("Disposing TPM device..."); + } _tpmDevice?.Dispose(); } diff --git a/UET/Redpoint.Tpm/Internal/DefaultTpmService.cs b/UET/Redpoint.Tpm/Internal/DefaultTpmService.cs index 54b2f4e3..4a531a6b 100644 --- a/UET/Redpoint.Tpm/Internal/DefaultTpmService.cs +++ b/UET/Redpoint.Tpm/Internal/DefaultTpmService.cs @@ -31,7 +31,7 @@ private static PolicyTree GetAikPolicyTree() public async Task<(byte[] ekPublicBytes, byte[] aikPublicBytes, ITpmOperationHandles operationHandles)> CreateRequestAsync() { - var handles = new DefaultTpmOperationHandles(); + var handles = new DefaultTpmOperationHandles(_logger); var returningHandles = false; try { @@ -51,7 +51,7 @@ private static PolicyTree GetAikPolicyTree() } // Create AIK. - _logger.LogTrace("Creating AIK..."); + _logger.LogTrace($"Creating AIK (alg {SecurityConstants.TpmHashAlgorithmId}, bits {SecurityConstants.RsaKeyBitsTpm})..."); var aikPublicTemplate = new TpmPublic( SecurityConstants.TpmHashAlgorithmId, ObjectAttr.Restricted | @@ -74,10 +74,14 @@ private static PolicyTree GetAikPolicyTree() handles._aikPublic = aikCreateResponse.outPublic; handles._aikHandle = aikCreateResponse.handle; + var ekPublicBytes = handles.EkPublic.GetTpmRepresentation(); + var aikPublicBytes = handles.AikPublic.GetTpmRepresentation(); + _logger.LogTrace($"CreateRequestAsync result (EK public bytes size: {ekPublicBytes.Length}, AIK public bytes size: {aikPublicBytes.Length}, AIK name hex: {Convert.ToHexStringLower(aikCreateResponse.outPublic.GetName())})"); + returningHandles = true; return ( - handles.EkPublic.GetTpmRepresentation(), - handles.AikPublic.GetTpmRepresentation(), + ekPublicBytes, + aikPublicBytes, handles); } finally @@ -143,20 +147,13 @@ public RSAParameters GetRsaParameters(byte[] publicKeyBytes) var ekPublicKey = new Marshaller(ekPublicBytes).Get<TpmPublic>(); var aikPublicKey = new Marshaller(aikPublicBytes).Get<TpmPublic>(); - _logger.LogTrace("Creating activation credentials for AIK..."); - var certInfo = ekPublicKey.CreateActivationCredentials( + _logger.LogTrace($"Creating activation credentials for AIK (unencrypted key length: {symmetricUnencryptedKey.Length}, AIK public key name length: {aikPublicKey.GetName().Length})..."); + var envelopingKey = ekPublicKey.CreateActivationCredentials( symmetricUnencryptedKey, aikPublicKey.GetName(), - out var aesEncryptedKey); - - var envelopingKeyMarshaller = new Marshaller(); - envelopingKeyMarshaller.Put(certInfo.integrityHMAC.Length, "integrityHMAC.Length"); - envelopingKeyMarshaller.Put(certInfo.integrityHMAC, "integrityHMAC"); - envelopingKeyMarshaller.Put(certInfo.encIdentity.Length, "encIdentity"); - envelopingKeyMarshaller.Put(certInfo.encIdentity, "encIdentity.Length"); - byte[] envelopingKey = envelopingKeyMarshaller.GetBytes(); + out var symmetricEncryptedKey); - return (envelopingKey, aesEncryptedKey, symmetricEncryptedData); + return (envelopingKey.EnvelopingKeyToBytes(_logger), symmetricEncryptedKey, symmetricEncryptedData); } public byte[] DecryptSecretKey(ITpmOperationHandles handles, byte[] envelopingKeyBytes, byte[] encryptedKey, byte[] encryptedData) @@ -164,24 +161,25 @@ public byte[] DecryptSecretKey(ITpmOperationHandles handles, byte[] envelopingKe try { // Get enveloping key. - IdObject envelopingKey = new IdObject(); - { - var envelopingKeyMarshaller = new Marshaller(envelopingKeyBytes); - int len = envelopingKeyMarshaller.Get<int>(); - envelopingKey.integrityHMAC = envelopingKeyMarshaller.GetArray<byte>(len); - len = envelopingKeyMarshaller.Get<int>(); - envelopingKey.encIdentity = envelopingKeyMarshaller.GetArray<byte>(len); - } + IdObject envelopingKey = envelopingKeyBytes.BytesToEnvelopingKey(_logger); + + _logger.LogTrace($"TPM device: {handles.TpmDevice}"); + _logger.LogTrace($"TPM: {handles.Tpm}"); + _logger.LogTrace($"EK handle: {handles.EkHandle}"); + _logger.LogTrace($"EK public key: {handles.EkPublic}"); + _logger.LogTrace($"AIK handle: {handles.AikHandle}"); + _logger.LogTrace($"AIK public key: {handles.AikPublic}"); // Create sessions for AIK usage. - _logger.LogTrace("Activating AIK..."); + _logger.LogTrace($"Starting auth session for AIK..."); var aikSession = handles.Tpm.StartAuthSessionEx( TpmSe.Policy, SecurityConstants.TpmHashAlgorithmId); - using var aikSessionAutoFlush = new TpmAutoFlush(handles.Tpm, aikSession); + using var aikSessionAutoFlush = new TpmAutoFlush(_logger, handles.Tpm, aikSession); aikSession.RunPolicy(handles.Tpm, GetAikPolicyTree(), "Activate"); // Determine policy tree for EK usage. + _logger.LogTrace($"Creating policy tree for EK..."); var ekPolicyTree = new PolicyTree(handles.EkPublic.nameAlg); ekPolicyTree.SetPolicyRoot(new TpmPolicySecret( TpmRh.Endorsement, @@ -191,13 +189,15 @@ public byte[] DecryptSecretKey(ITpmOperationHandles handles, byte[] envelopingKe null)); // Create session for EK usage. + _logger.LogTrace($"Starting auth session for EK (alg: {handles.EkPublic.nameAlg})..."); var ekSession = handles.Tpm.StartAuthSessionEx( TpmSe.Policy, handles.EkPublic.nameAlg); - using var ekSessionAutoFlush = new TpmAutoFlush(handles.Tpm, ekSession); + using var ekSessionAutoFlush = new TpmAutoFlush(_logger, handles.Tpm, ekSession); ekSession.RunPolicy(handles.Tpm, ekPolicyTree); // Activate the AIK credential. + _logger.LogTrace($"Activating AIK... (enveloping key bytes size: {envelopingKeyBytes.Length}, encrypted Key: {encryptedKey.Length})"); var symmetricUnencryptedKey = handles.Tpm[aikSession, ekSession] .ActivateCredential( handles.AikHandle, @@ -206,6 +206,7 @@ public byte[] DecryptSecretKey(ITpmOperationHandles handles, byte[] envelopingKe encryptedKey); // Use libsodium to decrypt the data with the decrypted symmetric key. + _logger.LogTrace($"Decrypting data with unencrypted symmetric key..."); { var symmetricAlgorithm = SecurityConstants.SymmetricAlgorithm; var symmetricKey = Key.Import( @@ -229,17 +230,20 @@ public byte[] DecryptSecretKey(ITpmOperationHandles handles, byte[] envelopingKe private class TpmAutoFlush : IDisposable { + private readonly ILogger _logger; private readonly Tpm2 _tpm; private readonly TpmHandle _handle; - public TpmAutoFlush(Tpm2 tpm, TpmHandle handle) + public TpmAutoFlush(ILogger logger, Tpm2 tpm, TpmHandle handle) { + _logger = logger; _tpm = tpm; _handle = handle; } public void Dispose() { + _logger.LogTrace($"Disposing TPM handle {_handle}."); _tpm.FlushContext(_handle); } } diff --git a/UET/Redpoint.Tpm/Internal/ITpmOperationHandles.cs b/UET/Redpoint.Tpm/Internal/ITpmOperationHandles.cs index 8a8b0c99..b66127dd 100644 --- a/UET/Redpoint.Tpm/Internal/ITpmOperationHandles.cs +++ b/UET/Redpoint.Tpm/Internal/ITpmOperationHandles.cs @@ -2,7 +2,7 @@ { using Tpm2Lib; - internal interface ITpmOperationHandles : IDisposable + public interface ITpmOperationHandles : IDisposable { Tpm2Device TpmDevice { get; } diff --git a/UET/Redpoint.Tpm/Internal/ITpmService.cs b/UET/Redpoint.Tpm/Internal/ITpmService.cs index 963d01a6..0d2cd32d 100644 --- a/UET/Redpoint.Tpm/Internal/ITpmService.cs +++ b/UET/Redpoint.Tpm/Internal/ITpmService.cs @@ -3,7 +3,7 @@ using System.Security.Cryptography; using System.Threading.Tasks; - internal interface ITpmService + public interface ITpmService { Task<(byte[] ekPublicBytes, byte[] aikPublicBytes, ITpmOperationHandles operationHandles)> CreateRequestAsync(); diff --git a/UET/Redpoint.YamlToJson/Redpoint.YamlToJson.csproj b/UET/Redpoint.YamlToJson/Redpoint.YamlToJson.csproj new file mode 100644 index 00000000..c2a45021 --- /dev/null +++ b/UET/Redpoint.YamlToJson/Redpoint.YamlToJson.csproj @@ -0,0 +1,15 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <Import Project="$(MSBuildThisFileDirectory)../Lib/Common.Build.props" /> + + <Import Project="$(MSBuildThisFileDirectory)../Lib/LibraryPackaging.Build.props" /> + <PropertyGroup> + <Description>Provides the YamlToJsonConverter API, which converts YAML to JSON in a trim-friendly way without serializing/deserializing to .NET objects.</Description> + <PackageTags>yaml, json, conversion</PackageTags> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="YamlDotNet" /> + </ItemGroup> + +</Project> diff --git a/UET/Redpoint.YamlToJson/YamlToJsonConverter.cs b/UET/Redpoint.YamlToJson/YamlToJsonConverter.cs new file mode 100644 index 00000000..bf4965ac --- /dev/null +++ b/UET/Redpoint.YamlToJson/YamlToJsonConverter.cs @@ -0,0 +1,187 @@ +namespace Redpoint.YamlToJson +{ + using System; + using System.IO; + using System.Text; + using System.Text.Json; + using System.Text.RegularExpressions; + using YamlDotNet.Core; + using YamlDotNet.Core.Tokens; + using YamlDotNet.RepresentationModel; + + public static class YamlToJsonConverter + { + public static string Convert(string yaml, JsonWriterOptions jsonWriterOptions = default) + { + using var yamlInputStream = new MemoryStream(); + using var jsonOutputStream = new MemoryStream(); + + using (var writer = new StreamWriter(yamlInputStream, Encoding.UTF8, leaveOpen: true)) + { + writer.Write(yaml); + } + + Convert(yamlInputStream, jsonOutputStream, jsonWriterOptions); + + jsonOutputStream.Seek(0, SeekOrigin.Begin); + + using (var reader = new StreamReader(jsonOutputStream)) + { + return reader.ReadToEnd(); + } + } + + public static void Convert(Stream yamlInputStream, Stream jsonOutputStream, JsonWriterOptions jsonWriterOptions = default) + { + YamlStream yamlStream; + using (var reader = new StreamReader(yamlInputStream)) + { + yamlStream = new YamlStream(); + yamlStream.Load(reader); + } + + using var jsonWriter = new Utf8JsonWriter(jsonOutputStream, jsonWriterOptions); + + if (yamlStream.Documents.Count == 1) + { + VisitNode(jsonWriter, yamlStream.Documents[0].RootNode); + } + else + { + jsonWriter.WriteStartArray(); + foreach (var document in yamlStream.Documents) + { + VisitNode(jsonWriter, document.RootNode); + } + jsonWriter.WriteEndArray(); + } + } + + private static void VisitNode(Utf8JsonWriter writer, YamlNode node) + { + switch (node) + { + case YamlScalarNode scalarNode: + VisitScalarNode(writer, scalarNode); + break; + case YamlSequenceNode sequenceNode: + VisitSequenceNode(writer, sequenceNode); + break; + case YamlMappingNode mappingNode: + VisitMappingNode(writer, mappingNode); + break; + default: + writer.WriteNullValue(); + break; + } + } + + private static readonly Regex _booleanTruePattern = new Regex("^(true|y|yes|on)$"); + private static readonly Regex _booleanFalsePattern = new Regex("^(false|n|no|off)$"); + + private static void VisitScalarNode(Utf8JsonWriter writer, YamlScalarNode scalarNode) + { + bool forceImplicitPlain = false; + if (scalarNode.Style == ScalarStyle.Plain && scalarNode.Tag.IsEmpty && + scalarNode.Value != null) + { + forceImplicitPlain = scalarNode.Value.Length switch + { + // we have an implicit null value without a tag stating it, fake it out + 0 => true, + 1 => scalarNode.Value == "~", + 4 => scalarNode.Value == "null" || scalarNode.Value == "Null" || scalarNode.Value == "NULL", + // for backwards compatability we won't be setting the Value property to null + _ => false + }; + } + if (forceImplicitPlain && scalarNode.Style == ScalarStyle.Plain && string.IsNullOrEmpty(scalarNode.Value)) + { + writer.WriteNullValue(); + return; + } + if (scalarNode.Tag.IsEmpty && scalarNode.Value == null && (scalarNode.Style == ScalarStyle.Plain || scalarNode.Style == ScalarStyle.Any)) + { + writer.WriteNullValue(); + return; + } + if (scalarNode.Value == null) + { + writer.WriteNullValue(); + return; + } + + // @todo: Respect tags. + + if (scalarNode.Style == ScalarStyle.SingleQuoted || + scalarNode.Style == ScalarStyle.DoubleQuoted || + scalarNode.Style == ScalarStyle.Literal || + scalarNode.Style == ScalarStyle.Folded) + { + // All of these styles are explicitly string values. + writer.WriteStringValue(scalarNode.Value); + return; + } + + if (_booleanTruePattern.IsMatch(scalarNode.Value)) + { + writer.WriteBooleanValue(true); + return; + } + if (_booleanFalsePattern.IsMatch(scalarNode.Value)) + { + writer.WriteBooleanValue(false); + return; + } + + if (long.TryParse(scalarNode.Value, out var longValue)) + { + writer.WriteNumberValue(longValue); + return; + } + if (ulong.TryParse(scalarNode.Value, out var ulongValue)) + { + writer.WriteNumberValue(ulongValue); + return; + } + if (double.TryParse(scalarNode.Value, out var doubleValue)) + { + writer.WriteNumberValue(doubleValue); + return; + } + + writer.WriteStringValue(scalarNode.Value); + return; + } + + private static void VisitSequenceNode(Utf8JsonWriter writer, YamlSequenceNode sequenceNode) + { + writer.WriteStartArray(); + + foreach (var node in sequenceNode.Children) + { + VisitNode(writer, node); + } + + writer.WriteEndArray(); + } + + private static void VisitMappingNode(Utf8JsonWriter writer, YamlMappingNode mappingNode) + { + writer.WriteStartObject(); + + foreach (var kv in mappingNode.Children) + { + if (kv.Key is YamlScalarNode keyScalarNode && + !string.IsNullOrWhiteSpace(keyScalarNode.Value)) + { + writer.WritePropertyName(keyScalarNode.Value); + + VisitNode(writer, kv.Value); + } + } + + writer.WriteEndObject(); + } + } +} diff --git a/UET/UET.sln b/UET/UET.sln index 1e8062c3..c5a5c662 100644 --- a/UET/UET.sln +++ b/UET/UET.sln @@ -382,14 +382,22 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.KubernetesManager. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.KubernetesManager.HostedService.Rkm", "Redpoint.KubernetesManager.HostedService.Rkm\Redpoint.KubernetesManager.HostedService.Rkm.csproj", "{299D0B37-9D9F-46E7-9246-553F039A9604}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.KubernetesManager.PxeBoot", "Redpoint.KubernetesManager.PxeBoot\Redpoint.KubernetesManager.PxeBoot.csproj", "{3220A8E5-F0FA-434E-AF43-C96040BBF241}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.ThirdParty.GitHub.JPMikkers.Dhcp", "Lib\Redpoint.ThirdParty.GitHub.JPMikkers.Dhcp\Redpoint.ThirdParty.GitHub.JPMikkers.Dhcp.csproj", "{FF246122-1FE3-476E-89B7-7ED216001BDA}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.Kestrel", "Redpoint.Kestrel\Redpoint.Kestrel.csproj", "{3CAD52BD-B610-4005-AC79-347190FE4E2C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.KubernetesManager.Configuration", "Redpoint.KubernetesManager.Configuration\Redpoint.KubernetesManager.Configuration.csproj", "{5396072D-DE85-42EA-B18E-CF984EE129DD}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.Tpm", "Redpoint.Tpm\Redpoint.Tpm.csproj", "{7AAC1EA6-6F9C-478B-9B3C-13FFBF479FFA}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.Tpm.Tests", "Redpoint.Tpm.Tests\Redpoint.Tpm.Tests.csproj", "{11640620-314D-4548-929D-481E6B501DD8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.YamlToJson", "Redpoint.YamlToJson\Redpoint.YamlToJson.csproj", "{899C913B-7A34-4A03-8222-C8F3D6E81CBD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Redpoint.ThirdParty.Tftp.Net", "Lib\Redpoint.ThirdParty.Tftp.Net\Redpoint.ThirdParty.Tftp.Net.csproj", "{B4E20A7F-64F7-46B2-8180-C00DB24731E3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1048,6 +1056,10 @@ Global {299D0B37-9D9F-46E7-9246-553F039A9604}.Debug|Any CPU.Build.0 = Debug|Any CPU {299D0B37-9D9F-46E7-9246-553F039A9604}.Release|Any CPU.ActiveCfg = Release|Any CPU {299D0B37-9D9F-46E7-9246-553F039A9604}.Release|Any CPU.Build.0 = Release|Any CPU + {3220A8E5-F0FA-434E-AF43-C96040BBF241}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3220A8E5-F0FA-434E-AF43-C96040BBF241}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3220A8E5-F0FA-434E-AF43-C96040BBF241}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3220A8E5-F0FA-434E-AF43-C96040BBF241}.Release|Any CPU.Build.0 = Release|Any CPU {FF246122-1FE3-476E-89B7-7ED216001BDA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {FF246122-1FE3-476E-89B7-7ED216001BDA}.Debug|Any CPU.Build.0 = Debug|Any CPU {FF246122-1FE3-476E-89B7-7ED216001BDA}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -1056,6 +1068,10 @@ Global {3CAD52BD-B610-4005-AC79-347190FE4E2C}.Debug|Any CPU.Build.0 = Debug|Any CPU {3CAD52BD-B610-4005-AC79-347190FE4E2C}.Release|Any CPU.ActiveCfg = Release|Any CPU {3CAD52BD-B610-4005-AC79-347190FE4E2C}.Release|Any CPU.Build.0 = Release|Any CPU + {5396072D-DE85-42EA-B18E-CF984EE129DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5396072D-DE85-42EA-B18E-CF984EE129DD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5396072D-DE85-42EA-B18E-CF984EE129DD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5396072D-DE85-42EA-B18E-CF984EE129DD}.Release|Any CPU.Build.0 = Release|Any CPU {7AAC1EA6-6F9C-478B-9B3C-13FFBF479FFA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {7AAC1EA6-6F9C-478B-9B3C-13FFBF479FFA}.Debug|Any CPU.Build.0 = Debug|Any CPU {7AAC1EA6-6F9C-478B-9B3C-13FFBF479FFA}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -1064,6 +1080,14 @@ Global {11640620-314D-4548-929D-481E6B501DD8}.Debug|Any CPU.Build.0 = Debug|Any CPU {11640620-314D-4548-929D-481E6B501DD8}.Release|Any CPU.ActiveCfg = Release|Any CPU {11640620-314D-4548-929D-481E6B501DD8}.Release|Any CPU.Build.0 = Release|Any CPU + {899C913B-7A34-4A03-8222-C8F3D6E81CBD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {899C913B-7A34-4A03-8222-C8F3D6E81CBD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {899C913B-7A34-4A03-8222-C8F3D6E81CBD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {899C913B-7A34-4A03-8222-C8F3D6E81CBD}.Release|Any CPU.Build.0 = Release|Any CPU + {B4E20A7F-64F7-46B2-8180-C00DB24731E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B4E20A7F-64F7-46B2-8180-C00DB24731E3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B4E20A7F-64F7-46B2-8180-C00DB24731E3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B4E20A7F-64F7-46B2-8180-C00DB24731E3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1202,7 +1226,10 @@ Global {95FADB3D-2623-484E-BD1F-2C6BFFB977D2} = {22C39CFD-6FF1-4E7B-8EE4-CCCF1D2ED461} {9E347546-446E-4795-BA67-4C0D864C5501} = {22C39CFD-6FF1-4E7B-8EE4-CCCF1D2ED461} {299D0B37-9D9F-46E7-9246-553F039A9604} = {22C39CFD-6FF1-4E7B-8EE4-CCCF1D2ED461} + {3220A8E5-F0FA-434E-AF43-C96040BBF241} = {22C39CFD-6FF1-4E7B-8EE4-CCCF1D2ED461} {FF246122-1FE3-476E-89B7-7ED216001BDA} = {39698A12-7C9B-47F5-BA1A-9F4884A770AF} + {5396072D-DE85-42EA-B18E-CF984EE129DD} = {22C39CFD-6FF1-4E7B-8EE4-CCCF1D2ED461} + {B4E20A7F-64F7-46B2-8180-C00DB24731E3} = {39698A12-7C9B-47F5-BA1A-9F4884A770AF} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8598A278-509A-48A6-A7B3-3E3B0D1011F1} diff --git a/UET/uet/Commands/Internal/InternalCommand.cs b/UET/uet/Commands/Internal/InternalCommand.cs index 893b301e..5594adb1 100644 --- a/UET/uet/Commands/Internal/InternalCommand.cs +++ b/UET/uet/Commands/Internal/InternalCommand.cs @@ -2,6 +2,7 @@ { using Redpoint.CommandLine; using Redpoint.KubernetesManager.HostedService.Rkm; + using Redpoint.KubernetesManager.PxeBoot; using System.CommandLine; using UET.Commands.Internal.CIBuild; using UET.Commands.Internal.CMakeUbaRun; @@ -19,6 +20,7 @@ using UET.Commands.Internal.InstallPlatformSdk; using UET.Commands.Internal.ListCustomObjects; using UET.Commands.Internal.Patch; + using UET.Commands.Internal.PxeBoot; using UET.Commands.Internal.RegisterGitLabRunner; using UET.Commands.Internal.RemoteZfsServer; using UET.Commands.Internal.RemoteZfsTest; @@ -94,6 +96,7 @@ internal sealed class InternalCommand : ICommandDescriptorProvider<UetGlobalComm builder.AddCommand<VerifyDllFileIntegrityCommand>(); builder.AddCommand<WindowsImagingCommand>(); builder.AddCommand<TpmCommand>(); + builder.AddCommandWithoutGlobalContext<PxeBootCommand>(); builder.AddCommand<ListCustomObjectsCommand>(); var command = new Command("internal", "Internal commands used by UET when it needs to call back into itself."); diff --git a/UET/uet/Commands/Internal/Tpm/TpmCreateAikCommand.cs b/UET/uet/Commands/Internal/Tpm/TpmCreateAikCommand.cs index 3707dac1..3ce29f8e 100644 --- a/UET/uet/Commands/Internal/Tpm/TpmCreateAikCommand.cs +++ b/UET/uet/Commands/Internal/Tpm/TpmCreateAikCommand.cs @@ -2,6 +2,8 @@ { using Microsoft.Extensions.Logging; using Redpoint.CommandLine; + using Redpoint.Tpm; + using Redpoint.Tpm.Internal; using System.CommandLine; using System.CommandLine.Invocation; using System.Security.Cryptography; @@ -15,6 +17,10 @@ internal sealed class TpmCreateAikCommand : ICommandDescriptorProvider<UetGlobal public static CommandDescriptor<UetGlobalCommandContext> Descriptor => UetCommandDescriptor.NewBuilder() .WithOptions<Options>() .WithInstance<TpmCreateAikCommandInstance>() + .WithRuntimeServices((_, services, _) => + { + services.AddTpm(); + }) .WithCommand( builder => { @@ -24,16 +30,34 @@ internal sealed class TpmCreateAikCommand : ICommandDescriptorProvider<UetGlobal internal sealed class Options { + public Option<bool> UseService; + public Option<int> Length; + + public Options() + { + UseService = new Option<bool>("--use-service", "Use service."); + UseService.AddAlias("-s"); + + Length = new Option<int>("--length", "Length of generated data to protect."); + Length.AddAlias("-l"); + Length.SetDefaultValue(32); + } } private sealed class TpmCreateAikCommandInstance : ICommandInstance { private readonly ILogger<TpmCreateAikCommandInstance> _logger; + private readonly Options _options; + private readonly ITpmService _tpmService; public TpmCreateAikCommandInstance( - ILogger<TpmCreateAikCommandInstance> logger) + ILogger<TpmCreateAikCommandInstance> logger, + Options options, + ITpmService tpmService) { _logger = logger; + _options = options; + _tpmService = tpmService; } private class TpmAutoFlush : IDisposable @@ -54,6 +78,39 @@ public void Dispose() } public async Task<int> ExecuteAsync(ICommandInvocationContext context) + { + if (context.ParseResult.GetValueForOption(_options.UseService)) + { + return await ExecuteServiceAsync(context); + } + else + { + return await ExecuteRawAsync(context); + } + } + + private async Task<int> ExecuteServiceAsync(ICommandInvocationContext context) + { + var (ekPublicBytes, aikPublicBytes, operationHandles) = await _tpmService.CreateRequestAsync(); + + var generatedBytes = RandomNumberGenerator.GetBytes(context.ParseResult.GetValueForOption(_options.Length)); + _logger.LogInformation($"Generated bytes: {Convert.ToHexStringLower(generatedBytes)}"); + + var (envelopingKeyBytes, encryptedKey, encryptedData) = _tpmService.Authorize( + ekPublicBytes, + aikPublicBytes, + generatedBytes); + + var decryptedGeneratedBytes = _tpmService.DecryptSecretKey( + operationHandles, + envelopingKeyBytes, + encryptedKey, + encryptedData); + _logger.LogInformation($"Decrypted generated bytes: {Convert.ToHexStringLower(generatedBytes)}"); + return 0; + } + + private async Task<int> ExecuteRawAsync(ICommandInvocationContext context) { using Tpm2Device tpmDevice = OperatingSystem.IsWindows() ? new TbsDevice() : new LinuxTpmDevice(); tpmDevice.Connect(); @@ -106,15 +163,29 @@ public async Task<int> ExecuteAsync(ICommandInvocationContext context) var aikHandle = aikCreateResponse.handle; using var aikHandleAutoFlush = new TpmAutoFlush(tpm, aikHandle); + { + var ekPublicBytes = ekPublicKey.GetTpmRepresentation(); + var aikPublicBytes = aikPublicKey.GetTpmRepresentation(); + _logger.LogTrace($"CreateRequestAsync result (EK public bytes size: {ekPublicBytes.Length}, AIK public bytes size: {aikPublicBytes.Length}, AIK name hex: {Convert.ToHexStringLower(aikCreateResponse.outPublic.GetName())})"); + + ekPublicKey = new Marshaller(ekPublicBytes).Get<TpmPublic>(); + aikPublicKey = new Marshaller(aikPublicBytes).Get<TpmPublic>(); + } + + // Generate bytes. + var generatedBytes = RandomNumberGenerator.GetBytes(context.ParseResult.GetValueForOption(_options.Length)); + _logger.LogInformation($"Generated bytes: {Convert.ToHexStringLower(generatedBytes)}"); + // Create activation credentials. - _logger.LogInformation("Creating activation credentials for AIK..."); - var certInfo = ekPublicKey.CreateActivationCredentials( - tpm.GetRandom(32), + _logger.LogInformation($"Creating activation credentials for AIK (AIK public key name length: {aikPublicKey.GetName().Length})..."); + var envelopingKey = ekPublicKey.CreateActivationCredentials( + generatedBytes, aikPublicKey.GetName(), out var encryptedSecret); + var envelopingKeyBytes = envelopingKey.EnvelopingKeyToBytes(_logger); // Create sessions for AIK usage. - _logger.LogInformation("Activating AIK..."); + _logger.LogTrace($"Starting auth session for AIK..."); var aikSession = tpm.StartAuthSessionEx( TpmSe.Policy, alg); @@ -122,6 +193,7 @@ public async Task<int> ExecuteAsync(ICommandInvocationContext context) aikSession.RunPolicy(tpm, aikPolicyTree, "Activate"); // Determine policy tree for EK usage. + _logger.LogTrace($"Creating policy tree for EK..."); var ekPolicyTree = new PolicyTree(ekPublicKey.nameAlg); ekPolicyTree.SetPolicyRoot(new TpmPolicySecret( TpmRh.Endorsement, @@ -131,6 +203,7 @@ public async Task<int> ExecuteAsync(ICommandInvocationContext context) null)); // Create session for EK usage. + _logger.LogTrace($"Starting auth session for EK (alg: {ekPublicKey.nameAlg})..."); var ekSession = tpm.StartAuthSessionEx( TpmSe.Policy, ekPublicKey.nameAlg); @@ -138,13 +211,17 @@ public async Task<int> ExecuteAsync(ICommandInvocationContext context) ekSession.RunPolicy(tpm, ekPolicyTree); // Activate the AIK credential. - var recoveredSecretKey = tpm[aikSession, ekSession] + _logger.LogInformation($"Activating AIK... (enveloping key bytes size: {envelopingKeyBytes.Length}, encrypted Key: {encryptedSecret.Length})"); + envelopingKey = envelopingKeyBytes.BytesToEnvelopingKey(_logger); + var recoveredGeneratedBytes = tpm[aikSession, ekSession] .ActivateCredential( aikHandle, ekHandle, - certInfo, + envelopingKey, encryptedSecret); + _logger.LogInformation($"Decrypted generated bytes: {Convert.ToHexStringLower(recoveredGeneratedBytes)}"); + // Export PEM. _logger.LogInformation($"EK: {ToPem(ekPublicKey)}"); _logger.LogInformation($"AIK: {ToPem(aikPublicKey)}"); diff --git a/UET/uet/uet.csproj b/UET/uet/uet.csproj index 2a40719e..90b5dee3 100644 --- a/UET/uet/uet.csproj +++ b/UET/uet/uet.csproj @@ -57,6 +57,7 @@ <ProjectReference Include="..\Redpoint.KubernetesManager.HostedService.Containerd\Redpoint.KubernetesManager.HostedService.Containerd.csproj" /> <ProjectReference Include="..\Redpoint.KubernetesManager.HostedService.Kubelet\Redpoint.KubernetesManager.HostedService.Kubelet.csproj" /> <ProjectReference Include="..\Redpoint.KubernetesManager.HostedService.Rkm\Redpoint.KubernetesManager.HostedService.Rkm.csproj" /> + <ProjectReference Include="..\Redpoint.KubernetesManager.PxeBoot\Redpoint.KubernetesManager.PxeBoot.csproj" /> <ProjectReference Include="..\Redpoint.KubernetesManager\Redpoint.KubernetesManager.csproj" /> <ProjectReference Include="..\Redpoint.MSBuildResolution\Redpoint.MSBuildResolution.csproj" /> <ProjectReference Include="..\Redpoint.PathResolution\Redpoint.PathResolution.csproj" />