diff --git a/Sources/APIServer/ContainerDNSHandler.swift b/Sources/APIServer/ContainerDNSHandler.swift index 78a207467..dd0996283 100644 --- a/Sources/APIServer/ContainerDNSHandler.swift +++ b/Sources/APIServer/ContainerDNSHandler.swift @@ -76,10 +76,12 @@ struct ContainerDNSHandler: DNSHandler { } private func answerHost(question: Question) async throws -> ResourceRecord? { - guard let ipAllocation = try await networkService.lookup(hostname: question.name) else { + guard let ipAllocation = try await networkService.lookup(hostname: question.name), + let ipv4Address = ipAllocation.ipv4Address + else { return nil } - let ipv4 = ipAllocation.ipv4Address.address.description + let ipv4 = ipv4Address.address.description guard let ip = try? IPv4Address(ipv4) else { throw DNSResolverError.serverError("failed to parse IP address: \(ipv4)") } diff --git a/Sources/ContainerCommands/Builder/BuilderStatus.swift b/Sources/ContainerCommands/Builder/BuilderStatus.swift index 3512fa5e0..ac2bb67c7 100644 --- a/Sources/ContainerCommands/Builder/BuilderStatus.swift +++ b/Sources/ContainerCommands/Builder/BuilderStatus.swift @@ -85,7 +85,7 @@ private struct PrintableBuilder: ListDisplayable { snapshot.id, snapshot.configuration.image.reference, snapshot.status.rawValue, - snapshot.networks.map { $0.ipv4Address.description }.joined(separator: ","), + snapshot.networks.map { $0.ipv4Address?.description ?? "" }.joined(separator: ","), "\(snapshot.configuration.resources.cpus)", "\(snapshot.configuration.resources.memoryInBytes / (1024 * 1024)) MB", ] diff --git a/Sources/ContainerCommands/Container/ContainerList.swift b/Sources/ContainerCommands/Container/ContainerList.swift index 39121f465..8a56ae9c1 100644 --- a/Sources/ContainerCommands/Container/ContainerList.swift +++ b/Sources/ContainerCommands/Container/ContainerList.swift @@ -64,7 +64,7 @@ extension PrintableContainer: ListDisplayable { self.configuration.platform.os, self.configuration.platform.architecture, self.status.rawValue, - self.networks.map { $0.ipv4Address.description }.joined(separator: ","), + self.networks.map { $0.ipv4Address?.description ?? "" }.joined(separator: ","), "\(self.configuration.resources.cpus)", "\(self.configuration.resources.memoryInBytes / (1024 * 1024)) MB", self.startedDate?.ISO8601Format() ?? "", diff --git a/Sources/ContainerResource/Network/Attachment.swift b/Sources/ContainerResource/Network/Attachment.swift index bbadad759..bce860544 100644 --- a/Sources/ContainerResource/Network/Attachment.swift +++ b/Sources/ContainerResource/Network/Attachment.swift @@ -23,9 +23,11 @@ public struct Attachment: Codable, Sendable { /// The hostname associated with the attachment. public let hostname: String /// The CIDR address describing the interface IPv4 address, with the prefix length of the subnet. - public let ipv4Address: CIDRv4 + /// Nil for bridge-mode attachments where the address is assigned by DHCP at runtime. + public let ipv4Address: CIDRv4? /// The IPv4 gateway address. - public let ipv4Gateway: IPv4Address + /// Nil for bridge-mode attachments where the gateway is discovered via DHCP at runtime. + public let ipv4Gateway: IPv4Address? /// The CIDR address describing the interface IPv6 address, with the prefix length of the subnet. /// The address is nil if the IPv6 subnet could not be determined at network creation time. public let ipv6Address: CIDRv6? @@ -37,8 +39,8 @@ public struct Attachment: Codable, Sendable { public init( network: String, hostname: String, - ipv4Address: CIDRv4, - ipv4Gateway: IPv4Address, + ipv4Address: CIDRv4?, + ipv4Gateway: IPv4Address?, ipv6Address: CIDRv6?, macAddress: MACAddress?, mtu: UInt32? = nil @@ -72,16 +74,12 @@ public struct Attachment: Codable, Sendable { network = try container.decode(String.self, forKey: .network) hostname = try container.decode(String.self, forKey: .hostname) - if let address = try? container.decode(CIDRv4.self, forKey: .ipv4Address) { - ipv4Address = address - } else { - ipv4Address = try container.decode(CIDRv4.self, forKey: .address) - } - if let gateway = try? container.decode(IPv4Address.self, forKey: .ipv4Gateway) { - ipv4Gateway = gateway - } else { - ipv4Gateway = try container.decode(IPv4Address.self, forKey: .gateway) - } + ipv4Address = + try container.decodeIfPresent(CIDRv4.self, forKey: .ipv4Address) + ?? container.decodeIfPresent(CIDRv4.self, forKey: .address) + ipv4Gateway = + try container.decodeIfPresent(IPv4Address.self, forKey: .ipv4Gateway) + ?? container.decodeIfPresent(IPv4Address.self, forKey: .gateway) ipv6Address = try container.decodeIfPresent(CIDRv6.self, forKey: .ipv6Address) macAddress = try container.decodeIfPresent(MACAddress.self, forKey: .macAddress) mtu = try container.decodeIfPresent(UInt32.self, forKey: .mtu) @@ -93,8 +91,8 @@ public struct Attachment: Codable, Sendable { try container.encode(network, forKey: .network) try container.encode(hostname, forKey: .hostname) - try container.encode(ipv4Address, forKey: .ipv4Address) - try container.encode(ipv4Gateway, forKey: .ipv4Gateway) + try container.encodeIfPresent(ipv4Address, forKey: .ipv4Address) + try container.encodeIfPresent(ipv4Gateway, forKey: .ipv4Gateway) try container.encodeIfPresent(ipv6Address, forKey: .ipv6Address) try container.encodeIfPresent(macAddress, forKey: .macAddress) try container.encodeIfPresent(mtu, forKey: .mtu) diff --git a/Sources/ContainerXPC/XPCMessage.swift b/Sources/ContainerXPC/XPCMessage.swift index 3c6a3dca8..d5f3ba385 100644 --- a/Sources/ContainerXPC/XPCMessage.swift +++ b/Sources/ContainerXPC/XPCMessage.swift @@ -275,6 +275,12 @@ extension XPCMessage { } } + public func xpcDictionary(key: String) -> xpc_object_t? { + lock.withLock { + xpc_dictionary_get_dictionary(self.object, key) + } + } + public func endpoint(key: String) -> xpc_endpoint_t? { lock.withLock { xpc_dictionary_get_value(self.object, key) diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 994933489..b759b8a60 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -27,12 +27,8 @@ import ContainerizationExtras import Foundation import Logging -enum Variant: String, ExpressibleByArgument { - case reserved - case allocationOnly -} - extension NetworkMode: ExpressibleByArgument {} +extension NetworkVariant: ExpressibleByArgument {} extension NetworkVmnetHelper { struct Start: AsyncParsableCommand { @@ -60,17 +56,26 @@ extension NetworkVmnetHelper { var ipv6Subnet: String? @Option(name: .long, help: "Variant of the network helper to use.") - var variant: Variant = { + var variant: NetworkVariant = { guard #available(macOS 26, *) else { return .allocationOnly } return .reserved }() + @Option(name: .customLong("option-json"), help: "UTF8-encoded JSON string that contains the configuration options for this network") + var stringifiedOptions: String = "{}" + var logRoot = LogRoot.path func run() async throws { let commandName = NetworkVmnetHelper._commandName + guard let encodedOptions = stringifiedOptions.data(using: .utf8), + let options = try? JSONSerialization.jsonObject(with: encodedOptions) as? [String: String] + else { + throw ContainerizationError(.invalidArgument, message: "failed to decode network configuration options from JSON string") + } + let logPath = logRoot.map { $0.appending("\(commandName)-\(id).log") } let log = ServiceLogger.bootstrap(category: "NetworkVmnetHelper", metadata: ["id": "\(id)"], debug: debug, logPath: logPath) log.info("starting helper", metadata: ["name": "\(commandName)"]) @@ -89,7 +94,9 @@ extension NetworkVmnetHelper { ipv4Subnet: ipv4Subnet, ipv6Subnet: ipv6Subnet, plugin: NetworkVmnetHelper._commandName, - options: ["variant": self.variant.rawValue] + options: options.merging( + ["variant": self.variant.rawValue], + uniquingKeysWith: { _, new in new }) ) let network = try Self.createNetwork( configuration: configuration, @@ -97,7 +104,7 @@ extension NetworkVmnetHelper { log: log ) try await network.start() - let service = try await DefaultNetworkService(network: network, log: log) + let service: NetworkService = try await Self.createNetworkService(network: network, variant: variant, log: log) let harness = NetworkHarness(service: service) let xpc = XPCServer( identifier: serviceIdentifier, @@ -122,7 +129,16 @@ extension NetworkVmnetHelper { } } - private static func createNetwork(configuration: NetworkConfiguration, variant: Variant, log: Logger) throws -> Network { + private static func createNetworkService(network: Network, variant: NetworkVariant, log: Logger) async throws -> NetworkService { + switch variant { + case .bridged, .bridgedViaHelper: + return try await BridgeNetworkService(network: network, variant: variant, log: log) + default: + return try await DefaultNetworkService(network: network, log: log) + } + } + + private static func createNetwork(configuration: NetworkConfiguration, variant: NetworkVariant, log: Logger) throws -> Network { switch variant { case .allocationOnly: return try AllocationOnlyVmnetNetwork(configuration: configuration, log: log) @@ -134,6 +150,8 @@ extension NetworkVmnetHelper { ) } return try ReservedVmnetNetwork(configuration: configuration, log: log) + case .bridged, .bridgedViaHelper: + return try BridgedVmnetNetwork(configuration: configuration, log: log) } } } diff --git a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift index 3c7938b8e..796bf2aff 100644 --- a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift +++ b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift @@ -68,6 +68,8 @@ extension RuntimeLinuxHelper { NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "allocationOnly"): IsolatedInterfaceStrategy() ] if #available(macOS 26, *) { + interfaceStrategies[NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "bridged")] = BridgedInterfaceStrategy(log: log) + interfaceStrategies[NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "bridgedViaHelper")] = BridgedInterfaceStrategy(log: log) interfaceStrategies[NetworkInterfaceKey(plugin: "container-network-vmnet", variant: "reserved")] = NonisolatedInterfaceStrategy(log: log) } diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 92655d229..20f8de2d9 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -428,6 +428,17 @@ public actor NetworksService { args += ["--variant", variant] } + // TODO: variant could possibly stay inside the options and does not need to be a dedicated commandline argument? + let options = configuration.options.filter({ key, _ in key != "variant" }) + if !options.isEmpty { + guard let encodedOptions = try? JSONSerialization.data(withJSONObject: options), + let stringifiedOptions = String(data: encodedOptions, encoding: .utf8) + else { + throw ContainerizationError(.internalError, message: "failed to encode network configuration options as json string") + } + args += ["--option-json", stringifiedOptions] + } + let entityPath = try store.entityPath(configuration.id) try pluginLoader.registerWithLaunchd( plugin: networkPlugin, diff --git a/Sources/Services/Network/Client/BridgeNetworkKeys.swift b/Sources/Services/Network/Client/BridgeNetworkKeys.swift new file mode 100644 index 000000000..fd600f9bd --- /dev/null +++ b/Sources/Services/Network/Client/BridgeNetworkKeys.swift @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +public enum BridgeNetworkKeys: String { + case hostInterface + case enableTso + case enableChecksumOffload + case bufferedPacketCount + case sandboxEndpoint +} diff --git a/Sources/Services/Network/Client/NetworkVariant.swift b/Sources/Services/Network/Client/NetworkVariant.swift new file mode 100644 index 000000000..a557081a1 --- /dev/null +++ b/Sources/Services/Network/Client/NetworkVariant.swift @@ -0,0 +1,22 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +public enum NetworkVariant: String, Sendable { + case reserved + case allocationOnly + case bridged + case bridgedViaHelper +} diff --git a/Sources/Services/Network/Server/BridgeNetworkService.swift b/Sources/Services/Network/Server/BridgeNetworkService.swift new file mode 100644 index 000000000..ebf6b629c --- /dev/null +++ b/Sources/Services/Network/Server/BridgeNetworkService.swift @@ -0,0 +1,488 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkClient +import ContainerResource +import ContainerXPC +import ContainerizationError +import ContainerizationExtras +import Foundation +import Logging +import vmnet + +internal final class Bridge { + + internal final class PacketBuffer { + public let buffers: UnsafeMutableRawPointer + public let iovs: UnsafeMutablePointer + public let packets: UnsafeMutablePointer + + public let maxPacketSize: Int + public let maxPktCount: Int + + public init(maxPktCount: Int, maxPktSize: Int) { + self.buffers = .allocate(byteCount: maxPktCount * maxPktSize, alignment: 16) + self.iovs = .allocate(capacity: maxPktCount) + self.packets = .allocate(capacity: maxPktCount) + for i in 0..sandbox + private let hostPacketBuffer: PacketBuffer + // sandbox->host + private let sandboxPacketBuffer: PacketBuffer + + public init( + identifier: String, + hostInterface: String, + logger: Logger, + enableTso: Bool, + enableChecksumOffload: Bool, + bufferedPacketCount: Int + ) throws { + var fds: [Int32] = [-1, -1] + let rc = fds.withUnsafeMutableBufferPointer { + socketpair(AF_UNIX, SOCK_DGRAM, 0, $0.baseAddress) + } + guard rc == 0 else { + throw ContainerizationError(.internalError, message: "unable to create socket pairs for UNIX bridge endpoints") + } + + let networkEndpoint = FileHandle(fileDescriptor: fds[0], closeOnDealloc: true) + let sandboxEndpoint = FileHandle(fileDescriptor: fds[1], closeOnDealloc: true) + + // Make our endpoint side non blocking, so the read source can drain it in a loop. + let flags = fcntl(networkEndpoint.fileDescriptor, F_GETFL) + _ = fcntl(networkEndpoint.fileDescriptor, F_SETFL, flags | O_NONBLOCK) + + // vmnet interface descriptor. + let desc = xpc_dictionary_create(nil, nil, 0) + xpc_dictionary_set_uint64( + desc, vmnet_operation_mode_key, + UInt64(vmnet.operating_modes_t.VMNET_BRIDGED_MODE.rawValue) + ) + xpc_dictionary_set_string(desc, vmnet_shared_interface_name_key, hostInterface) + xpc_dictionary_set_bool(desc, vmnet_enable_tso_key, enableTso) + xpc_dictionary_set_bool( + desc, vmnet_enable_checksum_offload_key, + enableChecksumOffload) + + // Unfortunately, apparently not understood currently - otherwise this would maybe provide us the opportunity to + // set a custom MAC address + // xpc_dictionary_set_bool(desc, vmnet_allocate_mac_address_key, false) + + // start interface synchronously — vmnet posts the completion on `queue`. + let queue = DispatchQueue(label: identifier, qos: .userInitiated) + let sema = DispatchSemaphore(value: 0) + var startStatus: vmnet_return_t = .VMNET_FAILURE + var maxPacketSize = 0 + var mtu: UInt32 = 0 + var mac = "" + + let iface = vmnet_start_interface(desc, queue) { status, param in + startStatus = status + if status == .VMNET_SUCCESS, let param = param { + maxPacketSize = Int(xpc_dictionary_get_uint64(param, vmnet_max_packet_size_key)) + mtu = UInt32(xpc_dictionary_get_uint64(param, vmnet_mtu_key)) + if let cstr = xpc_dictionary_get_string(param, vmnet_mac_address_key) { + mac = String(cString: cstr) + } + } + sema.signal() + } + sema.wait() + + guard startStatus == .VMNET_SUCCESS, let iface = iface else { + throw ContainerizationError(.internalError, message: "unable to start vmnet bridge interface") + } + guard let macAddress = try? MACAddress(mac) else { + throw ContainerizationError(.internalError, message: "vmnet returned an invalid mac address for vmnet bridge interface") + } + + self.log = logger + + self.identifier = identifier + self.networkEndpoint = networkEndpoint + self.sandboxEndpoint = sandboxEndpoint + self.macAddress = macAddress + self.mtu = mtu + self.interfaceRef = iface + self.queue = queue + + self.hostPacketBuffer = PacketBuffer(maxPktCount: bufferedPacketCount, maxPktSize: maxPacketSize) + self.sandboxPacketBuffer = PacketBuffer(maxPktCount: 1, maxPktSize: maxPacketSize) + } + + public func start() throws { + guard self.readSource == nil, !stopped else { + throw ContainerizationError(.invalidState, message: "bridge was already started") + } + + // TODO: despite Swift's claim that there are no function calls inside the event handlers that could throw exceptions, + // this is unfortunately simply not true. this is possibly connected due to some Objective-C legacy behavior + // of the vmnet_interface API. + // further investigation in catching these exceptions and making them visible might prove useful, since + // otherwise the network service can silently crash without any trace in the logs. the sandbox service will + // also keep running and will only notice that its interface has lost its carrier (quite fitting). only the + // console logging does provide then a crash dump including stack trace featuring some message like this: + // + // *** Terminating app due to uncaught exception 'NSFileHandleOperationException', reason: '*** -[NSConcreteFileHandle fileDescriptor]: Bad file descriptor' + + // host -> vm: vmnet fires an event when packets are buffered. + let st = vmnet_interface_set_event_callback( + interfaceRef, .VMNET_INTERFACE_PACKETS_AVAILABLE, queue + ) { [weak self] _, _ in + self?.relayFromHostToSandbox() + } + guard st == .VMNET_SUCCESS else { + throw ContainerizationError(.internalError, message: "vmnet_interface_set_event_callback failed") + } + + // vm -> host: dispatch source on the socket. + let src = DispatchSource.makeReadSource(fileDescriptor: networkEndpoint.fileDescriptor, queue: queue) + src.setEventHandler { [weak self] in self?.relayFromSandboxToHost() } + src.resume() + self.readSource = src + } + + public func stop() throws { + guard !stopped else { + throw ContainerizationError(.invalidState, message: "bridge was already stopped") + } + stopped = true + + guard let readSource else { + throw ContainerizationError(.internalError, message: "read source for network endpoint has not been created") + } + readSource.cancel() + + let sema = DispatchSemaphore(value: 0) + let st = vmnet_stop_interface(interfaceRef, queue, { _ in sema.signal() }) + sema.wait() + + guard st == .VMNET_SUCCESS else { + throw ContainerizationError(.internalError, message: "failed to stop vmnet bridge interface") + } + } + + // bridge input and output helpers + private func readFromHost() -> Int32 { + hostPacketBuffer.reset() + var count = Int32(hostPacketBuffer.maxPktCount) + let st = vmnet_read(interfaceRef, hostPacketBuffer.packets, &count) + guard st == .VMNET_SUCCESS, count > 0 else { + return -1 + } + return count + } + + private func writeToSandbox(count: Int) -> Int { + var written = 0 + for i in 0.. 0 else { + log.warning("received invalid number of packets from host: \(pktsIn)") + return + } + let pktsOut = writeToSandbox(count: pktsIn) + guard pktsOut == pktsIn else { + log.warning("could not forward all packets to sandbox: \(pktsIn) != \(pktsOut)") + return + } + } + + private func relayFromSandboxToHost() { + // vmnet-helper uses Apple-private system calls sendmsg_x/recmsg_x to forward multiple packets + // in a single system call on the bridge... that sounds intuitively as it will likely achieve quite a + // performance improvement, but it also might break without further notice + // + // in doubt: this is just a fallback for the likely even more optimized internal VirtualizationFramework + // bridging functionality + let pktSize = read( + networkEndpoint.fileDescriptor, sandboxPacketBuffer.buffers, + sandboxPacketBuffer.maxPacketSize) + guard pktSize > 0 else { + log.warning("failed to read from sandbox socket: \(pktSize) - EOF?") + return + } + + sandboxPacketBuffer.iovs.pointee.iov_len = pktSize + sandboxPacketBuffer.packets.pointee.vm_pkt_size = pktSize + sandboxPacketBuffer.packets.pointee.vm_flags = 0 + + var pktCount: Int32 = 1 + guard vmnet_write(interfaceRef, sandboxPacketBuffer.packets, &pktCount) == .VMNET_SUCCESS, pktCount == 1 else { + log.warning("failed to forward packet from sandbox to host") + return + } + } + +} + +public actor BridgeNetworkService: NetworkService { + + private let network: any Network + private let variant: NetworkVariant + private let log: Logger + private var allocationsBySession: [XPCServerSession: [(hostname: String, bridge: Bridge?)]] + private var macAddresses: [String: MACAddress] + + /// Set up a network service for the specified network. + public init( + network: any Network, + variant: NetworkVariant, + log: Logger + ) async throws { + guard await network.status != nil else { + throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") + } + + self.network = network + self.variant = variant + self.log = log + self.allocationsBySession = [:] + self.macAddresses = [:] + } + + @Sendable + public func status() async throws -> NetworkStatus { + guard let status = await network.status else { + throw ContainerizationError(.invalidState, message: "network \(network.id) is not running") + } + return status + } + + @Sendable + public func allocate( + hostname: String, + macAddress: MACAddress?, + session: XPCServerSession + ) async throws -> (attachment: Attachment, additionalData: XPCMessage?) { + log.debug("enter", metadata: ["func": "\(#function)"]) + defer { log.debug("exit", metadata: ["func": "\(#function)"]) } + + guard await network.status != nil else { + throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") + } + + // retrieve configuration from the network + // this is a bit awkward compared to the regular purpose of the network in managing allocations, since this is all + // handled externally by the network infrastructure on the other side of the bridge. therefore the network is for us + // simply a quite overblown struct/adapter that passes information along that are still unknown during creation time. + var additionalData: XPCMessage? + try network.withAdditionalData { + additionalData = $0 + } + + guard let bridgeDictionary = additionalData?.xpcDictionary(key: NetworkKeys.additionalData.rawValue) else { + throw ContainerizationError(.internalError, message: "did not receive bridge dictionary in network additional message") + } + let bridgeMessage = XPCMessage(object: bridgeDictionary) + + guard let hostInterface = bridgeMessage.string(key: BridgeNetworkKeys.hostInterface.rawValue) + else { + throw ContainerizationError(.invalidState, message: "bridge network is not assigned to a host interface") + } + + var attachment: Attachment + var bridge: Bridge? + if variant == .bridged { + let macAddress = macAddress ?? MACAddress((UInt64.random(in: 0...UInt64.max) & 0x0cff_ffff_ffff) | 0xf200_0000_0000) + attachment = Attachment( + network: network.id, + hostname: hostname, + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: macAddress, + mtu: nil, + ) + bridge = nil + } else { + let enableTso = + bridgeMessage.dataNoCopy(key: BridgeNetworkKeys.enableTso.rawValue) != nil ? bridgeMessage.bool(key: BridgeNetworkKeys.enableTso.rawValue) : nil + let enableChecksumOffload = + bridgeMessage.dataNoCopy(key: BridgeNetworkKeys.enableChecksumOffload.rawValue) != nil + ? bridgeMessage.bool(key: BridgeNetworkKeys.enableChecksumOffload.rawValue) : nil + let batchSize = + bridgeMessage.dataNoCopy(key: BridgeNetworkKeys.bufferedPacketCount.rawValue) != nil + ? Int(bridgeMessage.uint64(key: BridgeNetworkKeys.bufferedPacketCount.rawValue)) : nil + + bridge = try! Bridge( + identifier: "vmnet-bridge-\(network.id)-\(hostname)", hostInterface: hostInterface, logger: log, enableTso: enableTso ?? false, + enableChecksumOffload: enableChecksumOffload ?? false, bufferedPacketCount: batchSize ?? 64) + try bridge!.start() + + attachment = Attachment( + network: network.id, + hostname: hostname, + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: bridge!.macAddress, + mtu: bridge!.mtu + ) + + // add the sandbox-side fd endpoint to the bridge message + bridgeMessage.set(key: BridgeNetworkKeys.sandboxEndpoint.rawValue, value: bridge!.sandboxEndpoint) + + // close the sandbox endpoint's fd, since we do not need it anymore and it has been just dup-ed + // when we attached it to the XPC message above + try bridge!.sandboxEndpoint.close() + } + + log.info( + "allocated attachment", + metadata: [ + "hostname": "\(hostname)", + "hostInterface": "\(hostInterface)", + "macAddress": "\(attachment.macAddress!)", + "bridge": "\(bridge?.identifier ?? "vf-based")", + ]) + + if allocationsBySession[session] == nil { + allocationsBySession[session] = [] + await session.onDisconnect { [weak self] in + await self?.releaseSession(session) + } + } + allocationsBySession[session]!.append((hostname: hostname, bridge: bridge)) + macAddresses[hostname] = attachment.macAddress + + return (attachment: attachment, additionalData: additionalData) + } + + private func releaseSession(_ session: XPCServerSession) async { + guard let allocations = allocationsBySession.removeValue(forKey: session) else { + return + } + for allocation: (hostname: String, bridge: Bridge?) in allocations { + macAddresses[allocation.hostname] = nil + guard let bridge = allocation.bridge else { + continue + } + do { + try bridge.stop() + } catch let error as ContainerizationError { + log.error( + "failed to stop bridge for attachment", + metadata: [ + "error": Logger.MetadataValue.string(error.message) + ]) + } catch { + assert(false, "should never happen") + } + } + log.info("released session", metadata: ["allocations": "\(allocations.count)"]) + } + + @Sendable + public func lookup(hostname: String) async throws -> Attachment? { + log.debug("enter", metadata: ["func": "\(#function)"]) + defer { log.debug("exit", metadata: ["func": "\(#function)"]) } + + guard await network.status != nil else { + throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") + } + guard let macAddress = macAddresses[hostname] else { + log.warning("MAC address for hostname \(hostname) is not a valid attachment") + return nil + } + + let attachment = Attachment( + network: network.id, + hostname: hostname, + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: macAddress + ) + log.info( + "lookup attachment", + metadata: [ + "hostname": "\(hostname)", + "macAddress": "\(macAddress)", + ]) + + return attachment + } +} diff --git a/Sources/Services/Network/Server/DefaultNetworkService.swift b/Sources/Services/Network/Server/DefaultNetworkService.swift index 29f3fbe82..efbfc883b 100644 --- a/Sources/Services/Network/Server/DefaultNetworkService.swift +++ b/Sources/Services/Network/Server/DefaultNetworkService.swift @@ -83,8 +83,8 @@ public actor DefaultNetworkService: NetworkService { "allocated attachment", metadata: [ "hostname": "\(hostname)", - "ipv4Address": "\(attachment.ipv4Address)", - "ipv4Gateway": "\(attachment.ipv4Gateway)", + "ipv4Address": "\(attachment.ipv4Address?.description ?? "none")", + "ipv4Gateway": "\(attachment.ipv4Gateway?.description ?? "none")", "ipv6Address": "\(attachment.ipv6Address?.description ?? "unavailable")", "macAddress": "\(attachment.macAddress?.description ?? "unspecified")", ]) diff --git a/Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift b/Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift new file mode 100644 index 000000000..8ac016eea --- /dev/null +++ b/Sources/Services/NetworkVmnet/Server/BridgedVmnetNetwork.swift @@ -0,0 +1,155 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkClient +import ContainerNetworkServer +import ContainerResource +import ContainerXPC +import Containerization +import ContainerizationError +import ContainerizationExtras +import Foundation +import Logging +import Synchronization +import Virtualization +import XPC + +public final class BridgedVmnetNetwork: ContainerNetworkServer.Network { + // FIXME: NetworkPluginStatus requires non-optional ipv4Subnet/ipv4Gateway; use placeholder + // values until the type is refactored to make them optional. + private static let placeholderSubnet = try! CIDRv4("0.0.0.0/0") + private static let placeholderGateway = IPv4Address(0) + private static let placeholderSubnetv6 = try! CIDRv6("::/0") + + private let log: Logger + private let configuration: NetworkConfiguration + private let hostInterface: String + private let statusMutex: Mutex + private let enableTso: Bool? + private let enableChecksumOffload: Bool? + private let bufferedPacketCount: Int? + + public init(configuration: NetworkConfiguration, log: Logger) throws { + // TODO: ignore the network type for now - having both network type and plugin variant is a bit awkward... + // I imagine this will get resolved later on + // guard configuration.mode == .bridge else { + // throw ContainerizationError(.unsupported, message: "invalid network mode \(configuration.mode)") + // } + guard let variantStr = configuration.options["variant"], + let variant = NetworkVariant(rawValue: variantStr), + variant == .bridged || variant == .bridgedViaHelper + else { + throw ContainerizationError( + .unsupported, + message: "invalid network variant \(configuration.options["variant"] ?? "unspecified")") + } + + guard let hostInterface = configuration.options["hostInterface"] else { + throw ContainerizationError(.invalidArgument, message: "hostInterface must be given as a plugin option") + } + let available = VZBridgedNetworkInterface.networkInterfaces.map { $0.identifier } + guard available.contains(hostInterface) else { + let list = available.isEmpty ? "none available" : available.joined(separator: ", ") + throw ContainerizationError(.invalidArgument, message: "no host interface '\(hostInterface)'; available: \(list)") + } + self.hostInterface = hostInterface + + self.configuration = configuration + self.log = log + self.statusMutex = Mutex(nil) + + if variant == .bridged { + self.enableTso = nil + self.enableChecksumOffload = nil + self.bufferedPacketCount = nil + } else { + let rawEnableTso = configuration.options[BridgeNetworkKeys.enableTso.rawValue] + if rawEnableTso != nil { + guard let enableTso = Bool(rawEnableTso!) else { + throw ContainerizationError(.invalidArgument, message: "enableTso must be a boolean") + } + self.enableTso = enableTso + } else { + self.enableTso = nil + } + + let rawEnableChecksumOffload: String? = configuration.options[BridgeNetworkKeys.enableChecksumOffload.rawValue] + if rawEnableChecksumOffload != nil { + guard let enableChecksumOffload = Bool(rawEnableChecksumOffload!) else { + throw ContainerizationError(.invalidArgument, message: "enableChecksumOffload must be a boolean") + } + self.enableChecksumOffload = enableChecksumOffload + } else { + self.enableChecksumOffload = nil + } + + let rawBufferedPacketCount = configuration.options[BridgeNetworkKeys.bufferedPacketCount.rawValue] + if rawBufferedPacketCount != nil { + guard let bufferedPacketCount = Int(rawBufferedPacketCount!), bufferedPacketCount > 0 else { + throw ContainerizationError(.invalidArgument, message: "bufferedPacketCount must be a positive integer") + } + self.bufferedPacketCount = bufferedPacketCount + } else { + self.bufferedPacketCount = nil + } + } + } + + public nonisolated var id: String { configuration.id } + + public var status: NetworkStatus? { + self.statusMutex.withLock { $0 } + } + + public nonisolated func withAdditionalData(_ handler: (XPCMessage?) throws -> Void) throws { + let bridgeConfigMsg = XPCMessage(object: xpc_dictionary_create_empty()) + bridgeConfigMsg.set(key: BridgeNetworkKeys.hostInterface.rawValue, value: hostInterface) + if self.enableTso != nil { + bridgeConfigMsg.set(key: BridgeNetworkKeys.enableTso.rawValue, value: self.enableTso!) + } + if self.enableChecksumOffload != nil { + bridgeConfigMsg.set(key: BridgeNetworkKeys.enableChecksumOffload.rawValue, value: self.enableChecksumOffload!) + } + if self.bufferedPacketCount != nil { + bridgeConfigMsg.set(key: BridgeNetworkKeys.bufferedPacketCount.rawValue, value: UInt64(self.bufferedPacketCount!)) + } + + let msg: XPCMessage = XPCMessage(object: xpc_dictionary_create_empty()) + msg.set(key: NetworkKeys.additionalData.rawValue, value: bridgeConfigMsg.underlying) + try handler(msg) + } + + public func start() async throws { + try statusMutex.withLock { status in + + guard status == nil else { + throw ContainerizationError(.invalidArgument, message: "cannot start network \(configuration.id): already started") + } + + status = NetworkStatus( + ipv4Subnet: Self.placeholderSubnet, ipv4Gateway: Self.placeholderGateway, ipv6Subnet: Self.placeholderSubnetv6 + ) + + log.info( + "started bridged network", + metadata: [ + "id": "\(configuration.id)", + "hostInterface": "\(hostInterface)", + ] + ) + } + } +} diff --git a/Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift b/Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift new file mode 100644 index 000000000..29ff55f1b --- /dev/null +++ b/Sources/Services/RuntimeLinux/Server/BridgedInterfaceStrategy.swift @@ -0,0 +1,56 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkClient +import ContainerResource +import ContainerRuntimeClient +import ContainerXPC +import Containerization +import ContainerizationError +import Logging + +/// Interface strategy for containers that use macOS's custom network feature. +@available(macOS 26, *) +public struct BridgedInterfaceStrategy: InterfaceStrategy { + private let log: Logger + + public init(log: Logger) { + self.log = log + } + + public func toInterface(attachment: Attachment, interfaceIndex: Int, additionalData: XPCMessage?) throws -> Interface { + guard let bridgeDictionary = additionalData?.xpcDictionary(key: NetworkKeys.additionalData.rawValue) else { + throw ContainerizationError(.internalError, message: "did not receive bridge dictionary in interface additional message") + } + let bridgeData = XPCMessage(object: bridgeDictionary) + + guard let ifaceName = bridgeData.string(key: BridgeNetworkKeys.hostInterface.rawValue) + else { + throw ContainerizationError(.invalidState, message: "bridge network missing host interface name") + } + guard let containerBridgeEndpoint = bridgeData.fileHandle(key: BridgeNetworkKeys.sandboxEndpoint.rawValue) else { + return BridgedNetworkInterface( + hostInterfaceName: ifaceName, + macAddress: attachment.macAddress + ) + } + + return FileHandleNetworkInterface( + fileHandle: containerBridgeEndpoint, + macAddress: attachment.macAddress + ) + } +} diff --git a/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift b/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift index d180fac5c..ac975b48e 100644 --- a/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift +++ b/Sources/Services/RuntimeLinux/Server/IsolatedInterfaceStrategy.swift @@ -18,6 +18,7 @@ import ContainerResource import ContainerRuntimeClient import ContainerXPC import Containerization +import ContainerizationError /// Isolated container network interface strategy. This strategy prohibits /// container to container networking, but it is the only approach that @@ -25,10 +26,13 @@ import Containerization public struct IsolatedInterfaceStrategy: InterfaceStrategy { public init() {} - public func toInterface(attachment: Attachment, interfaceIndex: Int, additionalData: XPCMessage?) -> Interface { + public func toInterface(attachment: Attachment, interfaceIndex: Int, additionalData: XPCMessage?) throws -> Interface { + guard let ipv4Address = attachment.ipv4Address else { + throw ContainerizationError(.invalidState, message: "NAT attachment missing IPv4 address") + } let ipv4Gateway = interfaceIndex == 0 ? attachment.ipv4Gateway : nil return NATInterface( - ipv4Address: attachment.ipv4Address, + ipv4Address: ipv4Address, ipv4Gateway: ipv4Gateway, macAddress: attachment.macAddress, // https://github.com/apple/containerization/pull/38 diff --git a/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift b/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift index 38c1b6764..ea1602d65 100644 --- a/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift +++ b/Sources/Services/RuntimeLinux/Server/NonisolatedInterfaceStrategy.swift @@ -42,10 +42,13 @@ public struct NonisolatedInterfaceStrategy: InterfaceStrategy { throw ContainerizationError(.invalidState, message: "cannot deserialize custom network reference, status \(status)") } + guard let ipv4Address = attachment.ipv4Address else { + throw ContainerizationError(.invalidState, message: "NAT attachment missing IPv4 address") + } log.info("creating NATNetworkInterface with network reference") let ipv4Gateway = interfaceIndex == 0 ? attachment.ipv4Gateway : nil return NATNetworkInterface( - ipv4Address: attachment.ipv4Address, + ipv4Address: ipv4Address, ipv4Gateway: ipv4Gateway, reference: networkRef, macAddress: attachment.macAddress, diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 1f23e0966..356adb0a7 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -162,12 +162,6 @@ public actor RuntimeService { var kernel = try bundle.kernel kernel.commandLine.kernelArgs.append("oops=panic") kernel.commandLine.kernelArgs.append("lsm=lockdown,capability,landlock,yama,apparmor") - let vmm = VZVirtualMachineManager( - kernel: kernel, - initialFilesystem: bundle.initialFilesystem.asMount, - rosetta: config.rosetta, - logger: self.log - ) let networkBootstrapInfos = try message.networkBootstrapInfos() @@ -196,6 +190,12 @@ public actor RuntimeService { mtu: mtu ) } + + // enable DHCP if the attachment has not been assigned an explicit IP address + if attachment.ipv4Address == nil { + kernel.commandLine.kernelArgs.append("ip=::::\(attachment.hostname):eth\(index):dhcp") + } + guard let iStrategy = self.interfaceStrategies[NetworkInterfaceKey(plugin: info.plugin, variant: info.options["variant"])] else { throw ContainerizationError( .internalError, @@ -214,17 +214,23 @@ public actor RuntimeService { throw error } + let vmm = VZVirtualMachineManager( + kernel: kernel, + initialFilesystem: bundle.initialFilesystem.asMount, + rosetta: config.rosetta, + logger: self.log + ) + // Dynamically configure the DNS nameserver from a network if no explicit configuration + // For bridge networks (unspecified gateway), nameservers and domain come from DHCP (/proc/net/pnp). if let dns = config.dns, dns.nameservers.isEmpty { let defaultNameservers = self.getDefaultNameservers(from: attachments) - if !defaultNameservers.isEmpty { - config.dns = ContainerConfiguration.DNSConfiguration( - nameservers: defaultNameservers, - domain: dns.domain, - searchDomains: dns.searchDomains, - options: dns.options - ) - } + config.dns = ContainerConfiguration.DNSConfiguration( + nameservers: defaultNameservers.isEmpty ? dns.nameservers : defaultNameservers, + domain: defaultNameservers.isEmpty ? nil : dns.domain, + searchDomains: dns.searchDomains, + options: dns.options + ) } let stdio = message.stdio() @@ -261,11 +267,10 @@ public actor RuntimeService { // NOTE: We can support a user providing new entries eventually, but for now craft // a default /etc/hosts. var hostsEntries = [Hosts.Entry.localHostIPV4()] - if !interfaces.isEmpty { - let primaryIfaceAddr = interfaces[0].ipv4Address + if !interfaces.isEmpty, let ipv4Address = interfaces[0].ipv4Address, !ipv4Address.address.isUnspecified { hostsEntries.append( Hosts.Entry( - ipAddress: primaryIfaceAddr.address.description, + ipAddress: ipv4Address.address.description, hostnames: [czConfig.hostname ?? id], )) } @@ -877,7 +882,10 @@ public actor RuntimeService { let containerIPAddress: String switch publishedPort.hostAddress { case .v4(_): - containerIPAddress = attachment.ipv4Address.address.description + guard let ipv4Address = attachment.ipv4Address else { + throw ContainerizationError(.invalidState, message: "cannot configure IPv4 port forwarding for container with unknown IPv4 address") + } + containerIPAddress = ipv4Address.address.description case .v6(_): guard let ipv6Address = attachment.ipv6Address else { throw ContainerizationError(.invalidState, message: "cannot configure IPv6 port forwarding for container with unknown IPv6 address") @@ -1040,7 +1048,10 @@ public actor RuntimeService { private nonisolated func getDefaultNameservers(from attachments: [Attachment]) -> [String] { for attachment in attachments { - return [attachment.ipv4Gateway.description] + guard let ipv4Gateway: IPv4Address = attachment.ipv4Gateway else { + continue + } + return [ipv4Gateway.description] } return [] } diff --git a/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift b/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift index 2e30c74a9..f81df9e2f 100644 --- a/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift +++ b/Tests/CLITests/Subcommands/Networks/TestCLINetwork.swift @@ -72,7 +72,7 @@ class TestCLINetwork: CLITest { let container = try inspectContainer(name) #expect(container.networks.count > 0) - let cidrAddress = container.networks[0].ipv4Address + let cidrAddress = try #require(container.networks[0].ipv4Address) let url = "http://\(cidrAddress.address):\(port)" var request = HTTPClientRequest(url: url) request.method = .GET @@ -244,7 +244,7 @@ class TestCLINetwork: CLITest { let container = try inspectContainer(name) #expect(container.networks.count > 0) let curlImage = "docker.io/curlimages/curl:8.6.0" - let cidrAddress = container.networks[0].ipv4Address + let cidrAddress = try #require(container.networks[0].ipv4Address) let url = "http://\(cidrAddress.address):\(port)" let (_, _, _, succeed) = try run(arguments: [ "run", diff --git a/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift b/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift index 4d7ec61bd..2b9e12f3b 100644 --- a/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift +++ b/Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift @@ -610,7 +610,7 @@ class TestCLIRunCommand3: CLITest { .map { $0.joined(separator: " ") } let inspectOutput = try inspectContainer(name) - let ip = inspectOutput.networks[0].ipv4Address.address + let ip = try #require(inspectOutput.networks[0].ipv4Address).address let expectedNameserver = IPv4Address((ip.value & Prefix(length: 24)!.prefixMask32) + 1).description let defaultDomain = try getDefaultDomain() let expectedLines: [String] = [ @@ -672,7 +672,7 @@ class TestCLIRunCommand3: CLITest { } let inspectOutput = try inspectContainer(name) - let ip = inspectOutput.networks[0].ipv4Address.address + let ip = try #require(inspectOutput.networks[0].ipv4Address).address let output = try doExec(name: name, cmd: ["cat", "/etc/hosts"]) let lines = output.split(separator: "\n") diff --git a/Tests/ContainerResourceTests/AttachmentTest.swift b/Tests/ContainerResourceTests/AttachmentTest.swift new file mode 100644 index 000000000..60b7e50ef --- /dev/null +++ b/Tests/ContainerResourceTests/AttachmentTest.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Testing + +@testable import ContainerResource + +struct AttachmentTest { + @Test func testAttachmentNilIPFields() { + let attachment = Attachment( + network: "my-net", + hostname: "host1", + ipv4Address: nil, + ipv4Gateway: nil, + ipv6Address: nil, + macAddress: nil + ) + #expect(attachment.ipv4Address == nil) + #expect(attachment.ipv4Gateway == nil) + } +}