vminitd: Proper pty setup / mount /dev/console (#248)

Author: dcantah

Sources/Integration/Suite.swift

@@ -215,6 +215,7 @@ struct IntegrationSuite: AsyncParsableCommand {
             "nested virt": testNestedVirtualizationEnabled,
             "container manager": testContainerManagerCreate,
             "container reuse": testContainerReuse,
+            "container /dev/console": testContainerDevConsole,
         ]
 
         var passed = 0

Sources/Integration/VMTests.swift

@@ -257,6 +257,54 @@ extension IntegrationSuite {
         }
     }
 
+    func testContainerDevConsole() async throws {
+        let id = "test-container-devconsole"
+
+        let bs = try await bootstrap()
+
+        let manager = try ContainerManager(vmm: bs.vmm)
+        defer {
+            try? manager.delete(id)
+        }
+
+        let buffer = BufferWriter()
+        let container = try await manager.create(
+            id,
+            image: bs.image,
+            rootfs: bs.rootfs
+        ) { config in
+            // We mount devtmpfs by default, and while this includes creating
+            // /dev/console typically that'll be pointing to /dev/hvc0 (the
+            // virtio serial console). This is just a character device, so a trivial
+            // way to check that our bind mounted console setup worked is by just
+            // parsing `mount`'s output and looking for /dev/console as it wouldn't
+            // be there normally without our dance.
+            config.process.arguments = ["mount"]
+            config.process.terminal = true
+            config.process.stdout = buffer
+        }
+
+        try await container.create()
+        try await container.start()
+
+        let status = try await container.wait()
+        try await container.stop()
+        guard status == 0 else {
+            throw IntegrationError.assert(msg: "process status \(status) != 0")
+        }
+
+        guard let str = String(data: buffer.data, encoding: .utf8) else {
+            throw IntegrationError.assert(
+                msg: "failed to convert standard output to a UTF8 string")
+        }
+
+        let devConsole = "/dev/console"
+        guard str.contains(devConsole) else {
+            throw IntegrationError.assert(
+                msg: "process should have \(devConsole) in `mount` output")
+        }
+    }
+
     private func createMountDirectory() throws -> URL {
         let dir = FileManager.default.uniqueTemporaryDirectory(create: true)
         try "hello".write(to: dir.appendingPathComponent("hi.txt"), atomically: true, encoding: .utf8)

Sources/cctl/RunCommand.swift

@@ -68,7 +68,8 @@ extension Application {
         @Option(name: .long, help: "Current working directory")
         var cwd: String = "/"
 
-        @Argument var arguments: [String] = ["/bin/sh"]
+        @Argument(parsing: .captureForPassthrough)
+        var arguments: [String] = ["/bin/sh"]
 
         func run() async throws {
             let kernel = Kernel(

vminitd/Sources/vmexec/Console.swift

@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2025 Apple Inc. and the Containerization project authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import Foundation
+import Musl
+
+class Console {
+    let master: Int32
+    let slavePath: String
+
+    init() throws {
+        let masterFD = open("/dev/ptmx", O_RDWR | O_NOCTTY | O_CLOEXEC)
+        guard masterFD != -1 else {
+            throw App.Errno(stage: "open_ptmx")
+        }
+
+        guard unlockpt(masterFD) == 0 else {
+            throw App.Errno(stage: "unlockpt")
+        }
+
+        guard let slavePath = ptsname(masterFD) else {
+            throw App.Errno(stage: "ptsname")
+        }
+
+        self.master = masterFD
+        self.slavePath = String(cString: slavePath)
+    }
+
+    func configureStdIO() throws {
+        let path = self.slavePath
+        let slaveFD = open(path, O_RDWR)
+        guard slaveFD != -1 else {
+            throw App.Errno(stage: "open_pts")
+        }
+        defer { Musl.close(slaveFD) }
+
+        for fd: Int32 in 0...2 {
+            guard dup3(slaveFD, fd, 0) != -1 else {
+                throw App.Errno(stage: "dup3")
+            }
+        }
+    }
+
+    func close() throws {
+        guard Musl.close(self.master) == 0 else {
+            throw App.Errno(stage: "close")
+        }
+    }
+}

vminitd/Sources/vmexec/ExecCommand.swift

@@ -62,37 +62,68 @@ struct ExecCommand: ParsableCommand {
         process: ContainerizationOCI.Process,
         log: Logger
     ) throws {
-        // CLOEXEC the pipe fd that signals process readiness.
-        let syncfd = FileHandle(fileDescriptor: 3)
-        if fcntl(3, F_SETFD, FD_CLOEXEC) == -1 {
-            throw App.Errno(stage: "cloexec(syncfd)")
-        }
+        let syncPipe = FileHandle(fileDescriptor: 3)
+        let ackPipe = FileHandle(fileDescriptor: 4)
 
         try Self.enterNS(path: "/proc/\(self.parentPid)/ns/cgroup", nsType: CLONE_NEWCGROUP)
         try Self.enterNS(path: "/proc/\(self.parentPid)/ns/pid", nsType: CLONE_NEWPID)
         try Self.enterNS(path: "/proc/\(self.parentPid)/ns/uts", nsType: CLONE_NEWUTS)
         try Self.enterNS(path: "/proc/\(self.parentPid)/ns/mnt", nsType: CLONE_NEWNS)
 
-        let childPipe = Pipe()
-        try childPipe.setCloexec()
         let processID = fork()
 
         guard processID != -1 else {
-            try? childPipe.fileHandleForReading.close()
-            try? childPipe.fileHandleForWriting.close()
-            try? syncfd.close()
+            try? syncPipe.close()
+            try? ackPipe.close()
 
             throw App.Errno(stage: "fork")
         }
 
         if processID == 0 {  // child
-            try childPipe.fileHandleForReading.close()
-            try syncfd.close()
+            // Wait for the grandparent to tell us that they acked our pid.
+            guard let data = try ackPipe.read(upToCount: App.ackPid.count) else {
+                throw App.Failure(message: "read ack pipe")
+            }
+            guard let pidAckStr = String(data: data, encoding: .utf8) else {
+                throw App.Failure(message: "convert ack pipe data to string")
+            }
+
+            guard pidAckStr == App.ackPid else {
+                throw App.Failure(message: "received invalid acknowledgement string: \(pidAckStr)")
+            }
 
             guard setsid() != -1 else {
                 throw App.Errno(stage: "setsid()")
             }
 
+            if process.terminal {
+                let pty = try Console()
+                try pty.configureStdIO()
+                var masterFD = pty.master
+
+                let data = Data(bytes: &masterFD, count: MemoryLayout.size(ofValue: masterFD))
+                try syncPipe.write(contentsOf: data)
+                try syncPipe.close()
+
+                // Wait for the grandparent to tell us that they acked our console.
+                guard let data = try ackPipe.read(upToCount: App.ackConsole.count) else {
+                    throw App.Failure(message: "read ack pipe")
+                }
+
+                guard let consoleAckStr = String(data: data, encoding: .utf8) else {
+                    throw App.Failure(message: "convert ack pipe data to string")
+                }
+
+                guard consoleAckStr == App.ackConsole else {
+                    throw App.Failure(message: "received invalid acknowledgement string: \(consoleAckStr)")
+                }
+
+                guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else {
+                    throw App.Errno(stage: "setctty(0)")
+                }
+                try pty.close()
+            }
+
             // Apply O_CLOEXEC to all file descriptors except stdio.
             // This ensures that all unwanted fds we may have accidentally
             // inherited are marked close-on-exec so they stay out of the
@@ -106,26 +137,13 @@ struct ExecCommand: ParsableCommand {
             // Set uid, gid, and supplementary groups
             try App.setPermissions(user: process.user)
 
-            if process.terminal {
-                guard ioctl(0, UInt(TIOCSCTTY), 0) != -1 else {
-                    throw App.Errno(stage: "setctty()")
-                }
-            }
-
             try App.exec(process: process)
         } else {  // parent process
-            try childPipe.fileHandleForWriting.close()
-
-            // wait until the pipe is closed then carry on.
-            _ = try childPipe.fileHandleForReading.readToEnd()
-            try childPipe.fileHandleForReading.close()
-
-            // send our child's pid to our parent before we exit.
+            // Send our child's pid to our parent before we exit.
             var childPid = processID
             let data = Data(bytes: &childPid, count: MemoryLayout.size(ofValue: childPid))
 
-            try syncfd.write(contentsOf: data)
-            try syncfd.close()
+            try syncPipe.write(contentsOf: data)
         }
     }
 }

vminitd/Sources/vmexec/Mount.swift

@@ -36,8 +36,7 @@ struct ContainerMount {
     }
 
     func configureConsole() throws {
-        let ptmx = self.rootfs.standardizingPath.appendingPathComponent("/dev/ptmx")
-
+        let ptmx = self.rootfs.standardizingPath.appendingPathComponent("dev/ptmx")
         guard remove(ptmx) == 0 else {
             throw App.Errno(stage: "remove(ptmx)")
         }