diff --git a/Package.swift b/Package.swift index 62cf4da6..ebf38b22 100644 --- a/Package.swift +++ b/Package.swift @@ -27,6 +27,10 @@ var defaultTraits: Set = ["SubprocessFoundation"] defaultTraits.insert("SubprocessSpan") #endif +let packageSwiftSettings: [SwiftSetting] = [ + .define("SUBPROCESS_ASYNCIO_DISPATCH", .when(platforms: [.macOS, .custom("freebsd"), .openbsd])) +] + let package = Package( name: "Subprocess", platforms: [.macOS(.v13), .iOS("99.0")], @@ -58,7 +62,7 @@ let package = Package( .enableExperimentalFeature("NonescapableTypes"), .enableExperimentalFeature("LifetimeDependence"), .enableExperimentalFeature("Span"), - ] + ] + packageSwiftSettings ), .testTarget( name: "SubprocessTests", @@ -70,7 +74,7 @@ let package = Package( ], swiftSettings: [ .enableExperimentalFeature("Span"), - ] + ] + packageSwiftSettings ), .target( diff --git a/Sources/Subprocess/AsyncBufferSequence.swift b/Sources/Subprocess/AsyncBufferSequence.swift index 3076b12d..8fe78f1f 100644 --- a/Sources/Subprocess/AsyncBufferSequence.swift +++ b/Sources/Subprocess/AsyncBufferSequence.swift @@ -23,7 +23,7 @@ public struct AsyncBufferSequence: AsyncSequence, @unchecked Sendable { public typealias Failure = any Swift.Error public typealias Element = Buffer - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH internal typealias DiskIO = DispatchIO #elseif canImport(WinSDK) internal typealias DiskIO = HANDLE @@ -55,7 +55,7 @@ public struct AsyncBufferSequence: AsyncSequence, @unchecked Sendable { ) guard let data else { // We finished reading. Close the file descriptor now - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH try _safelyClose(.dispatchIO(self.diskIO)) #elseif canImport(WinSDK) try _safelyClose(.handle(self.diskIO)) @@ -137,7 +137,7 @@ extension AsyncBufferSequence { self.eofReached = true return nil } - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH // Unfortunately here we _have to_ copy the bytes out because // DispatchIO (rightfully) reuses buffer, which means `buffer.data` // has the same address on all iterations, therefore we can't directly diff --git a/Sources/Subprocess/Buffer.swift b/Sources/Subprocess/Buffer.swift index 8152ac56..74216445 100644 --- a/Sources/Subprocess/Buffer.swift +++ b/Sources/Subprocess/Buffer.swift @@ -17,7 +17,7 @@ extension AsyncBufferSequence { /// A immutable collection of bytes public struct Buffer: Sendable { - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH // We need to keep the backingData alive while Slice is alive internal let backingData: DispatchData internal let data: DispatchData.Region @@ -45,7 +45,7 @@ extension AsyncBufferSequence { internal static func createFrom(_ data: [UInt8]) -> [Buffer] { return [.init(data: data)] } - #endif // canImport(Darwin) + #endif // SUBPROCESS_ASYNCIO_DISPATCH } } @@ -92,7 +92,7 @@ extension AsyncBufferSequence.Buffer { // MARK: - Hashable, Equatable extension AsyncBufferSequence.Buffer: Equatable, Hashable { - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH public static func == (lhs: AsyncBufferSequence.Buffer, rhs: AsyncBufferSequence.Buffer) -> Bool { return lhs.data == rhs.data } @@ -104,7 +104,7 @@ extension AsyncBufferSequence.Buffer: Equatable, Hashable { // else Compiler generated conformances } -#if canImport(Darwin) +#if SUBPROCESS_ASYNCIO_DISPATCH extension DispatchData.Region { static func == (lhs: DispatchData.Region, rhs: DispatchData.Region) -> Bool { return lhs.withUnsafeBytes { lhsBytes in @@ -120,7 +120,7 @@ extension DispatchData.Region { } } } -#if !SubprocessFoundation +#if !canImport(Darwin) || !SubprocessFoundation /// `DispatchData.Region` is defined in Foundation, but we can't depend on Foundation when the SubprocessFoundation trait is disabled. extension DispatchData { typealias Region = _ContiguousBufferView diff --git a/Sources/Subprocess/CMakeLists.txt b/Sources/Subprocess/CMakeLists.txt index ea95c737..53de37cf 100644 --- a/Sources/Subprocess/CMakeLists.txt +++ b/Sources/Subprocess/CMakeLists.txt @@ -17,7 +17,7 @@ add_library(Subprocess Result.swift IO/Output.swift IO/Input.swift - IO/AsyncIO+Darwin.swift + IO/AsyncIO+Dispatch.swift IO/AsyncIO+Linux.swift IO/AsyncIO+Windows.swift Span+Subprocess.swift @@ -36,8 +36,13 @@ elseif(LINUX OR ANDROID) Platforms/Subprocess+Unix.swift) elseif(APPLE) target_sources(Subprocess PRIVATE + Platforms/Subprocess+BSD.swift Platforms/Subprocess+Darwin.swift Platforms/Subprocess+Unix.swift) +elseif(FREEBSD OR OPENBSD) + target_sources(Subprocess PRIVATE + Platforms/Subprocess+BSD.swift + Platforms/Subprocess+Unix.swift) endif() target_compile_options(Subprocess PRIVATE diff --git a/Sources/Subprocess/Configuration.swift b/Sources/Subprocess/Configuration.swift index 4953f48b..ed506e69 100644 --- a/Sources/Subprocess/Configuration.swift +++ b/Sources/Subprocess/Configuration.swift @@ -657,7 +657,7 @@ internal struct IODescriptor: ~Copyable { consuming func createIOChannel() -> IOChannel { let shouldClose = self.closeWhenDone self.closeWhenDone = false - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH // Transferring out the ownership of fileDescriptor means we don't have go close here let closeFd = self.descriptor let dispatchIO: DispatchIO = DispatchIO( @@ -708,10 +708,10 @@ internal struct IODescriptor: ~Copyable { } internal struct IOChannel: ~Copyable, @unchecked Sendable { - #if canImport(WinSDK) - typealias Channel = HANDLE - #elseif canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH typealias Channel = DispatchIO + #elseif canImport(WinSDK) + typealias Channel = HANDLE #else typealias Channel = FileDescriptor #endif @@ -733,10 +733,10 @@ internal struct IOChannel: ~Copyable, @unchecked Sendable { } closeWhenDone = false - #if canImport(WinSDK) - try _safelyClose(.handle(self.channel)) - #elseif canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH try _safelyClose(.dispatchIO(self.channel)) + #elseif canImport(WinSDK) + try _safelyClose(.handle(self.channel)) #else try _safelyClose(.fileDescriptor(self.channel)) #endif diff --git a/Sources/Subprocess/IO/AsyncIO+Darwin.swift b/Sources/Subprocess/IO/AsyncIO+Dispatch.swift similarity index 97% rename from Sources/Subprocess/IO/AsyncIO+Darwin.swift rename to Sources/Subprocess/IO/AsyncIO+Dispatch.swift index 11cf94a6..280fee91 100644 --- a/Sources/Subprocess/IO/AsyncIO+Darwin.swift +++ b/Sources/Subprocess/IO/AsyncIO+Dispatch.swift @@ -12,7 +12,7 @@ /// Darwin AsyncIO implementation based on DispatchIO // MARK: - macOS (DispatchIO) -#if canImport(Darwin) +#if SUBPROCESS_ASYNCIO_DISPATCH #if canImport(System) @preconcurrency import System @@ -166,4 +166,8 @@ final class AsyncIO: Sendable { } } +#if !canImport(Darwin) +extension DispatchData: @retroactive @unchecked Sendable { } +#endif + #endif diff --git a/Sources/Subprocess/IO/AsyncIO+Linux.swift b/Sources/Subprocess/IO/AsyncIO+Linux.swift index a667aa2c..b04b747e 100644 --- a/Sources/Subprocess/IO/AsyncIO+Linux.swift +++ b/Sources/Subprocess/IO/AsyncIO+Linux.swift @@ -11,7 +11,7 @@ /// Linux AsyncIO implementation based on epoll -#if canImport(Glibc) || canImport(Android) || canImport(Musl) +#if os(Linux) || os(Android) #if canImport(System) @preconcurrency import System @@ -266,6 +266,11 @@ final class AsyncIO: Sendable { targetEvent = EPOLL_EVENTS(EPOLLOUT) } + // Save the continuation (before calling epoll_ctl, so we don't miss any data) + _registration.withLock { storage in + storage[fileDescriptor.rawValue] = continuation + } + var event = epoll_event( events: targetEvent.rawValue, data: epoll_data(fd: fileDescriptor.rawValue) @@ -277,6 +282,10 @@ final class AsyncIO: Sendable { &event ) if rc != 0 { + _registration.withLock { storage in + storage.removeValue(forKey: fileDescriptor.rawValue) + } + let capturedError = errno let error = SubprocessError( code: .init(.asyncIOFailed( @@ -287,10 +296,6 @@ final class AsyncIO: Sendable { continuation.finish(throwing: error) return } - // Now save the continuation - _registration.withLock { storage in - storage[fileDescriptor.rawValue] = continuation - } case .failure(let setupError): continuation.finish(throwing: setupError) return diff --git a/Sources/Subprocess/IO/Output.swift b/Sources/Subprocess/IO/Output.swift index 1096a307..131d2840 100644 --- a/Sources/Subprocess/IO/Output.swift +++ b/Sources/Subprocess/IO/Output.swift @@ -148,7 +148,7 @@ public struct BytesOutput: OutputProtocol { internal func captureOutput( from diskIO: consuming IOChannel ) async throws -> [UInt8] { - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH var result: DispatchData? = nil #else var result: [UInt8]? = nil @@ -173,7 +173,7 @@ public struct BytesOutput: OutputProtocol { underlyingError: nil ) } - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH return result?.array() ?? [] #else return result ?? [] @@ -302,7 +302,7 @@ extension OutputProtocol { return try await bytesOutput.captureOutput(from: diskIO) as! Self.OutputType } - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH var result: DispatchData? = nil #else var result: [UInt8]? = nil @@ -328,7 +328,7 @@ extension OutputProtocol { ) } - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH return try self.output(from: result ?? .empty) #else return try self.output(from: result ?? []) @@ -353,7 +353,7 @@ extension OutputProtocol where OutputType == Void { #if SubprocessSpan extension OutputProtocol { - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH internal func output(from data: DispatchData) throws -> OutputType { guard !data.isEmpty else { let empty = UnsafeRawBufferPointer(start: nil, count: 0) @@ -380,7 +380,7 @@ extension OutputProtocol { return try self.output(from: span) } } - #endif // canImport(Darwin) + #endif // SUBPROCESS_ASYNCIO_DISPATCH } #endif diff --git a/Sources/Subprocess/Platforms/Subprocess+BSD.swift b/Sources/Subprocess/Platforms/Subprocess+BSD.swift new file mode 100644 index 00000000..c17dab74 --- /dev/null +++ b/Sources/Subprocess/Platforms/Subprocess+BSD.swift @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if os(macOS) || os(FreeBSD) || os(OpenBSD) + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +internal import Dispatch + +// MARK: - Process Monitoring +@Sendable +internal func monitorProcessTermination( + for processIdentifier: ProcessIdentifier +) async throws -> TerminationStatus { + switch Result(catching: { () throws(SubprocessError.UnderlyingError) -> TerminationStatus? in try processIdentifier.reap() }) { + case let .success(status?): + return status + case .success(nil): + break + case let .failure(error): + throw SubprocessError( + code: .init(.failedToMonitorProcess), + underlyingError: error + ) + } + return try await withCheckedThrowingContinuation { continuation in + let source = DispatchSource.makeProcessSource( + identifier: processIdentifier.value, + eventMask: [.exit], + queue: .global() + ) + source.setEventHandler { + source.cancel() + continuation.resume(with: Result(catching: { () throws(SubprocessError.UnderlyingError) -> TerminationStatus in + // NOTE_EXIT may be delivered slightly before the process becomes reapable, + // so we must call waitid without WNOHANG to avoid a narrow possibility of a race condition. + // If waitid does block, it won't do so for very long at all. + try processIdentifier.blockingReap() + }).mapError { underlyingError in + SubprocessError( + code: .init(.failedToMonitorProcess), + underlyingError: underlyingError + ) + }) + } + source.resume() + } +} + +#endif diff --git a/Sources/Subprocess/Platforms/Subprocess+Darwin.swift b/Sources/Subprocess/Platforms/Subprocess+Darwin.swift index 99ff9d65..cfaa6149 100644 --- a/Sources/Subprocess/Platforms/Subprocess+Darwin.swift +++ b/Sources/Subprocess/Platforms/Subprocess+Darwin.swift @@ -497,46 +497,4 @@ extension ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertib public var debugDescription: String { "\(self.value)" } } -// MARK: - Process Monitoring -@Sendable -internal func monitorProcessTermination( - for processIdentifier: ProcessIdentifier -) async throws -> TerminationStatus { - return try await withCheckedThrowingContinuation { continuation in - let source = DispatchSource.makeProcessSource( - identifier: processIdentifier.value, - eventMask: [.exit], - queue: .global() - ) - source.setEventHandler { - source.cancel() - var siginfo = siginfo_t() - let rc = waitid(P_PID, id_t(processIdentifier.value), &siginfo, WEXITED) - guard rc == 0 else { - continuation.resume( - throwing: SubprocessError( - code: .init(.failedToMonitorProcess), - underlyingError: .init(rawValue: errno) - ) - ) - return - } - switch siginfo.si_code { - case .init(CLD_EXITED): - continuation.resume(returning: .exited(siginfo.si_status)) - return - case .init(CLD_KILLED), .init(CLD_DUMPED): - continuation.resume(returning: .unhandledException(siginfo.si_status)) - case .init(CLD_TRAPPED), .init(CLD_STOPPED), .init(CLD_CONTINUED), .init(CLD_NOOP): - // Ignore these signals because they are not related to - // process exiting - break - default: - fatalError("Unexpected exit status: \(siginfo.si_code)") - } - } - source.resume() - } -} - #endif // canImport(Darwin) diff --git a/Sources/Subprocess/Platforms/Subprocess+Linux.swift b/Sources/Subprocess/Platforms/Subprocess+Linux.swift index 512eeed2..444a4c01 100644 --- a/Sources/Subprocess/Platforms/Subprocess+Linux.swift +++ b/Sources/Subprocess/Platforms/Subprocess+Linux.swift @@ -9,14 +9,6 @@ // //===----------------------------------------------------------------------===// -#if canImport(Glibc) || canImport(Android) || canImport(Musl) - -#if canImport(System) -@preconcurrency import System -#else -@preconcurrency import SystemPackage -#endif - #if canImport(Glibc) import Glibc let _subprocess_read = Glibc.read @@ -34,12 +26,19 @@ let _subprocess_write = Musl.write let _subprocess_close = Musl.close #endif +#if os(Linux) || os(Android) + +#if canImport(System) +@preconcurrency import System +#else +@preconcurrency import SystemPackage +#endif + internal import Dispatch import Synchronization import _SubprocessCShims -// Linux specific implementations #if canImport(Glibc) extension EPOLL_EVENTS { init(_ other: EPOLL_EVENTS) { @@ -76,245 +75,6 @@ extension Int32 { } #endif -extension Configuration { - internal func spawn( - withInput inputPipe: consuming CreatedPipe, - outputPipe: consuming CreatedPipe, - errorPipe: consuming CreatedPipe - ) throws -> SpawnResult { - // Ensure the waiter thread is running. - _setupMonitorSignalHandler() - - // Instead of checking if every possible executable path - // is valid, spawn each directly and catch ENOENT - let possiblePaths = self.executable.possibleExecutablePaths( - withPathValue: self.environment.pathValue() - ) - var inputPipeBox: CreatedPipe? = consume inputPipe - var outputPipeBox: CreatedPipe? = consume outputPipe - var errorPipeBox: CreatedPipe? = consume errorPipe - - return try self.preSpawn { args throws -> SpawnResult in - let (env, uidPtr, gidPtr, supplementaryGroups) = args - - var _inputPipe = inputPipeBox.take()! - var _outputPipe = outputPipeBox.take()! - var _errorPipe = errorPipeBox.take()! - - let inputReadFileDescriptor: IODescriptor? = _inputPipe.readFileDescriptor() - let inputWriteFileDescriptor: IODescriptor? = _inputPipe.writeFileDescriptor() - let outputReadFileDescriptor: IODescriptor? = _outputPipe.readFileDescriptor() - let outputWriteFileDescriptor: IODescriptor? = _outputPipe.writeFileDescriptor() - let errorReadFileDescriptor: IODescriptor? = _errorPipe.readFileDescriptor() - let errorWriteFileDescriptor: IODescriptor? = _errorPipe.writeFileDescriptor() - - for possibleExecutablePath in possiblePaths { - var processGroupIDPtr: UnsafeMutablePointer? = nil - if let processGroupID = self.platformOptions.processGroupID { - processGroupIDPtr = .allocate(capacity: 1) - processGroupIDPtr?.pointee = gid_t(processGroupID) - } - // Setup Arguments - let argv: [UnsafeMutablePointer?] = self.arguments.createArgs( - withExecutablePath: possibleExecutablePath - ) - defer { - for ptr in argv { ptr?.deallocate() } - } - // Setup input - let fileDescriptors: [CInt] = [ - inputReadFileDescriptor?.platformDescriptor() ?? -1, - inputWriteFileDescriptor?.platformDescriptor() ?? -1, - outputWriteFileDescriptor?.platformDescriptor() ?? -1, - outputReadFileDescriptor?.platformDescriptor() ?? -1, - errorWriteFileDescriptor?.platformDescriptor() ?? -1, - errorReadFileDescriptor?.platformDescriptor() ?? -1, - ] - - // Spawn - var pid: pid_t = 0 - var processDescriptor: PlatformFileDescriptor = -1 - let spawnError: CInt = possibleExecutablePath.withCString { exePath in - return (self.workingDirectory?.string).withOptionalCString { workingDir in - return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in - return fileDescriptors.withUnsafeBufferPointer { fds in - return _subprocess_fork_exec( - &pid, - &processDescriptor, - exePath, - workingDir, - fds.baseAddress!, - argv, - env, - uidPtr, - gidPtr, - processGroupIDPtr, - CInt(supplementaryGroups?.count ?? 0), - sgroups?.baseAddress, - self.platformOptions.createSession ? 1 : 0 - ) - } - } - } - } - // Spawn error - if spawnError != 0 { - if spawnError == ENOENT || spawnError == EACCES { - // Move on to another possible path - continue - } - // Throw all other errors - try self.safelyCloseMultiple( - inputRead: inputReadFileDescriptor, - inputWrite: inputWriteFileDescriptor, - outputRead: outputReadFileDescriptor, - outputWrite: outputWriteFileDescriptor, - errorRead: errorReadFileDescriptor, - errorWrite: errorWriteFileDescriptor - ) - throw SubprocessError( - code: .init(.spawnFailed), - underlyingError: .init(rawValue: spawnError) - ) - } - // After spawn finishes, close all child side fds - try self.safelyCloseMultiple( - inputRead: inputReadFileDescriptor, - inputWrite: nil, - outputRead: nil, - outputWrite: outputWriteFileDescriptor, - errorRead: nil, - errorWrite: errorWriteFileDescriptor - ) - let execution = Execution( - processIdentifier: .init( - value: pid, - processDescriptor: processDescriptor - ) - ) - return SpawnResult( - execution: execution, - inputWriteEnd: inputWriteFileDescriptor?.createIOChannel(), - outputReadEnd: outputReadFileDescriptor?.createIOChannel(), - errorReadEnd: errorReadFileDescriptor?.createIOChannel() - ) - } - - // If we reach this point, it means either the executable path - // or working directory is not valid. Since posix_spawn does not - // provide which one is not valid, here we make a best effort guess - // by checking whether the working directory is valid. This technically - // still causes TOUTOC issue, but it's the best we can do for error recovery. - try self.safelyCloseMultiple( - inputRead: inputReadFileDescriptor, - inputWrite: inputWriteFileDescriptor, - outputRead: outputReadFileDescriptor, - outputWrite: outputWriteFileDescriptor, - errorRead: errorReadFileDescriptor, - errorWrite: errorWriteFileDescriptor - ) - if let workingDirectory = self.workingDirectory?.string { - guard Configuration.pathAccessible(workingDirectory, mode: F_OK) else { - throw SubprocessError( - code: .init(.failedToChangeWorkingDirectory(workingDirectory)), - underlyingError: .init(rawValue: ENOENT) - ) - } - } - throw SubprocessError( - code: .init(.executableNotFound(self.executable.description)), - underlyingError: .init(rawValue: ENOENT) - ) - } - } -} - -// MARK: - ProcessIdentifier - -/// A platform independent identifier for a Subprocess. -public struct ProcessIdentifier: Sendable, Hashable { - /// The platform specific process identifier value - public let value: pid_t - public let processDescriptor: CInt - - internal init(value: pid_t, processDescriptor: PlatformFileDescriptor) { - self.value = value - self.processDescriptor = processDescriptor - } - - internal func close() { - if self.processDescriptor > 0 { - _ = _subprocess_close(self.processDescriptor) - } - } -} - -extension ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertible { - public var description: String { "\(self.value)" } - - public var debugDescription: String { "\(self.value)" } -} - -// MARK: - Platform Specific Options - -/// The collection of platform-specific settings -/// to configure the subprocess when running -public struct PlatformOptions: Sendable { - /// Set user ID for the subprocess - public var userID: uid_t? = nil - /// Set the real and effective group ID and the saved - /// set-group-ID of the subprocess, equivalent to calling - /// `setgid()` on the child process. - /// Group ID is used to control permissions, particularly - /// for file access. - public var groupID: gid_t? = nil - /// Set list of supplementary group IDs for the subprocess - public var supplementaryGroups: [gid_t]? = nil - /// Set the process group for the subprocess, equivalent to - /// calling `setpgid()` on the child process. - /// Process group ID is used to group related processes for - /// controlling signals. - public var processGroupID: pid_t? = nil - /// Creates a session and sets the process group ID - /// i.e. Detach from the terminal. - public var createSession: Bool = false - /// An ordered list of steps in order to tear down the child - /// process in case the parent task is cancelled before - /// the child process terminates. - /// Always ends in sending a `.kill` signal at the end. - public var teardownSequence: [TeardownStep] = [] - - public init() {} -} - -extension PlatformOptions: CustomStringConvertible, CustomDebugStringConvertible { - internal func description(withIndent indent: Int) -> String { - let indent = String(repeating: " ", count: indent * 4) - return """ - PlatformOptions( - \(indent) userID: \(String(describing: userID)), - \(indent) groupID: \(String(describing: groupID)), - \(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), - \(indent) processGroupID: \(String(describing: processGroupID)), - \(indent) createSession: \(createSession) - \(indent)) - """ - } - - public var description: String { - return self.description(withIndent: 0) - } - - public var debugDescription: String { - return self.description(withIndent: 0) - } -} - -// Special keys used in Error's user dictionary -extension String { - static let debugDescriptionErrorKey = "DebugDescription" -} - // MARK: - Process Monitoring @Sendable internal func monitorProcessTermination( @@ -366,31 +126,19 @@ internal func monitorProcessTermination( // Since Linux coalesce signals, it's possible by the time we request // monitoring the process has already exited. Check to make sure that // is not the case and only save continuation then. - var siginfo = siginfo_t() - // Use NOHANG here because the child process might still be running - if 0 == waitid(P_PID, id_t(processIdentifier.value), &siginfo, WEXITED | WNOHANG) { - // If si_pid and si_signo are both 0, the child is still running since we used WNOHANG - if siginfo.si_pid == 0 && siginfo.si_signo == 0 { - // Save this continuation to be called by signal hander - var newState = storage - newState.continuations[processIdentifier.value] = continuation - state = .started(newState) - return nil - } - - switch siginfo.si_code { - case .init(CLD_EXITED): - return .success(.exited(siginfo.si_status)) - case .init(CLD_KILLED), .init(CLD_DUMPED): - return .success(.unhandledException(siginfo.si_status)) - default: - fatalError("Unexpected exit status: \(siginfo.si_code)") - } - } else { - let waitidError = errno + switch Result(catching: { () throws(SubprocessError.UnderlyingError) -> TerminationStatus? in try processIdentifier.reap() }) { + case let .success(status?): + return .success(status) + case .success(nil): + // Save this continuation to be called by signal hander + var newState = storage + newState.continuations[processIdentifier.value] = continuation + state = .started(newState) + return nil + case let .failure(underlyingError): let error = SubprocessError( code: .init(.failedToMonitorProcess), - underlyingError: .init(rawValue: waitidError) + underlyingError: underlyingError ) return .failure(error) } @@ -431,28 +179,6 @@ private struct MonitorThreadContext: Sendable { } } -internal extension siginfo_t { - var si_status: Int32 { - #if canImport(Glibc) - return _sifields._sigchld.si_status - #elseif canImport(Musl) - return __si_fields.__si_common.__second.__sigchld.si_status - #elseif canImport(Bionic) - return _sifields._sigchld._status - #endif - } - - var si_pid: pid_t { - #if canImport(Glibc) - return _sifields._sigchld.si_pid - #elseif canImport(Musl) - return __si_fields.__si_common.__first.__piduid.si_pid - #elseif canImport(Bionic) - return _sifields._kill._pid - #endif - } -} - // Okay to be unlocked global mutable because this value is only set once like dispatch_once private nonisolated(unsafe) var _signalPipe: (readEnd: CInt, writeEnd: CInt) = (readEnd: -1, writeEnd: -1) // Okay to be unlocked global mutable because this value is only set once like dispatch_once @@ -690,23 +416,12 @@ internal func _setupMonitorSignalHandler() { } private func _blockAndWaitForProcessDescriptor(_ pidfd: CInt, context: MonitorThreadContext) { - var terminationStatus: Result - - var siginfo = siginfo_t() - if 0 == waitid(idtype_t(UInt32(P_PIDFD)), id_t(pidfd), &siginfo, WEXITED) { - switch siginfo.si_code { - case .init(CLD_EXITED): - terminationStatus = .success(.exited(siginfo.si_status)) - case .init(CLD_KILLED), .init(CLD_DUMPED): - terminationStatus = .success(.unhandledException(siginfo.si_status)) - default: - fatalError("Unexpected exit status: \(siginfo.si_code)") - } - } else { - let waitidErrno = errno - terminationStatus = .failure(SubprocessError( + var terminationStatus = Result(catching: { () throws(SubprocessError.UnderlyingError) in + try TerminationStatus(_waitid(idtype: idtype_t(UInt32(P_PIDFD)), id: id_t(pidfd), flags: WEXITED)) + }).mapError { underlyingError in + SubprocessError( code: .init(.failedToMonitorProcess), - underlyingError: .init(rawValue: waitidErrno)) + underlyingError: underlyingError ) } @@ -761,34 +476,22 @@ private func _reapAllKnownChildProcesses(_ signalFd: CInt, context: MonitorThrea var results: [ResultContinuation] = [] // Since Linux coalesce signals, we need to loop through all known child process // to check if they exited. - for knownChildPID in storage.continuations.keys { + loop: for (knownChildPID, continuation) in storage.continuations { let terminationStatus: Result - var siginfo = siginfo_t() - // Use `WNOHANG` here so waitid isn't blocking because we expect some - // child processes might be still running - if 0 == waitid(P_PID, id_t(knownChildPID), &siginfo, WEXITED | WNOHANG) { - // If si_pid and si_signo, the child is still running since we used WNOHANG - if siginfo.si_pid == 0 && siginfo.si_signo == 0 { - // Move on to the next child - continue - } - - switch siginfo.si_code { - case .init(CLD_EXITED): - terminationStatus = .success(.exited(siginfo.si_status)) - case .init(CLD_KILLED), .init(CLD_DUMPED): - terminationStatus = .success(.unhandledException(siginfo.si_status)) - default: - fatalError("Unexpected exit status: \(siginfo.si_code)") - } - } else { - let waitidErrno = errno - terminationStatus = .failure(SubprocessError( - code: .init(.failedToMonitorProcess), - underlyingError: .init(rawValue: waitidErrno)) - ) + switch Result(catching: { () throws(SubprocessError.UnderlyingError) -> TerminationStatus? in try _reap(pid: knownChildPID) }) { + case let .success(status?): + terminationStatus = .success(status) + case .success(nil): + // Move on to the next child + continue loop + case let .failure(error): + terminationStatus = .failure( + SubprocessError( + code: .init(.failedToMonitorProcess), + underlyingError: error + )) } - results.append((result: terminationStatus, continuation: storage.continuations[knownChildPID]!)) + results.append((result: terminationStatus, continuation: continuation)) // Now we have the exit code, remove saved continuations updatedContinuations.removeValue(forKey: knownChildPID) } @@ -820,47 +523,9 @@ internal func _isWaitprocessDescriptorSupported() -> Bool { /// reported that we don't have a child with the same selfPidfd; /// - EINVAL: in this case we know P_PIDFD is not supported because it does not /// recognize the `P_PIDFD` type + errno = 0 waitid(idtype_t(UInt32(P_PIDFD)), id_t(selfPidfd), &siginfo, WEXITED | WNOWAIT) return errno == ECHILD } -internal func pthread_create(_ body: @Sendable @escaping () -> ()) throws(SubprocessError.UnderlyingError) -> pthread_t { - final class Context { - let body: @Sendable () -> () - init(body: @Sendable @escaping () -> Void) { - self.body = body - } - } - #if canImport(Glibc) || canImport(Musl) - func proc(_ context: UnsafeMutableRawPointer?) -> UnsafeMutableRawPointer? { - (Unmanaged.fromOpaque(context!).takeRetainedValue() as! Context).body() - return nil - } - #elseif canImport(Bionic) - func proc(_ context: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { - (Unmanaged.fromOpaque(context).takeRetainedValue() as! Context).body() - return context - } - #endif - #if canImport(Glibc) || canImport(Bionic) - var thread = pthread_t() - #else - var thread: pthread_t? - #endif - let rc = pthread_create( - &thread, - nil, - proc, - Unmanaged.passRetained(Context(body: body)).toOpaque() - ) - if rc != 0 { - throw SubprocessError.UnderlyingError(rawValue: rc) - } - #if canImport(Glibc) || canImport(Bionic) - return thread - #else - return thread! - #endif -} - #endif // canImport(Glibc) || canImport(Android) || canImport(Musl) diff --git a/Sources/Subprocess/Platforms/Subprocess+Unix.swift b/Sources/Subprocess/Platforms/Subprocess+Unix.swift index e55ff510..b513dbc3 100644 --- a/Sources/Subprocess/Platforms/Subprocess+Unix.swift +++ b/Sources/Subprocess/Platforms/Subprocess+Unix.swift @@ -110,13 +110,13 @@ extension Execution { } let pid = shouldSendToProcessGroup ? -(processIdentifier.value) : processIdentifier.value - #if os(Linux) || os(Android) - // On linux, use pidfd_send_signal if possible + #if os(Linux) || os(Android) || os(FreeBSD) + // On platforms with process descriptors, use _subprocess_pdkill if possible if shouldSendToProcessGroup || self.processIdentifier.processDescriptor < 0 { - // pidfd_send_signal does not support sending signal to process group + // _subprocess_pdkill does not support sending signal to process group try _kill(pid, signal: signal) } else { - let rc = _pidfd_send_signal( + let rc = _subprocess_pdkill( processIdentifier.processDescriptor, signal.rawValue ) @@ -191,7 +191,7 @@ extension Environment { /// length = `key` + `=` + `value` + `\null` let totalLength = keyContainer.count + 1 + valueContainer.count + 1 let fullString: UnsafeMutablePointer = .allocate(capacity: totalLength) - #if os(Linux) || os(Android) + #if os(OpenBSD) || os(Linux) || os(Android) _ = _shims_snprintf(fullString, CInt(totalLength), "%s=%s", rawByteKey, rawByteValue) #else _ = snprintf(ptr: fullString, totalLength, "%s=%s", rawByteKey, rawByteValue) @@ -411,4 +411,373 @@ extension FileDescriptor { internal typealias PlatformFileDescriptor = CInt +// MARK: - Spawning + +#if !canImport(Darwin) +extension Configuration { + internal func spawn( + withInput inputPipe: consuming CreatedPipe, + outputPipe: consuming CreatedPipe, + errorPipe: consuming CreatedPipe + ) throws -> SpawnResult { + // Ensure the waiter thread is running. + #if os(Linux) || os(Android) + _setupMonitorSignalHandler() + #endif + + // Instead of checking if every possible executable path + // is valid, spawn each directly and catch ENOENT + let possiblePaths = self.executable.possibleExecutablePaths( + withPathValue: self.environment.pathValue() + ) + var inputPipeBox: CreatedPipe? = consume inputPipe + var outputPipeBox: CreatedPipe? = consume outputPipe + var errorPipeBox: CreatedPipe? = consume errorPipe + + return try self.preSpawn { args throws -> SpawnResult in + let (env, uidPtr, gidPtr, supplementaryGroups) = args + + var _inputPipe = inputPipeBox.take()! + var _outputPipe = outputPipeBox.take()! + var _errorPipe = errorPipeBox.take()! + + let inputReadFileDescriptor: IODescriptor? = _inputPipe.readFileDescriptor() + let inputWriteFileDescriptor: IODescriptor? = _inputPipe.writeFileDescriptor() + let outputReadFileDescriptor: IODescriptor? = _outputPipe.readFileDescriptor() + let outputWriteFileDescriptor: IODescriptor? = _outputPipe.writeFileDescriptor() + let errorReadFileDescriptor: IODescriptor? = _errorPipe.readFileDescriptor() + let errorWriteFileDescriptor: IODescriptor? = _errorPipe.writeFileDescriptor() + + for possibleExecutablePath in possiblePaths { + var processGroupIDPtr: UnsafeMutablePointer? = nil + if let processGroupID = self.platformOptions.processGroupID { + processGroupIDPtr = .allocate(capacity: 1) + processGroupIDPtr?.pointee = gid_t(processGroupID) + } + // Setup Arguments + let argv: [UnsafeMutablePointer?] = self.arguments.createArgs( + withExecutablePath: possibleExecutablePath + ) + defer { + for ptr in argv { ptr?.deallocate() } + } + // Setup input + let fileDescriptors: [CInt] = [ + inputReadFileDescriptor?.platformDescriptor() ?? -1, + inputWriteFileDescriptor?.platformDescriptor() ?? -1, + outputWriteFileDescriptor?.platformDescriptor() ?? -1, + outputReadFileDescriptor?.platformDescriptor() ?? -1, + errorWriteFileDescriptor?.platformDescriptor() ?? -1, + errorReadFileDescriptor?.platformDescriptor() ?? -1, + ] + + // Spawn + var pid: pid_t = 0 + var processDescriptor: PlatformFileDescriptor = -1 + let spawnError: CInt = possibleExecutablePath.withCString { exePath in + return (self.workingDirectory?.string).withOptionalCString { workingDir in + return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in + return fileDescriptors.withUnsafeBufferPointer { fds in + return _subprocess_fork_exec( + &pid, + &processDescriptor, + exePath, + workingDir, + fds.baseAddress!, + argv, + env, + uidPtr, + gidPtr, + processGroupIDPtr, + CInt(supplementaryGroups?.count ?? 0), + sgroups?.baseAddress, + self.platformOptions.createSession ? 1 : 0 + ) + } + } + } + } + // Spawn error + if spawnError != 0 { + if spawnError == ENOENT || spawnError == EACCES { + // Move on to another possible path + continue + } + // Throw all other errors + try self.safelyCloseMultiple( + inputRead: inputReadFileDescriptor, + inputWrite: inputWriteFileDescriptor, + outputRead: outputReadFileDescriptor, + outputWrite: outputWriteFileDescriptor, + errorRead: errorReadFileDescriptor, + errorWrite: errorWriteFileDescriptor + ) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: spawnError) + ) + } + // After spawn finishes, close all child side fds + try self.safelyCloseMultiple( + inputRead: inputReadFileDescriptor, + inputWrite: nil, + outputRead: nil, + outputWrite: outputWriteFileDescriptor, + errorRead: nil, + errorWrite: errorWriteFileDescriptor + ) + let execution = Execution( + processIdentifier: .init( + value: pid, + processDescriptor: processDescriptor + ) + ) + return SpawnResult( + execution: execution, + inputWriteEnd: inputWriteFileDescriptor?.createIOChannel(), + outputReadEnd: outputReadFileDescriptor?.createIOChannel(), + errorReadEnd: errorReadFileDescriptor?.createIOChannel() + ) + } + + // If we reach this point, it means either the executable path + // or working directory is not valid. Since posix_spawn does not + // provide which one is not valid, here we make a best effort guess + // by checking whether the working directory is valid. This technically + // still causes TOUTOC issue, but it's the best we can do for error recovery. + try self.safelyCloseMultiple( + inputRead: inputReadFileDescriptor, + inputWrite: inputWriteFileDescriptor, + outputRead: outputReadFileDescriptor, + outputWrite: outputWriteFileDescriptor, + errorRead: errorReadFileDescriptor, + errorWrite: errorWriteFileDescriptor + ) + if let workingDirectory = self.workingDirectory?.string { + guard Configuration.pathAccessible(workingDirectory, mode: F_OK) else { + throw SubprocessError( + code: .init(.failedToChangeWorkingDirectory(workingDirectory)), + underlyingError: .init(rawValue: ENOENT) + ) + } + } + throw SubprocessError( + code: .init(.executableNotFound(self.executable.description)), + underlyingError: .init(rawValue: ENOENT) + ) + } + } +} + +// MARK: - ProcessIdentifier + +/// A platform independent identifier for a Subprocess. +public struct ProcessIdentifier: Sendable, Hashable { + /// The platform specific process identifier value + public let value: pid_t + + #if os(Linux) || os(Android) || os(FreeBSD) + public let processDescriptor: CInt + #else + internal let processDescriptor: CInt // not used on other platforms + #endif + + internal init(value: pid_t, processDescriptor: PlatformFileDescriptor) { + self.value = value + self.processDescriptor = processDescriptor + } + + internal func close() { + if self.processDescriptor > 0 { + _ = _subprocess_close(self.processDescriptor) + } + } +} + +extension ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { "\(self.value)" } + + public var debugDescription: String { "\(self.value)" } +} + +// MARK: - Platform Specific Options + +/// The collection of platform-specific settings +/// to configure the subprocess when running +public struct PlatformOptions: Sendable { + /// Set user ID for the subprocess + public var userID: uid_t? = nil + /// Set the real and effective group ID and the saved + /// set-group-ID of the subprocess, equivalent to calling + /// `setgid()` on the child process. + /// Group ID is used to control permissions, particularly + /// for file access. + public var groupID: gid_t? = nil + /// Set list of supplementary group IDs for the subprocess + public var supplementaryGroups: [gid_t]? = nil + /// Set the process group for the subprocess, equivalent to + /// calling `setpgid()` on the child process. + /// Process group ID is used to group related processes for + /// controlling signals. + public var processGroupID: pid_t? = nil + /// Creates a session and sets the process group ID + /// i.e. Detach from the terminal. + public var createSession: Bool = false + /// An ordered list of steps in order to tear down the child + /// process in case the parent task is cancelled before + /// the child process terminates. + /// Always ends in sending a `.kill` signal at the end. + public var teardownSequence: [TeardownStep] = [] + + public init() {} +} + +extension PlatformOptions: CustomStringConvertible, CustomDebugStringConvertible { + internal func description(withIndent indent: Int) -> String { + let indent = String(repeating: " ", count: indent * 4) + return """ + PlatformOptions( + \(indent) userID: \(String(describing: userID)), + \(indent) groupID: \(String(describing: groupID)), + \(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), + \(indent) processGroupID: \(String(describing: processGroupID)), + \(indent) createSession: \(createSession) + \(indent)) + """ + } + + public var description: String { + return self.description(withIndent: 0) + } + + public var debugDescription: String { + return self.description(withIndent: 0) + } +} + +// Special keys used in Error's user dictionary +extension String { + static let debugDescriptionErrorKey = "DebugDescription" +} + +internal func pthread_create(_ body: @Sendable @escaping () -> ()) throws(SubprocessError.UnderlyingError) -> pthread_t { + final class Context { + let body: @Sendable () -> () + init(body: @Sendable @escaping () -> Void) { + self.body = body + } + } + #if canImport(Glibc) || canImport(Musl) + func proc(_ context: UnsafeMutableRawPointer?) -> UnsafeMutableRawPointer? { + (Unmanaged.fromOpaque(context!).takeRetainedValue() as! Context).body() + return nil + } + #elseif canImport(Bionic) + func proc(_ context: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { + (Unmanaged.fromOpaque(context).takeRetainedValue() as! Context).body() + return context + } + #endif + #if (os(Linux) && canImport(Glibc)) || canImport(Bionic) + var thread = pthread_t() + #else + var thread: pthread_t? + #endif + let rc = pthread_create( + &thread, + nil, + proc, + Unmanaged.passRetained(Context(body: body)).toOpaque() + ) + if rc != 0 { + throw SubprocessError.UnderlyingError(rawValue: rc) + } + #if (os(Linux) && canImport(Glibc)) || canImport(Bionic) + return thread + #else + return thread! + #endif +} + +#endif // !canImport(Darwin) + +extension ProcessIdentifier { + /// Reaps the zombie for the exited process. This function may block. + @available(*, noasync) + internal func blockingReap() throws(SubprocessError.UnderlyingError) -> TerminationStatus { + try _blockingReap(pid: value) + } + + /// Reaps the zombie for the exited process, or returns `nil` if the process is still running. This function will not block. + internal func reap() throws(SubprocessError.UnderlyingError) -> TerminationStatus? { + try _reap(pid: value) + } +} + +@available(*, noasync) +internal func _blockingReap(pid: pid_t) throws(SubprocessError.UnderlyingError) -> TerminationStatus { + return try TerminationStatus(_waitid(idtype: P_PID, id: id_t(pid), flags: WEXITED)) +} + +internal func _reap(pid: pid_t) throws(SubprocessError.UnderlyingError) -> TerminationStatus? { + let siginfo = try _waitid(idtype: P_PID, id: id_t(pid), flags: WEXITED | WNOHANG) + // If si_pid and si_signo are both 0, the child is still running since we used WNOHANG + if siginfo.si_pid == 0 && siginfo.si_signo == 0 { + return nil + } + return TerminationStatus(siginfo) +} + +internal func _waitid(idtype: idtype_t, id: id_t, flags: Int32) throws(SubprocessError.UnderlyingError) -> siginfo_t { + while true { + var siginfo = siginfo_t() + if waitid(idtype, id, &siginfo, flags) != -1 { + return siginfo + } else if errno != EINTR { + throw SubprocessError.UnderlyingError(rawValue: errno) + } + } +} + +internal extension TerminationStatus { + init(_ siginfo: siginfo_t) { + switch siginfo.si_code { + case .init(CLD_EXITED): + self = .exited(siginfo.si_status) + case .init(CLD_KILLED), .init(CLD_DUMPED): + self = .unhandledException(siginfo.si_status) + default: + fatalError("Unexpected exit status: \(siginfo.si_code)") + } + } +} + +#if os(OpenBSD) || os(Linux) || os(Android) +internal extension siginfo_t { + var si_status: Int32 { + #if os(OpenBSD) + return _data._proc._pdata._cld._status + #elseif canImport(Glibc) + return _sifields._sigchld.si_status + #elseif canImport(Musl) + return __si_fields.__si_common.__second.__sigchld.si_status + #elseif canImport(Bionic) + return _sifields._sigchld._status + #endif + } + + var si_pid: pid_t { + #if os(OpenBSD) + return _data._proc._pid + #elseif canImport(Glibc) + return _sifields._sigchld.si_pid + #elseif canImport(Musl) + return __si_fields.__si_common.__first.__piduid.si_pid + #elseif canImport(Bionic) + return _sifields._kill._pid + #endif + } +} +#endif + #endif // canImport(Darwin) || canImport(Glibc) || canImport(Android) || canImport(Musl) diff --git a/Sources/Subprocess/SubprocessFoundation/Input+Foundation.swift b/Sources/Subprocess/SubprocessFoundation/Input+Foundation.swift index c82d2d38..f124a7e8 100644 --- a/Sources/Subprocess/SubprocessFoundation/Input+Foundation.swift +++ b/Sources/Subprocess/SubprocessFoundation/Input+Foundation.swift @@ -129,7 +129,7 @@ extension StandardInputWriter { } -#if canImport(Darwin) +#if SUBPROCESS_ASYNCIO_DISPATCH extension AsyncIO { internal func write( _ data: Data, @@ -168,6 +168,6 @@ extension AsyncIO { return try await self._write(data, to: diskIO) } } -#endif // canImport(Darwin) +#endif // SUBPROCESS_ASYNCIO_DISPATCH #endif // SubprocessFoundation diff --git a/Sources/_SubprocessCShims/include/process_shims.h b/Sources/_SubprocessCShims/include/process_shims.h index 85c97bb3..584d5418 100644 --- a/Sources/_SubprocessCShims/include/process_shims.h +++ b/Sources/_SubprocessCShims/include/process_shims.h @@ -23,11 +23,18 @@ #if TARGET_OS_LINUX #include -#include -#include #include #endif // TARGET_OS_LINUX +#if TARGET_OS_FREEBSD +#include +#endif + +#if TARGET_OS_LINUX || TARGET_OS_FREEBSD +#include +#include +#endif // TARGET_OS_LINUX || TARGET_OS_FREEBSD + #if __has_include() vm_size_t _subprocess_vm_size(void); #endif @@ -72,7 +79,9 @@ void _subprocess_lock_environ(void); void _subprocess_unlock_environ(void); char * _Nullable * _Nullable _subprocess_get_environ(void); -#if TARGET_OS_LINUX +int _subprocess_pdkill(int pidfd, int signal); + +#if TARGET_OS_UNIX && !TARGET_OS_FREEBSD int _shims_snprintf( char * _Nonnull str, int len, @@ -80,9 +89,10 @@ int _shims_snprintf( char * _Nonnull str1, char * _Nonnull str2 ); +#endif +#if TARGET_OS_LINUX int _pidfd_open(pid_t pid); -int _pidfd_send_signal(int pidfd, int signal); // P_PIDFD is only defined on Linux Kernel 5.4 and above // Define our value if it's not available diff --git a/Sources/_SubprocessCShims/include/target_conditionals.h b/Sources/_SubprocessCShims/include/target_conditionals.h index aaa4a600..ba4e5c6e 100644 --- a/Sources/_SubprocessCShims/include/target_conditionals.h +++ b/Sources/_SubprocessCShims/include/target_conditionals.h @@ -28,10 +28,16 @@ #define TARGET_OS_LINUX 0 #endif +#if defined(__FreeBSD__) +#define TARGET_OS_FREEBSD 1 +#else +#define TARGET_OS_FREEBSD 0 +#endif + #if defined(__unix__) -#define TARGET_OS_BSD 1 +#define TARGET_OS_UNIX 1 #else -#define TARGET_OS_BSD 0 +#define TARGET_OS_UNIX 0 #endif #if defined(_WIN32) diff --git a/Sources/_SubprocessCShims/process_shims.c b/Sources/_SubprocessCShims/process_shims.c index 5fa1d2cc..d77d3723 100644 --- a/Sources/_SubprocessCShims/process_shims.c +++ b/Sources/_SubprocessCShims/process_shims.c @@ -42,6 +42,10 @@ #include #endif +#include +#include +#include + #if __has_include() #include #elif defined(_WIN32) @@ -262,12 +266,10 @@ int _subprocess_spawn( #endif // TARGET_OS_MAC -// MARK: - Linux (fork/exec + posix_spawn fallback) -#if TARGET_OS_LINUX || TARGET_OS_BSD -#ifndef __GLIBC_PREREQ -#define __GLIBC_PREREQ(maj, min) 0 -#endif +// MARK: - Linux/BSD (fork/exec + posix_spawn fallback) +#if TARGET_OS_UNIX && !TARGET_OS_MAC +#if TARGET_OS_LINUX #ifndef SYS_pidfd_open #define SYS_pidfd_open 434 #endif @@ -337,6 +339,29 @@ struct linux_dirent64 { static int _getdents64(int fd, struct linux_dirent64 *dirp, size_t nbytes) { return syscall(SYS_getdents64, fd, dirp, nbytes); } +#endif + +static pid_t _subprocess_pdfork(int *fdp) { +#if TARGET_OS_LINUX + return _clone3(fdp); // CLONE_PIDFD always sets close-on-exec on the fd +#elif TARGET_OS_FREEBSD + return pdfork(fdp, PD_CLOEXEC); +#else + errno = ENOSYS; + return -1; +#endif +} + +int _subprocess_pdkill(int pidfd, int signal) { +#if TARGET_OS_LINUX + return _pidfd_send_signal(pidfd, signal); +#elif TARGET_OS_FREEBSD + return pdkill(pidfd, signal); +#else + errno = ENOSYS; + return -1; +#endif +} static pthread_mutex_t _subprocess_fork_lock = PTHREAD_MUTEX_INITIALIZER; @@ -379,6 +404,7 @@ static int _subprocess_make_critical_mask(sigset_t *old_mask) { # define _SUBPROCESS_SIG_MAX (sizeof(sigset_t) * CHAR_BIT + 1) #endif +#if !TARGET_OS_FREEBSD int _shims_snprintf( char * _Nonnull str, int len, @@ -388,6 +414,7 @@ int _shims_snprintf( ) { return snprintf(str, len, format, str1, str2); } +#endif static int _positive_int_parse(const char *str) { char *end; @@ -403,6 +430,7 @@ static int _positive_int_parse(const char *str) { return (int)value; } +#if defined(__linux__) // Linux-specific version that uses syscalls directly and doesn't allocate heap memory. // Safe to use after vfork() and before execve() static int _highest_possibly_open_fd_dir_linux(const char *fd_dir) { @@ -451,6 +479,7 @@ static int _highest_possibly_open_fd_dir_linux(const char *fd_dir) { close(dir_fd); return highest_fd_so_far; } +#endif // This function is only used on systems with Linux kernel 5.9 or lower. // On newer systems, `close_range` is used instead. @@ -533,11 +562,11 @@ int _subprocess_fork_exec( // Finally, fork / clone int _pidfd = -1; - // First attempt to use clone3, only fall back to fork if clone3 is not available - pid_t childPid = _clone3(&_pidfd); + // First attempt to create a process file descriptor on supported platforms, only fall back to fork if those are not available + pid_t childPid = _subprocess_pdfork(&_pidfd); if (childPid < 0) { if (errno == ENOSYS) { - // clone3 is not implemented. Use fork instead + // process file descriptor is not implemented. Use fork instead #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated" childPid = fork(); @@ -683,11 +712,13 @@ int _subprocess_fork_exec( waitid(P_PID, childPid, &info, WEXITED); \ return capturedError +#if TARGET_OS_LINUX // On Linux 5.3 and lower, we have to fetch pidfd separately // Newer Linux supports clone3 which returns pidfd directly if (_pidfd < 0) { _pidfd = _pidfd_open(childPid); } +#endif // Parent process close(pipefd[1]); // Close unused write end @@ -738,7 +769,7 @@ int _subprocess_fork_exec( } } -#endif // TARGET_OS_LINUX +#endif // TARGET_OS_UNIX && !TARGET_OS_MAC #pragma mark - Environment Locking diff --git a/Tests/SubprocessTests/AsyncIOTests.swift b/Tests/SubprocessTests/AsyncIOTests.swift index 7a3a07ad..7ccbb846 100644 --- a/Tests/SubprocessTests/AsyncIOTests.swift +++ b/Tests/SubprocessTests/AsyncIOTests.swift @@ -138,8 +138,8 @@ extension SubprocessAsyncIOTests { extension SubprocessAsyncIOTests { @Test func testWriteToClosedPipe() async throws { var pipe = try CreatedPipe(closeWhenDone: true, purpose: .input) - var writeChannel = pipe.writeFileDescriptor()!.createIOChannel() - var readChannel = pipe.readFileDescriptor()!.createIOChannel() + var writeChannel = try _require(pipe.writeFileDescriptor()).createIOChannel() + var readChannel = try _require(pipe.readFileDescriptor()).createIOChannel() defer { try? readChannel.safelyClose() } @@ -176,8 +176,8 @@ extension SubprocessAsyncIOTests { @Test func testReadFromClosedPipe() async throws { var pipe = try CreatedPipe(closeWhenDone: true, purpose: .input) - var writeChannel = pipe.writeFileDescriptor()!.createIOChannel() - var readChannel = pipe.readFileDescriptor()!.createIOChannel() + var writeChannel = try _require(pipe.writeFileDescriptor()).createIOChannel() + var readChannel = try _require(pipe.readFileDescriptor()).createIOChannel() defer { try? writeChannel.safelyClose() } @@ -257,12 +257,12 @@ extension SubprocessAsyncIOTests { group.addTask { var readIOContainer: IOChannel? = readBox.take() - let readTestBed = TestBed(ioChannel: readIOContainer.take()!) + let readTestBed = try TestBed(ioChannel: _require(readIOContainer.take())) try await reader(readIO, readTestBed) } group.addTask { var writeIOContainer: IOChannel? = writeBox.take() - let writeTestBed = TestBed(ioChannel: writeIOContainer.take()!) + let writeTestBed = try TestBed(ioChannel: _require(writeIOContainer.take())) try await writer(writeIO, writeTestBed) } @@ -276,10 +276,10 @@ extension SubprocessAsyncIOTests { extension SubprocessAsyncIOTests.TestBed { consuming func finish() async throws { -#if canImport(WinSDK) - try _safelyClose(.handle(self.ioChannel.channel)) -#elseif canImport(Darwin) +#if SUBPROCESS_ASYNCIO_DISPATCH try _safelyClose(.dispatchIO(self.ioChannel.channel)) +#elseif canImport(WinSDK) + try _safelyClose(.handle(self.ioChannel.channel)) #else try _safelyClose(.fileDescriptor(self.ioChannel.channel)) #endif diff --git a/Tests/SubprocessTests/IntegrationTests.swift b/Tests/SubprocessTests/IntegrationTests.swift index d60be0e9..04f49332 100644 --- a/Tests/SubprocessTests/IntegrationTests.swift +++ b/Tests/SubprocessTests/IntegrationTests.swift @@ -429,7 +429,7 @@ extension SubprocessIntegrationTests { #expect(result.terminationStatus.isSuccess) // There shouldn't be any other environment variables besides // `PATH` that we set - let resultPath = result.standardOutput! + let resultPath = try #require(result.standardOutput) .trimmingNewLineAndQuotes() #if canImport(Darwin) // On Darwin, /var is linked to /private/var; /tmp is linked to /private/tmp @@ -1686,7 +1686,7 @@ extension SubprocessIntegrationTests { } // Generate at least 2 long lines that is longer than buffer size - func generateTestCases(count: Int) -> [TestCase] { + func generateTestCases(count: Int) throws -> [TestCase] { var targetSizes: [TestCaseSize] = TestCaseSize.allCases.flatMap { Array(repeating: $0, count: count / 3) } @@ -1701,7 +1701,7 @@ extension SubprocessIntegrationTests { for size in targetSizes { let components = generateString(size: size) // Choose a random new line - let newLine = newLineCharacters.randomElement()! + let newLine = try #require(newLineCharacters.randomElement()) let string = String(decoding: components + newLine, as: UTF8.self) testCases.append(( value: string, @@ -1717,7 +1717,7 @@ extension SubprocessIntegrationTests { FileManager.default.createFile(atPath: url.path(), contents: nil, attributes: nil) let fileHadle = try FileHandle(forWritingTo: url) for testCase in testCases { - fileHadle.write(testCase.value.data(using: .utf8)!) + fileHadle.write(Data(testCase.value.utf8)) } try fileHadle.close() #else @@ -1734,7 +1734,7 @@ extension SubprocessIntegrationTests { if FileManager.default.fileExists(atPath: testFilePath.path()) { try FileManager.default.removeItem(at: testFilePath) } - let testCases = generateTestCases(count: testCaseCount) + let testCases = try generateTestCases(count: testCaseCount) try writeTestCasesToFile(testCases, at: testFilePath) #if os(Windows) diff --git a/Tests/SubprocessTests/LinterTests.swift b/Tests/SubprocessTests/LinterTests.swift index d41be90d..8be35f8e 100644 --- a/Tests/SubprocessTests/LinterTests.swift +++ b/Tests/SubprocessTests/LinterTests.swift @@ -54,8 +54,8 @@ struct SubprocessLintingTest { else { return } - let sourcePath = String( - maybePath.prefix(upTo: maybePath.range(of: "/.build")!.lowerBound) + let sourcePath = try String( + maybePath.prefix(upTo: #require(maybePath.range(of: "/.build")).lowerBound) ) print("Linting \(sourcePath)") #if os(macOS) diff --git a/Tests/SubprocessTests/LinuxTests.swift b/Tests/SubprocessTests/LinuxTests.swift index a965cb0d..98f975bf 100644 --- a/Tests/SubprocessTests/LinuxTests.swift +++ b/Tests/SubprocessTests/LinuxTests.swift @@ -41,6 +41,10 @@ struct SubprocessLinuxTests { waitThread = try pthread_create { var suspendedStatus: Int32 = 0 let rc = waitpid(pid, &suspendedStatus, targetSignal) + if rc == -1 { + continuation.resume(throwing: SubprocessError.UnderlyingError(rawValue: errno)) + return + } handler(suspendedStatus) continuation.resume() } diff --git a/Tests/SubprocessTests/PlatformConformance.swift b/Tests/SubprocessTests/PlatformConformance.swift index 3c619d2e..04bd2705 100644 --- a/Tests/SubprocessTests/PlatformConformance.swift +++ b/Tests/SubprocessTests/PlatformConformance.swift @@ -36,7 +36,7 @@ protocol ProcessIdentifierProtocol: Sendable, Hashable, CustomStringConvertible, var value: pid_t { get } #endif - #if os(Linux) || os(Android) + #if os(Linux) || os(Android) || os(FreeBSD) var processDescriptor: PlatformFileDescriptor { get } #endif diff --git a/Tests/SubprocessTests/ProcessMonitoringTests.swift b/Tests/SubprocessTests/ProcessMonitoringTests.swift index 742e263c..1fab4ebe 100644 --- a/Tests/SubprocessTests/ProcessMonitoringTests.swift +++ b/Tests/SubprocessTests/ProcessMonitoringTests.swift @@ -19,8 +19,8 @@ import Darwin #elseif canImport(Glibc) import Glibc -#elseif canImport(Bionic) -import Bionic +#elseif canImport(Android) +import Android #elseif canImport(Musl) import Musl #elseif canImport(WinSDK) @@ -76,7 +76,7 @@ struct SubprocessProcessMonitoringTests { private func devNullInputPipe() throws -> CreatedPipe { #if os(Windows) let devnullFd: FileDescriptor = try .openDevNull(withAccessMode: .writeOnly) - let devnull = HANDLE(bitPattern: _get_osfhandle(devnullFd.rawValue))! + let devnull = try #require(HANDLE(bitPattern: _get_osfhandle(devnullFd.rawValue))) #else let devnull: FileDescriptor = try .openDevNull(withAccessMode: .readOnly) #endif @@ -89,7 +89,7 @@ struct SubprocessProcessMonitoringTests { private func devNullOutputPipe() throws -> CreatedPipe { #if os(Windows) let devnullFd: FileDescriptor = try .openDevNull(withAccessMode: .writeOnly) - let devnull = HANDLE(bitPattern: _get_osfhandle(devnullFd.rawValue))! + let devnull = try #require(HANDLE(bitPattern: _get_osfhandle(devnullFd.rawValue))) #else let devnull: FileDescriptor = try .openDevNull(withAccessMode: .writeOnly) #endif @@ -206,7 +206,7 @@ extension SubprocessProcessMonitoringTests { let processIdentifier = ProcessIdentifier( value: .max, processDescriptor: INVALID_HANDLE_VALUE, threadHandle: INVALID_HANDLE_VALUE ) - #elseif os(Linux) || os (FreeBSD) + #elseif os(Linux) || os(Android) || os(FreeBSD) let expectedError = SubprocessError( code: .init(.failedToMonitorProcess), underlyingError: .init(rawValue: ECHILD) diff --git a/Tests/SubprocessTests/TestSupport.swift b/Tests/SubprocessTests/TestSupport.swift index c13dc1d7..b7351202 100644 --- a/Tests/SubprocessTests/TestSupport.swift +++ b/Tests/SubprocessTests/TestSupport.swift @@ -20,6 +20,14 @@ import FoundationEssentials import Testing import Subprocess +// Workaround: https://github.com/swiftlang/swift-testing/issues/543 +internal func _require(_ value: consuming T?) throws -> T { + guard let value else { + throw SubprocessError.UnderlyingError(rawValue: .max) + } + return value +} + internal func randomString(length: Int, lettersOnly: Bool = false) -> String { let letters: String if lettersOnly { diff --git a/Tests/SubprocessTests/UnixTests.swift b/Tests/SubprocessTests/UnixTests.swift index ccb8013b..20ed2459 100644 --- a/Tests/SubprocessTests/UnixTests.swift +++ b/Tests/SubprocessTests/UnixTests.swift @@ -106,7 +106,7 @@ extension SubprocessUnixTests { let ids = try #require( idResult.standardOutput ).split(separator: ",") - .map { gid_t($0.trimmingCharacters(in: .whitespacesAndNewlines))! } + .map { try #require(gid_t($0.trimmingCharacters(in: .whitespacesAndNewlines))) } #expect(Set(ids) == expectedGroups) } @@ -253,7 +253,7 @@ extension SubprocessUnixTests { .send(signal: .interrupt, allowedDurationToNextStep: .milliseconds(100)) ] let result = try await Subprocess.run( - .path("/bin/bash"), + .name("bash"), arguments: [ "-c", """ @@ -350,6 +350,9 @@ extension SubprocessUnixTests { done """ var arguments = ["-c", shellScript, "--"] + #if os(FreeBSD) + arguments.append("") // FreeBSD /bin/sh interprets the first argument as the script name + #endif arguments.append(contentsOf: openedFileDescriptors.map { "\($0)" }) let result = try await Subprocess.run( @@ -385,7 +388,7 @@ extension SubprocessUnixTests { let result = try await pipe.writeEnd.closeAfter { // Spawn bash and then attempt to write to the write end try await Subprocess.run( - .path("/bin/bash"), + .name("bash"), arguments: [ "-c", """