From 83ae2dd52e8f3269a91748fa2b2366700cae22d0 Mon Sep 17 00:00:00 2001 From: Jake Petroules Date: Tue, 29 Jul 2025 17:22:47 -0700 Subject: [PATCH] Fix the build on FreeBSD (and OpenBSD) For now, we use Dispatch for AsyncIO and process termination monitoring, since BSDs (including macOS) all use kqueue as the Dispatch backend anyways. Closes #115 --- Package.swift | 8 +- Sources/Subprocess/AsyncBufferSequence.swift | 6 +- Sources/Subprocess/Buffer.swift | 10 +- Sources/Subprocess/CMakeLists.txt | 7 +- Sources/Subprocess/Configuration.swift | 14 +- ...IO+Darwin.swift => AsyncIO+Dispatch.swift} | 6 +- Sources/Subprocess/IO/AsyncIO+Linux.swift | 15 +- Sources/Subprocess/IO/Output.swift | 12 +- .../Subprocess/Platforms/Subprocess+BSD.swift | 62 +++ .../Platforms/Subprocess+Darwin.swift | 42 -- .../Platforms/Subprocess+Linux.swift | 413 ++---------------- .../Platforms/Subprocess+Unix.swift | 379 +++++++++++++++- .../Input+Foundation.swift | 4 +- .../_SubprocessCShims/include/process_shims.h | 18 +- .../include/target_conditionals.h | 10 +- Sources/_SubprocessCShims/process_shims.c | 49 ++- Tests/SubprocessTests/AsyncIOTests.swift | 18 +- Tests/SubprocessTests/IntegrationTests.swift | 10 +- Tests/SubprocessTests/LinterTests.swift | 4 +- Tests/SubprocessTests/LinuxTests.swift | 4 + .../SubprocessTests/PlatformConformance.swift | 2 +- .../ProcessMonitoringTests.swift | 10 +- Tests/SubprocessTests/TestSupport.swift | 8 + Tests/SubprocessTests/UnixTests.swift | 9 +- 24 files changed, 627 insertions(+), 493 deletions(-) rename Sources/Subprocess/IO/{AsyncIO+Darwin.swift => AsyncIO+Dispatch.swift} (97%) create mode 100644 Sources/Subprocess/Platforms/Subprocess+BSD.swift diff --git a/Package.swift b/Package.swift index 62cf4da6..ebf38b22 100644 --- a/Package.swift +++ b/Package.swift @@ -27,6 +27,10 @@ var defaultTraits: Set = ["SubprocessFoundation"] defaultTraits.insert("SubprocessSpan") #endif +let packageSwiftSettings: [SwiftSetting] = [ + .define("SUBPROCESS_ASYNCIO_DISPATCH", .when(platforms: [.macOS, .custom("freebsd"), .openbsd])) +] + let package = Package( name: "Subprocess", platforms: [.macOS(.v13), .iOS("99.0")], @@ -58,7 +62,7 @@ let package = Package( .enableExperimentalFeature("NonescapableTypes"), .enableExperimentalFeature("LifetimeDependence"), .enableExperimentalFeature("Span"), - ] + ] + packageSwiftSettings ), .testTarget( name: "SubprocessTests", @@ -70,7 +74,7 @@ let package = Package( ], swiftSettings: [ .enableExperimentalFeature("Span"), - ] + ] + packageSwiftSettings ), .target( diff --git a/Sources/Subprocess/AsyncBufferSequence.swift b/Sources/Subprocess/AsyncBufferSequence.swift index 3076b12d..8fe78f1f 100644 --- a/Sources/Subprocess/AsyncBufferSequence.swift +++ b/Sources/Subprocess/AsyncBufferSequence.swift @@ -23,7 +23,7 @@ public struct AsyncBufferSequence: AsyncSequence, @unchecked Sendable { public typealias Failure = any Swift.Error public typealias Element = Buffer - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH internal typealias DiskIO = DispatchIO #elseif canImport(WinSDK) internal typealias DiskIO = HANDLE @@ -55,7 +55,7 @@ public struct AsyncBufferSequence: AsyncSequence, @unchecked Sendable { ) guard let data else { // We finished reading. Close the file descriptor now - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH try _safelyClose(.dispatchIO(self.diskIO)) #elseif canImport(WinSDK) try _safelyClose(.handle(self.diskIO)) @@ -137,7 +137,7 @@ extension AsyncBufferSequence { self.eofReached = true return nil } - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH // Unfortunately here we _have to_ copy the bytes out because // DispatchIO (rightfully) reuses buffer, which means `buffer.data` // has the same address on all iterations, therefore we can't directly diff --git a/Sources/Subprocess/Buffer.swift b/Sources/Subprocess/Buffer.swift index 8152ac56..74216445 100644 --- a/Sources/Subprocess/Buffer.swift +++ b/Sources/Subprocess/Buffer.swift @@ -17,7 +17,7 @@ extension AsyncBufferSequence { /// A immutable collection of bytes public struct Buffer: Sendable { - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH // We need to keep the backingData alive while Slice is alive internal let backingData: DispatchData internal let data: DispatchData.Region @@ -45,7 +45,7 @@ extension AsyncBufferSequence { internal static func createFrom(_ data: [UInt8]) -> [Buffer] { return [.init(data: data)] } - #endif // canImport(Darwin) + #endif // SUBPROCESS_ASYNCIO_DISPATCH } } @@ -92,7 +92,7 @@ extension AsyncBufferSequence.Buffer { // MARK: - Hashable, Equatable extension AsyncBufferSequence.Buffer: Equatable, Hashable { - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH public static func == (lhs: AsyncBufferSequence.Buffer, rhs: AsyncBufferSequence.Buffer) -> Bool { return lhs.data == rhs.data } @@ -104,7 +104,7 @@ extension AsyncBufferSequence.Buffer: Equatable, Hashable { // else Compiler generated conformances } -#if canImport(Darwin) +#if SUBPROCESS_ASYNCIO_DISPATCH extension DispatchData.Region { static func == (lhs: DispatchData.Region, rhs: DispatchData.Region) -> Bool { return lhs.withUnsafeBytes { lhsBytes in @@ -120,7 +120,7 @@ extension DispatchData.Region { } } } -#if !SubprocessFoundation +#if !canImport(Darwin) || !SubprocessFoundation /// `DispatchData.Region` is defined in Foundation, but we can't depend on Foundation when the SubprocessFoundation trait is disabled. extension DispatchData { typealias Region = _ContiguousBufferView diff --git a/Sources/Subprocess/CMakeLists.txt b/Sources/Subprocess/CMakeLists.txt index ea95c737..53de37cf 100644 --- a/Sources/Subprocess/CMakeLists.txt +++ b/Sources/Subprocess/CMakeLists.txt @@ -17,7 +17,7 @@ add_library(Subprocess Result.swift IO/Output.swift IO/Input.swift - IO/AsyncIO+Darwin.swift + IO/AsyncIO+Dispatch.swift IO/AsyncIO+Linux.swift IO/AsyncIO+Windows.swift Span+Subprocess.swift @@ -36,8 +36,13 @@ elseif(LINUX OR ANDROID) Platforms/Subprocess+Unix.swift) elseif(APPLE) target_sources(Subprocess PRIVATE + Platforms/Subprocess+BSD.swift Platforms/Subprocess+Darwin.swift Platforms/Subprocess+Unix.swift) +elseif(FREEBSD OR OPENBSD) + target_sources(Subprocess PRIVATE + Platforms/Subprocess+BSD.swift + Platforms/Subprocess+Unix.swift) endif() target_compile_options(Subprocess PRIVATE diff --git a/Sources/Subprocess/Configuration.swift b/Sources/Subprocess/Configuration.swift index 4953f48b..ed506e69 100644 --- a/Sources/Subprocess/Configuration.swift +++ b/Sources/Subprocess/Configuration.swift @@ -657,7 +657,7 @@ internal struct IODescriptor: ~Copyable { consuming func createIOChannel() -> IOChannel { let shouldClose = self.closeWhenDone self.closeWhenDone = false - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH // Transferring out the ownership of fileDescriptor means we don't have go close here let closeFd = self.descriptor let dispatchIO: DispatchIO = DispatchIO( @@ -708,10 +708,10 @@ internal struct IODescriptor: ~Copyable { } internal struct IOChannel: ~Copyable, @unchecked Sendable { - #if canImport(WinSDK) - typealias Channel = HANDLE - #elseif canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH typealias Channel = DispatchIO + #elseif canImport(WinSDK) + typealias Channel = HANDLE #else typealias Channel = FileDescriptor #endif @@ -733,10 +733,10 @@ internal struct IOChannel: ~Copyable, @unchecked Sendable { } closeWhenDone = false - #if canImport(WinSDK) - try _safelyClose(.handle(self.channel)) - #elseif canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH try _safelyClose(.dispatchIO(self.channel)) + #elseif canImport(WinSDK) + try _safelyClose(.handle(self.channel)) #else try _safelyClose(.fileDescriptor(self.channel)) #endif diff --git a/Sources/Subprocess/IO/AsyncIO+Darwin.swift b/Sources/Subprocess/IO/AsyncIO+Dispatch.swift similarity index 97% rename from Sources/Subprocess/IO/AsyncIO+Darwin.swift rename to Sources/Subprocess/IO/AsyncIO+Dispatch.swift index 11cf94a6..280fee91 100644 --- a/Sources/Subprocess/IO/AsyncIO+Darwin.swift +++ b/Sources/Subprocess/IO/AsyncIO+Dispatch.swift @@ -12,7 +12,7 @@ /// Darwin AsyncIO implementation based on DispatchIO // MARK: - macOS (DispatchIO) -#if canImport(Darwin) +#if SUBPROCESS_ASYNCIO_DISPATCH #if canImport(System) @preconcurrency import System @@ -166,4 +166,8 @@ final class AsyncIO: Sendable { } } +#if !canImport(Darwin) +extension DispatchData: @retroactive @unchecked Sendable { } +#endif + #endif diff --git a/Sources/Subprocess/IO/AsyncIO+Linux.swift b/Sources/Subprocess/IO/AsyncIO+Linux.swift index a667aa2c..b04b747e 100644 --- a/Sources/Subprocess/IO/AsyncIO+Linux.swift +++ b/Sources/Subprocess/IO/AsyncIO+Linux.swift @@ -11,7 +11,7 @@ /// Linux AsyncIO implementation based on epoll -#if canImport(Glibc) || canImport(Android) || canImport(Musl) +#if os(Linux) || os(Android) #if canImport(System) @preconcurrency import System @@ -266,6 +266,11 @@ final class AsyncIO: Sendable { targetEvent = EPOLL_EVENTS(EPOLLOUT) } + // Save the continuation (before calling epoll_ctl, so we don't miss any data) + _registration.withLock { storage in + storage[fileDescriptor.rawValue] = continuation + } + var event = epoll_event( events: targetEvent.rawValue, data: epoll_data(fd: fileDescriptor.rawValue) @@ -277,6 +282,10 @@ final class AsyncIO: Sendable { &event ) if rc != 0 { + _registration.withLock { storage in + storage.removeValue(forKey: fileDescriptor.rawValue) + } + let capturedError = errno let error = SubprocessError( code: .init(.asyncIOFailed( @@ -287,10 +296,6 @@ final class AsyncIO: Sendable { continuation.finish(throwing: error) return } - // Now save the continuation - _registration.withLock { storage in - storage[fileDescriptor.rawValue] = continuation - } case .failure(let setupError): continuation.finish(throwing: setupError) return diff --git a/Sources/Subprocess/IO/Output.swift b/Sources/Subprocess/IO/Output.swift index 1096a307..131d2840 100644 --- a/Sources/Subprocess/IO/Output.swift +++ b/Sources/Subprocess/IO/Output.swift @@ -148,7 +148,7 @@ public struct BytesOutput: OutputProtocol { internal func captureOutput( from diskIO: consuming IOChannel ) async throws -> [UInt8] { - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH var result: DispatchData? = nil #else var result: [UInt8]? = nil @@ -173,7 +173,7 @@ public struct BytesOutput: OutputProtocol { underlyingError: nil ) } - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH return result?.array() ?? [] #else return result ?? [] @@ -302,7 +302,7 @@ extension OutputProtocol { return try await bytesOutput.captureOutput(from: diskIO) as! Self.OutputType } - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH var result: DispatchData? = nil #else var result: [UInt8]? = nil @@ -328,7 +328,7 @@ extension OutputProtocol { ) } - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH return try self.output(from: result ?? .empty) #else return try self.output(from: result ?? []) @@ -353,7 +353,7 @@ extension OutputProtocol where OutputType == Void { #if SubprocessSpan extension OutputProtocol { - #if canImport(Darwin) + #if SUBPROCESS_ASYNCIO_DISPATCH internal func output(from data: DispatchData) throws -> OutputType { guard !data.isEmpty else { let empty = UnsafeRawBufferPointer(start: nil, count: 0) @@ -380,7 +380,7 @@ extension OutputProtocol { return try self.output(from: span) } } - #endif // canImport(Darwin) + #endif // SUBPROCESS_ASYNCIO_DISPATCH } #endif diff --git a/Sources/Subprocess/Platforms/Subprocess+BSD.swift b/Sources/Subprocess/Platforms/Subprocess+BSD.swift new file mode 100644 index 00000000..c17dab74 --- /dev/null +++ b/Sources/Subprocess/Platforms/Subprocess+BSD.swift @@ -0,0 +1,62 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +#if os(macOS) || os(FreeBSD) || os(OpenBSD) + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +internal import Dispatch + +// MARK: - Process Monitoring +@Sendable +internal func monitorProcessTermination( + for processIdentifier: ProcessIdentifier +) async throws -> TerminationStatus { + switch Result(catching: { () throws(SubprocessError.UnderlyingError) -> TerminationStatus? in try processIdentifier.reap() }) { + case let .success(status?): + return status + case .success(nil): + break + case let .failure(error): + throw SubprocessError( + code: .init(.failedToMonitorProcess), + underlyingError: error + ) + } + return try await withCheckedThrowingContinuation { continuation in + let source = DispatchSource.makeProcessSource( + identifier: processIdentifier.value, + eventMask: [.exit], + queue: .global() + ) + source.setEventHandler { + source.cancel() + continuation.resume(with: Result(catching: { () throws(SubprocessError.UnderlyingError) -> TerminationStatus in + // NOTE_EXIT may be delivered slightly before the process becomes reapable, + // so we must call waitid without WNOHANG to avoid a narrow possibility of a race condition. + // If waitid does block, it won't do so for very long at all. + try processIdentifier.blockingReap() + }).mapError { underlyingError in + SubprocessError( + code: .init(.failedToMonitorProcess), + underlyingError: underlyingError + ) + }) + } + source.resume() + } +} + +#endif diff --git a/Sources/Subprocess/Platforms/Subprocess+Darwin.swift b/Sources/Subprocess/Platforms/Subprocess+Darwin.swift index 99ff9d65..cfaa6149 100644 --- a/Sources/Subprocess/Platforms/Subprocess+Darwin.swift +++ b/Sources/Subprocess/Platforms/Subprocess+Darwin.swift @@ -497,46 +497,4 @@ extension ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertib public var debugDescription: String { "\(self.value)" } } -// MARK: - Process Monitoring -@Sendable -internal func monitorProcessTermination( - for processIdentifier: ProcessIdentifier -) async throws -> TerminationStatus { - return try await withCheckedThrowingContinuation { continuation in - let source = DispatchSource.makeProcessSource( - identifier: processIdentifier.value, - eventMask: [.exit], - queue: .global() - ) - source.setEventHandler { - source.cancel() - var siginfo = siginfo_t() - let rc = waitid(P_PID, id_t(processIdentifier.value), &siginfo, WEXITED) - guard rc == 0 else { - continuation.resume( - throwing: SubprocessError( - code: .init(.failedToMonitorProcess), - underlyingError: .init(rawValue: errno) - ) - ) - return - } - switch siginfo.si_code { - case .init(CLD_EXITED): - continuation.resume(returning: .exited(siginfo.si_status)) - return - case .init(CLD_KILLED), .init(CLD_DUMPED): - continuation.resume(returning: .unhandledException(siginfo.si_status)) - case .init(CLD_TRAPPED), .init(CLD_STOPPED), .init(CLD_CONTINUED), .init(CLD_NOOP): - // Ignore these signals because they are not related to - // process exiting - break - default: - fatalError("Unexpected exit status: \(siginfo.si_code)") - } - } - source.resume() - } -} - #endif // canImport(Darwin) diff --git a/Sources/Subprocess/Platforms/Subprocess+Linux.swift b/Sources/Subprocess/Platforms/Subprocess+Linux.swift index 512eeed2..444a4c01 100644 --- a/Sources/Subprocess/Platforms/Subprocess+Linux.swift +++ b/Sources/Subprocess/Platforms/Subprocess+Linux.swift @@ -9,14 +9,6 @@ // //===----------------------------------------------------------------------===// -#if canImport(Glibc) || canImport(Android) || canImport(Musl) - -#if canImport(System) -@preconcurrency import System -#else -@preconcurrency import SystemPackage -#endif - #if canImport(Glibc) import Glibc let _subprocess_read = Glibc.read @@ -34,12 +26,19 @@ let _subprocess_write = Musl.write let _subprocess_close = Musl.close #endif +#if os(Linux) || os(Android) + +#if canImport(System) +@preconcurrency import System +#else +@preconcurrency import SystemPackage +#endif + internal import Dispatch import Synchronization import _SubprocessCShims -// Linux specific implementations #if canImport(Glibc) extension EPOLL_EVENTS { init(_ other: EPOLL_EVENTS) { @@ -76,245 +75,6 @@ extension Int32 { } #endif -extension Configuration { - internal func spawn( - withInput inputPipe: consuming CreatedPipe, - outputPipe: consuming CreatedPipe, - errorPipe: consuming CreatedPipe - ) throws -> SpawnResult { - // Ensure the waiter thread is running. - _setupMonitorSignalHandler() - - // Instead of checking if every possible executable path - // is valid, spawn each directly and catch ENOENT - let possiblePaths = self.executable.possibleExecutablePaths( - withPathValue: self.environment.pathValue() - ) - var inputPipeBox: CreatedPipe? = consume inputPipe - var outputPipeBox: CreatedPipe? = consume outputPipe - var errorPipeBox: CreatedPipe? = consume errorPipe - - return try self.preSpawn { args throws -> SpawnResult in - let (env, uidPtr, gidPtr, supplementaryGroups) = args - - var _inputPipe = inputPipeBox.take()! - var _outputPipe = outputPipeBox.take()! - var _errorPipe = errorPipeBox.take()! - - let inputReadFileDescriptor: IODescriptor? = _inputPipe.readFileDescriptor() - let inputWriteFileDescriptor: IODescriptor? = _inputPipe.writeFileDescriptor() - let outputReadFileDescriptor: IODescriptor? = _outputPipe.readFileDescriptor() - let outputWriteFileDescriptor: IODescriptor? = _outputPipe.writeFileDescriptor() - let errorReadFileDescriptor: IODescriptor? = _errorPipe.readFileDescriptor() - let errorWriteFileDescriptor: IODescriptor? = _errorPipe.writeFileDescriptor() - - for possibleExecutablePath in possiblePaths { - var processGroupIDPtr: UnsafeMutablePointer? = nil - if let processGroupID = self.platformOptions.processGroupID { - processGroupIDPtr = .allocate(capacity: 1) - processGroupIDPtr?.pointee = gid_t(processGroupID) - } - // Setup Arguments - let argv: [UnsafeMutablePointer?] = self.arguments.createArgs( - withExecutablePath: possibleExecutablePath - ) - defer { - for ptr in argv { ptr?.deallocate() } - } - // Setup input - let fileDescriptors: [CInt] = [ - inputReadFileDescriptor?.platformDescriptor() ?? -1, - inputWriteFileDescriptor?.platformDescriptor() ?? -1, - outputWriteFileDescriptor?.platformDescriptor() ?? -1, - outputReadFileDescriptor?.platformDescriptor() ?? -1, - errorWriteFileDescriptor?.platformDescriptor() ?? -1, - errorReadFileDescriptor?.platformDescriptor() ?? -1, - ] - - // Spawn - var pid: pid_t = 0 - var processDescriptor: PlatformFileDescriptor = -1 - let spawnError: CInt = possibleExecutablePath.withCString { exePath in - return (self.workingDirectory?.string).withOptionalCString { workingDir in - return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in - return fileDescriptors.withUnsafeBufferPointer { fds in - return _subprocess_fork_exec( - &pid, - &processDescriptor, - exePath, - workingDir, - fds.baseAddress!, - argv, - env, - uidPtr, - gidPtr, - processGroupIDPtr, - CInt(supplementaryGroups?.count ?? 0), - sgroups?.baseAddress, - self.platformOptions.createSession ? 1 : 0 - ) - } - } - } - } - // Spawn error - if spawnError != 0 { - if spawnError == ENOENT || spawnError == EACCES { - // Move on to another possible path - continue - } - // Throw all other errors - try self.safelyCloseMultiple( - inputRead: inputReadFileDescriptor, - inputWrite: inputWriteFileDescriptor, - outputRead: outputReadFileDescriptor, - outputWrite: outputWriteFileDescriptor, - errorRead: errorReadFileDescriptor, - errorWrite: errorWriteFileDescriptor - ) - throw SubprocessError( - code: .init(.spawnFailed), - underlyingError: .init(rawValue: spawnError) - ) - } - // After spawn finishes, close all child side fds - try self.safelyCloseMultiple( - inputRead: inputReadFileDescriptor, - inputWrite: nil, - outputRead: nil, - outputWrite: outputWriteFileDescriptor, - errorRead: nil, - errorWrite: errorWriteFileDescriptor - ) - let execution = Execution( - processIdentifier: .init( - value: pid, - processDescriptor: processDescriptor - ) - ) - return SpawnResult( - execution: execution, - inputWriteEnd: inputWriteFileDescriptor?.createIOChannel(), - outputReadEnd: outputReadFileDescriptor?.createIOChannel(), - errorReadEnd: errorReadFileDescriptor?.createIOChannel() - ) - } - - // If we reach this point, it means either the executable path - // or working directory is not valid. Since posix_spawn does not - // provide which one is not valid, here we make a best effort guess - // by checking whether the working directory is valid. This technically - // still causes TOUTOC issue, but it's the best we can do for error recovery. - try self.safelyCloseMultiple( - inputRead: inputReadFileDescriptor, - inputWrite: inputWriteFileDescriptor, - outputRead: outputReadFileDescriptor, - outputWrite: outputWriteFileDescriptor, - errorRead: errorReadFileDescriptor, - errorWrite: errorWriteFileDescriptor - ) - if let workingDirectory = self.workingDirectory?.string { - guard Configuration.pathAccessible(workingDirectory, mode: F_OK) else { - throw SubprocessError( - code: .init(.failedToChangeWorkingDirectory(workingDirectory)), - underlyingError: .init(rawValue: ENOENT) - ) - } - } - throw SubprocessError( - code: .init(.executableNotFound(self.executable.description)), - underlyingError: .init(rawValue: ENOENT) - ) - } - } -} - -// MARK: - ProcessIdentifier - -/// A platform independent identifier for a Subprocess. -public struct ProcessIdentifier: Sendable, Hashable { - /// The platform specific process identifier value - public let value: pid_t - public let processDescriptor: CInt - - internal init(value: pid_t, processDescriptor: PlatformFileDescriptor) { - self.value = value - self.processDescriptor = processDescriptor - } - - internal func close() { - if self.processDescriptor > 0 { - _ = _subprocess_close(self.processDescriptor) - } - } -} - -extension ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertible { - public var description: String { "\(self.value)" } - - public var debugDescription: String { "\(self.value)" } -} - -// MARK: - Platform Specific Options - -/// The collection of platform-specific settings -/// to configure the subprocess when running -public struct PlatformOptions: Sendable { - /// Set user ID for the subprocess - public var userID: uid_t? = nil - /// Set the real and effective group ID and the saved - /// set-group-ID of the subprocess, equivalent to calling - /// `setgid()` on the child process. - /// Group ID is used to control permissions, particularly - /// for file access. - public var groupID: gid_t? = nil - /// Set list of supplementary group IDs for the subprocess - public var supplementaryGroups: [gid_t]? = nil - /// Set the process group for the subprocess, equivalent to - /// calling `setpgid()` on the child process. - /// Process group ID is used to group related processes for - /// controlling signals. - public var processGroupID: pid_t? = nil - /// Creates a session and sets the process group ID - /// i.e. Detach from the terminal. - public var createSession: Bool = false - /// An ordered list of steps in order to tear down the child - /// process in case the parent task is cancelled before - /// the child process terminates. - /// Always ends in sending a `.kill` signal at the end. - public var teardownSequence: [TeardownStep] = [] - - public init() {} -} - -extension PlatformOptions: CustomStringConvertible, CustomDebugStringConvertible { - internal func description(withIndent indent: Int) -> String { - let indent = String(repeating: " ", count: indent * 4) - return """ - PlatformOptions( - \(indent) userID: \(String(describing: userID)), - \(indent) groupID: \(String(describing: groupID)), - \(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), - \(indent) processGroupID: \(String(describing: processGroupID)), - \(indent) createSession: \(createSession) - \(indent)) - """ - } - - public var description: String { - return self.description(withIndent: 0) - } - - public var debugDescription: String { - return self.description(withIndent: 0) - } -} - -// Special keys used in Error's user dictionary -extension String { - static let debugDescriptionErrorKey = "DebugDescription" -} - // MARK: - Process Monitoring @Sendable internal func monitorProcessTermination( @@ -366,31 +126,19 @@ internal func monitorProcessTermination( // Since Linux coalesce signals, it's possible by the time we request // monitoring the process has already exited. Check to make sure that // is not the case and only save continuation then. - var siginfo = siginfo_t() - // Use NOHANG here because the child process might still be running - if 0 == waitid(P_PID, id_t(processIdentifier.value), &siginfo, WEXITED | WNOHANG) { - // If si_pid and si_signo are both 0, the child is still running since we used WNOHANG - if siginfo.si_pid == 0 && siginfo.si_signo == 0 { - // Save this continuation to be called by signal hander - var newState = storage - newState.continuations[processIdentifier.value] = continuation - state = .started(newState) - return nil - } - - switch siginfo.si_code { - case .init(CLD_EXITED): - return .success(.exited(siginfo.si_status)) - case .init(CLD_KILLED), .init(CLD_DUMPED): - return .success(.unhandledException(siginfo.si_status)) - default: - fatalError("Unexpected exit status: \(siginfo.si_code)") - } - } else { - let waitidError = errno + switch Result(catching: { () throws(SubprocessError.UnderlyingError) -> TerminationStatus? in try processIdentifier.reap() }) { + case let .success(status?): + return .success(status) + case .success(nil): + // Save this continuation to be called by signal hander + var newState = storage + newState.continuations[processIdentifier.value] = continuation + state = .started(newState) + return nil + case let .failure(underlyingError): let error = SubprocessError( code: .init(.failedToMonitorProcess), - underlyingError: .init(rawValue: waitidError) + underlyingError: underlyingError ) return .failure(error) } @@ -431,28 +179,6 @@ private struct MonitorThreadContext: Sendable { } } -internal extension siginfo_t { - var si_status: Int32 { - #if canImport(Glibc) - return _sifields._sigchld.si_status - #elseif canImport(Musl) - return __si_fields.__si_common.__second.__sigchld.si_status - #elseif canImport(Bionic) - return _sifields._sigchld._status - #endif - } - - var si_pid: pid_t { - #if canImport(Glibc) - return _sifields._sigchld.si_pid - #elseif canImport(Musl) - return __si_fields.__si_common.__first.__piduid.si_pid - #elseif canImport(Bionic) - return _sifields._kill._pid - #endif - } -} - // Okay to be unlocked global mutable because this value is only set once like dispatch_once private nonisolated(unsafe) var _signalPipe: (readEnd: CInt, writeEnd: CInt) = (readEnd: -1, writeEnd: -1) // Okay to be unlocked global mutable because this value is only set once like dispatch_once @@ -690,23 +416,12 @@ internal func _setupMonitorSignalHandler() { } private func _blockAndWaitForProcessDescriptor(_ pidfd: CInt, context: MonitorThreadContext) { - var terminationStatus: Result - - var siginfo = siginfo_t() - if 0 == waitid(idtype_t(UInt32(P_PIDFD)), id_t(pidfd), &siginfo, WEXITED) { - switch siginfo.si_code { - case .init(CLD_EXITED): - terminationStatus = .success(.exited(siginfo.si_status)) - case .init(CLD_KILLED), .init(CLD_DUMPED): - terminationStatus = .success(.unhandledException(siginfo.si_status)) - default: - fatalError("Unexpected exit status: \(siginfo.si_code)") - } - } else { - let waitidErrno = errno - terminationStatus = .failure(SubprocessError( + var terminationStatus = Result(catching: { () throws(SubprocessError.UnderlyingError) in + try TerminationStatus(_waitid(idtype: idtype_t(UInt32(P_PIDFD)), id: id_t(pidfd), flags: WEXITED)) + }).mapError { underlyingError in + SubprocessError( code: .init(.failedToMonitorProcess), - underlyingError: .init(rawValue: waitidErrno)) + underlyingError: underlyingError ) } @@ -761,34 +476,22 @@ private func _reapAllKnownChildProcesses(_ signalFd: CInt, context: MonitorThrea var results: [ResultContinuation] = [] // Since Linux coalesce signals, we need to loop through all known child process // to check if they exited. - for knownChildPID in storage.continuations.keys { + loop: for (knownChildPID, continuation) in storage.continuations { let terminationStatus: Result - var siginfo = siginfo_t() - // Use `WNOHANG` here so waitid isn't blocking because we expect some - // child processes might be still running - if 0 == waitid(P_PID, id_t(knownChildPID), &siginfo, WEXITED | WNOHANG) { - // If si_pid and si_signo, the child is still running since we used WNOHANG - if siginfo.si_pid == 0 && siginfo.si_signo == 0 { - // Move on to the next child - continue - } - - switch siginfo.si_code { - case .init(CLD_EXITED): - terminationStatus = .success(.exited(siginfo.si_status)) - case .init(CLD_KILLED), .init(CLD_DUMPED): - terminationStatus = .success(.unhandledException(siginfo.si_status)) - default: - fatalError("Unexpected exit status: \(siginfo.si_code)") - } - } else { - let waitidErrno = errno - terminationStatus = .failure(SubprocessError( - code: .init(.failedToMonitorProcess), - underlyingError: .init(rawValue: waitidErrno)) - ) + switch Result(catching: { () throws(SubprocessError.UnderlyingError) -> TerminationStatus? in try _reap(pid: knownChildPID) }) { + case let .success(status?): + terminationStatus = .success(status) + case .success(nil): + // Move on to the next child + continue loop + case let .failure(error): + terminationStatus = .failure( + SubprocessError( + code: .init(.failedToMonitorProcess), + underlyingError: error + )) } - results.append((result: terminationStatus, continuation: storage.continuations[knownChildPID]!)) + results.append((result: terminationStatus, continuation: continuation)) // Now we have the exit code, remove saved continuations updatedContinuations.removeValue(forKey: knownChildPID) } @@ -820,47 +523,9 @@ internal func _isWaitprocessDescriptorSupported() -> Bool { /// reported that we don't have a child with the same selfPidfd; /// - EINVAL: in this case we know P_PIDFD is not supported because it does not /// recognize the `P_PIDFD` type + errno = 0 waitid(idtype_t(UInt32(P_PIDFD)), id_t(selfPidfd), &siginfo, WEXITED | WNOWAIT) return errno == ECHILD } -internal func pthread_create(_ body: @Sendable @escaping () -> ()) throws(SubprocessError.UnderlyingError) -> pthread_t { - final class Context { - let body: @Sendable () -> () - init(body: @Sendable @escaping () -> Void) { - self.body = body - } - } - #if canImport(Glibc) || canImport(Musl) - func proc(_ context: UnsafeMutableRawPointer?) -> UnsafeMutableRawPointer? { - (Unmanaged.fromOpaque(context!).takeRetainedValue() as! Context).body() - return nil - } - #elseif canImport(Bionic) - func proc(_ context: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { - (Unmanaged.fromOpaque(context).takeRetainedValue() as! Context).body() - return context - } - #endif - #if canImport(Glibc) || canImport(Bionic) - var thread = pthread_t() - #else - var thread: pthread_t? - #endif - let rc = pthread_create( - &thread, - nil, - proc, - Unmanaged.passRetained(Context(body: body)).toOpaque() - ) - if rc != 0 { - throw SubprocessError.UnderlyingError(rawValue: rc) - } - #if canImport(Glibc) || canImport(Bionic) - return thread - #else - return thread! - #endif -} - #endif // canImport(Glibc) || canImport(Android) || canImport(Musl) diff --git a/Sources/Subprocess/Platforms/Subprocess+Unix.swift b/Sources/Subprocess/Platforms/Subprocess+Unix.swift index e55ff510..b513dbc3 100644 --- a/Sources/Subprocess/Platforms/Subprocess+Unix.swift +++ b/Sources/Subprocess/Platforms/Subprocess+Unix.swift @@ -110,13 +110,13 @@ extension Execution { } let pid = shouldSendToProcessGroup ? -(processIdentifier.value) : processIdentifier.value - #if os(Linux) || os(Android) - // On linux, use pidfd_send_signal if possible + #if os(Linux) || os(Android) || os(FreeBSD) + // On platforms with process descriptors, use _subprocess_pdkill if possible if shouldSendToProcessGroup || self.processIdentifier.processDescriptor < 0 { - // pidfd_send_signal does not support sending signal to process group + // _subprocess_pdkill does not support sending signal to process group try _kill(pid, signal: signal) } else { - let rc = _pidfd_send_signal( + let rc = _subprocess_pdkill( processIdentifier.processDescriptor, signal.rawValue ) @@ -191,7 +191,7 @@ extension Environment { /// length = `key` + `=` + `value` + `\null` let totalLength = keyContainer.count + 1 + valueContainer.count + 1 let fullString: UnsafeMutablePointer = .allocate(capacity: totalLength) - #if os(Linux) || os(Android) + #if os(OpenBSD) || os(Linux) || os(Android) _ = _shims_snprintf(fullString, CInt(totalLength), "%s=%s", rawByteKey, rawByteValue) #else _ = snprintf(ptr: fullString, totalLength, "%s=%s", rawByteKey, rawByteValue) @@ -411,4 +411,373 @@ extension FileDescriptor { internal typealias PlatformFileDescriptor = CInt +// MARK: - Spawning + +#if !canImport(Darwin) +extension Configuration { + internal func spawn( + withInput inputPipe: consuming CreatedPipe, + outputPipe: consuming CreatedPipe, + errorPipe: consuming CreatedPipe + ) throws -> SpawnResult { + // Ensure the waiter thread is running. + #if os(Linux) || os(Android) + _setupMonitorSignalHandler() + #endif + + // Instead of checking if every possible executable path + // is valid, spawn each directly and catch ENOENT + let possiblePaths = self.executable.possibleExecutablePaths( + withPathValue: self.environment.pathValue() + ) + var inputPipeBox: CreatedPipe? = consume inputPipe + var outputPipeBox: CreatedPipe? = consume outputPipe + var errorPipeBox: CreatedPipe? = consume errorPipe + + return try self.preSpawn { args throws -> SpawnResult in + let (env, uidPtr, gidPtr, supplementaryGroups) = args + + var _inputPipe = inputPipeBox.take()! + var _outputPipe = outputPipeBox.take()! + var _errorPipe = errorPipeBox.take()! + + let inputReadFileDescriptor: IODescriptor? = _inputPipe.readFileDescriptor() + let inputWriteFileDescriptor: IODescriptor? = _inputPipe.writeFileDescriptor() + let outputReadFileDescriptor: IODescriptor? = _outputPipe.readFileDescriptor() + let outputWriteFileDescriptor: IODescriptor? = _outputPipe.writeFileDescriptor() + let errorReadFileDescriptor: IODescriptor? = _errorPipe.readFileDescriptor() + let errorWriteFileDescriptor: IODescriptor? = _errorPipe.writeFileDescriptor() + + for possibleExecutablePath in possiblePaths { + var processGroupIDPtr: UnsafeMutablePointer? = nil + if let processGroupID = self.platformOptions.processGroupID { + processGroupIDPtr = .allocate(capacity: 1) + processGroupIDPtr?.pointee = gid_t(processGroupID) + } + // Setup Arguments + let argv: [UnsafeMutablePointer?] = self.arguments.createArgs( + withExecutablePath: possibleExecutablePath + ) + defer { + for ptr in argv { ptr?.deallocate() } + } + // Setup input + let fileDescriptors: [CInt] = [ + inputReadFileDescriptor?.platformDescriptor() ?? -1, + inputWriteFileDescriptor?.platformDescriptor() ?? -1, + outputWriteFileDescriptor?.platformDescriptor() ?? -1, + outputReadFileDescriptor?.platformDescriptor() ?? -1, + errorWriteFileDescriptor?.platformDescriptor() ?? -1, + errorReadFileDescriptor?.platformDescriptor() ?? -1, + ] + + // Spawn + var pid: pid_t = 0 + var processDescriptor: PlatformFileDescriptor = -1 + let spawnError: CInt = possibleExecutablePath.withCString { exePath in + return (self.workingDirectory?.string).withOptionalCString { workingDir in + return supplementaryGroups.withOptionalUnsafeBufferPointer { sgroups in + return fileDescriptors.withUnsafeBufferPointer { fds in + return _subprocess_fork_exec( + &pid, + &processDescriptor, + exePath, + workingDir, + fds.baseAddress!, + argv, + env, + uidPtr, + gidPtr, + processGroupIDPtr, + CInt(supplementaryGroups?.count ?? 0), + sgroups?.baseAddress, + self.platformOptions.createSession ? 1 : 0 + ) + } + } + } + } + // Spawn error + if spawnError != 0 { + if spawnError == ENOENT || spawnError == EACCES { + // Move on to another possible path + continue + } + // Throw all other errors + try self.safelyCloseMultiple( + inputRead: inputReadFileDescriptor, + inputWrite: inputWriteFileDescriptor, + outputRead: outputReadFileDescriptor, + outputWrite: outputWriteFileDescriptor, + errorRead: errorReadFileDescriptor, + errorWrite: errorWriteFileDescriptor + ) + throw SubprocessError( + code: .init(.spawnFailed), + underlyingError: .init(rawValue: spawnError) + ) + } + // After spawn finishes, close all child side fds + try self.safelyCloseMultiple( + inputRead: inputReadFileDescriptor, + inputWrite: nil, + outputRead: nil, + outputWrite: outputWriteFileDescriptor, + errorRead: nil, + errorWrite: errorWriteFileDescriptor + ) + let execution = Execution( + processIdentifier: .init( + value: pid, + processDescriptor: processDescriptor + ) + ) + return SpawnResult( + execution: execution, + inputWriteEnd: inputWriteFileDescriptor?.createIOChannel(), + outputReadEnd: outputReadFileDescriptor?.createIOChannel(), + errorReadEnd: errorReadFileDescriptor?.createIOChannel() + ) + } + + // If we reach this point, it means either the executable path + // or working directory is not valid. Since posix_spawn does not + // provide which one is not valid, here we make a best effort guess + // by checking whether the working directory is valid. This technically + // still causes TOUTOC issue, but it's the best we can do for error recovery. + try self.safelyCloseMultiple( + inputRead: inputReadFileDescriptor, + inputWrite: inputWriteFileDescriptor, + outputRead: outputReadFileDescriptor, + outputWrite: outputWriteFileDescriptor, + errorRead: errorReadFileDescriptor, + errorWrite: errorWriteFileDescriptor + ) + if let workingDirectory = self.workingDirectory?.string { + guard Configuration.pathAccessible(workingDirectory, mode: F_OK) else { + throw SubprocessError( + code: .init(.failedToChangeWorkingDirectory(workingDirectory)), + underlyingError: .init(rawValue: ENOENT) + ) + } + } + throw SubprocessError( + code: .init(.executableNotFound(self.executable.description)), + underlyingError: .init(rawValue: ENOENT) + ) + } + } +} + +// MARK: - ProcessIdentifier + +/// A platform independent identifier for a Subprocess. +public struct ProcessIdentifier: Sendable, Hashable { + /// The platform specific process identifier value + public let value: pid_t + + #if os(Linux) || os(Android) || os(FreeBSD) + public let processDescriptor: CInt + #else + internal let processDescriptor: CInt // not used on other platforms + #endif + + internal init(value: pid_t, processDescriptor: PlatformFileDescriptor) { + self.value = value + self.processDescriptor = processDescriptor + } + + internal func close() { + if self.processDescriptor > 0 { + _ = _subprocess_close(self.processDescriptor) + } + } +} + +extension ProcessIdentifier: CustomStringConvertible, CustomDebugStringConvertible { + public var description: String { "\(self.value)" } + + public var debugDescription: String { "\(self.value)" } +} + +// MARK: - Platform Specific Options + +/// The collection of platform-specific settings +/// to configure the subprocess when running +public struct PlatformOptions: Sendable { + /// Set user ID for the subprocess + public var userID: uid_t? = nil + /// Set the real and effective group ID and the saved + /// set-group-ID of the subprocess, equivalent to calling + /// `setgid()` on the child process. + /// Group ID is used to control permissions, particularly + /// for file access. + public var groupID: gid_t? = nil + /// Set list of supplementary group IDs for the subprocess + public var supplementaryGroups: [gid_t]? = nil + /// Set the process group for the subprocess, equivalent to + /// calling `setpgid()` on the child process. + /// Process group ID is used to group related processes for + /// controlling signals. + public var processGroupID: pid_t? = nil + /// Creates a session and sets the process group ID + /// i.e. Detach from the terminal. + public var createSession: Bool = false + /// An ordered list of steps in order to tear down the child + /// process in case the parent task is cancelled before + /// the child process terminates. + /// Always ends in sending a `.kill` signal at the end. + public var teardownSequence: [TeardownStep] = [] + + public init() {} +} + +extension PlatformOptions: CustomStringConvertible, CustomDebugStringConvertible { + internal func description(withIndent indent: Int) -> String { + let indent = String(repeating: " ", count: indent * 4) + return """ + PlatformOptions( + \(indent) userID: \(String(describing: userID)), + \(indent) groupID: \(String(describing: groupID)), + \(indent) supplementaryGroups: \(String(describing: supplementaryGroups)), + \(indent) processGroupID: \(String(describing: processGroupID)), + \(indent) createSession: \(createSession) + \(indent)) + """ + } + + public var description: String { + return self.description(withIndent: 0) + } + + public var debugDescription: String { + return self.description(withIndent: 0) + } +} + +// Special keys used in Error's user dictionary +extension String { + static let debugDescriptionErrorKey = "DebugDescription" +} + +internal func pthread_create(_ body: @Sendable @escaping () -> ()) throws(SubprocessError.UnderlyingError) -> pthread_t { + final class Context { + let body: @Sendable () -> () + init(body: @Sendable @escaping () -> Void) { + self.body = body + } + } + #if canImport(Glibc) || canImport(Musl) + func proc(_ context: UnsafeMutableRawPointer?) -> UnsafeMutableRawPointer? { + (Unmanaged.fromOpaque(context!).takeRetainedValue() as! Context).body() + return nil + } + #elseif canImport(Bionic) + func proc(_ context: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { + (Unmanaged.fromOpaque(context).takeRetainedValue() as! Context).body() + return context + } + #endif + #if (os(Linux) && canImport(Glibc)) || canImport(Bionic) + var thread = pthread_t() + #else + var thread: pthread_t? + #endif + let rc = pthread_create( + &thread, + nil, + proc, + Unmanaged.passRetained(Context(body: body)).toOpaque() + ) + if rc != 0 { + throw SubprocessError.UnderlyingError(rawValue: rc) + } + #if (os(Linux) && canImport(Glibc)) || canImport(Bionic) + return thread + #else + return thread! + #endif +} + +#endif // !canImport(Darwin) + +extension ProcessIdentifier { + /// Reaps the zombie for the exited process. This function may block. + @available(*, noasync) + internal func blockingReap() throws(SubprocessError.UnderlyingError) -> TerminationStatus { + try _blockingReap(pid: value) + } + + /// Reaps the zombie for the exited process, or returns `nil` if the process is still running. This function will not block. + internal func reap() throws(SubprocessError.UnderlyingError) -> TerminationStatus? { + try _reap(pid: value) + } +} + +@available(*, noasync) +internal func _blockingReap(pid: pid_t) throws(SubprocessError.UnderlyingError) -> TerminationStatus { + return try TerminationStatus(_waitid(idtype: P_PID, id: id_t(pid), flags: WEXITED)) +} + +internal func _reap(pid: pid_t) throws(SubprocessError.UnderlyingError) -> TerminationStatus? { + let siginfo = try _waitid(idtype: P_PID, id: id_t(pid), flags: WEXITED | WNOHANG) + // If si_pid and si_signo are both 0, the child is still running since we used WNOHANG + if siginfo.si_pid == 0 && siginfo.si_signo == 0 { + return nil + } + return TerminationStatus(siginfo) +} + +internal func _waitid(idtype: idtype_t, id: id_t, flags: Int32) throws(SubprocessError.UnderlyingError) -> siginfo_t { + while true { + var siginfo = siginfo_t() + if waitid(idtype, id, &siginfo, flags) != -1 { + return siginfo + } else if errno != EINTR { + throw SubprocessError.UnderlyingError(rawValue: errno) + } + } +} + +internal extension TerminationStatus { + init(_ siginfo: siginfo_t) { + switch siginfo.si_code { + case .init(CLD_EXITED): + self = .exited(siginfo.si_status) + case .init(CLD_KILLED), .init(CLD_DUMPED): + self = .unhandledException(siginfo.si_status) + default: + fatalError("Unexpected exit status: \(siginfo.si_code)") + } + } +} + +#if os(OpenBSD) || os(Linux) || os(Android) +internal extension siginfo_t { + var si_status: Int32 { + #if os(OpenBSD) + return _data._proc._pdata._cld._status + #elseif canImport(Glibc) + return _sifields._sigchld.si_status + #elseif canImport(Musl) + return __si_fields.__si_common.__second.__sigchld.si_status + #elseif canImport(Bionic) + return _sifields._sigchld._status + #endif + } + + var si_pid: pid_t { + #if os(OpenBSD) + return _data._proc._pid + #elseif canImport(Glibc) + return _sifields._sigchld.si_pid + #elseif canImport(Musl) + return __si_fields.__si_common.__first.__piduid.si_pid + #elseif canImport(Bionic) + return _sifields._kill._pid + #endif + } +} +#endif + #endif // canImport(Darwin) || canImport(Glibc) || canImport(Android) || canImport(Musl) diff --git a/Sources/Subprocess/SubprocessFoundation/Input+Foundation.swift b/Sources/Subprocess/SubprocessFoundation/Input+Foundation.swift index c82d2d38..f124a7e8 100644 --- a/Sources/Subprocess/SubprocessFoundation/Input+Foundation.swift +++ b/Sources/Subprocess/SubprocessFoundation/Input+Foundation.swift @@ -129,7 +129,7 @@ extension StandardInputWriter { } -#if canImport(Darwin) +#if SUBPROCESS_ASYNCIO_DISPATCH extension AsyncIO { internal func write( _ data: Data, @@ -168,6 +168,6 @@ extension AsyncIO { return try await self._write(data, to: diskIO) } } -#endif // canImport(Darwin) +#endif // SUBPROCESS_ASYNCIO_DISPATCH #endif // SubprocessFoundation diff --git a/Sources/_SubprocessCShims/include/process_shims.h b/Sources/_SubprocessCShims/include/process_shims.h index 85c97bb3..584d5418 100644 --- a/Sources/_SubprocessCShims/include/process_shims.h +++ b/Sources/_SubprocessCShims/include/process_shims.h @@ -23,11 +23,18 @@ #if TARGET_OS_LINUX #include -#include -#include #include #endif // TARGET_OS_LINUX +#if TARGET_OS_FREEBSD +#include +#endif + +#if TARGET_OS_LINUX || TARGET_OS_FREEBSD +#include +#include +#endif // TARGET_OS_LINUX || TARGET_OS_FREEBSD + #if __has_include() vm_size_t _subprocess_vm_size(void); #endif @@ -72,7 +79,9 @@ void _subprocess_lock_environ(void); void _subprocess_unlock_environ(void); char * _Nullable * _Nullable _subprocess_get_environ(void); -#if TARGET_OS_LINUX +int _subprocess_pdkill(int pidfd, int signal); + +#if TARGET_OS_UNIX && !TARGET_OS_FREEBSD int _shims_snprintf( char * _Nonnull str, int len, @@ -80,9 +89,10 @@ int _shims_snprintf( char * _Nonnull str1, char * _Nonnull str2 ); +#endif +#if TARGET_OS_LINUX int _pidfd_open(pid_t pid); -int _pidfd_send_signal(int pidfd, int signal); // P_PIDFD is only defined on Linux Kernel 5.4 and above // Define our value if it's not available diff --git a/Sources/_SubprocessCShims/include/target_conditionals.h b/Sources/_SubprocessCShims/include/target_conditionals.h index aaa4a600..ba4e5c6e 100644 --- a/Sources/_SubprocessCShims/include/target_conditionals.h +++ b/Sources/_SubprocessCShims/include/target_conditionals.h @@ -28,10 +28,16 @@ #define TARGET_OS_LINUX 0 #endif +#if defined(__FreeBSD__) +#define TARGET_OS_FREEBSD 1 +#else +#define TARGET_OS_FREEBSD 0 +#endif + #if defined(__unix__) -#define TARGET_OS_BSD 1 +#define TARGET_OS_UNIX 1 #else -#define TARGET_OS_BSD 0 +#define TARGET_OS_UNIX 0 #endif #if defined(_WIN32) diff --git a/Sources/_SubprocessCShims/process_shims.c b/Sources/_SubprocessCShims/process_shims.c index 5fa1d2cc..d77d3723 100644 --- a/Sources/_SubprocessCShims/process_shims.c +++ b/Sources/_SubprocessCShims/process_shims.c @@ -42,6 +42,10 @@ #include #endif +#include +#include +#include + #if __has_include() #include #elif defined(_WIN32) @@ -262,12 +266,10 @@ int _subprocess_spawn( #endif // TARGET_OS_MAC -// MARK: - Linux (fork/exec + posix_spawn fallback) -#if TARGET_OS_LINUX || TARGET_OS_BSD -#ifndef __GLIBC_PREREQ -#define __GLIBC_PREREQ(maj, min) 0 -#endif +// MARK: - Linux/BSD (fork/exec + posix_spawn fallback) +#if TARGET_OS_UNIX && !TARGET_OS_MAC +#if TARGET_OS_LINUX #ifndef SYS_pidfd_open #define SYS_pidfd_open 434 #endif @@ -337,6 +339,29 @@ struct linux_dirent64 { static int _getdents64(int fd, struct linux_dirent64 *dirp, size_t nbytes) { return syscall(SYS_getdents64, fd, dirp, nbytes); } +#endif + +static pid_t _subprocess_pdfork(int *fdp) { +#if TARGET_OS_LINUX + return _clone3(fdp); // CLONE_PIDFD always sets close-on-exec on the fd +#elif TARGET_OS_FREEBSD + return pdfork(fdp, PD_CLOEXEC); +#else + errno = ENOSYS; + return -1; +#endif +} + +int _subprocess_pdkill(int pidfd, int signal) { +#if TARGET_OS_LINUX + return _pidfd_send_signal(pidfd, signal); +#elif TARGET_OS_FREEBSD + return pdkill(pidfd, signal); +#else + errno = ENOSYS; + return -1; +#endif +} static pthread_mutex_t _subprocess_fork_lock = PTHREAD_MUTEX_INITIALIZER; @@ -379,6 +404,7 @@ static int _subprocess_make_critical_mask(sigset_t *old_mask) { # define _SUBPROCESS_SIG_MAX (sizeof(sigset_t) * CHAR_BIT + 1) #endif +#if !TARGET_OS_FREEBSD int _shims_snprintf( char * _Nonnull str, int len, @@ -388,6 +414,7 @@ int _shims_snprintf( ) { return snprintf(str, len, format, str1, str2); } +#endif static int _positive_int_parse(const char *str) { char *end; @@ -403,6 +430,7 @@ static int _positive_int_parse(const char *str) { return (int)value; } +#if defined(__linux__) // Linux-specific version that uses syscalls directly and doesn't allocate heap memory. // Safe to use after vfork() and before execve() static int _highest_possibly_open_fd_dir_linux(const char *fd_dir) { @@ -451,6 +479,7 @@ static int _highest_possibly_open_fd_dir_linux(const char *fd_dir) { close(dir_fd); return highest_fd_so_far; } +#endif // This function is only used on systems with Linux kernel 5.9 or lower. // On newer systems, `close_range` is used instead. @@ -533,11 +562,11 @@ int _subprocess_fork_exec( // Finally, fork / clone int _pidfd = -1; - // First attempt to use clone3, only fall back to fork if clone3 is not available - pid_t childPid = _clone3(&_pidfd); + // First attempt to create a process file descriptor on supported platforms, only fall back to fork if those are not available + pid_t childPid = _subprocess_pdfork(&_pidfd); if (childPid < 0) { if (errno == ENOSYS) { - // clone3 is not implemented. Use fork instead + // process file descriptor is not implemented. Use fork instead #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated" childPid = fork(); @@ -683,11 +712,13 @@ int _subprocess_fork_exec( waitid(P_PID, childPid, &info, WEXITED); \ return capturedError +#if TARGET_OS_LINUX // On Linux 5.3 and lower, we have to fetch pidfd separately // Newer Linux supports clone3 which returns pidfd directly if (_pidfd < 0) { _pidfd = _pidfd_open(childPid); } +#endif // Parent process close(pipefd[1]); // Close unused write end @@ -738,7 +769,7 @@ int _subprocess_fork_exec( } } -#endif // TARGET_OS_LINUX +#endif // TARGET_OS_UNIX && !TARGET_OS_MAC #pragma mark - Environment Locking diff --git a/Tests/SubprocessTests/AsyncIOTests.swift b/Tests/SubprocessTests/AsyncIOTests.swift index 7a3a07ad..7ccbb846 100644 --- a/Tests/SubprocessTests/AsyncIOTests.swift +++ b/Tests/SubprocessTests/AsyncIOTests.swift @@ -138,8 +138,8 @@ extension SubprocessAsyncIOTests { extension SubprocessAsyncIOTests { @Test func testWriteToClosedPipe() async throws { var pipe = try CreatedPipe(closeWhenDone: true, purpose: .input) - var writeChannel = pipe.writeFileDescriptor()!.createIOChannel() - var readChannel = pipe.readFileDescriptor()!.createIOChannel() + var writeChannel = try _require(pipe.writeFileDescriptor()).createIOChannel() + var readChannel = try _require(pipe.readFileDescriptor()).createIOChannel() defer { try? readChannel.safelyClose() } @@ -176,8 +176,8 @@ extension SubprocessAsyncIOTests { @Test func testReadFromClosedPipe() async throws { var pipe = try CreatedPipe(closeWhenDone: true, purpose: .input) - var writeChannel = pipe.writeFileDescriptor()!.createIOChannel() - var readChannel = pipe.readFileDescriptor()!.createIOChannel() + var writeChannel = try _require(pipe.writeFileDescriptor()).createIOChannel() + var readChannel = try _require(pipe.readFileDescriptor()).createIOChannel() defer { try? writeChannel.safelyClose() } @@ -257,12 +257,12 @@ extension SubprocessAsyncIOTests { group.addTask { var readIOContainer: IOChannel? = readBox.take() - let readTestBed = TestBed(ioChannel: readIOContainer.take()!) + let readTestBed = try TestBed(ioChannel: _require(readIOContainer.take())) try await reader(readIO, readTestBed) } group.addTask { var writeIOContainer: IOChannel? = writeBox.take() - let writeTestBed = TestBed(ioChannel: writeIOContainer.take()!) + let writeTestBed = try TestBed(ioChannel: _require(writeIOContainer.take())) try await writer(writeIO, writeTestBed) } @@ -276,10 +276,10 @@ extension SubprocessAsyncIOTests { extension SubprocessAsyncIOTests.TestBed { consuming func finish() async throws { -#if canImport(WinSDK) - try _safelyClose(.handle(self.ioChannel.channel)) -#elseif canImport(Darwin) +#if SUBPROCESS_ASYNCIO_DISPATCH try _safelyClose(.dispatchIO(self.ioChannel.channel)) +#elseif canImport(WinSDK) + try _safelyClose(.handle(self.ioChannel.channel)) #else try _safelyClose(.fileDescriptor(self.ioChannel.channel)) #endif diff --git a/Tests/SubprocessTests/IntegrationTests.swift b/Tests/SubprocessTests/IntegrationTests.swift index d60be0e9..04f49332 100644 --- a/Tests/SubprocessTests/IntegrationTests.swift +++ b/Tests/SubprocessTests/IntegrationTests.swift @@ -429,7 +429,7 @@ extension SubprocessIntegrationTests { #expect(result.terminationStatus.isSuccess) // There shouldn't be any other environment variables besides // `PATH` that we set - let resultPath = result.standardOutput! + let resultPath = try #require(result.standardOutput) .trimmingNewLineAndQuotes() #if canImport(Darwin) // On Darwin, /var is linked to /private/var; /tmp is linked to /private/tmp @@ -1686,7 +1686,7 @@ extension SubprocessIntegrationTests { } // Generate at least 2 long lines that is longer than buffer size - func generateTestCases(count: Int) -> [TestCase] { + func generateTestCases(count: Int) throws -> [TestCase] { var targetSizes: [TestCaseSize] = TestCaseSize.allCases.flatMap { Array(repeating: $0, count: count / 3) } @@ -1701,7 +1701,7 @@ extension SubprocessIntegrationTests { for size in targetSizes { let components = generateString(size: size) // Choose a random new line - let newLine = newLineCharacters.randomElement()! + let newLine = try #require(newLineCharacters.randomElement()) let string = String(decoding: components + newLine, as: UTF8.self) testCases.append(( value: string, @@ -1717,7 +1717,7 @@ extension SubprocessIntegrationTests { FileManager.default.createFile(atPath: url.path(), contents: nil, attributes: nil) let fileHadle = try FileHandle(forWritingTo: url) for testCase in testCases { - fileHadle.write(testCase.value.data(using: .utf8)!) + fileHadle.write(Data(testCase.value.utf8)) } try fileHadle.close() #else @@ -1734,7 +1734,7 @@ extension SubprocessIntegrationTests { if FileManager.default.fileExists(atPath: testFilePath.path()) { try FileManager.default.removeItem(at: testFilePath) } - let testCases = generateTestCases(count: testCaseCount) + let testCases = try generateTestCases(count: testCaseCount) try writeTestCasesToFile(testCases, at: testFilePath) #if os(Windows) diff --git a/Tests/SubprocessTests/LinterTests.swift b/Tests/SubprocessTests/LinterTests.swift index d41be90d..8be35f8e 100644 --- a/Tests/SubprocessTests/LinterTests.swift +++ b/Tests/SubprocessTests/LinterTests.swift @@ -54,8 +54,8 @@ struct SubprocessLintingTest { else { return } - let sourcePath = String( - maybePath.prefix(upTo: maybePath.range(of: "/.build")!.lowerBound) + let sourcePath = try String( + maybePath.prefix(upTo: #require(maybePath.range(of: "/.build")).lowerBound) ) print("Linting \(sourcePath)") #if os(macOS) diff --git a/Tests/SubprocessTests/LinuxTests.swift b/Tests/SubprocessTests/LinuxTests.swift index a965cb0d..98f975bf 100644 --- a/Tests/SubprocessTests/LinuxTests.swift +++ b/Tests/SubprocessTests/LinuxTests.swift @@ -41,6 +41,10 @@ struct SubprocessLinuxTests { waitThread = try pthread_create { var suspendedStatus: Int32 = 0 let rc = waitpid(pid, &suspendedStatus, targetSignal) + if rc == -1 { + continuation.resume(throwing: SubprocessError.UnderlyingError(rawValue: errno)) + return + } handler(suspendedStatus) continuation.resume() } diff --git a/Tests/SubprocessTests/PlatformConformance.swift b/Tests/SubprocessTests/PlatformConformance.swift index 3c619d2e..04bd2705 100644 --- a/Tests/SubprocessTests/PlatformConformance.swift +++ b/Tests/SubprocessTests/PlatformConformance.swift @@ -36,7 +36,7 @@ protocol ProcessIdentifierProtocol: Sendable, Hashable, CustomStringConvertible, var value: pid_t { get } #endif - #if os(Linux) || os(Android) + #if os(Linux) || os(Android) || os(FreeBSD) var processDescriptor: PlatformFileDescriptor { get } #endif diff --git a/Tests/SubprocessTests/ProcessMonitoringTests.swift b/Tests/SubprocessTests/ProcessMonitoringTests.swift index 742e263c..1fab4ebe 100644 --- a/Tests/SubprocessTests/ProcessMonitoringTests.swift +++ b/Tests/SubprocessTests/ProcessMonitoringTests.swift @@ -19,8 +19,8 @@ import Darwin #elseif canImport(Glibc) import Glibc -#elseif canImport(Bionic) -import Bionic +#elseif canImport(Android) +import Android #elseif canImport(Musl) import Musl #elseif canImport(WinSDK) @@ -76,7 +76,7 @@ struct SubprocessProcessMonitoringTests { private func devNullInputPipe() throws -> CreatedPipe { #if os(Windows) let devnullFd: FileDescriptor = try .openDevNull(withAccessMode: .writeOnly) - let devnull = HANDLE(bitPattern: _get_osfhandle(devnullFd.rawValue))! + let devnull = try #require(HANDLE(bitPattern: _get_osfhandle(devnullFd.rawValue))) #else let devnull: FileDescriptor = try .openDevNull(withAccessMode: .readOnly) #endif @@ -89,7 +89,7 @@ struct SubprocessProcessMonitoringTests { private func devNullOutputPipe() throws -> CreatedPipe { #if os(Windows) let devnullFd: FileDescriptor = try .openDevNull(withAccessMode: .writeOnly) - let devnull = HANDLE(bitPattern: _get_osfhandle(devnullFd.rawValue))! + let devnull = try #require(HANDLE(bitPattern: _get_osfhandle(devnullFd.rawValue))) #else let devnull: FileDescriptor = try .openDevNull(withAccessMode: .writeOnly) #endif @@ -206,7 +206,7 @@ extension SubprocessProcessMonitoringTests { let processIdentifier = ProcessIdentifier( value: .max, processDescriptor: INVALID_HANDLE_VALUE, threadHandle: INVALID_HANDLE_VALUE ) - #elseif os(Linux) || os (FreeBSD) + #elseif os(Linux) || os(Android) || os(FreeBSD) let expectedError = SubprocessError( code: .init(.failedToMonitorProcess), underlyingError: .init(rawValue: ECHILD) diff --git a/Tests/SubprocessTests/TestSupport.swift b/Tests/SubprocessTests/TestSupport.swift index c13dc1d7..b7351202 100644 --- a/Tests/SubprocessTests/TestSupport.swift +++ b/Tests/SubprocessTests/TestSupport.swift @@ -20,6 +20,14 @@ import FoundationEssentials import Testing import Subprocess +// Workaround: https://github.com/swiftlang/swift-testing/issues/543 +internal func _require(_ value: consuming T?) throws -> T { + guard let value else { + throw SubprocessError.UnderlyingError(rawValue: .max) + } + return value +} + internal func randomString(length: Int, lettersOnly: Bool = false) -> String { let letters: String if lettersOnly { diff --git a/Tests/SubprocessTests/UnixTests.swift b/Tests/SubprocessTests/UnixTests.swift index ccb8013b..20ed2459 100644 --- a/Tests/SubprocessTests/UnixTests.swift +++ b/Tests/SubprocessTests/UnixTests.swift @@ -106,7 +106,7 @@ extension SubprocessUnixTests { let ids = try #require( idResult.standardOutput ).split(separator: ",") - .map { gid_t($0.trimmingCharacters(in: .whitespacesAndNewlines))! } + .map { try #require(gid_t($0.trimmingCharacters(in: .whitespacesAndNewlines))) } #expect(Set(ids) == expectedGroups) } @@ -253,7 +253,7 @@ extension SubprocessUnixTests { .send(signal: .interrupt, allowedDurationToNextStep: .milliseconds(100)) ] let result = try await Subprocess.run( - .path("/bin/bash"), + .name("bash"), arguments: [ "-c", """ @@ -350,6 +350,9 @@ extension SubprocessUnixTests { done """ var arguments = ["-c", shellScript, "--"] + #if os(FreeBSD) + arguments.append("") // FreeBSD /bin/sh interprets the first argument as the script name + #endif arguments.append(contentsOf: openedFileDescriptors.map { "\($0)" }) let result = try await Subprocess.run( @@ -385,7 +388,7 @@ extension SubprocessUnixTests { let result = try await pipe.writeEnd.closeAfter { // Spawn bash and then attempt to write to the write end try await Subprocess.run( - .path("/bin/bash"), + .name("bash"), arguments: [ "-c", """