From 4fac487c199f85f918e617a60f4ff0fd461fd249 Mon Sep 17 00:00:00 2001 From: Siddhant Khare Date: Tue, 8 Jul 2025 02:15:03 +0530 Subject: [PATCH 1/3] Fix image push tests permissions in CI --- Sources/ContainerizationOCI/Descriptor.swift | 2 +- .../RegistryClientTests.swift | 334 +++++++++++++----- .../ImageTests/ImageStoreTests.swift | 169 ++++++++- 3 files changed, 402 insertions(+), 103 deletions(-) diff --git a/Sources/ContainerizationOCI/Descriptor.swift b/Sources/ContainerizationOCI/Descriptor.swift index 648dc12c..59584d64 100644 --- a/Sources/ContainerizationOCI/Descriptor.swift +++ b/Sources/ContainerizationOCI/Descriptor.swift @@ -21,7 +21,7 @@ import Foundation /// Descriptor describes the disposition of targeted content. /// This structure provides `application/vnd.oci.descriptor.v1+json` mediatype /// when marshalled to JSON. -public struct Descriptor: Codable, Sendable, Equatable { +public struct Descriptor: Codable, Sendable, Equatable, Hashable { /// mediaType is the media type of the object this schema refers to. public let mediaType: String diff --git a/Tests/ContainerizationOCITests/RegistryClientTests.swift b/Tests/ContainerizationOCITests/RegistryClientTests.swift index c0567157..b5ae5bc0 100644 --- a/Tests/ContainerizationOCITests/RegistryClientTests.swift +++ b/Tests/ContainerizationOCITests/RegistryClientTests.swift @@ -21,6 +21,7 @@ import ContainerizationIO import Crypto import Foundation import NIO +import NIOCore import Synchronization import Testing @@ -166,113 +167,125 @@ struct OCIClientTests: ~Copyable { #expect(done) } - @Test(.disabled("External users cannot push images, disable while we find a better solution")) - func pushIndex() async throws { - let client = RegistryClient(host: "ghcr.io", authentication: Self.authentication) - let indexDescriptor = try await client.resolve(name: "apple/containerization/emptyimage", tag: "0.0.1") - let index: Index = try await client.fetch(name: "apple/containerization/emptyimage", descriptor: indexDescriptor) - - let platform = Platform(arch: "amd64", os: "linux") - - var manifestDescriptor: Descriptor? - for m in index.manifests where m.platform == platform { - manifestDescriptor = m - break - } - - #expect(manifestDescriptor != nil) - - let manifest: Manifest = try await client.fetch(name: "apple/containerization/emptyimage", descriptor: manifestDescriptor!) - let imgConfig: Image = try await client.fetch(name: "apple/containerization/emptyimage", descriptor: manifest.config) - - let layer = try #require(manifest.layers.first) - let blobPath = contentPath.appendingPathComponent(layer.digest) - let outputStream = OutputStream(toFileAtPath: blobPath.path, append: false) - #expect(outputStream != nil) + @Test func pushIndexWithMock() async throws { + // Create a mock client for testing push operations + let mockClient = MockRegistryClient() + + // Create test data for an index and its components + let testLayerData = "test layer content".data(using: .utf8)! + let layerDigest = SHA256.hash(data: testLayerData) + let layerDescriptor = Descriptor( + mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", + digest: "sha256:\(layerDigest.hexString)", + size: Int64(testLayerData.count) + ) - try await outputStream!.withThrowingOpeningStream { - try await client.fetchBlob(name: "apple/containerization/emptyimage", descriptor: layer) { (expected, body) in - var received: Int64 = 0 - for try await buffer in body { - received += Int64(buffer.readableBytes) + // Create test image config + let imageConfig = Image( + architecture: "amd64", + os: "linux", + config: ImageConfig(labels: ["test": "value"]), + rootfs: Rootfs(type: "layers", diffIDs: ["sha256:\(layerDigest.hexString)"]) + ) + let configData = try JSONEncoder().encode(imageConfig) + let configDigest = SHA256.hash(data: configData) + let configDescriptor = Descriptor( + mediaType: "application/vnd.docker.container.image.v1+json", + digest: "sha256:\(configDigest.hexString)", + size: Int64(configData.count) + ) - buffer.withUnsafeReadableBytes { pointer in - let unsafeBufferPointer = pointer.bindMemory(to: UInt8.self) - if let addr = unsafeBufferPointer.baseAddress { - outputStream!.write(addr, maxLength: buffer.readableBytes) - } - } - } + // Create test manifest + let manifest = Manifest( + schemaVersion: 2, + mediaType: "application/vnd.docker.distribution.manifest.v2+json", + config: configDescriptor, + layers: [layerDescriptor] + ) + let manifestData = try JSONEncoder().encode(manifest) + let manifestDigest = SHA256.hash(data: manifestData) + let manifestDescriptor = Descriptor( + mediaType: "application/vnd.docker.distribution.manifest.v2+json", + digest: "sha256:\(manifestDigest.hexString)", + size: Int64(manifestData.count), + platform: Platform(arch: "amd64", os: "linux") + ) - #expect(received == expected) - } - } + // Create test index + let index = Index( + schemaVersion: 2, + mediaType: "application/vnd.docker.distribution.manifest.list.v2+json", + manifests: [manifestDescriptor] + ) - let name = "apple/test-images/image-push" + let name = "test/image" let ref = "latest" - // Push the layer first. - do { - let content = try LocalContent(path: blobPath) - let generator = { - let stream = try ReadStream(url: content.path) - try stream.reset() - return stream.stream - } - try await client.push(name: name, ref: ref, descriptor: layer, streamGenerator: generator, progress: nil) - } catch let err as ContainerizationError { - guard err.code == .exists else { - throw err - } - } + // Test pushing individual components using the mock client - // Push the image configuration. - var imgConfigDesc: Descriptor? - do { - imgConfigDesc = try await self.pushDescriptor( - client: client, - name: name, - ref: ref, - content: imgConfig, - baseDescriptor: manifest.config - ) - } catch let err as ContainerizationError { - guard err.code != .exists else { - return - } - throw err - } + // Push layer + let layerStream = TestByteBufferSequence(data: testLayerData) + try await mockClient.push( + name: name, + ref: ref, + descriptor: layerDescriptor, + streamGenerator: { layerStream }, + progress: nil as ProgressHandler? + ) - // Push the image manifest. - let newManifest = Manifest( - schemaVersion: manifest.schemaVersion, - mediaType: manifest.mediaType!, - config: imgConfigDesc!, - layers: manifest.layers, - annotations: manifest.annotations + // Push config + let configStream = TestByteBufferSequence(data: configData) + try await mockClient.push( + name: name, + ref: ref, + descriptor: configDescriptor, + streamGenerator: { configStream }, + progress: nil as ProgressHandler? ) - let manifestDesc = try await self.pushDescriptor( - client: client, + + // Push manifest + let manifestStream = TestByteBufferSequence(data: manifestData) + try await mockClient.push( name: name, ref: ref, - content: newManifest, - baseDescriptor: manifestDescriptor! + descriptor: manifestDescriptor, + streamGenerator: { manifestStream }, + progress: nil as ProgressHandler? ) - // Push the index. - let newIndex = Index( - schemaVersion: index.schemaVersion, - mediaType: index.mediaType, - manifests: [manifestDesc], - annotations: index.annotations + // Push index + let indexData = try JSONEncoder().encode(index) + let indexDigest = SHA256.hash(data: indexData) + let indexDescriptor = Descriptor( + mediaType: "application/vnd.docker.distribution.manifest.list.v2+json", + digest: "sha256:\(indexDigest.hexString)", + size: Int64(indexData.count) ) - try await self.pushDescriptor( - client: client, + + let indexStream = TestByteBufferSequence(data: indexData) + try await mockClient.push( name: name, ref: ref, - content: newIndex, - baseDescriptor: indexDescriptor + descriptor: indexDescriptor, + streamGenerator: { indexStream }, + progress: nil as ProgressHandler? ) + + // Verify all push operations were recorded + #expect(mockClient.pushCalls.count == 4) + + // Verify content integrity + let storedLayerData = mockClient.getPushedContent(name: name, descriptor: layerDescriptor) + #expect(storedLayerData == testLayerData) + + let storedConfigData = mockClient.getPushedContent(name: name, descriptor: configDescriptor) + #expect(storedConfigData == configData) + + let storedManifestData = mockClient.getPushedContent(name: name, descriptor: manifestDescriptor) + #expect(storedManifestData == manifestData) + + let storedIndexData = mockClient.getPushedContent(name: name, descriptor: indexDescriptor) + #expect(storedIndexData == indexData) } @Test func resolveWithRetry() async throws { @@ -343,7 +356,7 @@ struct OCIClientTests: ~Copyable { ref: ref, descriptor: descriptor, streamGenerator: generator, - progress: nil + progress: nil as ProgressHandler? ) return descriptor } @@ -363,4 +376,143 @@ extension SHA256.Digest { let parts = self.description.split(separator: ": ") return "sha256:\(parts[1])" } + + var hexString: String { + self.compactMap { String(format: "%02x", $0) }.joined() + } +} + +// Helper to create ByteBuffer sequences for testing +struct TestByteBufferSequence: Sendable, AsyncSequence { + typealias Element = ByteBuffer + + private let data: Data + + init(data: Data) { + self.data = data + } + + func makeAsyncIterator() -> AsyncIterator { + AsyncIterator(data: data) + } + + struct AsyncIterator: AsyncIteratorProtocol { + private let data: Data + private var sent = false + + init(data: Data) { + self.data = data + } + + mutating func next() async throws -> ByteBuffer? { + guard !sent else { return nil } + sent = true + + var buffer = ByteBufferAllocator().buffer(capacity: data.count) + buffer.writeBytes(data) + return buffer + } + } +} + +// Helper class to create a mock ContentClient for testing +final class MockRegistryClient: ContentClient, @unchecked Sendable { + private var pushedContent: [String: [Descriptor: Data]] = [:] + private var fetchableContent: [String: [Descriptor: Data]] = [:] + + // Track push operations for verification + var pushCalls: [(name: String, ref: String, descriptor: Descriptor)] = [] + + func addFetchableContent(name: String, descriptor: Descriptor, content: T) throws { + let data = try JSONEncoder().encode(content) + if fetchableContent[name] == nil { + fetchableContent[name] = [:] + } + fetchableContent[name]![descriptor] = data + } + + func addFetchableData(name: String, descriptor: Descriptor, data: Data) { + if fetchableContent[name] == nil { + fetchableContent[name] = [:] + } + fetchableContent[name]![descriptor] = data + } + + func getPushedContent(name: String, descriptor: Descriptor) -> Data? { + pushedContent[name]?[descriptor] + } + + // MARK: - ContentClient Implementation + + func fetch(name: String, descriptor: Descriptor) async throws -> T { + guard let imageContent = fetchableContent[name], + let data = imageContent[descriptor] + else { + throw ContainerizationError(.notFound, message: "Content not found for \(name) with descriptor \(descriptor.digest)") + } + + return try JSONDecoder().decode(T.self, from: data) + } + + func fetchBlob(name: String, descriptor: Descriptor, into file: URL, progress: ProgressHandler?) async throws -> (Int64, SHA256.Digest) { + guard let imageContent = fetchableContent[name], + let data = imageContent[descriptor] + else { + throw ContainerizationError(.notFound, message: "Blob not found for \(name) with descriptor \(descriptor.digest)") + } + + try data.write(to: file) + let digest = SHA256.hash(data: data) + return (Int64(data.count), digest) + } + + func fetchData(name: String, descriptor: Descriptor) async throws -> Data { + guard let imageContent = fetchableContent[name], + let data = imageContent[descriptor] + else { + throw ContainerizationError(.notFound, message: "Data not found for \(name) with descriptor \(descriptor.digest)") + } + + return data + } + + func push( + name: String, + ref: String, + descriptor: Descriptor, + streamGenerator: () throws -> T, + progress: ProgressHandler? + ) async throws where T.Element == ByteBuffer { + // Record the push call for verification + pushCalls.append((name: name, ref: ref, descriptor: descriptor)) + + // Simulate reading the stream and storing the data + let stream = try streamGenerator() + var data = Data() + + for try await buffer in stream { + data.append(contentsOf: buffer.readableBytesView) + } + + // Verify the pushed data matches the expected descriptor + let actualDigest = SHA256.hash(data: data) + guard descriptor.digest == "sha256:\(actualDigest.hexString)" else { + throw ContainerizationError(.invalidArgument, message: "Digest mismatch: expected \(descriptor.digest), got sha256:\(actualDigest.hexString)") + } + + guard data.count == descriptor.size else { + throw ContainerizationError(.invalidArgument, message: "Size mismatch: expected \(descriptor.size), got \(data.count)") + } + + // Store the pushed content + if pushedContent[name] == nil { + pushedContent[name] = [:] + } + pushedContent[name]![descriptor] = data + + // Simulate progress reporting + if let progress = progress { + await progress(Int64(data.count), Int64(data.count)) + } + } } diff --git a/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift b/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift index ced63ee3..9b3a66ac 100644 --- a/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift +++ b/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift @@ -17,13 +17,143 @@ // import ContainerizationArchive +import ContainerizationError import ContainerizationExtras import ContainerizationOCI +import Crypto import Foundation +import NIOCore import Testing @testable import Containerization +// Test-specific extension to expose ExportOperation for testing +extension ImageStore { + func testPush(reference: String, client: ContentClient, platform: Platform? = nil) async throws { + let matcher = createPlatformMatcher(for: platform) + let img = try await self.get(reference: reference) + let allowedMediaTypes = [MediaTypes.dockerManifestList, MediaTypes.index] + guard allowedMediaTypes.contains(img.mediaType) else { + throw ContainerizationError(.internalError, message: "Cannot push image \(reference) with Index media type \(img.mediaType)") + } + let ref = try Reference.parse(reference) + let name = ref.path + guard let tag = ref.tag ?? ref.digest else { + throw ContainerizationError(.invalidArgument, message: "Invalid tag/digest for image reference \(reference)") + } + let operation = ExportOperation(name: name, tag: tag, contentStore: self.contentStore, client: client, progress: nil) + try await operation.export(index: img.descriptor, platforms: matcher) + } +} + +// Helper class to create a mock ContentClient for testing +final class MockRegistryClient: ContentClient, @unchecked Sendable { + private var pushedContent: [String: [Descriptor: Data]] = [:] + private var fetchableContent: [String: [Descriptor: Data]] = [:] + + // Track push operations for verification + var pushCalls: [(name: String, ref: String, descriptor: Descriptor)] = [] + + func addFetchableContent(name: String, descriptor: Descriptor, content: T) throws { + let data = try JSONEncoder().encode(content) + if fetchableContent[name] == nil { + fetchableContent[name] = [:] + } + fetchableContent[name]![descriptor] = data + } + + func addFetchableData(name: String, descriptor: Descriptor, data: Data) { + if fetchableContent[name] == nil { + fetchableContent[name] = [:] + } + fetchableContent[name]![descriptor] = data + } + + func getPushedContent(name: String, descriptor: Descriptor) -> Data? { + pushedContent[name]?[descriptor] + } + + // MARK: - ContentClient Implementation + + func fetch(name: String, descriptor: Descriptor) async throws -> T { + guard let imageContent = fetchableContent[name], + let data = imageContent[descriptor] + else { + throw ContainerizationError(.notFound, message: "Content not found for \(name) with descriptor \(descriptor.digest)") + } + + return try JSONDecoder().decode(T.self, from: data) + } + + func fetchBlob(name: String, descriptor: Descriptor, into file: URL, progress: ProgressHandler?) async throws -> (Int64, SHA256.Digest) { + guard let imageContent = fetchableContent[name], + let data = imageContent[descriptor] + else { + throw ContainerizationError(.notFound, message: "Blob not found for \(name) with descriptor \(descriptor.digest)") + } + + try data.write(to: file) + let digest = SHA256.hash(data: data) + return (Int64(data.count), digest) + } + + func fetchData(name: String, descriptor: Descriptor) async throws -> Data { + guard let imageContent = fetchableContent[name], + let data = imageContent[descriptor] + else { + throw ContainerizationError(.notFound, message: "Data not found for \(name) with descriptor \(descriptor.digest)") + } + + return data + } + + func push( + name: String, + ref: String, + descriptor: Descriptor, + streamGenerator: () throws -> T, + progress: ProgressHandler? + ) async throws where T.Element == ByteBuffer { + // Record the push call for verification + pushCalls.append((name: name, ref: ref, descriptor: descriptor)) + + // Simulate reading the stream and storing the data + let stream = try streamGenerator() + var data = Data() + + for try await buffer in stream { + data.append(contentsOf: buffer.readableBytesView) + } + + // Verify the pushed data matches the expected descriptor + let actualDigest = SHA256.hash(data: data) + guard descriptor.digest == "sha256:\(actualDigest.hexString)" else { + throw ContainerizationError(.invalidArgument, message: "Digest mismatch: expected \(descriptor.digest), got sha256:\(actualDigest.hexString)") + } + + guard data.count == descriptor.size else { + throw ContainerizationError(.invalidArgument, message: "Size mismatch: expected \(descriptor.size), got \(data.count)") + } + + // Store the pushed content + if pushedContent[name] == nil { + pushedContent[name] = [:] + } + pushedContent[name]![descriptor] = data + + // Simulate progress reporting + if let progress = progress { + await progress([ProgressEvent(event: "add-size", value: Int64(data.count))]) + } + } +} + +extension SHA256.Digest { + var hexString: String { + self.compactMap { String(format: "%02x", $0) }.joined() + } +} + @Suite public class ImageStoreTests: ContainsAuth { let store: ImageStore @@ -73,18 +203,35 @@ public class ImageStoreTests: ContainsAuth { try await self.store.save(references: [imageReference, expectedLoadedImage], out: tempFile) } - @Test(.disabled("External users cannot push images, disable while we find a better solution")) - func testImageStorePush() async throws { - guard let authentication = Self.authentication else { - return + @Test func testImageStorePushWithMock() async throws { + // Load a test image first to have something to push + let tarPath = Foundation.Bundle.module.url(forResource: "scratch", withExtension: "tar")! + let reader = try ArchiveReader(format: .pax, filter: .none, file: tarPath) + let tempDir = FileManager.default.uniqueTemporaryDirectory() + defer { + try? FileManager.default.removeItem(at: tempDir) } - let imageReference = "ghcr.io/apple/containerization/dockermanifestimage:0.0.2" + try reader.extractContents(to: tempDir) + + let loadedImages = try await self.store.load(from: tempDir) + let testImage = loadedImages.first! + + // Create a mock client to simulate registry interactions + let mockClient = MockRegistryClient() + + // Tag the image with a test registry reference + let testReference = "test-registry.local/test-image:latest" + try await self.store.tag(existing: testImage.reference, new: testReference) + + // Test push with mock client (using extension method) + try await self.store.testPush(reference: testReference, client: mockClient) + + // Verify that push operations were called + #expect(!mockClient.pushCalls.isEmpty) - let remoteImageName = "ghcr.io/apple/test-images/image-push" - let epoch = Int(Date().timeIntervalSince1970.description) - let tag = epoch != nil ? String(epoch!) : "latest" - let upstreamTag = "\(remoteImageName):\(tag)" - let _ = try await self.store.tag(existing: imageReference, new: upstreamTag) - try await self.store.push(reference: upstreamTag, auth: authentication) + // Verify that the correct image name and tag were used + let pushCall = mockClient.pushCalls.first! + #expect(pushCall.name == "test-registry.local/test-image") + #expect(pushCall.ref == "latest") } } From 7e45ae0c215d7e203234a76ce5bd0929cda3902f Mon Sep 17 00:00:00 2001 From: Siddhant Khare Date: Wed, 9 Jul 2025 10:21:01 +0530 Subject: [PATCH 2/3] address review comments --- .../RegistryClientTests.swift | 327 +++++------------- .../ImageTests/ImageStoreTests.swift | 17 +- 2 files changed, 100 insertions(+), 244 deletions(-) diff --git a/Tests/ContainerizationOCITests/RegistryClientTests.swift b/Tests/ContainerizationOCITests/RegistryClientTests.swift index b5ae5bc0..56035cb0 100644 --- a/Tests/ContainerizationOCITests/RegistryClientTests.swift +++ b/Tests/ContainerizationOCITests/RegistryClientTests.swift @@ -167,125 +167,113 @@ struct OCIClientTests: ~Copyable { #expect(done) } - @Test func pushIndexWithMock() async throws { - // Create a mock client for testing push operations - let mockClient = MockRegistryClient() - - // Create test data for an index and its components - let testLayerData = "test layer content".data(using: .utf8)! - let layerDigest = SHA256.hash(data: testLayerData) - let layerDescriptor = Descriptor( - mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip", - digest: "sha256:\(layerDigest.hexString)", - size: Int64(testLayerData.count) - ) + @Test(.disabled("External users cannot push images, disable while we find a better solution")) + func pushIndex() async throws { + let client = RegistryClient(host: "ghcr.io", authentication: Self.authentication) + let indexDescriptor = try await client.resolve(name: "apple/containerization/emptyimage", tag: "0.0.1") + let index: Index = try await client.fetch(name: "apple/containerization/emptyimage", descriptor: indexDescriptor) - // Create test image config - let imageConfig = Image( - architecture: "amd64", - os: "linux", - config: ImageConfig(labels: ["test": "value"]), - rootfs: Rootfs(type: "layers", diffIDs: ["sha256:\(layerDigest.hexString)"]) - ) - let configData = try JSONEncoder().encode(imageConfig) - let configDigest = SHA256.hash(data: configData) - let configDescriptor = Descriptor( - mediaType: "application/vnd.docker.container.image.v1+json", - digest: "sha256:\(configDigest.hexString)", - size: Int64(configData.count) - ) + let platform = Platform(arch: "amd64", os: "linux") - // Create test manifest - let manifest = Manifest( - schemaVersion: 2, - mediaType: "application/vnd.docker.distribution.manifest.v2+json", - config: configDescriptor, - layers: [layerDescriptor] - ) - let manifestData = try JSONEncoder().encode(manifest) - let manifestDigest = SHA256.hash(data: manifestData) - let manifestDescriptor = Descriptor( - mediaType: "application/vnd.docker.distribution.manifest.v2+json", - digest: "sha256:\(manifestDigest.hexString)", - size: Int64(manifestData.count), - platform: Platform(arch: "amd64", os: "linux") - ) + var manifestDescriptor: Descriptor? + for m in index.manifests where m.platform == platform { + manifestDescriptor = m + break + } - // Create test index - let index = Index( - schemaVersion: 2, - mediaType: "application/vnd.docker.distribution.manifest.list.v2+json", - manifests: [manifestDescriptor] - ) + #expect(manifestDescriptor != nil) + + let manifest: Manifest = try await client.fetch(name: "apple/containerization/emptyimage", descriptor: manifestDescriptor!) + let imgConfig: Image = try await client.fetch(name: "apple/containerization/emptyimage", descriptor: manifest.config) + + let layer = try #require(manifest.layers.first) + let blobPath = contentPath.appendingPathComponent(layer.digest) + let outputStream = OutputStream(toFileAtPath: blobPath.path, append: false) + #expect(outputStream != nil) + + try await outputStream!.withThrowingOpeningStream { + try await client.fetchBlob(name: "apple/containerization/emptyimage", descriptor: layer) { (expected, body) in + var received: Int64 = 0 + for try await buffer in body { + received += Int64(buffer.readableBytes) + + buffer.withUnsafeReadableBytes { pointer in + let unsafeBufferPointer = pointer.bindMemory(to: UInt8.self) + if let addr = unsafeBufferPointer.baseAddress { + outputStream!.write(addr, maxLength: buffer.readableBytes) + } + } + } + + #expect(received == expected) + } + } - let name = "test/image" + let name = "apple/test-images/image-push" let ref = "latest" - // Test pushing individual components using the mock client + // Push the layer first. + do { + let content = try LocalContent(path: blobPath) + let generator = { + let stream = try ReadStream(url: content.path) + try stream.reset() + return stream.stream + } + try await client.push(name: name, ref: ref, descriptor: layer, streamGenerator: generator, progress: nil) + } catch let err as ContainerizationError { + guard err.code == .exists else { + throw err + } + } - // Push layer - let layerStream = TestByteBufferSequence(data: testLayerData) - try await mockClient.push( - name: name, - ref: ref, - descriptor: layerDescriptor, - streamGenerator: { layerStream }, - progress: nil as ProgressHandler? - ) + // Push the image configuration. + var imgConfigDesc: Descriptor? + do { + imgConfigDesc = try await self.pushDescriptor( + client: client, + name: name, + ref: ref, + content: imgConfig, + baseDescriptor: manifest.config + ) + } catch let err as ContainerizationError { + guard err.code != .exists else { + return + } + throw err + } - // Push config - let configStream = TestByteBufferSequence(data: configData) - try await mockClient.push( - name: name, - ref: ref, - descriptor: configDescriptor, - streamGenerator: { configStream }, - progress: nil as ProgressHandler? + // Push the image manifest. + let newManifest = Manifest( + schemaVersion: manifest.schemaVersion, + mediaType: manifest.mediaType!, + config: imgConfigDesc!, + layers: manifest.layers, + annotations: manifest.annotations ) - - // Push manifest - let manifestStream = TestByteBufferSequence(data: manifestData) - try await mockClient.push( + let manifestDesc = try await self.pushDescriptor( + client: client, name: name, ref: ref, - descriptor: manifestDescriptor, - streamGenerator: { manifestStream }, - progress: nil as ProgressHandler? + content: newManifest, + baseDescriptor: manifestDescriptor! ) - // Push index - let indexData = try JSONEncoder().encode(index) - let indexDigest = SHA256.hash(data: indexData) - let indexDescriptor = Descriptor( - mediaType: "application/vnd.docker.distribution.manifest.list.v2+json", - digest: "sha256:\(indexDigest.hexString)", - size: Int64(indexData.count) + // Push the index. + let newIndex = Index( + schemaVersion: index.schemaVersion, + mediaType: index.mediaType, + manifests: [manifestDesc], + annotations: index.annotations ) - - let indexStream = TestByteBufferSequence(data: indexData) - try await mockClient.push( + try await self.pushDescriptor( + client: client, name: name, ref: ref, - descriptor: indexDescriptor, - streamGenerator: { indexStream }, - progress: nil as ProgressHandler? + content: newIndex, + baseDescriptor: indexDescriptor ) - - // Verify all push operations were recorded - #expect(mockClient.pushCalls.count == 4) - - // Verify content integrity - let storedLayerData = mockClient.getPushedContent(name: name, descriptor: layerDescriptor) - #expect(storedLayerData == testLayerData) - - let storedConfigData = mockClient.getPushedContent(name: name, descriptor: configDescriptor) - #expect(storedConfigData == configData) - - let storedManifestData = mockClient.getPushedContent(name: name, descriptor: manifestDescriptor) - #expect(storedManifestData == manifestData) - - let storedIndexData = mockClient.getPushedContent(name: name, descriptor: indexDescriptor) - #expect(storedIndexData == indexData) } @Test func resolveWithRetry() async throws { @@ -377,142 +365,7 @@ extension SHA256.Digest { return "sha256:\(parts[1])" } - var hexString: String { - self.compactMap { String(format: "%02x", $0) }.joined() - } -} - -// Helper to create ByteBuffer sequences for testing -struct TestByteBufferSequence: Sendable, AsyncSequence { - typealias Element = ByteBuffer - - private let data: Data - - init(data: Data) { - self.data = data - } - func makeAsyncIterator() -> AsyncIterator { - AsyncIterator(data: data) - } - - struct AsyncIterator: AsyncIteratorProtocol { - private let data: Data - private var sent = false - - init(data: Data) { - self.data = data - } - - mutating func next() async throws -> ByteBuffer? { - guard !sent else { return nil } - sent = true - - var buffer = ByteBufferAllocator().buffer(capacity: data.count) - buffer.writeBytes(data) - return buffer - } - } } -// Helper class to create a mock ContentClient for testing -final class MockRegistryClient: ContentClient, @unchecked Sendable { - private var pushedContent: [String: [Descriptor: Data]] = [:] - private var fetchableContent: [String: [Descriptor: Data]] = [:] - - // Track push operations for verification - var pushCalls: [(name: String, ref: String, descriptor: Descriptor)] = [] - - func addFetchableContent(name: String, descriptor: Descriptor, content: T) throws { - let data = try JSONEncoder().encode(content) - if fetchableContent[name] == nil { - fetchableContent[name] = [:] - } - fetchableContent[name]![descriptor] = data - } - - func addFetchableData(name: String, descriptor: Descriptor, data: Data) { - if fetchableContent[name] == nil { - fetchableContent[name] = [:] - } - fetchableContent[name]![descriptor] = data - } - - func getPushedContent(name: String, descriptor: Descriptor) -> Data? { - pushedContent[name]?[descriptor] - } - - // MARK: - ContentClient Implementation - - func fetch(name: String, descriptor: Descriptor) async throws -> T { - guard let imageContent = fetchableContent[name], - let data = imageContent[descriptor] - else { - throw ContainerizationError(.notFound, message: "Content not found for \(name) with descriptor \(descriptor.digest)") - } - - return try JSONDecoder().decode(T.self, from: data) - } - - func fetchBlob(name: String, descriptor: Descriptor, into file: URL, progress: ProgressHandler?) async throws -> (Int64, SHA256.Digest) { - guard let imageContent = fetchableContent[name], - let data = imageContent[descriptor] - else { - throw ContainerizationError(.notFound, message: "Blob not found for \(name) with descriptor \(descriptor.digest)") - } - - try data.write(to: file) - let digest = SHA256.hash(data: data) - return (Int64(data.count), digest) - } - - func fetchData(name: String, descriptor: Descriptor) async throws -> Data { - guard let imageContent = fetchableContent[name], - let data = imageContent[descriptor] - else { - throw ContainerizationError(.notFound, message: "Data not found for \(name) with descriptor \(descriptor.digest)") - } - - return data - } - func push( - name: String, - ref: String, - descriptor: Descriptor, - streamGenerator: () throws -> T, - progress: ProgressHandler? - ) async throws where T.Element == ByteBuffer { - // Record the push call for verification - pushCalls.append((name: name, ref: ref, descriptor: descriptor)) - - // Simulate reading the stream and storing the data - let stream = try streamGenerator() - var data = Data() - - for try await buffer in stream { - data.append(contentsOf: buffer.readableBytesView) - } - - // Verify the pushed data matches the expected descriptor - let actualDigest = SHA256.hash(data: data) - guard descriptor.digest == "sha256:\(actualDigest.hexString)" else { - throw ContainerizationError(.invalidArgument, message: "Digest mismatch: expected \(descriptor.digest), got sha256:\(actualDigest.hexString)") - } - - guard data.count == descriptor.size else { - throw ContainerizationError(.invalidArgument, message: "Size mismatch: expected \(descriptor.size), got \(data.count)") - } - - // Store the pushed content - if pushedContent[name] == nil { - pushedContent[name] = [:] - } - pushedContent[name]![descriptor] = data - - // Simulate progress reporting - if let progress = progress { - await progress(Int64(data.count), Int64(data.count)) - } - } -} diff --git a/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift b/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift index 9b3a66ac..6c79f8e4 100644 --- a/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift +++ b/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift @@ -127,8 +127,8 @@ final class MockRegistryClient: ContentClient, @unchecked Sendable { // Verify the pushed data matches the expected descriptor let actualDigest = SHA256.hash(data: data) - guard descriptor.digest == "sha256:\(actualDigest.hexString)" else { - throw ContainerizationError(.invalidArgument, message: "Digest mismatch: expected \(descriptor.digest), got sha256:\(actualDigest.hexString)") + guard descriptor.digest == "sha256:\(actualDigest.encoded)" else { + throw ContainerizationError(.invalidArgument, message: "Digest mismatch: expected \(descriptor.digest), got sha256:\(actualDigest.encoded)") } guard data.count == descriptor.size else { @@ -148,11 +148,7 @@ final class MockRegistryClient: ContentClient, @unchecked Sendable { } } -extension SHA256.Digest { - var hexString: String { - self.compactMap { String(format: "%02x", $0) }.joined() - } -} + @Suite public class ImageStoreTests: ContainsAuth { @@ -223,6 +219,10 @@ public class ImageStoreTests: ContainsAuth { let testReference = "test-registry.local/test-image:latest" try await self.store.tag(existing: testImage.reference, new: testReference) + // Get the actual image to verify layer count + let actualImage = try await self.store.get(reference: testReference) + let expectedDigests = actualImage.referencedDigests() + // Test push with mock client (using extension method) try await self.store.testPush(reference: testReference, client: mockClient) @@ -233,5 +233,8 @@ public class ImageStoreTests: ContainsAuth { let pushCall = mockClient.pushCalls.first! #expect(pushCall.name == "test-registry.local/test-image") #expect(pushCall.ref == "latest") + + // Verify that all layers of the test image have been pushed + #expect(mockClient.pushCalls.count == expectedDigests.count) } } From ff827414461d7acd00068ad82e6344bf67229a48 Mon Sep 17 00:00:00 2001 From: Siddhant Khare Date: Wed, 9 Jul 2025 10:35:38 +0530 Subject: [PATCH 3/3] final fix --- Tests/ContainerizationOCITests/RegistryClientTests.swift | 7 +------ .../ContainerizationTests/ImageTests/ImageStoreTests.swift | 4 +--- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/Tests/ContainerizationOCITests/RegistryClientTests.swift b/Tests/ContainerizationOCITests/RegistryClientTests.swift index 56035cb0..c0567157 100644 --- a/Tests/ContainerizationOCITests/RegistryClientTests.swift +++ b/Tests/ContainerizationOCITests/RegistryClientTests.swift @@ -21,7 +21,6 @@ import ContainerizationIO import Crypto import Foundation import NIO -import NIOCore import Synchronization import Testing @@ -344,7 +343,7 @@ struct OCIClientTests: ~Copyable { ref: ref, descriptor: descriptor, streamGenerator: generator, - progress: nil as ProgressHandler? + progress: nil ) return descriptor } @@ -364,8 +363,4 @@ extension SHA256.Digest { let parts = self.description.split(separator: ": ") return "sha256:\(parts[1])" } - - } - - diff --git a/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift b/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift index 6c79f8e4..9021bd02 100644 --- a/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift +++ b/Tests/ContainerizationTests/ImageTests/ImageStoreTests.swift @@ -148,8 +148,6 @@ final class MockRegistryClient: ContentClient, @unchecked Sendable { } } - - @Suite public class ImageStoreTests: ContainsAuth { let store: ImageStore @@ -233,7 +231,7 @@ public class ImageStoreTests: ContainsAuth { let pushCall = mockClient.pushCalls.first! #expect(pushCall.name == "test-registry.local/test-image") #expect(pushCall.ref == "latest") - + // Verify that all layers of the test image have been pushed #expect(mockClient.pushCalls.count == expectedDigests.count) }