From 6b57487e7616eb57b70f45273006ddcf17b5386c Mon Sep 17 00:00:00 2001 From: secustor Date: Wed, 25 Oct 2023 11:43:37 +0200 Subject: [PATCH] feat: initial prototype --- Package.resolved | 135 +++++++++++++++++ Package.swift | 8 +- Package@swift-4.2.swift | 3 +- Package@swift-5.0.swift | 3 +- .../XCLogParser/reporter/OTELReporter.swift | 140 +++++++++++++++++ Sources/XCLogParser/reporter/Reporter.swift | 3 + .../XCLogParserTests/OtelReporterTests.swift | 142 ++++++++++++++++++ Tests/XCLogParserTests/ReporterTests.swift | 3 + 8 files changed, 434 insertions(+), 3 deletions(-) create mode 100644 Sources/XCLogParser/reporter/OTELReporter.swift create mode 100644 Tests/XCLogParserTests/OtelReporterTests.swift diff --git a/Package.resolved b/Package.resolved index 900fb44..b124f3c 100644 --- a/Package.resolved +++ b/Package.resolved @@ -10,6 +10,15 @@ "version": "1.3.3" } }, + { + "package": "grpc-swift", + "repositoryURL": "https://github.com/grpc/grpc-swift.git", + "state": { + "branch": null, + "revision": "84bac657e9930d26e9124bac082f26586dc2d209", + "version": "1.19.1" + } + }, { "package": "Gzip", "repositoryURL": "https://github.com/1024jp/GzipSwift", @@ -19,6 +28,24 @@ "version": "5.1.1" } }, + { + "package": "opentelemetry-swift", + "repositoryURL": "https://github.com/open-telemetry/opentelemetry-swift", + "state": { + "branch": null, + "revision": "8d6fa745c186e3b7556f330b5236da25fa7c9d62", + "version": "1.7.0" + } + }, + { + "package": "Opentracing", + "repositoryURL": "https://github.com/undefinedlabs/opentracing-objc", + "state": { + "branch": null, + "revision": "18c1a35ca966236cee0c5a714a51a73ff33384c1", + "version": "0.5.2" + } + }, { "package": "PathKit", "repositoryURL": "https://github.com/kylef/PathKit.git", @@ -28,6 +55,15 @@ "version": "1.0.1" } }, + { + "package": "Reachability", + "repositoryURL": "https://github.com/ashleymills/Reachability.swift", + "state": { + "branch": null, + "revision": "c01bbdf2d633cf049ae1ed1a68a2020a8bda32e2", + "version": "5.1.0" + } + }, { "package": "Spectre", "repositoryURL": "https://github.com/kylef/Spectre.git", @@ -45,6 +81,105 @@ "revision": "fddd1c00396eed152c45a46bea9f47b98e59301d", "version": "1.2.0" } + }, + { + "package": "swift-atomics", + "repositoryURL": "https://github.com/apple/swift-atomics.git", + "state": { + "branch": null, + "revision": "cd142fd2f64be2100422d658e7411e39489da985", + "version": "1.2.0" + } + }, + { + "package": "swift-collections", + "repositoryURL": "https://github.com/apple/swift-collections.git", + "state": { + "branch": null, + "revision": "a902f1823a7ff3c9ab2fba0f992396b948eda307", + "version": "1.0.5" + } + }, + { + "package": "swift-log", + "repositoryURL": "https://github.com/apple/swift-log.git", + "state": { + "branch": null, + "revision": "532d8b529501fb73a2455b179e0bbb6d49b652ed", + "version": "1.5.3" + } + }, + { + "package": "swift-metrics", + "repositoryURL": "https://github.com/apple/swift-metrics.git", + "state": { + "branch": null, + "revision": "971ba26378ab69c43737ee7ba967a896cb74c0d1", + "version": "2.4.1" + } + }, + { + "package": "swift-nio", + "repositoryURL": "https://github.com/apple/swift-nio.git", + "state": { + "branch": null, + "revision": "3db5c4aeee8100d2db6f1eaf3864afdad5dc68fd", + "version": "2.59.0" + } + }, + { + "package": "swift-nio-extras", + "repositoryURL": "https://github.com/apple/swift-nio-extras.git", + "state": { + "branch": null, + "revision": "fb70a0f5e984f23be48b11b4f1909f3bee016178", + "version": "1.19.1" + } + }, + { + "package": "swift-nio-http2", + "repositoryURL": "https://github.com/apple/swift-nio-http2.git", + "state": { + "branch": null, + "revision": "9c22e4f810ce780453f563fba98e1a1039f83d56", + "version": "1.28.1" + } + }, + { + "package": "swift-nio-ssl", + "repositoryURL": "https://github.com/apple/swift-nio-ssl.git", + "state": { + "branch": null, + "revision": "320bd978cceb8e88c125dcbb774943a92f6286e9", + "version": "2.25.0" + } + }, + { + "package": "swift-nio-transport-services", + "repositoryURL": "https://github.com/apple/swift-nio-transport-services.git", + "state": { + "branch": null, + "revision": "e7403c35ca6bb539a7ca353b91cc2d8ec0362d58", + "version": "1.19.0" + } + }, + { + "package": "SwiftProtobuf", + "repositoryURL": "https://github.com/apple/swift-protobuf.git", + "state": { + "branch": null, + "revision": "3c54ab05249f59f2c6641dd2920b8358ea9ed127", + "version": "1.24.0" + } + }, + { + "package": "Thrift", + "repositoryURL": "https://github.com/undefinedlabs/Thrift-Swift", + "state": { + "branch": null, + "revision": "18ff09e6b30e589ed38f90a1af23e193b8ecef8e", + "version": "1.1.2" + } } ] }, diff --git a/Package.swift b/Package.swift index 98f46e7..22ad90c 100644 --- a/Package.swift +++ b/Package.swift @@ -5,7 +5,7 @@ import PackageDescription let package = Package( name: "XCLogParser", - platforms: [.macOS(.v10_13)], + platforms: [.macOS(.v10_15)], products: [ .executable(name: "xclogparser", targets: ["XCLogParserApp"]), .library(name: "XCLogParser", targets: ["XCLogParser"]) @@ -15,6 +15,7 @@ let package = Package( .package(url: "https://github.com/krzyzanowskim/CryptoSwift.git", .exact("1.3.3")), .package(url: "https://github.com/kylef/PathKit.git", from: "1.0.1"), .package(url: "https://github.com/apple/swift-argument-parser", from: "1.2.0"), + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", from: "1.7.0"), ], targets: [ .target( @@ -25,6 +26,11 @@ let package = Package( name: "XCLogParser", dependencies: [ .product(name: "Gzip", package: "GzipSwift"), + .product(name: "OpenTelemetryApi", package: "opentelemetry-swift"), + .product(name: "OpenTelemetrySdk", package: "opentelemetry-swift"), + .product(name: "StdoutExporter", package: "opentelemetry-swift"), + .product(name: "ResourceExtension", package: "opentelemetry-swift"), + .product(name: "OpenTelemetryProtocolExporter", package: "opentelemetry-swift"), "XcodeHasher", "PathKit" ] diff --git a/Package@swift-4.2.swift b/Package@swift-4.2.swift index 91b1f4b..a536699 100644 --- a/Package@swift-4.2.swift +++ b/Package@swift-4.2.swift @@ -15,6 +15,7 @@ let package = Package( .package(url: "https://github.com/krzyzanowskim/CryptoSwift.git", from: "0.15.0"), .package(url: "https://github.com/kylef/PathKit.git", from: "1.0.0"), .package(url: "https://github.com/antitypical/Result.git", from: "4.0.0"), + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", from: "1.7.0"), ], targets: [ .target( @@ -23,7 +24,7 @@ let package = Package( ), .target( name: "XCLogParser", - dependencies: ["Gzip", "XcodeHasher", "PathKit"] + dependencies: ["Gzip", "XcodeHasher", "PathKit", "OpenTelemetrySdk"] ), .target( name: "XCLogParserApp", diff --git a/Package@swift-5.0.swift b/Package@swift-5.0.swift index f5fe066..04b81d4 100644 --- a/Package@swift-5.0.swift +++ b/Package@swift-5.0.swift @@ -15,6 +15,7 @@ let package = Package( .package(url: "https://github.com/krzyzanowskim/CryptoSwift.git", .exact("1.3.3")), .package(url: "https://github.com/kylef/PathKit.git", from: "1.0.1"), .package(url: "https://github.com/apple/swift-argument-parser", .upToNextMinor(from: "0.3.0")), + .package(url: "https://github.com/open-telemetry/opentelemetry-swift", from: "1.7.0"), ], targets: [ .target( @@ -23,7 +24,7 @@ let package = Package( ), .target( name: "XCLogParser", - dependencies: ["Gzip", "XcodeHasher", "PathKit"] + dependencies: ["Gzip", "XcodeHasher", "PathKit", "OpenTelemetrySdk"] ), .target( name: "XCLogParserApp", diff --git a/Sources/XCLogParser/reporter/OTELReporter.swift b/Sources/XCLogParser/reporter/OTELReporter.swift new file mode 100644 index 0000000..3942e0d --- /dev/null +++ b/Sources/XCLogParser/reporter/OTELReporter.swift @@ -0,0 +1,140 @@ +// Copyright (c) 2019 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import Foundation +import OpenTelemetryApi +import OpenTelemetrySdk +import StdoutExporter +import OpenTelemetryProtocolExporterGrpc +import ResourceExtension +import GRPC +import NIO +import NIOHPACK + +public struct OTELReporter: LogReporter { + + public init() {} + + public func report(build: Any, output: ReporterOutput, rootOutput: String) throws { + guard let buildStep = build as? BuildStep else { + throw XCLogParserError.errorCreatingReport("Type not supported \(type(of: build))") + } + + let (tracer, processor) = createTracer() + + // recursively create spans + createSpan(tracer: tracer, parentSpan: nil, buildStep: buildStep) + + // ensure that all spans are exported before program shutdown + processor.forceFlush(timeout: TimeInterval(3000)) + + } +} + +func createSpan(tracer: Tracer, parentSpan: Span?, buildStep: BuildStep) { + // skip details as this would generate 10.000s of spans + // TODO make this configurable + if buildStep.type == BuildStepType.detail { + return + } + // do not create spans if they pulled from cache + // TODO make this a config option? + if buildStep.fetchedFromCache == true { + return + } + + let spanBuilder = tracer.spanBuilder(spanName: buildStep.title) + .setStartTime(time: Date(timeIntervalSince1970: buildStep.startTimestamp)) + + // set optional parentSpan + if let parentSpan = parentSpan { + spanBuilder.setParent(parentSpan) + } else { + spanBuilder.setNoParent() + } + + + let span = spanBuilder.startSpan() + + span.status = parseStatus(buildStep: buildStep) + + span.setAttribute(key: "xcode.build.step.type", value: buildStep.type.rawValue) + span.setAttribute(key: "xcode.build.domain", value: buildStep.domain) + span.setAttribute(key: "xcode.build.schema", value: buildStep.schema) + span.setAttribute(key: "xcode.build.cacheUsed", value: buildStep.fetchedFromCache) + + + // process child steps + for childStep in buildStep.subSteps { + createSpan(tracer: tracer, parentSpan: span, buildStep: childStep) + } + + // end span after processing childs, to allow modifications by them. + span.end(time: Date(timeIntervalSince1970: buildStep.endTimestamp)) +} + +func parseStatus(buildStep: BuildStep) -> Status { + let status = buildStep.buildStatus + switch status { + case "succeeded": + return Status.ok + case "failed": + return Status.error(description: "") + default: + return Status.error(description: "Build status is unexpected:" + status) + } +} + +func createTracer() -> (Tracer, SpanProcessor) { + let instrumentationName = "XCLogParser" + let instrumentationVersion = "semver:" + Version.current + + + // read out local machine information and set custom resource attributes + let resource = DefaultResources() + .get() + .merging( + other: Resource.init( + attributes: [ + "service.name" : AttributeValue("serviceName") // TODO read out from standardized OTEL env variables + ])) + + let configuration = ClientConnection.Configuration.default( + target: .hostAndPort("localhost", 4317), // TODO read out from standardized OTEL env variables + eventLoopGroup: MultiThreadedEventLoopGroup(numberOfThreads: 1) + ) + let client = ClientConnection(configuration: configuration) + let otlpTraceExporter = OtlpTraceExporter(channel: client) + + let stdoutExporter = StdoutExporter() + let spanExporter = MultiSpanExporter(spanExporters: [otlpTraceExporter, stdoutExporter]) + + let spanProcessor = SimpleSpanProcessor(spanExporter: spanExporter) + + OpenTelemetry.registerTracerProvider(tracerProvider: + TracerProviderBuilder() + .add(spanProcessor: spanProcessor) + .with(resource: resource) + .build() + ) + + let tracer = OpenTelemetry.instance.tracerProvider.get(instrumentationName: instrumentationName, instrumentationVersion: instrumentationVersion) + + return (tracer, spanProcessor) +} diff --git a/Sources/XCLogParser/reporter/Reporter.swift b/Sources/XCLogParser/reporter/Reporter.swift index 504cba7..35c33c3 100644 --- a/Sources/XCLogParser/reporter/Reporter.swift +++ b/Sources/XCLogParser/reporter/Reporter.swift @@ -26,6 +26,7 @@ public enum Reporter: String { case chromeTracer case html case issues + case otel public func makeLogReporter() -> LogReporter { switch self { @@ -41,6 +42,8 @@ public enum Reporter: String { return HtmlReporter() case .issues: return IssuesReporter() + case .otel: + return OTELReporter() } } } diff --git a/Tests/XCLogParserTests/OtelReporterTests.swift b/Tests/XCLogParserTests/OtelReporterTests.swift new file mode 100644 index 0000000..a3461ce --- /dev/null +++ b/Tests/XCLogParserTests/OtelReporterTests.swift @@ -0,0 +1,142 @@ +// Copyright (c) 2019 Spotify AB. +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import XCTest +@testable import XCLogParser + +class OtelOutputTests: XCTestCase { + let output = OTELReporter() + + func testTargetToTraceEvent() { + let root = getBuildStep() + + } + + private func getBuildStep() -> BuildStep { + let start = Date() + let end = start.addingTimeInterval(100 * 100) + let root = BuildStep(type: .main, + machineName: "", + buildIdentifier: "ABC", + identifier: "ABC1", + parentIdentifier: "", + domain: "", + title: "MyApp", + signature: "Build MyApp", + startDate: "", + endDate: "", + startTimestamp: start.timeIntervalSince1970, + endTimestamp: end.timeIntervalSince1970, + duration: 100 * 100, + detailStepType: .none, + buildStatus: "Build succeeded", + schema: "MyApp", + subSteps: getTargets(start: start), + warningCount: 0, + errorCount: 0, + architecture: "", + documentURL: "", + warnings: nil, + errors: nil, + notes: nil, + swiftFunctionTimes: nil, + fetchedFromCache: false, + compilationEndTimestamp: end.timeIntervalSince1970, + compilationDuration: 100 * 100, + clangTimeTraceFile: nil, + linkerStatistics: nil, + swiftTypeCheckTimes: nil + ) + return root + } + + // swiftlint:disable function_body_length + private func getTargets(start: Date) -> [BuildStep] { + let end = start.addingTimeInterval(50 * 100) + let target1 = BuildStep(type: .target, + machineName: "", + buildIdentifier: "ABC", + identifier: "ABC1_1", + parentIdentifier: "ABC1", + domain: "", + title: "MyTarget", + signature: "Build MyTarget", + startDate: "", + endDate: "", + startTimestamp: start.timeIntervalSince1970, + endTimestamp: end.timeIntervalSince1970, + duration: 50 * 100, + detailStepType: .none, + buildStatus: "Build succeeded", + schema: "MyApp", + subSteps: [BuildStep](), + warningCount: 0, + errorCount: 0, + architecture: "", + documentURL: "", + warnings: nil, + errors: nil, + notes: nil, + swiftFunctionTimes: nil, + fetchedFromCache: false, + compilationEndTimestamp: end.timeIntervalSince1970, + compilationDuration: 50 * 100, + clangTimeTraceFile: nil, + linkerStatistics: nil, + swiftTypeCheckTimes: nil + ) + + let end2 = end.addingTimeInterval(50 * 100) + let target2 = BuildStep(type: .target, + machineName: "", + buildIdentifier: "ABC", + identifier: "ABC1_2", + parentIdentifier: "ABC1", + domain: "", + title: "MyTarget2", + signature: "Build MyTarget2", + startDate: "", + endDate: "", + startTimestamp: end.timeIntervalSince1970, + endTimestamp: end2.timeIntervalSince1970, + duration: 50 * 100, + detailStepType: .none, + buildStatus: "Build succeeded", + schema: "MyApp", + subSteps: [BuildStep](), + warningCount: 0, + errorCount: 0, + architecture: "", + documentURL: "", + warnings: nil, + errors: nil, + notes: nil, + swiftFunctionTimes: nil, + fetchedFromCache: false, + compilationEndTimestamp: end2.timeIntervalSince1970, + compilationDuration: 50 * 100, + clangTimeTraceFile: nil, + linkerStatistics: nil, + swiftTypeCheckTimes: nil + ) + return [target1, target2] + + } + +} diff --git a/Tests/XCLogParserTests/ReporterTests.swift b/Tests/XCLogParserTests/ReporterTests.swift index f943567..2c106a0 100644 --- a/Tests/XCLogParserTests/ReporterTests.swift +++ b/Tests/XCLogParserTests/ReporterTests.swift @@ -38,6 +38,9 @@ class ReporterTests: XCTestCase { let issuesReporter = Reporter.issues.makeLogReporter() XCTAssertTrue(issuesReporter is IssuesReporter) + + let otelReporter = Reporter.otel.makeLogReporter() + XCTAssertTrue(otelReporter is OTELReporter) } }