Revert "Revert "Deprecate FileSystemProvider and RenderNodeProvider"" (#1117)

Author: d-ronnqvist

Sources/SwiftDocC/Indexing/Navigator/NavigatorIndex+Ext.swift

@@ -14,6 +14,7 @@ import Foundation
  This class provides a simple way to transform a `FileSystemProvider` into a `RenderNodeProvider` to feed an index builder.
  The data from the disk is fetched and processed in an efficient way to build a navigator index.
  */
+@available(*, deprecated, message: "This deprecated API will be removed after 6.2 is released.")
 public class FileSystemRenderNodeProvider: RenderNodeProvider {
 
     /// The internal `FileSystemProvider` reference.

Sources/SwiftDocC/Indexing/Navigator/NavigatorIndex.swift

@@ -12,6 +12,7 @@ import Foundation
 import Crypto
 
 /// A protocol to provide data to be indexed.
+@available(*, deprecated, message: "This deprecated API will be removed after 6.2 is released.")
 public protocol RenderNodeProvider {
     /// Get an instance of `RenderNode` to be processed by the index.
     /// - Note: Returning `nil` will end the indexing process.
@@ -21,8 +22,6 @@ public protocol RenderNodeProvider {
     func getProblems() -> [Problem]
 }
 
-
-
 /**
  A `NavigatorIndex` contains all the necessary information to display the data inside a navigator.
  The data ranges from the tree to the necessary pieces of information to filter the content and perform actions in a fast way.
@@ -479,8 +478,12 @@ extension NavigatorIndex {
     open class Builder {
 
         /// The data provider.
+        @available(*, deprecated, message: "This deprecated API will be removed after 6.2 is released")
         public let renderNodeProvider: RenderNodeProvider?
 
+        /// The documentation archive to build an index from.
+        public let archiveURL: URL?
+        
         /// The output URL.
         public let outputURL: URL
 
@@ -554,26 +557,38 @@ extension NavigatorIndex {
         /// Indicates if the page title should be used instead of the navigator title.
         private let usePageTitle: Bool
 
-        
         /// Maps the icon render references in the navigator items created by this builder
         /// to their image references.
         ///
         /// Use the `NavigatorItem.icon` render reference to look up the full image reference
         /// for any custom icons used in this navigator index.
         var iconReferences = [String : ImageReference]()
 
-        
         /// Create a new a builder with the given data provider and output URL.
         /// - Parameters:
-        ///    - renderNodeProvider: The `RenderNode` provider to use.
+        ///    - archiveURL: The location of the documentation archive that the builder builds an navigator index for.
         ///    - outputURL: The location where the builder will write the the built navigator index.
         ///    - bundleIdentifier: The bundle identifier of the documentation that the builder builds a navigator index for.
         ///    - sortRootChildrenByName: Configure the builder to sort root's children by name.
         ///    - groupByLanguage: Configure the builder to group the entries by language.
         ///    - writePathsOnDisk: Configure the builder to write each navigator item's path components to the location.
         ///    - usePageTitle: Configure the builder to use the "page title" instead of the "navigator title" as the title for each entry.
+        public init(archiveURL: URL? = nil, outputURL: URL, bundleIdentifier: String, sortRootChildrenByName: Bool = false, groupByLanguage: Bool = false, writePathsOnDisk: Bool = true, usePageTitle: Bool = false) {
+            self.archiveURL = archiveURL
+            self.renderNodeProvider = nil
+            self.outputURL = outputURL
+            self.bundleIdentifier = bundleIdentifier
+            self.sortRootChildrenByName = sortRootChildrenByName
+            self.groupByLanguage = groupByLanguage
+            self.writePathsOnDisk = writePathsOnDisk
+            self.usePageTitle = usePageTitle
+        }
+        
+        @available(*, deprecated, renamed: "init(archiveURL:outputURL:bundleIdentifier:sortRootChildrenByName:groupByLanguage:writePathsOnDisk:usePageTitle:)", message: "Use 'init(archiveURL:outputURL:bundleIdentifier:sortRootChildrenByName:groupByLanguage:writePathsOnDisk:usePageTitle:)' instead. This deprecated API will be removed after 6.2 is released")
+        @_disfavoredOverload
         public init(renderNodeProvider: RenderNodeProvider? = nil, outputURL: URL, bundleIdentifier: String, sortRootChildrenByName: Bool = false, groupByLanguage: Bool = false, writePathsOnDisk: Bool = true, usePageTitle: Bool = false) {
             self.renderNodeProvider = renderNodeProvider
+            self.archiveURL = nil
             self.outputURL = outputURL
             self.bundleIdentifier = bundleIdentifier
             self.sortRootChildrenByName = sortRootChildrenByName
@@ -1245,13 +1260,43 @@ extension NavigatorIndex {
             problems.append(problem)
         }
 
-        /**
-         Build the index using the passed instance of `RenderNodeProvider` if available.
-         - Returns: A list containing all the problems encountered during indexing.
-         - Note: If a provider is not available, this method would generate a fatal error.
-         */
+        
+        /// Build the index using the render nodes files in the provided documentation archive.
+        /// - Returns: A list containing all the errors encountered during indexing.
+        /// - Precondition: Either ``archiveURL`` or ``renderNodeProvider`` is set.
         public func build() -> [Problem] {
-            precondition(renderNodeProvider != nil, "Calling build without a renderNodeProvider set is not permitted.")
+            if let archiveURL {
+                return _build(archiveURL: archiveURL)
+            } else {
+                return (self as _DeprecatedRenderNodeProviderAccess)._legacyBuild()
+            }
+        }
+        
+        // After 6.2 is released, move this into `build()`.
+        private func _build(archiveURL: URL) -> [Problem] {
+            setup()
+            
+            let dataDirectory = archiveURL.appendingPathComponent(NodeURLGenerator.Path.dataFolderName, isDirectory: true)
+            for file in FileManager.default.recursiveFiles(startingPoint: dataDirectory) where file.pathExtension.lowercased() == "json" {
+                do {
+                    let data = try Data(contentsOf: file)
+                    let renderNode = try RenderNode.decode(fromJSON: data)
+                    try index(renderNode: renderNode)
+                } catch {
+                    problems.append(error.problem(source: file,
+                                                  severity: .warning,
+                                                  summaryPrefix: "RenderNode indexing process failed"))
+                }
+            }
+            
+            finalize()
+            
+            return problems
+        }
+        
+        @available(*, deprecated, message: "This deprecated API will be removed after 6.2 is released")
+        fileprivate func _legacyBuild() -> [Problem] {
+            precondition(renderNodeProvider != nil, "Calling `build()` without an `archiveURL` or `renderNodeProvider` set is not permitted.")
 
             setup()
 
@@ -1274,7 +1319,6 @@ extension NavigatorIndex {
             return availabilityIDs[Int(availabilityID)]
         }
     }
-    
 }
 
 fileprivate extension Error {
@@ -1343,3 +1387,9 @@ enum PathHasher: String {
         }
     }
 }
+
+private protocol _DeprecatedRenderNodeProviderAccess {
+    // This private function accesses the deprecated RenderNodeProvider
+    func _legacyBuild() -> [Problem]
+}
+extension NavigatorIndex.Builder: _DeprecatedRenderNodeProviderAccess {}

Sources/SwiftDocC/Infrastructure/Workspace/FileSystemProvider.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -11,12 +11,14 @@
 import Foundation
 
 /// A type that vends a tree of virtual filesystem objects.
+@available(*, deprecated, message: "Use 'FileManagerProtocol.recursiveFiles(startingPoint:)' instead. This deprecated API will be removed after 6.2 is released.")
 public protocol FileSystemProvider {
     /// The organization of the files that this provider provides.
     var fileSystem: FSNode { get }
 }
 
 /// An element in a virtual filesystem.
+@available(*, deprecated, message: "This deprecated API will be removed after 6.2 is released.")
 public enum FSNode {
     /// A file in a filesystem.
     case file(File)

Sources/SwiftDocC/Infrastructure/Workspace/LocalFileSystemDataProvider+BundleDiscovery.swift

@@ -142,6 +142,7 @@ extension LocalFileSystemDataProvider: DocumentationWorkspaceDataProvider {
     }
 }
 
+@available(*, deprecated, message: "This deprecated API will be removed after 6.2 is released.")
 fileprivate extension [FSNode] {
     /// Returns the first file that matches a given predicate.
     /// - Parameter predicate: A closure that takes a file as its argument and returns a Boolean value indicating whether the file should be returned from this function.

Sources/SwiftDocC/Infrastructure/Workspace/LocalFileSystemDataProvider.swift

@@ -11,6 +11,7 @@
 import Foundation
 
 /// A type that provides documentation bundles that it discovers by traversing the local file system.
+@available(*, deprecated, message: "This deprecated API will be removed after 6.2 is released.")
 public struct LocalFileSystemDataProvider: FileSystemProvider {
     public var identifier: String = UUID().uuidString
 

Sources/SwiftDocC/Utility/FileManagerProtocol+FilesSequence.swift

@@ -0,0 +1,75 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2024 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See https://swift.org/LICENSE.txt for license information
+ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import Foundation
+
+extension FileManagerProtocol {
+    /// Returns a sequence of all the files in the directory structure from the starting point.
+    /// - Parameters:
+    ///   - startingPoint: The file or directory that's the top of the directory structure that the file manager traverses.
+    ///   - options: Options for how the file manager enumerates the contents of directories. Defaults to `.skipsHiddenFiles`.
+    /// - Returns: A sequence of the files in the directory structure.
+    package func recursiveFiles(startingPoint: URL, options: FileManager.DirectoryEnumerationOptions = .skipsHiddenFiles) -> IteratorSequence<_FilesIterator> {
+        IteratorSequence(_FilesIterator(fileManager: self, startingPoint: startingPoint, options: options))
+    }
+}
+
+// FIXME: This should be private and `FileManagerProtocol.recursiveFiles(startingPoint:options:)` should return `some Sequence<ULR>`
+// but because of https://github.com/swiftlang/swift/issues/77955 it needs to be exposed as an explicit type to avoid a SIL Validation error in the Swift compiler.
+
+/// An iterator that traverses the directory structure and returns the files in breadth-first order.
+package struct _FilesIterator: IteratorProtocol {
+    /// The file manager that the iterator uses to traverse the directory structure.
+    private var fileManager: any FileManagerProtocol // This can't be a generic because of https://github.com/swiftlang/swift/issues/77955
+    private var options: FileManager.DirectoryEnumerationOptions
+    
+    private var foundFiles: [URL]
+    private var foundDirectories: [URL]
+    
+    fileprivate init(fileManager: any FileManagerProtocol, startingPoint: URL, options: FileManager.DirectoryEnumerationOptions) {
+        self.fileManager = fileManager
+        self.options = options
+        
+        // Check if the starting point is a file or a directory.
+        if fileManager.directoryExists(atPath: startingPoint.path) {
+            foundFiles       = []
+            foundDirectories = [startingPoint]
+        } else {
+            foundFiles       = [startingPoint]
+            foundDirectories = []
+        }
+    }
+    
+    package mutating func next() -> URL? {
+        // If the iterator has already found some files, return those first
+        if !foundFiles.isEmpty {
+            return foundFiles.removeFirst()
+        }
+        
+        // Otherwise, check the next found directory and add its contents
+        guard !foundDirectories.isEmpty else {
+            // Traversed the entire directory structure
+            return nil
+        }
+        
+        let directory = foundDirectories.removeFirst()
+        guard let (newFiles, newDirectories) = try? fileManager.contentsOfDirectory(at: directory, options: options) else {
+            // The iterator protocol doesn't have a mechanism for raising errors. If an error occurs we
+            return nil
+        }
+        
+        foundFiles.append(contentsOf: newFiles)
+        foundDirectories.append(contentsOf: newDirectories)
+        
+        // Iterate again after adding new found files and directories.
+        // This enables the iterator do recurse multiple layers of directories until it finds a file.
+        return next()
+    }
+}

Sources/SwiftDocCUtilities/Action/Actions/Convert/Indexer.swift

@@ -36,7 +36,7 @@ extension ConvertAction {
         init(outputURL: URL, bundleID: DocumentationBundle.Identifier) throws {
             let indexURL = outputURL.appendingPathComponent("index", isDirectory: true)
             indexBuilder = Synchronized<NavigatorIndex.Builder>(
-                NavigatorIndex.Builder(renderNodeProvider: nil,
+                NavigatorIndex.Builder(
                     outputURL: indexURL,
                     bundleIdentifier: bundleID.rawValue,
                     sortRootChildrenByName: true,

Sources/SwiftDocCUtilities/Action/Actions/IndexAction.swift

@@ -13,21 +13,21 @@ import SwiftDocC
 
 /// An action that creates an index of a documentation bundle.
 public struct IndexAction: AsyncAction {
-    let rootURL: URL
+    let archiveURL: URL
     let outputURL: URL
     let bundleIdentifier: String
 
     var diagnosticEngine: DiagnosticEngine
 
     /// Initializes the action with the given validated options, creates or uses the given action workspace & context.
-    public init(documentationBundleURL: URL, outputURL: URL, bundleIdentifier: String, diagnosticEngine: DiagnosticEngine = .init()) throws {
+    public init(archiveURL: URL, outputURL: URL, bundleIdentifier: String, diagnosticEngine: DiagnosticEngine = .init()) {
         // Initialize the action context.
-        self.rootURL = documentationBundleURL
+        self.archiveURL = archiveURL
         self.outputURL = outputURL
         self.bundleIdentifier = bundleIdentifier
 
         self.diagnosticEngine = diagnosticEngine
-        self.diagnosticEngine.add(DiagnosticConsoleWriter(formattingOptions: [], baseURL: documentationBundleURL))
+        self.diagnosticEngine.add(DiagnosticConsoleWriter(formattingOptions: [], baseURL: archiveURL))
     }
 
     /// Converts each eligible file from the source documentation bundle,
@@ -40,8 +40,7 @@ public struct IndexAction: AsyncAction {
     }
 
     private func buildIndex() throws -> [Problem] {
-        let dataProvider = try LocalFileSystemDataProvider(rootURL: rootURL)
-        let indexBuilder = NavigatorIndex.Builder(renderNodeProvider: FileSystemRenderNodeProvider(fileSystemProvider: dataProvider),
+        let indexBuilder = NavigatorIndex.Builder(archiveURL: archiveURL,
                                                   outputURL: outputURL,
                                                   bundleIdentifier: bundleIdentifier,
                                                   sortRootChildrenByName: true,

Sources/SwiftDocCUtilities/Action/Actions/TransformForStaticHostingAction.swift

@@ -92,8 +92,7 @@ struct TransformForStaticHostingAction: AsyncAction {
         )
 
         // Create a StaticHostableTransformer targeted at the archive data folder
-        let dataProvider = try LocalFileSystemDataProvider(rootURL: rootURL.appendingPathComponent(NodeURLGenerator.Path.dataFolderName))
-        let transformer = StaticHostableTransformer(dataProvider: dataProvider, fileManager: fileManager, outputURL: outputURL, indexHTMLData: indexHTMLData)
+        let transformer = StaticHostableTransformer(dataDirectory: rootURL.appendingPathComponent(NodeURLGenerator.Path.dataFolderName), fileManager: fileManager, outputURL: outputURL, indexHTMLData: indexHTMLData)
         try transformer.transform()
 
     }

Sources/SwiftDocCUtilities/ArgumentParsing/ActionExtensions/IndexAction+CommandInitialization.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -12,10 +12,10 @@ import Foundation
 
 extension IndexAction {
     /// Initializes ``IndexAction`` from the options in the ``Index`` command.
-    init(fromIndexCommand index: Docc.Index) throws {
+    init(fromIndexCommand index: Docc.Index) {
         // Initialize the `IndexAction` from the options provided by the `Index` command
-        try self.init(
-            documentationBundleURL: index.documentationBundle.urlOrFallback,
+        self.init(
+            archiveURL: index.documentationArchive.urlOrFallback,
             outputURL: index.outputURL,
             bundleIdentifier: index.bundleIdentifier)
     }