Add An Endpoint Coverage Report

Makefile

@@ -13,6 +13,7 @@ OUTPUT_DIRECTORY ?= $(CURRENT_MAKEFILE_DIR)/Sources/FreeAgentAPI/Generated
 API_VERSION_OUTPUT ?= $(OUTPUT_DIRECTORY)/APIVersion.swift
 OPENAPI_SOURCE_DIR ?= $(CURRENT_MAKEFILE_DIR)/openapi
 OPENAPI_SOURCE_ROOT ?= $(OPENAPI_SOURCE_DIR)/openapi.yaml
+COVERAGE_REPORT_PATH ?= $(CURRENT_MAKEFILE_DIR)/.build/coverage.md
 
 # Derived values (don't change these).
 CURRENT_MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
@@ -40,6 +41,7 @@ help:  # Display this help.
 	@-+echo "  OPENAPI_YAML_PATH (e.g. openapi-bundled.yaml)"
 	@-+echo "  OPENAPI_GENERATOR_CONFIG_PATH (e.g. openapi-generator-config.yaml)"
 	@-+echo "  OUTPUT_DIRECTORY (e.g. Sources/FreeAgentAPI/Generated)"
+	@-+echo "  COVERAGE_REPORT_PATH (e.g. coverage.md)"
 
 bundle: $(FREEAGENT_TOOLS_BIN) $(OPENAPI_SOURCE_FILES)  # Bundle split OpenAPI files into a single openapi.yaml.
 	$(FREEAGENT_TOOLS_BIN) bundle "$(OPENAPI_SOURCE_ROOT)" "$(OPENAPI_YAML_PATH)"
@@ -53,6 +55,10 @@ generate: bundle $(SWIFT_OPENAPI_GENERATOR_BIN) $(OPENAPI_GENERATOR_CONFIG_PATH)
 		|| (echo "error: info.version in $(OPENAPI_SOURCE_ROOT) must be a FreeAgent API version date (YYYY-MM-DD), found: '$(API_VERSION)'" >&2; exit 1)
 	printf '// Generated by make generate, do not modify.\npublic let apiVersion = "%s"\n' "$(API_VERSION)" > "$(API_VERSION_OUTPUT)"
 
+coverage: $(FREEAGENT_TOOLS_BIN)  # Report which documented FreeAgent endpoints the OpenAPI spec covers.
+	$(FREEAGENT_TOOLS_BIN) coverage --root "$(OPENAPI_SOURCE_ROOT)" --output "$(COVERAGE_REPORT_PATH)"
+	@cat "$(COVERAGE_REPORT_PATH)"
+
 build:  # Runs swift build.
 	swift build
 
@@ -87,6 +93,7 @@ dump:  # Dump all derived values used by the Makefile.
 	@echo "API_VERSION = $(API_VERSION)"
 	@echo "OPENAPI_SOURCE_DIR = $(OPENAPI_SOURCE_DIR)"
 	@echo "OPENAPI_SOURCE_ROOT = $(OPENAPI_SOURCE_ROOT)"
+	@echo "COVERAGE_REPORT_PATH = $(COVERAGE_REPORT_PATH)"
 
 $(SWIFT_OPENAPI_GENERATOR_CLONE_STAMP):
 	rm -rf "$(SWIFT_OPENAPI_GENERATOR_CLONE_DIR)" "$(OUTPUT_DIRECTORY)"
@@ -112,4 +119,4 @@ $(FREEAGENT_TOOLS_BIN): $(wildcard $(CURRENT_MAKEFILE_DIR)/Sources/FreeAgentTool
 $(OUTPUT_DIRECTORY):
 	mkdir -p "$@"
 
-.PHONY: help bundle generate build lint format clean clean-all dump
+.PHONY: help bundle generate coverage build lint format clean clean-all dump

Sources/FreeAgentTools/Commands/CoverageCommand.swift

@@ -0,0 +1,82 @@
+import ArgumentParser
+import Foundation
+#if canImport(FoundationNetworking)
+import FoundationNetworking
+#endif
+
+// MARK: - CoverageCommand
+
+struct CoverageCommand: AsyncParsableCommand {
+
+    // MARK: Internal
+
+    static let configuration = CommandConfiguration(
+        commandName: "coverage",
+        abstract: "Write a report of the documented FreeAgent endpoints the OpenAPI spec covers",
+        discussion: """
+            Reads every page linked from the documentation index and lists, per page, the endpoints the spec \
+            has and the ones it is missing, then the spec endpoints no page documents. An endpoint documented \
+            on several pages is listed under the page that documents the most endpoints.
+            """
+    )
+
+    @Option(name: .long, help: "Path to the root OpenAPI YAML file")
+    var root = "openapi/openapi.yaml"
+
+    @Option(name: .long, help: "URL of the FreeAgent API documentation index")
+    var docs = "https://dev.freeagent.com/docs"
+
+    @Option(name: .long, help: "Path to write the Markdown report to")
+    var output: String
+
+    func run() async throws {
+        guard let docsURL = URL(string: docs) else {
+            throw ValidationError("Not a URL: \(docs)")
+        }
+
+        let links = try await DocsPage.links(inIndex: Self.fetch(docsURL))
+        let pages = try await withThrowingTaskGroup(of: (Int, DocsPage).self) { group in
+            for (offset, link) in links.enumerated() {
+                group.addTask {
+                    let html = try await Self.fetch(docsURL.appending(path: link.slug))
+                    return (offset, DocsPage(slug: link.slug, title: link.title, html: html))
+                }
+            }
+
+            return try await group.reduce(into: [(Int, DocsPage)]()) { $0.append($1) }
+                .sorted { $0.0 < $1.0 }
+                .map(\.1)
+        }
+
+        let spec = try SpecPaths.endpoints(rootURL: URL(fileURLWithPath: root))
+        let report = CoverageReport(docsURL: docsURL, pages: pages, spec: spec)
+
+        try report.description.write(toFile: output, atomically: true, encoding: .utf8)
+    }
+
+    // MARK: Private
+
+    private static func fetch(_ url: URL) async throws -> String {
+        let (data, response) = try await URLSession.shared.data(from: url)
+        let status = (response as? HTTPURLResponse)?.statusCode
+
+        guard status == 200 else {
+            throw CoverageCommandError.unexpectedStatus(url, status)
+        }
+
+        return String(decoding: data, as: UTF8.self)
+    }
+}
+
+// MARK: - CoverageCommandError
+
+enum CoverageCommandError: Error, CustomStringConvertible {
+    case unexpectedStatus(URL, Int?)
+
+    var description: String {
+        switch self {
+        case .unexpectedStatus(let url, let status):
+            "\(url.absoluteString) answered with status \(status.map(String.init) ?? "unknown")"
+        }
+    }
+}

Sources/FreeAgentTools/FreeAgentTools.swift

@@ -1,13 +1,14 @@
 import ArgumentParser
 
 @main
-struct FreeAgentTools: ParsableCommand {
+struct FreeAgentTools: AsyncParsableCommand {
     static let configuration = CommandConfiguration(
         commandName: "FreeAgentTools",
         abstract: "Development tools for the FreeAgent OpenAPI spec",
         subcommands: [
             BundleCommand.self,
             ChangedSuitesCommand.self,
+            CoverageCommand.self,
         ]
     )
 }

Sources/FreeAgentTools/Models/CoverageReport.swift

@@ -0,0 +1,93 @@
+import Foundation
+
+// MARK: - CoverageReport
+
+struct CoverageReport: CustomStringConvertible {
+
+    // MARK: Lifecycle
+
+    init(docsURL: URL, pages: [DocsPage], spec: [Endpoint]) {
+        var owners = [String: String]()
+        let largestFirst = pages.enumerated().sorted { lhs, rhs in
+            lhs.element.endpoints.count != rhs.element.endpoints.count
+                ? lhs.element.endpoints.count > rhs.element.endpoints.count
+                : lhs.offset < rhs.offset
+        }
+        for (_, page) in largestFirst {
+            for endpoint in page.endpoints where owners[endpoint.key] == nil {
+                owners[endpoint.key] = page.slug
+            }
+        }
+
+        let specKeys = Set(spec.map(\.key))
+        self.docsURL = docsURL
+        sections = pages.compactMap { page in
+            let owned = page.endpoints.filter { owners[$0.key] == page.slug }
+            guard !owned.isEmpty else {
+                return nil
+            }
+
+            return CoverageSection(
+                slug: page.slug,
+                title: page.title,
+                covered: owned.filter { specKeys.contains($0.key) },
+                missing: owned.filter { !specKeys.contains($0.key) }
+            )
+        }
+        undocumented = spec
+            .filter { owners[$0.key] == nil }
+            .sorted { ($0.path, $0.method) < ($1.path, $1.method) }
+    }
+
+    // MARK: Internal
+
+    let docsURL: URL
+    let sections: [CoverageSection]
+    let undocumented: [Endpoint]
+
+    var description: String {
+        let documented = sections.reduce(0) { $0 + $1.covered.count + $1.missing.count }
+        let covered = sections.reduce(0) { $0 + $1.covered.count }
+        let percent = documented == 0 ? 0 : Int((Double(covered) / Double(documented) * 100).rounded())
+
+        var lines = [
+            "# FreeAgent API Coverage",
+            "",
+            "The spec covers \(covered) of the \(documented) endpoints documented at \(docsURL.absoluteString) (\(percent)%).",
+            "",
+            "| Page | Documented | Covered | Missing |",
+            "| --- | ---: | ---: | ---: |",
+        ]
+        lines += sections.map { section in
+            let link = "[\(section.title)](\(docsURL.appending(path: section.slug).absoluteString))"
+            let documented = section.covered.count + section.missing.count
+
+            return "| \(link) | \(documented) | \(section.covered.count) | \(section.missing.count) |"
+        }
+
+        let missing = sections.filter { !$0.missing.isEmpty }
+        if !missing.isEmpty {
+            lines += ["", "## Missing Endpoints"]
+            for section in missing {
+                lines += ["", "### \(section.title)", ""]
+                lines += section.missing.map { "- `\($0)`" }
+            }
+        }
+
+        if !undocumented.isEmpty {
+            lines += ["", "## Spec Endpoints Not In The Docs", ""]
+            lines += undocumented.map { "- `\($0)`" }
+        }
+
+        return lines.joined(separator: "\n") + "\n"
+    }
+}
+
+// MARK: - CoverageSection
+
+struct CoverageSection: Equatable {
+    let slug: String
+    let title: String
+    let covered: [Endpoint]
+    let missing: [Endpoint]
+}

Sources/FreeAgentTools/Models/DocsPage.swift

@@ -0,0 +1,45 @@
+import Foundation
+
+struct DocsPage: Sendable {
+
+    // MARK: Lifecycle
+
+    init(slug: String, title: String, endpoints: [Endpoint]) {
+        self.slug = slug
+        self.title = title
+        self.endpoints = endpoints
+    }
+
+    init(slug: String, title: String, html: String) {
+        var seen = Set<String>()
+        let endpoints = html.matches(of: Self.endpointPattern)
+            .map { Endpoint(method: String($0.output.1), path: String($0.output.2)) }
+            .filter { seen.insert($0.key).inserted }
+
+        self.init(slug: slug, title: title, endpoints: endpoints)
+    }
+
+    // MARK: Internal
+
+    let slug: String
+    let title: String
+    let endpoints: [Endpoint]
+
+    static func links(inIndex html: String) -> [(slug: String, title: String)] {
+        var seen = Set<String>()
+
+        return html.matches(of: linkPattern)
+            .map { (slug: String($0.output.1), title: String($0.output.2).replacingOccurrences(of: "&amp;", with: "&")) }
+            .filter { seen.insert($0.slug).inserted }
+    }
+
+    // MARK: Private
+
+    private static var linkPattern: Regex<(Substring, Substring, Substring)> {
+        #/<li><a href="/docs/([^"#]+)">([^<]+)</a></li>/#
+    }
+
+    private static var endpointPattern: Regex<(Substring, Substring, Substring)> {
+        #/<code[^>]*>(GET|POST|PUT|PATCH|DELETE)\s+https://api(?:\.sandbox)?\.freeagent\.com(/v2/[^\s?<"]*)/#
+    }
+}

Sources/FreeAgentTools/Models/Endpoint.swift

@@ -0,0 +1,25 @@
+struct Endpoint: Hashable, Sendable, CustomStringConvertible {
+
+    // MARK: Internal
+
+    let method: String
+    let path: String
+
+    var key: String {
+        let segments = path.split(separator: "/", omittingEmptySubsequences: false).map { segment in
+            Self.isPlaceholder(segment) ? "{}" : String(segment)
+        }
+
+        return "\(method) \(segments.joined(separator: "/"))"
+    }
+
+    var description: String {
+        "\(method) \(path)"
+    }
+
+    // MARK: Private
+
+    private static func isPlaceholder(_ segment: Substring) -> Bool {
+        segment.hasPrefix(":") || segment.hasPrefix("{") || (!segment.isEmpty && segment.allSatisfy(\.isNumber))
+    }
+}

Sources/FreeAgentTools/Models/SpecGraph.swift

@@ -44,6 +44,19 @@ struct SpecGraph {
         return node
     }
 
+    static func node(at pointer: String, in root: Node) throws -> Node {
+        var node = root
+        for key in pointer.dropFirst(2).split(separator: "/") {
+            guard let next = node[String(key)] else {
+                throw SpecGraphError.missingPointer(pointer)
+            }
+
+            node = next
+        }
+
+        return node
+    }
+
     func selection(changedFiles: [URL], baseRoot: Node?) -> SuiteSelection {
         var changed = Set<SpecLocation>()
         for location in changedFiles.map({ SpecLocation(file: $0) }) {
@@ -169,19 +182,6 @@ struct SpecGraph {
         return SpecLocation(file: URL(fileURLWithPath: file, relativeTo: location.file.deletingLastPathComponent()))
     }
 
-    private static func node(at pointer: String, in root: Node) throws -> Node {
-        var node = root
-        for key in pointer.dropFirst(2).split(separator: "/") {
-            guard let next = node[String(key)] else {
-                throw SpecGraphError.missingPointer(pointer)
-            }
-
-            node = next
-        }
-
-        return node
-    }
-
     private func closure(of location: SpecLocation) -> Set<SpecLocation> {
         var visited = Set<SpecLocation>()
         var pending = [location]

Sources/FreeAgentTools/Models/SpecPaths.swift

@@ -0,0 +1,40 @@
+import Foundation
+import Yams
+
+enum SpecPaths {
+
+    // MARK: Internal
+
+    static func endpoints(rootURL: URL) throws -> [Endpoint] {
+        let root = try SpecGraph.load(rootURL)
+        var endpoints = [Endpoint]()
+        for (path, item) in root["paths"]?.mapping ?? [:] {
+            guard let path = path.string else {
+                continue
+            }
+
+            let operations = try item["$ref"]?.string.map { try node(at: $0, relativeTo: rootURL) } ?? item
+            for method in methods where operations[method] != nil {
+                endpoints.append(Endpoint(method: method.uppercased(), path: path))
+            }
+        }
+
+        return endpoints
+    }
+
+    // MARK: Private
+
+    private static let methods = ["get", "post", "put", "patch", "delete"]
+
+    private static func node(at ref: String, relativeTo rootURL: URL) throws -> Node {
+        let parts = ref.split(separator: "#", maxSplits: 1, omittingEmptySubsequences: false)
+        let file = URL(fileURLWithPath: String(parts[0]), relativeTo: rootURL.deletingLastPathComponent())
+        let document = try SpecGraph.load(file)
+
+        guard parts.count == 2 else {
+            return document
+        }
+
+        return try SpecGraph.node(at: "#\(parts[1])", in: document)
+    }
+}