diff --git a/Makefile b/Makefile index 23824a1..7175628 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ OUTPUT_DIRECTORY ?= $(CURRENT_MAKEFILE_DIR)/Sources/FreeAgentAPI/Generated API_VERSION_OUTPUT ?= $(OUTPUT_DIRECTORY)/APIVersion.swift OPENAPI_SOURCE_DIR ?= $(CURRENT_MAKEFILE_DIR)/openapi OPENAPI_SOURCE_ROOT ?= $(OPENAPI_SOURCE_DIR)/openapi.yaml +COVERAGE_REPORT_PATH ?= $(CURRENT_MAKEFILE_DIR)/.build/coverage.md # Derived values (don't change these). CURRENT_MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) @@ -40,6 +41,7 @@ help: # Display this help. @-+echo " OPENAPI_YAML_PATH (e.g. openapi-bundled.yaml)" @-+echo " OPENAPI_GENERATOR_CONFIG_PATH (e.g. openapi-generator-config.yaml)" @-+echo " OUTPUT_DIRECTORY (e.g. Sources/FreeAgentAPI/Generated)" + @-+echo " COVERAGE_REPORT_PATH (e.g. coverage.md)" bundle: $(FREEAGENT_TOOLS_BIN) $(OPENAPI_SOURCE_FILES) # Bundle split OpenAPI files into a single openapi.yaml. $(FREEAGENT_TOOLS_BIN) bundle "$(OPENAPI_SOURCE_ROOT)" "$(OPENAPI_YAML_PATH)" @@ -53,6 +55,10 @@ generate: bundle $(SWIFT_OPENAPI_GENERATOR_BIN) $(OPENAPI_GENERATOR_CONFIG_PATH) || (echo "error: info.version in $(OPENAPI_SOURCE_ROOT) must be a FreeAgent API version date (YYYY-MM-DD), found: '$(API_VERSION)'" >&2; exit 1) printf '// Generated by make generate, do not modify.\npublic let apiVersion = "%s"\n' "$(API_VERSION)" > "$(API_VERSION_OUTPUT)" +coverage: $(FREEAGENT_TOOLS_BIN) # Report which documented FreeAgent endpoints the OpenAPI spec covers. + $(FREEAGENT_TOOLS_BIN) coverage --root "$(OPENAPI_SOURCE_ROOT)" --output "$(COVERAGE_REPORT_PATH)" + @cat "$(COVERAGE_REPORT_PATH)" + build: # Runs swift build. swift build @@ -87,6 +93,7 @@ dump: # Dump all derived values used by the Makefile. @echo "API_VERSION = $(API_VERSION)" @echo "OPENAPI_SOURCE_DIR = $(OPENAPI_SOURCE_DIR)" @echo "OPENAPI_SOURCE_ROOT = $(OPENAPI_SOURCE_ROOT)" + @echo "COVERAGE_REPORT_PATH = $(COVERAGE_REPORT_PATH)" $(SWIFT_OPENAPI_GENERATOR_CLONE_STAMP): rm -rf "$(SWIFT_OPENAPI_GENERATOR_CLONE_DIR)" "$(OUTPUT_DIRECTORY)" @@ -112,4 +119,4 @@ $(FREEAGENT_TOOLS_BIN): $(wildcard $(CURRENT_MAKEFILE_DIR)/Sources/FreeAgentTool $(OUTPUT_DIRECTORY): mkdir -p "$@" -.PHONY: help bundle generate build lint format clean clean-all dump +.PHONY: help bundle generate coverage build lint format clean clean-all dump diff --git a/Sources/FreeAgentTools/Commands/CoverageCommand.swift b/Sources/FreeAgentTools/Commands/CoverageCommand.swift new file mode 100644 index 0000000..2089f40 --- /dev/null +++ b/Sources/FreeAgentTools/Commands/CoverageCommand.swift @@ -0,0 +1,82 @@ +import ArgumentParser +import Foundation +#if canImport(FoundationNetworking) +import FoundationNetworking +#endif + +// MARK: - CoverageCommand + +struct CoverageCommand: AsyncParsableCommand { + + // MARK: Internal + + static let configuration = CommandConfiguration( + commandName: "coverage", + abstract: "Write a report of the documented FreeAgent endpoints the OpenAPI spec covers", + discussion: """ + Reads every page linked from the documentation index and lists, per page, the endpoints the spec \ + has and the ones it is missing, then the spec endpoints no page documents. An endpoint documented \ + on several pages is listed under the page that documents the most endpoints. + """ + ) + + @Option(name: .long, help: "Path to the root OpenAPI YAML file") + var root = "openapi/openapi.yaml" + + @Option(name: .long, help: "URL of the FreeAgent API documentation index") + var docs = "https://dev.freeagent.com/docs" + + @Option(name: .long, help: "Path to write the Markdown report to") + var output: String + + func run() async throws { + guard let docsURL = URL(string: docs) else { + throw ValidationError("Not a URL: \(docs)") + } + + let links = try await DocsPage.links(inIndex: Self.fetch(docsURL)) + let pages = try await withThrowingTaskGroup(of: (Int, DocsPage).self) { group in + for (offset, link) in links.enumerated() { + group.addTask { + let html = try await Self.fetch(docsURL.appending(path: link.slug)) + return (offset, DocsPage(slug: link.slug, title: link.title, html: html)) + } + } + + return try await group.reduce(into: [(Int, DocsPage)]()) { $0.append($1) } + .sorted { $0.0 < $1.0 } + .map(\.1) + } + + let spec = try SpecPaths.endpoints(rootURL: URL(fileURLWithPath: root)) + let report = CoverageReport(docsURL: docsURL, pages: pages, spec: spec) + + try report.description.write(toFile: output, atomically: true, encoding: .utf8) + } + + // MARK: Private + + private static func fetch(_ url: URL) async throws -> String { + let (data, response) = try await URLSession.shared.data(from: url) + let status = (response as? HTTPURLResponse)?.statusCode + + guard status == 200 else { + throw CoverageCommandError.unexpectedStatus(url, status) + } + + return String(decoding: data, as: UTF8.self) + } +} + +// MARK: - CoverageCommandError + +enum CoverageCommandError: Error, CustomStringConvertible { + case unexpectedStatus(URL, Int?) + + var description: String { + switch self { + case .unexpectedStatus(let url, let status): + "\(url.absoluteString) answered with status \(status.map(String.init) ?? "unknown")" + } + } +} diff --git a/Sources/FreeAgentTools/FreeAgentTools.swift b/Sources/FreeAgentTools/FreeAgentTools.swift index 453473a..5108afe 100644 --- a/Sources/FreeAgentTools/FreeAgentTools.swift +++ b/Sources/FreeAgentTools/FreeAgentTools.swift @@ -1,13 +1,14 @@ import ArgumentParser @main -struct FreeAgentTools: ParsableCommand { +struct FreeAgentTools: AsyncParsableCommand { static let configuration = CommandConfiguration( commandName: "FreeAgentTools", abstract: "Development tools for the FreeAgent OpenAPI spec", subcommands: [ BundleCommand.self, ChangedSuitesCommand.self, + CoverageCommand.self, ] ) } diff --git a/Sources/FreeAgentTools/Models/CoverageReport.swift b/Sources/FreeAgentTools/Models/CoverageReport.swift new file mode 100644 index 0000000..3e3636f --- /dev/null +++ b/Sources/FreeAgentTools/Models/CoverageReport.swift @@ -0,0 +1,93 @@ +import Foundation + +// MARK: - CoverageReport + +struct CoverageReport: CustomStringConvertible { + + // MARK: Lifecycle + + init(docsURL: URL, pages: [DocsPage], spec: [Endpoint]) { + var owners = [String: String]() + let largestFirst = pages.enumerated().sorted { lhs, rhs in + lhs.element.endpoints.count != rhs.element.endpoints.count + ? lhs.element.endpoints.count > rhs.element.endpoints.count + : lhs.offset < rhs.offset + } + for (_, page) in largestFirst { + for endpoint in page.endpoints where owners[endpoint.key] == nil { + owners[endpoint.key] = page.slug + } + } + + let specKeys = Set(spec.map(\.key)) + self.docsURL = docsURL + sections = pages.compactMap { page in + let owned = page.endpoints.filter { owners[$0.key] == page.slug } + guard !owned.isEmpty else { + return nil + } + + return CoverageSection( + slug: page.slug, + title: page.title, + covered: owned.filter { specKeys.contains($0.key) }, + missing: owned.filter { !specKeys.contains($0.key) } + ) + } + undocumented = spec + .filter { owners[$0.key] == nil } + .sorted { ($0.path, $0.method) < ($1.path, $1.method) } + } + + // MARK: Internal + + let docsURL: URL + let sections: [CoverageSection] + let undocumented: [Endpoint] + + var description: String { + let documented = sections.reduce(0) { $0 + $1.covered.count + $1.missing.count } + let covered = sections.reduce(0) { $0 + $1.covered.count } + let percent = documented == 0 ? 0 : Int((Double(covered) / Double(documented) * 100).rounded()) + + var lines = [ + "# FreeAgent API Coverage", + "", + "The spec covers \(covered) of the \(documented) endpoints documented at \(docsURL.absoluteString) (\(percent)%).", + "", + "| Page | Documented | Covered | Missing |", + "| --- | ---: | ---: | ---: |", + ] + lines += sections.map { section in + let link = "[\(section.title)](\(docsURL.appending(path: section.slug).absoluteString))" + let documented = section.covered.count + section.missing.count + + return "| \(link) | \(documented) | \(section.covered.count) | \(section.missing.count) |" + } + + let missing = sections.filter { !$0.missing.isEmpty } + if !missing.isEmpty { + lines += ["", "## Missing Endpoints"] + for section in missing { + lines += ["", "### \(section.title)", ""] + lines += section.missing.map { "- `\($0)`" } + } + } + + if !undocumented.isEmpty { + lines += ["", "## Spec Endpoints Not In The Docs", ""] + lines += undocumented.map { "- `\($0)`" } + } + + return lines.joined(separator: "\n") + "\n" + } +} + +// MARK: - CoverageSection + +struct CoverageSection: Equatable { + let slug: String + let title: String + let covered: [Endpoint] + let missing: [Endpoint] +} diff --git a/Sources/FreeAgentTools/Models/DocsPage.swift b/Sources/FreeAgentTools/Models/DocsPage.swift new file mode 100644 index 0000000..8dda2d1 --- /dev/null +++ b/Sources/FreeAgentTools/Models/DocsPage.swift @@ -0,0 +1,45 @@ +import Foundation + +struct DocsPage: Sendable { + + // MARK: Lifecycle + + init(slug: String, title: String, endpoints: [Endpoint]) { + self.slug = slug + self.title = title + self.endpoints = endpoints + } + + init(slug: String, title: String, html: String) { + var seen = Set() + let endpoints = html.matches(of: Self.endpointPattern) + .map { Endpoint(method: String($0.output.1), path: String($0.output.2)) } + .filter { seen.insert($0.key).inserted } + + self.init(slug: slug, title: title, endpoints: endpoints) + } + + // MARK: Internal + + let slug: String + let title: String + let endpoints: [Endpoint] + + static func links(inIndex html: String) -> [(slug: String, title: String)] { + var seen = Set() + + return html.matches(of: linkPattern) + .map { (slug: String($0.output.1), title: String($0.output.2).replacingOccurrences(of: "&", with: "&")) } + .filter { seen.insert($0.slug).inserted } + } + + // MARK: Private + + private static var linkPattern: Regex<(Substring, Substring, Substring)> { + #/
  • ([^<]+)
    • /# + } + + private static var endpointPattern: Regex<(Substring, Substring, Substring)> { + #/]*>(GET|POST|PUT|PATCH|DELETE)\s+https://api(?:\.sandbox)?\.freeagent\.com(/v2/[^\s?<"]*)/# + } +} diff --git a/Sources/FreeAgentTools/Models/Endpoint.swift b/Sources/FreeAgentTools/Models/Endpoint.swift new file mode 100644 index 0000000..badf1fa --- /dev/null +++ b/Sources/FreeAgentTools/Models/Endpoint.swift @@ -0,0 +1,25 @@ +struct Endpoint: Hashable, Sendable, CustomStringConvertible { + + // MARK: Internal + + let method: String + let path: String + + var key: String { + let segments = path.split(separator: "/", omittingEmptySubsequences: false).map { segment in + Self.isPlaceholder(segment) ? "{}" : String(segment) + } + + return "\(method) \(segments.joined(separator: "/"))" + } + + var description: String { + "\(method) \(path)" + } + + // MARK: Private + + private static func isPlaceholder(_ segment: Substring) -> Bool { + segment.hasPrefix(":") || segment.hasPrefix("{") || (!segment.isEmpty && segment.allSatisfy(\.isNumber)) + } +} diff --git a/Sources/FreeAgentTools/Models/SpecGraph.swift b/Sources/FreeAgentTools/Models/SpecGraph.swift index 8a68e1c..df77a72 100644 --- a/Sources/FreeAgentTools/Models/SpecGraph.swift +++ b/Sources/FreeAgentTools/Models/SpecGraph.swift @@ -44,6 +44,19 @@ struct SpecGraph { return node } + static func node(at pointer: String, in root: Node) throws -> Node { + var node = root + for key in pointer.dropFirst(2).split(separator: "/") { + guard let next = node[String(key)] else { + throw SpecGraphError.missingPointer(pointer) + } + + node = next + } + + return node + } + func selection(changedFiles: [URL], baseRoot: Node?) -> SuiteSelection { var changed = Set() for location in changedFiles.map({ SpecLocation(file: $0) }) { @@ -169,19 +182,6 @@ struct SpecGraph { return SpecLocation(file: URL(fileURLWithPath: file, relativeTo: location.file.deletingLastPathComponent())) } - private static func node(at pointer: String, in root: Node) throws -> Node { - var node = root - for key in pointer.dropFirst(2).split(separator: "/") { - guard let next = node[String(key)] else { - throw SpecGraphError.missingPointer(pointer) - } - - node = next - } - - return node - } - private func closure(of location: SpecLocation) -> Set { var visited = Set() var pending = [location] diff --git a/Sources/FreeAgentTools/Models/SpecPaths.swift b/Sources/FreeAgentTools/Models/SpecPaths.swift new file mode 100644 index 0000000..ab7cc98 --- /dev/null +++ b/Sources/FreeAgentTools/Models/SpecPaths.swift @@ -0,0 +1,40 @@ +import Foundation +import Yams + +enum SpecPaths { + + // MARK: Internal + + static func endpoints(rootURL: URL) throws -> [Endpoint] { + let root = try SpecGraph.load(rootURL) + var endpoints = [Endpoint]() + for (path, item) in root["paths"]?.mapping ?? [:] { + guard let path = path.string else { + continue + } + + let operations = try item["$ref"]?.string.map { try node(at: $0, relativeTo: rootURL) } ?? item + for method in methods where operations[method] != nil { + endpoints.append(Endpoint(method: method.uppercased(), path: path)) + } + } + + return endpoints + } + + // MARK: Private + + private static let methods = ["get", "post", "put", "patch", "delete"] + + private static func node(at ref: String, relativeTo rootURL: URL) throws -> Node { + let parts = ref.split(separator: "#", maxSplits: 1, omittingEmptySubsequences: false) + let file = URL(fileURLWithPath: String(parts[0]), relativeTo: rootURL.deletingLastPathComponent()) + let document = try SpecGraph.load(file) + + guard parts.count == 2 else { + return document + } + + return try SpecGraph.node(at: "#\(parts[1])", in: document) + } +} diff --git a/Tests/FreeAgentToolsTests/Models/CoverageReportTests.swift b/Tests/FreeAgentToolsTests/Models/CoverageReportTests.swift new file mode 100644 index 0000000..10806c0 --- /dev/null +++ b/Tests/FreeAgentToolsTests/Models/CoverageReportTests.swift @@ -0,0 +1,85 @@ +import Foundation +import Testing + +@testable import FreeAgentTools + +struct CoverageReportTests { + + // MARK: Internal + + @Test("splits each page's endpoints into covered and missing") + func sections() { + let report = report() + + #expect(report.sections.map(\.slug) == ["bills", "tasks"]) + #expect(report.sections[0].covered.map(\.description) == ["GET /v2/bills", "GET /v2/bills/:id"]) + #expect(report.sections[0].missing.map(\.description) == ["DELETE /v2/bills/:id"]) + } + + @Test("lists an endpoint documented on several pages under the page documenting the most") + func owner() { + let report = report() + + #expect(report.sections.allSatisfy { $0.slug != "introduction" }) + #expect(report.sections[1].covered.map(\.description) == ["GET /v2/tasks"]) + } + + @Test("lists the spec endpoints no page documents") + func undocumented() { + #expect(report().undocumented.map(\.description) == ["POST /v2/invoice_items", "DELETE /v2/tasks/{id}"]) + } + + @Test("writes a Markdown summary, the missing endpoints and the undocumented ones") + func markdown() { + #expect(report().description == """ + # FreeAgent API Coverage + + The spec covers 3 of the 4 endpoints documented at https://dev.freeagent.com/docs (75%). + + | Page | Documented | Covered | Missing | + | --- | ---: | ---: | ---: | + | [Bills](https://dev.freeagent.com/docs/bills) | 3 | 2 | 1 | + | [Tasks](https://dev.freeagent.com/docs/tasks) | 1 | 1 | 0 | + + ## Missing Endpoints + + ### Bills + + - `DELETE /v2/bills/:id` + + ## Spec Endpoints Not In The Docs + + - `POST /v2/invoice_items` + - `DELETE /v2/tasks/{id}` + + """) + } + + // MARK: Private + + private func report() -> CoverageReport { + let pages = [ + DocsPage(slug: "introduction", title: "Introduction", endpoints: [ + Endpoint(method: "GET", path: "/v2/bills") + ]), + DocsPage(slug: "bills", title: "Bills", endpoints: [ + Endpoint(method: "GET", path: "/v2/bills"), + Endpoint(method: "GET", path: "/v2/bills/:id"), + Endpoint(method: "DELETE", path: "/v2/bills/:id"), + ]), + DocsPage(slug: "tasks", title: "Tasks", endpoints: [ + Endpoint(method: "GET", path: "/v2/tasks") + ]), + DocsPage(slug: "currencies", title: "Currencies", endpoints: []), + ] + let spec = [ + Endpoint(method: "GET", path: "/v2/bills"), + Endpoint(method: "GET", path: "/v2/bills/{id}"), + Endpoint(method: "GET", path: "/v2/tasks"), + Endpoint(method: "DELETE", path: "/v2/tasks/{id}"), + Endpoint(method: "POST", path: "/v2/invoice_items"), + ] + + return CoverageReport(docsURL: URL(string: "https://dev.freeagent.com/docs")!, pages: pages, spec: spec) + } +} diff --git a/Tests/FreeAgentToolsTests/Models/DocsPageTests.swift b/Tests/FreeAgentToolsTests/Models/DocsPageTests.swift new file mode 100644 index 0000000..dd46e21 --- /dev/null +++ b/Tests/FreeAgentToolsTests/Models/DocsPageTests.swift @@ -0,0 +1,52 @@ +import Testing + +@testable import FreeAgentTools + +struct DocsPageTests { + + // MARK: Internal + + @Test("reads the sidebar's page links in order, once each") + func links() { + let links = DocsPage.links(inIndex: Self.index) + + #expect(links.map(\.slug) == ["introduction", "bills", "profit_and_loss"]) + #expect(links.map(\.title) == ["Introduction", "Bills", "Profit & Loss"]) + } + + @Test("reads each endpoint at the start of a code block, without its query, once each") + func endpoints() { + let page = DocsPage(slug: "bills", title: "Bills", html: Self.bills) + + #expect(page.endpoints.map(\.description) == [ + "GET /v2/bills", + "GET /v2/bills/:id", + "POST /v2/bills", + "DELETE /v2/bills/:id", + ]) + } + + // MARK: Private + + private static let index = """ +
  • + Index +
    • +
  • Learn more about OAuth 2.0
    • +
  • Introduction
    • +
  • Bills
    • +
  • Profit & Loss
    • +
  • Bills
    • +
  • API Terms
    • + """ + + private static let bills = """ +

    List all bills

    GET https://api.freeagent.com/v2/bills
+        
    GET https://api.freeagent.com/v2/bills?view=open
+

    Get a single bill

    GET  https://api.freeagent.com/v2/bills/:id
+

    Create a bill

    POST https://api.sandbox.freeagent.com/v2/bills
+

    Send a GET https://api.freeagent.com/v2/contacts request first.

    +

    Delete a bill

    DELETE https://api.freeagent.com/v2/bills/:id
+
    + """ +} diff --git a/Tests/FreeAgentToolsTests/Models/EndpointTests.swift b/Tests/FreeAgentToolsTests/Models/EndpointTests.swift new file mode 100644 index 0000000..e010ed5 --- /dev/null +++ b/Tests/FreeAgentToolsTests/Models/EndpointTests.swift @@ -0,0 +1,28 @@ +import Testing + +@testable import FreeAgentTools + +struct EndpointTests { + @Test( + "matches a documented placeholder or example ID with a spec placeholder", + arguments: ["/v2/tasks/:id", "/v2/tasks/{id}", "/v2/tasks/1"] + ) + func placeholder(path: String) { + #expect(Endpoint(method: "GET", path: path).key == "GET /v2/tasks/{}") + } + + @Test("keeps literal segments and the method apart") + func literals() { + let timeline = Endpoint(method: "GET", path: "/v2/invoices/timeline") + + #expect(timeline.key == "GET /v2/invoices/timeline") + #expect(timeline.key != Endpoint(method: "PUT", path: "/v2/invoices/timeline").key) + } + + @Test("names placeholders in every position") + func nestedPlaceholders() { + let endpoint = Endpoint(method: "PUT", path: "/v2/users/:user_id/self_assessment_returns/{period_ends_on}/mark_as_filed") + + #expect(endpoint.key == "PUT /v2/users/{}/self_assessment_returns/{}/mark_as_filed") + } +} diff --git a/Tests/FreeAgentToolsTests/Models/SpecPathsTests.swift b/Tests/FreeAgentToolsTests/Models/SpecPathsTests.swift new file mode 100644 index 0000000..aae3538 --- /dev/null +++ b/Tests/FreeAgentToolsTests/Models/SpecPathsTests.swift @@ -0,0 +1,70 @@ +import Foundation +import Testing + +@testable import FreeAgentTools + +struct SpecPathsTests { + + // MARK: Lifecycle + + init() throws { + directory = FileManager.default.temporaryDirectory.appending(path: UUID().uuidString) + try write(Self.root, to: "openapi.yaml") + try write(Self.bills, to: "paths/bills.yaml") + } + + // MARK: Internal + + @Test("reads the operations of every path, through a reference or inline") + func endpoints() throws { + let endpoints = try SpecPaths.endpoints(rootURL: directory.appending(path: "openapi.yaml")) + + #expect(endpoints.map(\.description) == [ + "GET /v2/bills", + "POST /v2/bills", + "GET /v2/bills/{id}", + "PUT /v2/bills/{id}", + "DELETE /v2/bills/{id}", + "GET /v2/company", + ]) + } + + // MARK: Private + + private static let root = """ + openapi: 3.1.0 + paths: + /v2/bills: + $ref: 'paths/bills.yaml#/list' + /v2/bills/{id}: + $ref: 'paths/bills.yaml#/show' + /v2/company: + get: + operationId: companyDetails + """ + + private static let bills = """ + list: + get: + operationId: listBills + post: + operationId: createBill + show: + get: + operationId: showBill + put: + operationId: updateBill + delete: + operationId: deleteBill + parameters: + - $ref: '#/components/parameters/ID' + """ + + private let directory: URL + + private func write(_ contents: String, to path: String) throws { + let url = directory.appending(path: path) + try FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true) + try contents.write(to: url, atomically: true, encoding: .utf8) + } +}