diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76cba2c0..cff505bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ jobs: fail-fast: false matrix: ocaml-compiler: ["4.14.x", "5.3.x"] + package: [mldoc, mldoc-cli, mldoc-js, mldoc-bench] steps: - uses: actions/checkout@v4 @@ -27,10 +28,22 @@ jobs: dune-cache: true - name: Install dependencies - run: opam install . --deps-only --with-doc --with-test + run: | + opam pin add -yn mldoc . + if [ "${{ matrix.package }}" = mldoc ]; then + opam install ./mldoc.opam --with-doc --with-test + else + opam install ./${{ matrix.package }}.opam + fi - - name: Build - run: opam exec -- dune build @install @JS @main + - uses: actions/setup-node@v4 + if: matrix.package == 'mldoc-js' + with: + node-version: '22' + + - name: Test installed package + run: opam exec -- sh test/package_smoke.sh ${{ matrix.package }} "$(opam var prefix)" - name: Test - run: opam exec -- dune runtest + if: matrix.package == 'mldoc' + run: opam exec -- dune runtest -p mldoc diff --git a/bench/dune b/bench/dune index 1a60565e..20265961 100644 --- a/bench/dune +++ b/bench/dune @@ -1,15 +1,28 @@ (executable (name bench) + (public_name mldoc-bench) + (package mldoc-bench) (libraries angstrom mldoc core core_bench core_unix.command_unix)) (executable (name time_parse) + (public_name mldoc-time-parse) + (package mldoc-bench) (libraries unix mldoc)) (executable (name time_graph) + (public_name mldoc-time-graph) + (package mldoc-bench) (libraries unix mldoc)) (alias (name bench) (deps bench.exe time_parse.exe time_graph.exe)) + +(install + (package mldoc-bench) + (section share) + (files + (../examples/doc.org as examples/doc.org) + (../examples/syntax.md as examples/syntax.md))) diff --git a/bin/dune b/bin/dune index 3edc0771..419e0590 100644 --- a/bin/dune +++ b/bin/dune @@ -1,6 +1,7 @@ (executable (name main) (public_name mldoc) + (package mldoc-cli) (libraries angstrom mldoc cmdliner lwt.unix)) (alias diff --git a/deploy-npm-package b/deploy-npm-package index 856c9b72..35c0fa14 100755 --- a/deploy-npm-package +++ b/deploy-npm-package @@ -1,6 +1,7 @@ #!/bin/sh -make +set -e +make js cp _build/default/js/lib.bc.js ./js/package/index.js cd ./js/package && npm publish diff --git a/js/dune b/js/dune index 2b634e8c..cda6a125 100644 --- a/js/dune +++ b/js/dune @@ -7,6 +7,11 @@ (js_of_ocaml (flags --target-env browser)) (libraries angstrom js_of_ocaml mldoc)) +(install + (package mldoc-js) + (section share) + (files (lib.bc.js as index.js))) + (alias (name JS) (deps lib.bc.js)) diff --git a/makefile b/makefile index f6c53b18..c8debe9e 100644 --- a/makefile +++ b/makefile @@ -1,24 +1,39 @@ DUNE ?= dune +OPAM ?= opam +.DEFAULT_GOAL := core all: - opam install --deps-only . - $(DUNE) build --profile=release @install @JS @main @bench + $(OPAM) install --working-dir --deps-only ./mldoc.opam ./mldoc-cli.opam ./mldoc-js.opam ./mldoc-bench.opam + $(DUNE) build -p mldoc,mldoc-cli,mldoc-js,mldoc-bench @install @JS @main @bench -check: tests +core: + $(OPAM) install --working-dir --deps-only ./mldoc.opam + $(DUNE) build -p mldoc @install + +cli: + $(OPAM) install --working-dir --deps-only ./mldoc.opam ./mldoc-cli.opam + $(DUNE) build -p mldoc,mldoc-cli @install @main + +js: + $(OPAM) install --working-dir --deps-only ./mldoc.opam ./mldoc-js.opam + $(DUNE) build -p mldoc,mldoc-js @install @JS + +check: test test: - opam install --deps-only -t . - $(DUNE) runtest + $(OPAM) install --working-dir --deps-only -t ./mldoc.opam + $(DUNE) runtest -p mldoc clean: dune clean -run: +run: cli $(DUNE) exec ./bin/main.exe bench: - $(DUNE) exec ./bench/bench.exe + $(OPAM) install --working-dir --deps-only ./mldoc.opam ./mldoc-bench.opam + $(DUNE) build -p mldoc,mldoc-bench @install @bench fmt: dune build @fmt --auto-promote -.PHONY: pin test all clean check bench fmt +.PHONY: core cli js test all clean check run bench fmt diff --git a/mldoc-bench.opam b/mldoc-bench.opam new file mode 100644 index 00000000..d25f1065 --- /dev/null +++ b/mldoc-bench.opam @@ -0,0 +1,20 @@ +opam-version: "2.0" +synopsis: "Benchmark and timing tools for Mldoc" +maintainer: ["logseq team"] +authors: ["tiensonqin " "RCmerci "] +license: "AGPL-3.0-only" +homepage: "https://github.com/logseq/mldoc" +bug-reports: "https://github.com/logseq/mldoc/issues" +available: os != "win32" +depends: [ + "dune" {>= "3.5"} + "ocaml" {>= "4.14.0"} + "mldoc" {= version} + "core" + "core_bench" + "core_unix" +] +build: [ + ["dune" "subst"] {dev} + ["dune" "build" "-p" name "-j" jobs "@install"] +] diff --git a/mldoc-cli.opam b/mldoc-cli.opam new file mode 100644 index 00000000..f2dd7cb1 --- /dev/null +++ b/mldoc-cli.opam @@ -0,0 +1,18 @@ +opam-version: "2.0" +synopsis: "Native command-line converter for Mldoc" +maintainer: ["logseq team"] +authors: ["tiensonqin " "RCmerci "] +license: "AGPL-3.0-only" +homepage: "https://github.com/logseq/mldoc" +bug-reports: "https://github.com/logseq/mldoc/issues" +depends: [ + "dune" {>= "3.5"} + "ocaml" {>= "4.14.0"} + "mldoc" {= version} + "cmdliner" {>= "1.1.0"} + "lwt" +] +build: [ + ["dune" "subst"] {dev} + ["dune" "build" "-p" name "-j" jobs "@install"] +] diff --git a/mldoc-js.opam b/mldoc-js.opam new file mode 100644 index 00000000..e070b1a1 --- /dev/null +++ b/mldoc-js.opam @@ -0,0 +1,18 @@ +opam-version: "2.0" +synopsis: "JavaScript export of Mldoc" +maintainer: ["logseq team"] +authors: ["tiensonqin " "RCmerci "] +license: "AGPL-3.0-only" +homepage: "https://github.com/logseq/mldoc" +bug-reports: "https://github.com/logseq/mldoc/issues" +depends: [ + "dune" {>= "3.5"} + "ocaml" {>= "4.14.0"} + "mldoc" {= version} + "js_of_ocaml" {>= "5.2.0"} + "js_of_ocaml-ppx" {build} +] +build: [ + ["dune" "subst"] {dev} + ["dune" "build" "-p" name "-j" jobs "@install"] +] diff --git a/mldoc.opam b/mldoc.opam index 95793d3f..79c804d6 100644 --- a/mldoc.opam +++ b/mldoc.opam @@ -5,22 +5,17 @@ description: "Another Org-mode and Markdown parser implemented using OCaml and Angstrom." maintainer: ["logseq team"] authors: ["tiensonqin " "RCmerci "] +license: "AGPL-3.0-only" homepage: "https://github.com/logseq/mldoc" bug-reports: "https://github.com/logseq/mldoc/issues" depends: [ "dune" {>= "3.5"} "ocaml" {>= "4.14.0"} "angstrom" {= "dev"} - "cmdliner" {>= "1.1.0"} - "core" {os != "win32"} - "core_bench" {os != "win32"} - "core_unix" {os != "win32"} - "js_of_ocaml" {>= "5.2.0"} - "js_of_ocaml-ppx" + "bigstringaf" "ppx_deriving_yojson" "uri" "yojson" - "lwt" "xmlm" {= "dev"} "odoc" {with-doc} "alcotest" {with-test} @@ -42,6 +37,6 @@ build: [ ] ] pin-depends: [ - [ "angstrom.dev" "git+https://github.com/logseq/angstrom#fork" ] + [ "angstrom.dev" "git+https://github.com/logseq/angstrom#3be9b966dc2bc9ccf9948d17a7b0df1cb526de15" ] [ "xmlm.dev" "git+https://github.com/logseq/xmlm#eb469d536e98c98f2754c0ff8813c92b3a17fe9f" ] ] diff --git a/readme.org b/readme.org index f672c4a6..7e962598 100644 --- a/readme.org +++ b/readme.org @@ -6,12 +6,64 @@ - [[https://logseq.com][Logseq]], a local-first notes app (In development) ** Build + The OCaml library remains named =mldoc=, with the same public modules and + parsing/export interfaces. Tools are installed through separate opam packages: + + | Package | Purpose and installed artifacts | + |---------------+----------------------------------------------------------------------| + | =mldoc= | OCaml library; no CLI, JS toolchain, or Core benchmark dependencies | + | =mldoc-cli= | Native =mldoc= command; adds Cmdliner and Lwt | + | =mldoc-js= | =$(opam var mldoc-js:share)/index.js=; adds js_of_ocaml and its PPX | + | =mldoc-bench= | =mldoc-bench=, =mldoc-time-parse=, =mldoc-time-graph=; adds Core packages | + + Companion packages require the matching version of the core library. + Core keeps the Angstrom lookahead/continuation fork and Xmlm attribute-newline + fork, both pinned to commits. Only the benchmark package is unavailable on + Windows because it depends on Jane Street's native benchmark tools. + #+begin_src sh - # compile + # default: build only the library make - # run test + # independently build optional targets + make cli + make js + make bench + + # all targets (includes native benchmarks) + make all + + # existing library regression tests make test + + # install the library from this checkout + opam install ./mldoc.opam + + # register this checkout's core, then install one matching companion + opam pin add -yn --kind=path mldoc . + opam install ./mldoc-cli.opam + # or: ./mldoc-js.opam / ./mldoc-bench.opam + #+end_src + + Use an explicit manifest for a single package. =opam install .= selects all + packages in the checkout. Make's optional targets build the local core and + selected tool together; a companion's opam build uses the installed library. + + The JS opam package provides the existing =Mldoc= JavaScript export; it does + not install the npm command wrapper. The npm package and its Node CLI remain + available as before. =deploy-npm-package= builds only the JS target. + + Installed benchmarks include their example inputs. Run fixture-generating + timing tools in a writable scratch directory, and pass an explicit synthetic + or intended graph directory to =mldoc-time-graph=: + + #+begin_src sh + bench_dir=$(mktemp -d) + cp -R "$(opam var mldoc-bench:share)/examples" "$bench_dir/" + cd "$bench_dir" + mldoc-bench -help + mldoc-time-parse + mldoc-time-graph /path/to/benchmark-fixture #+end_src ** Run utop @@ -35,8 +87,8 @@ *** For OCaml users #+BEGIN_SRC sh - # compile - make + # compile the native CLI + make cli # convert org file to html _build/default/bin/main.exe ./test.org -o ./test.html diff --git a/test/package_js_smoke.js b/test/package_js_smoke.js new file mode 100644 index 00000000..3a939b6c --- /dev/null +++ b/test/package_js_smoke.js @@ -0,0 +1,29 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const artifact = process.argv[2]; +const config = JSON.stringify({ format: 'Markdown', keep_line_break: false }); + +function check(mldoc) { + assert.ok(mldoc, 'existing Mldoc export'); + for (const input of ['', '- Package smoke λ with [[page]]\r\n']) { + const ast = JSON.parse(mldoc.parseJson(input, config)); + assert.ok(Array.isArray(ast)); + if (input) assert.ok(ast.length > 0); + } + assert.ok(Array.isArray(JSON.parse(mldoc.parseInlineJson('[[page]] λ', config)))); + const opml = JSON.parse(mldoc.parseOPML( + 'smoke' + + '' + )); + assert.ok(JSON.stringify(opml).includes('line1\\nline2'), 'OPML attribute newline'); + assert.ok(mldoc.export('markdown', '- Package smoke λ\n', config, '{}').includes('Package smoke')); +} + +check(require(artifact).Mldoc); +const browser = vm.createContext({ console, setTimeout, clearTimeout, TextEncoder, TextDecoder }); +vm.runInContext(fs.readFileSync(artifact, 'utf8'), browser, { filename: artifact }); +check(browser.Mldoc); +console.log('installed JS export (Node and browser context): OK'); diff --git a/test/package_smoke.sh b/test/package_smoke.sh new file mode 100644 index 00000000..be9fb19d --- /dev/null +++ b/test/package_smoke.sh @@ -0,0 +1,91 @@ +#!/bin/sh +set -eu + +# Exercise installed artifacts, outside the source/build tree. +package=${1:?usage: package_smoke.sh PACKAGE PREFIX} +prefix=${2:?usage: package_smoke.sh PACKAGE PREFIX} +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM + +case "$package" in + mldoc) + test -f "$prefix/lib/mldoc/META" + test ! -e "$prefix/bin/mldoc" + test ! -e "$prefix/bin/mldoc-bench" + test ! -e "$prefix/share/mldoc-js/index.js" + cat > "$work/dune-project" <<'EOF' +(lang dune 3.5) +EOF + cat > "$work/dune" <<'EOF' +(executable (name probe) (libraries mldoc)) +EOF + cat > "$work/probe.ml" <<'EOF' +let config format outline = + match Mldoc.Conf.of_yojson + (`Assoc [ "format", `String format; + "keep_line_break", `Bool false; + "parse_outline_only", `Bool outline ]) with + | Ok c -> c + | Error e -> failwith e + +let check format outline input = + let ast = Mldoc.Parser.parse (config format outline) input in + if input <> "" && ast = [] then failwith "empty parse"; + match Mldoc.Type.blocks_of_yojson (Mldoc.Type.blocks_to_yojson ast) with + | Ok decoded when decoded = ast -> () + | _ -> failwith "AST JSON round trip" + +let () = + List.iter (fun outline -> + List.iter (check "Markdown" outline) + [ ""; "- Package smoke λ with [[page]]\r\n id:: package-smoke\r\n" ]; + List.iter (check "Org" outline) + [ ""; "* Package smoke λ\r\nOrg paragraph\r\n" ]) [false; true]; + print_endline "installed library: OK" +EOF + OCAMLPATH="$prefix/lib${OCAMLPATH:+:$OCAMLPATH}" \ + dune exec --root "$work" ./probe.exe + ;; + mldoc-cli) + command_path="$prefix/bin/mldoc" + test -x "$command_path" + # The existing native CLI selects Org for stdin, so compare an Org file. + printf '%s\r\n' '* Package smoke λ' 'second line' > "$work/input.org" + "$command_path" "$work/input.org" -b markdown -o - > "$work/file.out" + "$command_path" - -b markdown -o - < "$work/input.org" > "$work/stdin.out" + cmp "$work/file.out" "$work/stdin.out" + test -s "$work/file.out" + "$command_path" "$work/input.org" -b markdown -o "$work/output.md" + cmp "$work/file.out" "$work/output.md" + "$command_path" - -b markdown -o - < /dev/null > "$work/empty.out" + if "$command_path" --not-a-mldoc-option > "$work/error.out" 2>&1; then + echo 'invalid CLI option unexpectedly succeeded' >&2 + exit 1 + fi + echo 'installed CLI: OK' + ;; + mldoc-js) + node "$script_dir/package_js_smoke.js" "$prefix/share/mldoc-js/index.js" + ;; + mldoc-bench) + mkdir "$work/examples" "$work/graph" "$work/graph/pages" + cp "$prefix/share/mldoc-bench/examples/doc.org" "$work/examples/doc.org" + cp "$prefix/share/mldoc-bench/examples/syntax.md" "$work/examples/syntax.md" + printf '%s\n' '- Synthetic package fixture' > "$work/graph/pages/page.md" + ( + cd "$work" + "$prefix/bin/mldoc-bench" -help > help.out + "$prefix/bin/mldoc-time-parse" > parse.out + "$prefix/bin/mldoc-time-graph" "$work/graph" > graph.out + test -s help.out + test -s parse.out + test -s graph.out + ) + echo 'installed benchmarks (synthetic fixtures): OK' + ;; + *) + echo "unknown package: $package" >&2 + exit 2 + ;; +esac