diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05af975e..82c86ff4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,3 +96,34 @@ jobs: # Must run from the repo root: with python/ as cwd, the source anydoc/ # package dir shadows the installed compiled module. - run: python -m unittest discover -s python/tests + + go: + name: Go bindings + runs-on: blacksmith-8vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - uses: actions/setup-go@v6 + with: + go-version: '1.22' + - name: Build the static library + run: cargo build -p anydoc-go --release + - name: Place the archive where cgo expects it + run: | + mkdir -p go/lib/linux_amd64_gnu + cp target/release/libanydoc_go.a go/lib/linux_amd64_gnu/ + - name: Committed header is up to date + # `cargo build` regenerates include/anydoc.h via cbindgen; any diff + # means the committed header is stale and must be rebuilt and committed. + run: | + if ! git diff --exit-code -- go/include/anydoc.h; then + echo "::error::go/include/anydoc.h is out of date. Run 'cargo build -p anydoc-go --release' and commit the result." + exit 1 + fi + - name: go vet + run: go vet ./... + working-directory: go + - name: go test + run: go test ./... + working-directory: go diff --git a/.github/workflows/prepare-go-archives.yml b/.github/workflows/prepare-go-archives.yml new file mode 100644 index 00000000..281dd4fe --- /dev/null +++ b/.github/workflows/prepare-go-archives.yml @@ -0,0 +1,86 @@ +name: Prepare Go Archives + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: Build Go archive (${{ matrix.target }}) + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-apple-darwin + host: blacksmith-6vcpu-macos-latest + lib_dir: darwin_amd64 + lib_name: libanydoc_go.a + - target: aarch64-apple-darwin + host: blacksmith-6vcpu-macos-latest + lib_dir: darwin_arm64 + lib_name: libanydoc_go.a + - target: x86_64-pc-windows-msvc + host: blacksmith-8vcpu-windows-2025 + lib_dir: windows_amd64 + lib_name: anydoc_go.lib + - target: x86_64-unknown-linux-gnu + host: blacksmith-8vcpu-ubuntu-2404 + lib_dir: linux_amd64_gnu + lib_name: libanydoc_go.a + - target: aarch64-unknown-linux-gnu + host: blacksmith-4vcpu-ubuntu-2404-arm + lib_dir: linux_arm64_gnu + lib_name: libanydoc_go.a + - target: x86_64-unknown-linux-musl + host: blacksmith-8vcpu-ubuntu-2404 + lib_dir: linux_amd64_musl + lib_name: libanydoc_go.a + zig: true + - target: aarch64-unknown-linux-musl + host: blacksmith-4vcpu-ubuntu-2404-arm + lib_dir: linux_arm64_musl + lib_name: libanydoc_go.a + zig: true + runs-on: ${{ matrix.host }} + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - uses: Swatinem/rust-cache@v2 + with: + key: go-${{ matrix.target }} + - uses: mlugg/setup-zig@v2 + if: matrix.zig == true + with: + version: 0.14.1 + - run: cargo build -p anydoc-go --release --target ${{ matrix.target }} + - name: Stage archive + shell: bash + run: | + mkdir -p staging/go/lib/${{ matrix.lib_dir }} + cp target/${{ matrix.target }}/release/${{ matrix.lib_name }} \ + staging/go/lib/${{ matrix.lib_dir }}/${{ matrix.lib_name }} + - uses: actions/upload-artifact@v7 + with: + name: go-lib-${{ matrix.target }} + path: staging + if-no-files-found: error + + bundle: + name: Bundle Go archives + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v8 + with: + pattern: go-lib-* + path: staging + merge-multiple: true + - uses: actions/upload-artifact@v7 + with: + name: go-libraries + path: staging + if-no-files-found: error diff --git a/.github/workflows/release-go.yml b/.github/workflows/release-go.yml new file mode 100644 index 00000000..c5c8ab6e --- /dev/null +++ b/.github/workflows/release-go.yml @@ -0,0 +1,82 @@ +name: Release Go Module + +on: + push: + tags: ['go/v*'] + workflow_dispatch: + +permissions: + contents: read + +jobs: + gate: + name: Version gate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Check versions + run: | + if [ "$GITHUB_REF_TYPE" = "tag" ]; then + sh scripts/check-versions.sh "v${GITHUB_REF_NAME#go/v}" + else + sh scripts/check-versions.sh + fi + + verify: + name: Verify Go archive (${{ matrix.target }}) + needs: gate + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-apple-darwin + host: blacksmith-6vcpu-macos-latest + lib_dir: darwin_amd64 + lib_name: libanydoc_go.a + - target: aarch64-apple-darwin + host: blacksmith-6vcpu-macos-latest + lib_dir: darwin_arm64 + lib_name: libanydoc_go.a + - target: x86_64-pc-windows-msvc + host: blacksmith-8vcpu-windows-2025 + lib_dir: windows_amd64 + lib_name: anydoc_go.lib + - target: x86_64-unknown-linux-gnu + host: blacksmith-8vcpu-ubuntu-2404 + lib_dir: linux_amd64_gnu + lib_name: libanydoc_go.a + - target: aarch64-unknown-linux-gnu + host: blacksmith-4vcpu-ubuntu-2404-arm + lib_dir: linux_arm64_gnu + lib_name: libanydoc_go.a + - target: x86_64-unknown-linux-musl + host: blacksmith-8vcpu-ubuntu-2404 + lib_dir: linux_amd64_musl + lib_name: libanydoc_go.a + zig: true + - target: aarch64-unknown-linux-musl + host: blacksmith-4vcpu-ubuntu-2404-arm + lib_dir: linux_arm64_musl + lib_name: libanydoc_go.a + zig: true + runs-on: ${{ matrix.host }} + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - uses: Swatinem/rust-cache@v2 + with: + key: go-${{ matrix.target }} + - uses: mlugg/setup-zig@v2 + if: matrix.zig == true + with: + version: 0.14.1 + - run: cargo build -p anydoc-go --release --target ${{ matrix.target }} + - name: Compare bundled archive + shell: bash + run: | + built=target/${{ matrix.target }}/release/${{ matrix.lib_name }} + bundled=go/lib/${{ matrix.lib_dir }}/${{ matrix.lib_name }} + test -f "$bundled" + cmp "$built" "$bundled" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 09c73f9b..e8e10854 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: version: ${{ steps.check.outputs.version }} steps: - uses: actions/checkout@v7 - - name: All five version locations agree (and match the tag) + - name: All seven version locations agree (and match the tag) id: check run: | if [ "$GITHUB_REF_TYPE" = "tag" ]; then @@ -442,11 +442,126 @@ jobs: # Tolerate re-running a release where PyPI already got some files. skip-existing: true + build-go: + name: Build Go static lib (${{ matrix.target }}) + needs: gate + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-apple-darwin + host: blacksmith-6vcpu-macos-latest + lib_dir: darwin_amd64 + lib_name: libanydoc_go.a + - target: aarch64-apple-darwin + host: blacksmith-6vcpu-macos-latest + lib_dir: darwin_arm64 + lib_name: libanydoc_go.a + - target: x86_64-pc-windows-msvc + host: blacksmith-8vcpu-windows-2025 + lib_dir: windows_amd64 + lib_name: anydoc_go.lib + # napi-cross is Node-specific; the Rust staticlib links glibc on the + # build host. manylinux2014 compatibility comes from building on + # ubuntu-2404 with the toolchain's default glibc target. + - target: x86_64-unknown-linux-gnu + host: blacksmith-8vcpu-ubuntu-2404 + lib_dir: linux_amd64_gnu + lib_name: libanydoc_go.a + - target: aarch64-unknown-linux-gnu + host: blacksmith-4vcpu-ubuntu-2404-arm + lib_dir: linux_arm64_gnu + lib_name: libanydoc_go.a + # musl targets cross-compile with the Zig toolchain, mirroring the + # Node build. The asset name carries the full triple so gnu and musl + # never collide on the release page. + - target: x86_64-unknown-linux-musl + host: blacksmith-8vcpu-ubuntu-2404 + zig: true + lib_dir: linux_amd64_musl + lib_name: libanydoc_go.a + - target: aarch64-unknown-linux-musl + host: blacksmith-4vcpu-ubuntu-2404-arm + zig: true + lib_dir: linux_arm64_musl + lib_name: libanydoc_go.a + runs-on: ${{ matrix.host }} + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - uses: Swatinem/rust-cache@v2 + with: + key: go-${{ matrix.target }} + - uses: mlugg/setup-zig@v2 + if: matrix.zig == true + with: + version: 0.14.1 + - name: Build staticlib + run: cargo build -p anydoc-go --release --target ${{ matrix.target }} + - name: Verify packaged archive + shell: bash + run: | + built=target/${{ matrix.target }}/release/${{ matrix.lib_name }} + packaged=go/lib/${{ matrix.lib_dir }}/${{ matrix.lib_name }} + test -f "$packaged" + cmp "$built" "$packaged" + + test-go: + name: Test Go bindings (${{ matrix.target }}) + needs: build-go + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + host: blacksmith-6vcpu-macos-latest + # Apple Silicon runs x64 Go through Rosetta, which loads the x64 lib. + - target: x86_64-apple-darwin + host: blacksmith-6vcpu-macos-latest + go_arch: amd64 + - target: x86_64-pc-windows-msvc + host: blacksmith-4vcpu-windows-2025 + # The Linux flavors test through `docker run` rather than container: + # jobs — JavaScript actions cannot run inside Alpine containers on + # arm64 runners. + - target: x86_64-unknown-linux-gnu + host: blacksmith-4vcpu-ubuntu-2404 + image: golang:1.22-slim + - target: aarch64-unknown-linux-gnu + host: blacksmith-4vcpu-ubuntu-2404-arm + image: golang:1.22-slim + - target: x86_64-unknown-linux-musl + host: blacksmith-4vcpu-ubuntu-2404 + image: golang:1.22-alpine + - target: aarch64-unknown-linux-musl + host: blacksmith-4vcpu-ubuntu-2404-arm + image: golang:1.22-alpine + runs-on: ${{ matrix.host }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v6 + if: ${{ !matrix.image }} + with: + go-version: '1.22' + architecture: ${{ matrix.go_arch || '' }} + - name: go test + if: ${{ !matrix.image }} + run: go test ./... + working-directory: go + - name: go test in container + if: ${{ matrix.image }} + run: > + docker run --rm -v ${{ github.workspace }}:/build -w /build/go + ${{ matrix.image }} sh -c + 'go test ${{ contains(matrix.target, 'musl') && '-tags musl' || '' }} ./...' + # The release page appears last: only once every registry publish has - # succeeded, carrying the Node addons, the wheels, and the sdist. + # succeeded, carrying the Node addons, wheels, and sdist. github-release: name: GitHub release - needs: [publish-npm, publish-npm-wasm, publish-crates, publish-pypi] + needs: [publish-npm, publish-npm-wasm, publish-crates, publish-pypi, test-go] if: github.event_name == 'push' runs-on: ubuntu-latest permissions: diff --git a/.gitignore b/.gitignore index ad264179..bc1bd8f6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ __pycache__/ .venv/ # maturin build output from the CI python job /python/wheels/ +# Go archives are generated by the maintainer-only release-preparation step. +# The archive payload lands in a follow-up release-prep PR. +/go/lib/ # wasm-pack build output: pkg/ is the npm package, www/pkg/ the demo site copy /wasm/pkg/ /wasm/www/pkg/ diff --git a/Cargo.lock b/Cargo.lock index f1bf05e8..0d393386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -104,6 +104,14 @@ dependencies = [ "zip", ] +[[package]] +name = "anydoc-go" +version = "0.1.6" +dependencies = [ + "anydoc", + "cbindgen", +] + [[package]] name = "anydoc-node" version = "0.1.0" @@ -228,6 +236,25 @@ dependencies = [ "cipher", ] +[[package]] +name = "cbindgen" +version = "0.29.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ecb53484c9c167ba674026b656d8a27d7657a58e6066aa902bfb1a4aa00ae20" +dependencies = [ + "clap", + "heck", + "indexmap", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.119", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.4.0" @@ -288,6 +315,33 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "codepage" version = "0.1.2" @@ -1125,9 +1179,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc153f5fd745cc038b5eed86622125969f8a39834a57bc96beaaf2512b1da729" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "libc", "once_cell", @@ -1139,18 +1193,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb77d9aa6d647507b55c69ee714d266d84c526c78ff0bc6dd8757f58591e64d1" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3160087aa5733bce7d9a729f8c55f85a2a53b74742a09613178eeebff0722253" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -1158,9 +1212,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91f9d455db760a9a0b0ddeaac25f1390b8a36ba73dfbda9f127cac6fc340d4d5" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1170,9 +1224,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.29.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e343bcec300ff262f5806a33a4e51b6d097a8a46435f512fcb83e95592581625" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", @@ -1262,9 +1316,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1365,6 +1419,28 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1422,6 +1498,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "syn" version = "2.0.119" @@ -1528,6 +1610,45 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow 1.0.4", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "ttf-parser" version = "0.25.1" @@ -1730,6 +1851,18 @@ dependencies = [ "windows-link", ] +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + [[package]] name = "zip" version = "8.6.0" @@ -1746,9 +1879,15 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.6" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zmij" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zopfli" diff --git a/Cargo.toml b/Cargo.toml index 3d4eb542..eefdc3cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["node", "python", "wasm"] +members = ["node", "python", "wasm", "go"] # cargo-fuzz keeps its own lockfile and nightly-only dependencies. exclude = ["fuzz"] diff --git a/README.md b/README.md index 21738c85..85c5e9c5 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,11 @@ [![Crates.io](https://img.shields.io/crates/v/anydoc.svg)](https://crates.io/crates/anydoc) [![npm](https://img.shields.io/npm/v/@firecrawl/anydoc.svg)](https://www.npmjs.com/package/@firecrawl/anydoc) [![PyPI](https://img.shields.io/pypi/v/firecrawl-anydoc.svg)](https://pypi.org/project/firecrawl-anydoc/) +[![Go Reference](https://pkg.go.dev/badge/github.com/firecrawl/anydoc/go.svg)](https://pkg.go.dev/github.com/firecrawl/anydoc/go) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![skills.sh](https://skills.sh/b/firecrawl/anydoc)](https://skills.sh/firecrawl/anydoc) -Fast Rust library that converts documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF) into clean GitHub-Flavored Markdown. Includes bindings for [Node.js](node/README.md), [Python](python/README.md), and the [browser](wasm/README.md) (WebAssembly). +Fast Rust library that converts documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF) into clean GitHub-Flavored Markdown. Includes bindings for [Node.js](node/README.md), [Python](python/README.md), [Go](go/README.md), and the [browser](wasm/README.md) (WebAssembly). Built by [Firecrawl](https://firecrawl.dev) to turn any office document into LLM-ready Markdown in single-digit milliseconds, with one consistent output no matter which format goes in. It powers [Firecrawl Parse](https://firecrawl.dev/parse), so if you'd rather not run it yourself, the hosted API gives you the same conversion plus our OCR models for the scanned pages anydoc can't read on its own. @@ -82,6 +83,31 @@ document = anydoc.to_document(data) > Full API reference: [python/README.md](python/README.md) +### Go + +```bash +go get github.com/firecrawl/anydoc/go@latest +``` + +```go +import anydoc "github.com/firecrawl/anydoc/go" + +// From a file path: +markdown, err := anydoc.ToMarkdown("report.docx") + +// From bytes, with the format detected from the content: +markdown, err = anydoc.ToMarkdownBytes(data, nil) + +// Or name it, which signature-less formats (CSV) need: +csv := anydoc.FormatCsv +markdown, err = anydoc.ToMarkdownBytes(data, &csv) + +// Or stop at the document model, which also carries embedded assets: +document, err := anydoc.ToDocument(data, nil) +``` + +> Full API reference: [go/README.md](go/README.md) + ### Browser (WebAssembly) ```bash @@ -132,7 +158,7 @@ let document = anydoc::to_document(&bytes, None)?; - **Embedded assets.** Images and embedded objects render as their alt text in the Markdown, and the raw bytes stay available on the document model, tagged with their media type. Images with an external URL become ordinary Markdown images. - **Content-based format detection.** The format is read from the bytes themselves (PDF header, RTF open group, OLE stream names, ZIP package mimetype), so mislabeled files still convert correctly. - **Fast.** Pure Rust, no ML models, no external services. Median conversion time is under 5ms per document. -- **Bindings that stay out of the way.** Node.js conversion runs on the libuv thread pool and never blocks the event loop; Python releases the GIL so other threads keep running. TypeScript types and Python stubs ship with the packages. +- **Bindings that stay out of the way.** Node.js conversion runs on the libuv thread pool and never blocks the event loop; Python releases the GIL so other threads keep running; Go links a prebuilt static library via cgo with no Rust toolchain required. TypeScript types and Python stubs ship with the packages. - **PDF support built in.** Text-based PDFs convert locally through [pdf-inspector](https://github.com/firecrawl/pdf-inspector), no OCR service required. - **Agent ready.** Ships as an [Agent Skill](#agent-skill): one `npx skills add firecrawl/anydoc` and any agent can read office documents. @@ -197,7 +223,7 @@ Format::from_extension("pptm"); // Some(Format::Pptx) Format::from_path(Path::new("report.odt")); // Some(Format::Odt) ``` -The same three functions exist in Node (`formatFromBytes`, ...) and Python (`anydoc.format_from_bytes`, ...). +The same three functions exist in Node (`formatFromBytes`, ...), Python (`anydoc.format_from_bytes`, ...), and Go (`anydoc.FormatFromBytes`, ...). ## Errors @@ -252,16 +278,20 @@ Because every format funnels through the same document model and serializer, out cargo test cd node && npm install && npm run build && npm test cd python && pip install maturin && maturin develop && python -m unittest discover -s tests +cd go && cargo build -p anydoc-go --release && go test ./... wasm-pack build wasm --release --target web --scope firecrawl && node --test wasm/test.mjs # see wasm/README.md ``` A committed fixture corpus under `tests/fixtures/` is snapshot-tested, `tests/robustness.rs` mutation-tests every fixture, and `fuzz/` carries cargo-fuzz targets per format. The speed and quality benchmark lives in [`bench/`](bench/README.md). -Releases are tagged `v`, which publishes the crate, the npm package, and the PyPI wheels from [`.github/workflows/release.yml`](.github/workflows/release.yml). The version lives in three places, bumped together for a release: +Root releases are tagged `v` for the crate, npm package, and PyPI wheels. The Go submodule is tagged separately as `go/v` after its bundled archives are prepared. The version lives in six places, bumped together for a release: - [`Cargo.toml`](Cargo.toml): the crate - [`node/package.json`](node/package.json): the npm package - [`python/Cargo.toml`](python/Cargo.toml): the wheel (`python/pyproject.toml` reads it) +- [`go/Cargo.toml`](go/Cargo.toml): the Go staticlib crate +- [`go/version.go`](go/version.go): the Go module version +- [`node/index.js`](node/index.js): the generated Node loader version guard (regenerated by `npm run build`) ## License diff --git a/go/Cargo.toml b/go/Cargo.toml new file mode 100644 index 00000000..137420ff --- /dev/null +++ b/go/Cargo.toml @@ -0,0 +1,22 @@ +# Go bindings for anydoc. +# +# `crate-type = ["staticlib"]` produces libanydoc_go.a, which the Go module +# links at build time via cgo. Not a cdylib: cgo wants an archive it can pull +# symbols from, not a loadable shared object. +[package] +name = "anydoc-go" +version = "0.1.6" +edition = "2024" +description = "Go bindings (C ABI) for anydoc" +license = "MIT" +repository = "https://github.com/firecrawl/anydoc" +publish = false + +[lib] +crate-type = ["staticlib"] + +[dependencies] +anydoc = { path = ".." } + +[build-dependencies] +cbindgen = "0.29" diff --git a/go/README.md b/go/README.md new file mode 100644 index 00000000..05e9ff99 --- /dev/null +++ b/go/README.md @@ -0,0 +1,162 @@ +# anydoc-go + +[![Go Reference](https://pkg.go.dev/badge/github.com/firecrawl/anydoc/go.svg)](https://pkg.go.dev/github.com/firecrawl/anydoc/go) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](../LICENSE) + +Go bindings for [anydoc](../README.md): convert documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF) to GitHub-Flavored Markdown, with full access to the parsed document model and embedded assets. + +## Install + +```bash +go get github.com/firecrawl/anydoc/go@latest +``` + +cgo must be enabled (`CGO_ENABLED=1`, the default when a C compiler is present). The module ships Rust static libraries for every supported platform, so no Rust toolchain or download step is required. + +Install the CLI with: + +```bash +go install github.com/firecrawl/anydoc/go/cmd/anydoc@latest +``` + +### musl (Alpine) + +On musl-based distros (Alpine, Azul), build with `-tags musl` so cgo selects the bundled musl archive: + +```bash +go build -tags musl ./... +``` + +### Building from source + +If you have a Rust toolchain and want to replace an archive, build its target +and copy the output to the matching directory. For example, on Apple Silicon: + +```bash +cargo build -p anydoc-go --release +cp target/release/libanydoc_go.a go/lib/darwin_arm64/ +``` + +Release archives live under `lib/`: `darwin_`, `windows_amd64`, and +`linux__{gnu,musl}`. They are committed to the module and release CI +rebuilds and compares every archive before publishing. + +### Preparing a release + +Maintainers run **Prepare Go Archives** from the Actions tab after the binding +code is merged. Its `go-libraries` artifact has the final `go/lib/...` layout. +Commit that payload in a release-prep PR, then create a `go/vX.Y.Z` tag. The +**Release Go Module** workflow rebuilds and compares every bundled archive. + +Regenerate the C header after ABI changes: + +```bash +sh scripts/gen-headers.sh +``` + +## Quick start + +```go +import anydoc "github.com/firecrawl/anydoc/go" + +// From a file path: +markdown, err := anydoc.ToMarkdown("report.docx") + +// From bytes, with the format detected from the content: +markdown, err = anydoc.ToMarkdownBytes(bytes, nil) + +// Or name it, which signature-less formats (CSV) need: +csv := anydoc.FormatCsv +markdown, err = anydoc.ToMarkdownBytes(csvBytes, &csv) + +// Or stop at the document model, which also carries embedded assets: +document, err := anydoc.ToDocument(bytes, nil) +for _, asset := range document.Assets { + _ = asset.Data // raw image/object bytes +} +``` + +## Command line + +After `go install`, run: + +```bash +anydoc report.docx +anydoc - --format csv < data.csv +``` + +The command follows the Node CLI: it accepts one file path or `-` for stdin, +uses `--format` for signature-less input, writes Markdown to stdout, and sends +only errors to stderr. + +## API + +### Format detection + +```go +anydoc.FormatFromBytes(data) // (Format, bool) +anydoc.FormatFromExtension(".pptm") // (Format, bool) +anydoc.FormatFromPath("report.odt") // (Format, bool) +``` + +### Conversion + +| Function | Description | +| --- | --- | +| `ToMarkdown(path string) (string, error)` | Convert a file to Markdown; format detected from content, extension as fallback. | +| `ToMarkdownBytes(data []byte, format *Format) (string, error)` | Convert in-memory bytes; `nil` format auto-detects. | +| `ToDocument(data []byte, format *Format) (*Document, error)` | Parse into the document model with embedded assets. Unsupported for PDF. | + +### Document model + +Concrete structs with a `Kind` tag, identical in shape to the [Node](../node/README.md) and [Python](../python/README.md) bindings: + +- `Document` — `Blocks`, `Notes`, `Assets` +- `Block` — `heading`, `paragraph`, `list`, `table`, `block_quote`, `code_block`, `rule` +- `Inline` — `text`, `link`, `image`, `anchor`, `note_ref`, `line_break` +- `Style` — `Bold`, `Italic`, `Strike`, `Code` +- `LinkTarget` — `external`, `relative`, `anchor` +- `ImageSource` — `external`, `asset`, `unavailable` +- `List` / `ListItem` — `marker` (`bullet`, `decimal`, `lower_alpha`, ...), `start`, `items` +- `Table` / `CellSlot` / `Cell` — canonical grid with `origin` and `covered` slots +- `Note` — `footnote`, `endnote` +- `Asset` — `id`, `media_type`, `origin_part`, `data` + +Every struct is plain Go, easy to JSON-encode, with no interface dispatch overhead. Variant enums surface as the same lowercase `Kind` strings the other bindings use. + +### Errors + +Conversion errors are typed `*anydoc.ConvertError` with `Kind` and `Detail` fields. Conversion kinds match the other bindings: `unsupported`, `malformed`, `encrypted`, `resource_limit`, `missing_part`, `io`, `pdf_no_model`. Go validates explicit formats before crossing the ABI and reports `unknown_format` for invalid `Format` values. + +```go +var ce *anydoc.ConvertError +if errors.As(err, &ce) { + fmt.Println(ce.Kind, ce.Detail) +} +``` + +## Supported platforms + +Prebuilt `libanydoc_go.a` assets are published for: + +| GOOS | GOARCH | +| --- | --- | +| darwin | arm64, amd64 | +| linux | amd64, arm64 (glibc and musl) | +| windows | amd64 | + +Windows cgo requires a C toolchain (MSVC or mingw). The Rust static lib is MSVC by default on Windows; cgo on Windows defaults to MSVC too, so they agree. + +## Development + +```bash +cargo build -p anydoc-go --release +cp target/release/libanydoc_go.a go/lib/$(go env GOOS)_$(go env GOARCH)/ +go test ./... +``` + +The C ABI surface lives in [`src/lib.rs`](src/lib.rs) with `#[repr(C)]` DTOs in [`src/model.rs`](src/model.rs). The committed header [`include/anydoc.h`](include/anydoc.h) is regenerated by `cargo build` via `cbindgen`; CI verifies it is up to date. + +## License + +[MIT](../LICENSE) diff --git a/go/anydoc.go b/go/anydoc.go new file mode 100644 index 00000000..bbfe54a7 --- /dev/null +++ b/go/anydoc.go @@ -0,0 +1,140 @@ +// Package anydoc converts documents (Word, PowerPoint, Excel, OpenDocument, +// RTF, EPUB, CSV, and PDF) to GitHub-Flavored Markdown, with full access to +// the parsed document model and embedded assets. +// +// This is the Go binding. It links a Rust static library through cgo; the +// module packages archives for every supported platform. See the README for +// the supported platform matrix and musl build tag. +package anydoc + +/* +#cgo CFLAGS: -I${SRCDIR}/include +#cgo darwin,arm64 LDFLAGS: -L${SRCDIR}/lib/darwin_arm64 -lanydoc_go -lm -lstdc++ +#cgo darwin,amd64 LDFLAGS: -L${SRCDIR}/lib/darwin_amd64 -lanydoc_go -lm -lstdc++ +#cgo windows,amd64 LDFLAGS: -L${SRCDIR}/lib/windows_amd64 -lanydoc_go + +#include +#include +#include +#include "include/anydoc.h" +*/ +import "C" + +import ( + "fmt" + "unsafe" +) + +// ToMarkdown converts a document file to Markdown. The format is detected +// from the file content; the extension is the fallback for signature-less +// formats (CSV) and unrecognizable containers. +func ToMarkdown(path string) (string, error) { + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + var out *C.char + var outLen C.uintptr_t + code := C.anydoc_to_markdown(cpath, &out, &outLen) + if code != errOK { + return "", convertError(code) + } + defer C.anydoc_string_free(out) + markdown, err := cStringN(out, outLen) + if err != nil { + return "", err + } + return markdown, nil +} + +// ToMarkdownBytes converts an in-memory document to Markdown. Pass a Format +// to select the parser, or nil to detect it from the content, which +// signature-less formats (CSV) have to name explicitly. +func ToMarkdownBytes(data []byte, format *Format) (string, error) { + if len(data) == 0 { + return "", &ConvertError{Kind: "unsupported", Detail: "empty input"} + } + tag := C.int(C.ANYDOC_FORMAT_NONE) + if format != nil { + tag = formatToTag(*format) + if tag == C.int(C.ANYDOC_FORMAT_NONE) { + return "", &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)} + } + } + var out *C.char + var outLen C.uintptr_t + code := C.anydoc_to_markdown_bytes((*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &out, &outLen) + if code != errOK { + return "", convertError(code) + } + defer C.anydoc_string_free(out) + markdown, err := cStringN(out, outLen) + if err != nil { + return "", err + } + return markdown, nil +} + +// ToDocument parses an in-memory document into the document model, which also +// carries the embedded assets. Pass a Format to select the parser, or nil to +// detect it from the content. +// +// Unsupported for PDF: PDF conversion produces Markdown directly and has no +// document-model form; use ToMarkdownBytes. +func ToDocument(data []byte, format *Format) (*Document, error) { + if len(data) == 0 { + return nil, &ConvertError{Kind: "unsupported", Detail: "empty input"} + } + tag := C.int(C.ANYDOC_FORMAT_NONE) + if format != nil { + tag = formatToTag(*format) + if tag == C.int(C.ANYDOC_FORMAT_NONE) { + return nil, &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)} + } + } + var buf *C.uint8_t + var bufLen C.uintptr_t + code := C.anydoc_to_document((*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &buf, &bufLen) + if code != errOK { + return nil, convertError(code) + } + defer C.anydoc_buffer_free(buf, bufLen) + if buf == nil || bufLen == 0 { + return &Document{}, nil + } + // Copy the C buffer into a Go-owned slice before decoding, so the C + // buffer can be freed immediately rather than tying its lifetime to the + // decoded Document. + raw, err := cGoBytes(unsafe.Pointer(buf), bufLen) + if err != nil { + return nil, err + } + return decodeDocument(raw) +} + +// cStringN reads a length-prefixed C buffer as a Go string. The bytes are +// copied, so the caller may free the C buffer immediately after. +func cStringN(s *C.char, n C.uintptr_t) (string, error) { + if s == nil || n == 0 { + return "", nil + } + b, err := cGoBytes(unsafe.Pointer(s), n) + if err != nil { + return "", err + } + return string(b), nil +} + +func cGoBytes(ptr unsafe.Pointer, n C.uintptr_t) ([]byte, error) { + // C.GoBytes accepts a C int length even on 64-bit Go builds. + if uint64(n) > 1<<31-1 { + return nil, &ConvertError{Kind: "resource_limit", Detail: fmt.Sprintf("FFI output is too large for Go: %d bytes", n)} + } + return C.GoBytes(ptr, C.int(n)), nil +} + +// cString reads a NUL-terminated C string as a Go string. +func cString(s *C.char) string { + if s == nil { + return "" + } + return C.GoString(s) +} diff --git a/go/anydoc_go_test.go b/go/anydoc_go_test.go new file mode 100644 index 00000000..6de34440 --- /dev/null +++ b/go/anydoc_go_test.go @@ -0,0 +1,332 @@ +// Smoke test: the bindings load and every entry point round-trips a fixture. +// +//go:build cgo + +package anydoc + +import ( + "bytes" + "errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +var ( + fixtureDir = filepath.Join("..", "tests", "fixtures") + outlineDocx = filepath.Join(fixtureDir, "docx", "handmade-outline.docx") + richDocx = filepath.Join(fixtureDir, "docx", "handmade-rich.docx") + csvSheet = filepath.Join(fixtureDir, "csv", "sheet.csv") +) + +func mustRead(t *testing.T, path string) []byte { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return b +} + +func TestToMarkdownDetectsFormatFromFileContent(t *testing.T) { + md, err := ToMarkdown(outlineDocx) + if err != nil { + t.Fatalf("ToMarkdown: %v", err) + } + if !regexp.MustCompile(`(?m)^# `).MatchString(md) { + t.Fatalf("expected a top-level heading, got:\n%s", md) + } +} + +func TestToMarkdownBytesConvertsInMemory(t *testing.T) { + docx := FormatDocx + md, err := ToMarkdownBytes(mustRead(t, richDocx), &docx) + if err != nil { + t.Fatalf("ToMarkdownBytes: %v", err) + } + if !strings.Contains(md, "| Quarter | Widgets |") { + t.Fatalf("expected the table header, got:\n%s", md) + } +} + +func TestToMarkdownBytesDetectsFormatWhenNoneNamed(t *testing.T) { + md, err := ToMarkdownBytes(mustRead(t, richDocx), nil) + if err != nil { + t.Fatalf("ToMarkdownBytes: %v", err) + } + if !strings.Contains(md, "| Quarter | Widgets |") { + t.Fatalf("expected the table header, got:\n%s", md) + } + + // CSV carries no signature, so it has to be named. + if _, err := ToMarkdownBytes(mustRead(t, csvSheet), nil); err == nil { + t.Fatalf("expected an error for unnamed CSV") + } + csv := FormatCsv + mdCsv, err := ToMarkdownBytes(mustRead(t, csvSheet), &csv) + if err != nil { + t.Fatalf("ToMarkdownBytes csv: %v", err) + } + if !strings.Contains(mdCsv, "| --- |") { + t.Fatalf("expected a CSV table, got:\n%s", mdCsv) + } +} + +func TestToDocumentExposesDocumentModel(t *testing.T) { + docx := FormatDocx + doc, err := ToDocument(mustRead(t, outlineDocx), &docx) + if err != nil { + t.Fatalf("ToDocument: %v", err) + } + var heading *Block + for i := range doc.Blocks { + if doc.Blocks[i].Kind == "heading" { + heading = &doc.Blocks[i] + break + } + } + if heading == nil { + t.Fatalf("no heading block in document: %+v", doc.Blocks) + } + if heading.Level == nil || *heading.Level < 1 || *heading.Level > 6 { + t.Fatalf("heading level out of range: %v", heading.Level) + } + if len(heading.Content) == 0 { + t.Fatalf("heading has no content") + } + if heading.Content[0].Kind != "text" || heading.Content[0].Text == nil { + t.Fatalf("first inline is not text: %+v", heading.Content[0]) + } + if heading.Content[0].Style == nil { + t.Fatalf("first inline has no style") + } +} + +func TestToDocumentCarriesEmbeddedAssetsAsBytes(t *testing.T) { + docx := FormatDocx + doc, err := ToDocument(mustRead(t, richDocx), &docx) + if err != nil { + t.Fatalf("ToDocument: %v", err) + } + imageIndex := -1 + for i := range doc.Assets { + if doc.Assets[i].MediaType == "image/png" { + imageIndex = i + break + } + } + if imageIndex < 0 { + t.Fatalf("no image/png asset in document: %+v", doc.Assets) + } + image := doc.Assets[imageIndex] + if len(image.Data) == 0 { + t.Fatalf("image asset has empty bytes") + } + if image.ID != uint64(imageIndex) { + t.Fatalf("asset id %d != index %d", image.ID, imageIndex) + } +} + +func TestFormatDetectionReadsContentExtensionAndPath(t *testing.T) { + f, ok := FormatFromBytes(mustRead(t, richDocx)) + if !ok || f != FormatDocx { + t.Fatalf("FormatFromBytes(rich) = %q ok=%v, want docx", f, ok) + } + // CSV carries no signature: only the extension names it. + if f, ok := FormatFromBytes(mustRead(t, csvSheet)); ok { + t.Fatalf("FormatFromBytes(csv) = %q ok=%v, want none", f, ok) + } + if f, ok := FormatFromExtension(".pptm"); !ok || f != FormatPptx { + t.Fatalf("FormatFromExtension(.pptm) = %q ok=%v, want pptx", f, ok) + } + if f, ok := FormatFromExtension("xls"); !ok || f != FormatXlsx { + t.Fatalf("FormatFromExtension(xls) = %q ok=%v, want xlsx", f, ok) + } + if f, ok := FormatFromPath("/tmp/report.odt"); !ok || f != FormatOdt { + t.Fatalf("FormatFromPath(report.odt) = %q ok=%v, want odt", f, ok) + } + if f, ok := FormatFromPath("/tmp/report.unknown"); ok { + t.Fatalf("FormatFromPath(report.unknown) = %q ok=%v, want none", f, ok) + } +} + +func TestConversionErrorsRaiseWithCrateErrorMessage(t *testing.T) { + docx := FormatDocx + _, err := ToMarkdownBytes([]byte("not a document"), &docx) + if err == nil { + t.Fatalf("expected an error") + } + var ce *ConvertError + if !errors.As(err, &ce) { + t.Fatalf("expected a *ConvertError, got %T: %v", err, err) + } + if !regexp.MustCompile(`malformed|unsupported`).MatchString(ce.Kind) { + t.Fatalf("error kind %q does not match malformed|unsupported", ce.Kind) + } +} + +func TestUnknownFormatErrors(t *testing.T) { + bad := Format("nope") + _, err := ToMarkdownBytes([]byte("x"), &bad) + if err == nil { + t.Fatalf("expected an error for unknown format") + } + var ce *ConvertError + if !errors.As(err, &ce) || ce.Kind != "unknown_format" { + t.Fatalf("expected unknown_format, got %v", err) + } +} + +func TestPDFToDocumentIsUnsupported(t *testing.T) { + pdfPath := filepath.Join(fixtureDir, "pdf", "text.pdf") + if _, err := os.Stat(pdfPath); err != nil { + t.Skipf("no pdf fixture: %v", err) + } + pdf := FormatPdf + _, err := ToDocument(mustRead(t, pdfPath), &pdf) + if err == nil { + t.Fatalf("expected an error for PDF ToDocument") + } + var ce *ConvertError + if !errors.As(err, &ce) || ce.Kind != "pdf_no_model" { + t.Fatalf("expected pdf_no_model, got %v", err) + } +} + +func TestPDFToMarkdownBytesWorks(t *testing.T) { + pdfPath := filepath.Join(fixtureDir, "pdf", "text.pdf") + if _, err := os.Stat(pdfPath); err != nil { + t.Skipf("no pdf fixture: %v", err) + } + pdf := FormatPdf + md, err := ToMarkdownBytes(mustRead(t, pdfPath), &pdf) + if err != nil { + t.Fatalf("ToMarkdownBytes pdf: %v", err) + } + if len(md) == 0 { + t.Fatalf("expected non-empty markdown") + } +} + +func TestTableGridDecodesWithSlots(t *testing.T) { + docx := FormatDocx + doc, err := ToDocument(mustRead(t, richDocx), &docx) + if err != nil { + t.Fatalf("ToDocument: %v", err) + } + var table *Table + for _, b := range doc.Blocks { + if b.Kind == "table" && b.Table != nil { + table = b.Table + break + } + } + if table == nil { + t.Fatalf("no table in rich docx") + } + if table.Kind != "data" { + t.Fatalf("table kind = %q, want data", table.Kind) + } + if table.HeaderRows == 0 { + t.Fatalf("expected at least one header row") + } + if len(table.Grid) == 0 || len(table.Grid[0]) == 0 { + t.Fatalf("table grid is empty") + } + // First row should be all origins. + for j, slot := range table.Grid[0] { + if slot.Kind != "origin" || slot.Cell == nil { + t.Fatalf("grid[0][%d] = %+v, want an origin cell", j, slot) + } + } +} + +func TestConvertCLI(t *testing.T) { + binary := filepath.Join(t.TempDir(), "anydoc") + if runtime.GOOS == "windows" { + binary += ".exe" + } + build := exec.Command("go", "build", "-o", binary, "./cmd/anydoc") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build anydoc command: %v\n%s", err, output) + } + + run := func(args []string, stdin []byte) (stdout, stderr string, code int) { + cmd := exec.Command(binary, args...) + cmd.Stdin = bytes.NewReader(stdin) + var out, err bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &err + runErr := cmd.Run() + if runErr == nil { + return out.String(), err.String(), 0 + } + exitErr, ok := runErr.(*exec.ExitError) + if !ok { + t.Fatalf("run anydoc %v: %v", args, runErr) + } + return out.String(), err.String(), exitErr.ExitCode() + } + + t.Run("converts a file to stdout", func(t *testing.T) { + stdout, stderr, code := run([]string{outlineDocx}, nil) + if code != 0 || !regexp.MustCompile(`(?m)^# `).MatchString(stdout) || stderr != "" { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + }) + + t.Run("writes output instead of stdout", func(t *testing.T) { + output := filepath.Join(t.TempDir(), "outline.md") + stdout, stderr, code := run([]string{outlineDocx, "-o", output}, nil) + if code != 0 || stdout != "" || stderr != "" { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + markdown := string(mustRead(t, output)) + if !regexp.MustCompile(`(?m)^# `).MatchString(markdown) { + t.Fatalf("output missing heading: %q", markdown) + } + }) + + t.Run("reads stdin with an explicit format", func(t *testing.T) { + stdout, stderr, code := run([]string{"-", "--format", "csv"}, mustRead(t, csvSheet)) + if code != 0 || !strings.Contains(stdout, "| --- |") || stderr != "" { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + }) + + t.Run("returns conversion errors", func(t *testing.T) { + _, stderr, code := run([]string{"no-such-file.docx"}, nil) + if code != 1 || !strings.HasPrefix(stderr, "anydoc: ") { + t.Fatalf("code=%d stderr=%q", code, stderr) + } + }) + + t.Run("returns usage errors", func(t *testing.T) { + for _, args := range [][]string{ + nil, + {"--frmat", "csv", csvSheet}, + {"--format", "nope", csvSheet}, + {outlineDocx, richDocx}, + } { + _, stderr, code := run(args, nil) + if code != 2 || !strings.HasPrefix(stderr, "anydoc: ") { + t.Fatalf("args=%v code=%d stderr=%q", args, code, stderr) + } + } + }) + + t.Run("prints help and version", func(t *testing.T) { + stdout, stderr, code := run([]string{"--help"}, nil) + if code != 0 || !strings.Contains(stdout, "Exit codes:") || stderr != "" { + t.Fatalf("help: code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + stdout, stderr, code = run([]string{"--version"}, nil) + if code != 0 || !regexp.MustCompile(`^\d+\.\d+\.\d+\n$`).MatchString(stdout) || stderr != "" { + t.Fatalf("version: code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + }) +} diff --git a/go/build.rs b/go/build.rs new file mode 100644 index 00000000..7054ba37 --- /dev/null +++ b/go/build.rs @@ -0,0 +1,37 @@ +//! Regenerate `include/anydoc.h` from the crate's C ABI on every build. +//! `cbindgen` is a build-dependency, so this runs whenever `cargo build +//! -p anydoc-go` runs inside the workspace. The committed header is the +//! source of truth for the Go side; consumers link the packaged static archive +//! and never run this. + +use std::path::PathBuf; + +fn main() { + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()); + let include_dir = manifest_dir.join("include"); + let header = include_dir.join("anydoc.h"); + + std::fs::create_dir_all(&include_dir).expect("create include directory"); + + let config = cbindgen::Config { + language: cbindgen::Language::C, + include_guard: Some("ANYDOC_H".to_string()), + autogen_warning: Some( + "/* Warning: this file is generated by `cargo build -p anydoc-go`. Do not edit by hand. */" + .to_string(), + ), + ..Default::default() + }; + + match cbindgen::Builder::new().with_crate(&manifest_dir).with_config(config).generate() { + Ok(bindings) => { + let _ = bindings.write_to_file(&header); + } + Err(err) => { + println!("cargo:warning=cbindgen failed to generate anydoc.h: {err}"); + } + } + + println!("cargo:rerun-if-changed=src/lib.rs"); + println!("cargo:rerun-if-changed=src/model.rs"); +} diff --git a/go/cmd/anydoc/main.go b/go/cmd/anydoc/main.go new file mode 100644 index 00000000..95035d7a --- /dev/null +++ b/go/cmd/anydoc/main.go @@ -0,0 +1,156 @@ +// Command anydoc converts a document to Markdown: +// +// anydoc [options] +// anydoc - [options] < document +package main + +import ( + "errors" + "fmt" + "io" + "os" + "strings" + "syscall" + + anydoc "github.com/firecrawl/anydoc/go" +) + +const formats = "doc, docx, odt, pdf, ppt, pptx, rtf, epub, xlsx, ods, odp, csv" + +const help = `anydoc: convert documents to GitHub-Flavored Markdown + +Usage: + anydoc [options] + anydoc - [options] < document + +Converts one document per invocation and writes the Markdown to stdout. +Pass - as the input to read the document from stdin. Never prompts; all +diagnostics go to stderr. + +Options: + -o, --output Write the Markdown to instead of stdout + -f, --format Name the input format instead of detecting it: + ` + formats + ` + (extension aliases like xls, docm, ppsx resolve + to these) + -h, --help Print this help and exit + -V, --version Print the version and exit + +The format is detected from the file content; the file extension is the +fallback for signature-less formats (CSV). stdin has no extension, so CSV +input from stdin needs --format csv. Scanned or image-only PDFs need OCR, +which anydoc does not do, and error as unsupported. + +Exit codes: + 0 success + 1 the document could not be read or converted + 2 usage error: unknown option, missing input, or invalid --format + +Examples: + anydoc report.docx + anydoc slides.pptx -o slides.md + anydoc - --format csv < data.csv + curl -s https://example.com/paper.pdf | anydoc - +` + +func fail(code int, format string, args ...any) { + fmt.Fprintf(os.Stderr, "anydoc: "+format+"\n", args...) + os.Exit(code) +} + +func main() { + args := os.Args[1:] + var input string + var output string + var formatName string + positionalOnly := false + for i := 0; i < len(args); i++ { + arg := args[i] + if positionalOnly || arg == "-" || !strings.HasPrefix(arg, "-") { + if input != "" { + fail(2, "one document per invocation: unexpected second input '%s'", arg) + } + input = arg + continue + } + if arg == "--" { + positionalOnly = true + continue + } + value := func() string { + if i+1 >= len(args) { + fail(2, "%s requires a value", arg) + } + i++ + return args[i] + } + switch arg { + case "-h", "--help": + fmt.Print(help) + os.Exit(0) + case "-V", "--version": + fmt.Println(anydoc.Version) + os.Exit(0) + case "-o", "--output": + output = value() + case "-f", "--format": + formatName = value() + default: + if strings.HasPrefix(arg, "--output=") { + output = strings.TrimPrefix(arg, "--output=") + } else if strings.HasPrefix(arg, "--format=") { + formatName = strings.TrimPrefix(arg, "--format=") + } else { + fail(2, "unknown option '%s' (see anydoc --help)", arg) + } + } + } + if input == "" { + fail(2, "missing input: pass a document path, or - for stdin (see anydoc --help)") + } + + var format *anydoc.Format + if formatName != "" { + f, ok := anydoc.FormatFromExtension(formatName) + if !ok { + fail(2, "invalid format '%s'; expected one of: %s", formatName, formats) + } + format = &f + } + + var markdown string + var err error + if input == "-" { + if info, statErr := os.Stdin.Stat(); statErr != nil { + fail(1, "%v", statErr) + } else if info.Mode()&os.ModeCharDevice != 0 { + fail(2, "stdin is a terminal; pipe or redirect a document into anydoc -") + } + data, readErr := io.ReadAll(os.Stdin) + if readErr != nil { + fail(1, "%v", readErr) + } + markdown, err = anydoc.ToMarkdownBytes(data, format) + } else if format != nil { + data, readErr := os.ReadFile(input) + if readErr != nil { + fail(1, "%v", readErr) + } + markdown, err = anydoc.ToMarkdownBytes(data, format) + } else { + markdown, err = anydoc.ToMarkdown(input) + } + if err != nil { + fail(1, "%v", err) + } + + if output != "" { + if err := os.WriteFile(output, []byte(markdown), 0o644); err != nil { + fail(1, "%v", err) + } + return + } + if _, err := fmt.Fprint(os.Stdout, markdown); err != nil && !errors.Is(err, syscall.EPIPE) { + fail(1, "%v", err) + } +} diff --git a/go/errors.go b/go/errors.go new file mode 100644 index 00000000..9000e837 --- /dev/null +++ b/go/errors.go @@ -0,0 +1,95 @@ +package anydoc + +/* +#include +#include "include/anydoc.h" +*/ +import "C" + +import "fmt" + +// Error codes mirroring the C ABI (ERR_*). Stable across versions. +const ( + errOK = C.ERR_OK + errUnsupported = C.ERR_UNSUPPORTED + errMalformed = C.ERR_MALFORMED + errEncrypted = C.ERR_ENCRYPTED + errResourceLimit = C.ERR_RESOURCE_LIMIT + errMissingPart = C.ERR_MISSING_PART + errIO = C.ERR_IO + errPDFNoModel = C.ERR_PDF_NO_MODEL + errInvalidArg = C.ERR_INVALID_ARG + errUnknownFormat = C.ERR_UNKNOWN_FORMAT +) + +// ConvertError is the typed error every conversion function returns. It +// carries the same variant kind the Node and Python bindings expose +// ("unsupported", "malformed", "encrypted", ...) plus the crate's +// human-readable detail. +type ConvertError struct { + // Kind is the lowercase variant name, matching the Node and Python + // bindings: "unsupported", "malformed", "encrypted", "resource_limit", + // "missing_part", "io", "pdf_no_model". Go also reports + // "unknown_format" for an invalid explicit Format. + Kind string + // Detail is the crate's Display output for the error. + Detail string +} + +func (e *ConvertError) Error() string { + if e == nil { + return "" + } + if e.Detail == "" { + return "anydoc: " + e.Kind + } + return "anydoc: " + e.Kind + ": " + e.Detail +} + +// convertError builds a ConvertError from an ABI error code, pulling the +// detail message from the thread-local last-error slot. +func convertError(code C.int) error { + if code == errOK { + return nil + } + kind := errorKind(code) + detail := lastError() + return &ConvertError{Kind: kind, Detail: detail} +} + +func errorKind(code C.int) string { + switch code { + case errUnsupported: + return "unsupported" + case errMalformed: + return "malformed" + case errEncrypted: + return "encrypted" + case errResourceLimit: + return "resource_limit" + case errMissingPart: + return "missing_part" + case errIO: + return "io" + case errPDFNoModel: + return "pdf_no_model" + case errInvalidArg: + return "invalid_argument" + case errUnknownFormat: + return "unknown_format" + default: + return fmt.Sprintf("unknown_error(%d)", int(code)) + } +} + +// lastError returns the human-readable message the last ABI call stashed on +// the current OS thread, or "" when there was none. The C side returns a +// freshly allocated string the caller must free. +func lastError() string { + ptr := C.anydoc_last_error() + if ptr == nil { + return "" + } + defer C.anydoc_string_free(ptr) + return cString(ptr) +} diff --git a/go/format.go b/go/format.go new file mode 100644 index 00000000..05bed7c8 --- /dev/null +++ b/go/format.go @@ -0,0 +1,143 @@ +package anydoc + +/* +#include "include/anydoc.h" +*/ +import "C" + +import ( + "strings" + "unsafe" +) + +// Format names a document format. The string values are the lowercase +// extension names, matching the Node and Python bindings ("doc", "docx", +// "pdf", ...). Use the exported constants rather than raw strings where +// possible: the constants are pinned to the C ABI's tag values at build +// time. +type Format string + +// Format constants. Every value is stable across versions. +const ( + FormatDoc Format = "doc" + FormatDocx Format = "docx" + FormatOdt Format = "odt" + FormatPdf Format = "pdf" + FormatPpt Format = "ppt" + FormatPptx Format = "pptx" + FormatRtf Format = "rtf" + FormatEpub Format = "epub" + FormatXlsx Format = "xlsx" + FormatOds Format = "ods" + FormatOdp Format = "odp" + FormatCsv Format = "csv" +) + +// formatToTag maps a Format to its C ABI tag. Returns ANYDOC_FORMAT_NONE for +// an unknown format. +func formatToTag(f Format) C.int { + switch f { + case FormatDoc: + return C.int(C.ANYDOC_FORMAT_DOC) + case FormatDocx: + return C.int(C.ANYDOC_FORMAT_DOCX) + case FormatOdt: + return C.int(C.ANYDOC_FORMAT_ODT) + case FormatPdf: + return C.int(C.ANYDOC_FORMAT_PDF) + case FormatPpt: + return C.int(C.ANYDOC_FORMAT_PPT) + case FormatPptx: + return C.int(C.ANYDOC_FORMAT_PPTX) + case FormatRtf: + return C.int(C.ANYDOC_FORMAT_RTF) + case FormatEpub: + return C.int(C.ANYDOC_FORMAT_EPUB) + case FormatXlsx: + return C.int(C.ANYDOC_FORMAT_XLSX) + case FormatOds: + return C.int(C.ANYDOC_FORMAT_ODS) + case FormatOdp: + return C.int(C.ANYDOC_FORMAT_ODP) + case FormatCsv: + return C.int(C.ANYDOC_FORMAT_CSV) + } + return C.int(C.ANYDOC_FORMAT_NONE) +} + +// tagToFormat maps a C ABI tag back to a Format. ok is false for NONE or an +// unknown tag. +func tagToFormat(tag C.int) (Format, bool) { + switch tag { + case C.ANYDOC_FORMAT_DOC: + return FormatDoc, true + case C.ANYDOC_FORMAT_DOCX: + return FormatDocx, true + case C.ANYDOC_FORMAT_ODT: + return FormatOdt, true + case C.ANYDOC_FORMAT_PDF: + return FormatPdf, true + case C.ANYDOC_FORMAT_PPT: + return FormatPpt, true + case C.ANYDOC_FORMAT_PPTX: + return FormatPptx, true + case C.ANYDOC_FORMAT_RTF: + return FormatRtf, true + case C.ANYDOC_FORMAT_EPUB: + return FormatEpub, true + case C.ANYDOC_FORMAT_XLSX: + return FormatXlsx, true + case C.ANYDOC_FORMAT_ODS: + return FormatOds, true + case C.ANYDOC_FORMAT_ODP: + return FormatOdp, true + case C.ANYDOC_FORMAT_CSV: + return FormatCsv, true + } + return "", false +} + +// FormatFromBytes detects the format from the content itself: the signature +// and identity each container specification designates (PDF header, RTF open +// group, OLE stream names, ZIP package mimetype/content types). Plain-text +// formats (CSV) carry no signature and return the zero Format and ok=false; +// so does anything unrecognized. +func FormatFromBytes(data []byte) (f Format, ok bool) { + if len(data) == 0 { + return "", false + } + var tag C.int + code := C.anydoc_format_from_bytes((*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), &tag) + if code != errOK { + return "", false + } + return tagToFormat(tag) +} + +// FormatFromExtension returns the format a bare extension names (with or +// without a leading dot), matched case-insensitively. ok is false for +// anything unrecognized. +func FormatFromExtension(ext string) (f Format, ok bool) { + ext = strings.TrimPrefix(ext, ".") + cext := C.CString(strings.ToLower(ext)) + defer C.free(unsafe.Pointer(cext)) + var tag C.int + code := C.anydoc_format_from_extension(cext, &tag) + if code != errOK { + return "", false + } + return tagToFormat(tag) +} + +// FormatFromPath returns the format a path's extension names. ok is false +// when the path has no extension or names nothing recognized. +func FormatFromPath(path string) (f Format, ok bool) { + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + var tag C.int + code := C.anydoc_format_from_path(cpath, &tag) + if code != errOK { + return "", false + } + return tagToFormat(tag) +} diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 00000000..d0780b92 --- /dev/null +++ b/go/go.mod @@ -0,0 +1,3 @@ +module github.com/firecrawl/anydoc/go + +go 1.22 diff --git a/go/include/anydoc.h b/go/include/anydoc.h new file mode 100644 index 00000000..86725e65 --- /dev/null +++ b/go/include/anydoc.h @@ -0,0 +1,249 @@ +#ifndef ANYDOC_H +#define ANYDOC_H + +/* Warning: this file is generated by `cargo build -p anydoc-go`. Do not edit by hand. */ + +#include +#include +#include +#include + +/** + * Success. + */ +#define ERR_OK 0 + +/** + * `ConvertError::Unsupported`. + */ +#define ERR_UNSUPPORTED 1 + +/** + * `ConvertError::Malformed`. + */ +#define ERR_MALFORMED 2 + +/** + * `ConvertError::Encrypted`. + */ +#define ERR_ENCRYPTED 3 + +/** + * `ConvertError::ResourceLimit`. + */ +#define ERR_RESOURCE_LIMIT 4 + +/** + * `ConvertError::MissingPart`. + */ +#define ERR_MISSING_PART 5 + +/** + * `ConvertError::Io`. + */ +#define ERR_IO 6 + +/** + * PDF was passed to `anydoc_to_document` (unsupported there). + */ +#define ERR_PDF_NO_MODEL 7 + +/** + * Null pointer or invalid argument passed to the ABI. + */ +#define ERR_INVALID_ARG 8 + +/** + * Unknown format name passed to the ABI. + */ +#define ERR_UNKNOWN_FORMAT 9 + +/** + * C-side format tag. Stable; mirrors the Node/Python lowercase string names + * via `format_name`. `ANYDOC_FORMAT_NONE` is the `Option::None` sentinel. + */ +#define ANYDOC_FORMAT_NONE -1 + +#define ANYDOC_FORMAT_DOC 0 + +#define ANYDOC_FORMAT_DOCX 1 + +#define ANYDOC_FORMAT_ODT 2 + +#define ANYDOC_FORMAT_PDF 3 + +#define ANYDOC_FORMAT_PPT 4 + +#define ANYDOC_FORMAT_PPTX 5 + +#define ANYDOC_FORMAT_RTF 6 + +#define ANYDOC_FORMAT_EPUB 7 + +#define ANYDOC_FORMAT_XLSX 8 + +#define ANYDOC_FORMAT_ODS 9 + +#define ANYDOC_FORMAT_ODP 10 + +#define ANYDOC_FORMAT_CSV 11 + +/** + * Tag values for `Block` variants. Stable across versions; the Go side maps + * these to the same lowercase string names the Node and Python bindings use. + */ +#define BLOCK_HEADING 0 + +#define BLOCK_PARAGRAPH 1 + +#define BLOCK_LIST 2 + +#define BLOCK_TABLE 3 + +#define BLOCK_QUOTE 4 + +#define BLOCK_CODE 5 + +#define BLOCK_RULE 6 + +#define INLINE_TEXT 0 + +#define INLINE_LINK 1 + +#define INLINE_IMAGE 2 + +#define INLINE_ANCHOR 3 + +#define INLINE_NOTEREF 4 + +#define INLINE_LINEBREAK 5 + +#define LINK_EXTERNAL 0 + +#define LINK_RELATIVE 1 + +#define LINK_ANCHOR 2 + +#define IMG_EXTERNAL 0 + +#define IMG_ASSET 1 + +#define IMG_UNAVAILABLE 2 + +#define MARKER_BULLET 0 + +#define MARKER_DECIMAL 1 + +#define MARKER_LOWER_ALPHA 2 + +#define MARKER_UPPER_ALPHA 3 + +#define MARKER_LOWER_ROMAN 4 + +#define MARKER_UPPER_ROMAN 5 + +#define TABLE_DATA 0 + +#define TABLE_LAYOUT 1 + +#define SLOT_ORIGIN 0 + +#define SLOT_COVERED 1 + +#define NOTE_FOOTNOTE 0 + +#define NOTE_ENDNOTE 1 + +/** + * Detect the format from the content. Writes a format tag to `*out` (one of + * `ANYDOC_FORMAT_*`), or `ANYDOC_FORMAT_NONE` when nothing matches. + * + * Returns `ERR_OK` on success (including no-match), `ERR_INVALID_ARG` on null + * pointers. + */ +int anydoc_format_from_bytes(const uint8_t *bytes, uintptr_t len, int *out); + +/** + * The format a bare extension names (no leading dot). Writes a format tag to + * `*out`, or `ANYDOC_FORMAT_NONE` when unrecognized. + */ +int anydoc_format_from_extension(const char *ext, int *out); + +/** + * The format a path's extension names. Writes a format tag to `*out`, or + * `ANYDOC_FORMAT_NONE` when unrecognized. + */ +int anydoc_format_from_path(const char *path, int *out); + +/** + * Convert a document file to Markdown. The format is detected from the file + * content, with the extension as the fallback for signature-less formats + * (CSV). + * + * On success writes a `*mut c_char` to `*out_str` and its byte length to + * `*out_len`. The caller must free the string with `anydoc_string_free`. + */ +int anydoc_to_markdown(const char *path, char **out_str, uintptr_t *out_len); + +/** + * Convert an in-memory document to Markdown. `format_tag` is one of the + * `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the + * content. + */ +int anydoc_to_markdown_bytes(const uint8_t *bytes, + uintptr_t len, + int format_tag, + char **out_str, + uintptr_t *out_len); + +/** + * Parse an in-memory document into the document model. `format_tag` is one of + * the `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the + * content. + * + * On success writes a heap-allocated byte buffer to `*out_buf` and its length + * to `*out_len`. The buffer holds the flat serialization (see `model`). The + * caller must free it with `anydoc_buffer_free`. + * + * Returns `ERR_PDF_NO_MODEL` for PDF input: PDF conversion produces Markdown + * directly and has no document-model form. Use `anydoc_to_markdown_bytes`. + */ +int anydoc_to_document(const uint8_t *bytes, + uintptr_t len, + int format_tag, + uint8_t **out_buf, + uintptr_t *out_len); + +/** + * Free a string returned by `anydoc_to_markdown*` or `anydoc_last_error`. + * Null is a no-op (the empty-string sentinel). + */ +void anydoc_string_free(char *s); + +/** + * Free a document buffer returned by `anydoc_to_document`. Null is a no-op. + * `len` must be the length written to `*out_len` at allocation time; it is + * validated against the header for safety. + */ +void anydoc_buffer_free(uint8_t *buf, uintptr_t len); + +/** + * Return the human-readable message for the last error on the current + * thread. The returned `*mut c_char` is freshly allocated and must be freed + * with `anydoc_string_free`. Returns null when there is no last error. + */ +char *anydoc_last_error(void); + +/** + * Return the lowercase name of a format tag, or null for an unknown tag. + * The returned string must be freed with `anydoc_string_free`. + */ +char *anydoc_format_name(int tag); + +/** + * Parse a lowercase format name into a tag. Writes the tag to `*out`, or + * `ANYDOC_FORMAT_NONE` when the name is unknown. + */ +int anydoc_format_from_name(const char *name, int *out); + +#endif /* ANYDOC_H */ diff --git a/go/lib_linux_gnu.go b/go/lib_linux_gnu.go new file mode 100644 index 00000000..1618280e --- /dev/null +++ b/go/lib_linux_gnu.go @@ -0,0 +1,9 @@ +//go:build linux && !musl + +package anydoc + +/* +#cgo amd64 LDFLAGS: -L${SRCDIR}/lib/linux_amd64_gnu -lanydoc_go -lm -lstdc++ -ldl -lpthread +#cgo arm64 LDFLAGS: -L${SRCDIR}/lib/linux_arm64_gnu -lanydoc_go -lm -lstdc++ -ldl -lpthread +*/ +import "C" diff --git a/go/lib_linux_musl.go b/go/lib_linux_musl.go new file mode 100644 index 00000000..5c710676 --- /dev/null +++ b/go/lib_linux_musl.go @@ -0,0 +1,9 @@ +//go:build linux && musl + +package anydoc + +/* +#cgo amd64 LDFLAGS: -L${SRCDIR}/lib/linux_amd64_musl -lanydoc_go -lm -lstdc++ -ldl -lpthread +#cgo arm64 LDFLAGS: -L${SRCDIR}/lib/linux_arm64_musl -lanydoc_go -lm -lstdc++ -ldl -lpthread +*/ +import "C" diff --git a/go/model.go b/go/model.go new file mode 100644 index 00000000..870d60e6 --- /dev/null +++ b/go/model.go @@ -0,0 +1,712 @@ +package anydoc + +/* +#include "include/anydoc.h" +*/ +import "C" + +import "fmt" + +// Document is a parsed document: its body, its notes, and the bytes of +// everything it embedded. A Document is self-contained — embedded assets +// carry their bytes, so it stays usable after the source archive is gone. +type Document struct { + Blocks []Block + // Notes are footnote and endnote bodies, referenced from text by an + // Inline with Kind == "note_ref". + Notes []Note + // Assets holds every embedded asset, indexed by Asset.ID as referenced + // by an Inline image source. + Assets []Asset +} + +// Block is one block-level piece of a document body. The Kind field selects +// which of the optional fields is populated; the rest are nil. +type Block struct { + // "heading", "paragraph", "list", "table", "block_quote", "code_block", + // or "rule". + Kind string + Level *uint8 // heading: 1-based outline depth + Anchor *string // heading: stable anchor id when the document targets it + Content []Inline // heading, paragraph + List *List // list + Table *Table // table + Blocks []Block // block_quote + Lang *string // code_block + Text *string // code_block +} + +// Inline is one span of inline content. The Kind field selects which optional +// fields are populated. +type Inline struct { + // "text", "link", "image", "anchor", "note_ref", or "line_break". + Kind string + Text *string // text + Style *Style // text + Content []Inline // link + Target *LinkTarget // link + Alt *string // image + Source *ImageSource // image + Anchor *string // anchor: the anchor id + NoteID *string // note_ref: the id of the note in Document.Notes +} + +// Style is a fully resolved character style. +type Style struct { + Bold bool + Italic bool + Strike bool + Code bool +} + +// LinkTarget is where a link points. +type LinkTarget struct { + // "external" (absolute URL with a scheme), "relative" (scheme-less + // relative reference, preserved as written), or "anchor" (internal + // target: a heading anchor or an "anchor" inline). + Kind string + Value string // the URL, relative reference, or anchor id +} + +// ImageSource is where an image's bytes live. +type ImageSource struct { + // "external" (absolute URL), "asset" (embedded image, carried in + // Document.Assets), or "unavailable" (no usable source; only the alt + // text remains). + Kind string + URL *string // external + AssetID *uint64 // asset: index into Document.Assets +} + +// List is a fully resolved list: numbering identity and marker resolution +// happen in the parser frontends. +type List struct { + // "bullet", "decimal", "lower_alpha", "upper_alpha", "lower_roman", or + // "upper_roman". + Marker string + Start uint64 + Items []ListItem +} + +// ListItem is one item of a List, which may hold nested blocks including +// further lists. +type ListItem struct { + Blocks []Block + Checked *bool // task-list state, when the item carries a checkbox + MarkerLabel *string // literal marker text overriding the level marker +} + +// Table is a canonical table grid: every logical grid position appears +// exactly once. Content and spans live on the origin slot, and each position +// a span covers holds a CellSlot with Kind == "covered" pointing back at that +// origin. +type Table struct { + Grid [][]CellSlot + HeaderRows uint32 // number of leading rows that are header rows (0 = none) + Kind string // "data" or "layout" +} + +// CellSlot is one position in a Table grid: either a cell or the shadow of +// one. +type CellSlot struct { + // "origin" (the cell itself) or "covered" (a position swallowed by a + // span, pointing back at the origin that covers it). + Kind string + Cell *Cell // origin + OriginRow *uint32 // covered: row of the covering origin + OriginCol *uint32 // covered: column of the covering origin +} + +// Cell is a table cell and the extent it spans. +type Cell struct { + Blocks []Block + ColSpan uint32 + RowSpan uint32 +} + +// Note is a footnote or endnote body, referenced from text by an Inline with +// Kind == "note_ref". +type Note struct { + ID string + Kind string // "footnote" or "endnote" + Blocks []Block +} + +// Asset is an embedded binary asset (image, object payload). Bytes are always +// retained, so a document stays self-contained. +type Asset struct { + ID uint64 // index into Document.Assets, as referenced by an image source + MediaType string + OriginPart string + Data []byte +} + +// ---- Decoding the flat FFI buffer ----------------------------------------- +// +// The buffer is the flat, length-prefixed serialization written by +// go/src/model.rs. Every field is little-endian. A decoder tracks the read +// cursor and panics on short buffers — the Rust side always produces a +// well-formed buffer, so a short read is a bug or a version skew, not user +// input. + +type decoder struct { + buf []byte + pos int +} + +func decodeDocument(raw []byte) (*Document, error) { + d := &decoder{buf: raw} + doc := &Document{} + var err error + if doc.Blocks, err = d.blocks(); err != nil { + return nil, err + } + if doc.Notes, err = d.notes(); err != nil { + return nil, err + } + if doc.Assets, err = d.assets(); err != nil { + return nil, err + } + return doc, nil +} + +func (d *decoder) need(n int) error { + if d.pos+n > len(d.buf) { + return fmt.Errorf("anydoc: truncated document buffer at offset %d (need %d bytes, have %d)", d.pos, n, len(d.buf)-d.pos) + } + return nil +} + +func (d *decoder) u8() (uint8, error) { + if err := d.need(1); err != nil { + return 0, err + } + v := d.buf[d.pos] + d.pos++ + return v, nil +} + +func (d *decoder) bool() (bool, error) { + v, err := d.u8() + if err != nil { + return false, err + } + return v != 0, nil +} + +func (d *decoder) u32() (uint32, error) { + if err := d.need(4); err != nil { + return 0, err + } + v := uint32(d.buf[d.pos]) | uint32(d.buf[d.pos+1])<<8 | uint32(d.buf[d.pos+2])<<16 | uint32(d.buf[d.pos+3])<<24 + d.pos += 4 + return v, nil +} + +func (d *decoder) u64() (uint64, error) { + if err := d.need(8); err != nil { + return 0, err + } + v := uint64(d.buf[d.pos]) | uint64(d.buf[d.pos+1])<<8 | uint64(d.buf[d.pos+2])<<16 | uint64(d.buf[d.pos+3])<<24 | + uint64(d.buf[d.pos+4])<<32 | uint64(d.buf[d.pos+5])<<40 | uint64(d.buf[d.pos+6])<<48 | uint64(d.buf[d.pos+7])<<56 + d.pos += 8 + return v, nil +} + +func (d *decoder) i32() (int32, error) { + u, err := d.u32() + if err != nil { + return 0, err + } + return int32(u), nil +} + +// optStr reads a length-prefixed string or a -1 sentinel (None). +func (d *decoder) optStr() (*string, error) { + length, err := d.i32() + if err != nil { + return nil, err + } + if length < 0 { + return nil, nil + } + s, err := d.strN(int(length)) + if err != nil { + return nil, err + } + return &s, nil +} + +func (d *decoder) str() (string, error) { + length, err := d.u32() + if err != nil { + return "", err + } + return d.strN(int(length)) +} + +func (d *decoder) strN(n int) (string, error) { + if err := d.need(n); err != nil { + return "", err + } + s := string(d.buf[d.pos : d.pos+n]) + d.pos += n + return s, nil +} + +func (d *decoder) bytes() ([]byte, error) { + length, err := d.u32() + if err != nil { + return nil, err + } + if err := d.need(int(length)); err != nil { + return nil, err + } + b := make([]byte, length) + copy(b, d.buf[d.pos:d.pos+int(length)]) + d.pos += int(length) + return b, nil +} + +func (d *decoder) blocks() ([]Block, error) { + n, err := d.u32() + if err != nil { + return nil, err + } + out := make([]Block, 0, n) + for i := uint32(0); i < n; i++ { + b, err := d.block() + if err != nil { + return nil, err + } + out = append(out, b) + } + return out, nil +} + +func (d *decoder) block() (Block, error) { + tag, err := d.i32() + if err != nil { + return Block{}, err + } + switch tag { + case C.BLOCK_HEADING: + level, err := d.u8() + if err != nil { + return Block{}, err + } + anchor, err := d.optStr() + if err != nil { + return Block{}, err + } + content, err := d.inlines() + if err != nil { + return Block{}, err + } + lv := level + return Block{Kind: "heading", Level: &lv, Anchor: anchor, Content: content}, nil + case C.BLOCK_PARAGRAPH: + content, err := d.inlines() + if err != nil { + return Block{}, err + } + return Block{Kind: "paragraph", Content: content}, nil + case C.BLOCK_LIST: + list, err := d.list() + if err != nil { + return Block{}, err + } + return Block{Kind: "list", List: list}, nil + case C.BLOCK_TABLE: + table, err := d.table() + if err != nil { + return Block{}, err + } + return Block{Kind: "table", Table: table}, nil + case C.BLOCK_QUOTE: + inner, err := d.blocks() + if err != nil { + return Block{}, err + } + return Block{Kind: "block_quote", Blocks: inner}, nil + case C.BLOCK_CODE: + lang, err := d.optStr() + if err != nil { + return Block{}, err + } + text, err := d.str() + if err != nil { + return Block{}, err + } + return Block{Kind: "code_block", Lang: lang, Text: &text}, nil + case C.BLOCK_RULE: + return Block{Kind: "rule"}, nil + default: + return Block{}, fmt.Errorf("anydoc: unknown block kind tag %d", tag) + } +} + +func (d *decoder) inlines() ([]Inline, error) { + n, err := d.u32() + if err != nil { + return nil, err + } + out := make([]Inline, 0, n) + for i := uint32(0); i < n; i++ { + in, err := d.inline() + if err != nil { + return nil, err + } + out = append(out, in) + } + return out, nil +} + +func (d *decoder) inline() (Inline, error) { + tag, err := d.i32() + if err != nil { + return Inline{}, err + } + switch tag { + case C.INLINE_TEXT: + text, err := d.str() + if err != nil { + return Inline{}, err + } + style, err := d.style() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "text", Text: &text, Style: &style}, nil + case C.INLINE_LINK: + content, err := d.inlines() + if err != nil { + return Inline{}, err + } + target, err := d.linkTarget() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "link", Content: content, Target: &target}, nil + case C.INLINE_IMAGE: + alt, err := d.str() + if err != nil { + return Inline{}, err + } + src, err := d.imageSource() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "image", Alt: &alt, Source: &src}, nil + case C.INLINE_ANCHOR: + id, err := d.str() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "anchor", Anchor: &id}, nil + case C.INLINE_NOTEREF: + id, err := d.str() + if err != nil { + return Inline{}, err + } + return Inline{Kind: "note_ref", NoteID: &id}, nil + case C.INLINE_LINEBREAK: + return Inline{Kind: "line_break"}, nil + default: + return Inline{}, fmt.Errorf("anydoc: unknown inline kind tag %d", tag) + } +} + +func (d *decoder) style() (Style, error) { + bold, err := d.bool() + if err != nil { + return Style{}, err + } + italic, err := d.bool() + if err != nil { + return Style{}, err + } + strike, err := d.bool() + if err != nil { + return Style{}, err + } + code, err := d.bool() + if err != nil { + return Style{}, err + } + return Style{Bold: bold, Italic: italic, Strike: strike, Code: code}, nil +} + +func (d *decoder) linkTarget() (LinkTarget, error) { + tag, err := d.i32() + if err != nil { + return LinkTarget{}, err + } + value, err := d.str() + if err != nil { + return LinkTarget{}, err + } + switch tag { + case C.LINK_EXTERNAL: + return LinkTarget{Kind: "external", Value: value}, nil + case C.LINK_RELATIVE: + return LinkTarget{Kind: "relative", Value: value}, nil + case C.LINK_ANCHOR: + return LinkTarget{Kind: "anchor", Value: value}, nil + default: + return LinkTarget{}, fmt.Errorf("anydoc: unknown link target tag %d", tag) + } +} + +func (d *decoder) imageSource() (ImageSource, error) { + tag, err := d.i32() + if err != nil { + return ImageSource{}, err + } + switch tag { + case C.IMG_EXTERNAL: + url, err := d.str() + if err != nil { + return ImageSource{}, err + } + return ImageSource{Kind: "external", URL: &url}, nil + case C.IMG_ASSET: + id, err := d.u64() + if err != nil { + return ImageSource{}, err + } + return ImageSource{Kind: "asset", AssetID: &id}, nil + case C.IMG_UNAVAILABLE: + return ImageSource{Kind: "unavailable"}, nil + default: + return ImageSource{}, fmt.Errorf("anydoc: unknown image source tag %d", tag) + } +} + +func (d *decoder) list() (*List, error) { + markerTag, err := d.i32() + if err != nil { + return nil, err + } + start, err := d.u64() + if err != nil { + return nil, err + } + n, err := d.u32() + if err != nil { + return nil, err + } + items := make([]ListItem, 0, n) + for i := uint32(0); i < n; i++ { + item, err := d.listItem() + if err != nil { + return nil, err + } + items = append(items, item) + } + marker, err := markerName(markerTag) + if err != nil { + return nil, err + } + return &List{Marker: marker, Start: start, Items: items}, nil +} + +func (d *decoder) listItem() (ListItem, error) { + blocks, err := d.blocks() + if err != nil { + return ListItem{}, err + } + checkedTag, err := d.i32() + if err != nil { + return ListItem{}, err + } + var checked *bool + switch checkedTag { + case 0: + f := false + checked = &f + case 1: + t := true + checked = &t + case -1: + checked = nil + default: + return ListItem{}, fmt.Errorf("anydoc: unknown checked tag %d", checkedTag) + } + label, err := d.optStr() + if err != nil { + return ListItem{}, err + } + return ListItem{Blocks: blocks, Checked: checked, MarkerLabel: label}, nil +} + +func (d *decoder) table() (*Table, error) { + headerRows, err := d.u32() + if err != nil { + return nil, err + } + kindTag, err := d.i32() + if err != nil { + return nil, err + } + rows, err := d.u32() + if err != nil { + return nil, err + } + grid := make([][]CellSlot, 0, rows) + for i := uint32(0); i < rows; i++ { + cols, err := d.u32() + if err != nil { + return nil, err + } + row := make([]CellSlot, 0, cols) + for j := uint32(0); j < cols; j++ { + slot, err := d.slot() + if err != nil { + return nil, err + } + row = append(row, slot) + } + grid = append(grid, row) + } + kind := "data" + if kindTag == C.TABLE_LAYOUT { + kind = "layout" + } + return &Table{Grid: grid, HeaderRows: headerRows, Kind: kind}, nil +} + +func (d *decoder) slot() (CellSlot, error) { + tag, err := d.i32() + if err != nil { + return CellSlot{}, err + } + switch tag { + case C.SLOT_ORIGIN: + cell, err := d.cell() + if err != nil { + return CellSlot{}, err + } + return CellSlot{Kind: "origin", Cell: &cell}, nil + case C.SLOT_COVERED: + r, err := d.u32() + if err != nil { + return CellSlot{}, err + } + c, err := d.u32() + if err != nil { + return CellSlot{}, err + } + rr := r + cc := c + return CellSlot{Kind: "covered", OriginRow: &rr, OriginCol: &cc}, nil + default: + return CellSlot{}, fmt.Errorf("anydoc: unknown cell slot tag %d", tag) + } +} + +func (d *decoder) cell() (Cell, error) { + colSpan, err := d.u32() + if err != nil { + return Cell{}, err + } + rowSpan, err := d.u32() + if err != nil { + return Cell{}, err + } + blocks, err := d.blocks() + if err != nil { + return Cell{}, err + } + return Cell{Blocks: blocks, ColSpan: colSpan, RowSpan: rowSpan}, nil +} + +func (d *decoder) notes() ([]Note, error) { + n, err := d.u32() + if err != nil { + return nil, err + } + out := make([]Note, 0, n) + for i := uint32(0); i < n; i++ { + note, err := d.note() + if err != nil { + return nil, err + } + out = append(out, note) + } + return out, nil +} + +func (d *decoder) note() (Note, error) { + id, err := d.str() + if err != nil { + return Note{}, err + } + kindTag, err := d.i32() + if err != nil { + return Note{}, err + } + blocks, err := d.blocks() + if err != nil { + return Note{}, err + } + kind := "footnote" + if kindTag == C.NOTE_ENDNOTE { + kind = "endnote" + } + return Note{ID: id, Kind: kind, Blocks: blocks}, nil +} + +func (d *decoder) assets() ([]Asset, error) { + n, err := d.u32() + if err != nil { + return nil, err + } + out := make([]Asset, 0, n) + for i := uint32(0); i < n; i++ { + asset, err := d.asset() + if err != nil { + return nil, err + } + out = append(out, asset) + } + return out, nil +} + +func (d *decoder) asset() (Asset, error) { + id, err := d.u64() + if err != nil { + return Asset{}, err + } + mediaType, err := d.str() + if err != nil { + return Asset{}, err + } + originPart, err := d.str() + if err != nil { + return Asset{}, err + } + data, err := d.bytes() + if err != nil { + return Asset{}, err + } + return Asset{ID: id, MediaType: mediaType, OriginPart: originPart, Data: data}, nil +} + +func markerName(tag int32) (string, error) { + switch tag { + case C.MARKER_BULLET: + return "bullet", nil + case C.MARKER_DECIMAL: + return "decimal", nil + case C.MARKER_LOWER_ALPHA: + return "lower_alpha", nil + case C.MARKER_UPPER_ALPHA: + return "upper_alpha", nil + case C.MARKER_LOWER_ROMAN: + return "lower_roman", nil + case C.MARKER_UPPER_ROMAN: + return "upper_roman", nil + } + return "", fmt.Errorf("anydoc: unknown list marker tag %d", tag) +} diff --git a/go/src/lib.rs b/go/src/lib.rs new file mode 100644 index 00000000..0628e045 --- /dev/null +++ b/go/src/lib.rs @@ -0,0 +1,537 @@ +//! C ABI surface for the Go bindings. +//! +//! Memory ownership rule: **Rust allocates, C ABI exposes a `_free` for every +//! allocation family, Go releases handles.** There are two allocation +//! families: +//! +//! - strings: `*mut c_char` + length out-params, returned by +//! `anydoc_to_markdown*` and `anydoc_last_error`. Released via +//! `anydoc_string_free`. Length is passed separately so binary-safe. +//! - document buffer: a single `*mut u8` + length, returned by +//! `anydoc_to_document`, holding the flat serialization from [`model`]. +//! Released via `anydoc_buffer_free`. +//! +//! Errors surface as stable non-zero codes (see `ERR_*`); the human-readable +//! message is available through `anydoc_last_error`, thread-local. + +#![allow(clippy::missing_safety_doc)] + +use std::ffi::CString; +use std::os::raw::{c_char, c_int}; +use std::ptr; + +use anydoc::{ConvertError, Format}; + +mod model; + +// ---- Error codes ----------------------------------------------------------- + +/// Success. +pub const ERR_OK: c_int = 0; +/// `ConvertError::Unsupported`. +pub const ERR_UNSUPPORTED: c_int = 1; +/// `ConvertError::Malformed`. +pub const ERR_MALFORMED: c_int = 2; +/// `ConvertError::Encrypted`. +pub const ERR_ENCRYPTED: c_int = 3; +/// `ConvertError::ResourceLimit`. +pub const ERR_RESOURCE_LIMIT: c_int = 4; +/// `ConvertError::MissingPart`. +pub const ERR_MISSING_PART: c_int = 5; +/// `ConvertError::Io`. +pub const ERR_IO: c_int = 6; +/// PDF was passed to `anydoc_to_document` (unsupported there). +pub const ERR_PDF_NO_MODEL: c_int = 7; +/// Null pointer or invalid argument passed to the ABI. +pub const ERR_INVALID_ARG: c_int = 8; +/// Unknown format name passed to the ABI. +pub const ERR_UNKNOWN_FORMAT: c_int = 9; + +thread_local! { + static LAST_ERROR: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +} + +fn set_last_error(msg: &str) { + let cstring = CString::new(msg).unwrap_or_else(|_| CString::new("").unwrap()); + LAST_ERROR.with(|cell| *cell.borrow_mut() = Some(cstring)); +} + +fn error_code(err: &ConvertError) -> c_int { + match err { + ConvertError::Unsupported(_) => ERR_UNSUPPORTED, + ConvertError::Malformed { .. } => ERR_MALFORMED, + ConvertError::Encrypted => ERR_ENCRYPTED, + ConvertError::ResourceLimit { .. } => ERR_RESOURCE_LIMIT, + ConvertError::MissingPart { .. } => ERR_MISSING_PART, + ConvertError::Io(_) => ERR_IO, + _ => ERR_UNSUPPORTED, + } +} + +fn fail(err: ConvertError) -> c_int { + let code = error_code(&err); + set_last_error(&err.to_string()); + code +} + +// ---- Format enum ----------------------------------------------------------- + +/// C-side format tag. Stable; mirrors the Node/Python lowercase string names +/// via `format_name`. `ANYDOC_FORMAT_NONE` is the `Option::None` sentinel. +pub const ANYDOC_FORMAT_NONE: c_int = -1; +pub const ANYDOC_FORMAT_DOC: c_int = 0; +pub const ANYDOC_FORMAT_DOCX: c_int = 1; +pub const ANYDOC_FORMAT_ODT: c_int = 2; +pub const ANYDOC_FORMAT_PDF: c_int = 3; +pub const ANYDOC_FORMAT_PPT: c_int = 4; +pub const ANYDOC_FORMAT_PPTX: c_int = 5; +pub const ANYDOC_FORMAT_RTF: c_int = 6; +pub const ANYDOC_FORMAT_EPUB: c_int = 7; +pub const ANYDOC_FORMAT_XLSX: c_int = 8; +pub const ANYDOC_FORMAT_ODS: c_int = 9; +pub const ANYDOC_FORMAT_ODP: c_int = 10; +pub const ANYDOC_FORMAT_CSV: c_int = 11; + +fn format_to_c(format: Format) -> c_int { + match format { + Format::Doc => ANYDOC_FORMAT_DOC, + Format::Docx => ANYDOC_FORMAT_DOCX, + Format::Odt => ANYDOC_FORMAT_ODT, + Format::Pdf => ANYDOC_FORMAT_PDF, + Format::Ppt => ANYDOC_FORMAT_PPT, + Format::Pptx => ANYDOC_FORMAT_PPTX, + Format::Rtf => ANYDOC_FORMAT_RTF, + Format::Epub => ANYDOC_FORMAT_EPUB, + Format::Excel => ANYDOC_FORMAT_XLSX, + Format::Ods => ANYDOC_FORMAT_ODS, + Format::Odp => ANYDOC_FORMAT_ODP, + Format::Csv => ANYDOC_FORMAT_CSV, + } +} + +fn format_from_c(tag: c_int) -> Option { + Some(match tag { + ANYDOC_FORMAT_DOC => Format::Doc, + ANYDOC_FORMAT_DOCX => Format::Docx, + ANYDOC_FORMAT_ODT => Format::Odt, + ANYDOC_FORMAT_PDF => Format::Pdf, + ANYDOC_FORMAT_PPT => Format::Ppt, + ANYDOC_FORMAT_PPTX => Format::Pptx, + ANYDOC_FORMAT_RTF => Format::Rtf, + ANYDOC_FORMAT_EPUB => Format::Epub, + ANYDOC_FORMAT_XLSX => Format::Excel, + ANYDOC_FORMAT_ODS => Format::Ods, + ANYDOC_FORMAT_ODP => Format::Odp, + ANYDOC_FORMAT_CSV => Format::Csv, + _ => return None, + }) +} + +/// Format names, lowercase, matching the Node/Python convention. +const FORMAT_NAMES: [&str; 12] = + ["doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv"]; + +fn format_name(format: Format) -> &'static str { + match format { + Format::Doc => FORMAT_NAMES[0], + Format::Docx => FORMAT_NAMES[1], + Format::Odt => FORMAT_NAMES[2], + Format::Pdf => FORMAT_NAMES[3], + Format::Ppt => FORMAT_NAMES[4], + Format::Pptx => FORMAT_NAMES[5], + Format::Rtf => FORMAT_NAMES[6], + Format::Epub => FORMAT_NAMES[7], + Format::Excel => FORMAT_NAMES[8], + Format::Ods => FORMAT_NAMES[9], + Format::Odp => FORMAT_NAMES[10], + Format::Csv => FORMAT_NAMES[11], + } +} + +fn parse_format_name(name: &str) -> Option { + FORMAT_NAMES + .iter() + .zip([ + Format::Doc, + Format::Docx, + Format::Odt, + Format::Pdf, + Format::Ppt, + Format::Pptx, + Format::Rtf, + Format::Epub, + Format::Excel, + Format::Ods, + Format::Odp, + Format::Csv, + ]) + .find(|(n, _)| **n == name) + .map(|(_, f)| f) +} + +// ---- String out-param helper ---------------------------------------------- + +/// Box a `String` into a heap allocation the caller owns and must free with +/// `anydoc_string_free`. Writes the byte length (excluding any NUL) to +/// `*out_len`. The buffer is NUL-terminated for safety with C string readers, +/// but binary-safe content relies on `out_len`. +/// +/// Returns a dangling null pointer for empty strings so the Go side can treat +/// a null result as "empty" without needing to free it. Length is still +/// written as 0. +fn string_out(s: String, out_str: *mut *mut c_char, out_len: *mut usize) -> c_int { + if out_str.is_null() { + set_last_error("anydoc: out_str is null"); + return ERR_INVALID_ARG; + } + let len = s.len(); + // SAFETY: we write the length unconditionally; null is the empty sentinel. + if !out_len.is_null() { + unsafe { *out_len = len }; + } + if len == 0 { + // Empty string: return null sentinel, no allocation to free. + unsafe { *out_str = ptr::null_mut() }; + return ERR_OK; + } + let cstring = match CString::new(s.into_bytes()) { + Ok(cs) => cs, + Err(_) => { + // Contains an interior NUL: fall back to lossy replacement so the + // caller still gets something. Markdown shouldn't contain NULs. + set_last_error("anydoc: output contained an interior NUL byte"); + return ERR_INVALID_ARG; + } + }; + let raw = cstring.into_raw(); + // into_raw returns a buffer of capacity len+1 (with the trailing NUL). + unsafe { *out_str = raw }; + ERR_OK +} + +// ---- C ABI functions ------------------------------------------------------- + +/// Detect the format from the content. Writes a format tag to `*out` (one of +/// `ANYDOC_FORMAT_*`), or `ANYDOC_FORMAT_NONE` when nothing matches. +/// +/// Returns `ERR_OK` on success (including no-match), `ERR_INVALID_ARG` on null +/// pointers. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_from_bytes( + bytes: *const u8, + len: usize, + out: *mut c_int, +) -> c_int { + if bytes.is_null() || out.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_format_from_bytes"); + return ERR_INVALID_ARG; + } + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + let tag = Format::from_bytes(slice).map_or(ANYDOC_FORMAT_NONE, format_to_c); + unsafe { *out = tag }; + ERR_OK +} + +/// The format a bare extension names (no leading dot). Writes a format tag to +/// `*out`, or `ANYDOC_FORMAT_NONE` when unrecognized. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_from_extension( + ext: *const c_char, + out: *mut c_int, +) -> c_int { + if ext.is_null() || out.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_format_from_extension"); + return ERR_INVALID_ARG; + } + let ext_str = match unsafe { std::ffi::CStr::from_ptr(ext) }.to_str() { + Ok(s) => s.trim_start_matches('.'), + Err(_) => { + set_last_error("anydoc: extension is not valid UTF-8"); + return ERR_INVALID_ARG; + } + }; + let tag = Format::from_extension(ext_str).map_or(ANYDOC_FORMAT_NONE, format_to_c); + unsafe { *out = tag }; + ERR_OK +} + +/// The format a path's extension names. Writes a format tag to `*out`, or +/// `ANYDOC_FORMAT_NONE` when unrecognized. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_from_path(path: *const c_char, out: *mut c_int) -> c_int { + if path.is_null() || out.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_format_from_path"); + return ERR_INVALID_ARG; + } + let path_str = match unsafe { std::ffi::CStr::from_ptr(path) }.to_str() { + Ok(s) => s, + Err(_) => { + set_last_error("anydoc: path is not valid UTF-8"); + return ERR_INVALID_ARG; + } + }; + let tag = + Format::from_path(std::path::Path::new(path_str)).map_or(ANYDOC_FORMAT_NONE, format_to_c); + unsafe { *out = tag }; + ERR_OK +} + +/// Convert a document file to Markdown. The format is detected from the file +/// content, with the extension as the fallback for signature-less formats +/// (CSV). +/// +/// On success writes a `*mut c_char` to `*out_str` and its byte length to +/// `*out_len`. The caller must free the string with `anydoc_string_free`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_to_markdown( + path: *const c_char, + out_str: *mut *mut c_char, + out_len: *mut usize, +) -> c_int { + if path.is_null() || out_str.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_to_markdown"); + return ERR_INVALID_ARG; + } + let path_str = match unsafe { std::ffi::CStr::from_ptr(path) }.to_str() { + Ok(s) => s, + Err(_) => { + set_last_error("anydoc: path is not valid UTF-8"); + return ERR_INVALID_ARG; + } + }; + match anydoc::to_markdown(path_str) { + Ok(md) => string_out(md, out_str, out_len), + Err(e) => fail(e), + } +} + +/// Convert an in-memory document to Markdown. `format_tag` is one of the +/// `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the +/// content. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_to_markdown_bytes( + bytes: *const u8, + len: usize, + format_tag: c_int, + out_str: *mut *mut c_char, + out_len: *mut usize, +) -> c_int { + if bytes.is_null() || out_str.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_to_markdown_bytes"); + return ERR_INVALID_ARG; + } + let format = if format_tag == ANYDOC_FORMAT_NONE { + None + } else { + match format_from_c(format_tag) { + Some(f) => Some(f), + None => { + set_last_error("anydoc: unknown format tag"); + return ERR_UNKNOWN_FORMAT; + } + } + }; + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + match anydoc::to_markdown_bytes(slice, format) { + Ok(md) => string_out(md, out_str, out_len), + Err(e) => fail(e), + } +} + +/// Parse an in-memory document into the document model. `format_tag` is one of +/// the `ANYDOC_FORMAT_*` constants, or `ANYDOC_FORMAT_NONE` to detect from the +/// content. +/// +/// On success writes a heap-allocated byte buffer to `*out_buf` and its length +/// to `*out_len`. The buffer holds the flat serialization (see `model`). The +/// caller must free it with `anydoc_buffer_free`. +/// +/// Returns `ERR_PDF_NO_MODEL` for PDF input: PDF conversion produces Markdown +/// directly and has no document-model form. Use `anydoc_to_markdown_bytes`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_to_document( + bytes: *const u8, + len: usize, + format_tag: c_int, + out_buf: *mut *mut u8, + out_len: *mut usize, +) -> c_int { + if bytes.is_null() || out_buf.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_to_document"); + return ERR_INVALID_ARG; + } + let format = if format_tag == ANYDOC_FORMAT_NONE { + None + } else { + match format_from_c(format_tag) { + Some(f) => Some(f), + None => { + set_last_error("anydoc: unknown format tag"); + return ERR_UNKNOWN_FORMAT; + } + } + }; + let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; + // Resolve the format up front so we can give the typed PDF error before + // parsing, matching the crate's `to_document` contract. + let resolved = match format { + Some(f) => f, + None => match Format::from_bytes(slice) { + Some(f) => f, + None => { + set_last_error( + "unsupported input: unrecognized file content: name the format explicitly", + ); + return ERR_UNSUPPORTED; + } + }, + }; + if resolved == Format::Pdf { + set_last_error( + "unsupported input: PDF conversion produces Markdown directly and has no document-model form; use to_markdown_bytes", + ); + return ERR_PDF_NO_MODEL; + } + let document = match anydoc::to_document(slice, resolved) { + Ok(d) => d, + Err(e) => return fail(e), + }; + let mut encoder = model::Encoder::new(); + model::write_document(&mut encoder, &document); + let body = encoder.into_vec(); + let len = body.len(); + // Layout: [usize len][usize cap][u8 data..len]. The caller receives a + // pointer to the data area and the length; `anydoc_buffer_free` walks back + // to the header to recover (len, cap) for a sound deallocation. + let header_size = std::mem::size_of::(); + let total = header_size + len; + let layout = std::alloc::Layout::from_size_align(total, std::mem::align_of::()) + .expect("document buffer layout"); + let base = unsafe { std::alloc::alloc(layout) as *mut BufferHeader }; + if base.is_null() { + set_last_error("anydoc: out of memory"); + return ERR_IO; + } + // The capacity we record is the *total* allocation size, so the free path + // can reconstruct a layout of the same size to pass back to `dealloc`. + unsafe { + std::ptr::write(base, BufferHeader { len, cap: total }); + let data = (base as *mut u8).add(header_size); + std::ptr::copy_nonoverlapping(body.as_ptr(), data, len); + *out_buf = data; + if !out_len.is_null() { + *out_len = len; + } + } + // `body` drops here, freeing its own backing allocation. + ERR_OK +} + +/// Header prepended to every document buffer, so `anydoc_buffer_free` can +/// soundly reclaim the allocation without any thread-local state. The data +/// the caller sees starts immediately after this header. +#[repr(C)] +struct BufferHeader { + len: usize, + /// Total allocation size (header + data), used to rebuild the Layout for + /// `dealloc`. + cap: usize, +} + +/// Free a string returned by `anydoc_to_markdown*` or `anydoc_last_error`. +/// Null is a no-op (the empty-string sentinel). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_string_free(s: *mut c_char) { + if s.is_null() { + return; + } + // SAFETY: the pointer was produced by `CString::into_raw`; reclaiming it + // with `from_raw` is the matching deallocator. + unsafe { + let _ = CString::from_raw(s); + } +} + +/// Free a document buffer returned by `anydoc_to_document`. Null is a no-op. +/// `len` must be the length written to `*out_len` at allocation time; it is +/// validated against the header for safety. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_buffer_free(buf: *mut u8, len: usize) { + if buf.is_null() { + return; + } + let header_size = std::mem::size_of::(); + // SAFETY: the pointer was produced by `anydoc_to_document` with a + // `BufferHeader` immediately before it. Walk back to read the header and + // rebuild the Layout, then dealloc the whole block. + unsafe { + let base = (buf.sub(header_size)) as *mut BufferHeader; + let header = base.read(); + if header.len != len { + // Length mismatch: caller bug. Leak rather than risk a bad free. + set_last_error("anydoc: anydoc_buffer_free length does not match allocation"); + return; + } + let layout = std::alloc::Layout::from_size_align_unchecked( + header.cap, + std::mem::align_of::(), + ); + std::alloc::dealloc(base as *mut u8, layout); + } +} + +/// Return the human-readable message for the last error on the current +/// thread. The returned `*mut c_char` is freshly allocated and must be freed +/// with `anydoc_string_free`. Returns null when there is no last error. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_last_error() -> *mut c_char { + LAST_ERROR + .with(|cell| cell.borrow_mut().take()) + .and_then(|cstring| { + // Re-allocate as a fresh owned C string the caller can free. + let bytes = cstring.into_bytes_with_nul(); + CString::from_vec_with_nul(bytes).ok().map(|cs| cs.into_raw()) + }) + .unwrap_or(ptr::null_mut()) +} + +// ---- Go-side helpers exposed for parity tests ----------------------------- + +/// Return the lowercase name of a format tag, or null for an unknown tag. +/// The returned string must be freed with `anydoc_string_free`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_name(tag: c_int) -> *mut c_char { + if tag == ANYDOC_FORMAT_NONE { + return ptr::null_mut(); + } + match format_from_c(tag) { + Some(f) => { + let name = format_name(f); + match CString::new(name) { + Ok(cs) => cs.into_raw(), + Err(_) => ptr::null_mut(), + } + } + None => ptr::null_mut(), + } +} + +/// Parse a lowercase format name into a tag. Writes the tag to `*out`, or +/// `ANYDOC_FORMAT_NONE` when the name is unknown. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn anydoc_format_from_name(name: *const c_char, out: *mut c_int) -> c_int { + if name.is_null() || out.is_null() { + set_last_error("anydoc: null pointer passed to anydoc_format_from_name"); + return ERR_INVALID_ARG; + } + let name_str = match unsafe { std::ffi::CStr::from_ptr(name) }.to_str() { + Ok(s) => s, + Err(_) => { + set_last_error("anydoc: format name is not valid UTF-8"); + return ERR_INVALID_ARG; + } + }; + let tag = parse_format_name(name_str).map_or(ANYDOC_FORMAT_NONE, format_to_c); + unsafe { *out = tag }; + ERR_OK +} diff --git a/go/src/model.rs b/go/src/model.rs new file mode 100644 index 00000000..5a97b044 --- /dev/null +++ b/go/src/model.rs @@ -0,0 +1,321 @@ +//! `#[repr(C)]` DTOs and a flat, self-describing serialization of the document +//! model. +//! +//! The document tree is recursive (blocks hold blocks, inlines hold inlines, +//! tables hold cells that hold blocks), so rather than surfacing it through a +//! forest of opaque handles and one `_free` per node type, the FFI serializes +//! the whole `Document` into a single length-prefixed byte buffer. That gives +//! exactly one allocation family for the tree (`anydoc_buffer_free`) and keeps +//! the C ABI tiny. Variant tags are written inline as `c_int`-compatible +//! values, mirroring the Node/Python `kind` string convention. +//! +//! Format: see `write_document` below. Every field is length-prefixed so the +//! Go side can decode without guessing. Strings carry their byte length as a +//! u32; slices carry a u32 count followed by that many elements. Booleans are +//! a single byte (0 or 1). Integers are little-endian fixed width. + +use std::os::raw::c_int; + +use anydoc::model; + +/// Tag values for `Block` variants. Stable across versions; the Go side maps +/// these to the same lowercase string names the Node and Python bindings use. +pub const BLOCK_HEADING: c_int = 0; +pub const BLOCK_PARAGRAPH: c_int = 1; +pub const BLOCK_LIST: c_int = 2; +pub const BLOCK_TABLE: c_int = 3; +pub const BLOCK_QUOTE: c_int = 4; +pub const BLOCK_CODE: c_int = 5; +pub const BLOCK_RULE: c_int = 6; + +pub const INLINE_TEXT: c_int = 0; +pub const INLINE_LINK: c_int = 1; +pub const INLINE_IMAGE: c_int = 2; +pub const INLINE_ANCHOR: c_int = 3; +pub const INLINE_NOTEREF: c_int = 4; +pub const INLINE_LINEBREAK: c_int = 5; + +pub const LINK_EXTERNAL: c_int = 0; +pub const LINK_RELATIVE: c_int = 1; +pub const LINK_ANCHOR: c_int = 2; + +pub const IMG_EXTERNAL: c_int = 0; +pub const IMG_ASSET: c_int = 1; +pub const IMG_UNAVAILABLE: c_int = 2; + +pub const MARKER_BULLET: c_int = 0; +pub const MARKER_DECIMAL: c_int = 1; +pub const MARKER_LOWER_ALPHA: c_int = 2; +pub const MARKER_UPPER_ALPHA: c_int = 3; +pub const MARKER_LOWER_ROMAN: c_int = 4; +pub const MARKER_UPPER_ROMAN: c_int = 5; + +pub const TABLE_DATA: c_int = 0; +pub const TABLE_LAYOUT: c_int = 1; + +pub const SLOT_ORIGIN: c_int = 0; +pub const SLOT_COVERED: c_int = 1; + +pub const NOTE_FOOTNOTE: c_int = 0; +pub const NOTE_ENDNOTE: c_int = 1; + +/// Encoder for the flat document buffer. Little-endian, length-prefixed. +pub struct Encoder { + buf: Vec, +} + +impl Encoder { + pub fn new() -> Self { + Encoder { buf: Vec::with_capacity(8 * 1024) } + } + + pub fn into_vec(self) -> Vec { + self.buf + } + + pub fn u8(&mut self, v: u8) { + self.buf.push(v); + } + + pub fn bool(&mut self, v: bool) { + self.buf.push(if v { 1 } else { 0 }); + } + + pub fn u32(&mut self, v: u32) { + self.buf.extend_from_slice(&v.to_le_bytes()); + } + + pub fn u64(&mut self, v: u64) { + self.buf.extend_from_slice(&v.to_le_bytes()); + } + + pub fn i32(&mut self, v: i32) { + self.buf.extend_from_slice(&v.to_le_bytes()); + } + + /// A length-prefixed UTF-8 string. `None` writes a -1 length sentinel. + pub fn opt_str(&mut self, v: &Option) { + match v { + Some(s) => self.str(s), + None => self.i32(-1), + } + } + + pub fn str(&mut self, s: &str) { + self.u32(s.len() as u32); + self.buf.extend_from_slice(s.as_bytes()); + } + + /// A length-prefixed byte slice. + pub fn bytes(&mut self, b: &[u8]) { + self.u32(b.len() as u32); + self.buf.extend_from_slice(b); + } +} + +pub fn write_document(e: &mut Encoder, doc: &model::Document) { + write_blocks(e, &doc.blocks); + e.u32(doc.notes.len() as u32); + for note in &doc.notes { + write_note(e, note); + } + e.u32(doc.assets.len() as u32); + for asset in &doc.assets { + write_asset(e, asset); + } +} + +fn write_blocks(e: &mut Encoder, blocks: &[model::Block]) { + e.u32(blocks.len() as u32); + for block in blocks { + write_block(e, block); + } +} + +fn write_block(e: &mut Encoder, block: &model::Block) { + match block { + model::Block::Heading { level, anchor, content } => { + e.i32(BLOCK_HEADING); + e.u8(*level); + e.opt_str(anchor); + write_inlines(e, content); + } + model::Block::Paragraph(content) => { + e.i32(BLOCK_PARAGRAPH); + write_inlines(e, content); + } + model::Block::List(list) => { + e.i32(BLOCK_LIST); + write_list(e, list); + } + model::Block::Table(table) => { + e.i32(BLOCK_TABLE); + write_table(e, table); + } + model::Block::BlockQuote(inner) => { + e.i32(BLOCK_QUOTE); + write_blocks(e, inner); + } + model::Block::CodeBlock { lang, text } => { + e.i32(BLOCK_CODE); + e.opt_str(lang); + e.str(text); + } + model::Block::Rule => { + e.i32(BLOCK_RULE); + } + } +} + +fn write_inlines(e: &mut Encoder, inlines: &[model::Inline]) { + e.u32(inlines.len() as u32); + for inline in inlines { + write_inline(e, inline); + } +} + +fn write_inline(e: &mut Encoder, inline: &model::Inline) { + match inline { + model::Inline::Text { text, style } => { + e.i32(INLINE_TEXT); + e.str(text); + write_style(e, *style); + } + model::Inline::Link { content, target } => { + e.i32(INLINE_LINK); + write_inlines(e, content); + write_link_target(e, target); + } + model::Inline::Image { alt, source } => { + e.i32(INLINE_IMAGE); + e.str(alt); + write_image_source(e, source); + } + model::Inline::Anchor(id) => { + e.i32(INLINE_ANCHOR); + e.str(id); + } + model::Inline::NoteRef(id) => { + e.i32(INLINE_NOTEREF); + e.str(id); + } + model::Inline::LineBreak => { + e.i32(INLINE_LINEBREAK); + } + } +} + +fn write_style(e: &mut Encoder, style: model::Style) { + e.bool(style.bold); + e.bool(style.italic); + e.bool(style.strike); + e.bool(style.code); +} + +fn write_link_target(e: &mut Encoder, target: &model::LinkTarget) { + match target { + model::LinkTarget::External(v) => { + e.i32(LINK_EXTERNAL); + e.str(v); + } + model::LinkTarget::Relative(v) => { + e.i32(LINK_RELATIVE); + e.str(v); + } + model::LinkTarget::Anchor(v) => { + e.i32(LINK_ANCHOR); + e.str(v); + } + } +} + +fn write_image_source(e: &mut Encoder, source: &model::ImageSource) { + match source { + model::ImageSource::External(url) => { + e.i32(IMG_EXTERNAL); + e.str(url); + } + model::ImageSource::Asset(id) => { + e.i32(IMG_ASSET); + e.u64(id.0 as u64); + } + model::ImageSource::Unavailable => { + e.i32(IMG_UNAVAILABLE); + } + } +} + +fn write_list(e: &mut Encoder, list: &model::List) { + e.i32(marker_tag(list.marker)); + e.u64(list.start); + e.u32(list.items.len() as u32); + for item in &list.items { + write_list_item(e, item); + } +} + +fn write_list_item(e: &mut Encoder, item: &model::ListItem) { + write_blocks(e, &item.blocks); + match item.checked { + None => e.i32(-1), + Some(false) => e.i32(0), + Some(true) => e.i32(1), + } + e.opt_str(&item.marker_label); +} + +fn write_table(e: &mut Encoder, table: &model::Table) { + e.u32(table.header_rows as u32); + e.i32(if table.kind == model::TableKind::Layout { TABLE_LAYOUT } else { TABLE_DATA }); + e.u32(table.grid.len() as u32); + for row in &table.grid { + e.u32(row.len() as u32); + for slot in row { + write_slot(e, slot); + } + } +} + +fn write_slot(e: &mut Encoder, slot: &model::CellSlot) { + match slot { + model::CellSlot::Origin(cell) => { + e.i32(SLOT_ORIGIN); + write_cell(e, cell); + } + model::CellSlot::Covered { origin_row, origin_col } => { + e.i32(SLOT_COVERED); + e.u32(*origin_row as u32); + e.u32(*origin_col as u32); + } + } +} + +fn write_cell(e: &mut Encoder, cell: &model::Cell) { + e.u32(cell.col_span); + e.u32(cell.row_span); + write_blocks(e, &cell.blocks); +} + +fn write_note(e: &mut Encoder, note: &model::Note) { + e.str(¬e.id); + e.i32(if note.kind == model::NoteKind::Endnote { NOTE_ENDNOTE } else { NOTE_FOOTNOTE }); + write_blocks(e, ¬e.blocks); +} + +fn write_asset(e: &mut Encoder, asset: &model::Asset) { + e.u64(asset.id.0 as u64); + e.str(&asset.media_type); + e.str(&asset.origin_part); + e.bytes(&asset.bytes); +} + +fn marker_tag(marker: model::MarkerKind) -> c_int { + match marker { + model::MarkerKind::Bullet => MARKER_BULLET, + model::MarkerKind::Decimal => MARKER_DECIMAL, + model::MarkerKind::LowerAlpha => MARKER_LOWER_ALPHA, + model::MarkerKind::UpperAlpha => MARKER_UPPER_ALPHA, + model::MarkerKind::LowerRoman => MARKER_LOWER_ROMAN, + model::MarkerKind::UpperRoman => MARKER_UPPER_ROMAN, + } +} diff --git a/go/version.go b/go/version.go new file mode 100644 index 00000000..0fc75924 --- /dev/null +++ b/go/version.go @@ -0,0 +1,6 @@ +package anydoc + +// Version is the version of the anydoc-go module, bumped in lockstep with the +// Rust crate version (go/Cargo.toml) and the GitHub release tag. The +// packaged static libraries. +const Version = "0.1.6" diff --git a/scripts/check-versions.sh b/scripts/check-versions.sh index ee6c6b97..7d500810 100644 --- a/scripts/check-versions.sh +++ b/scripts/check-versions.sh @@ -1,31 +1,22 @@ #!/bin/sh -# Verify that the release version agrees across the five places it is declared: -# 1. Cargo.toml [package].version -# 2. python/Cargo.toml [package].version -# 3. wasm/Cargo.toml [package].version (wasm-pack stamps it on @firecrawl/anydoc-wasm) -# 4. node/package.json .version -# 5. node/index.js generated version guard (contains `!== ''`) +# Verify that the release version agrees across the seven declared locations. # # Usage (from the repo root): # sh scripts/check-versions.sh # check agreement only # sh scripts/check-versions.sh v1.2.3 # also check that the tag matches -# -# On success prints the agreed version on stdout and exits 0. -# On failure prints actionable errors on stderr and exits 1. set -u err=0 report() { printf '%s\n' "$*" >&2; } -for f in Cargo.toml python/Cargo.toml wasm/Cargo.toml node/package.json node/index.js; do +for f in Cargo.toml python/Cargo.toml wasm/Cargo.toml node/package.json node/index.js go/Cargo.toml go/version.go; do if [ ! -f "$f" ]; then report "error: $f not found - run this script from the repository root." exit 1 fi done -# Read `version = "..."` only from the [package] section, not [dependencies] etc. toml_version() { awk ' /^\[package\]/ { in_package = 1; next } @@ -37,46 +28,41 @@ toml_version() { cargo_version=$(toml_version Cargo.toml) python_version=$(toml_version python/Cargo.toml) wasm_version=$(toml_version wasm/Cargo.toml) - -# npm keeps top-level "version" as the first version key in package.json. +go_version=$(toml_version go/Cargo.toml) package_json_version=$(sed -n 's/^[[:space:]]*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' node/package.json | head -n 1) - -# The generated Node loader embeds the version it was generated for in a guard -# containing the literal substring `!== ''`. guard_version=$(grep -o "!== '[^']*'" node/index.js | head -n 1 | sed "s/^!== '//; s/'\$//") +go_const_version=$(sed -n 's/^const Version = "\([^"]*\)".*/\1/p' go/version.go | head -n 1) [ -n "$cargo_version" ] || { report "error: could not read [package].version from Cargo.toml"; err=1; } [ -n "$python_version" ] || { report "error: could not read [package].version from python/Cargo.toml"; err=1; } [ -n "$wasm_version" ] || { report "error: could not read [package].version from wasm/Cargo.toml"; err=1; } [ -n "$package_json_version" ] || { report "error: could not read .version from node/package.json"; err=1; } -[ -n "$guard_version" ] || { report "error: could not find the version guard (!== '') in node/index.js - regenerate it with 'npm run build' in node/"; err=1; } +[ -n "$guard_version" ] || { report "error: could not find the version guard in node/index.js"; err=1; } +[ -n "$go_version" ] || { report "error: could not read [package].version from go/Cargo.toml"; err=1; } +[ -n "$go_const_version" ] || { report "error: could not read const Version from go/version.go"; err=1; } [ "$err" -eq 0 ] || exit 1 if [ "$python_version" != "$cargo_version" ] || \ [ "$wasm_version" != "$cargo_version" ] || \ [ "$package_json_version" != "$cargo_version" ] || \ - [ "$guard_version" != "$cargo_version" ]; then - report "error: the release version locations disagree:" + [ "$guard_version" != "$cargo_version" ] || \ + [ "$go_version" != "$cargo_version" ] || \ + [ "$go_const_version" != "$cargo_version" ]; then + report "error: release version locations disagree:" report " Cargo.toml [package].version = $cargo_version" report " python/Cargo.toml [package].version = $python_version" report " wasm/Cargo.toml [package].version = $wasm_version" report " node/package.json .version = $package_json_version" report " node/index.js version guard = $guard_version" - report "fix: set all five to the same version. Bump the three Cargo.toml files" - report " and node/package.json, then run 'npm run build' in node/ to" - report " regenerate node/index.js, and commit the result." + report " go/Cargo.toml [package].version = $go_version" + report " go/version.go const Version = $go_const_version" exit 1 fi -if [ "$#" -ge 1 ] && [ -n "$1" ]; then - tag=$1 - if [ "$tag" != "v$cargo_version" ]; then - report "error: tag '$tag' does not match the declared version '$cargo_version' (expected tag 'v$cargo_version')." - report "fix: either delete the tag and re-tag the commit that declares ${tag#v}," - report " or bump all four version locations to ${tag#v} and tag that commit." - exit 1 - fi +if [ "$#" -ge 1 ] && [ -n "$1" ] && [ "$1" != "v$cargo_version" ]; then + report "error: tag '$1' does not match the declared version '$cargo_version' (expected tag 'v$cargo_version')." + exit 1 fi printf '%s\n' "$cargo_version" diff --git a/scripts/gen-headers.sh b/scripts/gen-headers.sh new file mode 100755 index 00000000..4094c4c1 --- /dev/null +++ b/scripts/gen-headers.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Regenerate go/include/anydoc.h from the crate's C ABI. +# +# Run from the repo root: +# sh scripts/gen-headers.sh +# +# The header is committed; CI verifies it is up to date. End users who +# download the prebuilt libanydoc_go.a never run this. +set -e + +cargo build -p anydoc-go --release + +if [ ! -f go/include/anydoc.h ]; then + echo "error: go/include/anydoc.h was not generated" >&2 + exit 1 +fi + +echo "wrote go/include/anydoc.h" \ No newline at end of file diff --git a/skills/convert-documents-to-markdown/SKILL.md b/skills/convert-documents-to-markdown/SKILL.md index d4a0591e..8be16674 100644 --- a/skills/convert-documents-to-markdown/SKILL.md +++ b/skills/convert-documents-to-markdown/SKILL.md @@ -23,4 +23,4 @@ Rules: 3. Exit codes: 0 success, 1 the document could not be converted, 2 usage error. Failures print one `anydoc: ` line to stderr. The CLI never prompts. 4. For a large document, write to a file with `-o` and read the parts you need instead of streaming everything into context. 5. Scanned and image-only PDFs need OCR, which anydoc does not do; they fail as unsupported. The hosted [Firecrawl Parse](https://firecrawl.dev/parse) API handles those. -6. Inside a Node, Python, or Rust codebase, prefer the library over shelling out: `@firecrawl/anydoc` on npm, `firecrawl-anydoc` on PyPI, `anydoc` on crates.io. Each exposes the same `to_markdown` / `toMarkdown` API. +6. Inside a Node, Python, Go, or Rust codebase, prefer the library over shelling out: `@firecrawl/anydoc` on npm, `firecrawl-anydoc` on PyPI, `github.com/firecrawl/anydoc/go`, `anydoc` on crates.io. Each exposes the same `to_markdown` / `toMarkdown` API.