diff --git a/Cargo.lock b/Cargo.lock index 3319fd4..9261bbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,72 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "morph-da-codec-ffi" +version = "0.1.0" +dependencies = [ + "morph-da-decoder-ffi", + "morph-da-encoder-ffi", +] + [[package]] name = "morph-da-decoder-core" version = "0.1.0" @@ -23,6 +83,33 @@ dependencies = [ "morph-da-decoder-core", ] +[[package]] +name = "morph-da-encoder-core" +version = "0.1.0" +dependencies = [ + "morph-da-decoder-core", + "zstd", +] + +[[package]] +name = "morph-da-encoder-ffi" +version = "0.1.0" +dependencies = [ + "morph-da-encoder-core", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "ruzstd" version = "0.8.3" @@ -32,8 +119,54 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "git+https://github.com/morph-l2/zstd-rs.git?branch=zstd-dev#147bc25144f57278a538028df58afe8ae5e1dbe3" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "git+https://github.com/morph-l2/zstd-rs.git?branch=zstd-dev#147bc25144f57278a538028df58afe8ae5e1dbe3" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.9+zstd.1.5.5" +source = "git+https://github.com/morph-l2/zstd-rs.git?branch=zstd-dev#147bc25144f57278a538028df58afe8ae5e1dbe3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index da035bb..31b0650 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,9 @@ members = [ "crates/decoder/core", "crates/decoder/zstd-ffi", + "crates/encoder/core", + "crates/encoder/zstd-ffi", + "crates/codec-ffi", ] resolver = "3" @@ -12,6 +15,20 @@ license = "MIT OR Apache-2.0" publish = false [workspace.dependencies] +# Internal crates: pure-Rust logic lives in the `core` crates (reusable by other +# Rust projects as plain rlibs); the `ffi` crates hold the C ABI; `codec-ffi` is +# the single staticlib aggregator linked by Go. morph-da-decoder-core = { path = "crates/decoder/core" } +morph-da-decoder-ffi = { path = "crates/decoder/zstd-ffi" } +morph-da-encoder-core = { path = "crates/encoder/core" } +morph-da-encoder-ffi = { path = "crates/encoder/zstd-ffi" } + +# External: decoder uses the pure-Rust `ruzstd`; encoder wraps the C `libzstd` +# via the `zstd` crate. Their C symbols do not overlap. ruzstd = "0.8.3" -hex = "0.4.3" \ No newline at end of file +# `experimental` exposes LiteralCompressionMode / include_magicbytes (Magicless +# frame format), required by the Morph DA encoder parameter rules. +zstd = { git = "https://github.com/morph-l2/zstd-rs.git", branch = "zstd-dev", features = [ + "experimental", +] } +hex = "0.4.3" diff --git a/Makefile b/Makefile index 8980878..72eb950 100644 --- a/Makefile +++ b/Makefile @@ -19,23 +19,23 @@ else GOARCH := unsupported endif -FFI_LIB := target/release/libmorph_da_zstd_decode.a -GO_LIB := bindings/decoder/zstd/libmorph_da_zstd_decode_$(GOOS)_$(GOARCH).a -LINUX_AMD64_LIB := bindings/decoder/zstd/libmorph_da_zstd_decode_linux_amd64.a +FFI_LIB := target/release/libmorph_da_zstd.a +GO_LIB := bindings/codec/zstd/libmorph_da_zstd_$(GOOS)_$(GOARCH).a +LINUX_AMD64_LIB := bindings/codec/zstd/libmorph_da_zstd_linux_amd64.a CENTOS7_BUILD_OUT := build/out -CENTOS7_FFI_LIB := $(CENTOS7_BUILD_OUT)/libmorph_da_zstd_decode_linux_amd64.a +CENTOS7_FFI_LIB := $(CENTOS7_BUILD_OUT)/libmorph_da_zstd_linux_amd64.a DARWIN_DEPLOYMENT_TARGET ?= 11.0 ifeq ($(GOOS),darwin) FFI_BUILD_ENV := MACOSX_DEPLOYMENT_TARGET=$(DARWIN_DEPLOYMENT_TARGET) endif -.PHONY: build-zstd-ffi install-decoder-bindings build-zstd-ffi-linux-amd64 install-decoder-bindings-linux-amd64 linux-amd64-decoder-bindings test rust-test go-test clean +.PHONY: build-zstd-ffi install-codec-bindings build-zstd-ffi-linux-amd64 install-codec-bindings-linux-amd64 linux-amd64-codec-bindings test rust-test go-test clean build-zstd-ffi: - $(FFI_BUILD_ENV) cargo build -p morph-da-decoder-ffi --release + $(FFI_BUILD_ENV) cargo build -p morph-da-codec-ffi --release -install-decoder-bindings: build-zstd-ffi +install-codec-bindings: build-zstd-ffi cp $(FFI_LIB) $(GO_LIB) build-zstd-ffi-linux-amd64: @@ -45,19 +45,19 @@ build-zstd-ffi-linux-amd64: --output type=local,dest=$(CENTOS7_BUILD_OUT) \ . -install-decoder-bindings-linux-amd64: build-zstd-ffi-linux-amd64 +install-codec-bindings-linux-amd64: build-zstd-ffi-linux-amd64 cp $(CENTOS7_FFI_LIB) $(LINUX_AMD64_LIB) rust-test: cargo test --workspace -go-test: install-decoder-bindings - cd bindings/decoder && go test ./... +go-test: install-codec-bindings + cd bindings/codec && go test ./... test: rust-test go-test clean: cargo clean - rm -f bindings/decoder/zstd/*.a + rm -f bindings/codec/zstd/*.a rm -rf $(CENTOS7_BUILD_OUT) diff --git a/README.md b/README.md index 957583d..5d10c2d 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,29 @@ # morph-da-codec -Morph DA zstd decoder shared by Rust zkVM guest code and Go node code.(There will be encoder in the future) +Morph DA zstd codec (encoder + decoder) shared by Rust zkVM guest code and Go node code. + +The encoder and decoder are compiled into a single staticlib and exposed to Go through the `bindings/codec/zstd` package: + +- `CompressMorphDABatch` — compress a batch into a magic-less zstd frame; +- `DecompressMorphDABatch` — decompress a frame produced by the encoder. ## Semantics -- input payload omits zstd magic bytes; -- decoder prepends `28 b5 2f fd` internally; -- only the first complete zstd frame is decoded; -- bytes after the first frame are ignored; +Encoder and decoder share the same frame format: + +- the frame omits the zstd magic bytes (`28 b5 2f fd`); the decoder prepends them internally; +- a single frame is produced, with content size included so the decoder can validate it; +- literal compression is disabled (raw literal bytes) and the window log is capped at 17. + +Decoder specifics: + +- only the first complete frame is decoded; bytes after it are ignored; - first frame content size is required and validated. ## Build ```sh make build-zstd-ffi -make install-decoder-bindings +make install-codec-bindings make test ``` diff --git a/bindings/codec/README.md b/bindings/codec/README.md new file mode 100644 index 0000000..b8a30d6 --- /dev/null +++ b/bindings/codec/README.md @@ -0,0 +1,117 @@ +# Morph DA Codec Go Integration Guide + +Go bindings for the Morph DA zstd codec (compress + decompress) via cgo over a Rust FFI static library. + +## Package Paths + +```text +module: github.com/morph-l2/morph-da-codec/bindings/codec +codec: github.com/morph-l2/morph-da-codec/bindings/codec/zstd +``` + +## Core API + +```go +func DecompressMorphDABatch(payload []byte) ([]byte, error) +func CompressMorphDABatch(batch []byte) ([]byte, error) +``` + +## Adding the Dependency + +Import through Go modules: + +```sh +go get github.com/morph-l2/morph-da-codec/bindings/codec@ +``` + +For local development, add a `replace` directive to your application's `go.mod`: + +```go +require github.com/morph-l2/morph-da-codec/bindings/codec v0.0.0 + +replace github.com/morph-l2/morph-da-codec/bindings/codec => /absolute/path/to/morph-da-codec/bindings/codec +``` + +Then import: + +```go +import "github.com/morph-l2/morph-da-codec/bindings/codec/zstd" +``` + +## Decompression + +```go +decoded, err := zstd.DecompressMorphDABatch(payload) +if err != nil { + if errors.Is(err, zstd.ErrEmptyInput) { + // handle empty payload + } + log.Fatalf("decompress morph da batch: %v", err) +} +``` + +Semantics: + +- The input payload does not include the zstd magic bytes; the decoder prepends them internally (`28 b5 2f fd`). +- Only the first complete zstd frame is decoded; trailing bytes are ignored. +- The first frame must carry a content size, which is validated against the decompressed length. +- Empty input returns `ErrEmptyInput`. A `[0, 0, 0, ...]` payload returns an empty slice (an empty batch). + +## Compression + +```go +compressed, err := zstd.CompressMorphDABatch(batch) +if err != nil { + log.Fatalf("compress morph da batch: %v", err) +} +``` + +Semantics: + +- Produces a single magic-less zstd frame, matching what `DecompressMorphDABatch` expects (no `28 b5 2f fd` prefix). +- Empty input returns `ErrEmptyInput`. +- Round-trips: `DecompressMorphDABatch(CompressMorphDABatch(batch))` returns the original batch. + +## Errors + +- `zstd.ErrEmptyInput` — the input is empty. +- `zstd.ErrInvalidDecompressedLength` / `zstd.ErrInvalidCompressedLength` — the length returned by the FFI is invalid or exceeds the representable range of `int` on the current platform. +- Other errors come from the underlying Rust FFI layer, prefixed with `morph da zstd decode:` (decompress) or `morph da zstd encode:` (compress). + +## Native Static Library + +These bindings use cgo to call a Rust FFI static library. The platform-specific archive must exist in the `zstd` package directory: + +```text +libmorph_da_zstd_${GOOS}_${GOARCH}.a +``` + +Declared cgo link configurations cover: + +- `darwin/arm64` +- `linux/amd64` + +Build and install the library for the current host, then test: + +```sh +make install-codec-bindings # generate + install the static library +make go-test # run Go tests +make test # full test suite +``` + +## cgo Notes + +cgo must be enabled: + +```sh +CGO_ENABLED=1 go build ./... +CGO_ENABLED=1 go test ./... +``` + +Cross-compilation additionally requires: matching `GOOS`/`GOARCH`, a C toolchain for the target platform, and the target's `libmorph_da_zstd_${GOOS}_${GOARCH}.a` built and present under `zstd`. + +## Security and Resource Considerations + +- Output buffers are sized from the frame content size (decompress) or the compress bound. Avoid calling these APIs on untrusted, unbounded payloads; validate payload size, source, and batch boundaries first. +- Trailing bytes are ignored on decompress. If your semantics require exactly one frame, validate that in the upper layer. +- Deployment images and build artifacts must include the platform-specific `.a` file and a cgo build environment. diff --git a/bindings/codec/go.mod b/bindings/codec/go.mod new file mode 100644 index 0000000..d761984 --- /dev/null +++ b/bindings/codec/go.mod @@ -0,0 +1,3 @@ +module github.com/morph-l2/morph-da-codec/bindings/codec + +go 1.24 diff --git a/bindings/decoder/zstd/batch_test.go b/bindings/codec/zstd/batch_test.go similarity index 79% rename from bindings/decoder/zstd/batch_test.go rename to bindings/codec/zstd/batch_test.go index a0f3cbf..eec0076 100644 --- a/bindings/decoder/zstd/batch_test.go +++ b/bindings/codec/zstd/batch_test.go @@ -1,6 +1,7 @@ package zstd import ( + "crypto/sha256" "encoding/hex" "os" "path/filepath" @@ -33,10 +34,22 @@ func TestDecompressesBatch(t *testing.T) { batchData = append(batchData, decodeBlob(t, readHex(t, path))...) } + // compressedLen := compressedFrameLen(t, batchData) decoded, err := DecompressMorphDABatch(batchData) if err != nil { t.Fatal(err) } + + // Compression equivalence Test + recompressed, err := CompressMorphDABatch(decoded) + if err != nil { + t.Fatal(err) + } + originalCompressedHash := sha256.Sum256(batchData[:len(recompressed)]) + recompressedHash := sha256.Sum256(recompressed) + if originalCompressedHash != recompressedHash { + t.Fatalf("compressed hash mismatch: got %x, want %x ,recompressed len %d", recompressedHash, originalCompressedHash, len(recompressed)) + } t.Logf("decoded len %d", len(decoded)) }) } diff --git a/bindings/codec/zstd/compress.go b/bindings/codec/zstd/compress.go new file mode 100644 index 0000000..27363a3 --- /dev/null +++ b/bindings/codec/zstd/compress.go @@ -0,0 +1,73 @@ +package zstd + +/* +#include + +const char* morph_da_zstd_compress_bound( + const uint8_t* src, + uint64_t src_size, + uint64_t* output_size +); + +const char* morph_da_zstd_compress( + const uint8_t* src, + uint64_t src_size, + uint8_t* output_buf, + uint64_t* output_buf_size +); +*/ +import "C" + +import ( + "errors" + "fmt" + "unsafe" +) + +var ErrInvalidCompressedLength = errors.New("invalid compressed length") + +// CompressMorphDABatch compresses a Morph DA batch into a single magic-less +// zstd frame, matching the parameters DecompressMorphDABatch expects. +func CompressMorphDABatch(batch []byte) ([]byte, error) { + if len(batch) == 0 { + return nil, ErrEmptyInput + } + + src := (*C.uint8_t)(unsafe.Pointer(unsafe.SliceData(batch))) + srcSize := C.uint64_t(len(batch)) + + var bound C.uint64_t + if err := C.morph_da_zstd_compress_bound(src, srcSize, &bound); err != nil { + return nil, encodeError(err) + } + + boundLen := uint64(bound) + if uint64(int(boundLen)) != boundLen { + return nil, ErrInvalidCompressedLength + } + + output := make([]byte, int(boundLen)) + var outputSize = bound + var outputPtr *C.uint8_t + if len(output) != 0 { + outputPtr = (*C.uint8_t)(unsafe.Pointer(unsafe.SliceData(output))) + } + + if err := C.morph_da_zstd_compress(src, srcSize, outputPtr, &outputSize); err != nil { + return nil, encodeError(err) + } + + outputLen := uint64(outputSize) + if outputLen > uint64(len(output)) { + return nil, ErrInvalidCompressedLength + } + if uint64(int(outputLen)) != outputLen { + return nil, ErrInvalidCompressedLength + } + + return output[:int(outputLen)], nil +} + +func encodeError(err *C.char) error { + return fmt.Errorf("morph da zstd encode: %s", C.GoString(err)) +} diff --git a/bindings/codec/zstd/compress_test.go b/bindings/codec/zstd/compress_test.go new file mode 100644 index 0000000..c2789ea --- /dev/null +++ b/bindings/codec/zstd/compress_test.go @@ -0,0 +1,54 @@ +package zstd + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "testing" +) + +func TestCompressMorphDABatchRoundTrip(t *testing.T) { + original := bytes.Repeat([]byte("morph da batch payload \x00\x01\x02 round trip "), 64) + + compressed, err := CompressMorphDABatch(original) + if err != nil { + t.Fatal(err) + } + if len(compressed) == 0 { + t.Fatal("compressed output is empty") + } + // Magic-less frame: must not start with the zstd magic bytes 0x28 0xB5 0x2F 0xFD. + if bytes.HasPrefix(compressed, []byte{0x28, 0xB5, 0x2F, 0xFD}) { + t.Fatal("compressed output unexpectedly carries the zstd magic bytes") + } + + decoded, err := DecompressMorphDABatch(compressed) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(decoded, original) { + t.Fatalf("round-trip mismatch: got %d bytes, want %d", len(decoded), len(original)) + } +} + +func TestCompressMorphDABatchRejectsEmptyInput(t *testing.T) { + _, err := CompressMorphDABatch(nil) + if !errorsIs(err, ErrEmptyInput) { + t.Fatalf("expected ErrEmptyInput, got %v", err) + } +} + +func TestCompressMorphDABatchHelloWorldHash(t *testing.T) { + compressed, err := CompressMorphDABatch([]byte("hello world")) + if err != nil { + t.Fatal(err) + } + if len(compressed) == 0 { + t.Fatal("compressed output is empty") + } + + hash := sha256.Sum256(compressed) + if got, want := hex.EncodeToString(hash[:]), "5850cf750ce5ac2ccea3d4d0baeb85e3645297ca72df11c7c1c2b2dfbc1eb015"; got != want { + t.Fatalf("compressed hash mismatch: got %s, want %s", got, want) + } +} diff --git a/bindings/decoder/zstd/zstd.go b/bindings/codec/zstd/decompress.go similarity index 100% rename from bindings/decoder/zstd/zstd.go rename to bindings/codec/zstd/decompress.go diff --git a/bindings/decoder/zstd/zstd_test.go b/bindings/codec/zstd/decompress_test.go similarity index 100% rename from bindings/decoder/zstd/zstd_test.go rename to bindings/codec/zstd/decompress_test.go diff --git a/bindings/decoder/zstd/libmorph_da_zstd_decode_darwin_arm64.a b/bindings/codec/zstd/libmorph_da_zstd_darwin_arm64.a similarity index 90% rename from bindings/decoder/zstd/libmorph_da_zstd_decode_darwin_arm64.a rename to bindings/codec/zstd/libmorph_da_zstd_darwin_arm64.a index 3c65cc3..e5e3a56 100644 Binary files a/bindings/decoder/zstd/libmorph_da_zstd_decode_darwin_arm64.a and b/bindings/codec/zstd/libmorph_da_zstd_darwin_arm64.a differ diff --git a/bindings/decoder/zstd/libmorph_da_zstd_decode_darwin_arm64.go b/bindings/codec/zstd/libmorph_da_zstd_darwin_arm64.go similarity index 51% rename from bindings/decoder/zstd/libmorph_da_zstd_decode_darwin_arm64.go rename to bindings/codec/zstd/libmorph_da_zstd_darwin_arm64.go index e9b7cf0..dc6ccd0 100644 --- a/bindings/decoder/zstd/libmorph_da_zstd_decode_darwin_arm64.go +++ b/bindings/codec/zstd/libmorph_da_zstd_darwin_arm64.go @@ -3,6 +3,6 @@ package zstd /* -#cgo LDFLAGS: ${SRCDIR}/libmorph_da_zstd_decode_darwin_arm64.a +#cgo LDFLAGS: ${SRCDIR}/libmorph_da_zstd_darwin_arm64.a */ import "C" diff --git a/bindings/decoder/zstd/libmorph_da_zstd_decode_linux_amd64.a b/bindings/codec/zstd/libmorph_da_zstd_linux_amd64.a similarity index 90% rename from bindings/decoder/zstd/libmorph_da_zstd_decode_linux_amd64.a rename to bindings/codec/zstd/libmorph_da_zstd_linux_amd64.a index 5ba7a48..c182b16 100644 Binary files a/bindings/decoder/zstd/libmorph_da_zstd_decode_linux_amd64.a and b/bindings/codec/zstd/libmorph_da_zstd_linux_amd64.a differ diff --git a/bindings/codec/zstd/libmorph_da_zstd_linux_amd64.go b/bindings/codec/zstd/libmorph_da_zstd_linux_amd64.go new file mode 100644 index 0000000..b210ad7 --- /dev/null +++ b/bindings/codec/zstd/libmorph_da_zstd_linux_amd64.go @@ -0,0 +1,8 @@ +//go:build linux && amd64 && cgo && !musl + +package zstd + +/* +#cgo LDFLAGS: ${SRCDIR}/libmorph_da_zstd_linux_amd64.a -ldl -lpthread -lm +*/ +import "C" diff --git a/bindings/decoder/README.md b/bindings/decoder/README.md deleted file mode 100644 index c8a684b..0000000 --- a/bindings/decoder/README.md +++ /dev/null @@ -1,170 +0,0 @@ -# Morph DA Codec Go Integration Guide - -This document explains how to integrate the Morph DA first-frame zstd decoder into Go projects. - -## Package Paths - -Go module: - -```text -github.com/morph-l2/morph-da-codec/bindings/decoder -``` - -Decoder package path: - -```text -github.com/morph-l2/morph-da-codec/bindings/decoder/zstd -``` - -Core API: - -```go -func DecompressMorphDABatch(payload []byte) ([]byte, error) -``` - -## Adding the Dependency - -### Option 1: Import directly through Go modules - -Run the following command in your application project: - -```sh -go get github.com/morph-l2/morph-da-codec/bindings/decoder@ -``` - -Alternatively, add it manually to `go.mod`: - -```go -require github.com/morph-l2/morph-da-codec/bindings/decoder v0.0.0- -``` - -Then import it in your code: - -```go -import "github.com/morph-l2/morph-da-codec/bindings/decoder/zstd" -``` - -### Option 2: Local development integration - -If your application project needs to integrate with local source code during development, add a `replace` directive to the application's `go.mod`: - -```go -require github.com/morph-l2/morph-da-codec/bindings/decoder v0.0.0 - -replace github.com/morph-l2/morph-da-codec/bindings/decoder => /absolute/path/to/morph-da-codec/bindings/decoder -``` - -## Usage Example - -```go -package main - -import ( - "fmt" - "log" - - "github.com/morph-l2/morph-da-codec/bindings/decoder/zstd" -) - -func main() { - var payload []byte - - decoded, err := zstd.DecompressMorphDABatch(payload) - if err != nil { - log.Fatalf("decompress morph da batch: %v", err) - } - - fmt.Printf("decoded size: %d\n", len(decoded)) -} -``` - -If `payload` is empty, an `empty input` error is returned. - -If `payload` is `[0, 0, 0, ...]`, an empty byte slice is returned, which represents an empty batch. - -## Native Static Library Requirements - -This Go binding uses cgo to call a Rust FFI static library. Integrators must ensure that the static library for the target platform exists in the `bindings/decoder/zstd` package directory: - -```text -libmorph_da_zstd_decode_${GOOS}_${GOARCH}.a -``` - -The current Go binding declares cgo link configurations for the following platforms: - -- `darwin/arm64` -- `linux/amd64` - -Generate and install the static library for the current host platform in this repository: - -```sh -make install-decoder-bindings -``` - -Run Go tests: - -```sh -make go-test -``` - -Run the full test suite: - -```sh -make test -``` - -## cgo Notes - -Application projects must enable cgo: - -```sh -CGO_ENABLED=1 go test ./... -CGO_ENABLED=1 go build ./... -``` - -For cross-compilation, all of the following requirements must be satisfied: - -1. `GOOS` / `GOARCH` must match the target platform; -2. `CGO_ENABLED=1`; -3. A usable C toolchain for the target platform must be installed on the build host; -4. The target platform's `libmorph_da_zstd_decode_${GOOS}_${GOARCH}.a` must exist under `bindings/decoder/zstd`; -5. The Rust FFI static library must be correctly built for the target platform. - -## Decoding Semantics - -`DecompressMorphDABatch` takes a Morph DA batch compressed payload as input. Its semantics are as follows: - -- The input payload does not include zstd magic bytes; -- The decoder automatically prepends the zstd magic bytes internally: `28 b5 2f fd`; -- Only the first complete zstd frame is decoded; -- Trailing bytes after the first frame are ignored; -- The first frame must contain a content size, and the decompressed length is validated. - -## Error Handling - -Typical errors that may be returned: - -- `zstd.ErrEmptyInput`: the input is empty; -- `zstd.ErrInvalidDecompressedLength`: the decompressed length returned by the FFI is invalid or exceeds the representable range of `int` on the current Go platform; -- Other errors: decoding errors returned by the underlying zstd/Rust FFI layer. Their error messages are prefixed with `morph da zstd decode:`. - -Example: - -```go -decoded, err := zstd.DecompressMorphDABatch(payload) -if err != nil { - if errors.Is(err, zstd.ErrEmptyInput) { - // handle empty payload - } - return err -} - -_ = decoded -``` - -## Security and Resource Considerations - -- The decompressed output buffer is allocated at once based on the frame content size. Integrators should avoid calling this API without limits on untrusted oversized payloads; -- The upper-layer application is advised to validate the compressed payload size, source, and batch boundaries before calling this API; -- Trailing bytes are ignored. If the application semantics require the payload to contain exactly one frame, additional validation must be performed by the upper layer; -- This binding depends on a native static library. Deployment images or build artifacts must include the platform-specific `.a` file and a cgo build environment. \ No newline at end of file diff --git a/bindings/decoder/go.mod b/bindings/decoder/go.mod deleted file mode 100644 index eff6a3f..0000000 --- a/bindings/decoder/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/morph-l2/morph-da-codec/bindings/decoder - -go 1.24 diff --git a/bindings/decoder/zstd/libmorph_da_zstd_decode_linux_amd64.go b/bindings/decoder/zstd/libmorph_da_zstd_decode_linux_amd64.go deleted file mode 100644 index 6f35af0..0000000 --- a/bindings/decoder/zstd/libmorph_da_zstd_decode_linux_amd64.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build linux && amd64 && cgo && !musl - -package zstd - -/* -#cgo LDFLAGS: ${SRCDIR}/libmorph_da_zstd_decode_linux_amd64.a -ldl -lpthread -lm -*/ -import "C" diff --git a/build/Dockerfile b/build/Dockerfile index 4d6a3fc..c1703b5 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,5 +1,6 @@ # syntax=docker/dockerfile:1.6 +# Builds artifacts compatible with most Linux distros and kernels. FROM quay.io/pypa/manylinux2014_x86_64 AS builder ARG RUST_VERSION=1.93.0 @@ -20,11 +21,11 @@ COPY Cargo.toml Cargo.lock ./ COPY crates ./crates COPY bindings ./bindings -RUN cargo build -p morph-da-decoder-ffi --release --locked +RUN cargo build -p morph-da-codec-ffi --release --locked RUN mkdir -p /out && \ - cp target/release/libmorph_da_zstd_decode.a \ - /out/libmorph_da_zstd_decode_linux_amd64.a + cp target/release/libmorph_da_zstd.a \ + /out/libmorph_da_zstd_linux_amd64.a FROM scratch AS artifact diff --git a/crates/codec-ffi/Cargo.toml b/crates/codec-ffi/Cargo.toml new file mode 100644 index 0000000..30a56c9 --- /dev/null +++ b/crates/codec-ffi/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "morph-da-codec-ffi" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[lib] +# The ONLY staticlib in the workspace. Go links exactly this `.a`, so the final +# binary contains a single copy of the Rust runtime (no `__rust_alloc` clash). +# Produces `libmorph_da_zstd.a`, linked by the `bindings/codec` Go module. +name = "morph_da_zstd" +crate-type = ["staticlib"] + +[dependencies] +morph-da-decoder-ffi.workspace = true +morph-da-encoder-ffi.workspace = true diff --git a/crates/codec-ffi/src/lib.rs b/crates/codec-ffi/src/lib.rs new file mode 100644 index 0000000..47be0da --- /dev/null +++ b/crates/codec-ffi/src/lib.rs @@ -0,0 +1,10 @@ +//! Single staticlib aggregating the Morph DA decoder and encoder C ABIs. +//! +//! This crate holds no logic of its own. It exists solely to produce one `.a` +//! for Go to link, bundling exactly one copy of the Rust runtime. The +//! decoder/encoder FFI crates are plain rlibs; re-exporting their items here +//! keeps their `#[unsafe(no_mangle)]` symbols reachable from the staticlib root +//! so the linker does not strip them as dead code. + +pub use morph_da_decoder_ffi::*; +pub use morph_da_encoder_ffi::*; diff --git a/crates/decoder/zstd-ffi/Cargo.toml b/crates/decoder/zstd-ffi/Cargo.toml index 83f9543..60eb5eb 100644 --- a/crates/decoder/zstd-ffi/Cargo.toml +++ b/crates/decoder/zstd-ffi/Cargo.toml @@ -5,8 +5,10 @@ edition.workspace = true publish.workspace = true [lib] -name = "morph_da_zstd_decode" -crate-type = ["staticlib"] +# Plain rlib so it can be aggregated into the single `codec-ffi` staticlib. +# Producing a staticlib here would bundle a second copy of the Rust runtime and +# clash with the encoder's runtime at link time. +crate-type = ["rlib"] [dependencies] morph-da-decoder-core.workspace = true diff --git a/crates/encoder/core/Cargo.toml b/crates/encoder/core/Cargo.toml new file mode 100644 index 0000000..ab15c3f --- /dev/null +++ b/crates/encoder/core/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "morph-da-encoder-core" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[dependencies] +# Wraps the C `libzstd` for compression. Bundled into the codec staticlib; its +# C symbols (`ZSTD_*`) do not overlap with the decoder's pure-Rust `ruzstd`. +zstd.workspace = true + +[dev-dependencies] +# Round-trip tests: encode here, decode with the sibling decoder. +morph-da-decoder-core.workspace = true diff --git a/crates/encoder/core/src/lib.rs b/crates/encoder/core/src/lib.rs new file mode 100644 index 0000000..7a70ba7 --- /dev/null +++ b/crates/encoder/core/src/lib.rs @@ -0,0 +1,131 @@ +//! Morph DA zstd encoder (compression). +//! +//! Mirror of [`morph_da_decoder_core`](../../decoder/core): pure-Rust API with +//! no C ABI, so it can be reused directly by other Rust crates. Compression is +//! delegated to the C `libzstd` via the [`zstd`] crate. +//! +//! The frame is produced with magic bytes omitted, matching the decoder which +//! restores them before decoding. + +use std::{fmt, io::Write}; + +use zstd::{ + stream::Encoder, + zstd_safe::{CParameter, ParamSwitch}, +}; + +// we use offset window no more than = 17 +pub const CL_WINDOW_LIMIT: u32 = 17; + +/// zstd block size target. +pub const N_BLOCK_SIZE_TARGET: u32 = 124 * 1024; + +/// Errors that can occur while encoding a Morph DA zstd payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + EmptyInput, + OutputBufferTooSmall, + InternalEncoderError, +} + +impl Error { + /// Returns a static, human-readable description of the error. + pub const fn as_str(self) -> &'static str { + match self { + Self::EmptyInput => "empty input", + Self::OutputBufferTooSmall => "output buffer too small", + Self::InternalEncoderError => "internal encoder error", + } + } +} + +impl std::error::Error for Error {} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +pub type Result = std::result::Result; + +/// Builds the zstd encoder configured for Morph DA frames. +/// +/// All parameter calls use compile-time-constant values, so they cannot fail in +/// practice; failures here indicate a broken zstd build and are treated as +/// `expect`-level invariants. +pub(crate) fn init_zstd_encoder(target_block_size: u32) -> Encoder<'static, Vec> { + let mut encoder = Encoder::new(Vec::new(), 0).expect("infallible"); + + // disable compression of literals, i.e. literals will be raw bytes. + encoder + .set_parameter(CParameter::LiteralCompressionMode(ParamSwitch::Disable)) + .expect("infallible"); + // Set window log = 17 + encoder + .set_parameter(CParameter::WindowLog(CL_WINDOW_LIMIT)) + .expect("infallible"); + // set target block size to fit within a single block. + encoder + .set_parameter(CParameter::TargetCBlockSize(target_block_size)) + .expect("infallible"); + // do not include the checksum at the end of the encoded data. + encoder.include_checksum(false).expect("infallible"); + // do not include magic bytes at the start of the frame since we will have a + // single frame. + encoder.include_magicbytes(false).expect("infallible"); + // do not include dictionary id so we have more simple content + encoder.include_dictid(false).expect("infallible"); + // include the content size to know at decode time the expected size of + // decoded data. + encoder.include_contentsize(true).expect("infallible"); + + encoder +} + +/// Upper bound on the compressed size of `src_len` input bytes. +/// +/// `ZSTD_compressBound` accounts for worst-case expansion; our magic-less frame +/// is never larger, so this is a safe output-buffer size. +pub fn compressed_size_bound(src_len: usize) -> usize { + zstd::zstd_safe::compress_bound(src_len) +} + +/// Compresses a Morph DA batch into a single magic-less zstd frame. +pub fn compress_morph_da_zstd(batch: &[u8]) -> Result> { + let mut encoder = init_zstd_encoder(N_BLOCK_SIZE_TARGET); + encoder + .set_pledged_src_size(Some(batch.len() as u64)) + .map_err(|_| Error::InternalEncoderError)?; + encoder + .write_all(batch) + .map_err(|_| Error::InternalEncoderError)?; + + encoder.finish().map_err(|_| Error::InternalEncoderError) +} + +#[cfg(test)] +mod tests { + use super::*; + use morph_da_decoder_core::decompress_morph_da_zstd; + + #[test] + fn round_trips_through_decoder() { + let original = b"morph da batch payload \x00\x01\x02 repeated repeated repeated".repeat(50); + + let encoded = compress_morph_da_zstd(&original).expect("compress"); + // Magicless frame: must not start with the zstd magic bytes. + assert_ne!(&encoded[..4], &morph_da_decoder_core::ZSTD_MAGIC_BYTES); + + let decoded = decompress_morph_da_zstd(&encoded).expect("decompress"); + assert_eq!(decoded, original); + } + + #[test] + fn compresses_empty_input() { + // zstd produces a valid (small) frame for empty input; round-trip of an + // empty payload is handled by the FFI/Go layer, not core. + let encoded = compress_morph_da_zstd(b"").expect("compress empty"); + assert!(!encoded.is_empty()); + } +} diff --git a/crates/encoder/zstd-ffi/Cargo.toml b/crates/encoder/zstd-ffi/Cargo.toml new file mode 100644 index 0000000..6c5629f --- /dev/null +++ b/crates/encoder/zstd-ffi/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "morph-da-encoder-ffi" +version.workspace = true +edition.workspace = true +publish.workspace = true + +[lib] +# Plain rlib, aggregated into `codec-ffi` (see decoder/zstd-ffi for rationale). +crate-type = ["rlib"] + +[dependencies] +morph-da-encoder-core.workspace = true diff --git a/crates/encoder/zstd-ffi/src/lib.rs b/crates/encoder/zstd-ffi/src/lib.rs new file mode 100644 index 0000000..1fee9c7 --- /dev/null +++ b/crates/encoder/zstd-ffi/src/lib.rs @@ -0,0 +1,140 @@ +//! C ABI for the Morph DA zstd encoder. +//! +//! Mirror of [`morph_da_decoder_ffi`](../../decoder/zstd-ffi): thin +//! `#[unsafe(no_mangle)] extern "C"` wrappers over [`morph_da_encoder_core`]. +//! These are compiled into the single `codec-ffi` staticlib, not a staticlib of +//! their own. + +use std::{ + os::raw::{c_char, c_uchar}, + panic::{self, AssertUnwindSafe}, + ptr, slice, +}; + +use morph_da_encoder_core::{Error, compress_morph_da_zstd as compress, compressed_size_bound}; + +const OK: *const c_char = ptr::null(); + +const EMPTY_INPUT: &[u8] = b"empty input\0"; +const OUTPUT_BUFFER_TOO_SMALL: &[u8] = b"output buffer too small\0"; +const INTERNAL_ENCODER_ERROR: &[u8] = b"internal encoder error\0"; + +/// Computes an upper bound on the compressed size. +/// +/// # Safety +/// +/// `src` must be valid for `src_size` readable bytes unless `src_size == 0`. +/// `output_size` must be non-null and valid for writing one `u64`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn morph_da_zstd_compress_bound( + src: *const c_uchar, + src_size: u64, + output_size: *mut u64, +) -> *const c_char { + match panic::catch_unwind(AssertUnwindSafe(|| unsafe { + compress_bound_impl(src, src_size, output_size) + })) { + Ok(Ok(())) => OK, + Ok(Err(err)) => err_ptr(err), + Err(_) => internal_encoder_error_ptr(), + } +} + +/// Compresses into a caller-provided buffer. +/// +/// # Safety +/// +/// `src` must be valid for `src_size` readable bytes unless `src_size == 0`. +/// `output_buf_size` must be non-null and valid for one `u64`. +/// If output is produced, `output_buf` must be valid for the input capacity and +/// must not overlap `src`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn morph_da_zstd_compress( + src: *const c_uchar, + src_size: u64, + output_buf: *mut c_uchar, + output_buf_size: *mut u64, +) -> *const c_char { + match panic::catch_unwind(AssertUnwindSafe(|| unsafe { + compress_impl(src, src_size, output_buf, output_buf_size) + })) { + Ok(Ok(())) => OK, + Ok(Err(err)) => err_ptr(err), + Err(_) => internal_encoder_error_ptr(), + } +} + +unsafe fn compress_bound_impl( + src: *const c_uchar, + src_size: u64, + output_size: *mut u64, +) -> morph_da_encoder_core::Result<()> { + if output_size.is_null() { + return Err(Error::InternalEncoderError); + } + + let src = input_slice(src, src_size)?; + let bound = compressed_size_bound(src.len()); + unsafe { + *output_size = bound as u64; + } + Ok(()) +} + +unsafe fn compress_impl( + src: *const c_uchar, + src_size: u64, + output_buf: *mut c_uchar, + output_buf_size: *mut u64, +) -> morph_da_encoder_core::Result<()> { + if output_buf_size.is_null() { + return Err(Error::InternalEncoderError); + } + + let src = input_slice(src, src_size)?; + let capacity = unsafe { *output_buf_size }; + let encoded = compress(src)?; + + unsafe { + *output_buf_size = encoded.len() as u64; + } + + if encoded.len() as u64 > capacity { + return Err(Error::OutputBufferTooSmall); + } + if encoded.is_empty() { + return Ok(()); + } + if output_buf.is_null() { + return Err(Error::InternalEncoderError); + } + + unsafe { + ptr::copy_nonoverlapping(encoded.as_ptr(), output_buf, encoded.len()); + } + Ok(()) +} + +fn input_slice<'a>(src: *const c_uchar, src_size: u64) -> morph_da_encoder_core::Result<&'a [u8]> { + if src_size == 0 { + return Err(Error::EmptyInput); + } + if src.is_null() { + return Err(Error::InternalEncoderError); + } + + let len = usize::try_from(src_size).map_err(|_| Error::InternalEncoderError)?; + Ok(unsafe { slice::from_raw_parts(src, len) }) +} + +fn err_ptr(err: Error) -> *const c_char { + match err { + Error::EmptyInput => EMPTY_INPUT.as_ptr().cast(), + Error::OutputBufferTooSmall => OUTPUT_BUFFER_TOO_SMALL.as_ptr().cast(), + Error::InternalEncoderError => INTERNAL_ENCODER_ERROR.as_ptr().cast(), + } +} + +fn internal_encoder_error_ptr() -> *const c_char { + INTERNAL_ENCODER_ERROR.as_ptr().cast() +}