From 1bdaa4f7a2fea572a782b3a507c5f00da53fc8e6 Mon Sep 17 00:00:00 2001 From: Alona Enraght-Moony Date: Sun, 26 Jul 2026 17:14:01 +0000 Subject: [PATCH] rustdoc-json: Postcard output ## What `rustdoc --output-format=postcard` is like rustdoc-json, but using https://postcard.rs/ / https://docs.rs/postcard/1.1.1/ instead of JSON. ## Why JSON Size and speed isn't great. People [want](https://rust-lang.zulipchat.com/#narrow/channel/266220-t-rustdoc/topic/rustdoc_json.20ideas/with/524453896) [more](https://rust-lang.zulipchat.com/#narrow/channel/266220-t-rustdoc/topic/rustdoc-json.3A.20compressed.20output/with/462869885) [speed](https://rust-lang.zulipchat.com/#narrow/channel/266220-t-rustdoc/topic/.28De.29serialization.20speed.20of.20JSON.20docs), and smaller docs. There are proposals to make the JSON smaller (and therefor faster) by making field-names shorter, and omitting them when the value is the default. But ## How good is it? In a [very unscientific benchmark](https://github.com/aDotInTheVoid/rustdocjson-encoding-bench) for aws-sdk-ec2, it's ~3.6x smaller (255MiB vs 69 MiB) and ~1.8x faster to deserialize (1.6273 s vs 914.05 ms) ## What's the metaformat - 21 bytes of magic numbers - varint(u32) format version - `Crate` as usual This way, users can look at the magic number to check it's a rustdoc-json-postcard file, then read the version number to know if they can decode it. Only then can they deserialize the `Crate` itself. I plan to write a library that does this, so it's easy to do well. ## What about the name Right now, this means that the `rustdoc::json` module produces postcard & json output. And the overall feature is still called "rustdoc json". There should probably be a unified name for the both of these. Maybe something like "Rustdoc IR Output"? But that can wait for a follow-up commit I think. --- Cargo.lock | 2 + src/doc/rustdoc/src/unstable-features.md | 9 +++ src/librustdoc/Cargo.toml | 1 + src/librustdoc/config.rs | 37 ++++++---- src/librustdoc/core.rs | 3 +- src/librustdoc/json/mod.rs | 69 ++++++++++++------- src/librustdoc/lib.rs | 12 +++- src/rustdoc-json-types/lib.rs | 33 +++++++++ src/tools/compiletest/src/runtest.rs | 4 ++ .../compiletest/src/runtest/rustdoc_json.rs | 26 +++++-- src/tools/jsondoclint/Cargo.toml | 3 + src/tools/jsondoclint/src/main.rs | 59 +++++++++++++--- 12 files changed, 203 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5fc3a5ac71722..4605d4f7f82a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2232,6 +2232,7 @@ dependencies = [ "anyhow", "clap", "fs-err", + "postcard", "rustc-hash 2.1.1", "rustdoc-json-types", "serde", @@ -4990,6 +4991,7 @@ dependencies = [ "indexmap", "itertools", "minifier", + "postcard", "proc-macro2", "pulldown-cmark-escape", "regex", diff --git a/src/doc/rustdoc/src/unstable-features.md b/src/doc/rustdoc/src/unstable-features.md index ec03dba854578..d83af332c0b48 100644 --- a/src/doc/rustdoc/src/unstable-features.md +++ b/src/doc/rustdoc/src/unstable-features.md @@ -577,6 +577,15 @@ It can also be used with `--show-coverage`. Take a look at its [documentation](#--show-coverage-calculate-the-percentage-of-items-with-documentation) for more information. +### postcard + + * Tracking Issue: [#76578](https://github.com/rust-lang/rust/issues/76578) + +`--output-format postcard` emits documentation using [Postcard](https://postcard.jamesmunns.com/) as an encoding format. +This has exactly the same structure as the JSON format, but is a smaller file that's faster to serialize/deserialize. + +See the [API docs](https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc_json_types/postcard/index.html) for details. + ### doctest * Tracking issue: [#134529](https://github.com/rust-lang/rust/issues/134529) diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index 1da46d9f6328a..27924ac0e697c 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -15,6 +15,7 @@ base64 = "0.21.7" indexmap = { version = "2", features = ["serde"] } itertools = "0.15" minifier = { version = "0.4.0", default-features = false } +postcard = { version = "1.1.3", default-features = false, features = ["use-std"] } proc-macro2 = "1.0.103" pulldown-cmark-escape = { version = "0.11.0", features = ["simd"] } regex = "1" diff --git a/src/librustdoc/config.rs b/src/librustdoc/config.rs index 1584cacff688e..0a78c5bfd6e2b 100644 --- a/src/librustdoc/config.rs +++ b/src/librustdoc/config.rs @@ -34,13 +34,19 @@ pub(crate) enum OutputFormat { /// `--output-format=json` without `--show-coverage`. /// /// JSON description of crate API. - IrJson, + Ir(IrOutputFormat), /// `--output-format=json` with `--show-coverage`. CoverageJson, Html, Doctest, } +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum IrOutputFormat { + Json, + Postcard, +} + /// Either an input crate, markdown file, or nothing (--merge=finalize). pub(crate) enum InputMode { /// The `--merge=finalize` step does not need an input crate to rustdoc. @@ -314,7 +320,7 @@ pub(crate) enum EmitType { HtmlStaticFiles, HtmlNonStaticFiles, // not explicitly nameable by the user for now - IrJsonFiles, + IrFiles, CoverageJsonFiles, DepInfo(Option), } @@ -324,7 +330,7 @@ impl fmt::Display for EmitType { f.write_str(match self { Self::HtmlStaticFiles => "html-static-files", Self::HtmlNonStaticFiles => "html-non-static-files", - Self::IrJsonFiles => "ir-json-files", + Self::IrFiles => "ir-files", Self::CoverageJsonFiles => "coverage-json-files", Self::DepInfo(_) => "dep-info", }) @@ -469,9 +475,10 @@ impl Options { if show_coverage { OutputFormat::CoverageJson } else { - OutputFormat::IrJson + OutputFormat::Ir(IrOutputFormat::Json) } } + Some("postcard") => OutputFormat::Ir(IrOutputFormat::Postcard), Some("doctest") => OutputFormat::Doctest, Some(other) => dcx.fatal(format!("unknown output format `{other}`")), }; @@ -492,10 +499,15 @@ impl Options { // If `-Zunstable-options` is used, nothing to check after this point. (_, false, true) => {} (None | Some(OutputFormat::Html), false, _) => {} - (Some(OutputFormat::IrJson), false, false) => { - dcx.fatal( - "the -Z unstable-options flag must be passed to enable --output-format=json for documentation generation (see https://github.com/rust-lang/rust/issues/76578)", - ); + (Some(OutputFormat::Ir(irof)), false, false) => { + let flag = match irof { + IrOutputFormat::Json => "json", + IrOutputFormat::Postcard => "postcard", + }; + + dcx.fatal(format!( + "the -Z unstable-options flag must be passed to enable --output-format={flag} for documentation generation (see https://github.com/rust-lang/rust/issues/76578)", + )); } (Some(OutputFormat::Doctest), false, false) => { dcx.fatal( @@ -523,18 +535,19 @@ impl Options { match typ { EmitType::DepInfo(_) => match output_format { - OutputFormat::Html | OutputFormat::IrJson | OutputFormat::CoverageJson => {} + OutputFormat::Html | OutputFormat::Ir(_) | OutputFormat::CoverageJson => {} OutputFormat::Doctest => unreachable!(), }, EmitType::HtmlStaticFiles | EmitType::HtmlNonStaticFiles => match output_format { OutputFormat::Html => {} - OutputFormat::IrJson | OutputFormat::CoverageJson => dcx.fatal(format!( + OutputFormat::Ir(IrOutputFormat::Json) | OutputFormat::CoverageJson => dcx.fatal(format!( "the `--emit={typ}` flag is not supported with `--output-format=json`", )), + OutputFormat::Ir(IrOutputFormat::Postcard) => dcx.fatal(format!("the `--emit={typ}` flag is not supported with `--output-format=postcard`")), OutputFormat::Doctest => unreachable!(), }, - EmitType::IrJsonFiles | EmitType::CoverageJsonFiles => unreachable!(), + EmitType::IrFiles | EmitType::CoverageJsonFiles => unreachable!(), } // De-duplicate emit types and the last wins. @@ -550,7 +563,7 @@ impl Options { // will have already been rejected above. if emit.is_empty() { match output_format { - OutputFormat::IrJson => emit.push(EmitType::IrJsonFiles), + OutputFormat::Ir(_) => emit.push(EmitType::IrFiles), OutputFormat::CoverageJson => emit.push(EmitType::CoverageJsonFiles), OutputFormat::Html => { emit.push(EmitType::HtmlStaticFiles); diff --git a/src/librustdoc/core.rs b/src/librustdoc/core.rs index af1024580c544..098a3f114f858 100644 --- a/src/librustdoc/core.rs +++ b/src/librustdoc/core.rs @@ -145,7 +145,8 @@ impl<'tcx> DocContext<'tcx> { /// /// If another option like `--show-coverage` is enabled, it will return `false`. pub(crate) fn is_json_output(&self) -> bool { - self.output_format == OutputFormat::IrJson + // FIXME: Rename this method + matches!(self.output_format, OutputFormat::Ir(_)) } /// If `--document-private-items` was passed to rustdoc. diff --git a/src/librustdoc/json/mod.rs b/src/librustdoc/json/mod.rs index f161b31f94dcb..4b3014c1797a7 100644 --- a/src/librustdoc/json/mod.rs +++ b/src/librustdoc/json/mod.rs @@ -28,7 +28,7 @@ use tracing::{debug, trace}; use crate::clean::ItemKind; use crate::clean::types::{ExternalCrate, ExternalLocation}; -use crate::config::{EmitType, RenderOptions}; +use crate::config::{EmitType, IrOutputFormat, RenderOptions}; use crate::docfs::PathError; use crate::error::Error; use crate::formats::FormatRenderer; @@ -50,6 +50,7 @@ pub(crate) struct JsonRenderer<'tcx> { cache: Rc, imported_items: DefIdSet, id_interner: RefCell, + output_format: IrOutputFormat, } impl<'tcx> JsonRenderer<'tcx> { @@ -206,6 +207,7 @@ impl<'tcx> JsonRenderer<'tcx> { options: RenderOptions, cache: Cache, tcx: TyCtxt<'tcx>, + output_format: IrOutputFormat, ) -> Result<(Self, clean::Crate), Error> { debug!("Initializing json renderer"); @@ -218,6 +220,7 @@ impl<'tcx> JsonRenderer<'tcx> { out_dir: if options.output_to_stdout { None } else { Some(options.output) }, cache: Rc::new(cache), imported_items, + output_format, id_interner: Default::default(), }, krate, @@ -228,7 +231,7 @@ impl<'tcx> JsonRenderer<'tcx> { impl<'tcx> FormatRenderer<'tcx> for JsonRenderer<'tcx> { const DESCR: &'static str = "json"; const RUN_ON_MODULE: bool = false; - const NON_STATIC_FILE_EMIT_TYPE: EmitType = EmitType::IrJsonFiles; + const NON_STATIC_FILE_EMIT_TYPE: EmitType = EmitType::IrFiles; type ModuleData = (); @@ -379,36 +382,54 @@ impl<'tcx> FormatRenderer<'tcx> for JsonRenderer<'tcx> { target, format_version: types::FORMAT_VERSION, }; - if let Some(ref out_dir) = self.out_dir { - try_err!(create_dir_all(out_dir), out_dir); - - let mut p = out_dir.clone(); - p.push(output_crate.index.get(&output_crate.root).unwrap().name.clone().unwrap()); - p.set_extension("json"); - - serialize_and_write( - sess, - output_crate, - try_err!(File::create_buffered(&p), p), - &p.display().to_string(), - ) - } else { - serialize_and_write(sess, output_crate, BufWriter::new(stdout().lock()), "") - } + + let mut writer_file; + let mut writer_stdout; + + let (writer, name): (&mut dyn Write, String) = match &self.out_dir { + Some(out_dir) => { + try_err!(create_dir_all(out_dir), out_dir); + let mut p = out_dir.clone(); + p.push(output_crate.index.get(&output_crate.root).unwrap().name.clone().unwrap()); + p.set_extension(match self.output_format { + IrOutputFormat::Json => "json", + IrOutputFormat::Postcard => "postcard", + }); + + writer_file = try_err!(File::create_buffered(&p), p); + (&mut writer_file, p.display().to_string()) + } + None => { + writer_stdout = BufWriter::new(stdout().lock()); + (&mut writer_stdout, "".to_owned()) + } + }; + + serialize_and_write(sess, output_crate, self.output_format, writer, &name) } } -fn serialize_and_write( +fn serialize_and_write( sess: &Session, output_crate: types::Crate, - mut writer: BufWriter, + output_format: IrOutputFormat, + mut writer: &mut dyn Write, path: &str, ) -> Result<(), Error> { sess.time("rustdoc_json_serialize_and_write", || { - try_err!( - serde_json::ser::to_writer(&mut writer, &output_crate).map_err(|e| e.to_string()), - path - ); + match output_format { + IrOutputFormat::Json => try_err!( + serde_json::ser::to_writer(&mut writer, &output_crate).map_err(|e| e.to_string()), + path + ), + + IrOutputFormat::Postcard => { + let values: types::postcard::File = + (types::postcard::MAGIC, types::FORMAT_VERSION, output_crate); + + try_err!(postcard::to_io(&values, &mut writer), path); + } + } try_err!(writer.flush(), path); Ok(()) }) diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index 215552e8909be..7153ed63ac957 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -1015,8 +1015,16 @@ fn main_args(early_dcx: &mut EarlyDiagCtxt, at_args: &[String]) { }, ) }), - config::OutputFormat::IrJson => sess.time("render_json", || { - run_renderer(krate, render_opts, cache, tcx, json::JsonRenderer::init) + config::OutputFormat::Ir(irof) => sess.time("render_ir", || { + run_renderer( + krate, + render_opts, + cache, + tcx, + |krate, render_opts, cache, tcx| { + json::JsonRenderer::init(krate, render_opts, cache, tcx, irof) + }, + ) }), // Already handled above with doctest runners or coverage early return config::OutputFormat::Doctest | config::OutputFormat::CoverageJson => { diff --git a/src/rustdoc-json-types/lib.rs b/src/rustdoc-json-types/lib.rs index 3c20d392aab91..832660e1f99ee 100644 --- a/src/rustdoc-json-types/lib.rs +++ b/src/rustdoc-json-types/lib.rs @@ -117,6 +117,39 @@ pub type FxHashMap = HashMap; // re-export for use in src/librustdoc // Latest feature: Make `Stability` work with non-self-describing formats pub const FORMAT_VERSION: u32 = 61; +pub mod postcard { + //! Support for [Postcard](https://postcard.jamesmunns.com/) output from rustdoc. + //! + //! This is produced by calling `rustdoc --output-format=postcard`. + //! + //! This is smaller & faster to read than JSON, as it's a non-self-describing binary format. + //! + //! The root of the file is [`File`] (not [`crate::Crate`] like for JSON output). + + /// The [magic number](https://en.wikipedia.org/wiki/File_format#Magic_number) identifying + /// Rustdoc Postcard files. + pub const MAGIC: Magic = *b"\xDC\xDFRustdocIrPostcard\x01\x00"; + + /// The structure of a postcard output file. + /// + /// It contains: + /// - The magic constant, which won't change. + /// - The format version, which will change between rustdoc versions. + /// + /// See [`crate::FORMAT_VERSION`] for details. + /// - The information about the crate being documented. + /// + /// Postcard allows reading these fields from a file one at a time. + /// It's recommended to check the magic constant matches before reading the format version + /// (to check it's a rustdoc postcard file), and to check the format version matches before + /// reading the crate (to check the version of this library you have is compatible with the + /// version of rustdoc that wrote the file). + pub type File = (Magic, u32, crate::Crate); + + /// The type of the [MAGIC] constant. + pub type Magic = [u8; 21]; +} + /// The root of the emitted JSON blob. /// /// It contains all type/documentation information diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 73e3b87b37aa8..e1796ecbcefa8 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -1046,6 +1046,9 @@ impl<'test> TestCx<'test> { DocKind::Json => { rustdoc.arg("--output-format").arg("json").arg("-Zunstable-options"); } + DocKind::Postcard => { + rustdoc.arg("--output-format").arg("postcard").arg("-Zunstable-options"); + } } if let Some(ref linker) = self.config.target_linker { @@ -3082,6 +3085,7 @@ enum CompareOutcome { enum DocKind { Html, Json, + Postcard, } impl CompareOutcome { diff --git a/src/tools/compiletest/src/runtest/rustdoc_json.rs b/src/tools/compiletest/src/runtest/rustdoc_json.rs index 311c2c67f2dfa..99e27e8004b00 100644 --- a/src/tools/compiletest/src/runtest/rustdoc_json.rs +++ b/src/tools/compiletest/src/runtest/rustdoc_json.rs @@ -12,15 +12,20 @@ impl TestCx<'_> { panic!("failed to remove and recreate output directory `{out_dir}`: {e}") }); - let proc_res = self.document(&out_dir, DocKind::Json); + let run_rustdoc = |doc_kind| { + let proc_res = self.document(&out_dir, doc_kind); + if !self.config.capture { + writeln!(self.stdout, "{}", proc_res.format_info()); + } - if !self.config.capture { - writeln!(self.stdout, "{}", proc_res.format_info()); - } + if !proc_res.status.success() { + self.fatal_proc_rec("rustdoc failed!", &proc_res); + } - if !proc_res.status.success() { - self.fatal_proc_rec("rustdoc failed!", &proc_res); - } + proc_res + }; + + let proc_res = run_rustdoc(DocKind::Json); let mut cmd = ArgFileCommand::new(self.config.jsondocck_path.as_ref().unwrap()); cmd.arg("--doc-dir").arg(&out_dir).arg("--template").arg(&self.testpaths.file); @@ -28,6 +33,8 @@ impl TestCx<'_> { if !res.status.success() { self.fatal_proc_rec_general("jsondocck failed!", None, &res, || { + // FIXME: Include `--output-format=postcard` info here too. + // Alternatively, ditch this section altogether, how useful is it? writeln!(self.stdout, "Rustdoc Output:"); writeln!(self.stdout, "{}", proc_res.format_info()); }) @@ -36,8 +43,13 @@ impl TestCx<'_> { let mut json_out = out_dir.join(self.testpaths.file.file_stem().unwrap()); json_out.set_extension("json"); + run_rustdoc(DocKind::Postcard); + let mut postcard_out = json_out.clone(); + postcard_out.set_extension("postcard"); + let mut cmd = ArgFileCommand::new(self.config.jsondoclint_path.as_ref().unwrap()); cmd.arg(&json_out); + cmd.arg(&postcard_out); let res = self.run_command_to_procres(cmd); if !res.status.success() { diff --git a/src/tools/jsondoclint/Cargo.toml b/src/tools/jsondoclint/Cargo.toml index 848c0b37ae94e..1f1794d3c77f4 100644 --- a/src/tools/jsondoclint/Cargo.toml +++ b/src/tools/jsondoclint/Cargo.toml @@ -6,10 +6,13 @@ edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +# tidy-alphabetical-start anyhow = "1.0.62" clap = { version = "4.0.15", features = ["derive"] } fs-err = "2.8.1" +postcard = { version = "1.1.3", default-features = false, features = ["use-std"] } rustc-hash = "2.0.0" rustdoc-json-types = { version = "0.1.0", path = "../../rustdoc-json-types" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.85" +# tidy-alphabetical-end diff --git a/src/tools/jsondoclint/src/main.rs b/src/tools/jsondoclint/src/main.rs index f4278658200f0..19a966e4f864c 100644 --- a/src/tools/jsondoclint/src/main.rs +++ b/src/tools/jsondoclint/src/main.rs @@ -33,7 +33,10 @@ struct JsonOutput { #[derive(Parser)] struct Cli { /// The path to the json file to be linted - path: String, + json_path: String, + + /// The path of the postcard file to be linted + postcard_path: String, /// Show verbose output #[arg(long)] @@ -43,26 +46,34 @@ struct Cli { json_output: Option, } -fn main() -> Result<()> { - let Cli { path, verbose, json_output } = Cli::parse(); - +fn input_path(path: &str) -> PathBuf { // We convert `-` into `_` for the file name to be sure the JSON path will always be correct. let path = Path::new(&path); let filename = path.file_name().unwrap().to_str().unwrap().replace('-', "_"); let parent = path.parent().unwrap(); let path = parent.join(&filename); + path +} + +fn main() -> Result<()> { + let Cli { json_path, postcard_path, verbose, json_output } = Cli::parse(); + let json_path = input_path(&json_path); + let postcard_path = input_path(&postcard_path); - let contents = fs::read_to_string(&path)?; - let krate: Crate = serde_json::from_str(&contents)?; + let json_contents = fs::read_to_string(&json_path)?; + + let krate: Crate = serde_json::from_str(&json_contents)?; assert_eq!(krate.format_version, FORMAT_VERSION); - let krate_json: Value = serde_json::from_str(&contents)?; + check_postcard(&postcard_path, &krate)?; + + let krate_json: Value = serde_json::from_str(&json_contents)?; let mut validator = validator::Validator::new(&krate, krate_json); validator.check_crate(); if let Some(json_output) = json_output { - let output = JsonOutput { path: path.clone(), errors: validator.errs.clone() }; + let output = JsonOutput { path: json_path.clone(), errors: validator.errs.clone() }; let mut f = BufWriter::new(fs::File::create(json_output)?); serde_json::to_writer(&mut f, &output)?; f.flush()?; @@ -108,7 +119,37 @@ fn main() -> Result<()> { ErrorKind::Custom(msg) => eprintln!("{}: {}", err.id.0, msg), } } - bail!("Errors validating json {}", path.display()); + bail!("Errors validating json {}", json_path.display()); + } + + Ok(()) +} + +fn check_postcard(path: &Path, expected_krate: &Crate) -> Result<()> { + let postcard_bytes = fs::read(path)?; + + let (file, rest) = + postcard::take_from_bytes::(&postcard_bytes)?; + + if !rest.is_empty() { + bail!("Postcard file has {} leftover bytes", rest.len()); + } + + let (magic, format_version, krate) = file; + + let expected_magic = rustdoc_json_types::postcard::MAGIC; + if magic != expected_magic { + bail!("Postcard file has bad magic value, got {magic:?} but expected {expected_magic:?}"); + } + + let expected_format_version = rustdoc_json_types::FORMAT_VERSION; + if format_version != expected_format_version { + bail!( + "Postcard file has bad format version, got {format_version} but expected {expected_format_version}" + ); + } + if &krate != expected_krate { + bail!("Postcard file didn't contain same crate information as json file"); } Ok(())