rustdoc-json: Postcard output

Cargo.lock

@@ -2232,6 +2232,7 @@ dependencies = [
  "anyhow",
  "clap",
  "fs-err",
+ "postcard",
  "rustc-hash 2.1.1",
  "rustdoc-json-types",
  "serde",
@@ -4990,6 +4991,7 @@ dependencies = [
  "indexmap",
  "itertools",
  "minifier",
+ "postcard",
  "proc-macro2",
  "pulldown-cmark-escape",
  "regex",

src/doc/rustdoc/src/unstable-features.md

@@ -577,6 +577,15 @@ It can also be used with `--show-coverage`. Take a look at its
 [documentation](#--show-coverage-calculate-the-percentage-of-items-with-documentation) for more
 information.
 
+### postcard
+
+ * Tracking Issue: [#76578](https://github.com/rust-lang/rust/issues/76578)
+
+`--output-format postcard` emits documentation using [Postcard](https://postcard.jamesmunns.com/) as an encoding format.
+This has exactly the same structure as the JSON format, but is a smaller file that's faster to serialize/deserialize.
+
+See the [API docs](https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc_json_types/postcard/index.html) for details.
+
 ### doctest
 
  * Tracking issue: [#134529](https://github.com/rust-lang/rust/issues/134529)

src/librustdoc/Cargo.toml

@@ -15,6 +15,7 @@ base64 = "0.21.7"
 indexmap = { version = "2", features = ["serde"] }
 itertools = "0.15"
 minifier = { version = "0.4.0", default-features = false }
+postcard = { version = "1.1.3", default-features = false, features = ["use-std"] }
 proc-macro2 = "1.0.103"
 pulldown-cmark-escape = { version = "0.11.0", features = ["simd"] }
 regex = "1"

src/librustdoc/config.rs

@@ -34,13 +34,19 @@ pub(crate) enum OutputFormat {
     /// `--output-format=json` without `--show-coverage`.
     ///
     /// JSON description of crate API.
-    IrJson,
+    Ir(IrOutputFormat),
     /// `--output-format=json` with `--show-coverage`.
     CoverageJson,
     Html,
     Doctest,
 }
 
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub(crate) enum IrOutputFormat {
+    Json,
+    Postcard,
+}
+
 /// Either an input crate, markdown file, or nothing (--merge=finalize).
 pub(crate) enum InputMode {
     /// The `--merge=finalize` step does not need an input crate to rustdoc.
@@ -314,7 +320,7 @@ pub(crate) enum EmitType {
     HtmlStaticFiles,
     HtmlNonStaticFiles,
     // not explicitly nameable by the user for now
-    IrJsonFiles,
+    IrFiles,
     CoverageJsonFiles,
     DepInfo(Option<OutFileName>),
 }
@@ -324,7 +330,7 @@ impl fmt::Display for EmitType {
         f.write_str(match self {
             Self::HtmlStaticFiles => "html-static-files",
             Self::HtmlNonStaticFiles => "html-non-static-files",
-            Self::IrJsonFiles => "ir-json-files",
+            Self::IrFiles => "ir-files",
             Self::CoverageJsonFiles => "coverage-json-files",
             Self::DepInfo(_) => "dep-info",
         })
@@ -469,9 +475,10 @@ impl Options {
                 if show_coverage {
                     OutputFormat::CoverageJson
                 } else {
-                    OutputFormat::IrJson
+                    OutputFormat::Ir(IrOutputFormat::Json)
                 }
             }
+            Some("postcard") => OutputFormat::Ir(IrOutputFormat::Postcard),
             Some("doctest") => OutputFormat::Doctest,
             Some(other) => dcx.fatal(format!("unknown output format `{other}`")),
         };
@@ -492,10 +499,15 @@ impl Options {
             // If `-Zunstable-options` is used, nothing to check after this point.
             (_, false, true) => {}
             (None | Some(OutputFormat::Html), false, _) => {}
-            (Some(OutputFormat::IrJson), false, false) => {
-                dcx.fatal(
-                    "the -Z unstable-options flag must be passed to enable --output-format=json for documentation generation (see https://github.com/rust-lang/rust/issues/76578)",
-                );
+            (Some(OutputFormat::Ir(irof)), false, false) => {
+                let flag = match irof {
+                    IrOutputFormat::Json => "json",
+                    IrOutputFormat::Postcard => "postcard",
+                };
+
+                dcx.fatal(format!(
+                    "the -Z unstable-options flag must be passed to enable --output-format={flag} for documentation generation (see https://github.com/rust-lang/rust/issues/76578)",
+                ));
             }
             (Some(OutputFormat::Doctest), false, false) => {
                 dcx.fatal(
@@ -523,18 +535,19 @@ impl Options {
 
                 match typ {
                     EmitType::DepInfo(_) => match output_format {
-                        OutputFormat::Html | OutputFormat::IrJson | OutputFormat::CoverageJson => {}
+                        OutputFormat::Html | OutputFormat::Ir(_) | OutputFormat::CoverageJson => {}
                         OutputFormat::Doctest => unreachable!(),
                     },
                     EmitType::HtmlStaticFiles | EmitType::HtmlNonStaticFiles => match output_format
                     {
                         OutputFormat::Html => {}
-                        OutputFormat::IrJson | OutputFormat::CoverageJson => dcx.fatal(format!(
+                        OutputFormat::Ir(IrOutputFormat::Json) | OutputFormat::CoverageJson => dcx.fatal(format!(
                             "the `--emit={typ}` flag is not supported with `--output-format=json`",
                         )),
+                        OutputFormat::Ir(IrOutputFormat::Postcard) => dcx.fatal(format!("the `--emit={typ}` flag is not supported with `--output-format=postcard`")),
                         OutputFormat::Doctest => unreachable!(),
                     },
-                    EmitType::IrJsonFiles | EmitType::CoverageJsonFiles => unreachable!(),
+                    EmitType::IrFiles | EmitType::CoverageJsonFiles => unreachable!(),
                 }
 
                 // De-duplicate emit types and the last wins.
@@ -550,7 +563,7 @@ impl Options {
         // will have already been rejected above.
         if emit.is_empty() {
             match output_format {
-                OutputFormat::IrJson => emit.push(EmitType::IrJsonFiles),
+                OutputFormat::Ir(_) => emit.push(EmitType::IrFiles),
                 OutputFormat::CoverageJson => emit.push(EmitType::CoverageJsonFiles),
                 OutputFormat::Html => {
                     emit.push(EmitType::HtmlStaticFiles);

src/librustdoc/core.rs

@@ -145,7 +145,8 @@ impl<'tcx> DocContext<'tcx> {
     ///
     /// If another option like `--show-coverage` is enabled, it will return `false`.
     pub(crate) fn is_json_output(&self) -> bool {
-        self.output_format == OutputFormat::IrJson
+        // FIXME: Rename this method
+        matches!(self.output_format, OutputFormat::Ir(_))
     }
 
     /// If `--document-private-items` was passed to rustdoc.

src/librustdoc/json/mod.rs

@@ -28,7 +28,7 @@ use tracing::{debug, trace};
 
 use crate::clean::ItemKind;
 use crate::clean::types::{ExternalCrate, ExternalLocation};
-use crate::config::{EmitType, RenderOptions};
+use crate::config::{EmitType, IrOutputFormat, RenderOptions};
 use crate::docfs::PathError;
 use crate::error::Error;
 use crate::formats::FormatRenderer;
@@ -50,6 +50,7 @@ pub(crate) struct JsonRenderer<'tcx> {
     cache: Rc<Cache>,
     imported_items: DefIdSet,
     id_interner: RefCell<ids::IdInterner>,
+    output_format: IrOutputFormat,
 }
 
 impl<'tcx> JsonRenderer<'tcx> {
@@ -206,6 +207,7 @@ impl<'tcx> JsonRenderer<'tcx> {
         options: RenderOptions,
         cache: Cache,
         tcx: TyCtxt<'tcx>,
+        output_format: IrOutputFormat,
     ) -> Result<(Self, clean::Crate), Error> {
         debug!("Initializing json renderer");
 
@@ -218,6 +220,7 @@ impl<'tcx> JsonRenderer<'tcx> {
                 out_dir: if options.output_to_stdout { None } else { Some(options.output) },
                 cache: Rc::new(cache),
                 imported_items,
+                output_format,
                 id_interner: Default::default(),
             },
             krate,
@@ -228,7 +231,7 @@ impl<'tcx> JsonRenderer<'tcx> {
 impl<'tcx> FormatRenderer<'tcx> for JsonRenderer<'tcx> {
     const DESCR: &'static str = "json";
     const RUN_ON_MODULE: bool = false;
-    const NON_STATIC_FILE_EMIT_TYPE: EmitType = EmitType::IrJsonFiles;
+    const NON_STATIC_FILE_EMIT_TYPE: EmitType = EmitType::IrFiles;
 
     type ModuleData = ();
 
@@ -379,36 +382,54 @@ impl<'tcx> FormatRenderer<'tcx> for JsonRenderer<'tcx> {
             target,
             format_version: types::FORMAT_VERSION,
         };
-        if let Some(ref out_dir) = self.out_dir {
-            try_err!(create_dir_all(out_dir), out_dir);
-
-            let mut p = out_dir.clone();
-            p.push(output_crate.index.get(&output_crate.root).unwrap().name.clone().unwrap());
-            p.set_extension("json");
-
-            serialize_and_write(
-                sess,
-                output_crate,
-                try_err!(File::create_buffered(&p), p),
-                &p.display().to_string(),
-            )
-        } else {
-            serialize_and_write(sess, output_crate, BufWriter::new(stdout().lock()), "<stdout>")
-        }
+
+        let mut writer_file;
+        let mut writer_stdout;
+
+        let (writer, name): (&mut dyn Write, String) = match &self.out_dir {
+            Some(out_dir) => {
+                try_err!(create_dir_all(out_dir), out_dir);
+                let mut p = out_dir.clone();
+                p.push(output_crate.index.get(&output_crate.root).unwrap().name.clone().unwrap());
+                p.set_extension(match self.output_format {
+                    IrOutputFormat::Json => "json",
+                    IrOutputFormat::Postcard => "postcard",
+                });
+
+                writer_file = try_err!(File::create_buffered(&p), p);
+                (&mut writer_file, p.display().to_string())
+            }
+            None => {
+                writer_stdout = BufWriter::new(stdout().lock());
+                (&mut writer_stdout, "<stdout>".to_owned())
+            }
+        };
+
+        serialize_and_write(sess, output_crate, self.output_format, writer, &name)
     }
 }
 
-fn serialize_and_write<T: Write>(
+fn serialize_and_write(
     sess: &Session,
     output_crate: types::Crate,
-    mut writer: BufWriter<T>,
+    output_format: IrOutputFormat,
+    mut writer: &mut dyn Write,
     path: &str,
 ) -> Result<(), Error> {
     sess.time("rustdoc_json_serialize_and_write", || {
-        try_err!(
-            serde_json::ser::to_writer(&mut writer, &output_crate).map_err(|e| e.to_string()),
-            path
-        );
+        match output_format {
+            IrOutputFormat::Json => try_err!(
+                serde_json::ser::to_writer(&mut writer, &output_crate).map_err(|e| e.to_string()),
+                path
+            ),
+
+            IrOutputFormat::Postcard => {
+                let values: types::postcard::File =
+                    (types::postcard::MAGIC, types::FORMAT_VERSION, output_crate);
+
+                try_err!(postcard::to_io(&values, &mut writer), path);
+            }
+        }
         try_err!(writer.flush(), path);
         Ok(())
     })

src/librustdoc/lib.rs

@@ -1015,8 +1015,16 @@ fn main_args(early_dcx: &mut EarlyDiagCtxt, at_args: &[String]) {
                         },
                     )
                 }),
-                config::OutputFormat::IrJson => sess.time("render_json", || {
-                    run_renderer(krate, render_opts, cache, tcx, json::JsonRenderer::init)
+                config::OutputFormat::Ir(irof) => sess.time("render_ir", || {
+                    run_renderer(
+                        krate,
+                        render_opts,
+                        cache,
+                        tcx,
+                        |krate, render_opts, cache, tcx| {
+                            json::JsonRenderer::init(krate, render_opts, cache, tcx, irof)
+                        },
+                    )
                 }),
                 // Already handled above with doctest runners or coverage early return
                 config::OutputFormat::Doctest | config::OutputFormat::CoverageJson => {

src/rustdoc-json-types/lib.rs

@@ -117,6 +117,39 @@ pub type FxHashMap<K, V> = HashMap<K, V>; // re-export for use in src/librustdoc
 // Latest feature: Make `Stability` work with non-self-describing formats
 pub const FORMAT_VERSION: u32 = 61;
 
+pub mod postcard {
+    //! Support for [Postcard](https://postcard.jamesmunns.com/) output from rustdoc.
+    //!
+    //! This is produced by calling `rustdoc --output-format=postcard`.
+    //!
+    //! This is smaller & faster to read than JSON, as it's a non-self-describing binary format.
+    //!
+    //! The root of the file is [`File`] (not [`crate::Crate`] like for JSON output).
+
+    /// The [magic number](https://en.wikipedia.org/wiki/File_format#Magic_number) identifying
+    /// Rustdoc Postcard files.
+    pub const MAGIC: Magic = *b"\xDC\xDFRustdocIrPostcard\x01\x00";
+
+    /// The structure of a postcard output file.
+    ///
+    /// It contains:
+    /// - The magic constant, which won't change.
+    /// - The format version, which will change between rustdoc versions.
+    ///
+    ///   See [`crate::FORMAT_VERSION`] for details.
+    /// - The information about the crate being documented.
+    ///
+    /// Postcard allows reading these fields from a file one at a time.
+    /// It's recommended to check the magic constant matches before reading the format version
+    /// (to check it's a rustdoc postcard file), and to check the format version matches before
+    /// reading the crate (to check the version of this library you have is compatible with the
+    /// version of rustdoc that wrote the file).
+    pub type File = (Magic, u32, crate::Crate);
+
+    /// The type of the [MAGIC] constant.
+    pub type Magic = [u8; 21];
+}
+
 /// The root of the emitted JSON blob.
 ///
 /// It contains all type/documentation information

src/tools/compiletest/src/runtest.rs

@@ -1046,6 +1046,9 @@ impl<'test> TestCx<'test> {
             DocKind::Json => {
                 rustdoc.arg("--output-format").arg("json").arg("-Zunstable-options");
             }
+            DocKind::Postcard => {
+                rustdoc.arg("--output-format").arg("postcard").arg("-Zunstable-options");
+            }
         }
 
         if let Some(ref linker) = self.config.target_linker {
@@ -3082,6 +3085,7 @@ enum CompareOutcome {
 enum DocKind {
     Html,
     Json,
+    Postcard,
 }
 
 impl CompareOutcome {

src/tools/compiletest/src/runtest/rustdoc_json.rs

@@ -12,22 +12,29 @@ impl TestCx<'_> {
             panic!("failed to remove and recreate output directory `{out_dir}`: {e}")
         });
 
-        let proc_res = self.document(&out_dir, DocKind::Json);
+        let run_rustdoc = |doc_kind| {
+            let proc_res = self.document(&out_dir, doc_kind);
+            if !self.config.capture {
+                writeln!(self.stdout, "{}", proc_res.format_info());
+            }
 
-        if !self.config.capture {
-            writeln!(self.stdout, "{}", proc_res.format_info());
-        }
+            if !proc_res.status.success() {
+                self.fatal_proc_rec("rustdoc failed!", &proc_res);
+            }
 
-        if !proc_res.status.success() {
-            self.fatal_proc_rec("rustdoc failed!", &proc_res);
-        }
+            proc_res
+        };
+
+        let proc_res = run_rustdoc(DocKind::Json);
 
         let mut cmd = ArgFileCommand::new(self.config.jsondocck_path.as_ref().unwrap());
         cmd.arg("--doc-dir").arg(&out_dir).arg("--template").arg(&self.testpaths.file);
         let res = self.run_command_to_procres(cmd);
 
         if !res.status.success() {
             self.fatal_proc_rec_general("jsondocck failed!", None, &res, || {
+                // FIXME: Include `--output-format=postcard` info here too.
+                // Alternatively, ditch this section altogether, how useful is it?
                 writeln!(self.stdout, "Rustdoc Output:");
                 writeln!(self.stdout, "{}", proc_res.format_info());
             })
@@ -36,8 +43,13 @@ impl TestCx<'_> {
         let mut json_out = out_dir.join(self.testpaths.file.file_stem().unwrap());
         json_out.set_extension("json");
 
+        run_rustdoc(DocKind::Postcard);
+        let mut postcard_out = json_out.clone();
+        postcard_out.set_extension("postcard");
+
         let mut cmd = ArgFileCommand::new(self.config.jsondoclint_path.as_ref().unwrap());
         cmd.arg(&json_out);
+        cmd.arg(&postcard_out);
         let res = self.run_command_to_procres(cmd);
 
         if !res.status.success() {