Deduplicate headers across the per-target sysroots

cmake/wasi-sdk-sysroot.cmake

@@ -208,8 +208,10 @@ function(define_wasi_libc_sub sysroot target target_suffix lto)
     list(APPEND extra_cmake_args -DBUILD_SHARED=OFF)
   endif()
 
+  set(header_dir ${sysroot}/../libc-tmp-include)
   if (${sysroot} STREQUAL ${coop_threads_sysroot})
     list(APPEND extra_cmake_args -DENABLE_COOP_THREADS=ON)
+    set(header_dir ${sysroot}/include)
   endif()
 
   ExternalProject_Add(wasi-libc-${target}${target_suffix}-build
@@ -219,6 +221,7 @@ function(define_wasi_libc_sub sysroot target target_suffix lto)
       ${extra_cmake_args}
       -DTARGET_TRIPLE=${target}
       -DCMAKE_INSTALL_PREFIX=${sysroot}
+      -DCMAKE_INSTALL_INCLUDEDIR=${header_dir}
       -DCMAKE_C_FLAGS=${extra_cflags}
       -DCMAKE_ASM_FLAGS=${extra_cflags}
       -DBUILTINS_LIB=${libcompiler_rt_a}
@@ -249,8 +252,12 @@ function(define_wasi_libc target)
   endif()
 endfunction()
 
+add_custom_target(build-wasi-libc)
+add_custom_target(wasi-libc DEPENDS build-wasi-libc)
+
 foreach(target IN LISTS WASI_SDK_TARGETS)
   define_wasi_libc(${target})
+  add_dependencies(build-wasi-libc wasi-libc-${target})
 endforeach()
 
 # =============================================================================
@@ -323,9 +330,12 @@ function(define_libcxx_sub sysroot target target_suffix extra_target_flags extra
     set(shared OFF)
   endif()
 
-  # FIXME(WebAssembly/wasi-libc#813) - shared libraries don't work with coop
-  # threads right now.
+
+  set(header_dir ${sysroot}/../cpp-tmp-include/${target}${exnsuffix})
   if (${sysroot} STREQUAL ${coop_threads_sysroot})
+    set(header_dir ${sysroot}/include/${target}${exnsuffix})
+    # FIXME(WebAssembly/wasi-libc#813) - shared libraries don't work with coop
+    # threads right now.
     set(shared OFF)
   endif()
 
@@ -348,7 +358,7 @@ function(define_libcxx_sub sysroot target target_suffix extra_target_flags extra
       -DCMAKE_SYSROOT=${sysroot}
       # Ensure headers are installed in a target-specific path instead of a
       # target-generic path.
-      -DCMAKE_INSTALL_INCLUDEDIR=${sysroot}/include/${target}${exnsuffix}
+      -DCMAKE_INSTALL_INCLUDEDIR=${header_dir}
       -DCMAKE_STAGING_PREFIX=${sysroot}
       -DCMAKE_POSITION_INDEPENDENT_CODE=${pic}
       -DLIBCXX_ENABLE_THREADS:BOOL=ON
@@ -390,7 +400,7 @@ function(define_libcxx_sub sysroot target target_suffix extra_target_flags extra
     CMAKE_CACHE_ARGS
       -DLLVM_ENABLE_RUNTIMES:STRING=${runtimes}
     DEPENDS
-      wasi-libc-${target}
+      wasi-libc
       compiler-rt
     EXCLUDE_FROM_ALL ON
     USES_TERMINAL_CONFIGURE ON
@@ -451,10 +461,39 @@ function(define_libcxx target)
   endif()
 endfunction()
 
+add_custom_target(build-libcxx)
+add_custom_target(libcxx DEPENDS build-libcxx)
+
 foreach(target IN LISTS WASI_SDK_TARGETS)
   define_libcxx(${target})
+  add_dependencies(build-libcxx libcxx-${target})
 endforeach()
 
+# Add a top-level `build` target as well as `build-$target` targets.
+add_custom_target(build ALL)
+add_dependencies(build wasi-libc libcxx compiler-rt)
+
+# =============================================================================
+# sysroot header logic
+# =============================================================================
+
+set(dedupe_headers ${CMAKE_CURRENT_BINARY_DIR}/dedupe_headers${CMAKE_HOST_EXECUTABLE_SUFFIX})
+set(dedupe_headers_src ${CMAKE_CURRENT_SOURCE_DIR}/src/dedupe_headers.rs)
+add_custom_command(
+  OUTPUT ${dedupe_headers}
+  COMMAND rustc --edition 2024 ${dedupe_headers_src} -o ${dedupe_headers}
+  DEPENDS ${dedupe_headers_src})
+
+add_custom_target(dedupe-libc-headers
+  COMMAND ${dedupe_headers} ${wasi_sysroot}/../libc-tmp-include ${wasi_sysroot}/include
+  DEPENDS ${dedupe_headers} build-wasi-libc)
+add_dependencies(wasi-libc dedupe-libc-headers)
+
+add_custom_target(dedupe-libcxx-headers
+  COMMAND ${dedupe_headers} ${wasi_sysroot}/../cpp-tmp-include ${wasi_sysroot}/include
+  DEPENDS ${dedupe_headers} build-libcxx)
+add_dependencies(libcxx dedupe-libcxx-headers)
+
 # =============================================================================
 # misc build logic
 # =============================================================================
@@ -472,14 +511,6 @@ else()
           DESTINATION ${CMAKE_INSTALL_PREFIX}/clang-resource-dir)
 endif()
 
-# Add a top-level `build` target as well as `build-$target` targets.
-add_custom_target(build ALL)
-foreach(target IN LISTS WASI_SDK_TARGETS)
-  add_custom_target(build-${target})
-  add_dependencies(build-${target} libcxx-${target} wasi-libc-${target} compiler-rt)
-  add_dependencies(build build-${target})
-endforeach()
-
 # Install a `VERSION` file in the output prefix with a dump of version
 # information.
 execute_process(

src/dedupe_headers.rs

@@ -0,0 +1,179 @@
+//! Helper script to deduplicate directories of headers across wasi
+//! targets/builds.
+//!
+//! Right now each target is built in isolation, e.g. `wasm32-wasip{1,2,3}` and
+//! additionally each target has `libcxx` built with/without exceptions. This is
+//! quite a lot of header files in this full matrix (e.g. 6 copies of libcxx
+//! headers at least), but generally the files are all the same across all
+//! these targets. Clang's sysroot logic looks first in target-specific
+//! locations but then additionally consults more generic locations, and this
+//! script massages the input into an outupt directory suitable to have the same
+//! view according to clang.
+//!
+//! Specifically if the same header is present in every single target-specific
+//! directory, then it's copied up to the target-agnostic sysroot. This is
+//! done without symlinks to work well on Windows and means that the
+//! target-specific directories are generally quite small compared to the full
+//! complete list of directories.
+//!
+//! Note that special care is taken for the "eh" and "noeh" directories which
+//! are the libcxx builds with/without exceptions. If those are encountered
+//! then only files duplicated across both of them are lifted up.
+
+use std::ffi::OsString;
+use std::fs::{self, FileType};
+use std::path::{Path, PathBuf};
+
+macro_rules! debug {
+    ($($arg:tt)*) => {
+        if true {
+            eprintln!($($arg)*);
+        }
+    };
+}
+
+fn main() {
+    let mut args = std::env::args_os();
+    args.next().unwrap(); // skip exe name
+    let src = PathBuf::from(args.next().unwrap());
+    let dst = PathBuf::from(args.next().unwrap());
+
+    copy(
+        &src,
+        &mut src
+            .read_dir()
+            .unwrap()
+            .map(|d| d.unwrap().path())
+            .collect::<Vec<_>>(),
+        &dst,
+        &mut PathBuf::new(),
+    );
+}
+
+/// Copies all of the contents of each of the source directories from `srcs` to
+/// `dst`, deduplicating anything in common into `dst`.
+///
+/// The `src_root` option is used as a relative prefix for everything within
+/// `srcs` when some directories may differ. The `dst` is built up during the
+/// recursive traversal and is a relative destination from `dst_root`.
+fn copy(src_root: &Path, srcs: &mut Vec<PathBuf>, dst_root: &Path, dst: &mut PathBuf) {
+    let mut src_entries = srcs
+        .iter()
+        .map(|s| {
+            let mut entries = s
+                .read_dir()
+                .unwrap()
+                .map(|e| {
+                    let e = e.unwrap();
+                    (e.file_name(), e.file_type().unwrap())
+                })
+                .collect::<Vec<_>>();
+            entries.sort_by_key(|(name, _)| name.clone());
+            (entries.into_iter().peekable(), s.clone())
+        })
+        .collect::<Vec<_>>();
+    let ((first, src0), rest) = src_entries.split_first_mut().unwrap();
+
+    while let Some(pair @ (name, ft)) = first.peek() {
+        // Bring all iterators up to `name`
+        for (other, src) in rest.iter_mut() {
+            while let Some((e, ft)) = other.next_if(|(e, _)| e < name) {
+                rel_cp_r(src_root, src, ft, dst_root, &e);
+            }
+        }
+
+        // Test if all directories have either a directory for `name` or all
+        // have a file for `name`.
+        let src = src0.join(name);
+        let contents = if ft.is_dir() {
+            None
+        } else {
+            assert!(ft.is_file());
+            Some(fs::read(&src).unwrap())
+        };
+        let all_same = rest.iter_mut().all(|(other, other_src)| {
+            if other.peek() != Some(pair) {
+                return false;
+            }
+            match &contents {
+                Some(src0) => src0 == &fs::read(other_src.join(name)).unwrap(),
+                None => true,
+            }
+        });
+
+        // If everything is the same then this file or directory can be lifted
+        // up without its target prefix, otherwise copy this file for the
+        // `first` iterator and continue on.
+        if all_same {
+            if ft.is_dir() {
+                if name != "eh" {
+                    for src in srcs.iter_mut() {
+                        src.push(name);
+                    }
+                    if name == "noeh" {
+                        for mut src in srcs.clone() {
+                            src.pop();
+                            src.push("eh");
+                            srcs.push(src);
+                        }
+                    } else {
+                        fs::create_dir_all(&dst_root.join(&dst).join(name)).unwrap();
+                        dst.push(name);
+                    }
+                    copy(src_root, srcs, dst_root, dst);
+                    if name == "noeh" {
+                        srcs.truncate(srcs.len() / 2);
+                    } else {
+                        dst.pop();
+                    }
+                    for src in srcs.iter_mut() {
+                        src.pop();
+                    }
+                }
+            } else {
+                let dst = dst_root.join(&dst).join(name);
+                debug!("deduplicate: {dst:?}");
+                fs::copy(&src, &dst).unwrap();
+            }
+            for (other, _src) in rest.iter_mut() {
+                other.next();
+            }
+        } else {
+            rel_cp_r(src_root, src0, *ft, dst_root, name);
+        }
+
+        first.next();
+    }
+
+    // Copy over everything remaining in all other directories
+    for (other, src) in rest.iter_mut() {
+        for (e, ft) in other {
+            rel_cp_r(src_root, src, ft, dst_root, &e);
+        }
+    }
+}
+
+fn rel_cp_r(src_root: &Path, src: &Path, ft: FileType, dst: &Path, name: &OsString) {
+    let rel_path = src.strip_prefix(src_root).unwrap();
+    let dst = dst.join(rel_path);
+    fs::create_dir_all(&dst).unwrap();
+    cp_r(&mut src.join(name), ft, &mut dst.join(name));
+}
+
+fn cp_r(src: &mut PathBuf, ft: FileType, dst: &mut PathBuf) {
+    if ft.is_dir() {
+        fs::create_dir_all(&dst).unwrap();
+        for entry in src.read_dir().unwrap() {
+            let entry = entry.unwrap();
+            let name = entry.file_name();
+            src.push(&name);
+            dst.push(&name);
+            cp_r(src, entry.file_type().unwrap(), dst);
+            src.pop();
+            dst.pop();
+        }
+    } else {
+        debug!("unique: {dst:?}");
+        fs::copy(src, &dst).unwrap();
+    }
+}

tests/CMakeLists.txt

@@ -56,6 +56,7 @@ function(add_testcase test)
       # Add a new test executable based on `test`
       add_executable(${target_name} ${test})
       add_dependencies(build-tests ${target_name})
+      add_dependencies(${target_name} build)
 
       # Configure all the compile options necessary. For example `--target` here
       # if the target doesn't look like it's already in the name of the compiler
@@ -90,13 +91,6 @@ function(add_testcase test)
         else()
           target_compile_options(${target_name} PRIVATE -fno-exceptions)
         endif()
-        if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN)
-          add_dependencies(${target_name} libcxx-${target})
-        endif()
-      else()
-        if(NOT WASI_SDK_TEST_HOST_TOOLCHAIN)
-          add_dependencies(${target_name} wasi-libc-${target})
-        endif()
       endif()
 
       # Apply target-specific options.