diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..e70d4224a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,78 @@ +name: CI + +# Run on every push and pull request so each commit is validated. +on: + push: + pull_request: + +# Cancel an in-progress run if a newer commit is pushed to the same ref. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: build • test • coverage (OpenMPI) + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + # GitHub-hosted runners have fewer cores than the 8 ranks some suites + # launch, so allow MPI to oversubscribe. (rmaps_base for OpenMPI 4.x, + # the PRTE policy for OpenMPI 5.x — setting the unused one is harmless.) + OMPI_MCA_rmaps_base_oversubscribe: "1" + PRTE_MCA_rmaps_default_mapping_policy: ":oversubscribe" + steps: + - uses: actions/checkout@v4 + + - name: Install toolchain (CMake, OpenMPI, gcovr) + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + cmake build-essential openmpi-bin libopenmpi-dev gcovr + + - name: Configure (tests + coverage) + run: cmake -B build -DBLINK_TESTS=ON -DBLINK_COVERAGE=ON + + - name: Build + run: cmake --build build -j"$(nproc)" + + - name: Run tests + run: ctest --test-dir build --output-on-failure --no-tests=error + + # ---- coverage (best-effort: never flips the test verdict) ---- + - name: Coverage report (gcovr) + if: always() + continue-on-error: true + run: | + gcovr --root . --filter 'src/' \ + --exclude-unreachable-branches --exclude-throw-branches \ + --exclude-lines-by-pattern '.*MPI_Abort.*' \ + --xml-pretty -o coverage.xml \ + --print-summary | tee coverage_summary.txt + { + echo '## Coverage (src/)' + echo '```' + cat coverage_summary.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload coverage to Codecov + if: always() + continue-on-error: true + uses: codecov/codecov-action@v4 + with: + files: coverage.xml + fail_ci_if_error: false + env: + # Optional: set this repo secret to enable Codecov uploads/badge. + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + - name: Upload coverage artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage.xml + coverage_summary.txt + if-no-files-found: ignore diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..2df8eaaf2 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,46 @@ +name: CodeQL + +# Static analysis for C/C++. Runs on pushes/PRs to the main branches and +# weekly (to catch newly-published queries against unchanged code). +on: + push: + branches: [ main, dev ] + pull_request: + branches: [ main, dev ] + schedule: + - cron: '0 6 * * 1' # Mondays 06:00 UTC + +jobs: + analyze: + name: Analyze (c-cpp) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + security-events: write # upload results to the Security tab + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + cmake build-essential openmpi-bin libopenmpi-dev + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: c-cpp + + # Manual build (more reliable than autobuild for a CMake+MPI project). + # Tests are off so CodeQL focuses on the benchmark sources, not GTest. + - name: Build + run: | + cmake -B build -DBLINK_TESTS=OFF + cmake --build build -j"$(nproc)" + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:c-cpp" diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..6d0c0669c --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +build/ +build*/ + +# coverage artifacts (generated; uploaded to Codecov by CI, never committed) +coverage.xml +coverage*.html +*.gcda +*.gcno +*.gcov diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..34aba336c --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,113 @@ +cmake_minimum_required(VERSION 3.16) +project(blink C CXX) + +# On HPC systems where CC/CXX are already MPI wrappers (mpicc/mpicxx), +# FindMPI's link-test often fails because LD_LIBRARY_PATH isn't set up for +# CMake's subprocess. Pass -DBLINK_MPI_WRAPPER=ON to skip FindMPI and create +# stub INTERFACE targets instead — the wrapper already injects all flags. +option(BLINK_MPI_WRAPPER + "Assume CC/CXX are MPI compiler wrappers; skip find_package(MPI)" OFF) + +if(BLINK_MPI_WRAPPER) + # Prefer $ENV{MPICC} (always correct on HPC modules) over CMAKE_C_COMPILER + # (which may be the underlying gcc if the CMake cache is stale). + if(DEFINED ENV{MPICC}) + set(_mpi_cc "$ENV{MPICC}") + elseif(CMAKE_C_COMPILER MATCHES "mpi") + set(_mpi_cc "${CMAKE_C_COMPILER}") + else() + find_program(_mpi_cc NAMES mpicc DOC "MPI C compiler wrapper") + endif() + if(NOT _mpi_cc) + message(FATAL_ERROR + "BLINK_MPI_WRAPPER=ON but no MPI wrapper found.\n" + "Set the MPICC environment variable or put mpicc on PATH.") + endif() + message(STATUS "BLINK_MPI_WRAPPER=ON — querying '${_mpi_cc}' for MPI flags") + execute_process(COMMAND "${_mpi_cc}" --showme:incdirs + OUTPUT_VARIABLE _mpi_incdirs OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + execute_process(COMMAND "${_mpi_cc}" --showme:libdirs + OUTPUT_VARIABLE _mpi_libdirs OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + execute_process(COMMAND "${_mpi_cc}" --showme:libs + OUTPUT_VARIABLE _mpi_libs OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(NOT _mpi_incdirs OR NOT _mpi_libdirs) + message(FATAL_ERROR + "MPI wrapper '${_mpi_cc}' did not return include/lib dirs.\n" + "Check that it supports --showme:incdirs (OpenMPI wrapper required).") + endif() + string(REPLACE " " ";" _mpi_incdirs "${_mpi_incdirs}") + string(REPLACE " " ";" _mpi_libdirs "${_mpi_libdirs}") + string(REPLACE " " ";" _mpi_libs "${_mpi_libs}") + set(_mpi_lib_flags "") + foreach(_l ${_mpi_libs}) + list(APPEND _mpi_lib_flags "-l${_l}") + endforeach() + message(STATUS " includes : ${_mpi_incdirs}") + message(STATUS " libdirs : ${_mpi_libdirs}") + message(STATUS " libs : ${_mpi_lib_flags}") + foreach(_tgt MPI::MPI_C MPI::MPI_CXX) + if(NOT TARGET ${_tgt}) + add_library(${_tgt} INTERFACE IMPORTED) + endif() + set_target_properties(${_tgt} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${_mpi_incdirs}" + INTERFACE_LINK_DIRECTORIES "${_mpi_libdirs}" + INTERFACE_LINK_LIBRARIES "${_mpi_lib_flags}") + endforeach() +else() + find_package(MPI REQUIRED COMPONENTS C CXX) +endif() + +# All binaries land in /bin/ +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# Default to Release so -O3 is active unless the user overrides +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) +endif() + +option(BLINK_COVERAGE "Build with gcov coverage instrumentation (-O0 --coverage)" OFF) +if(BLINK_COVERAGE) + # -O0 for accurate line attribution; atomic counter updates so the multiple + # concurrent MPI ranks of a single benchmark binary don't corrupt the shared + # .gcda files when they dump coverage on exit. + add_compile_options(-O0 -g --coverage -fprofile-update=atomic) + add_link_options(--coverage) +else() + add_compile_options(-O3) +endif() +add_compile_definitions(_GNU_SOURCE) + +add_subdirectory(src) + +option(BLINK_TESTS "Build correctness tests (requires GTest)" ON) + +if(BLINK_TESTS) + find_package(GTest QUIET) + if(NOT GTest_FOUND) + message(STATUS "GTest not found via find_package — fetching with FetchContent") + include(FetchContent) + FetchContent_Declare( + googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG v1.14.0 + ) + set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) + if(NOT TARGET GTest::GTest) + add_library(GTest::GTest ALIAS gtest) + add_library(GTest::Main ALIAS gtest_main) + endif() + set(GTest_FOUND TRUE) + endif() + + if(GTest_FOUND) + enable_testing() + add_subdirectory(tests) + message(STATUS "GTest ready — correctness tests enabled (run: ctest --test-dir build)") + else() + message(WARNING "GTest unavailable — correctness tests skipped (pass -DBLINK_TESTS=OFF to silence)") + endif() +else() + message(STATUS "BLINK_TESTS=OFF — correctness tests skipped") +endif() diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..defa5e082 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024-2026 HLC-Lab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile deleted file mode 100644 index c65649e8f..000000000 --- a/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -# Directories -SRC_DIR = . -BIN_DIR = bin - -CC = mpicc -CXX = mpicxx - -# Source files -C_SRCS = $(wildcard $(SRC_DIR)/*.c) -CPP_SRCS = $(wildcard $(SRC_DIR)/*.cpp) - -# Executable names -C_BINS = $(patsubst $(SRC_DIR)/%.c, $(BIN_DIR)/%, $(C_SRCS)) -CPP_BINS = $(patsubst $(SRC_DIR)/%.cpp, $(BIN_DIR)/%, $(CPP_SRCS)) - -# Compilation flags -CFLAGS = -lm -D_GNU_SOURCE -O3 -CXXFLAGS = -lm -D_GNU_SOURCE -O3 # Matches CFLAGS, but you can add C++ specific flags like -std=c++17 here - -# Targets -all: $(C_BINS) $(CPP_BINS) - -# Rule for C files -$(BIN_DIR)/%: $(SRC_DIR)/%.c $(SRC_DIR)/common.h - @mkdir -p $(BIN_DIR) - $(CC) -o $@ $< $(CFLAGS) - -# Rule for C++ files -$(BIN_DIR)/%: $(SRC_DIR)/%.cpp $(SRC_DIR)/common.h - @mkdir -p $(BIN_DIR) - $(CXX) -o $@ $< $(CXXFLAGS) - -clean: - -rm -f $(C_BINS) $(CPP_BINS) diff --git a/README.md b/README.md new file mode 100644 index 000000000..0f3b610c5 --- /dev/null +++ b/README.md @@ -0,0 +1,412 @@ +
+ Blink logo +
+ +
+ +[![CI](https://github.com/hlc-lab/blink/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/hlc-lab/blink/actions/workflows/ci.yml?query=branch%3Adev) +[![CodeQL](https://github.com/hlc-lab/blink/actions/workflows/codeql.yml/badge.svg?branch=dev)](https://github.com/hlc-lab/blink/actions/workflows/codeql.yml?query=branch%3Adev) +[![codecov](https://codecov.io/gh/hlc-lab/blink/branch/dev/graph/badge.svg)](https://codecov.io/gh/hlc-lab/blink/tree/dev) +[![License: MIT](https://img.shields.io/github/license/hlc-lab/blink)](LICENSE) + +
+ + +Blink is a collection of MPI benchmarks designed for long-running, in-situ measurement of network behaviour under realistic traffic conditions. Each benchmark runs for a configurable number of iterations (or endlessly until interrupted), records per-iteration latency on every rank, and emits a CSV summary when it finishes — either naturally or via `SIGUSR1`. + +## How Blink differs from other benchmarking suites + +Most MPI benchmark suites run a fixed iteration sweep and report a single aggregate table (min / median / max per message size) before exiting. Blink takes a different approach on three axes: + +**Temporal evolution.** Blink records one measurement per iteration rather than collapsing everything into a final summary. This lets you observe how latency evolves over time, detect jitter events, and correlate performance spikes with external activity on the system. + +**Burst traffic modeling.** Real applications rarely issue communication at a steady, uniform rate. Blink supports configurable burst-pause cycles with pluggable inter-arrival distributions — exponential, Pareto, and log-normal — so the generated traffic pattern can more closely reflect the bursty, heavy-tailed nature of production workloads. + +**Long-running operation.** Blink is designed to run alongside real workloads or as a background monitor. It can run endlessly (`-endl`), keeps only the most recent *N* samples in a ring buffer (`-maxsamples`), and responds to `SIGUSR1` with a clean shutdown — collecting and printing whatever has been measured so far before calling `MPI_Finalize`. + +## Building + +Requirements: CMake ≥ 3.16, an MPI implementation (e.g. OpenMPI, MPICH, Intel MPI). + +```bash +cmake -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build -j$(nproc) +``` + +Binaries are placed in `build/bin/`. Adding a new `.c` or `.cpp` file anywhere under `src/` is enough — CMake picks it up automatically on the next configure. + + +## Testing + +Correctness tests are built automatically alongside the benchmarks (requires GTest >= 1.14, fetched automatically if not found on the system). Pass `-DBLINK_TESTS=OFF` to skip them. + +Each test is a single-process GTest binary that spawns the actual benchmark under `mpirun -n N -debug` via `popen()` and parses structured `DEBUG key=val` output. Most suites require **8 MPI ranks**; `test_pingpong` also covers a 2-rank case for `pingpong_b`. + +### Run all tests + +```bash +ctest --test-dir build --output-on-failure +``` + +| Useful flags | Effect | +|---|---| +| `-j ` | Run up to N test suites in parallel | +| `-V` | Always print test output (verbose) | +| `--output-on-failure` | Print output only on failure | +| `-R ` | Run only suites whose name matches | + +### Run a single test suite + +```bash +ctest --test-dir build -R test_collectives --output-on-failure +``` + +### Test suites + +| Suite | Benchmarks covered | Properties verified | +|---|---|---| +| `test_collectives` | `allgather_b/nb/comm_only`, `allreduce_b/nb`, `alltoall_b/nb/man/comm_only`, `barrier_b/nb`, `broadcast_b/nb`, `gather_b/nb`, `scatter_b/nb`, `reduce_b/nb`, `reduce_scatter_b/nb` | Coverage, communicator size, root rank, data integrity (plain + burst), b/nb agreement, allgather transfer-volume agreement | +| `test_pairwise` | `pairwise_b/nb/bsnbr` | Coverage, validity, symmetry (offpair/rpair), bijection, exact pairs, non-reciprocal modes (perm/rot) (plain + burst) | +| `test_ring` | `ring_nb`, `ring_bsnbr` | Coverage, validity, exact sequential neighbours, consistency (sequential + random), completeness (plain + burst) | +| `test_incast` | `incast_b/nb/bsnbr/get/put` | Coverage, roles (one receiver, N-1 senders), target correctness (plain + burst) | +| `test_pingpong` | `pingpong_b` (2 ranks), `pingpong_pairwise_b` (8 ranks) | Coverage, communicator size, data integrity (plain + burst) | +| `test_kpartners` | `kpartners_nb` | Coverage, communicator size, k value, slot coherence (plain + burst) | +| `test_stencil` | `stencil_2d_nb` | Coverage, communicator size, data integrity, neighbour-graph consistency (plain, periodic, burst) | +| `test_misc` | cross-cutting (`barrier_nb`, `incast_b`, `dist_test`, `alltoall_nb`) | Ring-buffer wrap, warm-up exclusion, pretty-print formatter, `-mrand` determinism, sampler self-check, SIGUSR1 clean shutdown | + +### Overriding MPI launcher + +The test binaries bake in the MPI launcher paths at configure time. Override them at runtime with environment variables: + +```bash +BLINK_MPIEXEC=/opt/mpi/bin/mpirun BLINK_NPROC_FLAG=-np BLINK_BIN_DIR=/custom/bin ctest --test-dir build +``` + +### Continuous integration + +Every push and pull request triggers the [CI workflow](.github/workflows/ci.yml): it +builds every benchmark and the test suite on Ubuntu with OpenMPI, runs `ctest`, and +produces a coverage report. The badges at the top of this file show the latest result. + +The build is instrumented via `-DBLINK_COVERAGE=ON` (`-O0 --coverage`, atomic counters +so concurrent MPI ranks don't corrupt the `.gcda` files); a [`gcovr`](https://gcovr.com) +summary of `src/` is printed to the run's job summary, attached as an artifact, and +uploaded to Codecov. To reproduce a coverage run locally: + +```bash +cmake -B build-cov -DBLINK_TESTS=ON -DBLINK_COVERAGE=ON +cmake --build build-cov -j$(nproc) +ctest --test-dir build-cov --output-on-failure +gcovr --root . --filter 'src/' --exclude-unreachable-branches --exclude-throw-branches \ + --exclude-lines-by-pattern '.*MPI_Abort.*' --print-summary +``` + +## Common flags + +Every benchmark understands the following flags: + +| Flag | Default | Description | +|------|---------|-------------| +| `-msgsize ` | `1024` | Message size in bytes | +| `-iter ` | `1` | Number of measured iterations (after warm-up) | +| `-warmup ` | `5` | Number of warm-up iterations (not recorded) | +| `-endl` | off | Run endlessly until `SIGUSR1` | +| `-grty ` | `1` | Measurement granularity — number of MPI calls batched per timed window | +| `-maxsamples ` | `1000` | Maximum recorded samples (ring buffer) | +| `-mrank ` | `0` | Rank that collects and prints results | +| `-mrand` | off | Pick master rank randomly | +| `-seed ` | `1` | RNG seed (shared across ranks) | +| `-blength ` | `0` | Mean burst length (0 = single shot per iteration). Bare number = seconds; suffixes `s`, `ms`, `us`, `ns` accepted (e.g. `400us`). | +| `-bldist ` | — | Randomise burst length using distribution `D` (`exp`, `pareto`, `lognormal`) | +| `-blshape ` | `1.5` | Shape parameter for burst distribution (α for Pareto, σ for log-normal) | +| `-bpause ` | `0` | Mean pause between bursts. Same unit conventions as `-blength`. | +| `-bpdist ` | — | Randomise pause length using distribution `D` (`exp`, `pareto`, `lognormal`) | +| `-bpshape ` | `1.5` | Shape parameter for pause distribution (α for Pareto, σ for log-normal) | +| `-pretty-print` | off | Human-readable table output instead of CSV (see [Output format](#output-format)) | +| `-plot` | off | Append an ASCII histogram of the per-iteration timing distribution (see [Output format](#output-format)) | +| `-plotstat ` | `max` | Which cross-rank statistic to histogram each iteration: `avg`, `min`, `max`, `median`, `mainrank` | +| `-plotbins ` | `10` | Number of histogram bins | +| `-plotbinsize ` | — | Fixed bin width with optional unit (e.g. `2ms`, `500us`, `0.001`); overrides `-plotbins`, linear bins only | +| `-plotlog` | off | Use logarithmic (geometric) bins — good for heavy-tailed latencies | +| `-h`, `--help`, `-help` | — | Print usage (common flags + benchmark-specific options if any) and exit | + +> **Tip:** every benchmark accepts `-h` (or `--help`); benchmarks with extra flags (`pairwise_*`, `kpartners_nb`, `ring_*`, `stencil_2d_nb`) append a `Benchmark-specific options` section listing them. + +### Burst distributions + +When `-bldist` or `-bpdist` is supplied, burst lengths / pauses are drawn independently on each iteration from the chosen distribution, parameterised so that the empirical mean equals `-blength` / `-bpause` regardless of the shape parameter: + +| Distribution | Flag value | Shape parameter | Notes | +|---|---|---|---| +| Exponential | `exp` | — (ignored) | Light-tailed; fully determined by its mean | +| Pareto | `pareto` | α (`-blshape` / `-bpshape`, must be > 1) | Heavy-tailed; α ≤ 2 → infinite variance | +| Log-normal | `lognormal` | σ (`-blshape` / `-bpshape`, > 0) | Log-symmetric; larger σ → heavier tail | + +```bash +# Pareto bursts (α=2, heavy tail) with exponential pauses +mpirun -n 8 build/bin/alltoall_nb -iter 500 \ + -blength 10ms -bldist pareto -blshape 2.0 \ + -bpause 50ms -bpdist exp +``` + +The sampler implementations can be verified independently: + +```bash +mpirun -n 1 build/bin/dist_test +``` + +## Auxiliary tools + +Besides the benchmarks, `src/tools/` builds a few helper binaries: + +| Binary | Purpose | +|--------|---------| +| `dist_test` | Verifies the burst-distribution samplers against their theoretical CDFs (ASCII histogram + mean / variance checks + KS test). Run single-rank: `mpirun -n 1 build/bin/dist_test`. | +| `checker` | An all-to-all workload (like `alltoall_b`) that additionally writes a per-iteration wall-clock timestamp log on the master rank (`checker_.log`, local time) — useful for correlating throughput dips with external system activity over a long run. Accepts the common flags. | +| `null_dummy` | Bare `MPI_Init` / `MPI_Finalize` with no communication; a baseline for measuring MPI startup / teardown overhead. | + +## Benchmarks + +Benchmarks are grouped by traffic pattern. The suffix convention is: + +| Suffix | Meaning | +|--------|---------| +| `_b` | Blocking MPI call | +| `_nb` | Non-blocking MPI call (`MPI_I*` + `MPI_Waitall`) | +| `_bsnbr` | Non-blocking send, blocking receive (anti-deadlock variant) | +| `_man` | Manual / hand-rolled implementation (no collective primitive) | +| `_comm_only` | Communication-only variant (no computation, C++) | +| `_get` / `_put` | One-sided (RMA) variant | + +### Collectives + +#### All-to-all — `alltoall/` +Every rank sends a distinct message to every other rank. + +| Binary | MPI primitive | +|--------|--------------| +| `alltoall_b` | `MPI_Alltoall` | +| `alltoall_nb` | `MPI_Ialltoall` | +| `alltoall_man` | `MPI_Isend` / `MPI_Irecv` (manual) | +| `alltoall_comm_only` | `MPI_Isend` / `MPI_Irecv` (C++, no memory init) | + +#### All-gather — `allgather/` +Every rank broadcasts its buffer; all ranks collect the full result. + +| Binary | MPI primitive | +|--------|--------------| +| `allgather_b` | `MPI_Allgather` | +| `allgather_nb` | `MPI_Iallgather` | +| `allgather_comm_only` | `MPI_Isend` / `MPI_Irecv` (C++) | + +#### All-reduce — `allreduce/` +Global reduction (`MPI_SUM` over `int`) delivered to all ranks. + +| Binary | MPI primitive | +|--------|--------------| +| `allreduce_b` | `MPI_Allreduce` | +| `allreduce_nb` | `MPI_Iallreduce` | + +#### Barrier — `barrier/` + +| Binary | MPI primitive | +|--------|--------------| +| `barrier_b` | `MPI_Barrier` | +| `barrier_nb` | `MPI_Ibarrier` | + +#### Broadcast — `broadcast/` +Root sends one buffer to all other ranks. + +| Binary | MPI primitive | +|--------|--------------| +| `broadcast_b` | `MPI_Bcast` | +| `broadcast_nb` | `MPI_Ibcast` | + +#### Gather — `gather/` +All ranks send to root; root collects all messages. + +| Binary | MPI primitive | +|--------|--------------| +| `gather_b` | `MPI_Gather` | +| `gather_nb` | `MPI_Igather` | + +#### Scatter — `scatter/` +Root distributes a distinct chunk to every rank. + +| Binary | MPI primitive | +|--------|--------------| +| `scatter_b` | `MPI_Scatter` | +| `scatter_nb` | `MPI_Iscatter` | + +#### Reduce — `reduce/` +Global reduction (`MPI_SUM` over `int`) to root only. + +| Binary | MPI primitive | +|--------|--------------| +| `reduce_b` | `MPI_Reduce` | +| `reduce_nb` | `MPI_Ireduce` | + +#### Reduce-scatter — `reduce_scatter/` +Reduction followed by a scatter of the result chunks. + +| Binary | MPI primitive | +|--------|--------------| +| `reduce_scatter_b` | `MPI_Reduce_scatter` | +| `reduce_scatter_nb` | `MPI_Ireduce_scatter` | + +--- + +### Point-to-point patterns + +#### Ping-pong — `pingpong/` +Latency measurement between a pair of ranks. + +| Binary | Description | +|--------|------------| +| `pingpong_b` | Single pair (rank 0 ↔ rank 1) | +| `pingpong_pairwise_b` | All pairs simultaneously | + +#### Incast — `incast/` +All non-root ranks send to root simultaneously. + +| Binary | Variant | +|--------|---------| +| `incast_b` | Blocking receive at root | +| `incast_nb` | Non-blocking (`MPI_Irecv` at root) | +| `incast_bsnbr` | Non-blocking send, blocking receive | +| `incast_get` | One-sided: root issues `MPI_Get` from each sender | +| `incast_put` | One-sided: senders issue `MPI_Put` to root window | + +#### Pairwise — `pairwise/` +Each rank exchanges messages with exactly one partner. Partners are chosen by offset or at random. + +| Binary | Variant | +|--------|---------| +| `pairwise_b` | Blocking (`MPI_Sendrecv`) | +| `pairwise_nb` | Non-blocking (`MPI_Isend` / `MPI_Irecv`) | +| `pairwise_bsnbr` | Non-blocking send, blocking receive | + +Extra flags: `-offset ` (default `1`) — fixed partner offset; `-mode ` (default `offpair`) — pairing mode. Supported modes: + +| Mode | Meaning | +|------|---------| +| `offpair` | Disjoint reciprocal pairs at fixed offset `O` (e.g. `(0,1) (2,3) …`) | +| `rpair` | Uniformly random disjoint pairs (same seed on all ranks) | +| `perm` | Random permutation: each rank sends to one partner (not necessarily reciprocal) | +| `rot` | Ring rotation: rank `i` sends to `(i+O) mod n` | + +#### Ring — `ring/` +Each rank simultaneously sends to both its left and right neighbours and receives one message from each (bidirectional ring exchange). + +| Binary | Variant | +|--------|---------| +| `ring_nb` | Non-blocking (`MPI_Isend` / `MPI_Irecv`) | +| `ring_bsnbr` | Non-blocking send, blocking receive | + +Extra flag: `-rring` — randomise the ring order instead of using rank order. + +--- + +### Synthetic / application-inspired patterns + +#### 2-D stencil — `stencil/` +Each rank exchanges halo data with its four Cartesian neighbours (north, south, west, east). Uses an `MPI_Cart_create` communicator. Boundary ranks have `MPI_PROC_NULL` neighbours which complete immediately with no data transfer, so no special-casing is needed for edge ranks. + +Binary: `stencil_2d_nb` + +Extra flags: + +| Flag | Default | Description | +|------|---------|-------------| +| `-dimx ` | `0` | Fix the X dimension of the process grid; Y is derived as `w_size / X`. `0` lets `MPI_Dims_create` choose automatically. | +| `-periodic` | off | Make the Cartesian grid periodic (toroidal) in both dimensions | + +```bash +mpirun -n 16 build/bin/stencil_2d_nb -msgsize 65536 -iter 100 -dimx 4 +``` + +#### k-random partners — `kpartners/` +Each rank communicates with *k* randomly chosen partners per iteration. The partner sets are derived from *k* independent random permutations (same seed on all ranks), guaranteeing that the resulting graph is k-regular: every rank sends exactly *k* messages and receives exactly *k* messages. + +Binary: `kpartners_nb` + +Extra flag: + +| Flag | Default | Description | +|------|---------|-------------| +| `-k ` | `1` | Number of random partners per rank (capped at `w_size - 1`) | + +```bash +mpirun -n 32 build/bin/kpartners_nb -k 4 -msgsize 4096 -iter 200 +``` + +--- + +## Output format + +### CSV (default) + +All benchmarks print a CSV block to stdout on the master rank. Each line corresponds to one measured iteration: + +``` +Average,Minimum,Maximum,Median,MainRank +,,,, +... +Ran N iterations. Measured M iterations. +``` + +Times are in seconds (9 decimal places). The `Average`, `Minimum`, `Maximum`, and `Median` columns are computed across all MPI ranks for that iteration. `MainRank` is the raw value recorded on the master rank. + +### Pretty-print + +Pass `-pretty-print` to get a human-readable table with auto-scaled units (`us` / `ms` / `s`) instead of CSV: + +```bash +mpirun -n 8 build/bin/alltoall_nb -iter 5 -pretty-print +``` + +``` + # avg min max median + ------------------------------------------------------------- + 1 12.45 us 9.12 us 18.67 us 11.34 us + 2 11.98 us 9.08 us 17.23 us 11.12 us + ------------------------------------------------------------- + 5 samples · 10 iterations total +``` + +The `MainRank` column is omitted in pretty mode. Pipe through a tool such as `sed 's/\x1b\[[0-9;]*m//g'` to strip ANSI colour codes if needed. + +### Distribution plot + +Pass `-plot` to **append** an ASCII histogram of the per-iteration timing distribution (the CSV / pretty table is still printed first). Each iteration contributes one value — the cross-rank statistic chosen with `-plotstat` (default `max`, i.e. the slowest rank, which is the completion time of a collective). A percentile footer summarises the tail regardless of how the bins fall. + +```bash +mpirun -n 8 build/bin/alltoall_nb -iter 400 -plot -plotstat max +``` + +``` + Per-iteration latency · stat=max · n=400 · linear bins + ---------------------------------------------------------------------- + [ 6.05 us - 32.88 us] ████████████████████████████████████████ 97.8% + [ 32.88 us - 59.70 us] 1.0% + ... + [ 247.49 us - 274.31 us] 0.2% + ---------------------------------------------------------------------- + n=400 · min 6.05 us · mean 9.09 us · p50 6.62 us · p90 7.15 us · p99 74.71 us · max 274.31 us +``` + +Binning options: + +- `-plotbins ` — number of bins (default `10`); always produces a readable, fixed-height histogram. +- `-plotbinsize ` — fixed bin **width** with an optional unit (`2ms`, `500us`, `100ns`, or a bare number = seconds). Edges are aligned to multiples of the width so plots from different runs line up. Overrides `-plotbins`; capped at 64 bins (the tail folds into the last bin, with a note) so a tiny width can't flood the terminal. +- `-plotlog` — logarithmic (geometric) bins, which reveal the shape of heavy-tailed latency distributions far better than linear bins. Cannot be combined with `-plotbinsize`. + +As with pretty-print, pipe through `sed 's/\x1b\[[0-9;]*m//g'` to strip the ANSI colours. + +## Early termination + +Send `SIGUSR1` to the master process to trigger a clean shutdown: remaining iterations are skipped, results collected so far are printed, and all ranks call `MPI_Finalize`. + +```bash +kill -USR1 +``` diff --git a/a2a_b.c b/a2a_b.c deleted file mode 100644 index d3a7c7b8b..000000000 --- a/a2a_b.c +++ /dev/null @@ -1,207 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" - -int main(int argc, char** argv){ - - /*init MPI world*/ - MPI_Init(&argc,&argv); - MPI_Comm_size(MPI_COMM_WORLD, &w_size); - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - - /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here - - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,k; - - /*read cmd line args*/ - for(i=1;i= 8 && msg_size % 8 == 0){ // Check if I can use 64-bit data types - large_count = msg_size / 8; - if (large_count >= ((u_int64_t) (1UL << 32)) - 1) { // If large_count can't be represented on 32 bits - if(my_rank == 0){ - printf("\tTransfer size (B): -1, Transfer Time (s): -1, Bandwidth (GB/s): -1, Iteration -1\n"); - } - return -1; - } - }else{ - if (msg_size >= ((u_int64_t) (1UL << 32)) - 1) { // If msg_size can't be represented on 32 bits - if(my_rank == 0){ - printf("\tTransfer size (B): -1, Transfer Time (s): -1, Bandwidth (GB/s): -1, Iteration -1\n"); - } - return -1; - } - } - - MPI_Barrier(MPI_COMM_WORLD); - do{ - for(k=0;k -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" - - - -void all2all_memcpy(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm){ - - int rank, size; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &size); - - int datatype_size; - MPI_Type_size(sendtype, &datatype_size); - - const char* sbuf = static_cast(sendbuf); - char* rbuf = static_cast(recvbuf); - - double mem_time = MPI_Wtime(); - // Copy local data directly (self-send) - memcpy(rbuf + rank * datatype_size * recvcount, - sbuf + rank * datatype_size * sendcount, - sendcount * datatype_size); - -} - -void custom_alltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, - void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) { - int rank, size; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &size); - - int datatype_size; - MPI_Type_size(sendtype, &datatype_size); - - const char* sbuf = static_cast(sendbuf); - char* rbuf = static_cast(recvbuf); - - std::vector requests; - for (int i = 0; i < size; ++i) { - if (i == rank) continue; - - MPI_Request req_recv; - MPI_Request req_send; - - MPI_Isend(sbuf + i * datatype_size * sendcount, sendcount, sendtype, i, 0, comm, &req_send); - MPI_Irecv(rbuf + i * datatype_size * recvcount, recvcount, recvtype, i, 0, comm, &req_recv); - - requests.push_back(req_send); - requests.push_back(req_recv); - } - - MPI_Waitall(static_cast(requests.size()), requests.data(), MPI_STATUSES_IGNORE); -} - -int main(int argc, char** argv){ - - /*init MPI world*/ - MPI_Init(&argc,&argv); - MPI_Comm_size(MPI_COMM_WORLD, &w_size); - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - - /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here - - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - size_t msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,k; - - /*read cmd line args*/ - for(i=1;i= 8 && msg_size % 8 == 0){ // Check if I can use 64-bit data types - large_count = msg_size / 8; - if (large_count >= ((u_int64_t) (1UL << 32)) - 1) { // If large_count can't be represented on 32 bits - if(my_rank == 0){ - printf("\tTransfer size (B): -1, Transfer Time (s): -1, Bandwidth (GB/s): -1, Iteration -1\n"); - } - return -1; - } - }else{ - if (msg_size >= ((u_int64_t) (1UL << 32)) - 1) { // If msg_size can't be represented on 32 bits - if(my_rank == 0){ - printf("\tTransfer size (B): -1, Transfer Time (s): -1, Bandwidth (GB/s): -1, Iteration -1\n"); - } - return -1; - } - } - - MPI_Barrier(MPI_COMM_WORLD); - do{ - for(k=0;k -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" - -int main(int argc, char** argv){ - - /*init MPI world*/ - MPI_Init(&argc,&argv); - MPI_Comm_size(MPI_COMM_WORLD, &w_size); - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - - /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here - - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,j,k; - - /*read cmd line args*/ - for(i=1;i -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" - -int main(int argc, char** argv){ - - /*init MPI world*/ - MPI_Init(&argc,&argv); - MPI_Comm_size(MPI_COMM_WORLD, &w_size); - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - - /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here - - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,j,k; - - /*read cmd line args*/ - for(i=1;i + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Blink + A BENCHMARK FOR LARGE-SCALE INTERCONNECTION NETWORKS + + diff --git a/bursty_noise_a2a.cpp b/bursty_noise_a2a.cpp deleted file mode 100644 index 30cabb556..000000000 --- a/bursty_noise_a2a.cpp +++ /dev/null @@ -1,129 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" - - -void all2all_memcpy(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm){ - - int rank, size; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &size); - - int datatype_size; - MPI_Type_size(sendtype, &datatype_size); - - const char* sbuf = static_cast(sendbuf); - char* rbuf = static_cast(recvbuf); - - double mem_time = MPI_Wtime(); - // Copy local data directly (self-send) - std::memcpy(rbuf + rank * datatype_size * recvcount, - sbuf + rank * datatype_size * sendcount, - sendcount * datatype_size); - -} - -void custom_alltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, - void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) { - int rank, size; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &size); - - int datatype_size; - MPI_Type_size(sendtype, &datatype_size); - - const char* sbuf = static_cast(sendbuf); - char* rbuf = static_cast(recvbuf); - - std::vector requests; - for (int i = 0; i < size; ++i) { - if (i == rank) continue; - - MPI_Request req_recv; - MPI_Request req_send; - - MPI_Isend(sbuf + i * datatype_size * sendcount, sendcount, sendtype, i, 0, comm, &req_send); - MPI_Irecv(rbuf + i * datatype_size * recvcount, recvcount, recvtype, i, 0, comm, &req_recv); - - requests.push_back(req_send); - requests.push_back(req_recv); - } - - MPI_Waitall(static_cast(requests.size()), requests.data(), MPI_STATUSES_IGNORE); -} - -int main(int argc, char** argv) { - MPI_Init(&argc, &argv); - - int size, rank; - MPI_Comm_size(MPI_COMM_WORLD, &size); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - const size_t BUFFER_SIZE = 2 * 1024 * 1024; // bytes per peer 2MiB - - // Each process will send a chunk to every other process - unsigned char *send_buffer = (unsigned char*) malloc(BUFFER_SIZE*size); - unsigned char *recv_buffer = (unsigned char*) malloc(BUFFER_SIZE*size); - if (send_buffer == NULL || recv_buffer == NULL) { - fprintf(stderr, "Memory allocation failed!\n"); - MPI_Abort(MPI_COMM_WORLD, 1); - return -1; - } - - srand(time(NULL)*rank); - for (int i = 0; i < BUFFER_SIZE*size; i++) { - send_buffer[i] = rand()*rank % size; - } - - double burst_pause; - double burst_length; - if(argc >= 2){ - burst_pause = atof(argv[1]); - }else{ - std::cerr << "Not enough arguments. Usage: ./bursty_noise_a2a " << std::endl; - return 1; - } - if(argc >= 3){ - burst_length = atof(argv[2]); - } else { - std::cerr << "Not enough arguments. Usage: ./bursty_noise_a2a " << std::endl; - return 1; - } - - - bool burst_pause_rand = false; - double burst_start_time; - double measure_start_time; - double burst_length_mean=burst_length; - double burst_pause_mean=burst_pause; - int burst_cont=0; - - while (1) { - burst_start_time=MPI_Wtime(); - do { - all2all_memcpy(send_buffer, BUFFER_SIZE, MPI_BYTE, recv_buffer, BUFFER_SIZE, MPI_BYTE, MPI_COMM_WORLD); - custom_alltoall(send_buffer, BUFFER_SIZE, MPI_BYTE, recv_buffer, BUFFER_SIZE, MPI_BYTE, MPI_COMM_WORLD); - - if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ - if(rank == 0){ /*master decides if burst should be continued*/ - burst_cont=((MPI_Wtime()-burst_start_time) -#include -#include -#include -#include -#include -#include -#include -#include "common.h" - -int main(int argc, char** argv) { - MPI_Init(&argc, &argv); - - int size, rank; - MPI_Comm_size(MPI_COMM_WORLD, &size); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - const size_t BUFFER_SIZE = 2 * 1024 * 1024; // bytes per peer 2MiB - - // Each process will send a chunk to every other process - unsigned char *buffer = (unsigned char*) malloc_align(BUFFER_SIZE); - if (buffer == NULL) { - fprintf(stderr, "Memory allocation failed!\n"); - MPI_Abort(MPI_COMM_WORLD, 1); - return -1; - } - - srand(time(NULL)*rank); - for (int i = 0; i < BUFFER_SIZE; i++) { - buffer[i] = rand()*rank % size; - } - - double burst_pause; - double burst_length; - if(argc >= 2){ - burst_pause = atof(argv[1]); - }else{ - std::cerr << "Not enough arguments. Usage: ./bursty_noise_a2a " << std::endl; - return 1; - } - if(argc >= 3){ - burst_length = atof(argv[2]); - } else { - std::cerr << "Not enough arguments. Usage: ./bursty_noise_a2a " << std::endl; - return 1; - } - - - bool burst_pause_rand = false; - double burst_start_time; - double measure_start_time; - double burst_length_mean=burst_length; - double burst_pause_mean=burst_pause; - int burst_cont=0; - - while (1) { - std::vector requests; - burst_start_time=MPI_Wtime(); - do { - // INCAST START - requests.clear(); - - if (rank == 0) { - - for (int sender = 1; sender < size; ++sender) { - MPI_Request req; - MPI_Irecv(buffer, BUFFER_SIZE, MPI_BYTE, sender, 0, MPI_COMM_WORLD, &req); - requests.push_back(req); - } - - } else { - MPI_Request req; - MPI_Isend(buffer, BUFFER_SIZE, MPI_BYTE, 0, 0, MPI_COMM_WORLD, &req); - requests.push_back(req); - } - - MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE); - // INCAST END - - if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ - if(rank == 0){ /*master decides if burst should be continued*/ - burst_cont=((MPI_Wtime()-burst_start_time) -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" - -int main(int argc, char** argv){ - - /*init MPI world*/ - MPI_Init(&argc,&argv); - MPI_Comm_size(MPI_COMM_WORLD, &w_size); - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - - /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here - - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,k; - - /*read cmd line args*/ - for(i=1;itm_hour - , time_str_tm->tm_min - , time_str_tm->tm_sec - , time_now.tv_usec); - fflush(fd_temp); - } - //----------------- - - MPI_Barrier(MPI_COMM_WORLD); - do{ - for(k=0;ktm_hour - , time_str_tm->tm_min - , time_str_tm->tm_sec - , time_now.tv_usec - , w_size - , curr_iters); - fflush(fd_temp); - } - //----------------- - if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ - if(my_rank==master_rank){ /*master decides if burst should be continued*/ - burst_cont=((MPI_Wtime()-burst_start_time) -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/*draw a exponentially distributed number with expectation=mean*/ -static double rand_expo(double mean) -{ - double lambda = 1.0 / mean; - double u = rand() / (RAND_MAX + 1.0); - return -log(1 - u) / lambda; -} - -/*sleep seconds given as double*/ -static int dsleep(double t) -{ - struct timespec t1, t2; - t1.tv_sec = (long)t; - t1.tv_nsec = (t - t1.tv_sec) * 1000000000L; - return nanosleep(&t1, &t2); -} - -/*double comparison function for quicksort*/ -int compare_doubles(const void *p1, const void *p2) -{ - if (*(double *)p1 < *(double *)p2) - return -1; - else if (*(double *)p1 > *(double *)p2) - return 1; - else - return 0; -} - -/*global variables because of signal handling*/ -static int my_rank; -static int w_size; -static int master_rank; -static int curr_iters; -static int warm_up_iters; -static int max_samples; -static double *durations; - -static void write_results() -{ - double duration_sum; - double duration_median; - int num_samples; - int i; - int start_index; - double *tmp_buf = NULL; - - if (curr_iters > max_samples) // Wrapped the sampling recording - { - num_samples = max_samples; - start_index = curr_iters % max_samples; - tmp_buf = (double *)malloc(sizeof(double)*num_samples); - // Copy the data from the durations buffer to tmp_buf (so that it is in the proper order) - memcpy(tmp_buf, &(durations[start_index]), sizeof(double)*(num_samples - start_index)); - memcpy(&tmp_buf[num_samples - start_index], durations, sizeof(double)*start_index); - } - else - { - num_samples = curr_iters - warm_up_iters; - start_index = warm_up_iters; - tmp_buf = (double *)malloc(sizeof(double)*num_samples); - memcpy(tmp_buf, &(durations[start_index]), sizeof(double)*num_samples); - } - - - double *all_data = (double *)malloc(sizeof(double)*num_samples*w_size); - double *sorting_buf = (double*)malloc(sizeof(double)*w_size); - - if(all_data == NULL || sorting_buf == NULL){ - fprintf(stderr,"Failed to allocate a buffer on rank %d\n",my_rank); - exit(-1); - } - - /*print file header*/ - if (my_rank == master_rank) - { - printf("Average,Minimum,Maximum,Median,MainRank\n"); - } - - // We need to first gather all the data - MPI_Gather(tmp_buf, num_samples, MPI_DOUBLE, all_data, num_samples, MPI_DOUBLE, master_rank, MPI_COMM_WORLD); - - if(my_rank == master_rank){ - for(i = 0; i < num_samples; i++){ - int j; - duration_sum = 0; - for(j = 0; j < w_size; j++){ - sorting_buf[j] = all_data[j*num_samples + i]; - duration_sum += sorting_buf[j]; - } - - qsort(sorting_buf, w_size, sizeof(double), compare_doubles); - if (w_size % 2 == 0) - { /*even: then median as mean of middle values*/ - duration_median = (sorting_buf[(w_size - 1) / 2] + sorting_buf[w_size / 2]) / 2; - } - else - { /*odd: else median as middle value*/ - duration_median = sorting_buf[(w_size - 1) / 2]; - } - printf("%.9f,%.9f,%.9f,%.9f,%.9f\n", duration_sum / w_size, sorting_buf[0], sorting_buf[w_size - 1], duration_median, tmp_buf[i]); - } - - printf("Ran %d iterations. Measured %d iterations.\n", curr_iters, num_samples); - fflush(stdout); - } - free(sorting_buf); - free(all_data); - free(tmp_buf); -} - -/*signal handler*/ -void sig_handler(int sig) -{ - write_results(); - MPI_Finalize(); - exit(0); -} - -/*use Fisher-Yates to permute array*/ -static void permute(int *a, int n) -{ - int j; - int t; - int i; - for (i = n; i > 1; i--) - { - j = rand() % i; - t = a[i - 1]; - a[i - 1] = a[j]; - a[j] = t; - } -} - -/*mathematical mod without negativ numbers*/ -static int mod(int a, int b) -{ - int c = a % b; - if (c < 0) - c += b; - return c; -} - -/*produce random pairs*/ -static void random_pairs(int *a, int n) -{ - int i, j; - for (i = 0; i < n; i++) - { - a[i] = -1; - } - if (n % 2 == 1) - { - n--; - a[n] = n; /*if odd last rank targets itself*/ - } - int k = 1; - int t; - for (i = 0; i < n; i++) - { - if (a[i] == -1) - { - t = rand() % (n - k); - for (j = i + 1; j < n; j++) - { - if (a[j] == -1) - { - if (t == 0) - { - a[i] = j; - a[j] = i; - k += 2; - break; - } - t--; - } - } - } - } -} - -/*produce fixed offset pairs*/ -static void offset_pairs(int *a, int n, int o) -{ - int i; - for (i = 0; i < n; i++) - { - a[i] = -1; - } - int t; - for (i = 0; i < n; i++) - { - t = mod(i + o, n); - if (a[i] == -1 && a[t] == -1) - { - if (n - i >= o) - { - a[i] = t; - a[t] = i; - } - } - } - for (i = 0; i < n; i++) - { - if (a[i] == -1) - a[i] = i; - } -} - -#define ALIGNMENT (sysconf(_SC_PAGESIZE)) -static void* malloc_align(size_t size){ - void *p = NULL; - int ret = posix_memalign(&p, ALIGNMENT, size); - if (ret != 0){ - fprintf(stderr, "Failed to allocate memory on rank\n"); - exit(-1); - } - return p; -} \ No newline at end of file diff --git a/null_dummy.c b/null_dummy.c deleted file mode 100644 index 13a24a747..000000000 --- a/null_dummy.c +++ /dev/null @@ -1 +0,0 @@ -int main(int argc, char** argv){} diff --git a/ping-pong_b.c b/ping-pong_b.c deleted file mode 100644 index 629e70883..000000000 --- a/ping-pong_b.c +++ /dev/null @@ -1,230 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" - -int main(int argc, char** argv){ - /*init MPI world*/ - MPI_Init(&argc,&argv); - MPI_Comm_size(MPI_COMM_WORLD, &w_size); - MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); - - /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here - - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,k; - - /*read cmd line args*/ - for(i=1;i max_samples) - { - num_samples = max_samples; - start_index = curr_iters % max_samples; - } - else - { - num_samples = curr_iters - warm_up_iters; - start_index = warm_up_iters; - } - printf("Time,Bandwidth\n"); - for(i = 0; i < num_samples; i++){ - float time = durations[(start_index + i) % max_samples]/2; - float bandwidth = ((msg_size * 8.0) / 1000000000.0) / time; - printf("%.9f,%.9f\n", time, bandwidth); - } - printf("Ran %d iterations. Measured %d iterations.\n", curr_iters, num_samples); - fflush(stdout); - } - - - /*free allocated buffers*/ - free(durations); - free(send_buf); - free(recv_buf); - - /*exit MPI library*/ - MPI_Finalize(); -} - diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 000000000..da1964139 --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,34 @@ +# Auto-discover every .c and .cpp file under src/ and build one executable per file. +# CONFIGURE_DEPENDS re-triggers configure when files are added or removed. +# common.h is in this directory (src/) and is found via the include path below. +# All executable names are derived from the source filename stem, e.g. +# alltoall/alltoall_nb.c -> bin/alltoall_nb + +# The shared runtime (common.c) is compiled once into a static library and +# linked into every benchmark — cleaner and faster than re-instantiating a +# header of static-inline functions in each TU, and it yields a single .gcda so +# gcov coverage of the shared code is counted accurately. MPI/m propagate to +# consumers via PUBLIC. C++ benchmarks link it too (common.h uses extern "C"). +add_library(blink_common STATIC common.c) +target_include_directories(blink_common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(blink_common PUBLIC MPI::MPI_C m) + +file(GLOB_RECURSE c_srcs CONFIGURE_DEPENDS "*.c") +file(GLOB_RECURSE cpp_srcs CONFIGURE_DEPENDS "*.cpp") +# common.c is the library, not a benchmark — don't build an executable from it. +list(REMOVE_ITEM c_srcs "${CMAKE_CURRENT_SOURCE_DIR}/common.c") + +foreach(src IN LISTS c_srcs cpp_srcs) + get_filename_component(name ${src} NAME_WE) + get_filename_component(ext ${src} EXT) + + add_executable(${name} ${src}) + target_include_directories(${name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(${name} PRIVATE blink_common) + + if(ext STREQUAL ".c") + target_link_libraries(${name} PRIVATE MPI::MPI_C m) + else() + target_link_libraries(${name} PRIVATE MPI::MPI_CXX m) + endif() +endforeach() diff --git a/src/allgather/allgather_b.c b/src/allgather/allgather_b.c new file mode 100644 index 000000000..6f728bbcc --- /dev/null +++ b/src/allgather/allgather_b.c @@ -0,0 +1,131 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + size_t send_buf_size, recv_buf_size; + unsigned char *send_buf; + unsigned char *recv_buf; + + send_buf_size=msg_size; + recv_buf_size=(size_t)w_size*msg_size; + + send_buf=(unsigned char*)malloc_align(send_buf_size); + recv_buf=(unsigned char*)malloc_align(recv_buf_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time)= warm_up_iters) record_duration(measure_total_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -255,13 +196,25 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + if (debug_mode) { + size_t _r, _b; + int _ok = 1; + for (_r = 0; _r < (size_t)w_size && _ok; _r++) + for (_b = 0; _b < msg_size_ints && _ok; _b++) + if (recv_buf[_r * msg_size_ints + _b] != (int)_r) _ok = 0; + printf("DEBUG rank=%d nprocs=%d bytes=%zu check=%s\n", + my_rank, w_size, (size_t)send_buf_size, _ok ? "OK" : "FAIL"); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/src/allgather/allgather_nb.c b/src/allgather/allgather_nb.c new file mode 100644 index 000000000..f4e989918 --- /dev/null +++ b/src/allgather/allgather_nb.c @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + size_t send_buf_size, recv_buf_size; + unsigned char *send_buf; + unsigned char *recv_buf; + MPI_Request *requests; + + send_buf_size=msg_size; + /* one full gathered block per batched (granularity) call so the concurrently + * outstanding MPI_Iallgather ops never share a receive buffer (-grty > 1). */ + recv_buf_size=(size_t)measure_granularity*w_size*msg_size; + + send_buf=(unsigned char*)malloc_align(send_buf_size); + recv_buf=(unsigned char*)malloc_align(recv_buf_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*measure_granularity); + + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + int msg_size_ints; + int send_buf_size, recv_buf_size; + int *send_buf; + int *recv_buf; + + if(msg_size%sizeof(int)!=0){ + if(my_rank==master_rank) + fprintf(stderr, "Msg-size (%d) must be divisible by size of int (%zu)\n",msg_size,sizeof(int)); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + msg_size_ints=msg_size/sizeof(int); + send_buf_size=msg_size; + recv_buf_size=msg_size; + + send_buf=(int*)malloc_align(send_buf_size); + recv_buf=(int*)malloc_align(recv_buf_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + int msg_size_ints; + int send_buf_size; + int *send_buf; + int *recv_buf; + MPI_Request *requests; + + if(msg_size%sizeof(int)!=0){ + if(my_rank==master_rank) + fprintf(stderr, "Msg-size (%d) must be divisible by size of int (%zu)\n",msg_size,sizeof(int)); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + msg_size_ints=msg_size/sizeof(int); + send_buf_size=msg_size; + + send_buf=(int*)malloc_align(send_buf_size); + /* one reduced result per batched (granularity) call so the concurrently + * outstanding MPI_Iallreduce ops never share a receive buffer (-grty > 1). */ + recv_buf=(int*)malloc_align((size_t)measure_granularity*msg_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*measure_granularity); + + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + size_t send_buf_size, recv_buf_size; + unsigned char *send_buf; + unsigned char *recv_buf; + + send_buf_size=(size_t)msg_size*w_size; + recv_buf_size=(size_t)msg_size*w_size; + + send_buf=(unsigned char*)malloc_align(send_buf_size); + recv_buf=(unsigned char*)malloc_align(recv_buf_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + + /*fill send buffer with dummies*/ + for(size_t bi=0;bi= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + + + +void all2all_memcpy(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm){ + + int rank, size; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &size); + + int datatype_size; + MPI_Type_size(sendtype, &datatype_size); + + const char* sbuf = static_cast(sendbuf); + char* rbuf = static_cast(recvbuf); + + // Copy local data directly (self-send) + memcpy(rbuf + (size_t)rank * datatype_size * recvcount, + sbuf + (size_t)rank * datatype_size * sendcount, + (size_t)sendcount * datatype_size); + +} + +void custom_alltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) { + int rank, size; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &size); + + int datatype_size; + MPI_Type_size(sendtype, &datatype_size); + + const char* sbuf = static_cast(sendbuf); + char* rbuf = static_cast(recvbuf); + + std::vector requests; + for (int i = 0; i < size; ++i) { + if (i == rank) continue; + + MPI_Request req_recv; + MPI_Request req_send; + + MPI_Isend(sbuf + i * datatype_size * sendcount, sendcount, sendtype, i, 0, comm, &req_send); + MPI_Irecv(rbuf + i * datatype_size * recvcount, recvcount, recvtype, i, 0, comm, &req_recv); + + requests.push_back(req_send); + requests.push_back(req_recv); + } + + MPI_Waitall(static_cast(requests.size()), requests.data(), MPI_STATUSES_IGNORE); +} + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + size_t send_buf_size, recv_buf_size; + unsigned char *send_buf; + unsigned char *recv_buf; + + send_buf_size=(size_t)msg_size*w_size; + recv_buf_size=(size_t)msg_size*w_size; + + send_buf=(unsigned char*)malloc_align(send_buf_size); + recv_buf=(unsigned char*)malloc_align(recv_buf_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + if(send_buf==NULL){ + fprintf(stderr,"Failed to allocate send_buf on rank %d\n",my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } else if (recv_buf==NULL){ + fprintf(stderr,"Failed to allocate recv_buf on rank %d\n",my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } else if (durations==NULL){ + fprintf(stderr,"Failed to allocate durations buffer on rank %d\n",my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + /*fill send buffer with dummies*/ + for(size_t bi=0;bi= warm_up_iters) record_duration(measure_total_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) 1). */ + recv_buf_size=(size_t)measure_granularity*w_size*msg_size; send_buf=(unsigned char*)malloc_align(send_buf_size); recv_buf=(unsigned char*)malloc_align(recv_buf_size); @@ -115,24 +54,20 @@ int main(int argc, char** argv){ recv_requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*w_size*measure_granularity); durations=(double *)malloc_align(sizeof(double)*max_samples); - if(send_buf==NULL || recv_buf==NULL || durations==NULL || send_requests==NULL || recv_requests==NULL){ - fprintf(stderr,"Failed to allocate a buffer on rank %d\n",my_rank); - exit(-1); - } /*fill send buffer with dummies*/ - for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -178,22 +115,23 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); - - /* - int l; - printf("%d\n",my_rank); - for(l=0;l 1). */ + recv_buf_size=(size_t)measure_granularity*msg_size*w_size; send_buf=(unsigned char*)malloc_align(send_buf_size); recv_buf=(unsigned char*)malloc_align(recv_buf_size); durations=(double *)malloc_align(sizeof(double)*max_samples); requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*measure_granularity); - if(send_buf==NULL || recv_buf==NULL || requests==NULL || durations==NULL){ - fprintf(stderr,"Failed to allocate a buffer on rank %d\n",my_rank); - exit(-1); - } /*fill send buffer with dummies*/ - for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -167,13 +104,23 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + if (debug_mode) { + int _r, _b, _ok = 1; + for (_r = 0; _r < w_size && _ok; _r++) + for (_b = 0; _b < msg_size && _ok; _b++) + if (recv_buf[(size_t)_r * msg_size + _b] != (unsigned char)_r) _ok = 0; + printf("DEBUG rank=%d nprocs=%d check=%s\n", my_rank, w_size, _ok ? "OK" : "FAIL"); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/barrier.c b/src/barrier/barrier_b.c similarity index 51% rename from barrier.c rename to src/barrier/barrier_b.c index cc12750df..5589ed9fa 100644 --- a/barrier.c +++ b/src/barrier/barrier_b.c @@ -18,77 +18,18 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,k; - - /*read cmd line args*/ - for(i=1;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -145,13 +84,19 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + if (debug_mode) { + printf("DEBUG rank=%d nprocs=%d check=OK\n", my_rank, w_size); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/src/barrier/barrier_nb.c b/src/barrier/barrier_nb.c new file mode 100644 index 000000000..f342b3a96 --- /dev/null +++ b/src/barrier/barrier_nb.c @@ -0,0 +1,115 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + MPI_Request *requests; + + durations=(double *)malloc_align(sizeof(double)*max_samples); + requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*measure_granularity); + + + /*print basic info to stdout*/ + if(my_rank==master_rank){ + if(endless){ + printf("Barrier (non-blocking) with %d processes, test iterations: endless.\n" + ,w_size); + }else{ + printf("Barrier (non-blocking) with %d processes, test iterations: %d.\n" + ,w_size,max_iters); + } + } + + /*measured iterations*/ + double burst_start_time; + double measure_start_time; + double burst_length_mean=burst_length; + double burst_pause_mean=burst_pause; + int burst_cont=0; + curr_iters=0; + measured_iters=0; + + MPI_Barrier(MPI_COMM_WORLD); + do{ + for(k=0;k= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time)= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -163,13 +98,24 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + if (debug_mode) { + int _ok = 1; + unsigned char _fill = (unsigned char)master_rank; + for (size_t _b = 0; _b < buf_size && _ok; _b++) + if (buf[_b] != _fill) _ok = 0; + printf("DEBUG rank=%d nprocs=%d root=%d check=%s\n", + my_rank, w_size, master_rank, _ok ? "OK" : "FAIL"); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/bdc_nb.c b/src/broadcast/broadcast_nb.c similarity index 53% rename from bdc_nb.c rename to src/broadcast/broadcast_nb.c index b5f1c51e6..7da950346 100644 --- a/bdc_nb.c +++ b/src/broadcast/broadcast_nb.c @@ -18,81 +18,18 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,k; - - /*read cmd line args*/ - for(i=1;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -166,13 +101,24 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + if (debug_mode) { + int _ok = 1; + unsigned char _fill = (unsigned char)master_rank; + for (size_t _b = 0; _b < buf_size && _ok; _b++) + if (buf[_b] != _fill) _ok = 0; + printf("DEBUG rank=%d nprocs=%d root=%d check=%s\n", + my_rank, w_size, master_rank, _ok ? "OK" : "FAIL"); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/src/common.c b/src/common.c new file mode 100644 index 000000000..908ae4c7d --- /dev/null +++ b/src/common.c @@ -0,0 +1,816 @@ +/* + * common.c — definitions of the shared Blink runtime (declared in common.h). + * + * Compiled once into the `blink_common` library and linked into every + * benchmark, rather than being a header full of `static inline` functions + * instantiated in each of the ~40 translation units. Besides being cleaner and + * faster to build, this gives accurate gcov coverage: a single .gcda for the + * shared code instead of per-TU copies that gcov cannot merge. + */ +#include "common.h" + +/* ── globals ─────────────────────────────────────────────────────────────── */ +int my_rank; +int w_size; +int master_rank = 0; +long long curr_iters; /* total outer iterations executed (warmup + measured) */ +long long measured_iters; /* number of recorded samples (excludes warmup) */ +int warm_up_iters = 5; +int max_samples = 1000; +double *durations; + +/* Async-signal-safe shutdown flag set by sig_handler; checked in measurement + * loops. The signal handler does no MPI / no malloc / no stdio — only sets + * this flag. The main thread observes it between iterations and exits the + * loop cleanly, then calls write_results() + MPI_Finalize() itself. */ +volatile sig_atomic_t shutdown_requested = 0; + +/* common benchmark parameters – initialised to defaults, set by parse_common_args() */ +int msg_size = 1024; +int measure_granularity = 1; +int rand_seed = 1; +int max_iters = 1; +int endless = 0; +double burst_length = 0.0; +int burst_length_rand = 0; /* set by -bldist */ +double burst_pause = 0.0; +int burst_pause_rand = 0; /* set by -bpdist */ +int pretty_output = 0; +int debug_mode = 0; + +/* runtime-distribution plot (-plot). */ +int plot_output = 0; +char plot_stat[16] = "max"; /* avg | min | max | median | mainrank */ +int plot_bins = 10; +double plot_bin_size = 0.0; /* 0 => use plot_bins */ +int plot_log = 0; + +/* distribution selection for burst length and pause (-bldist / -bpdist). */ +char burst_dist[16] = "exp"; +double burst_shape = 1.5; +char pause_dist[16] = "exp"; +double pause_shape = 1.5; + +/* Weak default — benchmarks that take their own flags override this with a + * strong definition at file scope (a one-liner `const char *benchmark_help = + * "..."` in the .c file). Benchmarks with no custom flags need do nothing. */ +__attribute__((weak)) const char *benchmark_help = NULL; + +#ifndef BLINK_PLOT_MAX_BINS +#define BLINK_PLOT_MAX_BINS 64 +#endif +#define BLINK_PLOT_BAR_MAX 40 +#define ALIGNMENT (sysconf(_SC_PAGESIZE)) + +/* ── random duration samplers ────────────────────────────────────────────── */ + +/* Exponential: shape parameter accepted for API uniformity but not used — + * the distribution is fully determined by its mean. */ +double rand_exp(double mean, double shape) +{ + (void)shape; + double u = (rand() + 0.5) / (RAND_MAX + 1.0); /* shift away from 0 and 1 */ + return -mean * log(1.0 - u); +} + +/* Pareto: shape = tail exponent α. Must be > 1 for a finite mean. + * Parameterised so that E[X] = mean regardless of α. + * Inverse-CDF method: x = x_m * u^(-1/α), u ~ Uniform(0,1). */ +double rand_pareto(double mean, double shape) +{ + double alpha = shape; + if (alpha <= 1.0) { + fprintf(stderr, "rand_pareto: shape (alpha) must be > 1, got %g\n", alpha); + exit(-1); + } + if (mean <= 0.0) { + fprintf(stderr, "rand_pareto: mean must be > 0, got %g\n", mean); + exit(-1); + } + double x_m = mean * (alpha - 1.0) / alpha; + double u = (rand() + 0.5) / (RAND_MAX + 1.0); /* shift away from 0 */ + return x_m * pow(u, -1.0 / alpha); +} + +/* Log-normal: shape = σ of the underlying normal. + * Parameterised so that E[X] = mean regardless of σ. + * Box-Muller transform. */ +double rand_lognormal(double mean, double sigma) +{ + if (mean <= 0.0) { + fprintf(stderr, "rand_lognormal: mean must be > 0, got %g\n", mean); + exit(-1); + } + if (sigma <= 0.0) { + fprintf(stderr, "rand_lognormal: sigma must be > 0, got %g\n", sigma); + exit(-1); + } + double mu = log(mean) - 0.5 * sigma * sigma; + double u1 = (rand() + 0.5) / (RAND_MAX + 1.0); /* shift away from 0 */ + double u2 = (rand() + 0.5) / (RAND_MAX + 1.0); + double z = sqrt(-2.0 * log(u1)) * cos(2.0 * M_PI * u2); + return exp(mu + sigma * z); +} + +/* Dispatch to the selected sampler. */ +double rand_duration(double mean, const char *dist, double shape) +{ + if (strcmp(dist, "pareto") == 0) return rand_pareto(mean, shape); + if (strcmp(dist, "lognormal") == 0) return rand_lognormal(mean, shape); + return rand_exp(mean, shape); +} + +/*sleep seconds given as double*/ +int dsleep(double t) +{ + struct timespec t1, t2; + t1.tv_sec = (long)t; + t1.tv_nsec = (t - t1.tv_sec) * 1000000000L; + return nanosleep(&t1, &t2); +} + +/*double comparison function for quicksort*/ +int compare_doubles(const void *p1, const void *p2) +{ + if (*(double *)p1 < *(double *)p2) + return -1; + else if (*(double *)p1 > *(double *)p2) + return 1; + else + return 0; +} + +/* Bounds-checked accessor for a flag's value argument. Aborts with a clear + * message instead of dereferencing argv[argc] (== NULL) when a value-taking + * flag is supplied as the final command-line token. */ +const char *arg_value(int argc, char **argv, int *i) +{ + if (*i + 1 >= argc) { + if (my_rank == master_rank) + fprintf(stderr, "Missing value for option %s\n", argv[*i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + return argv[++(*i)]; +} + +/* Validate a distribution name / shape pair selected via -bldist/-bpdist so a + * misconfiguration fails loudly at startup rather than silently producing + * garbage durations later (e.g. log-normal with a non-positive mean). */ +void validate_burst_dist(const char *what, const char *dist, + double mean, double shape) +{ + if (mean <= 0.0) { + if (my_rank == master_rank) + fprintf(stderr, "%s: randomised distribution requires a positive mean, got %g\n", + what, mean); + MPI_Abort(MPI_COMM_WORLD, -1); + } + if (strcmp(dist, "pareto") == 0 && shape <= 1.0) { + if (my_rank == master_rank) + fprintf(stderr, "%s: pareto shape (alpha) must be > 1, got %g\n", what, shape); + MPI_Abort(MPI_COMM_WORLD, -1); + } + if (strcmp(dist, "lognormal") == 0 && shape <= 0.0) { + if (my_rank == master_rank) + fprintf(stderr, "%s: lognormal shape (sigma) must be > 0, got %g\n", what, shape); + MPI_Abort(MPI_COMM_WORLD, -1); + } +} + +/* Print the help block: common options + benchmark_help (if the benchmark + * provides one). Only the master rank prints, to avoid w_size duplicated + * copies on stdout. Callers handle MPI_Finalize + exit afterwards. */ +void print_help(const char *progname) +{ + if (my_rank != 0) return; + const char *base = progname ? progname : "benchmark"; + const char *slash = strrchr(base, '/'); + if (slash) base = slash + 1; + fprintf(stdout, +"Usage: %s [options]\n" +"\n" +"Common options:\n" +" -mrank master / root rank (default 0)\n" +" -mrand choose master rank at random (uses -seed)\n" +" -msgsize message size in bytes (default 1024)\n" +" -iter number of measured iterations (default 1)\n" +" -warmup number of warm-up iterations (default 5)\n" +" -endl run endlessly until SIGUSR1\n" +" -seed RNG seed shared across ranks (default 1)\n" +" -grty ops per timed window (granularity, default 1)\n" +" -maxsamples cap on per-rank samples stored (default 1000)\n" +" -debug print per-rank DEBUG lines instead of timings\n" +" -pretty-print human-readable output (default: CSV row)\n" +"\n" +"Burst / pause (idle gaps between bursts of communication):\n" +" -blength burst length (default 0 = no bursting)\n" +" -bpause pause length between bursts (default 0)\n" +" -bldist randomise burst length (default: fixed)\n" +" -bpdist randomise pause length (default: fixed)\n" +" -blshape shape param for burst dist (pareto alpha / lognormal sigma)\n" +" -bpshape shape param for pause dist\n" +" (durations: bare number = seconds; suffixes 's', 'ms', 'us', 'ns' accepted)\n" +"\n" +"Per-iteration latency histogram (rank 0 only, end of run):\n" +" -plot emit ASCII histogram of per-iteration latencies\n" +" -plotstat cross-rank statistic (default: max)\n" +" -plotbins number of bins (default 10, max 64)\n" +" -plotbinsize bin width (e.g. 2ms, 500us, 0.001) — overrides -plotbins\n" +" -plotlog logarithmic bar heights\n" +"\n" +" -h, --help show this help and exit\n", + base); + + if (benchmark_help && *benchmark_help) { + fprintf(stdout, "\nBenchmark-specific options:\n%s", benchmark_help); + } + fflush(stdout); +} + +/* Wrap parse_duration with diagnostics + validation for CLI args that accept + * a duration. A bare number is seconds; suffixes s/ms/us/ns are recognised. + * Aborts with a clear message on malformed input or negative value (0 is OK + * because -blength 0 / -bpause 0 mean "no bursting"). */ +static double parse_duration_arg(const char *flag, const char *s) +{ + double v = parse_duration(s); + if (isnan(v) || v < 0.0) { + if (my_rank == master_rank) + fprintf(stderr, "%s must be a non-negative duration " + "(e.g. 0.001, 1ms, 500us, 100ns), got '%s'\n", flag, s); + MPI_Abort(MPI_COMM_WORLD, -1); + } + return v; +} + +/* Parse a duration: a number with an optional unit suffix (s, ms, us, ns). + * A bare number is interpreted as seconds (consistent with -blength/-bpause). + * Returns the value in seconds, or NAN on a malformed string / unknown unit. */ +double parse_duration(const char *s) +{ + char *end = NULL; + double v = strtod(s, &end); + if (end == s) return NAN; /* no leading number */ + while (*end == ' ') end++; + if (*end == '\0') return v; /* bare number => seconds */ + if (strcmp(end, "s") == 0) return v; + if (strcmp(end, "ms") == 0) return v * 1e-3; + if (strcmp(end, "us") == 0) return v * 1e-6; + if (strcmp(end, "ns") == 0) return v * 1e-9; + return NAN; /* unknown unit */ +} + +/* + * Parse the standard set of command-line flags shared by every benchmark. + * Unrecognised flags are compacted to the front of argv (after argv[0]) and + * the new argc is returned, so each benchmark can do a second pass for its + * own flags. Also seeds the RNG and resolves -mrand after parsing. + */ +int parse_common_args(int argc, char **argv) +{ + int new_argc = 1; /* always keep argv[0] (program name) */ + int do_rand_master = 0; + int i; + + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 + || strcmp(argv[i], "-help") == 0 + || strcmp(argv[i], "--help") == 0) { + print_help(argv[0]); + MPI_Finalize(); + exit(0); + } + else if (strcmp(argv[i], "-mrank") == 0) { master_rank = atoi(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-mrand") == 0) { do_rand_master = 1; } + else if (strcmp(argv[i], "-msgsize") == 0) { msg_size = atoi(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-endl") == 0) { endless = 1; } + else if (strcmp(argv[i], "-iter") == 0) { max_iters = atoi(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-warmup") == 0) { warm_up_iters = atoi(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-blength") == 0) { burst_length = parse_duration_arg("-blength", arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-bpause") == 0) { burst_pause = parse_duration_arg("-bpause", arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-bldist") == 0) { strncpy(burst_dist, arg_value(argc, argv, &i), 15); + burst_dist[15] = '\0'; + burst_length_rand = 1; } + else if (strcmp(argv[i], "-bpdist") == 0) { strncpy(pause_dist, arg_value(argc, argv, &i), 15); + pause_dist[15] = '\0'; + burst_pause_rand = 1; } + else if (strcmp(argv[i], "-blshape") == 0) { burst_shape = atof(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-bpshape") == 0) { pause_shape = atof(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-seed") == 0) { rand_seed = atoi(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-grty") == 0) { measure_granularity = atoi(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-maxsamples") == 0) { max_samples = atoi(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-pretty-print") == 0) { pretty_output = 1; } + else if (strcmp(argv[i], "-debug") == 0) { debug_mode = 1; } + else if (strcmp(argv[i], "-plot") == 0) { plot_output = 1; } + else if (strcmp(argv[i], "-plotstat") == 0) { strncpy(plot_stat, arg_value(argc, argv, &i), 15); + plot_stat[15] = '\0'; } + else if (strcmp(argv[i], "-plotbins") == 0) { plot_bins = atoi(arg_value(argc, argv, &i)); } + else if (strcmp(argv[i], "-plotlog") == 0) { plot_log = 1; } + else if (strcmp(argv[i], "-plotbinsize") == 0) { const char *_bs = arg_value(argc, argv, &i); + plot_bin_size = parse_duration(_bs); + if (isnan(plot_bin_size) || plot_bin_size <= 0.0) { + if (my_rank == master_rank) + fprintf(stderr, "-plotbinsize must be a positive duration " + "(e.g. 2ms, 500us, 0.001), got '%s'\n", _bs); + MPI_Abort(MPI_COMM_WORLD, -1); + } } + else { argv[new_argc++] = argv[i]; } /* pass through unknown flags */ + } + + /* fail fast on nonsensical numeric parameters */ + if (max_samples < 1) { + if (my_rank == master_rank) + fprintf(stderr, "-maxsamples must be >= 1, got %d\n", max_samples); + MPI_Abort(MPI_COMM_WORLD, -1); + } + if (measure_granularity < 1) { + if (my_rank == master_rank) + fprintf(stderr, "-grty must be >= 1, got %d\n", measure_granularity); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + /* a randomised burst/pause distribution needs a positive mean and a valid + * shape — validate now so we never feed log(0)/negative values to a sampler */ + if (burst_length_rand) + validate_burst_dist("-bldist", burst_dist, burst_length, burst_shape); + if (burst_pause_rand) + validate_burst_dist("-bpdist", pause_dist, burst_pause, pause_shape); + + /* seed RNG (shared across all ranks) and resolve randomised master */ + srand(rand_seed); + if (do_rand_master) + master_rank = rand() % w_size; + + /* validate the resolved master rank is a real rank. An out-of-range -mrank + * would otherwise become an invalid root in MPI_Gather/Bcast/Reduce, and can + * deadlock benchmarks that branch on (my_rank == master_rank). Gate the + * message on rank 0 because master_rank itself may be the bad value. */ + if (master_rank < 0 || master_rank >= w_size) { + if (my_rank == 0) + fprintf(stderr, "-mrank must be in [0, %d), got %d\n", w_size, master_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + /* validate -plot options */ + if (plot_output) { + if (strcmp(plot_stat, "avg") && strcmp(plot_stat, "min") && strcmp(plot_stat, "max") + && strcmp(plot_stat, "median") && strcmp(plot_stat, "mainrank")) { + if (my_rank == master_rank) + fprintf(stderr, "-plotstat must be one of avg|min|max|median|mainrank, got %s\n", plot_stat); + MPI_Abort(MPI_COMM_WORLD, -1); + } + if (plot_bin_size > 0.0 && plot_log) { + if (my_rank == master_rank) + fprintf(stderr, "-plotbinsize cannot be combined with -plotlog " + "(a constant width is meaningless on a log axis)\n"); + MPI_Abort(MPI_COMM_WORLD, -1); + } + if (plot_bin_size <= 0.0 && plot_bins < 1) { + if (my_rank == master_rank) + fprintf(stderr, "-plotbins must be >= 1, got %d\n", plot_bins); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + return new_argc; +} + +/* Convenience wrappers used by benchmark measurement loops. */ +double sample_burst_length(double mean) { return rand_duration(mean, burst_dist, burst_shape); } +double sample_pause_length(double mean) { return rand_duration(mean, pause_dist, pause_shape); } + +/* Record a single measured-iteration latency into the ring buffer. + * Skips writes during warm-up: callers gate this by `if (k >= warm_up_iters)` + * at the outer iteration level — see the standard benchmark loop. + * + * The ring buffer of size `max_samples` therefore only ever holds measured + * samples (no warmup pollution); the (oldest → newest) ordering is + * (measured_iters % max_samples) ... (measured_iters - 1) % max_samples. */ +void record_duration(double t) +{ + durations[measured_iters % max_samples] = t; + measured_iters++; +} + +/* ── output helpers ──────────────────────────────────────────────────────── */ + +/*format a duration (seconds) with auto-scaled units; "%9.2f UU" is normally + * 12 chars wide (9 is a minimum field width, so extreme outliers can exceed it)*/ +void format_duration(char *buf, size_t len, double t) +{ + if (t < 1e-3) + snprintf(buf, len, "%9.2f us", t * 1e6); + else if (t < 1.0) + snprintf(buf, len, "%9.2f ms", t * 1e3); + else + snprintf(buf, len, "%9.2f s", t); +} + +/* Strip the leading spaces format_duration adds (it right-pads to %9.2f). */ +const char *ltrim_spaces(const char *s) +{ + while (*s == ' ') s++; + return s; +} + +/* Print one histogram row: an aligned [lo - hi] range label, a bar scaled to + * the tallest bin (`maxc`), the empirical percentage, and — when `theory` >= 0 + * — a theoretical percentage alongside it (dist_test's CDF overlay). An + * infinite `hi` is rendered as "inf" for an open-ended overflow bin. Shared by + * the -plot histogram and dist_test's sampler-verification histogram. */ +void print_histogram_row(double lo, double hi, int count, int maxc, + int n, double theory) +{ + char lo_s[24], hi_s[24]; + int k, bar; + format_duration(lo_s, sizeof(lo_s), lo); + if (isinf(hi)) snprintf(hi_s, sizeof(hi_s), " %8s", "inf"); + else format_duration(hi_s, sizeof(hi_s), hi); + bar = (maxc > 0) ? (int)floor((double)count / maxc * BLINK_PLOT_BAR_MAX + 0.5) : 0; + printf(" [%s - %s] ", lo_s, hi_s); + for (k = 0; k < bar; k++) printf("\xe2\x96\x88"); /* full block */ + for (k = bar; k < BLINK_PLOT_BAR_MAX; k++) printf(" "); + if (theory >= 0.0) printf(" %5.1f%% (theory %5.1f%%)\n", 100.0 * count / n, 100.0 * theory); + else printf(" %5.1f%%\n", 100.0 * count / n); +} + +/* Render an ASCII histogram of `n` per-iteration timing samples (seconds), + * followed by a percentile footer. Binning modes: + * logscale : `bins` geometric bins between min and max (needs min > 0); + * bin_size > 0 : fixed-width linear bins, lower edge aligned to a multiple of + * bin_size, capped at BLINK_PLOT_MAX_BINS with the tail folded + * into the last bin (a note is printed when this triggers); + * otherwise : `bins` equal-width linear bins between min and max. + * `vals` is sorted in place — pass a scratch array you no longer need. */ +void print_runtime_histogram(double *vals, int n, const char *statname, + int bins, double bin_size, int logscale) +{ + int i; + printf("\n"); + if (n <= 0) { printf(" (-plot: no measured samples to plot)\n"); return; } + + qsort(vals, n, sizeof(double), compare_doubles); + double vmin = vals[0], vmax = vals[n - 1]; + + double sum = 0.0; + for (i = 0; i < n; i++) sum += vals[i]; + + if (vmax <= vmin) { /* degenerate: all identical */ + char one[24]; + format_duration(one, sizeof(one), vmin); + printf("\033[1m\033[36m Per-iteration latency \xc2\xb7 stat=%s \xc2\xb7 n=%d\033[0m\n", + statname, n); + printf(" all samples = %s\n", ltrim_spaces(one)); + return; + } + + int use_log = (logscale && vmin > 0.0); + int nb; + int overflow = 0; + double lo0 = vmin, width = 0.0, ratio = 1.0; + + if (use_log) { + nb = bins > 0 ? bins : 10; + ratio = pow(vmax / vmin, 1.0 / nb); + } else if (bin_size > 0.0) { + lo0 = floor(vmin / bin_size) * bin_size; + long need = (long)floor((vmax - lo0) / bin_size) + 1; + if (need < 1) need = 1; + if (need > BLINK_PLOT_MAX_BINS) { nb = BLINK_PLOT_MAX_BINS; overflow = 1; } + else { nb = (int)need; } + width = bin_size; + } else { + nb = bins > 0 ? bins : 10; + width = (vmax - vmin) / nb; + } + if (nb < 1) nb = 1; + + int *counts = (int *)calloc((size_t)nb, sizeof(int)); + if (!counts) { printf(" (-plot: allocation failed)\n"); return; } + for (i = 0; i < n; i++) { + int b = use_log ? (int)floor(log(vals[i] / vmin) / log(ratio)) + : (int)floor((vals[i] - lo0) / width); + if (b < 0) b = 0; + if (b >= nb) b = nb - 1; /* fold any overflow into the last bin */ + counts[b]++; + } + int maxc = 1; + for (i = 0; i < nb; i++) if (counts[i] > maxc) maxc = counts[i]; + + printf("\033[1m\033[36m Per-iteration latency \xc2\xb7 stat=%s \xc2\xb7 n=%d \xc2\xb7 %s\033[0m\n", + statname, n, use_log ? "log bins" : (bin_size > 0.0 ? "fixed bins" : "linear bins")); + if (overflow) + printf("\033[33m note: -plotbinsize would need %ld bins for the observed range; " + "capped at %d (tail folded into the last bin)\033[0m\n", + (long)floor((vmax - lo0) / bin_size) + 1, BLINK_PLOT_MAX_BINS); + printf("\033[2m ----------------------------------------------------------------------\033[0m\n"); + + for (i = 0; i < nb; i++) { + double lo = use_log ? vmin * pow(ratio, i) : lo0 + i * width; + double hi = use_log ? vmin * pow(ratio, i + 1) : lo0 + (i + 1) * width; + print_histogram_row(lo, hi, counts[i], maxc, n, -1.0); /* -1 => no theory overlay */ + } + printf("\033[2m ----------------------------------------------------------------------\033[0m\n"); + + char s_min[24], s_mean[24], s_p50[24], s_p90[24], s_p99[24], s_max[24]; + format_duration(s_min, sizeof(s_min), vmin); + format_duration(s_mean, sizeof(s_mean), sum / n); + format_duration(s_p50, sizeof(s_p50), vals[(int)(0.50 * (n - 1) + 0.5)]); + format_duration(s_p90, sizeof(s_p90), vals[(int)(0.90 * (n - 1) + 0.5)]); + format_duration(s_p99, sizeof(s_p99), vals[(int)(0.99 * (n - 1) + 0.5)]); + format_duration(s_max, sizeof(s_max), vmax); + printf(" n=%d \xc2\xb7 min %s \xc2\xb7 mean %s \xc2\xb7 p50 %s \xc2\xb7 p90 %s \xc2\xb7 p99 %s \xc2\xb7 max %s\n", + n, ltrim_spaces(s_min), ltrim_spaces(s_mean), ltrim_spaces(s_p50), + ltrim_spaces(s_p90), ltrim_spaces(s_p99), ltrim_spaces(s_max)); + free(counts); +} + +void write_results(void) +{ + double duration_sum; + double duration_median; + int num_samples; + int i; + int start_index; + double *tmp_buf = NULL; + + if (measured_iters > max_samples) /* wrapped the sampling ring buffer */ + { + num_samples = max_samples; + start_index = (int)(measured_iters % max_samples); + } + else + { + num_samples = (int)measured_iters; + start_index = 0; + } + + /* MPI_Gather below uses num_samples as BOTH send- and recv-count, which is + * only valid if every rank contributes the same count. Ranks normally stay + * in lockstep, but to be robust (e.g. a SIGUSR1 shutdown that reaches ranks + * at slightly different iterations) collectively agree on the common minimum + * so the gather can never mismatch. */ + { + int common_samples = num_samples; + MPI_Allreduce(&num_samples, &common_samples, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + /* keep only the most recent common_samples (drop the oldest extras) */ + start_index += (num_samples - common_samples); + num_samples = common_samples; + } + + tmp_buf = (double *)malloc(sizeof(double) * (num_samples > 0 ? num_samples : 1)); + /* copy in chronological order (oldest → newest) using circular indexing */ + for (i = 0; i < num_samples; i++) { + tmp_buf[i] = durations[(start_index + i) % max_samples]; + } + + double *all_data = (double *)malloc(sizeof(double)*(num_samples > 0 ? num_samples : 1)*w_size); + double *sorting_buf = (double *)malloc(sizeof(double)*w_size); + + if (all_data == NULL || sorting_buf == NULL) { + fprintf(stderr, "Failed to allocate a buffer on rank %d\n", my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + /* print header before the gather so it appears first */ + if (my_rank == master_rank) { + if (pretty_output) { + printf("\033[1m\033[36m" + " %5s %12s %12s %12s %12s" + "\033[0m\n", + "#", "avg", "min", "max", "median"); + printf("\033[2m" + " -------------------------------------------------------------" + "\033[0m\n"); + } else { + printf("Average,Minimum,Maximum,Median,MainRank\n"); + } + } + + MPI_Gather(tmp_buf, num_samples, MPI_DOUBLE, + all_data, num_samples, MPI_DOUBLE, + master_rank, MPI_COMM_WORLD); + + if (my_rank == master_rank) { + /* -plot: capture one chosen cross-rank statistic per iteration */ + int plot_stat_id = 0; /* 0=avg 1=min 2=max 3=median 4=mainrank */ + double *plot_vals = NULL; + if (plot_output) { + if (strcmp(plot_stat, "min") == 0) plot_stat_id = 1; + else if (strcmp(plot_stat, "max") == 0) plot_stat_id = 2; + else if (strcmp(plot_stat, "median") == 0) plot_stat_id = 3; + else if (strcmp(plot_stat, "mainrank") == 0) plot_stat_id = 4; + plot_vals = (double *)malloc(sizeof(double) * (num_samples > 0 ? num_samples : 1)); + if (plot_vals == NULL) { + fprintf(stderr, "Failed to allocate plot buffer on rank %d\n", my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + for (i = 0; i < num_samples; i++) { + int j; + duration_sum = 0; + for (j = 0; j < w_size; j++) { + sorting_buf[j] = all_data[j*num_samples + i]; + duration_sum += sorting_buf[j]; + } + + qsort(sorting_buf, w_size, sizeof(double), compare_doubles); + if (w_size % 2 == 0) /* even: median as mean of two middle values */ + duration_median = (sorting_buf[(w_size-1)/2] + sorting_buf[w_size/2]) / 2.0; + else /* odd: median is the middle value */ + duration_median = sorting_buf[(w_size-1)/2]; + + if (plot_output) { + switch (plot_stat_id) { + case 1: plot_vals[i] = sorting_buf[0]; break; + case 2: plot_vals[i] = sorting_buf[w_size - 1]; break; + case 3: plot_vals[i] = duration_median; break; + case 4: plot_vals[i] = tmp_buf[i]; break; + default: plot_vals[i] = duration_sum / w_size; break; + } + } + + if (pretty_output) { + char avg_s[24], min_s[24], max_s[24], med_s[24]; + format_duration(avg_s, sizeof(avg_s), duration_sum / w_size); + format_duration(min_s, sizeof(min_s), sorting_buf[0]); + format_duration(max_s, sizeof(max_s), sorting_buf[w_size-1]); + format_duration(med_s, sizeof(med_s), duration_median); + printf(" %5d %12s %12s %12s %12s\n", + i+1, avg_s, min_s, max_s, med_s); + } else { + printf("%.9f,%.9f,%.9f,%.9f,%.9f\n", + duration_sum / w_size, + sorting_buf[0], sorting_buf[w_size-1], + duration_median, tmp_buf[i]); + } + } + + if (pretty_output) { + printf("\033[2m" + " -------------------------------------------------------------" + "\033[0m\n"); + printf(" %d samples \xc2\xb7 %lld iterations total\n", num_samples, curr_iters); + } else { + printf("Ran %lld iterations. Measured %d iterations.\n", curr_iters, num_samples); + } + + /* -plot: additive ASCII histogram of the chosen per-iteration statistic */ + if (plot_output) { + print_runtime_histogram(plot_vals, num_samples, plot_stat, + plot_bins, plot_bin_size, plot_log); + free(plot_vals); + } + + fflush(stdout); + } + + free(sorting_buf); + free(all_data); + free(tmp_buf); +} + +/* Async-signal-safe handler: only sets a flag. The measurement loop in each + * benchmark observes the flag and exits cleanly; write_results() and + * MPI_Finalize() happen on the main thread, never from signal context. */ +void sig_handler(int sig) +{ + (void)sig; + shutdown_requested = 1; +} + +/* Install the SIGUSR1 shutdown handler with sigaction() rather than signal(), + * for portable, persistent (non-one-shot) semantics across platforms. */ +void install_shutdown_handler(void) +{ + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = sig_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + sigaction(SIGUSR1, &sa, NULL); +} + +/* Collectively decide whether to shut down. SIGUSR1 only sets the flag on the + * rank that received it, so a purely local check would make ranks leave the + * measurement loop at different iterations — desynchronising the per-iteration + * collectives and the final MPI_Gather (deadlock / mismatch). This all-reduce + * (called once per outer iteration, OUTSIDE the timed region) guarantees every + * rank breaks on the same iteration. Returns non-zero if any rank requested + * shutdown. */ +int check_shutdown(void) +{ + int local = (int)shutdown_requested; + int global = 0; + MPI_Allreduce(&local, &global, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); + return global; +} + +/* ── combinatorics helpers ───────────────────────────────────────────────── */ + +/*use Fisher-Yates to permute array*/ +void permute(int *a, int n) +{ + int j, t, i; + for (i = n; i > 1; i--) + { + j = rand() % i; + t = a[i - 1]; + a[i - 1] = a[j]; + a[j] = t; + } +} + +/*mathematical mod without negative numbers*/ +int mod(int a, int b) +{ + int c = a % b; + if (c < 0) + c += b; + return c; +} + +/* Produce a uniformly random matching of [0..n-1] (each entry pairs with + * exactly one other entry). Algorithm: shuffle the index list, then pair + * adjacent elements. This is provably uniform over the set of perfect + * matchings (and far simpler than the prior slot-counting version). + * + * For odd n the last element targets itself (it has no partner). + */ +void random_pairs(int *a, int n) +{ + int i; + int has_self = (n % 2 == 1); + int pair_n = has_self ? n - 1 : n; + int *shuf = (int *)malloc(sizeof(int) * n); + + if (shuf == NULL) { + fprintf(stderr, "random_pairs: malloc failed\n"); + MPI_Abort(MPI_COMM_WORLD, -1); + } + for (i = 0; i < n; i++) shuf[i] = i; + permute(shuf, n); + + for (i = 0; i < n; i++) a[i] = -1; + + for (i = 0; i < pair_n; i += 2) { + int x = shuf[i]; + int y = shuf[i + 1]; + a[x] = y; + a[y] = x; + } + if (has_self) { + /* odd n: one rank has no partner — target itself */ + int lone = shuf[n - 1]; + a[lone] = lone; + } + free(shuf); +} + +/* Produce fixed-offset pairs: walks i from 0 upward and pairs i with (i+o) + * whenever both ranks are still free and the partner is not a wrap-around + * (n - i >= o). This guarantees disjoint pairs (i, i+o), (i+2o, i+3o), … + * — for offset o=1 you get (0,1) (2,3) (4,5)…, for o=2 you get (0,2) + * (1,3) (4,6) (5,7)…, etc. + * + * Only non-wrapping reciprocal pairs (i, i+o) are formed; any rank that cannot + * be matched without wrapping around falls through to a self-pair (a[i] = i, + * i.e. it performs no communication). This happens for every offset > n/2, and + * also for many offsets <= n/2 when n is not a multiple of 2*o (e.g. n=12, o=4 + * leaves ranks 8..11 self-paired). Full disjoint tiling is only guaranteed when + * 2*o divides n (notably o=1). Recommended usage: 1 <= o <= n/2. + */ +void offset_pairs(int *a, int n, int o) +{ + int i, t; + for (i = 0; i < n; i++) + a[i] = -1; + for (i = 0; i < n; i++) + { + t = mod(i + o, n); + if (a[i] == -1 && a[t] == -1) + { + if (n - i >= o) + { + a[i] = t; + a[t] = i; + } + } + } + for (i = 0; i < n; i++) + { + if (a[i] == -1) + a[i] = i; /* no reciprocal partner — self */ + } +} + +void* malloc_align(size_t size) +{ + void *p = NULL; + int ret = posix_memalign(&p, ALIGNMENT, size); + if (ret != 0) { + fprintf(stderr, "Failed to allocate memory on rank\n"); + MPI_Abort(MPI_COMM_WORLD, -1); + } + return p; +} diff --git a/src/common.h b/src/common.h new file mode 100644 index 000000000..b039de690 --- /dev/null +++ b/src/common.h @@ -0,0 +1,99 @@ +#ifndef BLINK_COMMON_H +#define BLINK_COMMON_H + +/* Shared Blink runtime: declarations only. Definitions live in common.c, which + * is compiled once into the `blink_common` library and linked into every + * benchmark. (Previously this was a header full of `static inline` functions + * instantiated per translation unit — cleaner as a single compilation unit, and + * it gives accurate gcov coverage instead of per-TU copies gcov can't merge.) */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { /* common.c is C; let the C++ (comm_only) benchmarks link to it */ +#endif + +/* ── globals (defined in common.c) ───────────────────────────────────────── */ +extern int my_rank; +extern int w_size; +extern int master_rank; +extern long long curr_iters; /* total outer iterations (warmup + measured) */ +extern long long measured_iters; /* recorded samples (excludes warmup) */ +extern int warm_up_iters; +extern int max_samples; +extern double *durations; +extern volatile sig_atomic_t shutdown_requested; + +extern int msg_size; +extern int measure_granularity; +extern int rand_seed; +extern int max_iters; +extern int endless; +extern double burst_length; +extern int burst_length_rand; /* set by -bldist */ +extern double burst_pause; +extern int burst_pause_rand; /* set by -bpdist */ +extern int pretty_output; +extern int debug_mode; + +extern int plot_output; /* -plot */ +extern char plot_stat[16]; /* avg | min | max | median | mainrank */ +extern int plot_bins; +extern double plot_bin_size; /* 0 => use plot_bins */ +extern int plot_log; + +extern char burst_dist[16]; /* "exp" | "pareto" | "lognormal" */ +extern double burst_shape; +extern char pause_dist[16]; +extern double pause_shape; + +/* Benchmark-specific help text, printed after the common block by -h/--help. + * Weakly defined as NULL in common.c; benchmarks that take their own flags + * provide a strong definition at file scope (see e.g. src/pairwise/pairwise_b.c). + */ +extern const char *benchmark_help; + +/* ── functions (defined in common.c) ─────────────────────────────────────── */ +double rand_exp(double mean, double shape); +double rand_pareto(double mean, double shape); +double rand_lognormal(double mean, double sigma); +double rand_duration(double mean, const char *dist, double shape); +int dsleep(double t); +int compare_doubles(const void *p1, const void *p2); +const char *arg_value(int argc, char **argv, int *i); +void validate_burst_dist(const char *what, const char *dist, double mean, double shape); +double parse_duration(const char *s); +void print_help(const char *progname); +int parse_common_args(int argc, char **argv); +double sample_burst_length(double mean); +double sample_pause_length(double mean); +void record_duration(double t); +void format_duration(char *buf, size_t len, double t); +const char *ltrim_spaces(const char *s); +void print_histogram_row(double lo, double hi, int count, int maxc, int n, double theory); +void print_runtime_histogram(double *vals, int n, const char *statname, + int bins, double bin_size, int logscale); +void write_results(void); +void sig_handler(int sig); +void install_shutdown_handler(void); +int check_shutdown(void); +void permute(int *a, int n); +int mod(int a, int b); +void random_pairs(int *a, int n); +void offset_pairs(int *a, int n, int o); +void *malloc_align(size_t size); + +#ifdef __cplusplus +} +#endif + +#endif /* BLINK_COMMON_H */ diff --git a/src/gather/gather_b.c b/src/gather/gather_b.c new file mode 100644 index 000000000..3349ca1ac --- /dev/null +++ b/src/gather/gather_b.c @@ -0,0 +1,136 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + /*all ranks need a send buffer; only root needs recv buffer large enough for all ranks*/ + unsigned char *send_buf; + unsigned char *recv_buf=NULL; + + send_buf=(unsigned char*)malloc_align(msg_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + if(my_rank==master_rank){ + recv_buf=(unsigned char*)malloc_align((size_t)w_size*msg_size); + if(recv_buf==NULL){ + fprintf(stderr,"Failed to allocate recv_buf on rank %d\n",my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + /*all ranks need a send buffer; only root needs recv buffer large enough for all ranks*/ + unsigned char *send_buf; + unsigned char *recv_buf=NULL; + MPI_Request *requests; + + send_buf=(unsigned char*)malloc_align(msg_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*measure_granularity); + + if(my_rank==master_rank){ + /* one full gathered block per batched (granularity) call so the + * concurrently outstanding MPI_Igather ops never share a buffer. */ + recv_buf=(unsigned char*)malloc_align((size_t)measure_granularity*w_size*msg_size); + if(recv_buf==NULL){ + fprintf(stderr,"Failed to allocate recv_buf on rank %d\n",my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time)= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -173,13 +108,42 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + /* debug: deterministic post-loop exchange that verifies each sender's + * payload arrived intact at the receiver. Uses specific source/tag so + * we can attribute each slot to a known sender. Barrier first to + * drain any measurement-loop ANY_TAG receives so they don't consume + * our debug-tagged sends. */ + if (debug_mode) { + MPI_Barrier(MPI_COMM_WORLD); + int _ok = 1; + if (my_rank == master_rank) { + int s, _b, slot = 0; + for (s = 0; s < w_size; s++) { + if (s == master_rank) continue; + MPI_Recv(&recv_buf[slot*msg_size], msg_size, MPI_BYTE, + s, 0xD, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + for (_b = 0; _b < msg_size && _ok; _b++) + if (recv_buf[(size_t)slot*msg_size + _b] != (unsigned char)s) _ok = 0; + slot++; + } + printf("DEBUG rank=%d nprocs=%d nsenders=%d check=%s\n", + my_rank, w_size, w_size - 1, _ok ? "OK" : "FAIL"); + } else { + MPI_Send(send_buf, msg_size, MPI_BYTE, master_rank, 0xD, MPI_COMM_WORLD); + printf("DEBUG rank=%d nprocs=%d target=%d check=OK\n", + my_rank, w_size, master_rank); + } + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/inc_bsnbr.c b/src/incast/incast_bsnbr.c similarity index 55% rename from inc_bsnbr.c rename to src/incast/incast_bsnbr.c index 78a6e8bc7..91bcf6c33 100644 --- a/inc_bsnbr.c +++ b/src/incast/incast_bsnbr.c @@ -18,81 +18,18 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,j,k; - - /*read cmd line args*/ - for(i=1;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -175,13 +110,42 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + /* debug: deterministic post-loop exchange that verifies each sender's + * payload arrived intact at the receiver. Uses specific source/tag so + * we can attribute each slot to a known sender. Barrier first to + * drain any measurement-loop ANY_TAG receives so they don't consume + * our debug-tagged sends. */ + if (debug_mode) { + MPI_Barrier(MPI_COMM_WORLD); + int _ok = 1; + if (my_rank == master_rank) { + int s, _b, slot = 0; + for (s = 0; s < w_size; s++) { + if (s == master_rank) continue; + MPI_Recv(&recv_buf[slot*msg_size], msg_size, MPI_BYTE, + s, 0xD, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + for (_b = 0; _b < msg_size && _ok; _b++) + if (recv_buf[(size_t)slot*msg_size + _b] != (unsigned char)s) _ok = 0; + slot++; + } + printf("DEBUG rank=%d nprocs=%d nsenders=%d check=%s\n", + my_rank, w_size, w_size - 1, _ok ? "OK" : "FAIL"); + } else { + MPI_Send(send_buf, msg_size, MPI_BYTE, master_rank, 0xD, MPI_COMM_WORLD); + printf("DEBUG rank=%d nprocs=%d target=%d check=OK\n", + my_rank, w_size, master_rank); + } + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/inc_get.c b/src/incast/incast_get.c similarity index 52% rename from inc_get.c rename to src/incast/incast_get.c index ea618a217..646f31b84 100644 --- a/inc_get.c +++ b/src/incast/incast_get.c @@ -18,81 +18,18 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,j,k; - - /*read cmd line args*/ - for(i=1;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -178,13 +114,34 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + /* debug: master verifies each sender's window content arrived intact. + * recv_buf is already populated by the last measurement-loop Get, with + * sender j's bytes at offset j*msg_size*measure_granularity. */ + if (debug_mode) { + if (my_rank == master_rank) { + int s, _b, _ok = 1; + for (s = 0; s < w_size && _ok; s++) { + if (s == master_rank) continue; + for (_b = 0; _b < msg_size && _ok; _b++) + if (recv_buf[(size_t)s*msg_size*measure_granularity + _b] != (unsigned char)s) _ok = 0; + } + printf("DEBUG rank=%d nprocs=%d nsenders=%d check=%s\n", + my_rank, w_size, w_size - 1, _ok ? "OK" : "FAIL"); + } else { + printf("DEBUG rank=%d nprocs=%d target=%d check=OK\n", + my_rank, w_size, master_rank); + } + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/inc_nb.c b/src/incast/incast_nb.c similarity index 58% rename from inc_nb.c rename to src/incast/incast_nb.c index 516cd3943..242c28d8e 100644 --- a/inc_nb.c +++ b/src/incast/incast_nb.c @@ -18,81 +18,18 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,j,k; - - /*read cmd line args*/ - for(i=1;i 1). */ + recv_buf_size=(size_t)measure_granularity*(w_size-1)*msg_size; send_buf=(unsigned char*)malloc_align(send_buf_size); recv_buf=(unsigned char*)malloc_align(recv_buf_size); @@ -115,18 +54,14 @@ int main(int argc, char** argv){ recv_requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*(w_size-1)*measure_granularity); durations=(double *)malloc_align(sizeof(double)*max_samples); - if(send_buf==NULL || recv_buf==NULL || durations==NULL || send_requests==NULL || recv_requests==NULL){ - fprintf(stderr,"Failed to allocate a buffer on rank %d\n",my_rank); - exit(-1); - } - /*fill send buffer with dummies*/ + /*fill send buffer: rank-as-payload under -debug so receivers can verify*/ if(my_rank!=master_rank){ for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -182,13 +119,42 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + /* debug: deterministic post-loop exchange that verifies each sender's + * payload arrived intact at the receiver. Uses specific source/tag so + * we can attribute each slot to a known sender. Barrier first to + * drain any measurement-loop ANY_TAG receives so they don't consume + * our debug-tagged sends. */ + if (debug_mode) { + MPI_Barrier(MPI_COMM_WORLD); + int _ok = 1; + if (my_rank == master_rank) { + int s, _b, slot = 0; + for (s = 0; s < w_size; s++) { + if (s == master_rank) continue; + MPI_Recv(&recv_buf[slot*msg_size], msg_size, MPI_BYTE, + s, 0xD, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + for (_b = 0; _b < msg_size && _ok; _b++) + if (recv_buf[(size_t)slot*msg_size + _b] != (unsigned char)s) _ok = 0; + slot++; + } + printf("DEBUG rank=%d nprocs=%d nsenders=%d check=%s\n", + my_rank, w_size, w_size - 1, _ok ? "OK" : "FAIL"); + } else { + MPI_Send(send_buf, msg_size, MPI_BYTE, master_rank, 0xD, MPI_COMM_WORLD); + printf("DEBUG rank=%d nprocs=%d target=%d check=OK\n", + my_rank, w_size, master_rank); + } + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/inc_put.c b/src/incast/incast_put.c similarity index 50% rename from inc_put.c rename to src/incast/incast_put.c index 09e568574..8edcce2e8 100644 --- a/inc_put.c +++ b/src/incast/incast_put.c @@ -18,81 +18,18 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - int i,k; - - /*read cmd line args*/ - for(i=1;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -174,13 +111,34 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + /* debug: master verifies each sender's Put-payload landed at the right + * offset in its window. Senders contribute nothing here beyond having + * filled send_buf with their rank. */ + if (debug_mode) { + if (my_rank == master_rank) { + int s, _b, _ok = 1; + for (s = 0; s < w_size && _ok; s++) { + if (s == master_rank) continue; + for (_b = 0; _b < msg_size && _ok; _b++) + if (recv_buf[(size_t)s*msg_size*measure_granularity + _b] != (unsigned char)s) _ok = 0; + } + printf("DEBUG rank=%d nprocs=%d nsenders=%d check=%s\n", + my_rank, w_size, w_size - 1, _ok ? "OK" : "FAIL"); + } else { + printf("DEBUG rank=%d nprocs=%d target=%d check=OK\n", + my_rank, w_size, master_rank); + } + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/src/kpartners/kpartners_nb.c b/src/kpartners/kpartners_nb.c new file mode 100644 index 000000000..dd8dc4996 --- /dev/null +++ b/src/kpartners/kpartners_nb.c @@ -0,0 +1,275 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +const char *benchmark_help = +" -k number of random partners per rank (default 1, capped at w_size-1)\n"; + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + int npartners=1; /*number of random communication partners per rank (-k flag)*/ + + /*parse command line*/ + int i, k, p; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-k") == 0) { + npartners = atoi(arg_value(argc, argv, &i)); + } else { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + } + + /*validate npartners*/ + if(npartners<1){ + if(my_rank==master_rank){ + fprintf(stderr,"k (%d) must be >= 1\n",npartners); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + if(npartners>w_size-1){ + if(my_rank==master_rank){ + fprintf(stderr,"Warning: k (%d) capped to w_size-1 (%d)\n",npartners,w_size-1); + } + npartners=w_size-1; + } + + /* + * Build npartners random permutations of [0, w_size-1] such that the + * resulting communication graph is k-regular AND every rank has k + * DISTINCT partners (no self-pairings, no duplicate target across + * permutations). For permutation p, rank i sends to perms[p*w_size + i]. + * + * Algorithm: rejection sampling. Each permutation must + * (a) have no fixed point (perms[p][i] != i — avoids self-send), and + * (b) for every i, perms[p][i] != perms[q][i] for all q < p (avoids + * column duplicates, i.e. same target as an earlier permutation). + * If a generated permutation fails, retry. For small k relative to + * w_size this terminates quickly. After many attempts fail we fall + * back to a swap-repair pass. Partners are fixed for the entire run. + */ + int *perms; + perms=(int*)malloc_align(sizeof(int)*npartners*w_size); + if(perms==NULL){ + fprintf(stderr,"Failed to allocate permutation table on rank %d\n",my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + int attempts; + for(p=0;p= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +const char *benchmark_help = +" -mode pairing strategy (default: offpair)\n" +" offpair symmetric offset pairs (uses -offset)\n" +" rpair random reciprocal pairing (uses -seed)\n" +" rot rotation: rank r -> (r+offset) mod w_size; non-reciprocal\n" +" perm random permutation; non-reciprocal\n" +" -offset offset for offpair/rot modes (default 1)\n"; + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + const char *comm_mode="offpair"; + int target_offset=1; + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-offset") == 0) { + target_offset = atoi(arg_value(argc, argv, &i)); + } else if (strcmp(argv[i], "-mode") == 0) { + comm_mode = arg_value(argc, argv, &i); + } else { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + size_t send_buf_size, recv_buf_size; + unsigned char *send_buf; + unsigned char *recv_buf; + int *targets; + + send_buf_size=msg_size; + recv_buf_size=(size_t)measure_granularity*msg_size; + + send_buf=(unsigned char*)malloc_align(send_buf_size); + recv_buf=(unsigned char*)malloc_align(recv_buf_size); + targets=(int*)malloc_align(sizeof(int)*w_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + + /*fill send buffer: rank-as-payload under -debug so partners can verify*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time)= 0) { + MPI_Request reqs[2]; + MPI_Irecv(recv_buf, msg_size, MPI_BYTE, src, 0xD, MPI_COMM_WORLD, &reqs[0]); + MPI_Isend(send_buf, msg_size, MPI_BYTE, dst, 0xD, MPI_COMM_WORLD, &reqs[1]); + MPI_Waitall(2, reqs, MPI_STATUSES_IGNORE); + for (_b = 0; _b < msg_size && _ok; _b++) + if (recv_buf[_b] != (unsigned char)src) _ok = 0; + } + printf("DEBUG rank=%d nprocs=%d partner=%d check=%s\n", + my_rank, w_size, dst, _ok ? "OK" : "FAIL"); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: + /*write results to file*/ + MPI_Barrier(MPI_COMM_WORLD); + write_results(); + + /*free allocated buffers*/ + free(targets); + free(durations); + free(send_buf); + free(recv_buf); + + /*exit MPI library*/ + MPI_Finalize(); +} diff --git a/o2o_bsnbr.c b/src/pairwise/pairwise_bsnbr.c similarity index 51% rename from o2o_bsnbr.c rename to src/pairwise/pairwise_bsnbr.c index 469488406..b7e7a356b 100644 --- a/o2o_bsnbr.c +++ b/src/pairwise/pairwise_bsnbr.c @@ -10,6 +10,14 @@ #include #include "common.h" +const char *benchmark_help = +" -mode pairing strategy (default: offpair)\n" +" offpair symmetric offset pairs (uses -offset)\n" +" rpair random reciprocal pairing (uses -seed)\n" +" rot rotation: rank r -> (r+offset) mod w_size; non-reciprocal\n" +" perm random permutation; non-reciprocal\n" +" -offset offset for offpair/rot modes (default 1)\n"; + int main(int argc, char** argv){ /*init MPI world*/ @@ -18,90 +26,27 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - char *comm_mode="offpair"; + const char *comm_mode="offpair"; int target_offset=1; - - int i,k; - /*read cmd line args*/ - for(i=1;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -215,13 +145,39 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + /* debug: deterministic post-loop exchange to verify partner identity and + * payload integrity. Send to targets[my_rank] and receive from this rank's + * inverse source (the rank whose target is me). These differ in the + * non-reciprocal modes (perm/rot), so a non-blocking exchange is used to + * avoid deadlock. Barrier first to drain any measurement-loop ANY_TAG + * receives so they don't consume our debug-tagged sends. */ + if (debug_mode) { + MPI_Barrier(MPI_COMM_WORLD); + int dst = targets[my_rank]; + int src = -1, _t; + for (_t = 0; _t < w_size; _t++) if (targets[_t] == my_rank) { src = _t; break; } + int _ok = 1, _b; + if (dst != my_rank && src >= 0) { + MPI_Request reqs[2]; + MPI_Irecv(recv_buf, msg_size, MPI_BYTE, src, 0xD, MPI_COMM_WORLD, &reqs[0]); + MPI_Isend(send_buf, msg_size, MPI_BYTE, dst, 0xD, MPI_COMM_WORLD, &reqs[1]); + MPI_Waitall(2, reqs, MPI_STATUSES_IGNORE); + for (_b = 0; _b < msg_size && _ok; _b++) + if (recv_buf[_b] != (unsigned char)src) _ok = 0; + } + printf("DEBUG rank=%d nprocs=%d partner=%d check=%s\n", + my_rank, w_size, dst, _ok ? "OK" : "FAIL"); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/o2o_nb.c b/src/pairwise/pairwise_nb.c similarity index 55% rename from o2o_nb.c rename to src/pairwise/pairwise_nb.c index 34c1185d0..ea6ed9c9c 100644 --- a/o2o_nb.c +++ b/src/pairwise/pairwise_nb.c @@ -10,6 +10,14 @@ #include #include "common.h" +const char *benchmark_help = +" -mode pairing strategy (default: offpair)\n" +" offpair symmetric offset pairs (uses -offset)\n" +" rpair random reciprocal pairing (uses -seed)\n" +" rot rotation: rank r -> (r+offset) mod w_size; non-reciprocal\n" +" perm random permutation; non-reciprocal\n" +" -offset offset for offpair/rot modes (default 1)\n"; + int main(int argc, char** argv){ /*init MPI world*/ @@ -18,90 +26,27 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - - char *comm_mode="offpair"; + const char *comm_mode="offpair"; int target_offset=1; - - int i,k; - /*read cmd line args*/ - for(i=1;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -205,13 +148,39 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + /* debug: deterministic post-loop exchange to verify partner identity and + * payload integrity. Send to targets[my_rank] and receive from this rank's + * inverse source (the rank whose target is me). These differ in the + * non-reciprocal modes (perm/rot), so a non-blocking exchange is used to + * avoid deadlock. Barrier first to drain any measurement-loop ANY_TAG + * receives so they don't consume our debug-tagged sends. */ + if (debug_mode) { + MPI_Barrier(MPI_COMM_WORLD); + int dst = targets[my_rank]; + int src = -1, _t; + for (_t = 0; _t < w_size; _t++) if (targets[_t] == my_rank) { src = _t; break; } + int _ok = 1, _b; + if (dst != my_rank && src >= 0) { + MPI_Request reqs[2]; + MPI_Irecv(recv_buf, msg_size, MPI_BYTE, src, 0xD, MPI_COMM_WORLD, &reqs[0]); + MPI_Isend(send_buf, msg_size, MPI_BYTE, dst, 0xD, MPI_COMM_WORLD, &reqs[1]); + MPI_Waitall(2, reqs, MPI_STATUSES_IGNORE); + for (_b = 0; _b < msg_size && _ok; _b++) + if (recv_buf[_b] != (unsigned char)src) _ok = 0; + } + printf("DEBUG rank=%d nprocs=%d partner=%d check=%s\n", + my_rank, w_size, dst, _ok ? "OK" : "FAIL"); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); diff --git a/src/pingpong/pingpong_b.c b/src/pingpong/pingpong_b.c new file mode 100644 index 000000000..04a11a3c2 --- /dev/null +++ b/src/pingpong/pingpong_b.c @@ -0,0 +1,150 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + int send_buf_size, recv_buf_size; + unsigned char *send_buf; + unsigned char *recv_buf; + + send_buf_size=msg_size; + recv_buf_size=msg_size; + send_buf=(unsigned char*)malloc_align(send_buf_size); + recv_buf=(unsigned char*)malloc_align(recv_buf_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + + if(w_size!=2){ + fprintf(stderr,"Needs two processes to ping-pong. %d\n",my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration((MPI_Wtime()-measure_start_time)/2.0); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time)= warm_up_iters) record_duration((MPI_Wtime()-measure_start_time)/2.0); }else{ for(i=0;i= warm_up_iters) record_duration((MPI_Wtime()-measure_start_time)/2.0); }else{ for(i=0;i +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + int msg_size_ints; + int *send_buf; + int *recv_buf; + + if(msg_size%sizeof(int)!=0){ + if(my_rank==master_rank) + fprintf(stderr, "Msg-size (%d) must be divisible by size of int (%zu)\n",msg_size,sizeof(int)); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + msg_size_ints=msg_size/sizeof(int); + + send_buf=(int*)malloc_align(msg_size); + recv_buf=(int*)malloc_align(msg_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + int msg_size_ints; + int *send_buf; + int *recv_buf; + MPI_Request *requests; + + if(msg_size%sizeof(int)!=0){ + if(my_rank==master_rank) + fprintf(stderr, "Msg-size (%d) must be divisible by size of int (%zu)\n",msg_size,sizeof(int)); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + msg_size_ints=msg_size/sizeof(int); + + send_buf=(int*)malloc_align(msg_size); + /* one reduced result per batched (granularity) call so the concurrently + * outstanding MPI_Ireduce ops never share a receive buffer (-grty > 1). */ + recv_buf=(int*)malloc_align((size_t)measure_granularity*msg_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*measure_granularity); + + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + int msg_size_ints; + int *send_buf; + int *recv_buf; + int *recvcounts; + + if(msg_size%sizeof(int)!=0){ + if(my_rank==master_rank) + fprintf(stderr, "Msg-size (%d) must be divisible by size of int (%zu)\n",msg_size,sizeof(int)); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + msg_size_ints=msg_size/sizeof(int); + + /*each rank sends w_size*msg_size total, receives msg_size*/ + send_buf=(int*)malloc_align((size_t)w_size*msg_size); + recv_buf=(int*)malloc_align(msg_size); + recvcounts=(int*)malloc_align(sizeof(int)*w_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + + /*fill send buffer with dummies*/ + for(size_t bi=0;bi<(size_t)w_size*msg_size_ints;bi++){ + send_buf[bi] = debug_mode ? my_rank : 1; + } + + /*each rank receives msg_size_ints ints*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + int msg_size_ints; + int *send_buf; + int *recv_buf; + int *recvcounts; + MPI_Request *requests; + + if(msg_size%sizeof(int)!=0){ + if(my_rank==master_rank) + fprintf(stderr, "Msg-size (%d) must be divisible by size of int (%zu)\n",msg_size,sizeof(int)); + MPI_Abort(MPI_COMM_WORLD, -1); + } + + msg_size_ints=msg_size/sizeof(int); + + /*each rank sends w_size*msg_size total, receives msg_size*/ + send_buf=(int*)malloc_align((size_t)w_size*msg_size); + /* one received chunk per batched (granularity) call so the concurrently + * outstanding MPI_Ireduce_scatter ops never share a receive buffer (-grty > 1). */ + recv_buf=(int*)malloc_align((size_t)measure_granularity*msg_size); + recvcounts=(int*)malloc_align(sizeof(int)*w_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*measure_granularity); + + + /*fill send buffer with dummies*/ + for(size_t bi=0;bi<(size_t)w_size*msg_size_ints;bi++){ + send_buf[bi] = debug_mode ? my_rank : 1; + } + + /*each rank receives msg_size_ints ints*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) #include "common.h" +const char *benchmark_help = +" -rring randomise the ring topology (default: sequential 0->1->...->N-1)\n"; + int main(int argc, char** argv){ /*init MPI world*/ @@ -18,85 +21,24 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - bool rand_ring=false; - - int i,k; - /*read cmd line args*/ - for(i=1;i rank" map; build the inverse so each + * rank can find its position in the (possibly permuted) ring. Then the + * left/right neighbours are the ranks at adjacent positions, which + * guarantees a single Hamiltonian cycle in both directions and full + * reciprocity (A.left = B <=> B.right = A). */ + int *positions = (int*)malloc_align(sizeof(int)*w_size); + if (positions == NULL) { + fprintf(stderr, "Failed to allocate positions on rank %d\n", my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -192,19 +145,47 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + /* debug: deterministic post-loop exchange to verify both ring neighbours + * sent the expected payload (their rank). Uses specific sources so a + * broken topology (e.g. the old -rring bug) is detected. Barrier + * first to drain any measurement-loop ANY_TAG receives. */ + if (debug_mode) { + MPI_Barrier(MPI_COMM_WORLD); + /* reuse the already-allocated recv_buf (>= 2*msg_size) instead of a + * stack VLA, which would risk a stack overflow for large msg_size */ + unsigned char *left_buf = recv_buf; + unsigned char *right_buf = recv_buf + msg_size; + MPI_Request reqs[2]; + int _ok = 1, _b; + MPI_Irecv(left_buf, msg_size, MPI_BYTE, left_neighbor, 0xD, MPI_COMM_WORLD, &reqs[0]); + MPI_Irecv(right_buf, msg_size, MPI_BYTE, right_neighbor, 0xD, MPI_COMM_WORLD, &reqs[1]); + MPI_Send(send_buf, msg_size, MPI_BYTE, left_neighbor, 0xD, MPI_COMM_WORLD); + MPI_Send(send_buf, msg_size, MPI_BYTE, right_neighbor, 0xD, MPI_COMM_WORLD); + MPI_Waitall(2, reqs, MPI_STATUSES_IGNORE); + for (_b = 0; _b < msg_size && _ok; _b++) + if (left_buf[_b] != (unsigned char)left_neighbor) _ok = 0; + for (_b = 0; _b < msg_size && _ok; _b++) + if (right_buf[_b] != (unsigned char)right_neighbor) _ok = 0; + printf("DEBUG rank=%d nprocs=%d left=%d right=%d check=%s\n", + my_rank, w_size, left_neighbor, right_neighbor, _ok ? "OK" : "FAIL"); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); /*free allocated buffers*/ free(targets); + free(positions); free(durations); free(send_buf); free(recv_buf); diff --git a/ring_nb.c b/src/ring/ring_nb.c similarity index 52% rename from ring_nb.c rename to src/ring/ring_nb.c index fca46e7c8..06dea039a 100644 --- a/ring_nb.c +++ b/src/ring/ring_nb.c @@ -10,6 +10,9 @@ #include #include "common.h" +const char *benchmark_help = +" -rring randomise the ring topology (default: sequential 0->1->...->N-1)\n"; + int main(int argc, char** argv){ /*init MPI world*/ @@ -18,85 +21,24 @@ int main(int argc, char** argv){ MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /*register signal handler*/ - signal(SIGUSR1,sig_handler); //or SIGUSR1 here + install_shutdown_handler(); - /*default values*/ - int master_rank=0; - bool master_rand=false; - - int rand_seed=1; - - int msg_size=1024; - int measure_granularity=1; - max_samples=1000; - - warm_up_iters=5; - int max_iters=1; - bool endless=false; - - double burst_length=0.0; - bool burst_length_rand=false; - double burst_pause=0.0; - bool burst_pause_rand=false; - bool rand_ring=false; - - int i,k; - /*read cmd line args*/ - for(i=1;i rank" map; build the inverse so each + * rank can find its position in the (possibly permuted) ring. Then the + * left/right neighbours are the ranks at adjacent positions, which + * guarantees a single Hamiltonian cycle in both directions and full + * reciprocity (A.left = B <=> B.right = A). */ + int *positions = (int*)malloc_align(sizeof(int)*w_size); + if (positions == NULL) { + fprintf(stderr, "Failed to allocate positions on rank %d\n", my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); curr_iters++; if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ if(my_rank==master_rank){ /*master decides if burst should be continued*/ @@ -196,19 +150,47 @@ int main(int argc, char** argv){ }while(burst_cont); if(burst_pause!=0){ if(burst_pause_rand){ /*randomized break length*/ - burst_pause=rand_expo(burst_pause_mean); + burst_pause=sample_pause_length(burst_pause_mean); } dsleep(burst_pause); } } }while(endless); + /* debug: deterministic post-loop exchange to verify both ring neighbours + * sent the expected payload (their rank). Uses specific sources so a + * broken topology (e.g. the old -rring bug) is detected. Barrier + * first to drain any measurement-loop ANY_TAG receives. */ + if (debug_mode) { + MPI_Barrier(MPI_COMM_WORLD); + /* reuse the already-allocated recv_buf (>= 2*msg_size) instead of a + * stack VLA, which would risk a stack overflow for large msg_size */ + unsigned char *left_buf = recv_buf; + unsigned char *right_buf = recv_buf + msg_size; + MPI_Request reqs[4]; + int _ok = 1, _b; + MPI_Irecv(left_buf, msg_size, MPI_BYTE, left_neighbor, 0xD, MPI_COMM_WORLD, &reqs[0]); + MPI_Irecv(right_buf, msg_size, MPI_BYTE, right_neighbor, 0xD, MPI_COMM_WORLD, &reqs[1]); + MPI_Isend(send_buf, msg_size, MPI_BYTE, left_neighbor, 0xD, MPI_COMM_WORLD, &reqs[2]); + MPI_Isend(send_buf, msg_size, MPI_BYTE, right_neighbor, 0xD, MPI_COMM_WORLD, &reqs[3]); + MPI_Waitall(4, reqs, MPI_STATUSES_IGNORE); + for (_b = 0; _b < msg_size && _ok; _b++) + if (left_buf[_b] != (unsigned char)left_neighbor) _ok = 0; + for (_b = 0; _b < msg_size && _ok; _b++) + if (right_buf[_b] != (unsigned char)right_neighbor) _ok = 0; + printf("DEBUG rank=%d nprocs=%d left=%d right=%d check=%s\n", + my_rank, w_size, left_neighbor, right_neighbor, _ok ? "OK" : "FAIL"); + fflush(stdout); + MPI_Barrier(MPI_COMM_WORLD); + } +done: /*write results to file*/ MPI_Barrier(MPI_COMM_WORLD); write_results(); /*free allocated buffers*/ free(targets); + free(positions); free(durations); free(send_buf); free(recv_buf); diff --git a/src/scatter/scatter_b.c b/src/scatter/scatter_b.c new file mode 100644 index 000000000..0cdec93cb --- /dev/null +++ b/src/scatter/scatter_b.c @@ -0,0 +1,132 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + /*root needs a send buffer large enough for all ranks; others only need recv*/ + unsigned char *send_buf=NULL; + unsigned char *recv_buf; + + if(my_rank==master_rank){ + send_buf=(unsigned char*)malloc_align((size_t)w_size*msg_size); + if(send_buf==NULL){ + fprintf(stderr,"Failed to allocate send_buf on rank %d\n",my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + for(size_t bi=0;bi<(size_t)w_size*msg_size;bi++){ + send_buf[bi] = debug_mode ? (unsigned char)(bi / msg_size) : 'a'; + } + } + + recv_buf=(unsigned char*)malloc_align(msg_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + + /*print basic info to stdout*/ + if(my_rank==master_rank){ + if(endless){ + printf("Scatter with %d processes, sender rank: %d, msg-size: %d, test iterations: endless.\n" + ,w_size,master_rank,msg_size); + }else{ + printf("Scatter with %d processes, sender rank: %d, msg-size: %d, test iterations: %d.\n" + ,w_size,master_rank,msg_size,max_iters); + } + } + + /*measured iterations*/ + double burst_start_time; + double measure_start_time; + double burst_length_mean=burst_length; + double burst_pause_mean=burst_pause; + int burst_cont=0; + curr_iters=0; + measured_iters=0; + + MPI_Barrier(MPI_COMM_WORLD); + do{ + for(k=0;k= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + /*root needs a send buffer large enough for all ranks; others only need recv*/ + unsigned char *send_buf=NULL; + unsigned char *recv_buf; + MPI_Request *requests; + + if(my_rank==master_rank){ + send_buf=(unsigned char*)malloc_align((size_t)w_size*msg_size); + if(send_buf==NULL){ + fprintf(stderr,"Failed to allocate send_buf on rank %d\n",my_rank); + MPI_Abort(MPI_COMM_WORLD, -1); + } + for(size_t bi=0;bi<(size_t)w_size*msg_size;bi++){ + send_buf[bi] = debug_mode ? (unsigned char)(bi / msg_size) : 'a'; + } + } + + /* one received chunk per batched (granularity) call so the concurrently + * outstanding MPI_Iscatter ops never share a receive buffer (-grty > 1). */ + recv_buf=(unsigned char*)malloc_align((size_t)measure_granularity*msg_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*measure_granularity); + + + /*print basic info to stdout*/ + if(my_rank==master_rank){ + if(endless){ + printf("Scatter with %d processes, sender rank: %d, msg-size: %d, test iterations: endless.\n" + ,w_size,master_rank,msg_size); + }else{ + printf("Scatter with %d processes, sender rank: %d, msg-size: %d, test iterations: %d.\n" + ,w_size,master_rank,msg_size,max_iters); + } + } + + /*measured iterations*/ + double burst_start_time; + double measure_start_time; + double burst_length_mean=burst_length; + double burst_pause_mean=burst_pause; + int burst_cont=0; + curr_iters=0; + measured_iters=0; + + MPI_Barrier(MPI_COMM_WORLD); + do{ + for(k=0;k= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +const char *benchmark_help = +" -dimx first dim of the 2D Cartesian grid (default 0 = auto via MPI_Dims_create)\n" +" if non-zero, must divide w_size; w_size/dimx becomes the Y dim\n" +" -periodic enable wrap-around (toroidal) boundaries on both dims\n"; + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + int dimx=0; /*0 means auto-factor via MPI_Dims_create*/ + bool periodic=false; + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (strcmp(argv[i], "-dimx") == 0) { + dimx = atoi(arg_value(argc, argv, &i)); + } else if (strcmp(argv[i], "-periodic") == 0) { + periodic = true; + } else { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + } + + /*build 2D Cartesian communicator*/ + int dims[2]={0,0}; + int periods[2]={0,0}; + + if(dimx>0){ + if(w_size%dimx!=0){ + if(my_rank==master_rank){ + fprintf(stderr,"dimx (%d) does not divide w_size (%d)\n",dimx,w_size); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + dims[0]=dimx; + dims[1]=w_size/dimx; + }else{ + MPI_Dims_create(w_size,2,dims); + } + + if(periodic){ + periods[0]=1; + periods[1]=1; + } + + MPI_Comm cart_comm; + MPI_Cart_create(MPI_COMM_WORLD,2,dims,periods,0,&cart_comm); + + /*get 4-neighbor ranks (MPI_PROC_NULL for missing boundary neighbors)*/ + /*direction 0 = rows: north = row-1, south = row+1*/ + /*direction 1 = cols: west = col-1, east = col+1*/ + int north, south, west, east; + MPI_Cart_shift(cart_comm,0,1,&north,&south); + MPI_Cart_shift(cart_comm,1,1,&west,&east); + + int neighbors[4]={north,south,west,east}; + + /*pin to core*/ + /*cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(1, &mask); + sched_setaffinity(0, sizeof(mask), &mask);*/ + + /*allocate buffers*/ + /*4 directions, each of size msg_size; granularity batches them*/ + size_t send_buf_size, recv_buf_size; + unsigned char *send_buf; + unsigned char *recv_buf; + MPI_Request *send_requests; + MPI_Request *recv_requests; + + send_buf_size=msg_size; + recv_buf_size=(size_t)4*measure_granularity*msg_size; + + send_buf=(unsigned char*)malloc_align(send_buf_size); + recv_buf=(unsigned char*)malloc_align(recv_buf_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + send_requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*4*measure_granularity); + recv_requests=(MPI_Request*)malloc_align(sizeof(MPI_Request)*4*measure_granularity); + + + /*fill send buffer with dummies*/ + for(i=0;i= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); + curr_iters++; + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +int main(int argc, char** argv){ + + /*init MPI world*/ + MPI_Init(&argc,&argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + /*register signal handler*/ + install_shutdown_handler(); + + /*parse command line*/ + int i, k; + argc = parse_common_args(argc, argv); + for (i = 1; i < argc; i++) { + if (my_rank == master_rank) { + fprintf(stderr, "Unknown argument: %s\n", argv[i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + + /*allocate buffers*/ + size_t send_buf_size, recv_buf_size; + unsigned char *send_buf; + unsigned char *recv_buf; + + send_buf_size=(size_t)msg_size*w_size; + recv_buf_size=(size_t)msg_size*w_size; + + send_buf=(unsigned char*)malloc_align(send_buf_size); + recv_buf=(unsigned char*)malloc_align(recv_buf_size); + durations=(double *)malloc_align(sizeof(double)*max_samples); + + + /*fill send buffer with dummies*/ + for(size_t bi=0;bitm_hour + , time_str_tm->tm_min + , time_str_tm->tm_sec + , time_now.tv_usec); + fflush(fd_temp); + } + //----------------- + + MPI_Barrier(MPI_COMM_WORLD); + do{ + for(k=0;k= warm_up_iters) record_duration(MPI_Wtime()-measure_start_time); /*write result to ring buffer*/ + curr_iters++; + //----------------- + if(my_rank==master_rank){ + struct timeval time_now; + gettimeofday(&time_now, NULL); + struct tm *time_str_tm; + time_str_tm = localtime(&time_now.tv_sec); /* local time, consistent with the local-time log filename */ + fprintf(fd_temp, "%02i:%02i:%02i:%06li | %i | %lld\n" + , time_str_tm->tm_hour + , time_str_tm->tm_min + , time_str_tm->tm_sec + , time_now.tv_usec + , w_size + , curr_iters); + fflush(fd_temp); + } + //----------------- + if(burst_length!=0){ /*bcast needed for synch if bursts timed*/ + if(my_rank==master_rank){ /*master decides if burst should be continued*/ + burst_cont=((MPI_Wtime()-burst_start_time) +#include +#include +#include +#include +#include "common.h" + +#define N_SAMPLES 100000 +#define HIST_NBINS 8 + +/* Bin edges expressed as multiples of mean. The last bin is open-ended (overflow). */ +static const double EDGES[] = { 0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 5.0, 10.0 }; + +/* ── theoretical CDF functions ──────────────────────────────────────────── */ + +static double cdf_exp(double x, double mean, double shape) +{ + (void)shape; + if (x < 0.0) return 0.0; + return 1.0 - exp(-x / mean); +} + +static double cdf_pareto(double x, double mean, double alpha) +{ + double x_m = mean * (alpha - 1.0) / alpha; + if (x < x_m) return 0.0; + return 1.0 - pow(x_m / x, alpha); +} + +static double cdf_lognormal(double x, double mean, double sigma) +{ + if (x <= 0.0) return 0.0; + double mu = log(mean) - 0.5 * sigma * sigma; + double z = (log(x) - mu) / sigma; + return 0.5 * erfc(-z / M_SQRT2); +} + +typedef double (*cdf_fn)(double x, double p1, double p2); + +/* ── KS statistic D_n ───────────────────────────────────────────────────── */ + +static double ks_statistic(double *sorted, int n, cdf_fn cdf, double p1, double p2) +{ + double D = 0.0; + int i; + for (i = 0; i < n; i++) { + double F = cdf(sorted[i], p1, p2); + double d1 = fabs(F - (double)(i + 1) / n); + double d2 = fabs(F - (double)i / n); + double d = d1 > d2 ? d1 : d2; + if (d > D) D = d; + } + return D; +} + +static double ks_critical(int n) +{ + return 1.628 / sqrt((double)n); /* 1 % significance, Kolmogorov approx */ +} + +/* ── ASCII histogram ─────────────────────────────────────────────────────── */ + +static void print_histogram(double *sorted, int n, + cdf_fn cdf, double mean, double p1, double p2) +{ + int counts[HIST_NBINS] = {0}; + int b, i; + + /* bin the (already sorted) samples using linear edges relative to mean */ + b = 0; + for (i = 0; i < n; i++) { + while (b < HIST_NBINS - 1 && sorted[i] >= EDGES[b + 1] * mean) + b++; + counts[b]++; + } + + /* find the tallest bar for scaling */ + int max_count = 1; + for (b = 0; b < HIST_NBINS; b++) + if (counts[b] > max_count) max_count = counts[b]; + + printf("\n"); + for (b = 0; b < HIST_NBINS; b++) { + double lo = EDGES[b] * mean; + double hi = (b < HIST_NBINS - 1) ? EDGES[b + 1] * mean : INFINITY; + double theory = cdf(isinf(hi) ? 1e300 : hi, p1, p2) - cdf(lo, p1, p2); + /* shared renderer (common.h): range label, scaled bar, empirical %, + * and the theoretical % overlay (theory >= 0) */ + print_histogram_row(lo, hi, counts[b], max_count, n, theory); + } + printf("\n"); +} + +/* ── sampler wrappers (uniform signature) ───────────────────────────────── */ + +static double wrap_exp (double mean, double shape) { return rand_exp(mean, shape); } +static double wrap_pareto (double mean, double shape) { return rand_pareto(mean, shape); } +static double wrap_lognormal(double mean, double shape) { return rand_lognormal(mean, shape);} + +/* ── run one test case ──────────────────────────────────────────────────── */ + +static int run_test(const char *label, + cdf_fn cdf, + double (*sampler)(double, double), + double mean, + double shape, + double theoretical_var) /* NAN = skip variance check */ +{ + double *samples = malloc(sizeof(double) * N_SAMPLES); + int i, all_pass = 1; + + for (i = 0; i < N_SAMPLES; i++) + samples[i] = sampler(mean, shape); + + /* empirical mean */ + double sum = 0.0; + for (i = 0; i < N_SAMPLES; i++) sum += samples[i]; + double emp_mean = sum / N_SAMPLES; + + /* empirical variance */ + double sum2 = 0.0; + for (i = 0; i < N_SAMPLES; i++) { double d = samples[i] - emp_mean; sum2 += d * d; } + double emp_var = sum2 / (N_SAMPLES - 1); + + /* sort once — used by both histogram and KS test */ + qsort(samples, N_SAMPLES, sizeof(double), compare_doubles); + + /* histogram first so it appears right under the case header */ + print_histogram(samples, N_SAMPLES, cdf, mean, mean, shape); + + /* KS test */ + double D = ks_statistic(samples, N_SAMPLES, cdf, mean, shape); + double D_crit = ks_critical(N_SAMPLES); + + double mean_err = fabs(emp_mean - mean) / mean * 100.0; + int mean_ok = mean_err < 1.0; + int ks_ok = D < D_crit; + if (!mean_ok || !ks_ok) all_pass = 0; + + printf(" mean err=%5.2f%% %s | KS D=%.4f crit=%.4f %s\n", + mean_err, mean_ok ? "OK " : "FAIL", D, D_crit, ks_ok ? "OK" : "FAIL"); + + if (!isnan(theoretical_var)) { + double var_err = fabs(emp_var - theoretical_var) / theoretical_var * 100.0; + int var_ok = var_err < 10.0; + if (!var_ok) all_pass = 0; + printf(" variance: empirical=%.4e theoretical=%.4e err=%5.1f%% %s\n", + emp_var, theoretical_var, var_err, var_ok ? "OK" : "FAIL"); + } + + free(samples); + return all_pass; +} + +/* ── main ───────────────────────────────────────────────────────────────── */ + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &w_size); + MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); + + if (my_rank != 0) { MPI_Finalize(); return 0; } + + srand(42); /* fixed seed for reproducibility */ + + int total = 0, passed = 0; + double mean = 1e-3; /* 1 ms — representative MPI latency */ + + printf("=== Blink distribution sampler verification ===\n"); + printf(" n=%d samples | KS significance=1%% | KS critical=%.4f\n", + N_SAMPLES, ks_critical(N_SAMPLES)); + printf(" histogram bins: [0, 0.5, 1, 1.5, 2, 3, 5, 10, inf] × mean\n"); + printf(" bar height is relative to the tallest bin\n"); + + /* ── exponential ─────────────────────────────────────────────────────── */ + printf("\n--- exp(mean=1ms) [shape unused] ---\n"); + total++; passed += run_test("exp", cdf_exp, wrap_exp, mean, 1.0, mean * mean); + + /* ── Pareto ──────────────────────────────────────────────────────────── */ + printf("\n--- pareto(mean=1ms, alpha=1.5) [infinite variance] ---\n"); + total++; passed += run_test("pareto alpha=1.5", + cdf_pareto, wrap_pareto, mean, 1.5, NAN); + + printf("\n--- pareto(mean=1ms, alpha=2.5) [infinite kurtosis → var check skipped] ---\n"); + total++; passed += run_test("pareto alpha=2.5", + cdf_pareto, wrap_pareto, mean, 2.5, NAN); + + printf("\n--- pareto(mean=1ms, alpha=4.0) [finite kurtosis] ---\n"); + { + double alpha = 4.0; + double x_m = mean * (alpha - 1.0) / alpha; + double var = x_m * x_m * alpha / ((alpha-1.0)*(alpha-1.0)*(alpha-2.0)); + total++; passed += run_test("pareto alpha=4.0", + cdf_pareto, wrap_pareto, mean, alpha, var); + } + + /* ── log-normal ──────────────────────────────────────────────────────── */ + printf("\n--- lognormal(mean=1ms, sigma=0.5) ---\n"); + { + double sigma = 0.5; + double var = (exp(sigma*sigma) - 1.0) * mean * mean; + total++; passed += run_test("lognormal sigma=0.5", + cdf_lognormal, wrap_lognormal, mean, sigma, var); + } + + printf("\n--- lognormal(mean=1ms, sigma=1.0) ---\n"); + { + double sigma = 1.0; + double var = (exp(sigma*sigma) - 1.0) * mean * mean; + total++; passed += run_test("lognormal sigma=1.0", + cdf_lognormal, wrap_lognormal, mean, sigma, var); + } + + printf("\n--- lognormal(mean=1ms, sigma=1.5) [huge kurtosis → var check skipped] ---\n"); + { + double sigma = 1.5; + total++; passed += run_test("lognormal sigma=1.5", + cdf_lognormal, wrap_lognormal, mean, sigma, NAN); + } + + printf("\n%d / %d tests passed.\n", passed, total); + + MPI_Finalize(); + return (passed == total) ? 0 : 1; +} diff --git a/src/tools/null_dummy.c b/src/tools/null_dummy.c new file mode 100644 index 000000000..760f1d4ed --- /dev/null +++ b/src/tools/null_dummy.c @@ -0,0 +1,6 @@ +#include + +int main(int argc, char** argv){ + MPI_Init(&argc,&argv); + MPI_Finalize(); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 000000000..556c4e9ee --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,33 @@ +# ── Black-box benchmark test ────────────────────────────────────────────────── +# A regular single-process GTest that spawns mpirun -debug via +# popen() (see popen_helpers.h) and parses the structured debug output. The +# benchmark paths and mpiexec command are baked in at configure time +# (overridable via env vars). +# +# `nprocs` arg is forwarded to ctest's PROCESSORS property so `ctest -j` +# accounts for each suite's actual MPI process count and avoids +# oversubscribing the host. +function(add_benchmark_test name nprocs) + add_executable(${name} ${name}.cpp) + target_include_directories(${name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_compile_definitions(${name} PRIVATE + "BLINK_BIN_DIR=\"${CMAKE_BINARY_DIR}/bin\"" + "BLINK_MPIEXEC=\"${MPIEXEC_EXECUTABLE}\"" + "BLINK_NPROC_FLAG=\"${MPIEXEC_NUMPROC_FLAG}\"") + target_link_libraries(${name} PRIVATE GTest::GTest GTest::Main) + set_target_properties(${name} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + add_test(NAME ${name} COMMAND ${name}) + set_tests_properties(${name} PROPERTIES PROCESSORS ${nprocs}) +endfunction() + +# ── Registered tests ────────────────────────────────────────────────────────── +# nprocs is the MAXIMUM rank count any single test_*.cpp launches via mpirun. +add_benchmark_test(test_collectives 8) +add_benchmark_test(test_pairwise 8) +add_benchmark_test(test_ring 8) +add_benchmark_test(test_incast 8) +add_benchmark_test(test_pingpong 8) +add_benchmark_test(test_kpartners 8) +add_benchmark_test(test_stencil 8) +add_benchmark_test(test_misc 8) diff --git a/tests/popen_helpers.h b/tests/popen_helpers.h new file mode 100644 index 000000000..2b77a3c4b --- /dev/null +++ b/tests/popen_helpers.h @@ -0,0 +1,222 @@ +#pragma once +/* + * popen_helpers.h — shared helpers for spawning a benchmark under mpirun + * and parsing structured DEBUG output. + * + * Used by every test_*.cpp. Provides: + * - run_debug_capture(): launches the benchmark, returns parsed DEBUG + * fields keyed by rank, plus exit-status decoded via WIFEXITED/WEXITSTATUS. + * stderr is redirected into a temp file and surfaced on failure so the + * diagnostic is not lost. + * + * Configure-time defaults are baked in by CMake via -DBLINK_MPIEXEC=... + * etc. Each can be overridden at run time via environment variables + * (BLINK_MPIEXEC, BLINK_NPROC_FLAG, BLINK_BIN_DIR). + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace blink { + +inline std::string env_or(const char *key, const char *def) +{ + const char *v = std::getenv(key); + return v ? v : def; +} + +inline std::string mpiexec() { return env_or("BLINK_MPIEXEC", BLINK_MPIEXEC); } +inline std::string nproc_flag() { return env_or("BLINK_NPROC_FLAG", BLINK_NPROC_FLAG); } +inline std::string bin_dir() { return env_or("BLINK_BIN_DIR", BLINK_BIN_DIR); } + +/* A single parsed DEBUG line: rank => key/value map. */ +using DebugFields = std::map; +using DebugMap = std::map; + +/* Result of a debug run. */ +struct DebugRun { + DebugMap ranks; /* parsed DEBUG lines, keyed by rank */ + int exit_code; /* WEXITSTATUS, or -signum if killed */ + bool normal_exit; /* true iff WIFEXITED */ + std::string stdout_raw; /* full stdout */ + std::string stderr_raw; /* full stderr (drained from temp file) */ + std::string command; /* the command line that was run */ +}; + +/* Parse a single DEBUG line into key/value fields. Returns rank, or -1 + * if the line does not begin with "DEBUG rank=". */ +inline int parse_debug_line(const std::string& line, DebugFields& out) +{ + auto p = line.find("DEBUG rank="); + if (p == std::string::npos) return -1; + p += strlen("DEBUG rank="); + int rank = 0; + size_t consumed = 0; + try { rank = std::stoi(line.substr(p), &consumed); } + catch (...) { return -1; } + out["rank"] = std::to_string(rank); + p += consumed; + /* parse key=val tokens separated by whitespace */ + while (p < line.size()) { + while (p < line.size() && isspace((unsigned char)line[p])) p++; + size_t eq = line.find('=', p); + if (eq == std::string::npos) break; + std::string k = line.substr(p, eq - p); + size_t v_start = eq + 1; + size_t v_end = v_start; + while (v_end < line.size() && !isspace((unsigned char)line[v_end])) v_end++; + std::string v = line.substr(v_start, v_end - v_start); + out[k] = v; + p = v_end; + } + return rank; +} + +/* Spawn the benchmark, capture stdout + stderr, parse DEBUG lines, decode + * the wait status, and return a DebugRun. This does NOT assert anything — + * callers should check `run.normal_exit && run.exit_code == 0` themselves + * and use gtest macros (so context strings can be tailored per call). */ +inline DebugRun run_debug_capture(const std::string& binary, + int nprocs, + const std::string& extra = "") +{ + DebugRun r; + char errfile_template[] = "/tmp/blink_test_err_XXXXXX"; + int fd = mkstemp(errfile_template); + std::string errfile = (fd >= 0) ? errfile_template : "/dev/null"; + if (fd >= 0) close(fd); + + r.command = mpiexec() + " " + nproc_flag() + " " + + std::to_string(nprocs) + " " + + bin_dir() + "/" + binary + + " -debug -iter 1 -warmup 0 " + extra + + " 2>" + errfile; + + FILE *pipe = popen(r.command.c_str(), "r"); + if (!pipe) { + r.exit_code = -1; + r.normal_exit = false; + return r; + } + + char line[1024]; + while (fgets(line, sizeof(line), pipe)) { + r.stdout_raw += line; + DebugFields fields; + int rank = parse_debug_line(line, fields); + if (rank >= 0) r.ranks[rank] = std::move(fields); + } + int status = pclose(pipe); + if (status == -1) { + r.exit_code = -1; + r.normal_exit = false; + } else if (WIFEXITED(status)) { + r.exit_code = WEXITSTATUS(status); + r.normal_exit = true; + } else if (WIFSIGNALED(status)) { + r.exit_code = -WTERMSIG(status); + r.normal_exit = false; + } else { + r.exit_code = status; + r.normal_exit = false; + } + + /* drain stderr from the temp file */ + if (errfile != "/dev/null") { + FILE *ef = fopen(errfile.c_str(), "r"); + if (ef) { + char buf[1024]; + size_t n; + while ((n = fread(buf, 1, sizeof(buf), ef)) > 0) + r.stderr_raw.append(buf, n); + fclose(ef); + } + unlink(errfile.c_str()); + } + return r; +} + +/* Convenience: assert the benchmark ran successfully and return its parsed + * DEBUG lines. On failure, dump captured stderr / stdout to gtest output. */ +inline DebugMap run_debug(const std::string& binary, + int nprocs, + const std::string& extra = "") +{ + DebugRun r = run_debug_capture(binary, nprocs, extra); + EXPECT_TRUE(r.normal_exit) + << "command: " << r.command << "\n" + << "exit_code: " << r.exit_code << "\n" + << "stderr:\n" << r.stderr_raw << "\n" + << "stdout:\n" << r.stdout_raw; + EXPECT_EQ(r.exit_code, 0) + << "command: " << r.command << "\n" + << "stderr:\n" << r.stderr_raw << "\n" + << "stdout:\n" << r.stdout_raw; + return r.ranks; +} + +/* Look up an integer field for the given rank. Returns -1 if absent. */ +inline int get_int(const DebugMap& m, int rank, const std::string& key, int def = -1) +{ + auto rit = m.find(rank); + if (rit == m.end()) return def; + auto fit = rit->second.find(key); + if (fit == rit->second.end()) return def; + try { return std::stoi(fit->second); } + catch (...) { return def; } +} + +/* Look up a string field for the given rank. Returns "" if absent. */ +inline std::string get_str(const DebugMap& m, int rank, const std::string& key) +{ + auto rit = m.find(rank); + if (rit == m.end()) return ""; + auto fit = rit->second.find(key); + if (fit == rit->second.end()) return ""; + return fit->second; +} + +/* Assert that every rank emitted a check= field AND reported check=OK. + * Requiring the field to be present (rather than silently skipping ranks that + * omit it) makes data-integrity tests fail loudly if a benchmark ever stops + * emitting its self-check, instead of passing vacuously. */ +inline void check_all_ok(const DebugMap& m, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + auto it = fields.find("check"); + EXPECT_NE(it, fields.end()) + << ctx << ": rank " << r << " emitted no check= field"; + if (it == fields.end()) continue; + EXPECT_EQ(it->second, "OK") + << ctx << ": rank " << r << " check=" << it->second; + } +} + +/* Assert that ranks 0..n-1 are all present in the map. */ +inline void check_coverage(const DebugMap& m, int n, const std::string& ctx) +{ + ASSERT_EQ((int)m.size(), n) + << ctx << ": expected " << n << " DEBUG lines, got " << m.size(); + for (int r = 0; r < n; r++) + EXPECT_TRUE(m.count(r)) << ctx << ": rank " << r << " missing"; +} + +/* Assert that every rank that emits nprocs= reports the expected value. */ +inline void check_nprocs(const DebugMap& m, int expected, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + auto it = fields.find("nprocs"); + if (it == fields.end()) continue; + EXPECT_EQ(std::stoi(it->second), expected) + << ctx << ": rank " << r << " nprocs=" << it->second + << " expected " << expected; + } +} + +} // namespace blink diff --git a/tests/test_collectives.cpp b/tests/test_collectives.cpp new file mode 100644 index 000000000..b7c6d6460 --- /dev/null +++ b/tests/test_collectives.cpp @@ -0,0 +1,216 @@ +/* + * test_collectives.cpp — correctness tests for collective benchmarks. + * + * Strategy: spawn the actual benchmark binary with -debug and parse + * "DEBUG rank=X nprocs=Y [root=Z] check=OK|FAIL" lines. + * No MPI in this binary. + * + * Properties checked: + * Coverage — every rank 0..N-1 appears exactly once. + * Nprocs — every rank reports the correct communicator size. + * Root — rooted collectives use the expected master_rank. + * NonDefaultRoot — -mrank flag correctly changes the root. + * DataIntegrity — every rank reports check=OK (data-integrity verified + * by the benchmark itself in debug mode). + * Agreement — _nb variant reports the same result as the _b variant. + * + * Data filled in debug mode (inside the benchmark, not here): + * allgather/alltoall fill send chunk with my_rank byte; verify recv[r*M+j]==r + * allreduce/reduce fill send with my_rank (int); verify SUM == N*(N-1)/2 + * reduce_scatter same; scatter distributes one block to each rank + * broadcast root fills with master_rank byte; all verify + * gather fill send with my_rank byte; root verifies recv[r*M+j]==r + * scatter root prepares chunk r=byte(r); each rank verifies recv[j]==my_rank + * barrier reaches the debug print ⟹ barrier completed (deadlock check) + * + * Requires 8 MPI ranks. + */ +#include +#include +#include "popen_helpers.h" + +using blink::DebugMap; +using blink::run_debug; +using blink::get_int; +using blink::get_str; +using blink::check_all_ok; +using blink::check_coverage; +using blink::check_nprocs; + +/* ── shared helpers ─────────────────────────────────────────────────────────── */ + +static void check_root(const DebugMap& m, int expected_root, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + auto it = fields.find("root"); + if (it == fields.end()) continue; + EXPECT_EQ(std::stoi(it->second), expected_root) + << ctx << ": rank " << r << " reports root=" << it->second; + } +} + +/* ── unrooted collectives ───────────────────────────────────────────────────── */ + +class UnrootedCollectiveTest : public ::testing::TestWithParam { +protected: + static constexpr int N = 8; +}; + +TEST_P(UnrootedCollectiveTest, Coverage) { + auto m = run_debug(GetParam(), N); + check_coverage(m, N, GetParam()); +} +TEST_P(UnrootedCollectiveTest, Nprocs) { + auto m = run_debug(GetParam(), N); + check_nprocs(m, N, GetParam()); +} +TEST_P(UnrootedCollectiveTest, DataIntegrity) { + auto m = run_debug(GetParam(), N); + check_coverage(m, N, GetParam()); + check_all_ok(m, GetParam()); +} +TEST_P(UnrootedCollectiveTest, DataIntegrity_burst) { + auto m = run_debug(GetParam(), N, "-blength 0.001"); + check_coverage(m, N, GetParam() + " burst"); + check_all_ok(m, GetParam() + " burst"); +} +/* Granularity batching (-grty > 1): the _nb / manual variants issue grty + * concurrent operations per timed window. Each must write a DISJOINT receive + * slot — a shared buffer would be an MPI overlap violation that can corrupt + * data. This exercises that path (default -grty 1 would never catch it). */ +TEST_P(UnrootedCollectiveTest, DataIntegrity_grty) { + auto m = run_debug(GetParam(), N, "-grty 4"); + check_coverage(m, N, GetParam() + " grty=4"); + check_all_ok(m, GetParam() + " grty=4"); +} +/* Randomised burst length (-bldist) + a pause (-bpause): exercises the + * if(burst_length_rand) / if(burst_pause!=0) / if(burst_pause_rand) branches in + * this benchmark's measurement loop, which the fixed -blength burst test misses. */ +TEST_P(UnrootedCollectiveTest, DataIntegrity_burstdist) { + auto m = run_debug(GetParam(), N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, GetParam() + " burstdist"); + check_all_ok(m, GetParam() + " burstdist"); +} + +INSTANTIATE_TEST_SUITE_P( + Collectives, UnrootedCollectiveTest, + ::testing::Values( + "allgather_b", "allgather_nb", "allgather_comm_only", + "allreduce_b", "allreduce_nb", + "alltoall_b", "alltoall_nb", "alltoall_man", "alltoall_comm_only", + "barrier_b", "barrier_nb", + "reduce_scatter_b", "reduce_scatter_nb" + ) +); + +/* ── rooted collectives ─────────────────────────────────────────────────────── */ + +class RootedCollectiveTest : public ::testing::TestWithParam { +protected: + static constexpr int N = 8; + static constexpr int MASTER = 0; + static constexpr int ALT = 3; +}; + +TEST_P(RootedCollectiveTest, Coverage) { + auto m = run_debug(GetParam(), N); + check_coverage(m, N, GetParam()); +} +TEST_P(RootedCollectiveTest, Nprocs) { + auto m = run_debug(GetParam(), N); + check_nprocs(m, N, GetParam()); +} +TEST_P(RootedCollectiveTest, DefaultRoot) { + auto m = run_debug(GetParam(), N); + check_root(m, MASTER, GetParam() + " default root"); +} +TEST_P(RootedCollectiveTest, DataIntegrity) { + auto m = run_debug(GetParam(), N); + check_coverage(m, N, GetParam()); + check_all_ok(m, GetParam()); +} +TEST_P(RootedCollectiveTest, DataIntegrity_burst) { + auto m = run_debug(GetParam(), N, "-blength 0.001"); + check_coverage(m, N, GetParam() + " burst"); + check_all_ok(m, GetParam() + " burst"); +} +/* Granularity batching (-grty > 1): grty concurrent rooted ops per window, + * each into a disjoint receive slot. Guards the buffer-indexing fix. */ +TEST_P(RootedCollectiveTest, DataIntegrity_grty) { + auto m = run_debug(GetParam(), N, "-grty 4"); + check_coverage(m, N, GetParam() + " grty=4"); + check_all_ok(m, GetParam() + " grty=4"); +} +/* Randomised burst + pause: see the unrooted note. */ +TEST_P(RootedCollectiveTest, DataIntegrity_burstdist) { + auto m = run_debug(GetParam(), N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, GetParam() + " burstdist"); + check_all_ok(m, GetParam() + " burstdist"); +} +TEST_P(RootedCollectiveTest, NonDefaultRoot) { + auto m = run_debug(GetParam(), N, "-mrank " + std::to_string(ALT)); + check_coverage(m, N, GetParam() + " mrank=3"); + check_nprocs(m, N, GetParam() + " mrank=3"); + check_root(m, ALT, GetParam() + " mrank=3"); + check_all_ok(m, GetParam() + " mrank=3"); +} + +INSTANTIATE_TEST_SUITE_P( + Collectives, RootedCollectiveTest, + ::testing::Values( + "broadcast_b", "broadcast_nb", + "gather_b", "gather_nb", + "scatter_b", "scatter_nb", + "reduce_b", "reduce_nb" + ) +); + +/* ── nb variants agree with b variants ─────────────────────────────────────── */ + +class CollectiveAgreementTest : public ::testing::Test { +protected: + static constexpr int N = 8; + + static void check_agreement(const std::string& b_bin, const std::string& nb_bin) { + auto b = run_debug(b_bin, N); + auto nb = run_debug(nb_bin, N); + for (const auto& [r, fields_b] : b) { + if (!nb.count(r)) continue; + EXPECT_EQ(get_int(nb, r, "nprocs"), get_int(b, r, "nprocs")) + << "rank " << r << " nprocs differs: " << b_bin << " vs " << nb_bin; + EXPECT_EQ(get_int(nb, r, "root"), get_int(b, r, "root")) + << "rank " << r << " root differs: " << b_bin << " vs " << nb_bin; + EXPECT_EQ(get_str(nb, r, "check"), get_str(b, r, "check")) + << "rank " << r << " check differs: " << b_bin << " vs " << nb_bin; + } + } +}; + +TEST_F(CollectiveAgreementTest, Allgather) { check_agreement("allgather_b", "allgather_nb"); } +TEST_F(CollectiveAgreementTest, Allreduce) { check_agreement("allreduce_b", "allreduce_nb"); } +TEST_F(CollectiveAgreementTest, Alltoall) { check_agreement("alltoall_b", "alltoall_nb"); } +TEST_F(CollectiveAgreementTest, Barrier) { check_agreement("barrier_b", "barrier_nb"); } +TEST_F(CollectiveAgreementTest, Broadcast) { check_agreement("broadcast_b", "broadcast_nb"); } +TEST_F(CollectiveAgreementTest, Gather) { check_agreement("gather_b", "gather_nb"); } +TEST_F(CollectiveAgreementTest, Scatter) { check_agreement("scatter_b", "scatter_nb"); } +TEST_F(CollectiveAgreementTest, Reduce) { check_agreement("reduce_b", "reduce_nb"); } +TEST_F(CollectiveAgreementTest, ReduceScatter) { check_agreement("reduce_scatter_b", "reduce_scatter_nb"); } + +/* ── transfer-volume agreement ─────────────────────────────────────────────── */ +/* For a given -msgsize, every allgather variant must move the SAME per-rank + * payload. Regression guard: allgather_comm_only previously treated -msgsize as + * the total gathered size (msg_size/w_size per rank) while allgather_b/nb treat + * it as the per-rank contribution, making their numbers non-comparable. The + * benchmarks now emit bytes= under -debug. */ +TEST_F(CollectiveAgreementTest, AllgatherTransferVolume) { + const int M = 2048; /* multiple of sizeof(int) for the comm_only variant */ + const std::string sz = "-msgsize " + std::to_string(M); + auto b = run_debug("allgather_b", N, sz); + auto nb = run_debug("allgather_nb", N, sz); + auto co = run_debug("allgather_comm_only", N, sz); + for (int r = 0; r < N; r++) { + EXPECT_EQ(get_int(b, r, "bytes"), M) << "allgather_b rank " << r; + EXPECT_EQ(get_int(nb, r, "bytes"), M) << "allgather_nb rank " << r; + EXPECT_EQ(get_int(co, r, "bytes"), M) << "allgather_comm_only rank " << r; + } +} diff --git a/tests/test_incast.cpp b/tests/test_incast.cpp new file mode 100644 index 000000000..7d64143d5 --- /dev/null +++ b/tests/test_incast.cpp @@ -0,0 +1,329 @@ +/* + * test_incast.cpp — correctness tests for incast benchmarks. + * + * Strategy: spawn the actual benchmark binary with -debug and parse: + * "DEBUG rank=X nprocs=N nsenders=K check=OK|FAIL" (receiver) + * "DEBUG rank=X nprocs=N target=R check=OK" (senders) + * No MPI in this binary. + * + * A rank is the receiver iff its parsed fields contain "nsenders". + * + * Properties checked: + * Coverage — every rank 0..N-1 appears exactly once. + * Roles — exactly one receiver (master_rank), N-1 senders. + * Targets — every sender targets master_rank. + * Count — receiver's nsenders equals N-1. + * NonDefault — -mrank flag correctly changes the receiver. + * DataIntegrity — every rank reports check=OK. + * Agreement — incast_nb / incast_bsnbr / incast_get / incast_put report + * the same topology as incast_b. + * + * Requires 8 MPI ranks. + */ +#include +#include +#include "popen_helpers.h" + +using blink::DebugMap; +using blink::DebugFields; +using blink::run_debug; +using blink::get_int; +using blink::get_str; +using blink::check_all_ok; +using blink::check_coverage; + +/* ── role helpers ───────────────────────────────────────────────────────────── */ + +static bool is_receiver(const DebugMap& m, int r) +{ + auto it = m.find(r); + if (it == m.end()) return false; + return it->second.count("nsenders") > 0; +} + +static void check_roles(const DebugMap& m, int n, int master, const std::string& ctx) +{ + int receiver_count = 0; + for (const auto& [r, fields] : m) { + if (fields.count("nsenders")) { + receiver_count++; + EXPECT_EQ(r, master) + << ctx << ": rank " << r << " is receiver but master_rank=" << master; + int nsenders = get_int(m, r, "nsenders"); + EXPECT_EQ(nsenders, n - 1) + << ctx << ": receiver reports nsenders=" << nsenders + << " expected " << (n - 1); + } else { + int target = get_int(m, r, "target"); + EXPECT_EQ(target, master) + << ctx << ": rank " << r << " sender targets " << target + << " expected master=" << master; + } + } + EXPECT_EQ(receiver_count, 1) << ctx << ": expected exactly 1 receiver"; +} + +/* ── incast_b ───────────────────────────────────────────────────────────────── */ + +class IncastBTest : public ::testing::Test { +protected: + static constexpr int N = 8; + static constexpr int MASTER = 0; +}; + +TEST_F(IncastBTest, Coverage) { + auto m = run_debug("incast_b", N); + check_coverage(m, N, "incast_b"); + check_all_ok(m, "incast_b"); +} +TEST_F(IncastBTest, Roles) { + auto m = run_debug("incast_b", N); + check_roles(m, N, MASTER, "incast_b"); +} +TEST_F(IncastBTest, Burst) { + auto m = run_debug("incast_b", N, "-blength 0.001"); + check_coverage(m, N, "incast_b burst"); + check_roles(m, N, MASTER, "incast_b burst"); + check_all_ok(m, "incast_b burst"); +} +TEST_F(IncastBTest, Burstdist) { + auto m = run_debug("incast_b", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "incast_b burstdist"); + check_roles(m, N, MASTER, "incast_b burstdist"); + check_all_ok(m, "incast_b burstdist"); +} +TEST_F(IncastBTest, NonDefaultMaster) { + const int ALT = 3; + auto m = run_debug("incast_b", N, "-mrank " + std::to_string(ALT)); + check_coverage(m, N, "incast_b mrank=3"); + check_roles(m, N, ALT, "incast_b mrank=3"); + check_all_ok(m, "incast_b mrank=3"); +} + +/* ── incast_nb ──────────────────────────────────────────────────────────────── */ + +class IncastNbTest : public ::testing::Test { +protected: + static constexpr int N = 8; + static constexpr int MASTER = 0; +}; + +TEST_F(IncastNbTest, Coverage) { + auto m = run_debug("incast_nb", N); + check_coverage(m, N, "incast_nb"); + check_all_ok(m, "incast_nb"); +} +TEST_F(IncastNbTest, Roles) { + auto m = run_debug("incast_nb", N); + check_roles(m, N, MASTER, "incast_nb"); +} +TEST_F(IncastNbTest, Burst) { + auto m = run_debug("incast_nb", N, "-blength 0.001"); + check_coverage(m, N, "incast_nb burst"); + check_roles(m, N, MASTER, "incast_nb burst"); + check_all_ok(m, "incast_nb burst"); +} +TEST_F(IncastNbTest, Burstdist) { + auto m = run_debug("incast_nb", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "incast_nb burstdist"); + check_roles(m, N, MASTER, "incast_nb burstdist"); + check_all_ok(m, "incast_nb burstdist"); +} +/* -grty>1: the root posts grty*(N-1) concurrent Irecvs, each into a disjoint + * slot. A shared slot would be an MPI overlap violation; this guards it. */ +TEST_F(IncastNbTest, Grty) { + auto m = run_debug("incast_nb", N, "-grty 4"); + check_coverage(m, N, "incast_nb grty=4"); + check_roles(m, N, MASTER, "incast_nb grty=4"); + check_all_ok(m, "incast_nb grty=4"); +} +TEST_F(IncastNbTest, NonDefaultMaster) { + const int ALT = 3; + auto m = run_debug("incast_nb", N, "-mrank " + std::to_string(ALT)); + check_coverage(m, N, "incast_nb mrank=3"); + check_roles(m, N, ALT, "incast_nb mrank=3"); + check_all_ok(m, "incast_nb mrank=3"); +} +TEST_F(IncastNbTest, AgreesWith_IncastB) { + auto b = run_debug("incast_b", N); + auto nb = run_debug("incast_nb", N); + for (const auto& [r, fields_b] : b) { + if (!nb.count(r)) continue; + bool b_rx = is_receiver(b, r); + bool nb_rx = is_receiver(nb, r); + EXPECT_EQ(nb_rx, b_rx) + << "rank " << r << " role differs between incast_b and incast_nb"; + if (!b_rx) + EXPECT_EQ(get_int(nb, r, "target"), get_int(b, r, "target")) + << "rank " << r << " target differs between incast_b and incast_nb"; + } +} + +/* ── incast_bsnbr ───────────────────────────────────────────────────────────── */ + +class IncastBsnbrTest : public ::testing::Test { +protected: + static constexpr int N = 8; + static constexpr int MASTER = 0; +}; + +TEST_F(IncastBsnbrTest, Coverage) { + auto m = run_debug("incast_bsnbr", N); + check_coverage(m, N, "incast_bsnbr"); + check_all_ok(m, "incast_bsnbr"); +} +TEST_F(IncastBsnbrTest, Roles) { + auto m = run_debug("incast_bsnbr", N); + check_roles(m, N, MASTER, "incast_bsnbr"); +} +TEST_F(IncastBsnbrTest, Burst) { + auto m = run_debug("incast_bsnbr", N, "-blength 0.001"); + check_coverage(m, N, "incast_bsnbr burst"); + check_roles(m, N, MASTER, "incast_bsnbr burst"); + check_all_ok(m, "incast_bsnbr burst"); +} +TEST_F(IncastBsnbrTest, Burstdist) { + auto m = run_debug("incast_bsnbr", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "incast_bsnbr burstdist"); + check_roles(m, N, MASTER, "incast_bsnbr burstdist"); + check_all_ok(m, "incast_bsnbr burstdist"); +} +TEST_F(IncastBsnbrTest, NonDefaultMaster) { + const int ALT = 3; + auto m = run_debug("incast_bsnbr", N, "-mrank " + std::to_string(ALT)); + check_coverage(m, N, "incast_bsnbr mrank=3"); + check_roles(m, N, ALT, "incast_bsnbr mrank=3"); + check_all_ok(m, "incast_bsnbr mrank=3"); +} +TEST_F(IncastBsnbrTest, AgreesWith_IncastB) { + auto b = run_debug("incast_b", N); + auto bsnbr = run_debug("incast_bsnbr", N); + for (const auto& [r, fields_b] : b) { + if (!bsnbr.count(r)) continue; + bool b_rx = is_receiver(b, r); + bool bs_rx = is_receiver(bsnbr, r); + EXPECT_EQ(bs_rx, b_rx) + << "rank " << r << " role differs between incast_b and incast_bsnbr"; + if (!b_rx) + EXPECT_EQ(get_int(bsnbr, r, "target"), get_int(b, r, "target")) + << "rank " << r << " target differs between incast_b and incast_bsnbr"; + } +} + +/* ── incast_get ─────────────────────────────────────────────────────────────── */ + +class IncastGetTest : public ::testing::Test { +protected: + static constexpr int N = 8; + static constexpr int MASTER = 0; +}; + +TEST_F(IncastGetTest, Coverage) { + auto m = run_debug("incast_get", N); + check_coverage(m, N, "incast_get"); + check_all_ok(m, "incast_get"); +} +TEST_F(IncastGetTest, Roles) { + auto m = run_debug("incast_get", N); + check_roles(m, N, MASTER, "incast_get"); +} +TEST_F(IncastGetTest, Burst) { + auto m = run_debug("incast_get", N, "-blength 0.001"); + check_coverage(m, N, "incast_get burst"); + check_roles(m, N, MASTER, "incast_get burst"); + check_all_ok(m, "incast_get burst"); +} +TEST_F(IncastGetTest, Burstdist) { + auto m = run_debug("incast_get", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "incast_get burstdist"); + check_roles(m, N, MASTER, "incast_get burstdist"); + check_all_ok(m, "incast_get burstdist"); +} +/* -grty>1: the root issues grty Gets per sender into distinct window-offset + * slots; guards the RMA displacement math (j*M*grty + i*M). */ +TEST_F(IncastGetTest, Grty) { + auto m = run_debug("incast_get", N, "-grty 4"); + check_coverage(m, N, "incast_get grty=4"); + check_roles(m, N, MASTER, "incast_get grty=4"); + check_all_ok(m, "incast_get grty=4"); +} +TEST_F(IncastGetTest, NonDefaultMaster) { + const int ALT = 3; + auto m = run_debug("incast_get", N, "-mrank " + std::to_string(ALT)); + check_coverage(m, N, "incast_get mrank=3"); + check_roles(m, N, ALT, "incast_get mrank=3"); + check_all_ok(m, "incast_get mrank=3"); +} +TEST_F(IncastGetTest, AgreesWith_IncastB) { + auto b = run_debug("incast_b", N); + auto get = run_debug("incast_get", N); + for (const auto& [r, fields_b] : b) { + if (!get.count(r)) continue; + bool b_rx = is_receiver(b, r); + bool g_rx = is_receiver(get, r); + EXPECT_EQ(g_rx, b_rx) + << "rank " << r << " role differs between incast_b and incast_get"; + if (!b_rx) + EXPECT_EQ(get_int(get, r, "target"), get_int(b, r, "target")) + << "rank " << r << " target differs between incast_b and incast_get"; + } +} + +/* ── incast_put ─────────────────────────────────────────────────────────────── */ + +class IncastPutTest : public ::testing::Test { +protected: + static constexpr int N = 8; + static constexpr int MASTER = 0; +}; + +TEST_F(IncastPutTest, Coverage) { + auto m = run_debug("incast_put", N); + check_coverage(m, N, "incast_put"); + check_all_ok(m, "incast_put"); +} +TEST_F(IncastPutTest, Roles) { + auto m = run_debug("incast_put", N); + check_roles(m, N, MASTER, "incast_put"); +} +TEST_F(IncastPutTest, Burst) { + auto m = run_debug("incast_put", N, "-blength 0.001"); + check_coverage(m, N, "incast_put burst"); + check_roles(m, N, MASTER, "incast_put burst"); + check_all_ok(m, "incast_put burst"); +} +TEST_F(IncastPutTest, Burstdist) { + auto m = run_debug("incast_put", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "incast_put burstdist"); + check_roles(m, N, MASTER, "incast_put burstdist"); + check_all_ok(m, "incast_put burstdist"); +} +/* -grty>1: each sender issues grty Puts into distinct window-offset slots; + * guards the RMA displacement math (rank*M*grty + i*M). */ +TEST_F(IncastPutTest, Grty) { + auto m = run_debug("incast_put", N, "-grty 4"); + check_coverage(m, N, "incast_put grty=4"); + check_roles(m, N, MASTER, "incast_put grty=4"); + check_all_ok(m, "incast_put grty=4"); +} +TEST_F(IncastPutTest, NonDefaultMaster) { + const int ALT = 3; + auto m = run_debug("incast_put", N, "-mrank " + std::to_string(ALT)); + check_coverage(m, N, "incast_put mrank=3"); + check_roles(m, N, ALT, "incast_put mrank=3"); + check_all_ok(m, "incast_put mrank=3"); +} +TEST_F(IncastPutTest, AgreesWith_IncastB) { + auto b = run_debug("incast_b", N); + auto put = run_debug("incast_put", N); + for (const auto& [r, fields_b] : b) { + if (!put.count(r)) continue; + bool b_rx = is_receiver(b, r); + bool p_rx = is_receiver(put, r); + EXPECT_EQ(p_rx, b_rx) + << "rank " << r << " role differs between incast_b and incast_put"; + if (!b_rx) + EXPECT_EQ(get_int(put, r, "target"), get_int(b, r, "target")) + << "rank " << r << " target differs between incast_b and incast_put"; + } +} diff --git a/tests/test_kpartners.cpp b/tests/test_kpartners.cpp new file mode 100644 index 000000000..e94301039 --- /dev/null +++ b/tests/test_kpartners.cpp @@ -0,0 +1,77 @@ +/* + * test_kpartners.cpp — correctness tests for kpartners_nb. + * + * Strategy: spawn kpartners_nb with -debug and parse + * "DEBUG rank=X nprocs=Y k=K check=OK|FAIL" lines. + * + * Properties checked: + * Coverage — every rank 0..N-1 appears exactly once. + * Nprocs — every rank reports the correct communicator size. + * K — every rank reports the expected number of partners. + * DataIntegrity — every rank reports check=OK. In the new schema this is + * a meaningful check that partner identity was honoured. + * + * Requires 8 MPI ranks. + */ +#include +#include +#include "popen_helpers.h" + +using blink::DebugMap; +using blink::run_debug; +using blink::get_int; +using blink::check_all_ok; +using blink::check_coverage; +using blink::check_nprocs; + +/* ── helpers ────────────────────────────────────────────────────────────────── */ + +static void check_k(const DebugMap& m, int expected_k, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + int k = get_int(m, r, "k"); + EXPECT_EQ(k, expected_k) + << ctx << ": rank " << r << " reports k=" << k; + } +} + +/* ── tests ──────────────────────────────────────────────────────────────────── */ + +class KpartnersTest : public ::testing::Test { +protected: + static constexpr int N = 8; +}; + +TEST_F(KpartnersTest, Coverage) { + auto m = run_debug("kpartners_nb", N); + check_coverage(m, N, "k=1"); +} +TEST_F(KpartnersTest, Nprocs) { + auto m = run_debug("kpartners_nb", N); + check_nprocs(m, N, "k=1"); +} +TEST_F(KpartnersTest, K_default) { + auto m = run_debug("kpartners_nb", N); + check_k(m, 1, "k=1"); +} +TEST_F(KpartnersTest, DataIntegrity_k1) { + auto m = run_debug("kpartners_nb", N); + check_coverage(m, N, "k=1"); + check_all_ok(m, "k=1"); +} +TEST_F(KpartnersTest, DataIntegrity_k3) { + auto m = run_debug("kpartners_nb", N, "-k 3"); + check_coverage(m, N, "k=3"); + check_k(m, 3, "k=3"); + check_all_ok(m, "k=3"); +} +TEST_F(KpartnersTest, DataIntegrity_burst) { + auto m = run_debug("kpartners_nb", N, "-blength 0.001"); + check_coverage(m, N, "k=1 burst"); + check_all_ok(m, "k=1 burst"); +} +TEST_F(KpartnersTest, DataIntegrity_burstdist) { + auto m = run_debug("kpartners_nb", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "k=1 burstdist"); + check_all_ok(m, "k=1 burstdist"); +} diff --git a/tests/test_misc.cpp b/tests/test_misc.cpp new file mode 100644 index 000000000..7b7c30b59 --- /dev/null +++ b/tests/test_misc.cpp @@ -0,0 +1,433 @@ +/* + * test_misc.cpp — miscellaneous correctness tests: + * - ring-buffer wrap (iter > maxsamples) + * - pretty-print formatter doesn't crash and produces a recognisable table + * - -mrand picks a deterministic master rank for a given seed + * - sampler/dist_test runs (validates the burst distribution implementation) + * + * All tests use 8 ranks except where noted. + */ +#include +#include +#include +#include +#include +#include /* std::count */ +#include "popen_helpers.h" + +using blink::run_debug_capture; +using blink::DebugRun; + +/* Helper: count lines in s containing a substring */ +static int count_lines_containing(const std::string& s, const std::string& needle) +{ + int n = 0; + size_t p = 0; + while ((p = s.find(needle, p)) != std::string::npos) { n++; p += needle.size(); } + return n; +} + +/* ── ring-buffer wrap ───────────────────────────────────────────────────────── */ + +/* + * Configure max_samples to be smaller than the iteration count, ensuring the + * ring buffer wraps. The benchmark should still print exactly max_samples + * "Average,..." data lines (one per recorded sample), and the summary line + * should report "Measured " iterations. + */ +TEST(RingBufferWrap, BarrierWrapsCorrectly) +{ + /* run barrier_nb (cheap) with iter=50 maxsamples=10 warmup=0 — buffer + * should hold the last 10 samples. */ + blink::DebugRun r = run_debug_capture("barrier_nb", 8, + "-iter 50 -maxsamples 10 -warmup 0"); + /* run_debug_capture does NOT assert; do the assertions explicitly */ + ASSERT_TRUE(r.normal_exit) << "stderr:\n" << r.stderr_raw; + ASSERT_EQ(r.exit_code, 0) << "stderr:\n" << r.stderr_raw; + + /* the master rank's CSV body has lines of the form "%.9f,%.9f,..." */ + int data_lines = 0; + size_t p = 0; + while ((p = r.stdout_raw.find('\n', p)) != std::string::npos) { + /* heuristic: a data line starts with a digit and has 4 commas */ + size_t line_start = r.stdout_raw.rfind('\n', p > 0 ? p - 1 : 0); + size_t s = (line_start == std::string::npos) ? 0 : line_start + 1; + std::string line = r.stdout_raw.substr(s, p - s); + if (!line.empty() && isdigit((unsigned char)line[0])) + if (std::count(line.begin(), line.end(), ',') == 4) + data_lines++; + p++; + } + EXPECT_EQ(data_lines, 10) + << "ring buffer wrap: expected 10 measured samples (maxsamples), got " + << data_lines << "\nstdout:\n" << r.stdout_raw; + + EXPECT_NE(r.stdout_raw.find("Ran 50 iterations. Measured 10 iterations."), + std::string::npos) + << "summary line mismatch.\nstdout:\n" << r.stdout_raw; +} + +/* ── pretty-print formatter ─────────────────────────────────────────────────── */ + +TEST(PrettyPrint, FormatterDoesNotCrash) +{ + DebugRun r = run_debug_capture("barrier_nb", 8, "-iter 3 -pretty-print"); + ASSERT_TRUE(r.normal_exit) << "stderr:\n" << r.stderr_raw; + ASSERT_EQ(r.exit_code, 0); + /* pretty-print uses a column-header row and a separator line */ + EXPECT_NE(r.stdout_raw.find("avg"), std::string::npos); + EXPECT_NE(r.stdout_raw.find("median"), std::string::npos); + EXPECT_NE(r.stdout_raw.find("samples"),std::string::npos); +} + +/* ── -mrand determinism ─────────────────────────────────────────────────────── */ + +TEST(MasterRand, SameSeedSameMaster) +{ + /* with -mrand and a fixed seed, the chosen master_rank should be + * deterministic across runs. We grep the "receiver rank: R" line from + * incast_b's own startup banner. */ + DebugRun r1 = run_debug_capture("incast_b", 8, "-mrand -seed 42"); + DebugRun r2 = run_debug_capture("incast_b", 8, "-mrand -seed 42"); + ASSERT_TRUE(r1.normal_exit); + ASSERT_TRUE(r2.normal_exit); + auto p1 = r1.stdout_raw.find("receiver rank:"); + auto p2 = r2.stdout_raw.find("receiver rank:"); + ASSERT_NE(p1, std::string::npos) << r1.stdout_raw; + ASSERT_NE(p2, std::string::npos) << r2.stdout_raw; + int m1, m2; + sscanf(r1.stdout_raw.c_str() + p1, "receiver rank: %d", &m1); + sscanf(r2.stdout_raw.c_str() + p2, "receiver rank: %d", &m2); + EXPECT_EQ(m1, m2) + << "same -seed should give same -mrand master, got " << m1 << " vs " << m2; +} + +/* ── dist_test sampler validation ───────────────────────────────────────────── */ + +TEST(SamplerValidation, DistTestRuns) +{ + /* dist_test is single-rank. If it exits 0 the sampler implementations + * are internally consistent (mean / KS test). */ + DebugRun r = run_debug_capture("dist_test", 1, ""); + /* dist_test may not accept -debug; treat absence of failure as success */ + ASSERT_EQ(r.exit_code, 0) + << "dist_test failed.\nstderr:\n" << r.stderr_raw + << "\nstdout:\n" << r.stdout_raw; +} + +/* ── warm-up exclusion ──────────────────────────────────────────────────────── */ + +/* + * Warm-up iterations must NOT be recorded. With -iter 5 -warmup 3 the benchmark + * runs 8 outer iterations but records only the last 5; the summary must report + * "Ran 8 iterations. Measured 5 iterations." and emit exactly 5 CSV data lines. + */ +TEST(WarmupExclusion, WarmupSamplesNotRecorded) +{ + DebugRun r = run_debug_capture("barrier_nb", 8, "-iter 5 -warmup 3 -maxsamples 1000"); + ASSERT_TRUE(r.normal_exit) << "stderr:\n" << r.stderr_raw; + ASSERT_EQ(r.exit_code, 0) << "stderr:\n" << r.stderr_raw; + + int data_lines = 0; + size_t p = 0; + while ((p = r.stdout_raw.find('\n', p)) != std::string::npos) { + size_t line_start = r.stdout_raw.rfind('\n', p > 0 ? p - 1 : 0); + size_t s = (line_start == std::string::npos) ? 0 : line_start + 1; + std::string line = r.stdout_raw.substr(s, p - s); + if (!line.empty() && isdigit((unsigned char)line[0])) + if (std::count(line.begin(), line.end(), ',') == 4) + data_lines++; + p++; + } + EXPECT_EQ(data_lines, 5) + << "expected 5 recorded samples (3 warmup excluded), got " << data_lines + << "\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stdout_raw.find("Ran 8 iterations. Measured 5 iterations."), + std::string::npos) + << "summary should report 8 run / 5 measured.\nstdout:\n" << r.stdout_raw; +} + +/* ── SIGUSR1 clean shutdown ─────────────────────────────────────────────────── */ + +/* + * Regression test for the endless-mode SIGUSR1 shutdown. The shutdown flag is + * set only on the rank that receives the signal, so all ranks must agree + * collectively (check_shutdown) to leave the measurement loop on the same + * iteration. Sending SIGUSR1 to exactly ONE rank of an endless run must shut + * the whole job down cleanly (no deadlock) and still print collected results. + * + * Orchestrated in POSIX sh via popen: launch the endless run in the background, + * signal one rank (a process whose comm is exactly "alltoall_nb" — i.e. a rank, + * not the launcher), then wait up to 30s for a clean exit, killing it if it + * hangs so the test itself can never block indefinitely. + */ +TEST(Shutdown, Sigusr1ToSingleRankShutsDownCleanly) +{ + char outtmpl[] = "/tmp/blink_endl_XXXXXX"; + int ofd = mkstemp(outtmpl); + ASSERT_GE(ofd, 0); + close(ofd); + std::string outfile = outtmpl; + std::string bin = blink::bin_dir() + "/alltoall_nb"; + + std::string cmd = + "\"" + blink::mpiexec() + "\" " + blink::nproc_flag() + " 4 \"" + bin + "\"" + " -endl -msgsize 1024 -seed 999983 > \"" + outfile + "\" 2>&1 &\n" + "MPIPID=$!\n" + "sleep 3\n" + "RP=\"\"\n" + "for q in $(pgrep -f 'alltoall_nb -endl'); do\n" + " if [ -r /proc/$q/comm ] && [ \"$(cat /proc/$q/comm)\" = alltoall_nb ]; then RP=$q; break; fi\n" + "done\n" + "[ -n \"$RP\" ] && kill -USR1 \"$RP\"\n" + "CLEAN=HANG\n" + "for i in $(seq 1 30); do kill -0 \"$MPIPID\" 2>/dev/null || { CLEAN=CLEAN; break; }; sleep 1; done\n" + "if [ \"$CLEAN\" = HANG ]; then kill -9 \"$MPIPID\" 2>/dev/null; pkill -9 -f 'alltoall_nb -endl' 2>/dev/null; fi\n" + "echo SIGNALED=$RP\n" + "echo RESULT=$CLEAN\n"; + + FILE *pp = popen(cmd.c_str(), "r"); + ASSERT_NE(pp, nullptr); + std::string out; + char buf[512]; + size_t n; + while ((n = fread(buf, 1, sizeof(buf), pp)) > 0) out.append(buf, n); + pclose(pp); + + std::string bench; + FILE *of = fopen(outfile.c_str(), "r"); + if (of) { while ((n = fread(buf, 1, sizeof(buf), of)) > 0) bench.append(buf, n); fclose(of); } + unlink(outfile.c_str()); + + EXPECT_EQ(out.find("SIGNALED=\n"), std::string::npos) + << "no rank process was found to signal.\norchestration:\n" << out; + EXPECT_NE(out.find("RESULT=CLEAN"), std::string::npos) + << "endless job did not shut down after SIGUSR1 to one rank (deadlock?).\n" + << "orchestration:\n" << out << "\nbenchmark output:\n" << bench; + EXPECT_NE(bench.find("Measured"), std::string::npos) + << "no results footer after clean shutdown.\nbenchmark output:\n" << bench; +} + +/* ── CLI validation guards ──────────────────────────────────────────────────── */ + +/* + * An out-of-range -mrank must fail loudly (clean MPI_Abort with a diagnostic), + * never crash or run with an invalid collective root. Regression guard for the + * master_rank range check added to parse_common_args(). + */ +TEST(CliGuards, OutOfRangeMrankAbortsCleanly) +{ + DebugRun r = run_debug_capture("alltoall_b", 8, "-mrank 99"); + EXPECT_FALSE(r.normal_exit && r.exit_code == 0) + << "out-of-range -mrank should abort, but the run exited 0.\n" + << "stdout:\n" << r.stdout_raw; + EXPECT_NE(r.stderr_raw.find("mrank must be in"), std::string::npos) + << "expected an -mrank range diagnostic on stderr.\nstderr:\n" << r.stderr_raw; +} + +/* + * A value-taking flag supplied as the final token must be rejected with a + * "Missing value" message rather than dereferencing argv[argc] (== NULL). + * Regression guard for arg_value() and the benchmark-specific parsers that + * now route through it. + */ +TEST(CliGuards, MissingFlagValueAbortsCleanly) +{ + DebugRun r = run_debug_capture("pairwise_b", 8, "-mode"); + EXPECT_FALSE(r.normal_exit && r.exit_code == 0) + << "missing -mode value should abort, but the run exited 0.\n" + << "stdout:\n" << r.stdout_raw; + EXPECT_NE(r.stderr_raw.find("Missing value for option -mode"), std::string::npos) + << "expected a 'Missing value' diagnostic on stderr.\nstderr:\n" << r.stderr_raw; +} + +/* ── -plot distribution histogram ───────────────────────────────────────────── */ + +TEST(Plot, RendersHistogramAdditively) +{ + DebugRun r = run_debug_capture("alltoall_nb", 8, "-iter 100 -plot"); + ASSERT_TRUE(r.normal_exit) << "stderr:\n" << r.stderr_raw; + ASSERT_EQ(r.exit_code, 0) << "stderr:\n" << r.stderr_raw; + EXPECT_NE(r.stdout_raw.find("Per-iteration latency"), std::string::npos) + << "no plot header.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stdout_raw.find("stat=max"), std::string::npos) /* default stat */ + << "default -plotstat should be max.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stdout_raw.find("Measured"), std::string::npos) /* additive */ + << "plot should be additive (the listing is still printed).\nstdout:\n" << r.stdout_raw; +} + +TEST(Plot, StatAndBinningSelectors) +{ + DebugRun avg = run_debug_capture("alltoall_nb", 8, "-iter 100 -plot -plotstat avg"); + EXPECT_EQ(avg.exit_code, 0) << avg.stderr_raw; + EXPECT_NE(avg.stdout_raw.find("stat=avg"), std::string::npos) << avg.stdout_raw; + + DebugRun lg = run_debug_capture("alltoall_nb", 8, "-iter 100 -plot -plotlog"); + EXPECT_EQ(lg.exit_code, 0) << lg.stderr_raw; + EXPECT_NE(lg.stdout_raw.find("log bins"), std::string::npos) << lg.stdout_raw; + + DebugRun fx = run_debug_capture("alltoall_nb", 8, "-iter 100 -plot -plotbinsize 1us"); + EXPECT_EQ(fx.exit_code, 0) << fx.stderr_raw; + EXPECT_NE(fx.stdout_raw.find("fixed bins"), std::string::npos) << fx.stdout_raw; +} + +TEST(Plot, InvalidStatAbortsCleanly) +{ + DebugRun r = run_debug_capture("alltoall_nb", 8, "-plot -plotstat bogus"); + EXPECT_FALSE(r.normal_exit && r.exit_code == 0) + << "invalid -plotstat should abort.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stderr_raw.find("plotstat must be one of"), std::string::npos) + << "expected a -plotstat diagnostic.\nstderr:\n" << r.stderr_raw; +} + +TEST(Plot, BinsizeWithLogConflictAborts) +{ + DebugRun r = run_debug_capture("alltoall_nb", 8, "-plot -plotbinsize 1us -plotlog"); + EXPECT_FALSE(r.normal_exit && r.exit_code == 0) + << "combining -plotbinsize with -plotlog should abort.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stderr_raw.find("cannot be combined with -plotlog"), std::string::npos) + << "expected a conflict diagnostic.\nstderr:\n" << r.stderr_raw; +} + +/* ── burst / pause distribution sampling ───────────────────────────────────── + * Exercises the randomised inter-arrival path from the benchmark side: + * sample_burst_length/sample_pause_length -> rand_duration -> rand_{exp,pareto, + * lognormal}, plus dsleep(). barrier_nb is the cheapest benchmark to drive it. */ +TEST(BurstSampling, RandomisedBurstAndPause) +{ + for (const char *d : {"exp", "pareto", "lognormal"}) { + std::string extra = std::string("-iter 5 -blength 0.0004 -bldist ") + d; + if (std::string(d) != "exp") extra += " -blshape 1.6"; /* shape ignored for exp */ + DebugRun r = run_debug_capture("barrier_nb", 8, extra); + EXPECT_TRUE(r.normal_exit && r.exit_code == 0) + << "barrier_nb -bldist " << d << " failed.\nstderr:\n" << r.stderr_raw; + } + /* randomised pause drives dsleep() + sample_pause_length() */ + DebugRun p = run_debug_capture("barrier_nb", 8, "-iter 4 -bpause 0.0004 -bpdist exp"); + EXPECT_TRUE(p.normal_exit && p.exit_code == 0) + << "barrier_nb -bpdist failed.\nstderr:\n" << p.stderr_raw; +} + +TEST(BurstSampling, InvalidShapeAbortsCleanly) +{ + DebugRun r = run_debug_capture("barrier_nb", 8, "-blength 0.001 -bldist pareto -blshape 0.5"); + EXPECT_FALSE(r.normal_exit && r.exit_code == 0) + << "a Pareto shape <= 1 should abort.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stderr_raw.find("pareto shape"), std::string::npos) + << "expected a pareto-shape diagnostic.\nstderr:\n" << r.stderr_raw; +} + +/* + * Duration unit suffixes (s, ms, us, ns) — previously dropped silently by atof + * (so "-blength 1ms" became 1 SECOND, not 1 ms). parse_duration_arg() now + * recognises them; a bare number is still seconds for backwards compatibility. + */ +TEST(BurstSampling, DurationSuffixesAccepted) +{ + DebugRun r = run_debug_capture("barrier_nb", 4, + "-iter 3 -blength 400us -bpause 300us -bldist exp -bpdist exp"); + EXPECT_TRUE(r.normal_exit && r.exit_code == 0) + << "barrier_nb with unit-suffix durations failed.\nstderr:\n" << r.stderr_raw; +} + +TEST(BurstSampling, InvalidDurationAbortsCleanly) +{ + /* old atof would have silently returned 0.0; we now abort instead. */ + DebugRun r = run_debug_capture("barrier_nb", 2, "-blength garbage"); + EXPECT_FALSE(r.normal_exit && r.exit_code == 0) + << "non-numeric -blength should abort.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stderr_raw.find("must be a non-negative duration"), std::string::npos) + << "expected a duration diagnostic.\nstderr:\n" << r.stderr_raw; +} + +/* ── -h / --help / -help ──────────────────────────────────────────────────── + * + * The help system: weak `benchmark_help` symbol in common.c overridden by + * benchmarks with custom flags (pairwise, kpartners, ring, stencil). + * parse_common_args handles -h/-help/--help by printing on rank 0 and calling + * MPI_Finalize + exit(0) on every rank. + */ + +TEST(Help, CommonHelpPrintsAndExitsZero) +{ + /* barrier_nb has NO custom args — only the common block should appear */ + DebugRun r = run_debug_capture("barrier_nb", 2, "-h"); + ASSERT_TRUE(r.normal_exit) << "stderr:\n" << r.stderr_raw; + EXPECT_EQ(r.exit_code, 0) << "stderr:\n" << r.stderr_raw; + EXPECT_NE(r.stdout_raw.find("Usage: barrier_nb"), std::string::npos) + << "expected 'Usage: barrier_nb'.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stdout_raw.find("-mrank"), std::string::npos) + << "common help should list -mrank.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stdout_raw.find("-blength"), std::string::npos) + << "common help should list burst flags.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stdout_raw.find("-plot"), std::string::npos) + << "common help should list plot flags.\nstdout:\n" << r.stdout_raw; + /* benchmarks without custom args must NOT show the benchmark-specific section */ + EXPECT_EQ(r.stdout_raw.find("Benchmark-specific options"), std::string::npos) + << "barrier_nb has no custom args — the section should be absent.\nstdout:\n" + << r.stdout_raw; +} + +TEST(Help, BenchmarkSpecificSectionAppears) +{ + /* pairwise_b defines benchmark_help with -mode and -offset */ + DebugRun r = run_debug_capture("pairwise_b", 8, "--help"); + ASSERT_TRUE(r.normal_exit) << "stderr:\n" << r.stderr_raw; + EXPECT_EQ(r.exit_code, 0) << "stderr:\n" << r.stderr_raw; + EXPECT_NE(r.stdout_raw.find("Benchmark-specific options"), std::string::npos) + << "pairwise_b --help should include the benchmark-specific section.\nstdout:\n" + << r.stdout_raw; + EXPECT_NE(r.stdout_raw.find("-mode"), std::string::npos) + << "pairwise_b --help should mention -mode.\nstdout:\n" << r.stdout_raw; + EXPECT_NE(r.stdout_raw.find("-offset"), std::string::npos) + << "pairwise_b --help should mention -offset.\nstdout:\n" << r.stdout_raw; +} + +TEST(Help, AllThreeFlagFormsAccepted) +{ + for (const char *flag : {"-h", "-help", "--help"}) { + DebugRun r = run_debug_capture("barrier_nb", 2, flag); + EXPECT_TRUE(r.normal_exit && r.exit_code == 0) + << "flag '" << flag << "' should print help and exit 0.\nstderr:\n" + << r.stderr_raw; + EXPECT_NE(r.stdout_raw.find("Usage:"), std::string::npos) + << "flag '" << flag << "' — expected 'Usage:' in stdout.\nstdout:\n" + << r.stdout_raw; + } +} + +TEST(Help, OnlyMasterRankPrints) +{ + /* With -np 4, help must appear exactly once, not 4×. Catches a regression + * where print_help would skip its rank-0 guard. */ + DebugRun r = run_debug_capture("barrier_nb", 4, "-h"); + EXPECT_EQ(r.exit_code, 0) << "stderr:\n" << r.stderr_raw; + int usage_count = 0; + size_t p = 0; + while ((p = r.stdout_raw.find("Usage:", p)) != std::string::npos) { + usage_count++; + p += 6; + } + EXPECT_EQ(usage_count, 1) + << "help should print once (rank 0 only), got " << usage_count + << " copies.\nstdout:\n" << r.stdout_raw; +} + +/* ── auxiliary tools (checker, null_dummy) ──────────────────────────────────── */ + +TEST(Tools, CheckerRuns) +{ + /* checker = all-to-all plus a per-iteration timestamp log; just exercise it. */ + DebugRun r = run_debug_capture("checker", 8, "-iter 10"); + EXPECT_TRUE(r.normal_exit) << "stderr:\n" << r.stderr_raw; + EXPECT_EQ(r.exit_code, 0) << "stderr:\n" << r.stderr_raw; + EXPECT_NE(r.stdout_raw.find("Measured"), std::string::npos) + << "checker produced no results footer.\nstdout:\n" << r.stdout_raw; +} + +TEST(Tools, NullDummyRuns) +{ + DebugRun r = run_debug_capture("null_dummy", 1, ""); + EXPECT_TRUE(r.normal_exit) << "stderr:\n" << r.stderr_raw; + EXPECT_EQ(r.exit_code, 0) << "stderr:\n" << r.stderr_raw; +} diff --git a/tests/test_pairwise.cpp b/tests/test_pairwise.cpp new file mode 100644 index 000000000..b23c6d62f --- /dev/null +++ b/tests/test_pairwise.cpp @@ -0,0 +1,266 @@ +/* + * test_pairwise.cpp — correctness tests for pairwise benchmarks. + * + * Strategy: spawn the *actual* benchmark binary under mpirun with -debug, + * parse the "DEBUG rank=X nprocs=N partner=Y check=OK|FAIL" lines it emits, + * and assert the pairing properties below. Breaking pairwise_b.c / + * pairwise_nb.c / pairwise_bsnbr.c will break these tests. + * + * Properties checked: + * Coverage — every rank 0..N-1 appears exactly once in the output. + * Validity — every reported partner is in [0, N-1]. + * Symmetry — A→B implies B→A (holds for offpair and rpair modes). + * Bijection — no partner value appears more than once. + * ExactPairs — for deterministic modes we verify the specific expected mapping. + * DataIntegrity — every rank reports check=OK. + */ +#include +#include +#include +#include +#include "popen_helpers.h" + +using blink::DebugMap; +using blink::run_debug; +using blink::get_int; +using blink::check_all_ok; +using blink::check_coverage; + +/* ── shared assertion helpers ───────────────────────────────────────────────── */ + +static void check_validity(const DebugMap& m, int n, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + int p = get_int(m, r, "partner"); + EXPECT_GE(p, 0) << ctx << ": rank " << r << " has partner " << p << " < 0"; + EXPECT_LT(p, n) << ctx << ": rank " << r << " has partner " << p << " >= " << n; + } +} + +static void check_symmetric(const DebugMap& m, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + int p = get_int(m, r, "partner"); + auto it = m.find(p); + if (it == m.end()) continue; /* coverage failures reported elsewhere */ + int pp = get_int(m, p, "partner"); + EXPECT_EQ(pp, r) + << ctx << ": rank " << r << "→" << p + << " but rank " << p << "→" << pp << " (not symmetric)"; + } +} + +static void check_bijection(const DebugMap& m, const std::string& ctx) +{ + std::set seen; + for (const auto& [r, fields] : m) { + int p = get_int(m, r, "partner"); + EXPECT_TRUE(seen.insert(p).second) + << ctx << ": partner " << p << " assigned to more than one rank"; + } +} + +static void check_exact(const DebugMap& m, + const std::map& expected, + const std::string& ctx) +{ + for (const auto& [r, p] : expected) { + auto it = m.find(r); + if (it == m.end()) continue; /* coverage failures reported elsewhere */ + int got = get_int(m, r, "partner"); + EXPECT_EQ(got, p) + << ctx << ": rank " << r << " expected partner " << p + << " but got " << got; + } +} + +/* ── pairwise_b ─────────────────────────────────────────────────────────────── */ + +class PairwiseBTest : public ::testing::Test { +protected: + static constexpr int N = 8; +}; + +TEST_F(PairwiseBTest, Offpair_Offset1_Coverage) { + auto m = run_debug("pairwise_b", N, "-mode offpair -offset 1"); + check_coverage(m, N, "offpair offset=1"); + check_all_ok(m, "offpair offset=1"); +} +TEST_F(PairwiseBTest, Offpair_Offset1_Validity) { + auto m = run_debug("pairwise_b", N, "-mode offpair -offset 1"); + check_validity(m, N, "offpair offset=1"); +} +TEST_F(PairwiseBTest, Offpair_Offset1_Symmetric) { + auto m = run_debug("pairwise_b", N, "-mode offpair -offset 1"); + check_symmetric(m, "offpair offset=1"); +} +TEST_F(PairwiseBTest, Offpair_Offset1_ExactPairs) { + auto m = run_debug("pairwise_b", N, "-mode offpair -offset 1"); + /* offset_pairs(n=8, offset=1) → 0↔1, 2↔3, 4↔5, 6↔7 */ + check_exact(m, {{0,1},{1,0},{2,3},{3,2},{4,5},{5,4},{6,7},{7,6}}, "offpair offset=1"); +} + +TEST_F(PairwiseBTest, Offpair_Offset2_Symmetric) { + auto m = run_debug("pairwise_b", N, "-mode offpair -offset 2"); + check_symmetric(m, "offpair offset=2"); +} +TEST_F(PairwiseBTest, Offpair_Offset2_ExactPairs) { + auto m = run_debug("pairwise_b", N, "-mode offpair -offset 2"); + /* offset_pairs(n=8, offset=2) → 0↔2, 1↔3, 4↔6, 5↔7 */ + check_exact(m, {{0,2},{2,0},{1,3},{3,1},{4,6},{6,4},{5,7},{7,5}}, "offpair offset=2"); +} + +TEST_F(PairwiseBTest, RandomPair_Coverage) { + auto m = run_debug("pairwise_b", N, "-mode rpair -seed 1"); + check_coverage(m, N, "rpair"); + check_all_ok(m, "rpair"); +} +TEST_F(PairwiseBTest, RandomPair_Validity) { + auto m = run_debug("pairwise_b", N, "-mode rpair -seed 1"); + check_validity(m, N, "rpair"); +} +TEST_F(PairwiseBTest, RandomPair_Symmetric) { + auto m = run_debug("pairwise_b", N, "-mode rpair -seed 1"); + check_symmetric(m, "rpair"); +} +TEST_F(PairwiseBTest, RandomPair_Bijection) { + auto m = run_debug("pairwise_b", N, "-mode rpair -seed 1"); + check_bijection(m, "rpair"); +} +TEST_F(PairwiseBTest, Burst) { + auto m = run_debug("pairwise_b", N, "-mode offpair -offset 1 -blength 0.001"); + check_coverage(m, N, "pairwise_b burst"); + check_symmetric(m, "pairwise_b burst"); + check_all_ok(m, "pairwise_b burst"); +} +TEST_F(PairwiseBTest, Burstdist) { + auto m = run_debug("pairwise_b", N, "-mode offpair -offset 1 -iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "pairwise_b burstdist"); + check_all_ok(m, "pairwise_b burstdist"); +} + +/* ── non-reciprocal modes (rot / perm) ────────────────────────────────────── + * rot and perm are permutations, not symmetric pairings, so the partner graph + * is a bijection but NOT symmetric (A→B does not imply B→A). These exercise + * the debug path that must avoid deadlock with a non-reciprocal partner. */ +TEST_F(PairwiseBTest, Rot_Offset1_Coverage) { + auto m = run_debug("pairwise_b", N, "-mode rot -offset 1"); + check_coverage(m, N, "rot offset=1"); + check_all_ok(m, "rot offset=1"); +} +TEST_F(PairwiseBTest, Rot_Offset1_Validity) { + auto m = run_debug("pairwise_b", N, "-mode rot -offset 1"); + check_validity(m, N, "rot offset=1"); +} +TEST_F(PairwiseBTest, Rot_Offset1_Bijection) { + auto m = run_debug("pairwise_b", N, "-mode rot -offset 1"); + check_bijection(m, "rot offset=1"); +} +TEST_F(PairwiseBTest, Rot_Offset1_ExactPairs) { + auto m = run_debug("pairwise_b", N, "-mode rot -offset 1"); + /* rot: rank r sends to (r+1)%N */ + check_exact(m, {{0,1},{1,2},{2,3},{3,4},{4,5},{5,6},{6,7},{7,0}}, "rot offset=1"); +} +TEST_F(PairwiseBTest, Perm_Coverage) { + auto m = run_debug("pairwise_b", N, "-mode perm -seed 1"); + check_coverage(m, N, "perm"); + check_all_ok(m, "perm"); +} +TEST_F(PairwiseBTest, Perm_Validity) { + auto m = run_debug("pairwise_b", N, "-mode perm -seed 1"); + check_validity(m, N, "perm"); +} +TEST_F(PairwiseBTest, Perm_Bijection) { + auto m = run_debug("pairwise_b", N, "-mode perm -seed 1"); + check_bijection(m, "perm"); +} + +/* ── pairwise_nb — same pairing logic, non-blocking MPI ────────────────────── */ + +class PairwiseNbTest : public ::testing::Test { +protected: + static constexpr int N = 8; +}; + +TEST_F(PairwiseNbTest, Offpair_Offset1_Symmetric) { + auto m = run_debug("pairwise_nb", N, "-mode offpair -offset 1"); + check_symmetric(m, "nb offpair offset=1"); + check_all_ok(m, "nb offpair offset=1"); +} +TEST_F(PairwiseNbTest, Offpair_Offset1_ExactPairs) { + auto m = run_debug("pairwise_nb", N, "-mode offpair -offset 1"); + check_exact(m, {{0,1},{1,0},{2,3},{3,2},{4,5},{5,4},{6,7},{7,6}}, "nb offpair offset=1"); +} +TEST_F(PairwiseNbTest, RandomPair_Symmetric) { + auto m = run_debug("pairwise_nb", N, "-mode rpair -seed 1"); + check_symmetric(m, "nb rpair"); + check_all_ok(m, "nb rpair"); +} +TEST_F(PairwiseNbTest, Burst) { + auto m = run_debug("pairwise_nb", N, "-mode offpair -offset 1 -blength 0.001"); + check_coverage(m, N, "pairwise_nb burst"); + check_symmetric(m, "pairwise_nb burst"); + check_all_ok(m, "pairwise_nb burst"); +} +TEST_F(PairwiseNbTest, Burstdist) { + auto m = run_debug("pairwise_nb", N, "-mode offpair -offset 1 -iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "pairwise_nb burstdist"); + check_all_ok(m, "pairwise_nb burstdist"); +} +TEST_F(PairwiseNbTest, Rot_Offset1_DataIntegrity) { + auto m = run_debug("pairwise_nb", N, "-mode rot -offset 1"); + check_coverage(m, N, "nb rot offset=1"); + check_bijection(m, "nb rot offset=1"); + check_all_ok(m, "nb rot offset=1"); +} +TEST_F(PairwiseNbTest, Perm_DataIntegrity) { + auto m = run_debug("pairwise_nb", N, "-mode perm -seed 1"); + check_coverage(m, N, "nb perm"); + check_bijection(m, "nb perm"); + check_all_ok(m, "nb perm"); +} + +/* ── pairwise_bsnbr — same pairing logic, non-blocking send / blocking recv ── */ + +class PairwiseBsnbrTest : public ::testing::Test { +protected: + static constexpr int N = 8; +}; + +TEST_F(PairwiseBsnbrTest, Offpair_Offset1_Symmetric) { + auto m = run_debug("pairwise_bsnbr", N, "-mode offpair -offset 1"); + check_symmetric(m, "bsnbr offpair offset=1"); + check_all_ok(m, "bsnbr offpair offset=1"); +} +TEST_F(PairwiseBsnbrTest, Offpair_Offset1_ExactPairs) { + auto m = run_debug("pairwise_bsnbr", N, "-mode offpair -offset 1"); + check_exact(m, {{0,1},{1,0},{2,3},{3,2},{4,5},{5,4},{6,7},{7,6}}, "bsnbr offpair offset=1"); +} +TEST_F(PairwiseBsnbrTest, RandomPair_Symmetric) { + auto m = run_debug("pairwise_bsnbr", N, "-mode rpair -seed 1"); + check_symmetric(m, "bsnbr rpair"); + check_all_ok(m, "bsnbr rpair"); +} +TEST_F(PairwiseBsnbrTest, Burst) { + auto m = run_debug("pairwise_bsnbr", N, "-mode offpair -offset 1 -blength 0.001"); + check_coverage(m, N, "pairwise_bsnbr burst"); + check_symmetric(m, "pairwise_bsnbr burst"); + check_all_ok(m, "pairwise_bsnbr burst"); +} +TEST_F(PairwiseBsnbrTest, Burstdist) { + auto m = run_debug("pairwise_bsnbr", N, "-mode offpair -offset 1 -iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "pairwise_bsnbr burstdist"); + check_all_ok(m, "pairwise_bsnbr burstdist"); +} +TEST_F(PairwiseBsnbrTest, Rot_Offset1_DataIntegrity) { + auto m = run_debug("pairwise_bsnbr", N, "-mode rot -offset 1"); + check_coverage(m, N, "bsnbr rot offset=1"); + check_bijection(m, "bsnbr rot offset=1"); + check_all_ok(m, "bsnbr rot offset=1"); +} +TEST_F(PairwiseBsnbrTest, Perm_DataIntegrity) { + auto m = run_debug("pairwise_bsnbr", N, "-mode perm -seed 1"); + check_coverage(m, N, "bsnbr perm"); + check_bijection(m, "bsnbr perm"); + check_all_ok(m, "bsnbr perm"); +} diff --git a/tests/test_pingpong.cpp b/tests/test_pingpong.cpp new file mode 100644 index 000000000..5852afe54 --- /dev/null +++ b/tests/test_pingpong.cpp @@ -0,0 +1,89 @@ +/* + * test_pingpong.cpp — correctness tests for pingpong benchmarks. + * + * Strategy: spawn the benchmark with -debug and parse + * "DEBUG rank=X nprocs=Y [partner=Z] check=OK|FAIL" lines. + * + * Properties checked: + * Coverage — every rank 0..N-1 appears exactly once. + * Nprocs — every rank reports the correct communicator size. + * DataIntegrity — every rank reports check=OK. + * Each rank fills send_buf with its own rank byte; + * after the ping-pong it verifies recv_buf contains + * the partner's rank byte throughout. + * + * pingpong_b requires exactly 2 MPI ranks. + * pingpong_pairwise_b requires 8 MPI ranks (even number). + */ +#include +#include +#include "popen_helpers.h" + +using blink::run_debug; +using blink::check_all_ok; +using blink::check_coverage; +using blink::check_nprocs; + +/* ── pingpong_b (2 ranks) ───────────────────────────────────────────────────── */ + +class PingpongBTest : public ::testing::Test { +protected: + static constexpr int N = 2; +}; + +TEST_F(PingpongBTest, Coverage) { + auto m = run_debug("pingpong_b", N); + check_coverage(m, N, "pingpong_b"); +} +TEST_F(PingpongBTest, Nprocs) { + auto m = run_debug("pingpong_b", N); + check_nprocs(m, N, "pingpong_b"); +} +TEST_F(PingpongBTest, DataIntegrity) { + auto m = run_debug("pingpong_b", N); + check_coverage(m, N, "pingpong_b"); + check_all_ok(m, "pingpong_b"); +} + +TEST_F(PingpongBTest, DataIntegrity_burst) { + auto m = run_debug("pingpong_b", N, "-blength 0.001"); + check_coverage(m, N, "pingpong_b burst"); + check_all_ok(m, "pingpong_b burst"); +} + +TEST_F(PingpongBTest, DataIntegrity_burstdist) { + auto m = run_debug("pingpong_b", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "pingpong_b burstdist"); + check_all_ok(m, "pingpong_b burstdist"); +} + +/* ── pingpong_pairwise_b (8 ranks) ─────────────────────────────────────────── */ + +class PingpongPairwiseBTest : public ::testing::Test { +protected: + static constexpr int N = 8; +}; + +TEST_F(PingpongPairwiseBTest, Coverage) { + auto m = run_debug("pingpong_pairwise_b", N); + check_coverage(m, N, "pingpong_pairwise_b"); +} +TEST_F(PingpongPairwiseBTest, Nprocs) { + auto m = run_debug("pingpong_pairwise_b", N); + check_nprocs(m, N, "pingpong_pairwise_b"); +} +TEST_F(PingpongPairwiseBTest, DataIntegrity) { + auto m = run_debug("pingpong_pairwise_b", N); + check_coverage(m, N, "pingpong_pairwise_b"); + check_all_ok(m, "pingpong_pairwise_b"); +} +TEST_F(PingpongPairwiseBTest, DataIntegrity_burst) { + auto m = run_debug("pingpong_pairwise_b", N, "-blength 0.001"); + check_coverage(m, N, "pingpong_pairwise_b burst"); + check_all_ok(m, "pingpong_pairwise_b burst"); +} +TEST_F(PingpongPairwiseBTest, DataIntegrity_burstdist) { + auto m = run_debug("pingpong_pairwise_b", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "pingpong_pairwise_b burstdist"); + check_all_ok(m, "pingpong_pairwise_b burstdist"); +} diff --git a/tests/test_ring.cpp b/tests/test_ring.cpp new file mode 100644 index 000000000..ed0ab622c --- /dev/null +++ b/tests/test_ring.cpp @@ -0,0 +1,244 @@ +/* + * test_ring.cpp — correctness tests for ring benchmarks. + * + * Strategy: spawn the actual benchmark binary with -debug and parse + * "DEBUG rank=X nprocs=N left=L right=R check=OK|FAIL" lines. + * No MPI in this binary. + * + * Properties checked: + * Coverage — every rank 0..N-1 appears exactly once. + * Validity — left and right are valid ranks in [0, N-1]. + * Exact values — sequential ring: rank r has left=(r-1+N)%N, right=(r+1)%N. + * Consistency — if rank A says right=B then rank B must say left=A, + * and vice-versa (the neighbour relation is coherent). + * Completeness — every rank appears as someone's left exactly once AND + * as someone's right exactly once (proper ring, no orphans). + * DataIntegrity — every rank reports check=OK. + * + * Requires 8 MPI ranks. + */ +#include +#include +#include +#include +#include "popen_helpers.h" + +using blink::DebugMap; +using blink::run_debug; +using blink::get_int; +using blink::check_all_ok; +using blink::check_coverage; + +/* ── shared assertion helpers ───────────────────────────────────────────────── */ + +static void check_validity(const DebugMap& m, int n, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + int l = get_int(m, r, "left"); + int rr = get_int(m, r, "right"); + EXPECT_GE(l, 0) << ctx << ": rank " << r << " left=" << l; + EXPECT_LT(l, n) << ctx << ": rank " << r << " left=" << l; + EXPECT_GE(rr, 0) << ctx << ": rank " << r << " right=" << rr; + EXPECT_LT(rr, n) << ctx << ": rank " << r << " right=" << rr; + } +} + +/* If A.right = B then B.left must equal A (and A.left = C implies C.right = A). */ +static void check_consistency(const DebugMap& m, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + int l = get_int(m, r, "left"); + int rr = get_int(m, r, "right"); + + /* check right edge: r → rr */ + auto it = m.find(rr); + if (it != m.end()) { + int neighbour_left = get_int(m, rr, "left"); + EXPECT_EQ(neighbour_left, r) + << ctx << ": rank " << r << " says right=" << rr + << " but rank " << rr << " says left=" << neighbour_left; + } + + /* check left edge: r → l */ + it = m.find(l); + if (it != m.end()) { + int neighbour_right = get_int(m, l, "right"); + EXPECT_EQ(neighbour_right, r) + << ctx << ": rank " << r << " says left=" << l + << " but rank " << l << " says right=" << neighbour_right; + } + } +} + +/* Every rank appears as someone's left exactly once AND right exactly once. */ +static void check_completeness(const DebugMap& m, const std::string& ctx) +{ + std::map as_left, as_right; + for (const auto& [r, fields] : m) { + as_left[get_int(m, r, "left")]++; + as_right[get_int(m, r, "right")]++; + } + for (const auto& [r, _] : m) { + EXPECT_EQ(as_left.count(r) ? as_left[r] : 0, 1) + << ctx << ": rank " << r << " appears as left neighbour " + << (as_left.count(r) ? as_left[r] : 0) << " time(s) (expected 1)"; + EXPECT_EQ(as_right.count(r) ? as_right[r] : 0, 1) + << ctx << ": rank " << r << " appears as right neighbour " + << (as_right.count(r) ? as_right[r] : 0) << " time(s) (expected 1)"; + } +} + +static void check_exact_sequential(const DebugMap& m, int n, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + int l = get_int(m, r, "left"); + int rr = get_int(m, r, "right"); + EXPECT_EQ(l, (r - 1 + n) % n) + << ctx << ": rank " << r << " expected left=" << (r-1+n)%n + << " got " << l; + EXPECT_EQ(rr, (r + 1) % n) + << ctx << ": rank " << r << " expected right=" << (r+1)%n + << " got " << rr; + } +} + +/* ── ring_nb ────────────────────────────────────────────────────────────────── */ + +class RingNbTest : public ::testing::Test { +protected: + static constexpr int N = 8; +}; + +TEST_F(RingNbTest, Sequential_Coverage) { + auto m = run_debug("ring_nb", N); + check_coverage(m, N, "ring_nb sequential"); + check_all_ok(m, "ring_nb sequential"); +} +TEST_F(RingNbTest, Sequential_Validity) { + auto m = run_debug("ring_nb", N); + check_validity(m, N, "ring_nb sequential"); +} +TEST_F(RingNbTest, Sequential_ExactNeighbors) { + auto m = run_debug("ring_nb", N); + check_exact_sequential(m, N, "ring_nb sequential"); +} +TEST_F(RingNbTest, Sequential_Consistency) { + auto m = run_debug("ring_nb", N); + check_consistency(m, "ring_nb sequential"); +} +TEST_F(RingNbTest, Sequential_Completeness) { + auto m = run_debug("ring_nb", N); + check_completeness(m, "ring_nb sequential"); +} + +TEST_F(RingNbTest, Random_Coverage) { + auto m = run_debug("ring_nb", N, "-rring -seed 1"); + check_coverage(m, N, "ring_nb random"); + check_all_ok(m, "ring_nb random"); +} +TEST_F(RingNbTest, Random_Validity) { + auto m = run_debug("ring_nb", N, "-rring -seed 1"); + check_validity(m, N, "ring_nb random"); +} +/* The random ring is built from an explicit position<->rank inverse map + * (positions[targets[i]] = i in ring_nb.c), so the neighbour relation is fully + * reciprocal (A.right=B <=> B.left=A) — the same consistency property the + * sequential ring has. Both consistency and completeness must hold. */ +TEST_F(RingNbTest, Random_Consistency) { + auto m = run_debug("ring_nb", N, "-rring -seed 1"); + check_consistency(m, "ring_nb random"); +} +TEST_F(RingNbTest, Random_Completeness) { + auto m = run_debug("ring_nb", N, "-rring -seed 1"); + check_completeness(m, "ring_nb random"); +} +TEST_F(RingNbTest, Burst) { + auto m = run_debug("ring_nb", N, "-blength 0.001"); + check_coverage(m, N, "ring_nb burst"); + check_consistency(m, "ring_nb burst"); + check_all_ok(m, "ring_nb burst"); +} +TEST_F(RingNbTest, Burstdist) { + auto m = run_debug("ring_nb", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "ring_nb burstdist"); + check_consistency(m, "ring_nb burstdist"); + check_all_ok(m, "ring_nb burstdist"); +} +/* Random ring must differ from sequential (sanity: permutation actually happened). */ +TEST_F(RingNbTest, Random_DiffersFromSequential) { + auto seq = run_debug("ring_nb", N); + auto rnd = run_debug("ring_nb", N, "-rring -seed 1"); + bool any_diff = false; + for (const auto& [r, fields] : seq) { + if (!rnd.count(r)) continue; + if (get_int(rnd, r, "left") != get_int(seq, r, "left") || + get_int(rnd, r, "right") != get_int(seq, r, "right")) + any_diff = true; + } + EXPECT_TRUE(any_diff) << "random ring produced identical topology to sequential"; +} + +/* ── ring_bsnbr — same setup code, different MPI calls ─────────────────────── */ + +class RingBsnbrTest : public ::testing::Test { +protected: + static constexpr int N = 8; +}; + +TEST_F(RingBsnbrTest, Sequential_ExactNeighbors) { + auto m = run_debug("ring_bsnbr", N); + check_exact_sequential(m, N, "ring_bsnbr sequential"); + check_all_ok(m, "ring_bsnbr sequential"); +} +TEST_F(RingBsnbrTest, Sequential_Consistency) { + auto m = run_debug("ring_bsnbr", N); + check_consistency(m, "ring_bsnbr sequential"); +} +TEST_F(RingBsnbrTest, Sequential_Completeness) { + auto m = run_debug("ring_bsnbr", N); + check_completeness(m, "ring_bsnbr sequential"); +} +TEST_F(RingBsnbrTest, Random_Consistency) { + auto m = run_debug("ring_bsnbr", N, "-rring -seed 1"); + check_consistency(m, "ring_bsnbr random"); +} +TEST_F(RingBsnbrTest, Random_Completeness) { + auto m = run_debug("ring_bsnbr", N, "-rring -seed 1"); + check_completeness(m, "ring_bsnbr random"); + check_all_ok(m, "ring_bsnbr random"); +} +/* Both variants must agree on the same topology for the same seed. */ +TEST_F(RingBsnbrTest, AgreesWith_RingNb_Sequential) { + auto nb = run_debug("ring_nb", N); + auto bsnbr = run_debug("ring_bsnbr", N); + for (const auto& [r, fields] : nb) { + if (!bsnbr.count(r)) continue; + EXPECT_EQ(get_int(bsnbr, r, "left"), get_int(nb, r, "left")) + << "rank " << r << " left differs between ring_nb and ring_bsnbr"; + EXPECT_EQ(get_int(bsnbr, r, "right"), get_int(nb, r, "right")) + << "rank " << r << " right differs between ring_nb and ring_bsnbr"; + } +} +TEST_F(RingBsnbrTest, Burst) { + auto m = run_debug("ring_bsnbr", N, "-blength 0.001"); + check_coverage(m, N, "ring_bsnbr burst"); + check_consistency(m, "ring_bsnbr burst"); + check_all_ok(m, "ring_bsnbr burst"); +} +TEST_F(RingBsnbrTest, Burstdist) { + auto m = run_debug("ring_bsnbr", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "ring_bsnbr burstdist"); + check_consistency(m, "ring_bsnbr burstdist"); + check_all_ok(m, "ring_bsnbr burstdist"); +} +TEST_F(RingBsnbrTest, AgreesWith_RingNb_Random) { + auto nb = run_debug("ring_nb", N, "-rring -seed 42"); + auto bsnbr = run_debug("ring_bsnbr", N, "-rring -seed 42"); + for (const auto& [r, fields] : nb) { + if (!bsnbr.count(r)) continue; + EXPECT_EQ(get_int(bsnbr, r, "left"), get_int(nb, r, "left")) + << "rank " << r << " left differs (random, seed=42)"; + EXPECT_EQ(get_int(bsnbr, r, "right"), get_int(nb, r, "right")) + << "rank " << r << " right differs (random, seed=42)"; + } +} diff --git a/tests/test_stencil.cpp b/tests/test_stencil.cpp new file mode 100644 index 000000000..74bd1c912 --- /dev/null +++ b/tests/test_stencil.cpp @@ -0,0 +1,145 @@ +/* + * test_stencil.cpp — correctness tests for stencil_2d_nb. + * + * Strategy: spawn stencil_2d_nb with -debug and parse + * "DEBUG rank=X nprocs=Y north=N south=S west=W east=E check=OK|FAIL" lines. + * Absent neighbours (boundary ranks, non-periodic) are printed as MPI_PROC_NULL + * which is negative; the test treats any negative value as "no neighbour". + * + * Properties checked: + * Coverage — every rank 0..N-1 appears exactly once. + * Nprocs — every rank reports the correct communicator size. + * DataIntegrity — every rank reports check=OK. + * For each present neighbour d, recv_buf[d*msg_size..] + * must equal neighbour's rank byte throughout. + * Consistency — the neighbour graph is symmetric: + * if rank A's north = B then rank B's south = A, etc. + * + * Requires 8 MPI ranks (auto grid via MPI_Dims_create). + */ +#include +#include +#include "popen_helpers.h" + +using blink::DebugMap; +using blink::run_debug; +using blink::get_int; +using blink::check_all_ok; +using blink::check_coverage; +using blink::check_nprocs; + +/* ── helpers ────────────────────────────────────────────────────────────────── */ + +static void check_consistency(const DebugMap& m, const std::string& ctx) +{ + for (const auto& [r, fields] : m) { + int north = get_int(m, r, "north"); + int south = get_int(m, r, "south"); + int west = get_int(m, r, "west"); + int east = get_int(m, r, "east"); + + if (north >= 0) { + ASSERT_TRUE(m.count(north)) << ctx << ": rank " << north << " missing"; + int n_south = get_int(m, north, "south"); + EXPECT_EQ(n_south, r) + << ctx << ": rank " << r << " north=" << north + << " but rank " << north << " south=" << n_south; + } + if (south >= 0) { + ASSERT_TRUE(m.count(south)) << ctx << ": rank " << south << " missing"; + int s_north = get_int(m, south, "north"); + EXPECT_EQ(s_north, r) + << ctx << ": rank " << r << " south=" << south + << " but rank " << south << " north=" << s_north; + } + if (west >= 0) { + ASSERT_TRUE(m.count(west)) << ctx << ": rank " << west << " missing"; + int w_east = get_int(m, west, "east"); + EXPECT_EQ(w_east, r) + << ctx << ": rank " << r << " west=" << west + << " but rank " << west << " east=" << w_east; + } + if (east >= 0) { + ASSERT_TRUE(m.count(east)) << ctx << ": rank " << east << " missing"; + int e_west = get_int(m, east, "west"); + EXPECT_EQ(e_west, r) + << ctx << ": rank " << r << " east=" << east + << " but rank " << east << " west=" << e_west; + } + } +} + +/* ── tests ──────────────────────────────────────────────────────────────────── */ + +class StencilTest : public ::testing::Test { +protected: + static constexpr int N = 8; +}; + +TEST_F(StencilTest, Coverage) { + auto m = run_debug("stencil_2d_nb", N); + check_coverage(m, N, "stencil default"); +} +TEST_F(StencilTest, Nprocs) { + auto m = run_debug("stencil_2d_nb", N); + check_nprocs(m, N, "stencil default"); +} +TEST_F(StencilTest, DataIntegrity) { + auto m = run_debug("stencil_2d_nb", N); + check_coverage(m, N, "stencil default"); + check_all_ok(m, "stencil default"); +} +TEST_F(StencilTest, Consistency) { + auto m = run_debug("stencil_2d_nb", N); + check_coverage(m, N, "stencil default"); + check_consistency(m, "stencil default"); +} +TEST_F(StencilTest, DataIntegrity_periodic) { + auto m = run_debug("stencil_2d_nb", N, "-periodic"); + check_coverage(m, N, "stencil periodic"); + check_all_ok(m, "stencil periodic"); +} +TEST_F(StencilTest, Consistency_periodic) { + auto m = run_debug("stencil_2d_nb", N, "-periodic"); + check_coverage(m, N, "stencil periodic"); + check_consistency(m, "stencil periodic"); +} +TEST_F(StencilTest, DataIntegrity_burst) { + auto m = run_debug("stencil_2d_nb", N, "-blength 0.001"); + check_coverage(m, N, "stencil burst"); + check_all_ok(m, "stencil burst"); +} +TEST_F(StencilTest, DataIntegrity_burstdist) { + auto m = run_debug("stencil_2d_nb", N, "-iter 3 -blength 0.0004 -bldist exp -bpause 0.0003 -bpdist exp"); + check_coverage(m, N, "stencil burstdist"); + check_all_ok(m, "stencil burstdist"); +} +TEST_F(StencilTest, Consistency_burst) { + auto m = run_debug("stencil_2d_nb", N, "-blength 0.001"); + check_coverage(m, N, "stencil burst"); + check_consistency(m, "stencil burst"); +} +/* + * Periodic vs non-periodic distinction: non-periodic must produce at least + * one boundary rank with an MPI_PROC_NULL (negative) neighbour, and periodic + * must produce none. A silently-ignored -periodic flag would fail this. + */ +TEST_F(StencilTest, PeriodicityDistinct) { + auto plain = run_debug("stencil_2d_nb", N); + auto periodic = run_debug("stencil_2d_nb", N, "-periodic"); + + int plain_nulls = 0, periodic_nulls = 0; + for (const auto& [r, fields] : plain) { + for (const char *d : {"north","south","west","east"}) + if (get_int(plain, r, d) < 0) plain_nulls++; + } + for (const auto& [r, fields] : periodic) { + for (const char *d : {"north","south","west","east"}) + if (get_int(periodic, r, d) < 0) periodic_nulls++; + } + EXPECT_GT(plain_nulls, 0) + << "non-periodic stencil should have at least one boundary neighbour=MPI_PROC_NULL"; + EXPECT_EQ(periodic_nulls, 0) + << "periodic stencil should have no MPI_PROC_NULL neighbours (toroidal grid), " + << "got " << periodic_nulls << " — is -periodic silently ignored?"; +}