Skip to content

Commit 457ccaa

Browse files
aetosdios27aetosdios27
authored
test: run microbenchmarks in extended CI (#25836)
## Which issue does this PR close? Closes #24868. ## Rationale for this change Microbenchmarks can compile successfully but panic when executed. Running their cases in CI catches stale benchmark fixtures and invocation assumptions without collecting performance measurements. ## What changes are included in this PR? - Add `cargo xtask ci step test microbenchmarks` and a required extended CI job. Following the existing workflow convention, it runs in the merge queue, on release-branch pushes, and manually; it skips PR events. - Execute Criterion bench targets in test mode with `--no-fail-fast`, including the physical-plan benchmarks gated by `test_utils`. The `ci-bench` profile retains workspace debug assertions and overflow checks with light optimization to keep larger cases practical. - Use the checked-in ten-row ClickBench fixture for SQL planner smoke tests through `CLICKBENCH_DATA_PATH`. Ordinary performance runs retain their full-dataset default. The separate SQL benchmark harness, which requires generated datasets, remains covered by its existing jobs. - Fix benchmark failures exposed by execution: timestamp timezone metadata, Spark slice input coercion, string return types, and repeat row counts/error assertions. Slice coercion happens before timing the UDF. - Clarify the Spark ListView benchmark labels and check the repeat overflow diagnostic outside the timed loop. Patch two certificate dependencies to resolve the security audit failure. ## What is the testing strategy for this PR? - The full `cargo xtask ci step test microbenchmarks` run passed all 131 benchmark targets on upstream `99193cbd0`. After rebasing onto `6c1727fcd`, the Spark slice and repeat benchmark smoke tests passed again, including the new labels and overflow diagnostic assertion. - `cargo fmt --all` and `cargo clippy --all-targets --all-features -- -D warnings` pass with the pinned Rust 1.99 toolchain on the final rebase. - `./ci/scripts/security_audit.sh` passes after patching `rustls-webpki` to 0.103.16 and `webpki-root-certs` to 1.0.10. - Extended workspace tests were run with the documented features. The run stopped at the existing `sort_with_mem_limit_1_runner` RSS assertion: the core integration target had 1,463 passing tests, one failure, and one ignored test. Its RSS monitor reported a zero-byte baseline and a 192.2 MB peak against a 190.7 MB limit. The unchanged test passed in isolation on a clean upstream build, but the full run still fails locally. No tests were skipped and no memory-test code or limits were changed. ## Are there any user-facing changes? No query behavior or public API changes. Benchmark contributors gain a local smoke-test command and automated execution coverage.
1 parent 20463f3 commit 457ccaa

8 files changed

Lines changed: 160 additions & 37 deletions

File tree

‎.asf.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ github:
6969
- "cargo check datafusion-spark features"
7070
- "cargo test (amd64)"
7171
- "cargo test 'extended_tests' (amd64)"
72+
- "microbenchmark tests (amd64)"
7273
- "cargo test hash collisions (amd64)"
7374
- "cargo test datafusion-cli (amd64)"
7475
- "cargo examples (amd64)"

‎.github/workflows/extended.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,28 @@ jobs:
9090
- name: Cleanup
9191
run: cargo clean
9292

93+
# Execute Criterion benchmarks once to catch panics without measuring performance.
94+
linux-test-microbenchmarks:
95+
name: microbenchmark tests (amd64)
96+
# Long-running job: run in the merge queue, on release pushes, or manually.
97+
if: github.event_name != 'pull_request'
98+
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
99+
container:
100+
image: amd64/rust
101+
steps:
102+
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
103+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
104+
with:
105+
ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
106+
submodules: true
107+
fetch-depth: 1
108+
- name: Setup Rust toolchain
109+
uses: ./.github/actions/setup-builder
110+
with:
111+
rust-version: stable
112+
- name: Test microbenchmarks
113+
run: cargo xtask ci step test microbenchmarks
114+
93115
# Check answers are correct when hash values collide
94116
hash-collisions:
95117
name: cargo test hash collisions (amd64)

‎Cargo.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,3 +409,8 @@ debug = false
409409
debug-assertions = false
410410
strip = "debuginfo"
411411
incremental = false
412+
413+
# Execute benchmark tests with debug checks, without release-mode build costs.
414+
[profile.ci-bench]
415+
inherits = "ci"
416+
opt-level = 1

‎datafusion/core/benches/sql_planner.rs‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -203,17 +203,10 @@ fn register_defs(ctx: SessionContext, defs: Vec<TableDef>) -> SessionContext {
203203
ctx
204204
}
205205

206-
fn register_clickbench_hits_table(rt: &Runtime) -> SessionContext {
206+
fn register_clickbench_hits_table(rt: &Runtime, path: &str) -> SessionContext {
207207
let ctx = SessionContext::new();
208208

209209
// use an external table for clickbench benchmarks
210-
let path =
211-
if PathBuf::from(format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")).exists() {
212-
format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")
213-
} else {
214-
format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")
215-
};
216-
217210
let sql =
218211
format!("CREATE EXTERNAL TABLE hits_raw STORED AS PARQUET LOCATION '{path}'");
219212

@@ -355,15 +348,23 @@ fn union_orderby_query(n: usize) -> String {
355348
}
356349

357350
fn criterion_benchmark(c: &mut Criterion) {
351+
// CI uses the small checked-in fixture; performance runs use the full dataset.
352+
let clickbench_data_path =
353+
std::env::var("CLICKBENCH_DATA_PATH").unwrap_or_else(|_| {
354+
let path = format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}");
355+
if PathBuf::from(&path).exists() {
356+
path
357+
} else {
358+
format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")
359+
}
360+
});
358361
// verify that we can load the clickbench data prior to running the benchmark
359-
if !PathBuf::from(format!("{BENCHMARKS_PATH_1}{CLICKBENCH_DATA_PATH}")).exists()
360-
&& !PathBuf::from(format!("{BENCHMARKS_PATH_2}{CLICKBENCH_DATA_PATH}")).exists()
361-
{
362-
panic!(
363-
"benchmarks/data/hits_partitioned/ could not be loaded. Please run \
364-
'benchmarks/bench.sh data clickbench_partitioned' prior to running this benchmark"
365-
)
366-
}
362+
assert!(
363+
PathBuf::from(&clickbench_data_path).exists(),
364+
"{clickbench_data_path} could not be loaded. Please run \
365+
'benchmarks/bench.sh data clickbench_partitioned' prior to running this benchmark \
366+
or set CLICKBENCH_DATA_PATH to an existing dataset"
367+
);
367368

368369
let ctx = create_context();
369370
let rt = Runtime::new().unwrap();
@@ -693,7 +694,7 @@ fn criterion_benchmark(c: &mut Criterion) {
693694
)))
694695
.collect::<Vec<_>>();
695696

696-
let clickbench_ctx = register_clickbench_hits_table(&rt);
697+
let clickbench_ctx = register_clickbench_hits_table(&rt, &clickbench_data_path);
697698

698699
// for (i, sql) in clickbench_queries.iter().enumerate() {
699700
// c.bench_function(&format!("logical_plan_clickbench_q{}", i + 1), |b| {

‎datafusion/functions/benches/datetime_expressions/to_timestamp.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use std::sync::Arc;
2121
use arrow::array::builder::StringBuilder;
2222
use arrow::array::{Array, ArrayRef, StringArray};
2323
use arrow::compute::cast;
24-
use arrow::datatypes::{DataType, Field, TimeUnit};
24+
use arrow::datatypes::{DataType, Field};
2525
use criterion::{Criterion, criterion_group};
2626
use datafusion_common::config::ConfigOptions;
2727
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
@@ -108,15 +108,19 @@ fn data_with_formats() -> (StringArray, StringArray, StringArray, StringArray) {
108108
)
109109
}
110110
fn criterion_benchmark(c: &mut Criterion) {
111-
let return_field =
112-
Field::new("f", DataType::Timestamp(TimeUnit::Nanosecond, None), true).into();
113111
let arg_field = Field::new("a", DataType::Utf8, false).into();
114112
let arg_fields = vec![arg_field];
115113
let mut options = ConfigOptions::default();
116114
options.execution.time_zone = Some("UTC".parse().unwrap());
117115
let config_options = Arc::new(options);
118116

119117
let to_timestamp_udf = to_timestamp(config_options.as_ref());
118+
let return_field = Field::new(
119+
"f",
120+
to_timestamp_udf.return_type(&[DataType::Utf8]).unwrap(),
121+
true,
122+
)
123+
.into();
120124

121125
c.bench_function("to_timestamp_no_formats_utf8", |b| {
122126
let to_timestamp_udf = Arc::clone(&to_timestamp_udf);

‎datafusion/functions/benches/string_expressions/repeat.rs‎

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ fn create_args<O: OffsetSizeTrait>(
6060

6161
fn invoke_repeat_with_args(
6262
args: Vec<ColumnarValue>,
63-
repeat_times: i64,
6463
) -> Result<ColumnarValue, DataFusionError> {
6564
let arg_fields = args
6665
.iter()
@@ -69,11 +68,22 @@ fn invoke_repeat_with_args(
6968
.collect::<Vec<_>>();
7069
let config_options = Arc::new(ConfigOptions::default());
7170

71+
let return_type = string::repeat().return_type(
72+
&args
73+
.iter()
74+
.map(ColumnarValue::data_type)
75+
.collect::<Vec<_>>(),
76+
)?;
77+
let number_rows = match &args[0] {
78+
ColumnarValue::Array(array) => array.len(),
79+
ColumnarValue::Scalar(_) => 1,
80+
};
81+
7282
string::repeat().invoke_with_args(ScalarFunctionArgs {
7383
args,
7484
arg_fields,
75-
number_rows: repeat_times as usize,
76-
return_field: Field::new("f", DataType::Utf8, true).into(),
85+
number_rows,
86+
return_field: Field::new("f", return_type, true).into(),
7787
config_options: Arc::clone(&config_options),
7888
})
7989
}
@@ -111,7 +121,7 @@ fn criterion_benchmark(c: &mut Criterion) {
111121
Field::new("b", DataType::Int64, false).into(),
112122
],
113123
number_rows: 1,
114-
return_field: Field::new("f", DataType::Utf8, true).into(),
124+
return_field: Field::new("f", DataType::Utf8View, true).into(),
115125
config_options: Arc::clone(&config_options),
116126
};
117127
b.iter(|| black_box(repeat_fn.invoke_with_args(args.clone()).unwrap()))
@@ -131,7 +141,7 @@ fn criterion_benchmark(c: &mut Criterion) {
131141
|b| {
132142
b.iter(|| {
133143
let args_cloned = args.clone();
134-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
144+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
135145
})
136146
},
137147
);
@@ -142,7 +152,7 @@ fn criterion_benchmark(c: &mut Criterion) {
142152
|b| {
143153
b.iter(|| {
144154
let args_cloned = args.clone();
145-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
155+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
146156
})
147157
},
148158
);
@@ -153,7 +163,7 @@ fn criterion_benchmark(c: &mut Criterion) {
153163
|b| {
154164
b.iter(|| {
155165
let args_cloned = args.clone();
156-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
166+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
157167
})
158168
},
159169
);
@@ -173,7 +183,7 @@ fn criterion_benchmark(c: &mut Criterion) {
173183
|b| {
174184
b.iter(|| {
175185
let args_cloned = args.clone();
176-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
186+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
177187
})
178188
},
179189
);
@@ -184,7 +194,7 @@ fn criterion_benchmark(c: &mut Criterion) {
184194
|b| {
185195
b.iter(|| {
186196
let args_cloned = args.clone();
187-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
197+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
188198
})
189199
},
190200
);
@@ -195,7 +205,7 @@ fn criterion_benchmark(c: &mut Criterion) {
195205
|b| {
196206
b.iter(|| {
197207
let args_cloned = args.clone();
198-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
208+
black_box(invoke_repeat_with_args(args_cloned).unwrap())
199209
})
200210
},
201211
);
@@ -210,12 +220,25 @@ fn criterion_benchmark(c: &mut Criterion) {
210220
group.measurement_time(Duration::from_secs(10));
211221

212222
let args = create_args::<i32>(size, 2, repeat_times, false);
223+
// Verify once outside the timed loop that the failure is the expected
224+
// overflow error, and not an unrelated error that also happens to fail.
225+
let err = invoke_repeat_with_args(args.clone())
226+
.expect_err("repeat should reject an overflowing output size")
227+
.to_string();
228+
assert!(
229+
err.contains("string size overflow on repeat"),
230+
"unexpected error: {err}"
231+
);
213232
group.bench_function(
214233
format!("repeat_string overflow [size={size}, repeat_times={repeat_times}]"),
215234
|b| {
216235
b.iter(|| {
217236
let args_cloned = args.clone();
218-
black_box(invoke_repeat_with_args(args_cloned, repeat_times))
237+
black_box(
238+
invoke_repeat_with_args(args_cloned).expect_err(
239+
"repeat should reject an overflowing output size",
240+
),
241+
)
219242
})
220243
},
221244
);

‎datafusion/spark/benches/slice.rs‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ use arrow::datatypes::{DataType, Field, Int64Type};
2323
use criterion::{Criterion, criterion_group, criterion_main};
2424
use datafusion_common::ScalarValue;
2525
use datafusion_common::config::ConfigOptions;
26-
use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
26+
use datafusion_expr::type_coercion::functions::fields_with_udf;
27+
use datafusion_expr::{ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs};
2728
use datafusion_spark::function::array::slice;
2829
use rand::rngs::StdRng;
2930
use rand::{Rng, SeedableRng};
@@ -115,6 +116,19 @@ fn array_slice_benchmark(
115116
<Arc<Field>>::from(Field::new(format!("arg_{idx}"), arg.data_type(), true))
116117
})
117118
.collect::<Vec<_>>();
119+
// Apply planner coercion (including ListView to List) before timing the UDF.
120+
let arg_fields = fields_with_udf(&arg_fields, array_slice.as_ref()).unwrap();
121+
let args = args
122+
.iter()
123+
.zip(&arg_fields)
124+
.map(|(arg, field)| arg.cast_to(field.data_type(), None).unwrap())
125+
.collect::<Vec<_>>();
126+
let return_field = array_slice
127+
.return_field_from_args(ReturnFieldArgs {
128+
arg_fields: &arg_fields,
129+
scalar_arguments: &vec![None; arg_fields.len()],
130+
})
131+
.unwrap();
118132
c.bench_function(name, |b| {
119133
b.iter(|| {
120134
black_box(
@@ -123,8 +137,7 @@ fn array_slice_benchmark(
123137
args: args.clone(),
124138
arg_fields: arg_fields.clone(),
125139
number_rows: size,
126-
return_field: Field::new_list_field(args[0].data_type(), true)
127-
.into(),
140+
return_field: Arc::clone(&return_field),
128141
config_options: Arc::new(ConfigOptions::default()),
129142
})
130143
.unwrap(),
@@ -161,7 +174,10 @@ fn criterion_benchmark(c: &mut Criterion) {
161174
let scalar_to = ColumnarValue::Scalar(ScalarValue::from(child_array_size as i64 / 2));
162175

163176
for input in [list_array, list_view_array] {
164-
let input_type = input.data_type().to_string();
177+
let input_type = match input.data_type() {
178+
DataType::ListView(_) => "ListView -> List (coercion excluded)".to_string(),
179+
_ => input.data_type().to_string(),
180+
};
165181

166182
array_slice_benchmark(
167183
&format!("slice: input {input_type}, array args, no stride"),

‎xtask/src/ci_steps.rs‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ static CI_STEPS: &[StepInfo] = &[
5555
},
5656
StepInfo {
5757
command: "test",
58-
help_usage: "test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]",
58+
help_usage: "test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]",
5959
help_examples: &["test cli", "test workspace"],
6060
help_description: "Run a CI test suite",
6161
error_message: "Cargo test step failed",
@@ -290,6 +290,38 @@ impl StepContext {
290290
"integration-tests",
291291
]);
292292
}
293+
"microbenchmarks" => {
294+
// The SQL benchmark harness needs separately generated datasets
295+
// and is covered by the benchmark result verification job.
296+
// Select only bench targets (not lib/bin unit tests), and tell
297+
// Criterion to execute each benchmark once instead of timing it.
298+
// Keep debug assertions and overflow checks enabled without the
299+
// cost of unoptimized execution on the larger benchmark datasets.
300+
command.env(
301+
"CLICKBENCH_DATA_PATH",
302+
self.root
303+
.join("datafusion/core/tests/data/clickbench_hits_10.parquet"),
304+
);
305+
command.args([
306+
"test",
307+
"--profile",
308+
"ci-bench",
309+
"--no-fail-fast",
310+
"--workspace",
311+
"--exclude",
312+
"datafusion-examples",
313+
"--exclude",
314+
"datafusion-benchmarks",
315+
"--exclude",
316+
"datafusion-cli",
317+
"--bench",
318+
"*",
319+
"--features",
320+
"datafusion-physical-plan/test_utils",
321+
"--",
322+
"--test",
323+
]);
324+
}
293325
"benchmark-plan" => {
294326
command
295327
.args([
@@ -616,7 +648,7 @@ fn required_env(name: &str) -> Result<String> {
616648
mod tests {
617649
use super::*;
618650

619-
const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]";
651+
const TEST_USAGE: &str = "usage: cargo xtask ci step test <workspace|cli|doctest|ffi|microbenchmarks|benchmark-plan|benchmark-sqllogic|postgres|substrait|substrait-optimized|extended|hash-collisions|sqlite> [--explain]";
620652

621653
fn args(args: &[&str]) -> Vec<String> {
622654
args.iter().map(|arg| (*arg).to_string()).collect()
@@ -796,6 +828,25 @@ cargo test \
796828
--lib \
797829
--tests \
798830
--features integration-tests
831+
")
832+
},
833+
},
834+
Cmd {
835+
command: &["test", "microbenchmarks"],
836+
expected: |actual| {
837+
insta::assert_snapshot!(actual, @r"
838+
cd /workspace && \
839+
CLICKBENCH_DATA_PATH=/workspace/datafusion/core/tests/data/clickbench_hits_10.parquet \
840+
cargo test \
841+
--profile ci-bench \
842+
--no-fail-fast \
843+
--workspace \
844+
--exclude datafusion-examples \
845+
--exclude datafusion-benchmarks \
846+
--exclude datafusion-cli \
847+
--bench '*' \
848+
--features datafusion-physical-plan/test_utils \
849+
-- --test
799850
")
800851
},
801852
},

0 commit comments

Comments
 (0)