diff --git a/docs/testing-guide.md b/docs/testing-guide.md index d1d3cc5b..ae11fce7 100644 --- a/docs/testing-guide.md +++ b/docs/testing-guide.md @@ -171,6 +171,44 @@ class CompatibilityTest { The versions from `@AdditionallyRunWithGradle` are merged with the globally configured versions. When applied to both a class and a method, all versions are combined. Duplicate versions are automatically deduplicated. +#### Restricting to Specific Versions + +Use `@RestrictToGradleVersionsEqualTo` to restrict the test matrix to only run on specific Gradle versions. Unlike `@AdditionallyRunWithGradle` which adds versions, this annotation restricts which versions from the matrix will actually run. + +```java +@GradlePluginTests +class RestrictedVersionTest { + @Test + @RestrictToGradleVersionsEqualTo(value = "8.14.3", reason = "This test only applies to Gradle 8.14.3") + void test_only_on_specific_version(GradleInvoker gradle, RootProject project) { + // This test only runs on 8.14.3, even if other versions are in the matrix + } + + @Test + @RestrictToGradleVersionsEqualTo({"8.10", "8.14.3"}) + void test_on_subset_of_versions(GradleInvoker gradle, RootProject project) { + // This test only runs on 8.10 and 8.14.3 + } +} +``` + +**Key differences from `@AdditionallyRunWithGradle`:** +- `@AdditionallyRunWithGradle` **adds** versions to the test matrix +- `@RestrictToGradleVersionsEqualTo` **restricts** the existing matrix to only include specified versions + +**Important:** If you specify a version that isn't in the test matrix, the test simply won't run for that version. To run a specific version that isn't in the matrix, use both annotations together: + +```java +@Test +@AdditionallyRunWithGradle("8.5") // Add 8.5 to the matrix +@RestrictToGradleVersionsEqualTo("8.5") // Restrict to only run 8.5 +void test_only_on_8_5(GradleInvoker gradle, RootProject project) { + // Runs exclusively on Gradle 8.5 +} +``` + +The annotation can be applied at the class level to restrict all tests in the class, or at the method level for individual tests. Method-level restrictions are applied in addition to class-level restrictions. + ## File Operations ### Working with Files diff --git a/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/AdditionallyRunWithGradleCondition.java b/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/AdditionallyRunWithGradleCondition.java index 87dc26ec..732673cf 100644 --- a/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/AdditionallyRunWithGradleCondition.java +++ b/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/AdditionallyRunWithGradleCondition.java @@ -28,6 +28,10 @@ *

When a method has its own {@link AdditionallyRunWithGradle} annotation, this condition ensures * that only the method-specific versions (plus base and class-level versions) run for that method. * For methods without the annotation, only base and class-level versions run. + * + *

When a method has filter annotations like {@link RestrictToGradleVersionsEqualTo}, the allowed versions + * are filtered accordingly. Class-level filter annotations are handled in {@link GradleVersioningClassTemplate} + * to filter the test matrix upfront. */ final class AdditionallyRunWithGradleCondition implements ExecutionCondition { @@ -39,7 +43,7 @@ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext con } GradleVersion currentVersion = GradleVersionStore.gradleVersion(context); - Set versionsForThisMethod = GradleVersions.versionsForMethod(context); + Set versionsForThisMethod = GradleVersions.filteredVersionsForMethod(context); if (versionsForThisMethod.contains(currentVersion)) { return ConditionEvaluationResult.enabled( diff --git a/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/GradleVersioningClassTemplate.java b/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/GradleVersioningClassTemplate.java index 6f53c927..0a8dfcbb 100644 --- a/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/GradleVersioningClassTemplate.java +++ b/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/GradleVersioningClassTemplate.java @@ -33,7 +33,7 @@ public boolean supportsClassTemplate(ExtensionContext context) { @Override public Stream provideClassTemplateInvocationContexts( ExtensionContext context) { - return GradleVersions.allVersions(context).stream().map(GradleVersionInvocationContext::new); + return GradleVersions.allFilteredVersions(context).stream().map(GradleVersionInvocationContext::new); } private record GradleVersionInvocationContext(GradleVersion gradleVersion) diff --git a/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/GradleVersions.java b/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/GradleVersions.java index 7b606b33..66a8d6e3 100644 --- a/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/GradleVersions.java +++ b/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/GradleVersions.java @@ -36,11 +36,12 @@ final class GradleVersions { private static final String GRADLE_VERSIONS_CONFIG_PARAM = "com.palantir.gradle.testing.gradle_versions_to_test"; /** - * Returns all Gradle versions for the test class, including versions from all methods. + * Returns all Gradle versions for the test class, including versions from all methods, + * with the class-level {@link RestrictToGradleVersionsEqualTo} filter applied. * *

This is used to build the complete test matrix for the class. */ - static Set allVersions(ExtensionContext context) { + static Set allFilteredVersions(ExtensionContext context) { Set versions = configuredVersions(context); context.getTestClass().ifPresent(clazz -> { @@ -49,23 +50,27 @@ static Set allVersions(ExtensionContext context) { Arrays.stream(clazz.getDeclaredMethods()) .filter(GradleVersions::isTestMethod) .forEach(method -> versions.addAll(versionsFromAnnotation(method))); + + applyFilter(versions, clazz); }); return versions; } /** - * Returns all Gradle versions for a specific method, including class-level versions. + * Returns all Gradle versions for a specific method, including class-level versions, + * with the method-level {@link RestrictToGradleVersionsEqualTo} filter applied. * *

This is used to determine if a method should run for a given Gradle version. */ - static Set versionsForMethod(ExtensionContext context) { + static Set filteredVersionsForMethod(ExtensionContext context) { Set versions = configuredVersions(context); context.getTestClass().ifPresent(clazz -> versions.addAll(versionsFromAnnotation(clazz))); context.getTestMethod().ifPresent(method -> { versions.addAll(versionsFromAnnotation(method)); + applyFilter(versions, method); }); return versions; @@ -87,6 +92,16 @@ private static Set versionsFromAnnotation(AnnotatedElement elemen .collect(Collectors.toCollection(LinkedHashSet::new)); } + private static void applyFilter(Set versions, AnnotatedElement element) { + AnnotationSupport.findAnnotation(element, RestrictToGradleVersionsEqualTo.class) + .ifPresent(annotation -> { + Set allowed = Arrays.stream(annotation.value()) + .map(GradleVersion::new) + .collect(Collectors.toSet()); + versions.retainAll(allowed); + }); + } + private static boolean isTestMethod(Method method) { return AnnotationSupport.isAnnotated(method, Test.class); } diff --git a/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/RestrictToGradleVersionsEqualTo.java b/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/RestrictToGradleVersionsEqualTo.java new file mode 100644 index 00000000..8262e388 --- /dev/null +++ b/gradle-plugin-testing-junit/src/main/java/com/palantir/gradle/testing/junit/RestrictToGradleVersionsEqualTo.java @@ -0,0 +1,49 @@ +/* + * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.gradle.testing.junit; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation for restricting Gradle versions to only run the specified versions. + * + *

Unlike {@link AdditionallyRunWithGradle} which adds versions to the test matrix, this annotation filters the + * available versions to only include the specified ones. If a specified version is not in the test matrix + * (from configuration or {@code @AdditionallyRunWithGradle}), it will simply not run. + * + *

To run a specific version that isn't in the matrix, use both annotations: + * {@code @AdditionallyRunWithGradle("8.5")} and {@code @RestrictToGradleVersionsEqualTo("8.5")}. + */ +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface RestrictToGradleVersionsEqualTo { + + /** + * The Gradle versions to restrict to. + * @return an array of Gradle version strings (e.g., "7.6.5", "8.0") + */ + String[] value(); + + /** + * Optional reason explaining why this test is restricted to these specific Gradle versions. + * @return the reason for restricting to these specific versions + */ + String reason() default ""; +} diff --git a/gradle-plugin-testing-junit/src/test/java/com/palantir/gradle/testing/ete/RestrictToGradleVersionsEqualToTest.java b/gradle-plugin-testing-junit/src/test/java/com/palantir/gradle/testing/ete/RestrictToGradleVersionsEqualToTest.java new file mode 100644 index 00000000..5a22a803 --- /dev/null +++ b/gradle-plugin-testing-junit/src/test/java/com/palantir/gradle/testing/ete/RestrictToGradleVersionsEqualToTest.java @@ -0,0 +1,139 @@ +/* + * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.gradle.testing.ete; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.palantir.example.ClassLevelRestrictToGradleVersionsEqualToFixtureTest; +import com.palantir.example.RestrictToEqualToAndAdditionallyRunWithGradleFixtureTest; +import com.palantir.example.RestrictToGradleVersionsEqualToFixtureTest; +import java.util.List; +import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.junit.platform.engine.TestDescriptor; +import org.junit.platform.engine.discovery.DiscoverySelectors; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.platform.testkit.engine.Event; + +final class RestrictToGradleVersionsEqualToTest { + + @Test + void restrict_to_gradle_versions_equal_to_filters_to_specified_version() { + EngineExecutionResults executionResults = EngineTestKit.engine("junit-jupiter") + .selectors(DiscoverySelectors.selectClass(RestrictToGradleVersionsEqualToFixtureTest.class)) + .configurationParameter("com.palantir.gradle.testing.gradle_versions_to_test", "7.6.5,8.0") + .configurationParameter("com.palantir.gradle.testing.configuration_cache_enabled", "false") + .execute(); + + List finished = executionResults.testEvents().finished().stream().toList(); + List skipped = executionResults.testEvents().skipped().stream().toList(); + + assertThat(finished) + .satisfiesExactlyInAnyOrder( + // test_without_restrict_annotation runs on both base versions + ranWithNameAndVersion( + RestrictToGradleVersionsEqualToFixtureTest.class, + "test without restrict annotation", + "7.6.5"), + ranWithNameAndVersion( + RestrictToGradleVersionsEqualToFixtureTest.class, + "test without restrict annotation", + "8.0"), + // test_with_restrict_annotation_filtering_to_existing_version only runs on 8.0 + ranWithNameAndVersion( + RestrictToGradleVersionsEqualToFixtureTest.class, + "test with restrict annotation filtering to existing version", + "8.0")); + + assertThat(skipped).hasSize(3); + assertThat(skipped) + .satisfiesExactlyInAnyOrder( + // 7.6.5 skipped for "restrict to 8.0" test + skippedWithNameAndVersion( + "test with restrict annotation filtering to existing version", "7.6.5"), + // 7.6.5 skipped for "restrict to 8.5" test (nonexistent in matrix) + skippedWithNameAndVersion( + "test with restrict annotation filtering to nonexisting version", "7.6.5"), + // 8.0 skipped for "restrict to 8.5" test (nonexistent in matrix) + skippedWithNameAndVersion( + "test with restrict annotation filtering to nonexisting version", "8.0")); + } + + @Test + void restrict_to_equal_to_and_additionally_run_with_gradle_combined_adds_then_filters() { + EngineExecutionResults executionResults = EngineTestKit.engine("junit-jupiter") + .selectors( + DiscoverySelectors.selectClass(RestrictToEqualToAndAdditionallyRunWithGradleFixtureTest.class)) + .configurationParameter("com.palantir.gradle.testing.gradle_versions_to_test", "7.6.5,8.0") + .configurationParameter("com.palantir.gradle.testing.configuration_cache_enabled", "false") + .execute(); + + List finished = executionResults.testEvents().finished().stream().toList(); + List skipped = executionResults.testEvents().skipped().stream().toList(); + + assertThat(finished) + .satisfiesExactly(ranWithNameAndVersion( + RestrictToEqualToAndAdditionallyRunWithGradleFixtureTest.class, + "test with both annotations adding and restricting", + "8.5")); + + assertThat(skipped).hasSize(2); + assertThat(skipped) + .satisfiesExactlyInAnyOrder( + skippedWithNameAndVersion("test with both annotations adding and restricting", "7.6.5"), + skippedWithNameAndVersion("test with both annotations adding and restricting", "8.0")); + } + + @Test + void class_level_restrict_to_gradle_versions_equal_to_filters_matrix_upfront() { + EngineExecutionResults executionResults = EngineTestKit.engine("junit-jupiter") + .selectors(DiscoverySelectors.selectClass(ClassLevelRestrictToGradleVersionsEqualToFixtureTest.class)) + .configurationParameter("com.palantir.gradle.testing.gradle_versions_to_test", "7.6.5,8.0") + .configurationParameter("com.palantir.gradle.testing.configuration_cache_enabled", "false") + .execute(); + + List finished = executionResults.testEvents().finished().stream().toList(); + List skipped = executionResults.testEvents().skipped().stream().toList(); + + // Class-level @RestrictToGradleVersionsEqualTo("8.0") filters matrix to only 8.0 + // No tests should be skipped - 7.6.5 is not even in the matrix + assertThat(finished) + .satisfiesExactly(ranWithNameAndVersion( + ClassLevelRestrictToGradleVersionsEqualToFixtureTest.class, + "test runs only on restricted version", + "8.0")); + + assertThat(skipped).isEmpty(); + } + + public static Consumer ranWithNameAndVersion( + Class testClass, String displayNameContains, String gradleVersion) { + return event -> { + assertThat(event.getTestDescriptor().getDisplayName()).contains(displayNameContains); + Assertions.assertThatRanWithCorrectGradleVersion(testClass, event, gradleVersion, displayNameContains); + }; + } + + public static Consumer skippedWithNameAndVersion(String displayNameContains, String gradleVersion) { + return event -> { + assertThat(event.getTestDescriptor().getDisplayName()).contains(displayNameContains); + assertThat(event.getTestDescriptor().getParent().map(TestDescriptor::getDisplayName)) + .hasValue("Gradle " + gradleVersion); + }; + } +} diff --git a/gradle-plugin-testing-junit/src/testFixtures/java/com/palantir/example/ClassLevelRestrictToGradleVersionsEqualToFixtureTest.java b/gradle-plugin-testing-junit/src/testFixtures/java/com/palantir/example/ClassLevelRestrictToGradleVersionsEqualToFixtureTest.java new file mode 100644 index 00000000..b7c534ad --- /dev/null +++ b/gradle-plugin-testing-junit/src/testFixtures/java/com/palantir/example/ClassLevelRestrictToGradleVersionsEqualToFixtureTest.java @@ -0,0 +1,42 @@ +/* + * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.example; + +import com.palantir.gradle.testing.execution.GradleInvoker; +import com.palantir.gradle.testing.junit.GradlePluginTests; +import com.palantir.gradle.testing.junit.RestrictToGradleVersionsEqualTo; +import com.palantir.gradle.testing.project.RootProject; +import org.junit.jupiter.api.Test; + +/** + * Test fixture for testing class-level {@link RestrictToGradleVersionsEqualTo} annotation behavior. + * When applied at class level, the annotation restricts the test matrix upfront. + */ +@GradlePluginTests +@RestrictToGradleVersionsEqualTo("8.0") +public class ClassLevelRestrictToGradleVersionsEqualToFixtureTest { + + @Test + void test_runs_only_on_restricted_version(GradleInvoker gradleInvoker, RootProject rootProject) { + rootProject.buildGradle().append(""" + import org.gradle.util.GradleVersion + println "GradleVersion: ${GradleVersion.current().version}" + """); + + throw new RuntimeException(gradleInvoker.withArgs().buildsSuccessfully().output()); + } +} diff --git a/gradle-plugin-testing-junit/src/testFixtures/java/com/palantir/example/RestrictToEqualToAndAdditionallyRunWithGradleFixtureTest.java b/gradle-plugin-testing-junit/src/testFixtures/java/com/palantir/example/RestrictToEqualToAndAdditionallyRunWithGradleFixtureTest.java new file mode 100644 index 00000000..03675f68 --- /dev/null +++ b/gradle-plugin-testing-junit/src/testFixtures/java/com/palantir/example/RestrictToEqualToAndAdditionallyRunWithGradleFixtureTest.java @@ -0,0 +1,45 @@ +/* + * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.example; + +import com.palantir.gradle.testing.execution.GradleInvoker; +import com.palantir.gradle.testing.junit.AdditionallyRunWithGradle; +import com.palantir.gradle.testing.junit.GradlePluginTests; +import com.palantir.gradle.testing.junit.RestrictToGradleVersionsEqualTo; +import com.palantir.gradle.testing.project.RootProject; +import org.junit.jupiter.api.Test; + +/** + * Test fixture for testing {@link RestrictToGradleVersionsEqualTo} combined with {@link AdditionallyRunWithGradle}. + * This is in a separate fixture so that the {@link AdditionallyRunWithGradle} doesn't affect the test matrix of + * other tests. + */ +@GradlePluginTests +public class RestrictToEqualToAndAdditionallyRunWithGradleFixtureTest { + + @Test + @AdditionallyRunWithGradle("8.5") + @RestrictToGradleVersionsEqualTo("8.5") + void test_with_both_annotations_adding_and_restricting(GradleInvoker gradleInvoker, RootProject rootProject) { + rootProject.buildGradle().append(""" + import org.gradle.util.GradleVersion + println "GradleVersion: ${GradleVersion.current().version}" + """); + + throw new RuntimeException(gradleInvoker.withArgs().buildsSuccessfully().output()); + } +} diff --git a/gradle-plugin-testing-junit/src/testFixtures/java/com/palantir/example/RestrictToGradleVersionsEqualToFixtureTest.java b/gradle-plugin-testing-junit/src/testFixtures/java/com/palantir/example/RestrictToGradleVersionsEqualToFixtureTest.java new file mode 100644 index 00000000..1c3c5126 --- /dev/null +++ b/gradle-plugin-testing-junit/src/testFixtures/java/com/palantir/example/RestrictToGradleVersionsEqualToFixtureTest.java @@ -0,0 +1,66 @@ +/* + * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.example; + +import com.palantir.gradle.testing.execution.GradleInvoker; +import com.palantir.gradle.testing.junit.GradlePluginTests; +import com.palantir.gradle.testing.junit.RestrictToGradleVersionsEqualTo; +import com.palantir.gradle.testing.project.RootProject; +import org.junit.jupiter.api.Test; + +/** + * Test fixture for testing {@link RestrictToGradleVersionsEqualTo} annotation behavior. + * This fixture is designed to be run with base versions 7.6.5 and 8.0 via configuration parameter. + */ +@GradlePluginTests +public class RestrictToGradleVersionsEqualToFixtureTest { + + @Test + void test_without_restrict_annotation(GradleInvoker gradleInvoker, RootProject rootProject) { + rootProject.buildGradle().append(""" + import org.gradle.util.GradleVersion + println "GradleVersion: ${GradleVersion.current().version}" + """); + + // This exception is just so we can pass the output back up to the JUnit testkit-based test + throw new RuntimeException(gradleInvoker.withArgs().buildsSuccessfully().output()); + } + + @Test + @RestrictToGradleVersionsEqualTo("8.0") + void test_with_restrict_annotation_filtering_to_existing_version( + GradleInvoker gradleInvoker, RootProject rootProject) { + rootProject.buildGradle().append(""" + import org.gradle.util.GradleVersion + println "GradleVersion: ${GradleVersion.current().version}" + """); + + throw new RuntimeException(gradleInvoker.withArgs().buildsSuccessfully().output()); + } + + @Test + @RestrictToGradleVersionsEqualTo("8.5") + void test_with_restrict_annotation_filtering_to_nonexisting_version( + GradleInvoker gradleInvoker, RootProject rootProject) { + rootProject.buildGradle().append(""" + import org.gradle.util.GradleVersion + println "GradleVersion: ${GradleVersion.current().version}" + """); + + throw new RuntimeException(gradleInvoker.withArgs().buildsSuccessfully().output()); + } +}