diff --git a/perfkitbenchmarker/configs/default_benchmark_config.yaml b/perfkitbenchmarker/configs/default_benchmark_config.yaml
index cbd9ae1c41..80939e1681 100644
--- a/perfkitbenchmarker/configs/default_benchmark_config.yaml
+++ b/perfkitbenchmarker/configs/default_benchmark_config.yaml
@@ -415,9 +415,9 @@ sparksql_tpcds_1t_tuned: &sparksql_tpcds_1t_tuned
dpb_export_job_stats: True
hadoop_hdfs_client_readahead: 0
hadoop_hdfs_replication: 1
- hadoop_version: 3.4.2
+ hadoop_version: '3.4'
openjdk_version: 17
- spark_version: 3.5.8
+ spark_version: '3.5'
sparksql_readahead_kb: 1024
dpb_service:
service_type: unmanaged_spark_cluster
@@ -443,10 +443,10 @@ sparksql_tpcds_1t_tuned_2026:
<<: *sparksql_tpcds_1t_tuned
flags:
<<: *sparksql_tpcds_1t_tuned_flags
- hadoop_version: 3.5.0
+ hadoop_version: '3.5'
# Spark 4.2 supports Java 25, but Hadoop 3.5 does not.
openjdk_version: 21
- spark_version: 4.2.1
+ spark_version: '4.2'
sysbench_oltp_base: &sysbench_oltp_base
sysbench_table_size: 50000000
@@ -721,7 +721,7 @@ hadoop_dfsio:
hadoop_hdfs_replication: 1
hadoop_map_slots_per_core: 1
hadoop_reduce_slots_per_core: 1
- hadoop_version: 3.4.1
+ hadoop_version: '3.4'
num_cpus_override: 16
openjdk_version: 11
yarn_scheduler: org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
diff --git a/perfkitbenchmarker/linux_packages/hadoop.py b/perfkitbenchmarker/linux_packages/hadoop.py
index b822cb6e17..f97d617761 100644
--- a/perfkitbenchmarker/linux_packages/hadoop.py
+++ b/perfkitbenchmarker/linux_packages/hadoop.py
@@ -40,9 +40,8 @@
_VERSION = flags.DEFINE_string(
'hadoop_version',
- '3.4.1',
- 'Version of Hadoop. Default is 3.4.1 to get around'
- ' https://issues.apache.org/jira/projects/INFRA/issues/INFRA-27182.',
+ None,
+ 'Version of Hadoop. Defaults to latest. Can be a version prefix like 3.4.',
)
_URL_OVERRIDE = flags.DEFINE_string(
'hadoop_bin_url', None, 'Specify to override url from HADOOP_URL_BASE.'
@@ -108,8 +107,7 @@
START_HADOOP_SCRIPT = 'hadoop/start-hadoop.sh.j2'
HADOOP_URL_BASE = 'https://dlcdn.apache.org/hadoop/common'
-HADOOP_STABLE_URL = HADOOP_URL_BASE + '/stable'
-HADOOP_TAR_PATTERN = re.compile(r'hadoop-([0-9.]+)\.t(ar\.)?gz')
+HADOOP_VERSION_DIR_PATTERN = re.compile(r'hadoop-([0-9.]+)/')
HADOOP_DIR = posixpath.join(linux_packages.INSTALL_DIR, 'hadoop')
HADOOP_BIN = posixpath.join(HADOOP_DIR, 'bin')
@@ -127,8 +125,11 @@
@functools.lru_cache()
def HadoopVersion() -> version.Version:
"""Get provided or latest stable HadoopVersion."""
+ requested_version = None
if _VERSION.value:
- return version.Version(_VERSION.value)
+ requested_version = version.Version(str(_VERSION.value))
+ if len(requested_version.release) >= 3:
+ return requested_version
if _URL_OVERRIDE.value:
extracted_version = re.search(
r'[0-9]+\.[0-9]+\.[0-9]+', _URL_OVERRIDE.value
@@ -140,20 +141,28 @@ def HadoopVersion() -> version.Version:
'Cannot parse version out of --hadoop_bin_url please pass '
'--hadoop_version as well.'
)
- response = requests.get(HADOOP_STABLE_URL)
+ response = requests.get(HADOOP_URL_BASE)
if not response.ok:
raise errors.Setup.MissingExecutableError(
- 'Could not load ' + HADOOP_STABLE_URL
+ 'Could not load ' + HADOOP_URL_BASE
)
soup = bs4.BeautifulSoup(response.content, 'html.parser')
- link = soup.find('a', href=HADOOP_TAR_PATTERN)
- if link:
- match = re.match(HADOOP_TAR_PATTERN, link['href']) # pyrefly: ignore[bad-index, no-matching-overload]
+ found_versions = []
+ for link in soup.find_all('a', href=HADOOP_VERSION_DIR_PATTERN):
+ match = re.match(HADOOP_VERSION_DIR_PATTERN, link.get('href')) # pyrefly: ignore[missing-attribute]
if match:
- return version.Version(match.group(1))
- raise errors.Setup.MissingExecutableError(
- 'Could not find valid hadoop version at ' + HADOOP_STABLE_URL
- )
+ found_version = version.Version(match.group(1))
+ if (
+ requested_version is None
+ or found_version.release[: len(requested_version.release)]
+ == requested_version.release
+ ):
+ found_versions.append(found_version)
+ if not found_versions:
+ raise errors.Setup.MissingExecutableError(
+ 'Could not find valid hadoop versions at ' + HADOOP_URL_BASE
+ )
+ return max(found_versions)
def _GetHadoopURL():
diff --git a/perfkitbenchmarker/linux_packages/spark.py b/perfkitbenchmarker/linux_packages/spark.py
index 9624d11f45..d4b91b1957 100644
--- a/perfkitbenchmarker/linux_packages/spark.py
+++ b/perfkitbenchmarker/linux_packages/spark.py
@@ -39,7 +39,9 @@
FLAGS = flags.FLAGS
_SPARK_VERSION_FLAG = flags.DEFINE_string(
- 'spark_version', None, 'Version of spark. Defaults to latest.'
+ 'spark_version',
+ None,
+ 'Version of spark. Defaults to latest. Can be a version prefix like 3.5.',
)
_SHUFFLE_PARTITIONS = flags.DEFINE_integer(
@@ -71,8 +73,11 @@
@functools.lru_cache()
def SparkVersion() -> version.Version:
"""Get passed Spark version or latest available."""
+ requested_version = None
if _SPARK_VERSION_FLAG.value:
- return version.Version(_SPARK_VERSION_FLAG.value)
+ requested_version = version.Version(_SPARK_VERSION_FLAG.value)
+ if len(requested_version.release) >= 3:
+ return requested_version
# Find latest version
# N.B. requests handles a redirect from spark to spark/.
response = requests.get(SPARK_DOWNLOADS)
@@ -85,7 +90,13 @@ def SparkVersion() -> version.Version:
for link in soup.find_all('a', href=SPARK_VERSION_DIR_PATTERN):
match = re.match(SPARK_VERSION_DIR_PATTERN, link.get('href')) # pyrefly: ignore[missing-attribute]
if match:
- found_versions.append(version.Version(match.group(1)))
+ found_version = version.Version(match.group(1))
+ if (
+ requested_version is None
+ or found_version.release[: len(requested_version.release)]
+ == requested_version.release
+ ):
+ found_versions.append(found_version)
if not found_versions:
raise errors.Setup.MissingExecutableError(
'Could not find valid spark versions at ' + SPARK_DOWNLOADS
diff --git a/tests/linux_packages/hadoop_test.py b/tests/linux_packages/hadoop_test.py
index 725fa8110e..bb6bd3be74 100644
--- a/tests/linux_packages/hadoop_test.py
+++ b/tests/linux_packages/hadoop_test.py
@@ -9,33 +9,22 @@
from tests import pkb_common_test_case
import requests_mock
-HADOOP_STABLE_DIR = """\
+HADOOP_VERSION_LIST = """\
- Index of /hadoop/common/stable
+ Index of /hadoop/common
-Index of /hadoop/common/stable
+Index of /hadoop/common
Name Last modified Size Description
![[PARENTDIR]](/icons/back.gif)
Parent Directory -
-![[TXT]](/icons/text.gif)
CHANGELOG.md 2022-05-11 16:49 5.2K
- CHANGELOG.md.asc 2022-05-11 16:49 833
- CHANGELOG.md.sha512 2022-05-11 16:49 153
- RELEASENOTES.md 2022-05-11 16:49 2.0K
- RELEASENOTES.md.asc 2022-05-11 16:49 833
- RELEASENOTES.md.sha512 2022-05-11 16:49 156
- hadoop-3.3.3-rat.txt 2022-05-11 16:49 2.0M
- hadoop-3.3.3-rat.txt.asc 2022-05-11 16:49 833
- hadoop-3.3.3-rat.txt.sha512 2022-05-11 16:49 161
-![[ ]](/icons/compressed.gif)
hadoop-3.3.3-site.tar.gz 2022-05-11 16:49 42M
- hadoop-3.3.3-site.tar.gz.asc 2022-05-11 16:49 833
- hadoop-3.3.3-site.tar.gz.sha512 2022-05-11 16:49 165
- hadoop-3.3.3-src.tar.gz 2022-05-11 16:49 34M
- hadoop-3.3.3-src.tar.gz.asc 2022-05-11 16:49 833
- hadoop-3.3.3-src.tar.gz.sha512 2022-05-11 16:49 164
- hadoop-3.3.3.tar.gz 2022-05-11 16:49 615M
- hadoop-3.3.3.tar.gz.asc 2022-05-11 16:49 833
- hadoop-3.3.3.tar.gz.sha512 2022-05-11 16:49 160
+![[DIR]](/icons/folder.gif)
hadoop-2.10.2/ 2022-06-17 11:13 -
+ hadoop-3.2.4/ 2022-06-17 11:12 -
+ hadoop-3.3.5/ 2022-06-17 11:12 -
+ hadoop-3.3.6/ 2022-07-15 14:43 -
+ hadoop-3.4.0/ 2024-03-15 14:43 -
+ hadoop-3.4.1/ 2024-10-15 14:43 -
+![[ ]](/icons/unknown.gif)
KEYS 2022-06-15 08:08 102K
"""
@@ -48,41 +37,38 @@ def setUp(self):
hadoop.HadoopVersion.cache_clear()
@requests_mock.Mocker()
- @flagsaver.flagsaver(hadoop_version='')
def testDefaultHadoopVersion(self, mock_requests):
mock_requests.get(
- 'https://dlcdn.apache.org/hadoop/common/stable',
- text=HADOOP_STABLE_DIR,
+ 'https://dlcdn.apache.org/hadoop/common',
+ text=HADOOP_VERSION_LIST,
)
for _ in range(5):
observed = hadoop.HadoopVersion()
- self.assertEqual(version.Version('3.3.3'), observed)
+ self.assertEqual(version.Version('3.4.1'), observed)
# test caching
self.assertEqual(1, mock_requests.call_count)
@requests_mock.Mocker()
- @flagsaver.flagsaver(hadoop_version='')
def testHadoopVersionConnectionError(self, mock_requests):
mock_requests.get(
- 'https://dlcdn.apache.org/hadoop/common/stable', status_code=404
+ 'https://dlcdn.apache.org/hadoop/common', status_code=404
)
with self.assertRaisesRegex(
errors.Setup.MissingExecutableError,
- 'Could not load https://dlcdn.apache.org/hadoop/common/stable',
+ 'Could not load https://dlcdn.apache.org/hadoop/common',
):
hadoop.HadoopVersion()
@requests_mock.Mocker()
- @flagsaver.flagsaver(hadoop_version='')
def testHadoopVersionParsingError(self, mock_requests):
mock_requests.get(
- 'https://dlcdn.apache.org/hadoop/common/stable',
+ 'https://dlcdn.apache.org/hadoop/common',
text='bar',
)
with self.assertRaisesRegex(
errors.Setup.MissingExecutableError,
- 'Could not find valid hadoop version at '
- 'https://dlcdn.apache.org/hadoop/common/stable',
+ 'Could not find valid hadoop versions at '
+ 'https://dlcdn.apache.org/hadoop/common',
):
hadoop.HadoopVersion()
@@ -102,6 +88,30 @@ def testHadoopVersionUrlOverride(self, mock_requests):
self.assertFalse(mock_requests.called)
self.assertEqual(version.Version('4.2.0'), observed)
+ @requests_mock.Mocker()
+ @flagsaver.flagsaver(hadoop_version='3.3')
+ def testHadoopVersionPrefix(self, mock_requests):
+ mock_requests.get(
+ 'https://dlcdn.apache.org/hadoop/common', text=HADOOP_VERSION_LIST
+ )
+ for _ in range(5):
+ observed = hadoop.HadoopVersion()
+ self.assertEqual(version.Version('3.3.6'), observed)
+ self.assertEqual(1, mock_requests.call_count)
+
+ @requests_mock.Mocker()
+ @flagsaver.flagsaver(hadoop_version='3.9')
+ def testHadoopVersionPrefixNotFound(self, mock_requests):
+ mock_requests.get(
+ 'https://dlcdn.apache.org/hadoop/common', text=HADOOP_VERSION_LIST
+ )
+ with self.assertRaisesRegex(
+ errors.Setup.MissingExecutableError,
+ 'Could not find valid hadoop versions at '
+ 'https://dlcdn.apache.org/hadoop/common',
+ ):
+ hadoop.HadoopVersion()
+
if __name__ == '__main__':
unittest.main()
diff --git a/tests/linux_packages/spark_test.py b/tests/linux_packages/spark_test.py
index eb1b037a1d..557b6540a1 100644
--- a/tests/linux_packages/spark_test.py
+++ b/tests/linux_packages/spark_test.py
@@ -22,6 +22,7 @@
spark-3.0.3/ 2022-06-17 11:12 -
spark-3.1.3/ 2022-06-17 11:12 -
spark-3.2.2/ 2022-07-15 14:43 -
+ spark-3.2.4/ 2022-07-15 14:43 -
spark-3.3.0/ 2022-06-17 11:11 -
KEYS 2022-06-15 08:08 102K
@@ -75,6 +76,30 @@ def testSparkVersionProvider(self, mock_requests):
self.assertFalse(mock_requests.called)
self.assertEqual(version.Version('4.2.0'), observed)
+ @requests_mock.Mocker()
+ @flagsaver.flagsaver(spark_version='3.2')
+ def testSparkVersionPrefix(self, mock_requests):
+ mock_requests.get(
+ 'https://dlcdn.apache.org/spark', text=SPARK_VERSION_LIST
+ )
+ for _ in range(5):
+ observed = spark.SparkVersion()
+ self.assertEqual(version.Version('3.2.4'), observed)
+ self.assertEqual(1, mock_requests.call_count)
+
+ @requests_mock.Mocker()
+ @flagsaver.flagsaver(spark_version='3.9')
+ def testSparkVersionPrefixNotFound(self, mock_requests):
+ mock_requests.get(
+ 'https://dlcdn.apache.org/spark', text=SPARK_VERSION_LIST
+ )
+ with self.assertRaisesRegex(
+ errors.Setup.MissingExecutableError,
+ 'Could not find valid spark versions at '
+ 'https://dlcdn.apache.org/spark',
+ ):
+ spark.SparkVersion()
+
if __name__ == '__main__':
unittest.main()