Skip to content

Support passing major.minor for Hadoop and Spark versions. #7242

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Nov 10, 2026
Merged

Support passing major.minor for Hadoop and Spark versions. #7242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions perfkitbenchmarker/configs/default_benchmark_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,9 +415,9 @@ sparksql_tpcds_1t_tuned: &sparksql_tpcds_1t_tuned
dpb_export_job_stats: True
hadoop_hdfs_client_readahead: 0
hadoop_hdfs_replication: 1
hadoop_version: 3.4.2
hadoop_version: '3.4'
openjdk_version: 17
spark_version: 3.5.8
spark_version: '3.5'
sparksql_readahead_kb: 1024
dpb_service:
service_type: unmanaged_spark_cluster
Expand All @@ -443,10 +443,10 @@ sparksql_tpcds_1t_tuned_2026:
<<: *sparksql_tpcds_1t_tuned
flags:
<<: *sparksql_tpcds_1t_tuned_flags
hadoop_version: 3.5.0
hadoop_version: '3.5'
# Spark 4.2 supports Java 25, but Hadoop 3.5 does not.
openjdk_version: 21
spark_version: 4.2.1
spark_version: '4.2'

sysbench_oltp_base: &sysbench_oltp_base
sysbench_table_size: 50000000
Expand Down Expand Up @@ -721,7 +721,7 @@ hadoop_dfsio:
hadoop_hdfs_replication: 1
hadoop_map_slots_per_core: 1
hadoop_reduce_slots_per_core: 1
hadoop_version: 3.4.1
hadoop_version: '3.4'
num_cpus_override: 16
openjdk_version: 11
yarn_scheduler: org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
Expand Down
39 changes: 24 additions & 15 deletions perfkitbenchmarker/linux_packages/hadoop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,8 @@

_VERSION = flags.DEFINE_string(
'hadoop_version',
'3.4.1',
'Version of Hadoop. Default is 3.4.1 to get around'
' https://issues.apache.org/jira/projects/INFRA/issues/INFRA-27182.',
None,
'Version of Hadoop. Defaults to latest. Can be a version prefix like 3.4.',
)
_URL_OVERRIDE = flags.DEFINE_string(
'hadoop_bin_url', None, 'Specify to override url from HADOOP_URL_BASE.'
Expand Down Expand Up @@ -108,8 +107,7 @@
START_HADOOP_SCRIPT = 'hadoop/start-hadoop.sh.j2'

HADOOP_URL_BASE = 'https://dlcdn.apache.org/hadoop/common'
HADOOP_STABLE_URL = HADOOP_URL_BASE + '/stable'
HADOOP_TAR_PATTERN = re.compile(r'hadoop-([0-9.]+)\.t(ar\.)?gz')
HADOOP_VERSION_DIR_PATTERN = re.compile(r'hadoop-([0-9.]+)/')

HADOOP_DIR = posixpath.join(linux_packages.INSTALL_DIR, 'hadoop')
HADOOP_BIN = posixpath.join(HADOOP_DIR, 'bin')
Expand All @@ -127,8 +125,11 @@
@functools.lru_cache()
def HadoopVersion() -> version.Version:
"""Get provided or latest stable HadoopVersion."""
requested_version = None
if _VERSION.value:
return version.Version(_VERSION.value)
requested_version = version.Version(str(_VERSION.value))
if len(requested_version.release) >= 3:
return requested_version
if _URL_OVERRIDE.value:
extracted_version = re.search(
r'[0-9]+\.[0-9]+\.[0-9]+', _URL_OVERRIDE.value
Expand All @@ -140,20 +141,28 @@ def HadoopVersion() -> version.Version:
'Cannot parse version out of --hadoop_bin_url please pass '
'--hadoop_version as well.'
)
response = requests.get(HADOOP_STABLE_URL)
response = requests.get(HADOOP_URL_BASE)
if not response.ok:
raise errors.Setup.MissingExecutableError(
'Could not load ' + HADOOP_STABLE_URL
'Could not load ' + HADOOP_URL_BASE
)
soup = bs4.BeautifulSoup(response.content, 'html.parser')
link = soup.find('a', href=HADOOP_TAR_PATTERN)
if link:
match = re.match(HADOOP_TAR_PATTERN, link['href']) # pyrefly: ignore[bad-index, no-matching-overload]
found_versions = []
for link in soup.find_all('a', href=HADOOP_VERSION_DIR_PATTERN):
match = re.match(HADOOP_VERSION_DIR_PATTERN, link.get('href')) # pyrefly: ignore[missing-attribute]
if match:
return version.Version(match.group(1))
raise errors.Setup.MissingExecutableError(
'Could not find valid hadoop version at ' + HADOOP_STABLE_URL
)
found_version = version.Version(match.group(1))
if (
requested_version is None
or found_version.release[: len(requested_version.release)]
== requested_version.release
):
found_versions.append(found_version)
if not found_versions:
raise errors.Setup.MissingExecutableError(
'Could not find valid hadoop versions at ' + HADOOP_URL_BASE
)
return max(found_versions)


def _GetHadoopURL():
Expand Down
17 changes: 14 additions & 3 deletions perfkitbenchmarker/linux_packages/spark.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@
FLAGS = flags.FLAGS

_SPARK_VERSION_FLAG = flags.DEFINE_string(
'spark_version', None, 'Version of spark. Defaults to latest.'
'spark_version',
None,
'Version of spark. Defaults to latest. Can be a version prefix like 3.5.',
)

_SHUFFLE_PARTITIONS = flags.DEFINE_integer(
Expand Down Expand Up @@ -71,8 +73,11 @@
@functools.lru_cache()
def SparkVersion() -> version.Version:
"""Get passed Spark version or latest available."""
requested_version = None
if _SPARK_VERSION_FLAG.value:
return version.Version(_SPARK_VERSION_FLAG.value)
requested_version = version.Version(_SPARK_VERSION_FLAG.value)
if len(requested_version.release) >= 3:
return requested_version
# Find latest version
# N.B. requests handles a redirect from spark to spark/.
response = requests.get(SPARK_DOWNLOADS)
Expand All @@ -85,7 +90,13 @@ def SparkVersion() -> version.Version:
for link in soup.find_all('a', href=SPARK_VERSION_DIR_PATTERN):
match = re.match(SPARK_VERSION_DIR_PATTERN, link.get('href')) # pyrefly: ignore[missing-attribute]
if match:
found_versions.append(version.Version(match.group(1)))
found_version = version.Version(match.group(1))
if (
requested_version is None
or found_version.release[: len(requested_version.release)]
== requested_version.release
):
found_versions.append(found_version)
if not found_versions:
raise errors.Setup.MissingExecutableError(
'Could not find valid spark versions at ' + SPARK_DOWNLOADS
Expand Down
74 changes: 42 additions & 32 deletions tests/linux_packages/hadoop_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,33 +9,22 @@
from tests import pkb_common_test_case
import requests_mock

HADOOP_STABLE_DIR = """\
HADOOP_VERSION_LIST = """\
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<html>
<head>
<title>Index of /hadoop/common/stable</title>
<title>Index of /hadoop/common</title>
</head>
<body>
<h1>Index of /hadoop/common/stable</h1>
<h1>Index of /hadoop/common</h1>
<pre><img src="/icons/blank.gif" alt="Icon "> <a href="?C=N;O=D">Name</a> <a href="?C=M;O=A">Last modified</a> <a href="?C=S;O=A">Size</a> <a href="?C=D;O=A">Description</a><hr><img src="/icons/back.gif" alt="[PARENTDIR]"> <a href="/hadoop/common/">Parent Directory</a> -
<img src="/icons/text.gif" alt="[TXT]"> <a href="CHANGELOG.md">CHANGELOG.md</a> 2022-05-11 16:49 5.2K
<img src="/icons/text.gif" alt="[TXT]"> <a href="CHANGELOG.md.asc">CHANGELOG.md.asc</a> 2022-05-11 16:49 833
<img src="/icons/text.gif" alt="[TXT]"> <a href="CHANGELOG.md.sha512">CHANGELOG.md.sha512</a> 2022-05-11 16:49 153
<img src="/icons/text.gif" alt="[TXT]"> <a href="RELEASENOTES.md">RELEASENOTES.md</a> 2022-05-11 16:49 2.0K
<img src="/icons/text.gif" alt="[TXT]"> <a href="RELEASENOTES.md.asc">RELEASENOTES.md.asc</a> 2022-05-11 16:49 833
<img src="/icons/text.gif" alt="[TXT]"> <a href="RELEASENOTES.md.sha512">RELEASENOTES.md.sha512</a> 2022-05-11 16:49 156
<img src="/icons/text.gif" alt="[TXT]"> <a href="hadoop-3.3.3-rat.txt">hadoop-3.3.3-rat.txt</a> 2022-05-11 16:49 2.0M
<img src="/icons/text.gif" alt="[TXT]"> <a href="hadoop-3.3.3-rat.txt.asc">hadoop-3.3.3-rat.txt.asc</a> 2022-05-11 16:49 833
<img src="/icons/text.gif" alt="[TXT]"> <a href="hadoop-3.3.3-rat.txt.sha512">hadoop-3.3.3-rat.txt.sha512</a> 2022-05-11 16:49 161
<img src="/icons/compressed.gif" alt="[ ]"> <a href="hadoop-3.3.3-site.tar.gz">hadoop-3.3.3-site.tar.gz</a> 2022-05-11 16:49 42M
<img src="/icons/text.gif" alt="[TXT]"> <a href="hadoop-3.3.3-site.tar.gz.asc">hadoop-3.3.3-site.tar.gz.asc</a> 2022-05-11 16:49 833
<img src="/icons/text.gif" alt="[TXT]"> <a href="hadoop-3.3.3-site.tar.gz.sha512">hadoop-3.3.3-site.tar.gz.sha512</a> 2022-05-11 16:49 165
<img src="/icons/compressed.gif" alt="[ ]"> <a href="hadoop-3.3.3-src.tar.gz">hadoop-3.3.3-src.tar.gz</a> 2022-05-11 16:49 34M
<img src="/icons/text.gif" alt="[TXT]"> <a href="hadoop-3.3.3-src.tar.gz.asc">hadoop-3.3.3-src.tar.gz.asc</a> 2022-05-11 16:49 833
<img src="/icons/text.gif" alt="[TXT]"> <a href="hadoop-3.3.3-src.tar.gz.sha512">hadoop-3.3.3-src.tar.gz.sha512</a> 2022-05-11 16:49 164
<img src="/icons/compressed.gif" alt="[ ]"> <a href="hadoop-3.3.3.tar.gz">hadoop-3.3.3.tar.gz</a> 2022-05-11 16:49 615M
<img src="/icons/text.gif" alt="[TXT]"> <a href="hadoop-3.3.3.tar.gz.asc">hadoop-3.3.3.tar.gz.asc</a> 2022-05-11 16:49 833
<img src="/icons/text.gif" alt="[TXT]"> <a href="hadoop-3.3.3.tar.gz.sha512">hadoop-3.3.3.tar.gz.sha512</a> 2022-05-11 16:49 160
<img src="/icons/folder.gif" alt="[DIR]"> <a href="hadoop-2.10.2/">hadoop-2.10.2/</a> 2022-06-17 11:13 -
<img src="/icons/folder.gif" alt="[DIR]"> <a href="hadoop-3.2.4/">hadoop-3.2.4/</a> 2022-06-17 11:12 -
<img src="/icons/folder.gif" alt="[DIR]"> <a href="hadoop-3.3.5/">hadoop-3.3.5/</a> 2022-06-17 11:12 -
<img src="/icons/folder.gif" alt="[DIR]"> <a href="hadoop-3.3.6/">hadoop-3.3.6/</a> 2022-07-15 14:43 -
<img src="/icons/folder.gif" alt="[DIR]"> <a href="hadoop-3.4.0/">hadoop-3.4.0/</a> 2024-03-15 14:43 -
<img src="/icons/folder.gif" alt="[DIR]"> <a href="hadoop-3.4.1/">hadoop-3.4.1/</a> 2024-10-15 14:43 -
<img src="/icons/unknown.gif" alt="[ ]"> <a href="KEYS">KEYS</a> 2022-06-15 08:08 102K
<hr></pre>
</body></html>
"""
Expand All @@ -48,41 +37,38 @@ def setUp(self):
hadoop.HadoopVersion.cache_clear()

@requests_mock.Mocker()
@flagsaver.flagsaver(hadoop_version='')
def testDefaultHadoopVersion(self, mock_requests):
mock_requests.get(
'https://dlcdn.apache.org/hadoop/common/stable',
text=HADOOP_STABLE_DIR,
'https://dlcdn.apache.org/hadoop/common',
text=HADOOP_VERSION_LIST,
)
for _ in range(5):
observed = hadoop.HadoopVersion()
self.assertEqual(version.Version('3.3.3'), observed)
self.assertEqual(version.Version('3.4.1'), observed)
# test caching
self.assertEqual(1, mock_requests.call_count)

@requests_mock.Mocker()
@flagsaver.flagsaver(hadoop_version='')
def testHadoopVersionConnectionError(self, mock_requests):
mock_requests.get(
'https://dlcdn.apache.org/hadoop/common/stable', status_code=404
'https://dlcdn.apache.org/hadoop/common', status_code=404
)
with self.assertRaisesRegex(
errors.Setup.MissingExecutableError,
'Could not load https://dlcdn.apache.org/hadoop/common/stable',
'Could not load https://dlcdn.apache.org/hadoop/common',
):
hadoop.HadoopVersion()

@requests_mock.Mocker()
@flagsaver.flagsaver(hadoop_version='')
def testHadoopVersionParsingError(self, mock_requests):
mock_requests.get(
'https://dlcdn.apache.org/hadoop/common/stable',
'https://dlcdn.apache.org/hadoop/common',
text='<html><body><a href="foo">bar</a></body></html>',
)
with self.assertRaisesRegex(
errors.Setup.MissingExecutableError,
'Could not find valid hadoop version at '
'https://dlcdn.apache.org/hadoop/common/stable',
'Could not find valid hadoop versions at '
'https://dlcdn.apache.org/hadoop/common',
):
hadoop.HadoopVersion()

Expand All @@ -102,6 +88,30 @@ def testHadoopVersionUrlOverride(self, mock_requests):
self.assertFalse(mock_requests.called)
self.assertEqual(version.Version('4.2.0'), observed)

@requests_mock.Mocker()
@flagsaver.flagsaver(hadoop_version='3.3')
def testHadoopVersionPrefix(self, mock_requests):
mock_requests.get(
'https://dlcdn.apache.org/hadoop/common', text=HADOOP_VERSION_LIST
)
for _ in range(5):
observed = hadoop.HadoopVersion()
self.assertEqual(version.Version('3.3.6'), observed)
self.assertEqual(1, mock_requests.call_count)

@requests_mock.Mocker()
@flagsaver.flagsaver(hadoop_version='3.9')
def testHadoopVersionPrefixNotFound(self, mock_requests):
mock_requests.get(
'https://dlcdn.apache.org/hadoop/common', text=HADOOP_VERSION_LIST
)
with self.assertRaisesRegex(
errors.Setup.MissingExecutableError,
'Could not find valid hadoop versions at '
'https://dlcdn.apache.org/hadoop/common',
):
hadoop.HadoopVersion()


if __name__ == '__main__':
unittest.main()
25 changes: 25 additions & 0 deletions tests/linux_packages/spark_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
<img src="/icons/folder.gif" alt="[DIR]"> <a href="spark-3.0.3/">spark-3.0.3/</a> 2022-06-17 11:12 -
<img src="/icons/folder.gif" alt="[DIR]"> <a href="spark-3.1.3/">spark-3.1.3/</a> 2022-06-17 11:12 -
<img src="/icons/folder.gif" alt="[DIR]"> <a href="spark-3.2.2/">spark-3.2.2/</a> 2022-07-15 14:43 -
<img src="/icons/folder.gif" alt="[DIR]"> <a href="spark-3.2.4/">spark-3.2.4/</a> 2022-07-15 14:43 -
<img src="/icons/folder.gif" alt="[DIR]"> <a href="spark-3.3.0/">spark-3.3.0/</a> 2022-06-17 11:11 -
<img src="/icons/unknown.gif" alt="[ ]"> <a href="KEYS">KEYS</a> 2022-06-15 08:08 102K
<hr></pre>
Expand Down Expand Up @@ -75,6 +76,30 @@ def testSparkVersionProvider(self, mock_requests):
self.assertFalse(mock_requests.called)
self.assertEqual(version.Version('4.2.0'), observed)

@requests_mock.Mocker()
@flagsaver.flagsaver(spark_version='3.2')
def testSparkVersionPrefix(self, mock_requests):
mock_requests.get(
'https://dlcdn.apache.org/spark', text=SPARK_VERSION_LIST
)
for _ in range(5):
observed = spark.SparkVersion()
self.assertEqual(version.Version('3.2.4'), observed)
self.assertEqual(1, mock_requests.call_count)

@requests_mock.Mocker()
@flagsaver.flagsaver(spark_version='3.9')
def testSparkVersionPrefixNotFound(self, mock_requests):
mock_requests.get(
'https://dlcdn.apache.org/spark', text=SPARK_VERSION_LIST
)
with self.assertRaisesRegex(
errors.Setup.MissingExecutableError,
'Could not find valid spark versions at '
'https://dlcdn.apache.org/spark',
):
spark.SparkVersion()


if __name__ == '__main__':
unittest.main()
Loading