diff --git a/obstore/python/obstore/_get.pyi b/obstore/python/obstore/_get.pyi index ff0cbf37..ed062d17 100644 --- a/obstore/python/obstore/_get.pyi +++ b/obstore/python/obstore/_get.pyi @@ -328,19 +328,33 @@ def get_range( ) -> Bytes: """Return the bytes that are stored at the specified location in the given byte range. - If the given range is zero-length or starts after the end of the object, an error - will be returned. Additionally, if the range ends after the end of the object, the - entire remainder of the object will be returned. Otherwise, the exact requested - range will be returned. + The requested range may be bounded, open-ended, or relative to the end of the + object: + + - `start` plus either `end` or `length` requests a specific range of bytes. + If the given range is zero-length or starts after the end of the object, an + error will be returned. Additionally, if the range ends after the end of the + object, the entire remainder of the object will be returned. Otherwise, the + exact requested range will be returned. + - A non-negative `start` on its own requests all bytes from `start` onwards. + This is equivalent to `bytes={start}-` as an HTTP header. + - A negative `start` on its own requests the last `abs(start)` bytes. Note that + here, `abs(start)` is _the size of the request_, not a byte offset. This is + equivalent to `bytes=-{abs(start)}` as an HTTP header. Args: store: The ObjectStore instance to use. path: The path within ObjectStore to retrieve. Keyword Args: - start: The start of the byte range. - end: The end of the byte range (exclusive). Either `end` or `length` must be non-None. - length: The number of bytes of the byte range. Either `end` or `length` must be non-None. + start: The start of the byte range. If negative, the last `abs(start)` bytes + of the object are requested, and `end` and `length` must both be None. + end: The end of the byte range (exclusive). Mutually exclusive with `length`. + Defaults to None, in which case the range continues to the end of the + object. + length: The number of bytes of the byte range. Mutually exclusive with `end`. + Defaults to None, in which case the range continues to the end of the + object. Returns: A `Bytes` object implementing the Python buffer protocol, allowing @@ -366,27 +380,53 @@ def get_ranges( path: str, *, starts: Sequence[int], - ends: Sequence[int] | None = None, - lengths: Sequence[int] | None = None, - coalesce: int = 1024 * 1024, + ends: Sequence[int | None] | None = None, + lengths: Sequence[int | None] | None = None, + coalesce: int | None = None, ) -> list[Bytes]: """Return the bytes stored at the specified location in the given byte ranges. + Each range is described by one element of `starts` and the element at the same + position in `ends` or `lengths`, and follows the same semantics as + [get_range][obstore.get_range]. In particular, a negative element of `starts` + requests the last `abs(start)` bytes, and a range whose end is left unspecified + continues to the end of the object. Because ranges are matched up by position, + every sequence given must have the same length as `starts`; a mismatch raises + `ValueError`. + + `ends` and `lengths` may both be given at once, so long as at most one of the two + is non-None for any individual range — for example `ends=[10, None], + lengths=[None, 5]` bounds the first range by its end offset and the second by its + length. Omitting both reads every range to the end of the object. + To improve performance this will: - Transparently combine ranges less than `coalesce` bytes apart into a single underlying request (defaults to 1MB) - Make multiple `fetch` requests in parallel (up to maximum of 10) + !!! note + + Combining ranges requires knowing where each one ends, so if any requested + range is open-ended or relative to the end of the object, one additional + `head` request is made to resolve the size of the object. Requests made up + entirely of bounded ranges do not incur this cost. + Args: store: The ObjectStore instance to use. path: The path within ObjectStore to retrieve. Other Args: - starts: A sequence of `int` where each offset starts. - ends: A sequence of `int` where each offset ends (exclusive). Either `ends` or `lengths` must be non-None. - lengths: A sequence of `int` with the number of bytes of each byte range. Either `ends` or `lengths` must be non-None. - coalesce: Maximum distance in bytes between ranges that will be coalesced into a single request. Defaults to 1MiB. Set to `0` to disable coalescing. + starts: A sequence of `int` where each offset starts. A negative value + requests the last `abs(start)` bytes for that range, in which case the + corresponding elements of `ends` and `lengths` must be None. + ends: A sequence of `int` where each offset ends (exclusive). An element may + be None to continue that range to the end of the object. + lengths: A sequence of `int` with the number of bytes of each byte range. An + element may be None to continue that range to the end of the object. + coalesce: Maximum distance in bytes between ranges that will be coalesced + into a single request. Defaults to None, in which case obstore applies + its own default of 1MiB. Set to `0` to disable coalescing. Returns: A sequence of `Bytes`, one for each range. This `Bytes` object implements the @@ -400,9 +440,9 @@ async def get_ranges_async( path: str, *, starts: Sequence[int], - ends: Sequence[int] | None = None, - lengths: Sequence[int] | None = None, - coalesce: int = 1024 * 1024, + ends: Sequence[int | None] | None = None, + lengths: Sequence[int | None] | None = None, + coalesce: int | None = None, ) -> list[Bytes]: """Call `get_ranges` asynchronously. diff --git a/obstore/python/obstore/fsspec.py b/obstore/python/obstore/fsspec.py index d6c76a92..3e9409bc 100644 --- a/obstore/python/obstore/fsspec.py +++ b/obstore/python/obstore/fsspec.py @@ -35,9 +35,10 @@ import asyncio import warnings from collections import defaultdict +from collections.abc import Iterable from functools import cached_property, lru_cache from pathlib import Path -from typing import TYPE_CHECKING, Literal, overload +from typing import TYPE_CHECKING, Literal, cast, overload from urllib.parse import urlparse import fsspec.asyn @@ -49,7 +50,7 @@ if TYPE_CHECKING: import sys - from collections.abc import Coroutine, Iterable + from collections.abc import Coroutine, Sequence from datetime import datetime from typing import Any @@ -113,6 +114,35 @@ """A type hint for all supported protocols.""" +def _needs_object_size(start: int | None, end: int | None) -> bool: + """Whether a range must be resolved against the object size first. + + A negative `start` on its own is a suffix request, which obstore supports + directly. Anything else counting back from the end of the object is not. + """ + return end is not None and (end < 0 or (start is not None and start < 0)) + + +def _apply_object_size( + start: int | None, + end: int | None, + size: int | None, +) -> tuple[int, int | None]: + """Rewrite the bounds of a range that count back from the end of an object. + + A `size` of None leaves the range alone, for objects whose size was never + needed. + """ + if start is None: + start = 0 + if size is None: + return start, end + return ( + max(0, size + start) if start < 0 else start, + max(0, size + end) if end is not None and end < 0 else end, + ) + + class FsspecStore(fsspec.asyn.AsyncFileSystem): """An fsspec implementation based on a obstore Store. @@ -389,12 +419,19 @@ async def _cat_file( resp = await store.get_async(path) return (await resp.bytes_async()).to_bytes() - if start is None or end is None: - raise NotImplementedError( - "cat_file not implemented for start=None xor end=None", - ) - - range_bytes = await store.get_range_async(path, start=start, end=end) + if _needs_object_size(start, end): + # See `_resolve_ranges` for why this needs the object size. + size = (await store.head_async(path))["size"] + if start is not None and start < 0: + start = max(0, size + start) + if end is not None and end < 0: + end = max(0, size + end) + + range_bytes = await store.get_range_async( + path, + start=0 if start is None else start, + end=end, + ) return range_bytes.to_bytes() async def _cat( # type: ignore (fsspec has bad typing) @@ -426,23 +463,30 @@ async def _cat( # type: ignore (fsspec has bad typing) async def _cat_ranges( # noqa: PLR0913, PLR0917 # type: ignore (fsspec has bad typing) self, paths: list[str], - starts: list[int] | int, - ends: list[int] | int, - max_gap=None, # noqa: ANN001, ARG002 - batch_size=None, # noqa: ANN001, ARG002 - on_error="return", # noqa: ANN001, ARG002 + starts: Sequence[int | None] | int | None, + ends: Sequence[int | None] | int | None, + max_gap: int | None = None, + batch_size: int | None = None, # noqa: ARG002 + on_error: str = "return", # noqa: ARG002 **_kwargs: Any, ) -> list[bytes]: - if isinstance(starts, int): + # A non-iterable start or end applies to every path, per fsspec's + # AsyncFileSystem._cat_ranges. + if not isinstance(starts, Iterable): starts = [starts] * len(paths) - if isinstance(ends, int): + if not isinstance(ends, Iterable): ends = [ends] * len(paths) if not len(paths) == len(starts) == len(ends): raise ValueError - per_file_requests: dict[str, list[tuple[int, int, int]]] = defaultdict(list) - # When upgrading to Python 3.10, use strict=True - for idx, (path, start, end) in enumerate(zip(paths, starts, ends)): + resolved = await self._resolve_ranges(paths, starts, ends) + + per_file_requests: dict[str, list[tuple[int, int | None, int]]] = defaultdict( + list, + ) + for idx, (path, (start, end)) in enumerate( + zip(paths, resolved, strict=True), + ): per_file_requests[path].append((start, end, idx)) futs: list[Coroutine[Any, Any, list[Bytes]]] = [] @@ -451,23 +495,65 @@ async def _cat_ranges( # noqa: PLR0913, PLR0917 # type: ignore (fsspec has bad store = self._construct_store(bucket) offsets = [r[0] for r in ranges] - ends = [r[1] for r in ranges] - fut = store.get_ranges_async(path_no_bucket, starts=offsets, ends=ends) + file_ends = [r[1] for r in ranges] + # fsspec's `max_gap` is obstore's `coalesce`: the largest gap between + # two ranges that may still be served by a single request. + fut = store.get_ranges_async( + path_no_bucket, + starts=offsets, + ends=file_ends, + coalesce=max_gap, + ) futs.append(fut) result = await asyncio.gather(*futs) output_buffers: list[bytes] = [b""] * len(paths) - # When upgrading to Python 3.10, use strict=True - for per_file_request, buffers in zip(per_file_requests.items(), result): + for per_file_request, buffers in zip( + per_file_requests.items(), + result, + strict=True, + ): path, ranges = per_file_request - # When upgrading to Python 3.10, use strict=True - for buffer, ranges_ in zip(buffers, ranges): + for buffer, ranges_ in zip(buffers, ranges, strict=True): initial_index = ranges_[2] output_buffers[initial_index] = buffer.to_bytes() return output_buffers + async def _resolve_ranges( + self, + paths: list[str], + starts: Sequence[int | None], + ends: Sequence[int | None], + ) -> list[tuple[int, int | None]]: + """Rewrite fsspec's range vocabulary into the subset obstore accepts. + + `start` is normalized from None to 0. Ranges counting back from the end of + the object are resolved against its size, at the cost of one `head` request + per object; a lone negative `start` is left alone, since obstore expresses + that directly as a suffix request. + """ + sized_paths = sorted( + { + path + for path, start, end in zip(paths, starts, ends, strict=True) + if _needs_object_size(start, end) + }, + ) + sizes = dict( + zip( + sized_paths, + # fsspec types `_sizes` as `list[None]`; the values are really sizes. + cast("list[int]", await self._sizes(sized_paths)), + strict=True, + ), + ) + return [ + _apply_object_size(start, end, sizes.get(path)) + for path, start, end in zip(paths, starts, ends, strict=True) + ] + async def _put_file( self, lpath: str, diff --git a/obstore/python/obstore/store.py b/obstore/python/obstore/store.py index 72fe1b80..764d2c42 100644 --- a/obstore/python/obstore/store.py +++ b/obstore/python/obstore/store.py @@ -244,9 +244,9 @@ def get_ranges( path: str, *, starts: Sequence[int], - ends: Sequence[int] | None = None, - lengths: Sequence[int] | None = None, - coalesce: int = 1024 * 1024, + ends: Sequence[int | None] | None = None, + lengths: Sequence[int | None] | None = None, + coalesce: int | None = None, ) -> list[Bytes]: """Return the bytes stored at the specified location in the given byte ranges. @@ -266,9 +266,9 @@ async def get_ranges_async( path: str, *, starts: Sequence[int], - ends: Sequence[int] | None = None, - lengths: Sequence[int] | None = None, - coalesce: int = 1024 * 1024, + ends: Sequence[int | None] | None = None, + lengths: Sequence[int | None] | None = None, + coalesce: int | None = None, ) -> list[Bytes]: """Call `get_ranges` asynchronously. diff --git a/obstore/src/get.rs b/obstore/src/get.rs index afdcc978..7de648be 100644 --- a/obstore/src/get.rs +++ b/obstore/src/get.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use bytes::Bytes; use chrono::{DateTime, Utc}; use futures::stream::{BoxStream, Fuse}; -use futures::StreamExt; +use futures::{StreamExt, TryFutureExt}; use object_store::{ coalesce_ranges, Attributes, GetOptions, GetRange, GetResult, ObjectMeta, ObjectStore, ObjectStoreExt, OBJECT_STORE_COALESCE_DEFAULT, @@ -117,7 +117,7 @@ impl<'py> FromPyObject<'_, 'py> for PyGetRange { } else if let Ok(suffix_range) = obj.extract::() { Ok(Self(suffix_range.into())) } else { - Err(PyValueError::new_err("Unexpected input for byte range.\nExpected two-integer tuple or list, or dict with 'offset' or 'suffix' key." )) + Err(PyValueError::new_err("Unexpected input for byte range.\nExpected two-integer tuple or list, or dict with 'offset' or 'suffix' key.")) } } } @@ -382,15 +382,20 @@ pub(crate) fn get_range( py: Python, store: PyObjectStore, path: PyPath, - start: u64, + start: i64, end: Option, length: Option, -) -> PyObjectStoreResult { +) -> PyObjectStoreResult { let runtime = get_runtime(); let range = params_to_range(start, end, length)?; py.detach(|| { - let out = runtime.block_on(store.as_ref().get_range(path.as_ref(), range))?; - Ok::<_, PyObjectStoreError>(pyo3_bytes::PyBytes::new(out)) + let out = runtime.block_on( + store + .as_ref() + .get_opts(path.as_ref(), GetOptions::new().with_range(range.into())) + .and_then(GetResult::bytes), + )?; + Ok::<_, PyObjectStoreError>(PyBytes::new(out)) }) } @@ -400,7 +405,7 @@ pub(crate) fn get_range_async( py: Python, store: PyObjectStore, path: PyPath, - start: u64, + start: i64, end: Option, length: Option, ) -> PyResult> { @@ -408,36 +413,64 @@ pub(crate) fn get_range_async( pyo3_async_runtimes::tokio::future_into_py(py, async move { let out = store .as_ref() - .get_range(path.as_ref(), range) + .get_opts(path.as_ref(), GetOptions::new().with_range(range.into())) + .and_then(GetResult::bytes) .await .map_err(PyObjectStoreError::ObjectStoreError)?; - Ok(pyo3_bytes::PyBytes::new(out)) + Ok(PyBytes::new(out)) }) } fn params_to_range( - start: u64, + start: i64, end: Option, length: Option, -) -> PyObjectStoreResult> { +) -> PyObjectStoreResult { + if start < 0 { + if end.is_some() || length.is_some() { + return Err( + PyValueError::new_err("end and length must be None if start is negative.").into(), + ); + } + return Ok(GetRange::Suffix(start.unsigned_abs())); + } + + let start = start as u64; match (end, length) { (Some(_), Some(_)) => { Err(PyValueError::new_err("end and length cannot both be non-None.").into()) } - (None, None) => Err(PyValueError::new_err("Either end or length must be non-None.").into()), - (Some(end), None) => validate_range(start..end), - (None, Some(length)) => validate_range(start..start + length), + (None, None) => Ok(GetRange::Offset(start)), + (Some(end), None) => validate_range(start..end).map(GetRange::Bounded), + (None, Some(length)) => validate_range(start..start + length).map(GetRange::Bounded), } } async fn _get_ranges( store: PyObjectStore, path: PyPath, - ranges: &[Range], + ranges: &[GetRange], coalesce: u64, ) -> PyObjectStoreResult> { + let mut len: Option = None; + let mut resolved = Vec::with_capacity(ranges.len()); + for range in ranges { + resolved.push(match range { + GetRange::Bounded(r) => r.clone(), + other => { + let size = match len { + Some(len) => len, + None => *len.insert(store.as_ref().head(path.as_ref()).await?.size), + }; + other + .as_range(size) + .map_err(|err| PyValueError::new_err(err.to_string()))? + } + }); + } + let out = coalesce_ranges( - ranges, + &resolved, |range| store.as_ref().get_range(path.as_ref(), range), coalesce, ) @@ -446,61 +479,80 @@ async fn _get_ranges( } #[pyfunction] -#[pyo3(signature = (store, path, *, starts, ends=None, lengths=None, coalesce=OBJECT_STORE_COALESCE_DEFAULT))] +#[pyo3(signature = (store, path, *, starts, ends=None, lengths=None, coalesce=None))] pub(crate) fn get_ranges( py: Python, store: PyObjectStore, path: PyPath, - starts: Vec, - ends: Option>, - lengths: Option>, - coalesce: u64, + starts: Vec, + ends: Option>>, + lengths: Option>>, + coalesce: Option, ) -> PyObjectStoreResult> { let runtime = get_runtime(); let ranges = params_to_ranges(starts, ends, lengths)?; + let coalesce = coalesce.unwrap_or(OBJECT_STORE_COALESCE_DEFAULT); py.detach(|| runtime.block_on(_get_ranges(store, path, &ranges, coalesce))) } #[pyfunction] -#[pyo3(signature = (store, path, *, starts, ends=None, lengths=None, coalesce=OBJECT_STORE_COALESCE_DEFAULT))] +#[pyo3(signature = (store, path, *, starts, ends=None, lengths=None, coalesce=None))] pub(crate) fn get_ranges_async( py: Python, store: PyObjectStore, path: PyPath, - starts: Vec, - ends: Option>, - lengths: Option>, - coalesce: u64, + starts: Vec, + ends: Option>>, + lengths: Option>>, + coalesce: Option, ) -> PyResult> { let ranges = params_to_ranges(starts, ends, lengths)?; + let coalesce = coalesce.unwrap_or(OBJECT_STORE_COALESCE_DEFAULT); pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(_get_ranges(store, path, &ranges, coalesce).await?) }) } fn params_to_ranges( - starts: Vec, - ends: Option>, - lengths: Option>, -) -> PyObjectStoreResult>> { - match (ends, lengths) { - (Some(_), Some(_)) => { - Err(PyValueError::new_err("ends and lengths cannot both be non-None.").into()) - } - (None, None) => { - Err(PyValueError::new_err("Either ends or lengths must be non-None.").into()) + starts: Vec, + ends: Option>>, + lengths: Option>>, +) -> PyObjectStoreResult> { + for (name, len) in [("ends", ends.as_ref()), ("lengths", lengths.as_ref())] + .into_iter() + .filter_map(|(name, seq)| seq.map(|seq| (name, seq.len()))) + { + if len != starts.len() { + return Err(PyValueError::new_err(format!( + "starts and {name} must have the same length, got {} and {len}.", + starts.len(), + )) + .into()); } + } + + match (ends, lengths) { + // Consistent with `get_range`: an unbounded range reads to the end of the + // object. + (None, None) => starts + .into_iter() + .map(|start| params_to_range(start, None, None)) + .collect(), + (Some(ends), Some(lengths)) => starts + .into_iter() + .zip(ends) + .zip(lengths) + .map(|((start, end), length)| params_to_range(start, end, length)) + .collect(), (Some(ends), None) => starts .into_iter() .zip(ends) - .map(|(start, end)| start..end) - .map(validate_range) + .map(|(start, end)| params_to_range(start, end, None)) .collect(), (None, Some(lengths)) => starts .into_iter() .zip(lengths) - .map(|(start, length)| start..start + length) - .map(validate_range) + .map(|(start, length)| params_to_range(start, None, length)) .collect(), } } diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 3da2f5d3..beba5d36 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -13,6 +13,7 @@ from fsspec.registry import _registry from obstore.fsspec import FsspecStore, register +from obstore.store import ObjectStoreMethods from tests.conftest import TEST_BUCKET_NAME if TYPE_CHECKING: @@ -572,6 +573,56 @@ def test_multi_file_ops(minio_bucket: tuple[S3Config, ClientConfig]): assert out == [f"{bucket}/afile"] +def test_cat_file(fs: FsspecStore): + data = os.urandom(10000) + path = f"{TEST_BUCKET_NAME}/data1" + fs.pipe_file(path, data) + + assert fs.cat_file(path) == data + assert fs.cat_file(path, start=100, end=200) == data[100:200] + + # Either bound on its own. + assert fs.cat_file(path, start=100) == data[100:] + assert fs.cat_file(path, end=200) == data[:200] + + # Bounds counted back from the end of the object. + assert fs.cat_file(path, start=-100) == data[-100:] + assert fs.cat_file(path, start=100, end=-100) == data[100:-100] + assert fs.cat_file(path, start=-200, end=-100) == data[-200:-100] + + +def test_cat_ranges_max_gap(fs: FsspecStore, monkeypatch: pytest.MonkeyPatch): + data = os.urandom(10000) + path = f"{TEST_BUCKET_NAME}/data1" + fs.pipe_file(path, data) + + seen: list[int | None] = [] + original = ObjectStoreMethods.get_ranges_async + + async def spy(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003 + seen.append(kwargs.get("coalesce")) + return await original(self, *args, **kwargs) + + monkeypatch.setattr(ObjectStoreMethods, "get_ranges_async", spy) + + # Unset: forwarded as None, which lets obstore apply its own default. + assert fs.cat_ranges([path, path], [0, 200], [100, 300]) == [ + data[0:100], + data[200:300], + ] + assert seen == [None] + + # Set: forwarded verbatim as obstore's `coalesce`, including 0 to disable + # coalescing entirely. + for max_gap in (0, 1000): + seen.clear() + assert fs.cat_ranges([path, path], [0, 200], [100, 300], max_gap=max_gap) == [ + data[0:100], + data[200:300], + ] + assert seen == [max_gap] + + def test_cat_ranges_one(fs: FsspecStore): data1 = os.urandom(10000) fs.pipe_file(f"{TEST_BUCKET_NAME}/data1", data1) @@ -631,14 +682,15 @@ def test_cat_ranges_two(fs: FsspecStore): assert out == [data1[10:20], data2[10:20]] -@pytest.mark.xfail(reason="negative and mixed ranges not implemented") def test_cat_ranges_mixed(fs: FsspecStore): data1 = os.urandom(10000) data2 = os.urandom(10000) - fs.pipe({"data1": data1, "data2": data2}) + path1 = f"{TEST_BUCKET_NAME}/data1" + path2 = f"{TEST_BUCKET_NAME}/data2" + fs.pipe({path1: data1, path2: data2}) # single range in each file - out = fs.cat_ranges(["data1", "data1", "data2"], [-10, None, 10], [None, -10, -10]) + out = fs.cat_ranges([path1, path1, path2], [-10, None, 10], [None, -10, -10]) assert out == [data1[-10:], data1[:-10], data2[10:-10]] diff --git a/tests/test_get.py b/tests/test_get.py index a301d32e..edfa9f1c 100644 --- a/tests/test_get.py +++ b/tests/test_get.py @@ -113,6 +113,19 @@ def test_get_range(): view = memoryview(buffer) assert view == data[5:15] + buffer = store.get_range(path, start=4300) + view = memoryview(buffer) + assert view == data[4300:4400] + + buffer = store.get_range(path, start=-100) + view = memoryview(buffer) + assert view == data[4300:4400] + + # A suffix longer than the object yields the whole object. + buffer = store.get_range(path, start=-999999) + view = memoryview(buffer) + assert view == data + def test_get_ranges(): store = MemoryStore() @@ -130,6 +143,27 @@ def test_get_ranges(): for start, end, buffer in zip(starts, ends, buffers): assert memoryview(buffer) == data[start:end] + # A `None` element leaves that one range open-ended; a negative start makes it + # a suffix request. Omitting `ends` and `lengths` reads every range to the end. + buffers = store.get_ranges(path, starts=[5, 4300, -100], ends=[15, None, None]) + assert [memoryview(b) for b in buffers] == [ + data[5:15], + data[4300:4400], + data[4300:4400], + ] + + buffers = store.get_ranges(path, starts=[4300, -100]) + assert [memoryview(b) for b in buffers] == [data[4300:4400], data[4300:4400]] + + # `ends` and `lengths` may be mixed, at most one per range. + buffers = store.get_ranges( + path, + starts=[5, 20], + ends=[15, None], + lengths=[None, 10], + ) + assert [memoryview(b) for b in buffers] == [data[5:15], data[20:30]] + lengths = [10, 10, 10, 10] buffers = store.get_ranges(path, starts=starts, lengths=lengths) @@ -207,6 +241,12 @@ def test_get_range_invalid_range(): with pytest.raises(ValueError, match="Invalid range"): store.get_range(path, start=10, length=0) + with pytest.raises(ValueError, match="end and length must be None"): + store.get_range(path, start=-10, end=10) + + with pytest.raises(ValueError, match="end and length must be None"): + store.get_range(path, start=-10, length=10) + def test_get_ranges_invalid_range(): store = MemoryStore() @@ -224,6 +264,12 @@ def test_get_ranges_invalid_range(): with pytest.raises(ValueError, match="Invalid range"): store.get_ranges(path, starts=[10, 20], lengths=[10, 0]) + with pytest.raises(ValueError, match="starts and ends must have the same length"): + store.get_ranges(path, starts=[10, 20], ends=[30]) + + with pytest.raises(ValueError, match="starts and lengths must have the same"): + store.get_ranges(path, starts=[10], lengths=[10, 20]) + def test_access_getresult_attributes_after_reading_stream(): store = MemoryStore()