From 8dfb4955efaab8902599b69af9c0da9c908e40e5 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 22 Aug 2026 12:17:35 +0200 Subject: [PATCH 01/27] Caterva2 subscriber -> Caterca2 server --- bench/ndarray/cat2-block-granularity.py | 80 ++--- doc/guides/remote_arrays.md | 14 +- doc/reference/byterangendsource.rst | 4 +- doc/reference/c2array.rst | 2 +- src/blosc2/c2array.py | 104 +++---- src/blosc2/core.py | 2 +- src/blosc2/proxy.py | 4 +- src/blosc2/proxy_source.py | 6 +- src/blosc2/schunk.py | 2 +- tests/conftest.py | 2 +- tests/ndarray/test_c2array_blocks.py | 388 ++++++++++++------------ tests/ndarray/test_c2array_writes.py | 188 ++++++------ 12 files changed, 398 insertions(+), 398 deletions(-) diff --git a/bench/ndarray/cat2-block-granularity.py b/bench/ndarray/cat2-block-granularity.py index 2983d3211..5284ab735 100644 --- a/bench/ndarray/cat2-block-granularity.py +++ b/bench/ndarray/cat2-block-granularity.py @@ -14,10 +14,10 @@ compresses and decompresses independently, and Caterva2 serves a *stored* dataset straight from its file, so ``api/fetch`` honours a ``Range`` header and a slice can fetch only the blocks it touches. Whether that pays depends on -three things this script measures against a subscriber of your choosing: +three things this script measures against a server of your choosing: - whether the dataset **serves ranges at all**. One stored from a file does; - one the subscriber computes -- a lazy expression, an HDF5 leaf, a ``.b2z`` + one the server computes -- a lazy expression, an HDF5 leaf, a ``.b2z`` member -- is streamed, cannot honour a range, and keeps to whole chunks; - the **request plan**: how many requests each mode issues and how many bytes they carry, read from the frame's own chunk headers for a few hundred bytes; @@ -26,7 +26,7 @@ Usage ----- - # a local array, served by a stand-in subscriber over loopback + # a local array, served by a stand-in server over loopback python cat2-block-granularity.py mydata.b2nd # ... with a network put back in front of every request @@ -35,7 +35,7 @@ # ... served the way a computed dataset is, which is what the fallback costs python cat2-block-granularity.py mydata.b2nd --streamed - # against a real subscriber + # against a real server python cat2-block-granularity.py @public/examples/kevlar-tomo.b2nd \\ --urlbase https://cat2.cloud/demo @@ -57,17 +57,17 @@ RFC 7233 lets one ``Range`` header name many spans, and Caterva2 answers ``multipart/byteranges``. No object store offers this. -The stand-in subscriber answers ``api/info``, ``api/fetch`` and ``api/chunk`` +The stand-in server answers ``api/info``, ``api/fetch`` and ``api/chunk`` the way Caterva2 does, ranges and multipart included (it sorts and merges the spans it is given, as Starlette does, which is what the client has to survive). -Its request and byte counts are exact. Its *times* are not a subscriber's: -loopback answers in a fraction of a millisecond, where a subscriber over a WAN +Its request and byte counts are exact. Its *times* are not a server's: +loopback answers in a fraction of a millisecond, where a server over a WAN takes tens of milliseconds, which is the regime the whole trade lives in. ``--latency-ms`` and ``--bandwidth-mbs`` put a stated network back in front of each request; cat2.cloud from Europe measures about ``--latency-ms 45 --bandwidth-mbs 10``. The simulated bandwidth is *per request*, so eight parallel ones get eight times as much of it -- which is about right for an -object store and about wrong for one subscriber, and is why ``multipart`` can +object store and about wrong for one server, and is why ``multipart`` can come out behind ``blocks`` there while it wins against the real thing. ``--write`` measures the other direction: an array is laid out empty and filled @@ -75,7 +75,7 @@ things, and the first is the only one that goes over the wire: - the **fill**, serial and then ``--concurrency`` writers at once. The - subscriber serializes the writes themselves -- each takes the frame's + server serializes the writes themselves -- each takes the frame's exclusive lock -- so what overlaps is the round trip, and the gain is whatever share of a write that was. Over loopback it is almost none; put a network in front with ``--latency-ms`` and it is most of it; @@ -88,7 +88,7 @@ its chunks. The offsets are one decompress whatever the count; the walk is a read per chunk, so the two cross over as an array grows. -Against a real subscriber ``--write`` needs ``--write-target``: an empty +Against a real server ``--write`` needs ``--write-target``: an empty pre-sized array to fill, since laying one out is not this script's business on someone else's server. Only the serial fill runs there -- a slot is written once, so a second timed fill needs a second array. @@ -118,7 +118,7 @@ # -# A stand-in subscriber, so this runs with no service to point at +# A stand-in server, so this runs with no service to point at # @@ -126,7 +126,7 @@ """What a frame codes in a chunk's flags byte for a slot never written to.""" -class Subscriber: +class Cat2Server: """Caterva2's read endpoints over one local .b2nd file, and its write one.""" def __init__(self, urlpath, streamed=False, writable=False): @@ -139,14 +139,14 @@ def __init__(self, urlpath, streamed=False, writable=False): self.writable = writable self.array = blosc2.open(str(self.path), mode="a" if writable else "r", locking=writable) self.lock = threading.Lock() - # A dataset the subscriber would compute rather than store: served by a + # A dataset the server would compute rather than store: served by a # body builder, which has no way to honour a Range self.streamed = streamed def close(self): """Let go of the file this held open, so the scratch tree can be removed. - A writable subscriber keeps one handle for its whole life, and a run that + A writable server keeps one handle for its whole life, and a run that fills several arrays leaves one behind per array otherwise -- still holding files that `shutil.rmtree` then unlinks under them. """ @@ -222,49 +222,49 @@ def _dataset(self): target = getattr(self.server, "target", None) if target is not None and self.path.split("?")[0].endswith(target.name): return target - return self.server.subscriber + return self.server.cat2 def do_POST(self): # BaseHTTPRequestHandler's own spelling - sub = self._dataset() + srv = self._dataset() endpoint = self.path.split("/")[2].split("?")[0] - if endpoint != "chunk" or not sub.writable: + if endpoint != "chunk" or not srv.writable: self._send(404, b"") return nchunk = int(self.path.split("nchunk=")[1]) body = self.rfile.read(int(self.headers.get("Content-Length", 0))) - status, answer = sub.write_chunk(nchunk, body) + status, answer = srv.write_chunk(nchunk, body) self._send(status, json.dumps(answer).encode()) def do_GET(self): # BaseHTTPRequestHandler's own spelling - sub = self._dataset() + srv = self._dataset() endpoint = self.path.split("/")[2] if endpoint == "info": - self._send(200, json.dumps(sub.meta()).encode()) + self._send(200, json.dumps(srv.meta()).encode()) elif endpoint == "chunk": nchunk = int(self.path.split("nchunk=")[1]) - self._send(200, sub.array.schunk.get_chunk(nchunk)) + self._send(200, srv.array.schunk.get_chunk(nchunk)) elif endpoint == "fetch": - self._fetch(sub) + self._fetch(srv) else: self._send(404, b"") - def _fetch(self, sub): + def _fetch(self, srv): wanted = self.headers.get("Range") - if sub.streamed: + if srv.streamed: # What the streaming paths answer since they were made honest: a 416 # instead of the whole body with a 200 that no client could notice if wanted: self._send(416, b"", [("Accept-Ranges", "none")]) else: - self._send(200, sub.read(0, sub.size - 1), [("Accept-Ranges", "none")]) + self._send(200, srv.read(0, srv.size - 1), [("Accept-Ranges", "none")]) return if not wanted: - self._send(200, sub.read(0, sub.size - 1), [("Accept-Ranges", "bytes")]) + self._send(200, srv.read(0, srv.size - 1), [("Accept-Ranges", "bytes")]) return spans = [] for span in wanted.removeprefix("bytes=").split(","): start, end = (int(n) for n in span.split("-")) - spans.append((start, min(end, sub.size - 1))) + spans.append((start, min(end, srv.size - 1))) # Starlette sorts the spans and merges the ones that touch, and answers a # plain 206 when only one is left, so a client cannot count on a part per # span nor on the order it asked in @@ -279,17 +279,17 @@ def _fetch(self, sub): start, end = merged[0] self._send( 206, - sub.read(start, end), - [("Content-Range", f"bytes {start}-{end}/{sub.size}"), ("Accept-Ranges", "bytes")], + srv.read(start, end), + [("Content-Range", f"bytes {start}-{end}/{srv.size}"), ("Accept-Ranges", "bytes")], ) return body = b"" for start, end in merged: body += ( f"--{self.BOUNDARY}\r\nContent-Type: application/octet-stream\r\n" - f"Content-Range: bytes {start}-{end}/{sub.size}\r\n\r\n" + f"Content-Range: bytes {start}-{end}/{srv.size}\r\n\r\n" ).encode() - body += sub.read(start, end) + b"\r\n" + body += srv.read(start, end) + b"\r\n" body += f"--{self.BOUNDARY}--\r\n".encode() self._send( 206, @@ -309,7 +309,7 @@ def stand_in(urlpath, streamed=False): here. """ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) - server.subscriber = Subscriber(urlpath, streamed) + server.cat2 = Cat2Server(urlpath, streamed) server.target = None threading.Thread(target=server.serve_forever, daemon=True).start() urlbase = f"http://127.0.0.1:{server.server_address[1]}/" @@ -348,7 +348,7 @@ def chunk_cbytes(source, nchunks): Read rather than guessed at: the distance to the next chunk is an upper bound only, and a frame with a hole in it would make the chunk mode look - dearer than it is. One request for the lot where the subscriber takes + dearer than it is. One request for the lot where the server takes several ranges, which is the same trick the fetch path uses. """ live = [n for n in nchunks if int(source._offsets[n]) >= 0] @@ -360,7 +360,7 @@ def chunk_cbytes(source, nchunks): def request_plan(proxy, array, item): - """What each mode asks the subscriber for, to serve *item*. + """What each mode asks the server for, to serve *item*. Follows `Proxy._fetch_by_block` step for step -- which chunks the slice touches, which of those are worth taking apart, one read for the block @@ -485,7 +485,7 @@ def timed_fill(open_array, chunks, writers, latency, bandwidth): """Write *chunks* into a pre-sized array, and say what it cost. One `C2Array` per writer, as separate processes would have. What overlaps - is the round trip: the subscriber serializes the writes themselves, since + is the round trip: the server serializes the writes themselves, since each one takes the frame's exclusive lock. """ tally = {"requests": 0, "bytes": 0} @@ -561,7 +561,7 @@ def connection_setup(urlbase, path, token, reps): """ import httpx - url = c2array._sub_url(urlbase, f"api/info/{path}") + url = c2array._server_url(urlbase, f"api/info/{path}") headers = c2array._auth_headers(token) pooled = c2array._sync_client() @@ -581,8 +581,8 @@ def median(get): def main(): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("dataset", help="a remote dataset path with --urlbase, else a local .b2nd") - parser.add_argument("--urlbase", help="a Caterva2 subscriber; without it, one is stood in") - parser.add_argument("--username", help="log in to the subscriber as this user") + parser.add_argument("--urlbase", help="a Caterva2 server; without it, one is stood in") + parser.add_argument("--username", help="log in to the server as this user") parser.add_argument("--password", help="the password to log in with") parser.add_argument("--token", help="an authorization cookie, instead of logging in") parser.add_argument( @@ -617,7 +617,7 @@ def main(): if args.urlbase: urlbase, path = args.urlbase, args.dataset if args.write and not args.write_target: - parser.error("--write against a subscriber needs --write-target: an empty array to fill") + parser.error("--write against a server needs --write-target: an empty array to fill") else: server, urlbase, path = stand_in(args.dataset, args.streamed) token = args.token @@ -654,7 +654,7 @@ def presize(serve=False): if serve: if server.target is not None: server.target.close() # its handle is done with; the next array gets its own - server.target = Subscriber(path, writable=True) + server.target = Cat2Server(path, writable=True) return path return presize diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index ffbb92053..b6c89ee1f 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -7,7 +7,7 @@ A Blosc2 array that lives on a server does not have to be downloaded to be used. | Where the array lives | How to open it | |---|---| | Any URL fsspec reaches — `s3://`, `gs://`, `https://`, `zip://`… | `blosc2.open(url, lazy=True)` | -| A [Caterva2](https://ironarray.io/caterva2) subscriber | `blosc2.C2Array(path, urlbase=...)` | +| A [Caterva2](https://ironarray.io/caterva2) server | `blosc2.C2Array(path, urlbase=...)` | | Anything else | A `read_range()` of your own — see [Your own transport](#your-own-transport) | ```python @@ -16,7 +16,7 @@ import blosc2 # An object store, a web server, a zip on either of them a = blosc2.open("s3://bucket/big.b2nd", lazy=True) -# A Caterva2 subscriber +# A Caterva2 server b = blosc2.C2Array( "@public/examples/lung-jpeg2000_10x.b2nd", urlbase="https://cat2.cloud/demo" ) @@ -25,7 +25,7 @@ a.shape, a.dtype # metadata only; nothing was downloaded a[100:110, :50] # a NumPy array, fetched now ``` -`https://` means a plain web server — nginx, a CDN, an S3 website endpoint — anything that answers a `Range` request. A Caterva2 subscriber is *not* reached that way: it names its datasets by root and path, so use {ref}`C2Array` (or `blosc2.URLPath` with {func}`blosc2.open`). +`https://` means a plain web server — nginx, a CDN, an S3 website endpoint — anything that answers a `Range` request. A Caterva2 server is *not* reached that way: it names its datasets by root and path, so use {ref}`C2Array` (or `blosc2.URLPath` with {func}`blosc2.open`). ## The cache @@ -48,13 +48,13 @@ You do not ask for this; it happens when it pays: - On S3, block reads are **5–17x faster** on arrays with multi-megabyte chunks, and **2–5x** on 1 MB ones. - On cat2.cloud's `kevlar-tomo.b2nd`, a corner slice costs **0.031 MB instead of 2.723 MB**, and a slice touching ten chunks takes **0.14 s against 1.01 s**. -It is never a loss. Two thresholds decide it — a chunk under a megabyte is one cheap request anyway, and wanting more than half a chunk's blocks is wanting the chunk — and both are answered from metadata already in hand. Where blocks are not available, the read falls back to whole chunks by itself: that happens for a dataset a Caterva2 subscriber *computes* rather than stores (a lazy expression, an HDF5 leaf, a `.b2z` member), and for a server that stops honouring ranges. +It is never a loss. Two thresholds decide it — a chunk under a megabyte is one cheap request anyway, and wanting more than half a chunk's blocks is wanting the chunk — and both are answered from metadata already in hand. Where blocks are not available, the read falls back to whole chunks by itself: that happens for a dataset a Caterva2 server *computes* rather than stores (a lazy expression, an HDF5 leaf, a `.b2z` member), and for a server that stops honouring ranges. Fetches also overlap: a lazy proxy runs 8 at a time by default. Pass `max_concurrency=1` for a local protocol with no latency to hide. ## When the remote changes underneath -A cache is only good while the bytes it was filled from are still there. Sources that can name their bytes — an fsspec URL by its token, a Caterva2 array by an identifier the subscriber keeps — are checked against what the cache recorded: +A cache is only good while the bytes it was filled from are still there. Sources that can name their bytes — an fsspec URL by its token, a Caterva2 array by an identifier the server keeps — are checked against what the cache recorded: ```python p = blosc2.Proxy(src, urlpath="cache.b2nd", mode="a") @@ -66,7 +66,7 @@ p = blosc2.Proxy(src, urlpath="cache.b2nd", mode="a") ## Filling an array from several writers -A Caterva2 array can be *written*, one chunk at a time, by as many processes as it has chunks. Lay the array out empty first — {func}`blosc2.uninit` writes a couple of hundred bytes whatever the shape — upload it to the subscriber, then have each writer post the chunks it owns: +A Caterva2 array can be *written*, one chunk at a time, by as many processes as it has chunks. Lay the array out empty first — {func}`blosc2.uninit` writes a couple of hundred bytes whatever the shape — upload it to the server, then have each writer post the chunks it owns: ```python import blosc2 @@ -121,7 +121,7 @@ for nchunk in np.flatnonzero(~written): ... # the work still to do, after a crash ``` -What this buys: the subscriber serializes the writes themselves, so what overlaps is the round trip — which over a network is nearly all of the cost. Against a real subscriber, a fill went from **244 ms per chunk serially to 32 ms with 8 writers, 7.6x**. Over loopback, where there is no round trip to hide, it is 1.0x. +What this buys: the server serializes the writes themselves, so what overlaps is the round trip — which over a network is nearly all of the cost. Against a real server, a fill went from **244 ms per chunk serially to 32 ms with 8 writers, 7.6x**. Over loopback, where there is no round trip to hide, it is 1.0x. ## Your own transport diff --git a/doc/reference/byterangendsource.rst b/doc/reference/byterangendsource.rst index 82970ff78..cb1eae8b5 100644 --- a/doc/reference/byterangendsource.rst +++ b/doc/reference/byterangendsource.rst @@ -8,7 +8,7 @@ Blosc2 frame it can read byte ranges of, instead of transferring the whole container. It knows the frame format and nothing about where the frame lives: subclasses supply ``read_range(offset, size)`` and nothing else. :ref:`FsspecNDSource` reads through fsspec, and :ref:`C2Array` reads over HTTP -ranges from a Caterva2 subscriber. For other sources, see :ref:`ProxyNDSource` +ranges from a Caterva2 server. For other sources, see :ref:`ProxyNDSource` and :ref:`ProxySource`. .. currentmodule:: blosc2 @@ -22,7 +22,7 @@ When a range read is refused ---------------------------- A transport that reads byte ranges may be answered with something other than the -bytes asked for: a subscriber that now streams the dataset, a server that is too +bytes asked for: a server that now streams the dataset, a server that is too busy to serve it, a body that cannot be taken apart. Those raise ``blosc2.proxy_source.NotRanged``, which a :ref:`Proxy` catches for itself -- whatever the fetch is still missing comes as whole chunks -- and which a caller diff --git a/doc/reference/c2array.rst b/doc/reference/c2array.rst index 47d6a3ddb..9f8caaef2 100644 --- a/doc/reference/c2array.rst +++ b/doc/reference/c2array.rst @@ -8,7 +8,7 @@ This is a class for remote arrays. This kind of array can also work as operand o Wrapped in a :ref:`Proxy`, a stored remote array is read at block granularity: the proxy asks for the blocks a slice touches rather than the chunks they live in, which for a multi-megabyte chunk is a small fraction of the bytes. That -rests on the subscriber serving the dataset from a file, ``Range`` header and +rests on the server serving the dataset from a file, ``Range`` header and auth cookie both honoured; a dataset it computes instead (a lazy expression, an HDF5 leaf) is fetched a whole chunk at a time, as everything was before. Which one this is takes at most one request to find out, and is decided once -- diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index beb9f45ba..3a5a5514d 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -32,11 +32,11 @@ _is_transient, ) -_subscriber_data = { +_server_data = { "urlbase": os.environ.get("BLOSC_C2URLBASE"), "auth_token": "", } -"""Caterva2 subscriber data saved by context manager.""" +"""Caterva2 server data saved by context manager.""" TIMEOUT = 15 """Default timeout for HTTP requests.""" @@ -122,17 +122,17 @@ def c2context( auth_token: (str | None) = None, ) -> None: """ - Context manager that sets parameters in Caterva2 subscriber requests. + Context manager that sets parameters in Caterva2 server requests. A parameter not specified or set to ``None`` will inherit the value from the previous context manager, defaulting to an environment variable (see below) if supported by that parameter. Parameters set to an empty string will not be used in requests (without a default either). - If the subscriber requires authorization for requests, you can either + If the server requires authorization for requests, you can either provide an `auth_token` (which you should have obtained previously from the - subscriber), or both `username` and `password` to obtain the token by - logging in to the subscriber. The token will be reused until it is explicitly + server), or both `username` and `password` to obtain the token by + logging in to the server. The token will be reused until it is explicitly reset or requested again in a later context manager invocation. Please note that this manager is reentrant but not safe for concurrent use. @@ -140,14 +140,14 @@ def c2context( Parameters ---------- urlbase : str | None - The base URL to be used when a C2Array instance does not have a subscriber + The base URL to be used when a C2Array instance does not have a server URL base set. If not specified, it defaults to the value of the ``BLOSC_C2URLBASE`` environment variable. username : str | None - The username for logging in to the subscriber to obtain an authorization token. + The username for logging in to the server to obtain an authorization token. If not specified, it defaults to the value of the ``BLOSC_C2USERNAME`` environment variable. password : str | None - The password for logging in to the subscriber to obtain an authorization token. + The password for logging in to the server to obtain an authorization token. If not specified, it defaults to the value of the ``BLOSC_C2PASSWORD`` environment variable. auth_token : str | None The authorization token to be used when a C2Array instance does not have an @@ -158,8 +158,8 @@ def c2context( out: None """ - global _subscriber_data - print("_subscriber_data", _subscriber_data) + global _server_data + print("_server_data", _server_data) # Perform login to get an authorization token. if not auth_token: @@ -171,23 +171,23 @@ def c2context( auth_token = login(username, password, urlbase) try: - old_sub_data = _subscriber_data - new_sub_data = old_sub_data.copy() # inherit old values + old_server_data = _server_data + new_server_data = old_server_data.copy() # inherit old values if urlbase is not None: - new_sub_data["urlbase"] = urlbase - elif old_sub_data["urlbase"] is None: + new_server_data["urlbase"] = urlbase + elif old_server_data["urlbase"] is None: # The variable may have gotten a value after program start. - new_sub_data["urlbase"] = os.environ.get("BLOSC_C2URLBASE") + new_server_data["urlbase"] = os.environ.get("BLOSC_C2URLBASE") if auth_token is not None: - new_sub_data["auth_token"] = auth_token - _subscriber_data = new_sub_data + new_server_data["auth_token"] = auth_token + _server_data = new_server_data yield finally: - _subscriber_data = old_sub_data + _server_data = old_server_data def _auth_headers(auth_token, headers=None): - auth_token = auth_token or _subscriber_data["auth_token"] + auth_token = auth_token or _server_data["auth_token"] if auth_token: headers = headers.copy() if headers else {} headers["Cookie"] = auth_token @@ -202,7 +202,7 @@ def _xget(url, params=None, headers=None, auth_token=None, timeout=TIMEOUT): def _xpost(url, json=None, auth_token=None, timeout=TIMEOUT): - auth_token = auth_token or _subscriber_data["auth_token"] + auth_token = auth_token or _server_data["auth_token"] headers = {"Cookie": auth_token} if auth_token else None response = _sync_client().post(url, json=json, headers=headers, timeout=timeout) response.raise_for_status() @@ -232,7 +232,7 @@ def _xpost_bytes(url, content, params=None, auth_token=None, timeout=TIMEOUT): """POST a body of bytes through the pooled client, and read what came back. `_xpost` sends JSON, which a compressed chunk is not: it goes as it is, and - the subscriber reads it as the chunk it will store. + the server reads it as the chunk it will store. """ response = _sync_client().post( url, params=params, content=content, headers=_chunk_headers(auth_token), timeout=timeout @@ -246,15 +246,15 @@ async def _axpost_bytes(client, url, content, params=None, auth_token=None): return _chunk_written(response, url, params and params.get("nchunk")) -def _sub_url(urlbase, path): - urlbase = urlbase or _subscriber_data["urlbase"] +def _server_url(urlbase, path): + urlbase = urlbase or _server_data["urlbase"] if not urlbase: - raise RuntimeError("No default Caterva2 subscriber set") + raise RuntimeError("No default Caterva2 server set") return f"{urlbase}{path}" if urlbase.endswith("/") else f"{urlbase}/{path}" def login(username, password, urlbase): - url = _sub_url(urlbase, "auth/jwt/login") + url = _server_url(urlbase, "auth/jwt/login") creds = {"username": username, "password": password} # Not the pooled client: this is the one request whose Set-Cookie matters, # and it belongs to the caller rather than to every later request @@ -264,14 +264,14 @@ def login(username, password, urlbase): def info(path, urlbase, params=None, headers=None, model=None, auth_token=None): - url = _sub_url(urlbase, f"api/info/{path}") + url = _server_url(urlbase, f"api/info/{path}") response = _xget(url, params, headers, auth_token) json = response.json() return json if model is None else model(**json) def fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False): - url = _sub_url(urlbase, f"api/fetch/{path}") + url = _server_url(urlbase, f"api/fetch/{path}") response = _xget(url, params=params, auth_token=auth_token) data = response.content # Try different deserialization methods @@ -311,7 +311,7 @@ def slice_to_string(slice_): """A block source that has not been asked for yet, as against one that failed.""" MAX_RANGES_PER_REQUEST = 64 -"""How many byte ranges one request to a subscriber may ask for. +"""How many byte ranges one request to a server may ask for. There is no limit in the protocol, and the saving grows with the count -- but a `Range` header is a header, which servers and proxies cap the length of (8 KB is @@ -424,7 +424,7 @@ def _span_of(parts: list[tuple[int, bytes, int | None]], offset: int, size: int, class ChunkAlreadyWritten(ValueError): """A chunk was written to a slot of a remote array that already held content. - A subscriber that accepts chunk writes accepts each slot exactly once: the + A server that accepts chunk writes accepts each slot exactly once: the frame's own offsets say whether a slot was ever written, and a second write would move every chunk that came after it. So a writer that finds this has lost a race, or is repeating work another writer already did; either way the @@ -441,7 +441,7 @@ class C2NDSource(ByteRangeNDSource): cookie composes with it. That is everything :ref:`ByteRangeNDSource` needs, so a slice costs the blocks it touches instead of the chunks they live in. - A dataset the subscriber *builds* -- a lazy expression, an HDF5 leaf, a + A dataset the server *builds* -- a lazy expression, an HDF5 leaf, a ``.b2z`` member -- is streamed instead, and a streamed response ignores the ``Range`` header and answers with the whole body. :meth:`read_range` refuses such an answer without reading it off the socket, and :ref:`C2Array` then @@ -453,7 +453,7 @@ class C2NDSource(ByteRangeNDSource): max_ranges = MAX_RANGES_PER_REQUEST def __init__(self, array: C2Array, max_concurrency: int = REMOTE_MAX_CONCURRENCY): - self._url = _sub_url(array.urlbase, f"api/fetch/{array.path}") + self._url = _server_url(array.urlbase, f"api/fetch/{array.path}") self._auth_token = array.auth_token # Answers that did not carry their parts, in a row; see `read_ranges` self._misses = 0 @@ -551,7 +551,7 @@ def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | N The path to the remote NDArray file (root + file path) as a posix path. urlbase: str - The base URL (slash-terminated) of the subscriber to query. + The base URL (slash-terminated) of the server to query. auth_token: str An optional token to authorize requests via HTTP. Currently, it will be sent as an HTTP cookie. @@ -798,7 +798,7 @@ async def aclose(self) -> None: self._aclient = None # -- Writing chunks. A pre-sized array is filled a chunk at a time, by as - # many writers as there are chunks to fill; the subscriber serializes them + # many writers as there are chunks to fill; the server serializes them # and refuses a slot that was already written. def update_chunk(self, nchunk: int, chunk: bytes) -> dict: @@ -814,7 +814,7 @@ def update_chunk(self, nchunk: int, chunk: bytes) -> dict: The chunk must match the array's geometry -- its chunkshape, its typesize and its blocksize -- which is what compressing against - :attr:`cparams` and :attr:`blocks` gives; the subscriber checks it and + :attr:`cparams` and :attr:`blocks` gives; the server checks it and refuses anything else rather than storing a chunk the array cannot read. Parameters @@ -829,7 +829,7 @@ def update_chunk(self, nchunk: int, chunk: bytes) -> dict: Returns ------- out: dict - What the subscriber reports of the array's state now. Carries + What the server reports of the array's state now. Carries ``written`` and ``nchunks`` where it counts them, so a writer can see a fill finish without asking again. @@ -852,13 +852,13 @@ def update_chunk(self, nchunk: int, chunk: bytes) -> dict: The blocksize is spelled out because :func:`blosc2.compress2` picks its own when it is not: left to choose it takes the whole chunk, and a chunk - blocked differently from the array is one the subscriber refuses. + blocked differently from the array is one the server refuses. """ url = self._chunk_url() try: return _xpost_bytes(url, chunk, params={"nchunk": nchunk}, auth_token=self.auth_token) finally: - # However it went. A refusal is the answer of a subscriber that has + # However it went. A refusal is the answer of a server that has # already stored someone else's chunk in that slot, and a request that # failed on the way home may have stored this one; either way what # this handle read of the array is no longer what the array is @@ -868,7 +868,7 @@ async def aupdate_chunk(self, nchunk: int, chunk: bytes) -> dict: """Write one compressed chunk asynchronously; see :meth:`update_chunk`. The same request, off the event loop, so a writer with many chunks to - send can have several in flight. The subscriber serializes them at the + send can have several in flight. The server serializes them at the far end regardless -- what overlaps is the round trip, which for a chunk-sized body is most of the cost. """ @@ -891,7 +891,7 @@ def written_chunks(self) -> np.ndarray: :meth:`ByteRangeNDSource.written_chunks`. Read out of the frame's own offsets, which is where a fill records - itself: no endpoint of its own, and nothing for the subscriber to keep in + itself: no endpoint of its own, and nothing for the server to keep in step with the array. Read afresh every time, since the point of asking is to see what other writers have done since -- which is a couple of range reads, the header first (a write moves the frame's length, and the @@ -912,7 +912,7 @@ def written_chunks(self) -> np.ndarray: def _chunk_url(self) -> str: """Where a chunk of this array is read from, and written to.""" - return _sub_url(self.urlbase, f"api/chunk/{self.path}") + return _server_url(self.urlbase, f"api/chunk/{self.path}") def _forget_index(self) -> None: """Drop what this handle read of a frame it has since written to.""" @@ -1004,7 +1004,7 @@ def stamp(self) -> str | None: Geometry cannot tell a dataset that was replaced from the one a cache was filled from: a shape and a partitioning survive a rewrite, while every cached chunk -- and, in block mode, every offset they were fetched by -- - goes stale. The subscriber's own mtime does tell, and `api/info` carries + goes stale. The server's own mtime does tell, and `api/info` carries it, so this costs no request of its own; the compressed size goes in with it, since a rewrite within the same clock tick is what an mtime cannot see. What it names is the array as this handle last looked at it -- @@ -1012,7 +1012,7 @@ def stamp(self) -> str | None: says so, and what a `Proxy` calls before judging a cache by it. Two questions, and they want different answers. *Which array is this* is - answered by the nonce a subscriber writes into an array's vlmeta the first + answered by the nonce a server writes into an array's vlmeta the first time a chunk is written to it: a size and an mtime can both be repeated by a different array that came to sit at the same path, and a cache served against one of those is stale without ever saying so. *Has it @@ -1031,7 +1031,7 @@ def stamp(self) -> str | None: one it had; when a writer fills that slot, both are wrong, and nothing in the cache marks them apart from the chunks that are still good. - None when the subscriber reports no mtime and the array carries no nonce, + None when the server reports no mtime and the array carries no nonce, which leaves the cache checked on its geometry alone, as every source without a stamp is. """ @@ -1069,7 +1069,7 @@ def serves_blocks(self) -> bool: """Whether blocks are worth asking this dataset for, as far as info can say. What `api/info` already carries, and no request of its own: a dataset the - subscriber *computes* reports an expression where a stored one reports a + server *computes* reports an expression where a stored one reports a geometry, and a frame of small chunks would never have one taken apart -- blosc2 declines to split a chunk below ``BLOCK_MIN_CBYTES``, so the block path would end in whole chunks anyway, by the longer road. @@ -1112,7 +1112,7 @@ def block_source(self) -> C2NDSource | None: """The frame reader behind the block methods, or None if there is none. Built on the first request for it and never rebuilt. The fallback has to - be permanent: a subscriber that streams this dataset answers a range + be permanent: a server that streams this dataset answers a range request with the whole body, so retrying would pay a full download to rediscover the same answer. @@ -1148,10 +1148,10 @@ def _open_block_source(self): """Decide, at whatever cost it takes, whether this dataset serves ranges. None for a dataset that does not serve ranges, which is an answer for - good; `_UNTRIED` for a subscriber that could not say, which is not. + good; `_UNTRIED` for a server that could not say, which is not. """ httpx = _httpx() - # A dataset the subscriber computes has no frame to read at all, and + # A dataset the server computes has no frame to read at all, and # `api/info` says so for free. Whether its chunks are worth taking apart # is a separate judgement, made by whoever asks -- see `block_source` if not self._reports_geometry: @@ -1169,7 +1169,7 @@ def _open_block_source(self): # asking, and whole chunks read the dataset either way return _UNTRIED if exc.transient else None except httpx.HTTPStatusError as exc: - # A busy or broken subscriber said nothing about how this is served + # A busy or broken server said nothing about how this is served return _UNTRIED if _is_transient(exc.response.status_code) else None except httpx.TransportError: # Nothing was downloaded to find this out, so asking again is cheap @@ -1214,7 +1214,7 @@ def wants_blocks(self, nchunk: int, nwanted: int) -> bool: @property def max_ranges(self) -> int: - """How many ranges one request to this subscriber may carry.""" + """How many ranges one request to this server may carry.""" source = self.block_source() return 1 if source is None else source.max_ranges @@ -1239,7 +1239,7 @@ def read_range(self, offset: int, size: int) -> bytes: return source.read_range(offset, size) def read_ranges(self, spans: Sequence[tuple[int, int]]) -> list[bytes]: - """The bytes of every span, in one request where the subscriber allows it.""" + """The bytes of every span, in one request where the server allows it.""" with self._ranged() as source: return source.read_ranges(spans) @@ -1247,7 +1247,7 @@ def read_ranges(self, spans: Sequence[tuple[int, int]]) -> list[bytes]: def _ranged(self, index_only: bool = False): """The block source, retired if it turns out to serve ranges no longer. - The subscriber can stop serving a dataset from a file between one fetch + The server can stop serving a dataset from a file between one fetch and the next -- replaced by a lazy expression, moved into a container it streams out of -- and the answer to a range request is where that shows. A refusal that says so for good puts the array back where it was before diff --git a/src/blosc2/core.py b/src/blosc2/core.py index 856684d65..feed6ebef 100644 --- a/src/blosc2/core.py +++ b/src/blosc2/core.py @@ -657,7 +657,7 @@ def is_fsspec_url(urlpath: object) -> bool: `http(s)://` included: a frame behind a plain web server is a frame like any other, and fsspec reads it in ranges wherever the server answers them. A - Caterva2 subscriber is not reached this way -- its datasets are named by root + Caterva2 server is not reached this way -- its datasets are named by root and path rather than by URL, so :ref:`C2Array` is entered through :ref:`URLPath`, which `open` dispatches on before it ever gets here. """ diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 8aa897021..249645edd 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -91,7 +91,7 @@ def __init__( A source that can name the exact bytes it reads, as :ref:`FsspecNDSource` does with its ``stamp`` (fsspec's token) and - :ref:`C2Array` with the subscriber's mtime, is checked against that + :ref:`C2Array` with the server's mtime, is checked against that too: a cache built from different bytes raises, even when the geometry still fits. For every other source geometry is all there is to check, so a source whose contents changed underneath while its geometry did @@ -552,7 +552,7 @@ def fetch( chunks, whenever that is the cheaper way round -- see the thresholds in `blosc2.proxy_source`. The chunks left in the cache then hold only those blocks, and read as zeros elsewhere until the rest are fetched. A source - that stops answering range reads partway (a subscriber that now computes + that stops answering range reads partway (a server that now computes the dataset, or is too busy to serve it from its file) does not fail the fetch: the chunks it was asked for come whole instead. diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index a1f93dd03..5464245fe 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -548,7 +548,7 @@ class ByteRangeNDSource(ProxyNDSource): only has to say how to read bytes: :meth:`read_range` is the one abstract method, and the transport behind it decides nothing about the rest. :ref:`FsspecNDSource` reads them with fsspec, and :ref:`C2Array` reads them - over HTTP ranges from a Caterva2 subscriber, carrying its auth cookie. + over HTTP ranges from a Caterva2 server, carrying its auth cookie. A subclass sets its transport up first and then calls this constructor, which reads the frame's header through it -- one small read, and everything @@ -583,7 +583,7 @@ class ByteRangeNDSource(ProxyNDSource): The frame is there to be read in pieces -- that is what an open of one settles -- so a :ref:`Proxy` over it goes straight to the block path. A - source that only sometimes serves blocks (:ref:`C2Array`, whose subscriber + source that only sometimes serves blocks (:ref:`C2Array`, whose server may compute the dataset rather than store it) overrides this. """ @@ -591,7 +591,7 @@ class ByteRangeNDSource(ProxyNDSource): """How many ranges one request of this transport may carry. One means one request each, which is all any object store offers. A - subscriber answering ``multipart/byteranges`` takes more -- see + server answering ``multipart/byteranges`` takes more -- see :meth:`read_ranges` -- and then a slice costs a couple of requests rather than a couple per chunk it touches. """ diff --git a/src/blosc2/schunk.py b/src/blosc2/schunk.py index 9f8b5d786..1db3449ca 100644 --- a/src/blosc2/schunk.py +++ b/src/blosc2/schunk.py @@ -2086,7 +2086,7 @@ def open( urlpath: str | pathlib.Path | :ref:`URLPath` The path where the :ref:`SChunk` (or :ref:`NDArray`) is stored. If it is a remote Caterva2 array, a :ref:`URLPath` must be passed: - a subscriber names its datasets by root and path rather than by URL. + a server names its datasets by root and path rather than by URL. Any URL with a scheme (``s3://``, ``gs://``, ``https://``, ``zip://``, ``memory://``...) is opened through fsspec; see the `Notes` section for the limits. diff --git a/tests/conftest.py b/tests/conftest.py index 59915b07c..c5c4f2f4a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,7 +90,7 @@ def pytest_configure(config): @pytest.fixture(scope="session") def cat2_context(): # You may use the URL and credentials for an already existing user - # in a different Caterva2 subscriber. + # in a different Caterva2 server. urlbase = os.environ.get("BLOSC_C2URLBASE", "https://cat2.cloud/testing/") c2params = {"urlbase": urlbase, "username": None, "password": None} with blosc2.c2context(**c2params): diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 0386dd4c8..dc3c6071b 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -5,14 +5,14 @@ # SPDX-License-Identifier: BSD-3-Clause ####################################################################### -"""Block-granular reads of a C2Array, against a stand-in for a subscriber. +"""Block-granular reads of a C2Array, against a stand-in for a server. The server here answers the two endpoints the block path uses -- `api/info` for the geometry and `api/fetch` for the bytes -- the way Caterva2 does: a stored dataset comes back through a file response that honours `Range`, and one the -subscriber would compute comes back as a stream that ignores it. Which is the +server would compute comes back as a stream that ignores it. Which is the distinction the whole arrangement rests on, and the one thing a test against a -live subscriber could not switch off at will. +live server could not switch off at will. """ import contextlib @@ -28,13 +28,13 @@ import blosc2 -# The stand-in subscriber binds a real socket, and Pyodide has no listen(2): +# The stand-in server binds a real socket, and Pyodide has no listen(2): # node asks for the `ws` module that is not there, and takes the runtime down # with it rather than raising pytestmark = pytest.mark.skipif(blosc2.IS_WASM, reason="no listening sockets on wasm32") -class _Subscriber: +class _Cat2Server: """A Caterva2-shaped server over one .b2nd file.""" def __init__( @@ -57,13 +57,13 @@ def __init__( self.multipart = multipart # False: answer only the first range asked for self.bad_parts = bad_parts # ... and this many multi-range answers do too self.merge_ranges = merge_ranges # as Starlette does with ranges that touch - self.geometry = geometry # False: report a dataset the subscriber computes, + self.geometry = geometry # False: report a dataset the server computes, # which has no chunks or blocks of its own to report self.log = [] # (endpoint, status, bytes served) self.reload() def reload(self): - """Pick up the file as it is now, as a subscriber would on the next request. + """Pick up the file as it is now, as a server would on the next request. A leaf is served out of its window in the container, which is what makes it look to a client exactly like a dataset of its own: byte 0 of what it @@ -121,7 +121,7 @@ def handle(self): super().handle() def _send(self, status, body, headers=(), endpoint=""): - self.server.subscriber.log.append((endpoint, status, len(body))) + self.server.cat2.log.append((endpoint, status, len(body))) self.send_response(status) for name, value in headers: self.send_header(name, value) @@ -130,55 +130,55 @@ def _send(self, status, body, headers=(), endpoint=""): self.wfile.write(body) def do_GET(self): - sub = self.server.subscriber - if sub.cookie and self.headers.get("Cookie") != sub.cookie: + srv = self.server.cat2 + if srv.cookie and self.headers.get("Cookie") != srv.cookie: self._send(401, b"unauthorized", endpoint="auth") return endpoint = self.path.split("/")[2] if endpoint == "info": - self._send(200, json.dumps(sub.meta).encode(), endpoint="info") + self._send(200, json.dumps(srv.meta).encode(), endpoint="info") elif endpoint == "chunk": nchunk = int(self.path.split("nchunk=")[1]) - self._send(200, sub.array.schunk.get_chunk(nchunk), endpoint="chunk") + self._send(200, srv.array.schunk.get_chunk(nchunk), endpoint="chunk") elif endpoint == "fetch": - self._fetch(sub) + self._fetch(srv) else: self._send(404, b"", endpoint=endpoint) - def _fetch(self, sub): - if sub.fetch_failures: - # A subscriber too busy to answer says nothing about how it serves - sub.fetch_failures -= 1 + def _fetch(self, srv): + if srv.fetch_failures: + # A server too busy to answer says nothing about how it serves + srv.fetch_failures -= 1 self._send(503, b"busy", endpoint="fetch") return wanted = self.headers.get("Range") - if not wanted or not sub.ranges: + if not wanted or not srv.ranges: # What a StreamingResponse does with a Range header: nothing at all - self._send(200, sub.frame, endpoint="fetch") + self._send(200, srv.frame, endpoint="fetch") return spans = [] for span in wanted.removeprefix("bytes=").split(","): start, end = (int(n) for n in span.split("-")) - spans.append((start, min(end, len(sub.frame) - 1))) + spans.append((start, min(end, len(srv.frame) - 1))) # Starlette sorts the spans and merges the ones that touch, so a client # cannot count on getting a part per span, nor on the order it asked in spans.sort() merged = [spans[0]] for start, end in spans[1:]: - if start <= merged[-1][1] + 1 and sub.merge_ranges: + if start <= merged[-1][1] + 1 and srv.merge_ranges: merged[-1] = (merged[-1][0], max(merged[-1][1], end)) else: merged.append((start, end)) - partial = not sub.multipart - if len(merged) > 1 and sub.bad_parts: - sub.bad_parts -= 1 # an answer that carries only the first part, once + partial = not srv.multipart + if len(merged) > 1 and srv.bad_parts: + srv.bad_parts -= 1 # an answer that carries only the first part, once partial = True if len(merged) == 1 or partial: start, end = merged[0] # ... and answers a plain 206 when one is left self._send( 206, - sub.frame[start : end + 1], - [("Content-Range", f"bytes {start}-{end}/{len(sub.frame)}"), ("Accept-Ranges", "bytes")], + srv.frame[start : end + 1], + [("Content-Range", f"bytes {start}-{end}/{len(srv.frame)}"), ("Accept-Ranges", "bytes")], endpoint="fetch", ) return @@ -187,9 +187,9 @@ def _fetch(self, sub): for start, end in merged: body += ( f"--{boundary}\r\nContent-Type: application/octet-stream\r\n" - f"Content-Range: bytes {start}-{end}/{len(sub.frame)}\r\n\r\n" + f"Content-Range: bytes {start}-{end}/{len(srv.frame)}\r\n\r\n" ).encode() - body += sub.frame[start : end + 1] + b"\r\n" + body += srv.frame[start : end + 1] + b"\r\n" body += f"--{boundary}--\r\n".encode() self._send( 206, @@ -203,10 +203,10 @@ def _fetch(self, sub): def _serve(tmp_path, data, chunks, blocks, name="ds.b2nd", key=None, **kwargs): - """A C2Array over *data*, served by a subscriber stand-in on localhost. + """A C2Array over *data*, served by a server stand-in on localhost. With *key*, the array is a leaf of a TreeStore container instead of a file - of its own, and the subscriber serves it from its window -- which is what + of its own, and the server serves it from its window -- which is what Caterva2 does, and what the client is meant not to notice. """ urlpath = str(tmp_path / name) @@ -215,33 +215,33 @@ def _serve(tmp_path, data, chunks, blocks, name="ds.b2nd", key=None, **kwargs): else: with blosc2.TreeStore(urlpath, mode="w") as tstore: tstore[key] = blosc2.asarray(data, chunks=chunks, blocks=blocks) - server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) - server.subscriber = _Subscriber(urlpath, key=key, **kwargs) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + httpd.cat2 = _Cat2Server(urlpath, key=key, **kwargs) # A short poll interval, because `shutdown()` waits for one to elapse before # the serve loop notices: at the default 0.5 s that is half a second of doing # nothing per test, and this file has enough of them for that to be most of # what it costs - threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True).start() - urlbase = f"http://127.0.0.1:{server.server_address[1]}/" + threading.Thread(target=httpd.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True).start() + urlbase = f"http://127.0.0.1:{httpd.server_address[1]}/" path = f"@public/{name}{key or ''}" array = blosc2.C2Array(path, urlbase=urlbase, auth_token=kwargs.get("cookie")) - return array, server.subscriber, server + return array, httpd.cat2, httpd @pytest.fixture -def subscriber(tmp_path): +def server(tmp_path): """Serve one array; the test parametrizes with `_serve`'s arguments.""" servers = [] def build(*args, **kwargs): - array, sub, server = _serve(tmp_path, *args, **kwargs) - servers.append(server) - return array, sub + array, srv, httpd = _serve(tmp_path, *args, **kwargs) + servers.append(httpd) + return array, srv yield build - for server in servers: - server.shutdown() - server.server_close() + for httpd in servers: + httpd.shutdown() + httpd.server_close() @pytest.fixture @@ -254,24 +254,24 @@ def _incompressible(shape, seed=0): return np.random.default_rng(seed).random(shape) -def _bytes(sub, endpoint): - return sum(n for kind, _, n in sub.log if kind == endpoint) +def _bytes(srv, endpoint): + return sum(n for kind, _, n in srv.log if kind == endpoint) -def test_blocks_are_read_over_ranges(subscriber, any_chunk_wants_blocks): +def test_blocks_are_read_over_ranges(server, any_chunk_wants_blocks): data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") - sub.log.clear() + srv.log.clear() assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) # The frame index (header, then offsets), one read for the chunk's block # offsets, and one for the block the slice lands in - assert [kind for kind, _, _ in sub.log] == ["fetch"] * 4 - assert {status for _, status, _ in sub.log} == {206} - assert not _bytes(sub, "chunk") + assert [kind for kind, _, _ in srv.log] == ["fetch"] * 4 + assert {status for _, status, _ in srv.log} == {206} + assert not _bytes(srv, "chunk") # A block of a chunk, not the chunk: an eighth of it here, and never the frame - assert _bytes(sub, "fetch") < sub.array.schunk.cbytes / 8 + assert _bytes(srv, "fetch") < srv.array.schunk.cbytes / 8 # ... and the rest of the array still arrives correctly afterwards assert np.array_equal(p[...], data) @@ -286,78 +286,78 @@ def test_blocks_are_read_over_ranges(subscriber, any_chunk_wants_blocks): ], ids=["1d", "2d", "3d", "point"], ) -def test_block_reads_are_correct(subscriber, any_chunk_wants_blocks, shape, chunks, blocks, item): +def test_block_reads_are_correct(server, any_chunk_wants_blocks, shape, chunks, blocks, item): data = _incompressible(shape) - array, _ = subscriber(data, chunks=chunks, blocks=blocks) + array, _ = server(data, chunks=chunks, blocks=blocks) p = blosc2.Proxy(array, mode="w") assert np.array_equal(p[item], data[item]) assert np.array_equal(p[...], data) -def test_blocks_carry_the_auth_cookie(subscriber, any_chunk_wants_blocks): +def test_blocks_carry_the_auth_cookie(server, any_chunk_wants_blocks): # fsspec's HTTP filesystem cannot carry this, which is why the block reads of - # a C2Array are its own rather than an fsspec URL pointed at the subscriber + # a C2Array are its own rather than an fsspec URL pointed at the server data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20), cookie="token=sikrit") + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), cookie="token=sikrit") p = blosc2.Proxy(array, mode="w") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) assert array.block_source() is not None - assert not any(status == 401 for _, status, _ in sub.log) + assert not any(status == 401 for _, status, _ in srv.log) -def test_a_streamed_dataset_falls_back_to_chunks(subscriber, any_chunk_wants_blocks): +def test_a_streamed_dataset_falls_back_to_chunks(server, any_chunk_wants_blocks): # A lazy expression, an HDF5 leaf or a .b2z member is built rather than # stored, and the response that carries it ignores Range data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20), ranges=False) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), ranges=False) p = blosc2.Proxy(array, mode="w") - sub.log.clear() + srv.log.clear() assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) assert array.block_source() is None - assert [kind for kind, _, _ in sub.log] == ["fetch", "chunk"] + assert [kind for kind, _, _ in srv.log] == ["fetch", "chunk"] # The probe must not read the body it refused: the whole dataset is what it # would have downloaded to find out that ranges are not served - assert _bytes(sub, "fetch") == len(sub.frame) # served, but never read + assert _bytes(srv, "fetch") == len(srv.frame) # served, but never read # And it is never probed again, whatever else is asked for assert np.array_equal(p[...], data) - assert sum(1 for kind, _, _ in sub.log if kind == "fetch") == 1 + assert sum(1 for kind, _, _ in srv.log if kind == "fetch") == 1 -def test_a_computed_dataset_is_ruled_out_without_a_request(subscriber, any_chunk_wants_blocks): - # api/info tells a stored dataset from one the subscriber computes: the +def test_a_computed_dataset_is_ruled_out_without_a_request(server, any_chunk_wants_blocks): + # api/info tells a stored dataset from one the server computes: the # latter reports `expression` and `operands` where this reports a geometry data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) del array.meta["chunks"] - sub.log.clear() + srv.log.clear() assert array.block_source() is None - assert not sub.log + assert not srv.log -def test_small_chunks_are_fetched_whole(subscriber): +def test_small_chunks_are_fetched_whole(server): # Below the threshold a chunk is one cheap request, so blocks would only add # a round trip: nothing goes looking for the frame index, let alone a block data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") - sub.log.clear() + srv.log.clear() assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) assert array.block_source() is None - assert [kind for kind, _, _ in sub.log] == ["chunk"] + assert [kind for kind, _, _ in srv.log] == ["chunk"] -def test_a_dataset_that_serves_no_blocks_keeps_the_chunkwise_bitmap(tmp_path, subscriber): +def test_a_dataset_that_serves_no_blocks_keeps_the_chunkwise_bitmap(tmp_path, server): # Nothing will ever ask this one for a block, so its cache records chunks: # the bitmap an older blosc2 also reads, and none of the per-block # bookkeeping that would be kept only to say `all of them` every time data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) assert not array.serves_blocks # decided from api/info, without a request cache = str(tmp_path / "chunkwise-cache.b2nd") p = blosc2.Proxy(array, urlpath=cache, mode="w") @@ -369,26 +369,26 @@ def test_a_dataset_that_serves_no_blocks_keeps_the_chunkwise_bitmap(tmp_path, su assert "proxy-fetched-blocks" not in kept -def test_blocks_accumulate_in_a_chunk(subscriber, any_chunk_wants_blocks): +def test_blocks_accumulate_in_a_chunk(server, any_chunk_wants_blocks): data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) - served = len(sub.log) + served = len(srv.log) # A different block of the same chunk: what is already cached stays cached assert np.array_equal(p[0:5, 100:110], data[0:5, 100:110]) - assert len(sub.log) > served - served = len(sub.log) + assert len(srv.log) > served + served = len(srv.log) # Both are now in the same cached chunk, and both are still right assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) assert np.array_equal(p[0:5, 100:110], data[0:5, 100:110]) - assert len(sub.log) == served + assert len(srv.log) == served -def test_blocks_survive_a_reopened_cache(tmp_path, subscriber, any_chunk_wants_blocks): +def test_blocks_survive_a_reopened_cache(tmp_path, server, any_chunk_wants_blocks): data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "c2-cache.b2nd") p = blosc2.Proxy(array, urlpath=cache, mode="a") @@ -397,87 +397,87 @@ def test_blocks_survive_a_reopened_cache(tmp_path, subscriber, any_chunk_wants_b # A partly filled chunk survives, so the blocks in it do not travel again p = blosc2.Proxy(array, urlpath=cache, mode="a") - served = len(sub.log) + served = len(srv.log) assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) - assert len(sub.log) == served + assert len(srv.log) == served # ... and the ones missing from it still do assert np.array_equal(p[0:5, 100:110], data[0:5, 100:110]) - assert len(sub.log) > served + assert len(srv.log) > served assert np.array_equal(p[...], data) -def test_a_cache_that_holds_the_slice_costs_no_request(tmp_path, subscriber, any_chunk_wants_blocks): +def test_a_cache_that_holds_the_slice_costs_no_request(tmp_path, server, any_chunk_wants_blocks): # Re-running a script over a cache that already covers the slice: `api/info` # is all it takes -- the one the proxy spends looking again at an array that # could have been written to since (see `refresh_stamp`). Nothing opens the # frame, because opening it is what `block_source` puts off until a fetch # actually wants a chunk -- and this fetch wants none. data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "held.b2nd") item = (slice(0, 5), slice(0, 10)) blosc2.Proxy(array, urlpath=cache, mode="a").fetch(item) - # A later run: its own array over the same subscriber, its own source + # A later run: its own array over the same server, its own source again = blosc2.C2Array(array.path, urlbase=array.urlbase) - sub.log.clear() + srv.log.clear() p = blosc2.Proxy(again, urlpath=cache, mode="a") p.fetch(item) assert np.array_equal(p[item], data[item]) - assert [kind for kind, _, _ in sub.log] == ["info"] + assert [kind for kind, _, _ in srv.log] == ["info"] # ... and a slice the cache does not hold opens the frame then: the header, # the layout of the chunk it lands in, and the blocks. Not where the chunks # are -- the earlier run left that in the cache assert np.array_equal(p[100:105, 0:10], data[100:105, 0:10]) - assert [kind for kind, _, _ in sub.log] == ["info"] + ["fetch"] * 3 + assert [kind for kind, _, _ in srv.log] == ["info"] + ["fetch"] * 3 -def test_a_kept_index_halves_a_warm_fetch(tmp_path, subscriber, any_chunk_wants_blocks): +def test_a_kept_index_halves_a_warm_fetch(tmp_path, server, any_chunk_wants_blocks): # A later run wanting different blocks of chunks a previous one half filled: # where the chunks are and where those blocks are both came out of the cache, # so what travels is the header and the blocks, and nothing between data = _incompressible((400, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "kept.b2nd") blosc2.Proxy(array, urlpath=cache, mode="a").fetch((slice(None), slice(0, 10))) again = blosc2.C2Array(array.path, urlbase=array.urlbase) - sub.log.clear() + srv.log.clear() p = blosc2.Proxy(again, urlpath=cache, mode="a") assert np.array_equal(p[:, 100:110], data[:, 100:110]) # The proxy's look at the array, then the header and the blocks -- nothing between - assert [kind for kind, _, _ in sub.log] == ["info", "fetch", "fetch"] + assert [kind for kind, _, _ in srv.log] == ["info", "fetch", "fetch"] assert np.array_equal(p[...], data) # ... and the rest still reads right -def test_a_kept_index_does_not_open_the_frame_to_be_taken_up(tmp_path, subscriber, any_chunk_wants_blocks): +def test_a_kept_index_does_not_open_the_frame_to_be_taken_up(tmp_path, server, any_chunk_wants_blocks): # Handing the index to the source would build the source to receive it, and # building it reads the header -- a request, at the very moment of a run that # may go on to fetch nothing. It waits with the array until there is a source data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "unopened.b2nd") item = (slice(0, 5), slice(0, 10)) blosc2.Proxy(array, urlpath=cache, mode="a").fetch(item) again = blosc2.C2Array(array.path, urlbase=array.urlbase) - sub.log.clear() + srv.log.clear() p = blosc2.Proxy(again, urlpath=cache, mode="a") assert again._pending_index is not None # taken out of the cache, not yet used - assert [kind for kind, _, _ in sub.log] == ["info"] # the proxy's look, and no frame read + assert [kind for kind, _, _ in srv.log] == ["info"] # the proxy's look, and no frame read p.fetch(item) - assert [kind for kind, _, _ in sub.log] == ["info"] + assert [kind for kind, _, _ in srv.log] == ["info"] -def test_a_whole_chunk_cache_is_adopted(tmp_path, subscriber, any_chunk_wants_blocks): +def test_a_whole_chunk_cache_is_adopted(tmp_path, server, any_chunk_wants_blocks): # A cache left by a run that fetched whole chunks (which is every run before # this existed) holds complete chunks, so nothing in it is fetched again data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "chunkwise.b2nd") - array._block_source = None # as if the subscriber served no ranges + array._block_source = None # as if the server served no ranges p = blosc2.Proxy(array, urlpath=cache, mode="a") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) del p @@ -485,59 +485,59 @@ def test_a_whole_chunk_cache_is_adopted(tmp_path, subscriber, any_chunk_wants_bl # The same dataset, opened afresh: this one takes the blocks path array = blosc2.C2Array(array.path, urlbase=array.urlbase) p = blosc2.Proxy(array, urlpath=cache, mode="a") - served = len(sub.log) + served = len(srv.log) assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) - assert len(sub.log) == served + assert len(srv.log) == served assert np.array_equal(p[...], data) -def test_blocks_per_chunk_costs_no_request(subscriber): +def test_blocks_per_chunk_costs_no_request(server): data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) - sub.log.clear() + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + srv.log.clear() assert array.blocks_per_chunk == math.prod((100 // 10, 200 // 20)) - assert not sub.log + assert not srv.log -def test_read_range_says_so_when_there_are_no_ranges(subscriber, any_chunk_wants_blocks): +def test_read_range_says_so_when_there_are_no_ranges(server, any_chunk_wants_blocks): data = _incompressible((200, 200)) - array, _ = subscriber(data, chunks=(100, 200), blocks=(10, 20), ranges=False) + array, _ = server(data, chunks=(100, 200), blocks=(10, 20), ranges=False) with pytest.raises(ValueError, match="not served in byte ranges"): array.read_range(0, 32) -def test_a_whole_wave_travels_in_one_request(subscriber, any_chunk_wants_blocks): +def test_a_whole_wave_travels_in_one_request(server, any_chunk_wants_blocks): # A column through every chunk: each one wants a handful of blocks that lie # apart in the file, which without batching is a request each data = _incompressible((400, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") assert array.block_source() is not None # the header, read once per array - sub.log.clear() + srv.log.clear() assert np.array_equal(p[:, 0:10], data[:, 0:10]) # One request for where the four chunks are, one for the layouts of all of # them, one for all their blocks -- three waves, not three per chunk - assert [kind for kind, _, _ in sub.log] == ["fetch", "fetch", "fetch"] - assert {status for _, status, _ in sub.log} == {206} - assert _bytes(sub, "fetch") < sub.array.schunk.cbytes / 4 + assert [kind for kind, _, _ in srv.log] == ["fetch", "fetch", "fetch"] + assert {status for _, status, _ in srv.log} == {206} + assert _bytes(srv, "fetch") < srv.array.schunk.cbytes / 4 # Which is the whole of the difference: one request per range otherwise - other, sub2 = subscriber(data, chunks=(100, 200), blocks=(10, 20), name="unbatched.b2nd") + other, sub2 = server(data, chunks=(100, 200), blocks=(10, 20), name="unbatched.b2nd") other.block_source().max_ranges = 1 q = blosc2.Proxy(other, mode="w") sub2.log.clear() assert np.array_equal(q[:, 0:10], data[:, 0:10]) - assert len(sub2.log) > 4 * len(sub.log) + assert len(sub2.log) > 4 * len(srv.log) -def test_merged_and_reordered_parts_are_read_correctly(subscriber, any_chunk_wants_blocks): +def test_merged_and_reordered_parts_are_read_correctly(server, any_chunk_wants_blocks): # The server sorts the spans and merges the ones that touch, so the answer # carries fewer parts than were asked for and in an order of its own data = _incompressible((400, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") assert np.array_equal(p[:, 0:10], data[:, 0:10]) @@ -545,55 +545,55 @@ def test_merged_and_reordered_parts_are_read_correctly(subscriber, any_chunk_wan assert array.max_ranges > 1 # ... and it never had to stop batching -def test_a_server_that_answers_one_range_stops_being_batched(subscriber, any_chunk_wants_blocks): - # A subscriber that takes the first span of a multi-range request and ignores +def test_a_server_that_answers_one_range_stops_being_batched(server, any_chunk_wants_blocks): + # A server that takes the first span of a multi-range request and ignores # the rest: the answer does not cover what was asked for, which is noticed # and never repeated data = _incompressible((400, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20), multipart=False) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), multipart=False) p = blosc2.Proxy(array, mode="w") assert np.array_equal(p[:, 0:10], data[:, 0:10]) assert array.max_ranges == 1 - served = len(sub.log) + served = len(srv.log) assert np.array_equal(p[:, 100:110], data[:, 100:110]) # One request per range from here on, and no second attempt at batching - assert len(sub.log) > served + 2 + assert len(srv.log) > served + 2 assert np.array_equal(p[...], data) # --- a cache is checked against the bytes it was filled from ---------------- -def _replace(sub, data, chunks, blocks): +def _replace(srv, data, chunks, blocks): """Rewrite the served dataset, as an upload of new data would.""" - blosc2.asarray(data, chunks=chunks, blocks=blocks, urlpath=sub.path, mode="w") - stat = pathlib.Path(sub.path).stat() - os.utime(sub.path, (stat.st_atime, stat.st_mtime + 10)) # a tick the clock cannot swallow - sub.reload() + blosc2.asarray(data, chunks=chunks, blocks=blocks, urlpath=srv.path, mode="w") + stat = pathlib.Path(srv.path).stat() + os.utime(srv.path, (stat.st_atime, stat.st_mtime + 10)) # a tick the clock cannot swallow + srv.reload() -def test_a_cache_is_stamped_with_the_remote_mtime(subscriber): +def test_a_cache_is_stamped_with_the_remote_mtime(server): data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") - assert array.stamp == f"{sub.mtime}:{sub.array.schunk.cbytes}" + assert array.stamp == f"{srv.mtime}:{srv.array.schunk.cbytes}" assert p.schunk.vlmeta["proxy-stamp"] == array.stamp -def test_a_cache_from_other_bytes_is_refused(tmp_path, subscriber, any_chunk_wants_blocks): +def test_a_cache_from_other_bytes_is_refused(tmp_path, server, any_chunk_wants_blocks): # Same shape, same partitioning, different data: geometry cannot tell, and # every cached chunk (and the offsets it was fetched by) is stale data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "stamped.b2nd") p = blosc2.Proxy(array, urlpath=cache, mode="a") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) del p other = _incompressible((200, 200), seed=1) - _replace(sub, other, chunks=(100, 200), blocks=(10, 20)) + _replace(srv, other, chunks=(100, 200), blocks=(10, 20)) replaced = blosc2.C2Array(array.path, urlbase=array.urlbase) assert replaced.stamp != array.stamp @@ -606,38 +606,38 @@ def test_a_cache_from_other_bytes_is_refused(tmp_path, subscriber, any_chunk_wan assert np.array_equal(p[...], other) -def test_a_cache_of_bytes_that_were_replaced_is_emptied(tmp_path, subscriber, any_chunk_wants_blocks): +def test_a_cache_of_bytes_that_were_replaced_is_emptied(tmp_path, server, any_chunk_wants_blocks): # `blosc2.open` rebuilds the proxy over the cache as it stands, with no # `mode="a"` to refuse it by: the chunks in there were fetched from a frame # that is gone, so what the cache says it holds is dropped rather than served data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "emptied.b2nd") p = blosc2.Proxy(array, urlpath=cache, mode="w") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) del p other = _incompressible((200, 200), seed=1) - _replace(sub, other, chunks=(100, 200), blocks=(10, 20)) + _replace(srv, other, chunks=(100, 200), blocks=(10, 20)) reopened = blosc2.open(cache, mode="a") assert np.array_equal(reopened[0:5, 0:10], other[0:5, 0:10]) assert np.array_equal(reopened[...], other) -def test_a_cache_emptied_of_replaced_bytes_stays_emptied(tmp_path, subscriber, any_chunk_wants_blocks): +def test_a_cache_emptied_of_replaced_bytes_stays_emptied(tmp_path, server, any_chunk_wants_blocks): # The stamp is written on the way in, so the run after this one finds a cache # whose stamp fits and believes what it says it holds: emptying it has to # reach the file, not just the proxy that noticed data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "stillemptied.b2nd") p = blosc2.Proxy(array, urlpath=cache, mode="w") assert np.array_equal(p[...], data) # every chunk of it, fetched and recorded del p other = _incompressible((200, 200), seed=1) - _replace(sub, other, chunks=(100, 200), blocks=(10, 20)) + _replace(srv, other, chunks=(100, 200), blocks=(10, 20)) noticed = blosc2.open(cache, mode="a") # opened over the new bytes, dropped unread del noticed @@ -646,19 +646,19 @@ def test_a_cache_emptied_of_replaced_bytes_stays_emptied(tmp_path, subscriber, a assert np.array_equal(again[...], other) -def test_a_read_only_cache_of_replaced_bytes_reads_past_it(tmp_path, subscriber, any_chunk_wants_blocks): +def test_a_read_only_cache_of_replaced_bytes_reads_past_it(tmp_path, server, any_chunk_wants_blocks): # Nothing may be written to a cache opened read-only, so it cannot be emptied # either -- but nothing of it is believed either, and the read falls through # to the source rather than coming back with the bytes that are gone data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "readonlyreplaced.b2nd") p = blosc2.Proxy(array, urlpath=cache, mode="w") assert np.array_equal(p[...], data) del p other = _incompressible((200, 200), seed=1) - _replace(sub, other, chunks=(100, 200), blocks=(10, 20)) + _replace(srv, other, chunks=(100, 200), blocks=(10, 20)) reopened = blosc2.open(cache, mode="r") assert np.array_equal(reopened[:], other) # read past the cache, off the source @@ -666,9 +666,9 @@ def test_a_read_only_cache_of_replaced_bytes_reads_past_it(tmp_path, subscriber, assert reopened.schunk.vlmeta["proxy-stamp"] != blosc2.C2Array(array.path, urlbase=array.urlbase).stamp -def test_a_cache_from_the_same_bytes_is_adopted(tmp_path, subscriber, any_chunk_wants_blocks): +def test_a_cache_from_the_same_bytes_is_adopted(tmp_path, server, any_chunk_wants_blocks): data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "unchanged.b2nd") p = blosc2.Proxy(array, urlpath=cache, mode="a") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) @@ -677,16 +677,16 @@ def test_a_cache_from_the_same_bytes_is_adopted(tmp_path, subscriber, any_chunk_ again = blosc2.C2Array(array.path, urlbase=array.urlbase) assert again.stamp == array.stamp p = blosc2.Proxy(again, urlpath=cache, mode="a") - served = len(sub.log) + served = len(srv.log) assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) - assert len(sub.log) == served # what it holds was not fetched again + assert len(srv.log) == served # what it holds was not fetched again -def test_no_stamp_when_the_subscriber_reports_no_mtime(tmp_path, subscriber): +def test_no_stamp_when_the_server_reports_no_mtime(tmp_path, server): # Then the cache is checked on geometry alone, as every unstamped source is data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) - sub.mtime = None # the subscriber itself reports none, and goes on doing so + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + srv.mtime = None # the server itself reports none, and goes on doing so array = blosc2.C2Array(array.path, urlbase=array.urlbase) assert array.stamp is None @@ -698,11 +698,11 @@ def test_no_stamp_when_the_subscriber_reports_no_mtime(tmp_path, subscriber): assert blosc2.Proxy(array, urlpath=cache, mode="a") is not None -def test_a_read_only_cache_is_not_stamped(tmp_path, subscriber): +def test_a_read_only_cache_is_not_stamped(tmp_path, server): # `blosc2.open(path, mode="r")` rebuilds the proxy over a cache that may not # be written to; recording the stamp there raised instead of opening it data = _incompressible((200, 200)) - array, _sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, _sub = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "readonly.b2nd") p = blosc2.Proxy(array, urlpath=cache, mode="w") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) @@ -712,35 +712,35 @@ def test_a_read_only_cache_is_not_stamped(tmp_path, subscriber): assert np.array_equal(reopened[0:5, 0:10], data[0:5, 0:10]) -def test_blocks_of_a_container_leaf(subscriber, any_chunk_wants_blocks): - """A leaf of a .b2z is a whole frame inside the container, and a subscriber +def test_blocks_of_a_container_leaf(server, any_chunk_wants_blocks): + """A leaf of a .b2z is a whole frame inside the container, and a server serves it from that window -- so the client reads its blocks knowing nothing about containers, which is the whole of what it takes.""" data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20), name="tree.b2z", key="/g/leaf") + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), name="tree.b2z", key="/g/leaf") p = blosc2.Proxy(array, mode="w") assert array.block_source() is not None - sub.log.clear() + srv.log.clear() assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) - assert not _bytes(sub, "chunk") - assert _bytes(sub, "fetch") < sub.array.schunk.cbytes / 8 + assert not _bytes(srv, "chunk") + assert _bytes(srv, "fetch") < srv.array.schunk.cbytes / 8 assert np.array_equal(p[...], data) -def test_a_container_leaf_is_stamped_like_any_other(subscriber): +def test_a_container_leaf_is_stamped_like_any_other(server): data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20), name="tree.b2z", key="/g/leaf") + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), name="tree.b2z", key="/g/leaf") # A leaf has no mtime of its own: the container's is what says it changed - assert array.stamp == f"{sub.mtime}:{sub.array.schunk.cbytes}" + assert array.stamp == f"{srv.mtime}:{srv.array.schunk.cbytes}" -def test_a_busy_subscriber_is_asked_again(subscriber, any_chunk_wants_blocks): +def test_a_busy_server_is_asked_again(server, any_chunk_wants_blocks): # A 503 to the probe says nothing about whether the dataset is served from a # file, and cost no download to find out -- unlike the streamed 200 the # permanent fallback exists for, so this one is asked again on the next fetch data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20), fetch_failures=1) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), fetch_failures=1) p = blosc2.Proxy(array, mode="w") assert array.block_source() is None # the probe was refused ... @@ -749,90 +749,90 @@ def test_a_busy_subscriber_is_asked_again(subscriber, any_chunk_wants_blocks): assert np.array_equal(p[...], data) -def test_a_streamed_dataset_is_not_asked_again(subscriber, any_chunk_wants_blocks): +def test_a_streamed_dataset_is_not_asked_again(server, any_chunk_wants_blocks): # The other half of the same rule: a 200 is the dataset itself, and asking # again would pay for the whole of it to be told the same thing data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20), ranges=False) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), ranges=False) p = blosc2.Proxy(array, mode="w") assert array.block_source() is None assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) assert array.block_source() is None - assert not any(status == 206 for _, status, _ in sub.log) + assert not any(status == 206 for _, status, _ in srv.log) -def test_a_dataset_that_stops_being_stored_falls_back_to_chunks(subscriber, any_chunk_wants_blocks): - # A subscriber can stop serving a dataset from a file between one fetch and +def test_a_dataset_that_stops_being_stored_falls_back_to_chunks(server, any_chunk_wants_blocks): + # A server can stop serving a dataset from a file between one fetch and # the next -- replaced by a lazy expression, moved into a container it # streams out of. The fetch that runs into it reads the chunks it was after # whole rather than failing, and nothing asks for a range again data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) assert array.block_source() is not None - sub.ranges = False - sub.log.clear() + srv.ranges = False + srv.log.clear() assert np.array_equal(p[50:55, 0:10], data[50:55, 0:10]) assert array.block_source() is None # retired: whole chunks read every dataset - assert _bytes(sub, "chunk") + assert _bytes(srv, "chunk") assert np.array_equal(p[...], data) # The one refused request is the whole of what the change cost: the body it # answered with was never read, and nothing asked for a range again - assert [status for kind, status, _ in sub.log if kind == "fetch"] == [200] + assert [status for kind, status, _ in srv.log if kind == "fetch"] == [200] -def test_a_subscriber_too_busy_for_a_range_keeps_its_source(subscriber, any_chunk_wants_blocks): +def test_a_server_too_busy_for_a_range_keeps_its_source(server, any_chunk_wants_blocks): # The other half of the rule that governs the probe: a 503 says nothing about # how the dataset is served, so the fetch falls back for now and the next one # asks for blocks again data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) source = array.block_source() - sub.fetch_failures = 1 - sub.log.clear() + srv.fetch_failures = 1 + srv.log.clear() assert np.array_equal(p[50:55, 0:10], data[50:55, 0:10]) - assert _bytes(sub, "chunk") # served whole, since the range was refused + assert _bytes(srv, "chunk") # served whole, since the range was refused assert array.block_source() is source - sub.log.clear() + srv.log.clear() assert np.array_equal(p[0:5, 100:110], data[0:5, 100:110]) - assert not _bytes(sub, "chunk") # ... and blocks are asked for again + assert not _bytes(srv, "chunk") # ... and blocks are asked for again def test_a_proxy_over_a_cache_survives_a_dataset_that_became_computed( - tmp_path, subscriber, any_chunk_wants_blocks + tmp_path, server, any_chunk_wants_blocks ): # `blosc2.open` rebuilds the proxy over its own cache, and the source it - # rebuilds may by then be a dataset the subscriber computes -- which reports + # rebuilds may by then be a dataset the server computes -- which reports # no partitioning at all, so nothing may go asking one for it data = _incompressible((200, 200)) - array, sub = subscriber(data, chunks=(100, 200), blocks=(10, 20)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) cache = str(tmp_path / "computed-cache.b2nd") blosc2.Proxy(array, urlpath=cache, mode="w").fetch((slice(0, 5), slice(0, 10))) - sub.geometry = False + srv.geometry = False assert not blosc2.C2Array(array.path, urlbase=array.urlbase).serves_blocks reopened = blosc2.open(cache) assert isinstance(reopened, blosc2.Proxy) assert np.array_equal(reopened[0:5, 0:10], data[0:5, 0:10]) # out of the cache -def test_one_answer_that_misses_its_parts_does_not_end_batching(subscriber, any_chunk_wants_blocks): +def test_one_answer_that_misses_its_parts_does_not_end_batching(server, any_chunk_wants_blocks): # Batching is worth an order of magnitude, so a single truncated answer is # worth retrying a range at a time rather than giving up the whole of it data = _incompressible((400, 400)) - array, sub = subscriber(data, chunks=(200, 200), blocks=(10, 20), bad_parts=1) + array, srv = server(data, chunks=(200, 200), blocks=(10, 20), bad_parts=1) p = blosc2.Proxy(array, mode="w") item = (slice(190, 210), slice(190, 210)) # a corner of each of the four chunks assert np.array_equal(p[item], data[item]) - assert not sub.bad_parts # the answer that carried one part was asked for ... + assert not srv.bad_parts # the answer that carried one part was asked for ... assert array.max_ranges > 1 # ... and cost the batching nothing assert np.array_equal(p[...], data) @@ -906,17 +906,17 @@ def test_a_part_that_ends_where_the_frame_does_is_kept(): blosc2.c2array._span_of([(100, b"12345", None)], 100, 6, "url") -def test_a_small_frame_is_read_over_ranges(subscriber, any_chunk_wants_blocks): +def test_a_small_frame_is_read_over_ranges(server, any_chunk_wants_blocks): # The whole of the frame arrives in the first read an open asks for, which # is the clipped answer above, and the dataset is served in blocks all the same data = np.arange(200, dtype="i4").reshape(20, 10) - array, sub = subscriber(data, chunks=(10, 10), blocks=(5, 10)) - assert len(sub.frame) < blosc2.proxy_source._FRAME_PREFETCH + array, srv = server(data, chunks=(10, 10), blocks=(5, 10)) + assert len(srv.frame) < blosc2.proxy_source._FRAME_PREFETCH p = blosc2.Proxy(array, mode="w") assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) assert array.block_source() is not None - assert not _bytes(sub, "chunk") + assert not _bytes(srv, "chunk") def test_the_shared_client_keeps_no_cookies(): diff --git a/tests/ndarray/test_c2array_writes.py b/tests/ndarray/test_c2array_writes.py index 8a9b1a3f1..aa17f574d 100644 --- a/tests/ndarray/test_c2array_writes.py +++ b/tests/ndarray/test_c2array_writes.py @@ -7,7 +7,7 @@ """Filling a pre-sized remote array a chunk at a time, from several writers. -The stand-in here answers the write contract a subscriber is meant to answer: +The stand-in here answers the write contract a server is meant to answer: one chunk per request, into a slot nothing was written to yet, refused with a 409 otherwise. That refusal is the whole of the coordination -- the frame's own offsets say which slots are free, so two writers that both believe they own a @@ -39,7 +39,7 @@ SHAPE = (CHUNKS[0] * NCHUNKS,) -class _Subscriber: +class _Cat2Server: """A Caterva2-shaped server over one .b2nd file, that also accepts writes.""" def __init__(self, path): @@ -121,7 +121,7 @@ def handle(self): super().handle() def _send(self, status, body, headers=(), endpoint=""): - self.server.subscriber.log.append((endpoint, status)) + self.server.cat2.log.append((endpoint, status)) self.send_response(status) for name, value in headers: self.send_header(name, value) @@ -130,43 +130,43 @@ def _send(self, status, body, headers=(), endpoint=""): self.wfile.write(body) def do_GET(self): - sub = self.server.subscriber + srv = self.server.cat2 endpoint = self.path.split("/")[2] if endpoint == "info": - self._send(200, json.dumps(sub.meta).encode(), endpoint="info") + self._send(200, json.dumps(srv.meta).encode(), endpoint="info") elif endpoint == "chunk": nchunk = int(self.path.split("nchunk=")[1]) - with sub.lock: - self._send(200, sub.array.schunk.get_chunk(nchunk), endpoint="chunk") + with srv.lock: + self._send(200, srv.array.schunk.get_chunk(nchunk), endpoint="chunk") elif endpoint == "fetch": - self._fetch(sub) + self._fetch(srv) else: self._send(404, b"", endpoint=endpoint) def do_POST(self): - sub = self.server.subscriber + srv = self.server.cat2 endpoint = self.path.split("/")[2].split("?")[0] if endpoint != "chunk": self._send(404, b"", endpoint=endpoint) return nchunk = int(self.path.split("nchunk=")[1]) body = self.rfile.read(int(self.headers.get("Content-Length", 0))) - status, answer = sub.write_chunk(nchunk, body) + status, answer = srv.write_chunk(nchunk, body) self._send(status, json.dumps(answer).encode(), endpoint="write") - def _fetch(self, sub): + def _fetch(self, srv): """Ranges over the frame's bytes, or the slice itself when none is asked. - Both halves of what a subscriber serves: `C2Array.__getitem__` asks for a + Both halves of what a server serves: `C2Array.__getitem__` asks for a slice and gets a cframe of it, while the block path asks for byte ranges of the file. A fill has to be visible through both. """ query = parse_qs(urlparse(self.path).query) - frame = pathlib.Path(sub.path).read_bytes() + frame = pathlib.Path(srv.path).read_bytes() wanted = self.headers.get("Range") if not wanted: - with sub.lock: - array = sub.array + with srv.lock: + array = srv.array sliced = array[_parse_slice(query.get("slice_", [""])[0], array.ndim)] self._send(200, blosc2.asarray(sliced).to_cframe(), endpoint="fetch") return @@ -227,21 +227,21 @@ def _parse_slice(text, ndim): @pytest.fixture -def subscriber(tmp_path): +def server(tmp_path): """A pre-sized, unwritten array and a server over it.""" path = tmp_path / "run.b2nd" presized = blosc2.uninit(SHAPE, dtype=np.int32, chunks=CHUNKS, blocks=BLOCKS, urlpath=str(path)) del presized # the server's handle is to be the only one over this file - sub = _Subscriber(path) - server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) - server.subscriber = sub - threading.Thread(target=server.serve_forever, daemon=True).start() - urlbase = f"http://127.0.0.1:{server.server_address[1]}/" + srv = _Cat2Server(path) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + httpd.cat2 = srv + threading.Thread(target=httpd.serve_forever, daemon=True).start() + urlbase = f"http://127.0.0.1:{httpd.server_address[1]}/" try: - yield blosc2.C2Array("run.b2nd", urlbase=urlbase), sub + yield blosc2.C2Array("run.b2nd", urlbase=urlbase), srv finally: - server.shutdown() - server.server_close() + httpd.shutdown() + httpd.server_close() def _chunk(nchunk, value=None): @@ -250,32 +250,32 @@ def _chunk(nchunk, value=None): return blosc2.compress2(data, typesize=4, blocksize=BLOCKS[0] * 4) -def test_a_chunk_written_is_read_back(subscriber): - array, sub = subscriber +def test_a_chunk_written_is_read_back(server): + array, srv = server array.update_chunk(2, _chunk(2)) assert np.all(array[2 * CHUNKS[0] : 3 * CHUNKS[0]] == 2) # ... and nothing else was touched assert np.all(array[0 : CHUNKS[0]] == 0) -def test_a_second_write_is_refused(subscriber): - array, sub = subscriber +def test_a_second_write_is_refused(server): + array, srv = server array.update_chunk(1, _chunk(1)) with pytest.raises(blosc2.ChunkAlreadyWritten): array.update_chunk(1, _chunk(1, value=99)) assert np.all(array[CHUNKS[0] : 2 * CHUNKS[0]] == 1) # the first write stands -def test_a_chunk_of_the_wrong_shape_is_refused(subscriber): - array, sub = subscriber +def test_a_chunk_of_the_wrong_shape_is_refused(server): + array, srv = server wrong = blosc2.compress2(np.zeros(CHUNKS[0] // 2, dtype=np.int32), typesize=4) with pytest.raises(Exception): # noqa: B017 -- an HTTP 400, whatever httpx calls it array.update_chunk(0, wrong) assert not array.written_chunks().any() -def test_written_chunks_tracks_the_fill(subscriber): - array, sub = subscriber +def test_written_chunks_tracks_the_fill(server): + array, srv = server assert list(array.written_chunks()) == [False] * NCHUNKS array.update_chunk(3, _chunk(3)) assert list(array.written_chunks()) == [False, False, False, True, False, False] @@ -283,14 +283,14 @@ def test_written_chunks_tracks_the_fill(subscriber): assert list(array.written_chunks()) == [True, False, False, True, False, False] -def test_a_written_chunk_of_zeros_counts_as_written(subscriber): +def test_a_written_chunk_of_zeros_counts_as_written(server): """The reason a pre-sized array is filled with `uninit` and not with `zeros`. Compressing an all-zero buffer gives a run-length chunk, so a slot written with one is special again -- but tagged as zeros, not as uninitialized, which is what keeps it distinguishable from a slot nobody has reached yet. """ - array, sub = subscriber + array, srv = server array.update_chunk(4, _chunk(4, value=0)) assert array.written_chunks()[4] assert np.all(array[4 * CHUNKS[0] : 5 * CHUNKS[0]] == 0) @@ -298,9 +298,9 @@ def test_a_written_chunk_of_zeros_counts_as_written(subscriber): array.update_chunk(4, _chunk(4)) -def test_a_fill_leaves_the_chunks_before_it_where_they_were(subscriber): +def test_a_fill_leaves_the_chunks_before_it_where_they_were(server): """What makes an append-only fill cheap to read alongside.""" - array, sub = subscriber + array, srv = server array.update_chunk(0, _chunk(0, value=42)) placed = array.get_chunk(0) for nchunk in range(1, NCHUNKS): @@ -311,8 +311,8 @@ def test_a_fill_leaves_the_chunks_before_it_where_they_were(subscriber): assert np.all(array[nchunk * CHUNKS[0] : (nchunk + 1) * CHUNKS[0]] == nchunk) -def test_concurrent_writers_fill_the_array(subscriber): - array, sub = subscriber +def test_concurrent_writers_fill_the_array(server): + array, srv = server urlbase = array.urlbase def fill(nchunk): @@ -329,8 +329,8 @@ def fill(nchunk): np.testing.assert_array_equal(array[:], expected) -def test_two_writers_racing_for_one_chunk_leave_one_winner(subscriber): - array, sub = subscriber +def test_two_writers_racing_for_one_chunk_leave_one_winner(server): + array, srv = server urlbase = array.urlbase barrier = threading.Barrier(2) @@ -351,8 +351,8 @@ def fill(value): assert stored[0] in (7, 8) -def test_a_reader_sees_chunks_that_land_after_it_read(subscriber): - array, sub = subscriber +def test_a_reader_sees_chunks_that_land_after_it_read(server): + array, srv = server array.update_chunk(0, _chunk(0)) assert np.all(array[0 : CHUNKS[0]] == 0) # reads, and indexes, the frame array.update_chunk(1, _chunk(1)) @@ -360,8 +360,8 @@ def test_a_reader_sees_chunks_that_land_after_it_read(subscriber): @pytest.mark.asyncio -async def test_chunks_can_be_written_off_the_event_loop(subscriber): - array, sub = subscriber +async def test_chunks_can_be_written_off_the_event_loop(server): + array, srv = server answer = await array.aupdate_chunk(2, _chunk(2)) assert answer["written"] == 1 with pytest.raises(blosc2.ChunkAlreadyWritten): @@ -375,7 +375,7 @@ def _fill(array, values=None): array.update_chunk(nchunk, _chunk(nchunk, value=None if values is None else values)) -def test_a_filling_array_is_stamped_afresh_on_every_write(subscriber): +def test_a_filling_array_is_stamped_afresh_on_every_write(server): """A cache of an array still being filled has to be thrown away, not kept. What it holds of a chunk nobody had written is the zeros an unwritten chunk @@ -383,7 +383,7 @@ def test_a_filling_array_is_stamped_afresh_on_every_write(subscriber): both are wrong, and nothing in the cache tells them from the chunks that are still good. """ - array, sub = subscriber + array, srv = server stamps = [] for nchunk in range(3): array.update_chunk(nchunk, _chunk(nchunk)) @@ -391,9 +391,9 @@ def test_a_filling_array_is_stamped_afresh_on_every_write(subscriber): assert len(set(stamps)) == len(stamps) -def test_a_complete_array_keeps_one_stamp(subscriber): +def test_a_complete_array_keeps_one_stamp(server): """Once every slot is claimed the array cannot change, so a cache of it stands.""" - array, sub = subscriber + array, srv = server _fill(array) def stamp(): @@ -402,12 +402,12 @@ def stamp(): complete = stamp() assert complete.startswith("n") # An mtime that moved for reasons of its own is not a reason to refetch - os.utime(sub.path, (time.time() + 10, time.time() + 10)) - sub.reload() + os.utime(srv.path, (time.time() + 10, time.time() + 10)) + srv.reload() assert stamp() == complete -def test_two_arrays_at_one_path_are_told_apart(subscriber, tmp_path): +def test_two_arrays_at_one_path_are_told_apart(server, tmp_path): """The hole a size and an mtime leave, which is what the nonce closes. Both arrays here are filled with constant chunks, so they compress to exactly @@ -415,24 +415,24 @@ def test_two_arrays_at_one_path_are_told_apart(subscriber, tmp_path): separates them, and a cache of the first served against the second would be wrong in every chunk. """ - array, sub = subscriber + array, srv = server _fill(array, values=1) first = blosc2.C2Array("run.b2nd", urlbase=array.urlbase) - first_stamp, first_size = first.stamp, pathlib.Path(sub.path).stat().st_size + first_stamp, first_size = first.stamp, pathlib.Path(srv.path).stat().st_size # A different array comes to sit at the same path, of the same size replacement = tmp_path / "replacement.b2nd" presized = blosc2.uninit(SHAPE, dtype=np.int32, chunks=CHUNKS, blocks=BLOCKS, urlpath=str(replacement)) del presized - sub.array = blosc2.open(str(replacement), mode="a", locking=True) - sub.path = str(replacement) + srv.array = blosc2.open(str(replacement), mode="a", locking=True) + srv.path = str(replacement) for nchunk in range(NCHUNKS): - sub.write_chunk(nchunk, _chunk(nchunk, value=2)) - sub.reload() + srv.write_chunk(nchunk, _chunk(nchunk, value=2)) + srv.reload() - assert pathlib.Path(sub.path).stat().st_size == first_size # same bytes on disk - os.utime(sub.path, (first.meta["mtime"], first.meta["mtime"])) - sub.reload() + assert pathlib.Path(srv.path).stat().st_size == first_size # same bytes on disk + os.utime(srv.path, (first.meta["mtime"], first.meta["mtime"])) + srv.reload() second = blosc2.C2Array("run.b2nd", urlbase=array.urlbase) assert second.meta["mtime"] == first.meta["mtime"] # ... and the same mtime assert second.stamp != first_stamp @@ -442,41 +442,41 @@ def test_an_array_with_no_nonce_is_stamped_as_before(tmp_path): """An ordinary dataset, never filled a chunk at a time, is unchanged by this.""" path = tmp_path / "plain.b2nd" blosc2.asarray(np.arange(4000, dtype=np.int32), chunks=(1000,), blocks=(250,), urlpath=str(path)) - sub = _Subscriber(path) - server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) - server.subscriber = sub - threading.Thread(target=server.serve_forever, daemon=True).start() + srv = _Cat2Server(path) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + httpd.cat2 = srv + threading.Thread(target=httpd.serve_forever, daemon=True).start() try: - array = blosc2.C2Array("plain.b2nd", urlbase=f"http://127.0.0.1:{server.server_address[1]}/") - assert array.stamp == f"{sub.mtime}:{array.meta['schunk']['cbytes']}" + array = blosc2.C2Array("plain.b2nd", urlbase=f"http://127.0.0.1:{httpd.server_address[1]}/") + assert array.stamp == f"{srv.mtime}:{array.meta['schunk']['cbytes']}" finally: - server.shutdown() - server.server_close() + httpd.shutdown() + httpd.server_close() -def test_a_cache_of_a_complete_array_survives_a_second_run(subscriber, tmp_path): +def test_a_cache_of_a_complete_array_survives_a_second_run(server, tmp_path): """What the nonce is for: the finished array is the one read again and again. The cache is reopened after the array's mtime has moved under it, which is what a republish or a copy does. Nothing was refetched -- the stamp says it is the same array, and a complete one cannot have changed. """ - array, sub = subscriber + array, srv = server _fill(array) cache = str(tmp_path / "cache.b2nd") proxy = blosc2.Proxy(blosc2.C2Array("run.b2nd", urlbase=array.urlbase), urlpath=cache, mode="w") expected = proxy[:] del proxy - os.utime(sub.path, (time.time() + 10, time.time() + 10)) - sub.reload() - sub.log.clear() + os.utime(srv.path, (time.time() + 10, time.time() + 10)) + srv.reload() + srv.log.clear() proxy = blosc2.Proxy(blosc2.C2Array("run.b2nd", urlbase=array.urlbase), urlpath=cache, mode="a") np.testing.assert_array_equal(proxy[:], expected) - assert not [entry for entry in sub.log if entry[0] in ("chunk", "fetch")] + assert not [entry for entry in srv.log if entry[0] in ("chunk", "fetch")] -def test_a_handle_that_writes_stamps_what_it_wrote(subscriber): +def test_a_handle_that_writes_stamps_what_it_wrote(server): """A writer's own view of the array has to move when the array does. `meta` is read when the array is opened, and `stamp` is built from exactly @@ -484,14 +484,14 @@ def test_a_handle_that_writes_stamps_what_it_wrote(subscriber): answer for the array as it was before its own writes -- and a `Proxy` given that handle would adopt a cache built against them. """ - array, sub = subscriber + array, srv = server before = array.stamp array.update_chunk(0, _chunk(0)) assert array.stamp != before assert array.stamp == blosc2.C2Array("run.b2nd", urlbase=array.urlbase).stamp -def test_asking_about_blocks_does_not_close_the_door_on_the_index(subscriber): +def test_asking_about_blocks_does_not_close_the_door_on_the_index(server): """Two questions, one source, and the answer to one must not answer the other. `serves_blocks` weighs whether splitting a chunk into blocks would pay, which @@ -499,21 +499,21 @@ def test_asking_about_blocks_does_not_close_the_door_on_the_index(subscriber): anyway. Deciding that at the call rather than remembering it is what keeps the block path from shutting the index path down. """ - array, sub = subscriber + array, srv = server assert not array.serves_blocks # chunks here are far under BLOCK_MIN_CBYTES assert array.max_ranges == 1 # the block path, asked first, and declining assert array.block_source() is None assert list(array.written_chunks()) == [False] * NCHUNKS # still answerable -def test_a_filling_stamp_can_never_read_as_a_complete_one(subscriber): +def test_a_filling_stamp_can_never_read_as_a_complete_one(server): """The two branches must not be able to produce the same string. A cache built while chunks were unwritten holds the zeros they read as; if the completed array stamped the same, that cache would be adopted against it and those zeros served as data. """ - array, sub = subscriber + array, srv = server array.update_chunk(0, _chunk(0)) filling = blosc2.C2Array("run.b2nd", urlbase=array.urlbase).stamp assert ":f:" in filling @@ -523,7 +523,7 @@ def test_a_filling_stamp_can_never_read_as_a_complete_one(subscriber): assert ":c:" in complete assert complete != filling - # ... including when the subscriber reports no mtime at all, which is what + # ... including when the server reports no mtime at all, which is what # left the two able to collide handle = blosc2.C2Array("run.b2nd", urlbase=array.urlbase) handle.meta["mtime"] = None @@ -544,14 +544,14 @@ def blocks_are_worth_it(monkeypatch): monkeypatch.setattr(blosc2.proxy_source, "BLOCK_MIN_CBYTES", 0) -def test_blocks_of_a_chunk_written_since_the_index_was_read(subscriber, tmp_path, blocks_are_worth_it): +def test_blocks_of_a_chunk_written_since_the_index_was_read(server, tmp_path, blocks_are_worth_it): """A `Proxy` reading blocks has to see a slot that was filled under it. - `__getitem__` asks the subscriber for a slice and never touches the frame, + `__getitem__` asks the server for a slice and never touches the frame, so a read that goes through it says nothing about the index. This one goes through the offsets, the chunk's block starts and a range read of the block. """ - array, sub = subscriber + array, srv = server array.update_chunk(1, _chunk(1)) proxy = blosc2.Proxy(array, urlpath=str(tmp_path / "blocks.b2nd"), mode="w") assert array.serves_blocks @@ -561,7 +561,7 @@ def test_blocks_of_a_chunk_written_since_the_index_was_read(subscriber, tmp_path np.testing.assert_array_equal(proxy[2 * CHUNKS[0] : 2 * CHUNKS[0] + 10], np.full(10, 2, dtype=np.int32)) -def test_an_index_a_write_moved_is_not_handed_to_a_cache(subscriber, blocks_are_worth_it): +def test_an_index_a_write_moved_is_not_handed_to_a_cache(server, blocks_are_worth_it): """What `_index_state` keeps is where the chunks are, which a write moves. A cache adopts these against a stamp that says the array has not changed @@ -569,7 +569,7 @@ def test_an_index_a_write_moved_is_not_handed_to_a_cache(subscriber, blocks_are_ index read before the write that completed it. Nothing downstream can catch that, so what is stale is not handed over at all. """ - array, sub = subscriber + array, srv = server array.update_chunk(0, _chunk(0, value=7)) array.chunk_layout(0) # builds the source and reads the frame's offsets kept = array._index_state()["offsets"] @@ -581,7 +581,7 @@ def test_an_index_a_write_moved_is_not_handed_to_a_cache(subscriber, blocks_are_ assert array._index_state()["offsets"] not in (b"", kept) # ... and worth keeping again -def test_a_cache_over_a_handle_that_outlived_a_write_is_not_kept(subscriber, tmp_path): +def test_a_cache_over_a_handle_that_outlived_a_write_is_not_kept(server, tmp_path): """A handle names the array as it last looked, and a proxy has to look again. `meta` is read when the handle is opened and never again of itself, so a @@ -589,7 +589,7 @@ def test_a_cache_over_a_handle_that_outlived_a_write_is_not_kept(subscriber, tmp of the array as it was -- which the cache built under that stamp matches, and the bytes no longer do. """ - array, sub = subscriber + array, srv = server reader = blosc2.C2Array("run.b2nd", urlbase=array.urlbase) array.update_chunk(0, _chunk(0, value=4)) cache = str(tmp_path / "outlived.b2nd") @@ -604,14 +604,14 @@ def test_a_cache_over_a_handle_that_outlived_a_write_is_not_kept(subscriber, tmp np.testing.assert_array_equal(proxy[CHUNKS[0] : 2 * CHUNKS[0]], np.full(CHUNKS[0], 1, dtype=np.int32)) -def test_written_chunks_does_not_answer_out_of_a_proxy_cache(subscriber, tmp_path, blocks_are_worth_it): +def test_written_chunks_does_not_answer_out_of_a_proxy_cache(server, tmp_path, blocks_are_worth_it): """The one question whose whole point is what other writers have done. A `Proxy` hands its cached index to the array before there is a source to put it in, and the source takes it up as it is built. A fill read through that is the fill as of whenever the cache was written. """ - array, sub = subscriber + array, srv = server array.update_chunk(0, _chunk(0)) cache = str(tmp_path / "pending.b2nd") blosc2.Proxy(array, urlpath=cache, mode="w")[0:10] # leaves the offsets in the cache @@ -623,9 +623,9 @@ def test_written_chunks_does_not_answer_out_of_a_proxy_cache(subscriber, tmp_pat assert list(reader.written_chunks()) == [True, True, False, False, False, False] -def test_a_writer_that_lost_a_race_stops_believing_what_it_read(subscriber): +def test_a_writer_that_lost_a_race_stops_believing_what_it_read(server): """The refusal is the one answer that proves another writer moved the frame.""" - array, sub = subscriber + array, srv = server loser = blosc2.C2Array("run.b2nd", urlbase=array.urlbase) before = loser.stamp array.update_chunk(3, _chunk(3)) @@ -635,13 +635,13 @@ def test_a_writer_that_lost_a_race_stops_believing_what_it_read(subscriber): assert loser.stamp == blosc2.C2Array("run.b2nd", urlbase=array.urlbase).stamp -def test_a_write_that_lands_while_the_handle_looks_is_not_forgotten(subscriber, monkeypatch): +def test_a_write_that_lands_while_the_handle_looks_is_not_forgotten(server, monkeypatch): """Reading `api/info` is a round trip, and a write of this handle's can land inside it. Such an answer describes the array as it was before that write: keeping it would leave the handle believing it is current with nothing left to say otherwise. """ - array, sub = subscriber + array, srv = server array.update_chunk(0, _chunk(0)) # the handle now has a look to catch up on real, raced = blosc2.c2array.info, [] From 95825325fc4a1c1feac68e030b4d38690db00694 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 22 Aug 2026 17:51:10 +0200 Subject: [PATCH 02/27] Weigh blocks against chunks by what a fetch saves, not by one chunk BLOCK_MIN_CBYTES prices one extra round trip, and charged it to a single chunk. That is right where every range is its own request, and wrong where a transport carries many: block mode is then two waves whatever a slice touches, so the cost is the fetch's and not the chunk's. `wants_blocks` takes the wave it belongs to and, where `max_ranges > 1`, compares what the whole fetch skips against the same budget. A slice touching one chunk asks exactly what it asked before. Measured against a Caterva2 server at 45 ms and 10 MB/s: a dataset of 193 KB chunks reads a slab of 81 of them in 6.4 MB against 13.3 MB whole, and one of 650 KB chunks a slab of 36 in 6.4 MB against 23.3 MB. The fsspec path keeps the per-chunk test, which is measured right for it: 193 KB chunks stay at 0.70x against S3 out to 121 of them. `C2Array.serves_blocks` no longer refuses a dataset for the average size of its chunks. It decided from one number, once, that no future slice would be worth taking apart -- and it was the reason the two datasets above were never asked. It now answers the only question it can answer before a slice exists: whether the server has a frame to read ranges of. It also reads `accept_ranges` where the server reports it, so a dataset mounted from a peer -- stored there, re-serialized here, and so refusing ranges -- costs no request to rule out. `Proxy.traffic` counts what crossed the wire, in bytes and requests, at the transport: the frame index and the block offsets are in it as well as the data. Bytes are the half of this trade nothing reported, and the half a shared uplink runs out of -- blocks and chunks take similar time on a fast link and differ by the compression ratio in traffic. Measured on cat2.cloud, a pencil through `examples/cube-1k-1k-1k.b2nd` now costs 1.92 MB against 10.26 MB, and 17 requests against 102. Co-Authored-By: Claude Opus 5 --- src/blosc2/__init__.py | 2 + src/blosc2/c2array.py | 80 +++++++++++------ src/blosc2/proxy.py | 41 ++++++++- src/blosc2/proxy_source.py | 125 +++++++++++++++++++++++++-- tests/ndarray/test_c2array_async.py | 1 + tests/ndarray/test_c2array_blocks.py | 112 ++++++++++++++++++++++-- tests/ndarray/test_c2array_writes.py | 14 +-- 7 files changed, 326 insertions(+), 49 deletions(-) diff --git a/src/blosc2/__init__.py b/src/blosc2/__init__.py index 182111bc6..307e3bde0 100644 --- a/src/blosc2/__init__.py +++ b/src/blosc2/__init__.py @@ -593,6 +593,7 @@ def _raise(exc): ProxyNDSource, ByteRangeNDSource, FsspecNDSource, + Traffic, ) from .indexing import Index @@ -889,6 +890,7 @@ def _raise(exc): "Operand", "ByteRangeNDSource", "FsspecNDSource", + "Traffic", "Proxy", "ProxyNDField", "ProxyNDSource", diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 3a5a5514d..050700dab 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -457,7 +457,9 @@ def __init__(self, array: C2Array, max_concurrency: int = REMOTE_MAX_CONCURRENCY self._auth_token = array.auth_token # Answers that did not carry their parts, in a row; see `read_ranges` self._misses = 0 - super().__init__(self._url, max_concurrency) + # The array's own tally, so that what it reads through `api/chunk` and + # what this reads through `api/fetch` add up to what the dataset cost + super().__init__(self._url, max_concurrency, traffic=array.traffic) # A `Proxy` mixes the two: the block grid and the fetched bitmap come from # the array's `api/info`, while the header sections and `bstarts` come from # this frame. They have to be the same dataset for that to mean anything, @@ -521,6 +523,7 @@ def _get(self, spans: list[tuple[int, int]]) -> list[bytes]: response.status_code, ) response.read() + self.traffic.charge(len(response.content)) parts = _byteranges(response) return [_span_of(parts, offset, size, self._url) for offset, size in spans] @@ -598,6 +601,9 @@ def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | N self._meta_lock = threading.Lock() # An index a `Proxy` handed over before the source existed; see _adopt_index self._pending_index = None + # What this handle has read off the server, whichever endpoint it used; + # the block source built later is handed this same tally + self.traffic = blosc2.proxy_source.Traffic() # Try to 'open' the remote path try: @@ -759,6 +765,7 @@ def get_chunk(self, nchunk: int) -> bytes: url = self._chunk_url() params = {"nchunk": nchunk} response = _xget(url, params=params, auth_token=self.auth_token) + self.traffic.charge(len(response.content)) return response.content async def aget_chunk(self, nchunk: int) -> bytes: @@ -789,6 +796,7 @@ async def aget_chunk(self, nchunk: int) -> bytes: self._aclient = _httpx().AsyncClient(timeout=TIMEOUT) response = await self._aclient.get(url, params=params, headers=headers) response.raise_for_status() + self.traffic.charge(len(response.content)) return response.content async def aclose(self) -> None: @@ -1070,31 +1078,47 @@ def serves_blocks(self) -> bool: What `api/info` already carries, and no request of its own: a dataset the server *computes* reports an expression where a stored one reports a - geometry, and a frame of small chunks would never have one taken apart -- - blosc2 declines to split a chunk below ``BLOCK_MIN_CBYTES``, so the block - path would end in whole chunks anyway, by the longer road. + geometry, and only a stored one has a frame to read ranges of. That is + the whole question here. Whether taking a particular chunk apart pays is + a different one, and it is asked per fetch by + :meth:`ByteRangeNDSource.wants_blocks`, which knows what the slice + touches; this cannot, since it is read before any slice exists. + + It used to answer no as well for a frame whose chunks averaged under + ``BLOCK_MIN_CBYTES``, which decided from one number, once, that no future + slice of that dataset would ever be worth taking apart. That forfeited + the bytes the block path exists to save: measured against a Caterva2 + server, a dataset of 193 KB chunks reads a slab of 81 of them in 6.4 MB + against 13.3 MB whole, and one of 650 KB chunks a slab of 36 in 6.4 MB + against 23.3 MB -- 2.1x and 3.6x the traffic, on every such read, for the + life of the dataset. Where the link is what is scarce, and a server's + uplink is shared by everyone reading through it, those are the bytes that + decide how many readers it can hold. The judgement was never wrong, only + made too early and too widely: a point read of a small chunk really does + cost more than it saves, and `wants_blocks` still refuses it. Read off `api/info` again where this handle has written since it last - looked, which is the one case where the answer moves under it: a pre-sized - array holds almost nothing until it is filled, and a writer that took the - open-time figure would go on calling its own filled array too small to - take apart. That is one request to a handle that has just written, and - none at all to a reader -- which is what the promise below needs. + looked, which is the one case where the answer moves under it: a dataset + may be laid out before it is stored. That is one request to a handle that + has just written, and none at all to a reader -- which is what the promise + below needs. False is the whole answer; True is only that it is worth one request to find out, which :meth:`block_source` spends. A :ref:`Proxy` reads this when it is built, to decide whether its cache records blocks or chunks, so it must cost nothing and must not depend on what has been fetched. + + A server that reports ``accept_ranges`` spares even that request where + the answer is no: a dataset this server mounts from a peer reports the + peer's geometry, being stored there, but is fetched from its owner and + re-serialized here, so a range read of it is refused. Nothing else in + what `api/info` says can tell the two apart. A server that reports + nothing is an older one, and then this asks as it always did. """ - if not self._reports_geometry: - return False - try: - nchunks = math.prod(math.ceil(s / c) for s, c in zip(self.shape, self.chunks, strict=True)) - return bool(nchunks) and self.cbytes / nchunks >= blosc2.proxy_source.BLOCK_MIN_CBYTES - except (KeyError, TypeError, ValueError, ZeroDivisionError): - # An `api/info` without the fields these read describes a dataset - # whole chunks work for, as they do for every dataset there is + self._refresh_meta() # `meta` is what carries it, so read it current + if self.meta.get("accept_ranges") == "none": return False + return self._reports_geometry @property def _reports_geometry(self) -> bool: @@ -1116,11 +1140,12 @@ def block_source(self) -> C2NDSource | None: request with the whole body, so retrying would pay a full download to rediscover the same answer. - A frame whose chunks are too small to be worth taking apart says no here - without building anything, and without remembering that it said so: the - judgement is about *blocks*, and the same frame's index is still worth - reading. Deciding it at the call rather than caching it is what keeps - the two questions from answering each other. + Every stored frame says yes: whether a given chunk of it is worth taking + apart is decided per fetch, by `wants_blocks`, and not here. So this and + :meth:`_index_source` now come to the same answer, and both remain because + they ask for different reasons -- one for the blocks of a chunk, one for + the offsets that say where the chunks are. Neither remembers a no, since + a dataset laid out empty becomes a stored one as it is filled. """ return self._source() if self.serves_blocks else None @@ -1128,9 +1153,10 @@ def _index_source(self) -> C2NDSource | None: """The same reader, built for any stored frame however small its chunks. Reading the frame's index is not the same question as reading blocks of - its chunks: a frame of chunks too small to take apart still has offsets, - and they still say which chunks hold anything. Whatever is built here is - the source the block path uses too -- there is only ever one. + its chunks, though a stored frame now answers yes to both: the offsets + say which chunks hold anything, which is worth knowing whatever is done + with them. Whatever is built here is the source the block path uses too + -- there is only ever one. """ return self._source() if self._reports_geometry else None @@ -1207,10 +1233,10 @@ def _index_state(self, keep=()) -> dict | None: return self._pending_index return source._index_state(keep) - def wants_blocks(self, nchunk: int, nwanted: int) -> bool: + def wants_blocks(self, nchunk: int, nwanted: int, wave=None) -> bool: """Whether fetching *nwanted* blocks of a chunk beats fetching all of it.""" source = self.block_source() - return source is not None and source.wants_blocks(nchunk, nwanted) + return source is not None and source.wants_blocks(nchunk, nwanted, wave) @property def max_ranges(self) -> int: diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 249645edd..bce2923a4 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -228,6 +228,35 @@ def __init__( for key in vlmeta or (): self._schunk_cache.vlmeta[key] = vlmeta[key] + @property + def traffic(self) -> "blosc2.proxy_source.Traffic | None": + """What this proxy has read off its source, or None for a local one. + + Cumulative bytes and requests since the source was opened, counted at the + transport, so the frame index and the block offsets are in it as well as + the data, and the metadata call that opened the handle is not. What a + slice cost in traffic is the difference between two readings of this, or + one reading after :meth:`Traffic.reset`. + + It is what says whether block granularity is doing anything for a given + dataset and access pattern: whole chunks and blocks of them take similar + time on a fast link and differ by the compression ratio in bytes, and + bytes are what a shared uplink runs out of. + + None where nothing crosses a wire -- a proxy over a local array -- since + a counter that only ever reads zero would say the traffic was free rather + than that it was never measured. + + Examples + -------- + >>> proxy = blosc2.open(url, lazy=True) # doctest: +SKIP + >>> proxy.traffic.reset() # doctest: +SKIP + >>> _ = proxy[0, 0, 0] # doctest: +SKIP + >>> proxy.traffic # doctest: +SKIP + Traffic(requests=2, nbytes=20480) + """ + return getattr(self.src, "traffic", None) + def __enter__(self) -> "Proxy": """Enter a context manager and return this proxy.""" return self @@ -626,7 +655,17 @@ def _fetch_by_block(self, item, max_concurrency: int | None): missing = self._missing_blocks(item) if not missing: return self._cache - wanted = {n: bs for n, bs in missing.items() if self.src.wants_blocks(n, len(bs))} + # A transport that batches ranges pays the block path's fixed cost once + # for the whole fetch, so what it wants asked is the wave rather than the + # chunk; see `ByteRangeNDSource._wave_saves`. It is also the only kind of + # source this module hands the wave to, so one written to the two-argument + # protocol is never called with three. + if getattr(self.src, "max_ranges", 1) > 1: + wave = {n: len(bs) for n, bs in missing.items()} + asks = lambda n, nwanted: self.src.wants_blocks(n, nwanted, wave) # noqa: E731 + else: + asks = self.src.wants_blocks + wanted = {n: bs for n, bs in missing.items() if asks(n, len(bs))} whole = [n for n in missing if n not in wanted] layouts = dict(zip(wanted, self._chunk_layouts(list(wanted), max_concurrency), strict=True)) diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index 5464245fe..e78ee2e4b 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -24,7 +24,7 @@ import struct import threading from abc import ABC, abstractmethod -from collections.abc import Sequence +from collections.abc import Mapping, Sequence try: from itertools import batched @@ -91,6 +91,51 @@ def batched(iterable, n): BLOCK_HOT_CHUNKS = 8 +class Traffic: + """What crossed the wire, counted where it crossed. + + Bytes, not wall time, are what a shared uplink runs out of, and they are the + half of the block-granularity trade that nothing else reports: a slice that + reads one block of a chunk and one that reads the whole chunk take about the + same time on a fast link and differ by the compression ratio in traffic. + Whoever pays for the link is the one who needs to see that, so it is counted + rather than inferred -- and counted at the transport, so the frame index and + the block offsets, which no caller ever asks for by name, are in it too. + + What crosses the wire to *carry data*, which is every range read and every + chunk: the one metadata call that opens a handle (`api/info`, a few hundred + bytes, once) is not in it, being neither what a slice costs nor anything the + block path can change. + + Cumulative from the moment a source is built. Take two readings and subtract, + or :meth:`reset` between them. + """ + + __slots__ = ("_lock", "nbytes", "requests") + + def __init__(self): + self.requests = 0 + self.nbytes = 0 + # Requests overlap in a thread pool, so the two counters are bumped + # together or the totals drift apart under any real fetch + self._lock = threading.Lock() + + def charge(self, nbytes: int) -> None: + """Record one request that carried *nbytes*.""" + with self._lock: + self.requests += 1 + self.nbytes += nbytes + + def reset(self) -> None: + """Start counting again from zero.""" + with self._lock: + self.requests = 0 + self.nbytes = 0 + + def __repr__(self) -> str: + return f"Traffic(requests={self.requests}, nbytes={self.nbytes})" + + def _is_transient(status: int | None) -> bool: """Whether a status says the server was busy, rather than answering. @@ -160,8 +205,10 @@ class ProxyNDSource(ABC): A source whose transport can ask for several ranges at once says so with ``max_ranges`` and serves ``read_ranges(spans)`` and ``chunk_layouts(nchunks)`` as well; :ref:`Proxy` then sends a whole wave of - reads as one request. Both are optional, and a source without them is asked - one range at a time exactly as before. + reads as one request, and asks ``wants_blocks(nchunk, nwanted, wave)`` with + the fetch that chunk belongs to, since a shared round trip is the wave's to + weigh and not the chunk's. All are optional, and a source without them is + asked one range at a time, and two arguments at a time, exactly as before. A block read that the transport cannot answer raises ``NotRanged``, and :ref:`Proxy` then fetches the chunks it was after whole. @@ -596,9 +643,18 @@ class ByteRangeNDSource(ProxyNDSource): than a couple per chunk it touches. """ - def __init__(self, urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY): + def __init__( + self, + urlpath: str, + max_concurrency: int = REMOTE_MAX_CONCURRENCY, + traffic: Traffic | None = None, + ): self.max_concurrency = max_concurrency self.urlpath = urlpath + # Taken rather than made where a caller already has one, so that what an + # open costs -- the frame header, read a few lines down -- is counted with + # everything the source goes on to read, and not into a tally thrown away + self.traffic = traffic if traffic is not None else Traffic() # Exact ranges, not a file handle: a buffered one reads a whole block per # seek (50 MiB on s3fs by default), which would undo the point of a lazy # open. Chunk reads are stateless, so the index below is the only state a @@ -613,6 +669,9 @@ def __init__(self, urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY): # a b2nd metalayer -- is in the header that was just read. self._index = None self._index_lock = threading.Lock() + # The last wave `_wave_saves` was asked about, and what it came to: one + # fetch asks once per chunk for an answer that is the same every time + self._wave_saved = None # Set when the frame is written to under this handle: the header moves as # well as the offsets, so both are read again before the next lookup self._stale = False @@ -869,20 +928,70 @@ def read_ranges(self, spans: Sequence[tuple[int, int]]) -> list[bytes]: """ return [self.read_range(offset, size) for offset, size in spans] - def wants_blocks(self, nchunk: int, nwanted: int) -> bool: + def wants_blocks(self, nchunk: int, nwanted: int, wave: Mapping[int, int] | None = None) -> bool: """Whether fetching *nwanted* blocks of a chunk beats fetching all of it. Answered without reading anything, so a chunk that says no costs exactly what it costs today: the number of blocks a slice touches is geometry, and an upper bound on the chunk's compressed size is already in hand from the frame's offsets. See the thresholds at the top of this module. + + *wave* is the whole fetch this chunk belongs to, ``{nchunk: nwanted}``, + which a transport that batches ranges is asked with; see + :meth:`_wave_saves` for what it is used for and why. """ offsets, extents = self._frame_index() # once, rather than twice under the lock if int(offsets[nchunk]) < 0: return False # a run-length chunk has no bytes in the file to skip if nwanted > self.blocks_per_chunk * BLOCK_MAX_FRACTION: return False - return int(extents[nchunk]) >= BLOCK_MIN_CBYTES + if wave is None or self.max_ranges <= 1: + return int(extents[nchunk]) >= BLOCK_MIN_CBYTES + return self._wave_saves(wave) >= BLOCK_MIN_CBYTES + + def _wave_saves(self, wave: Mapping[int, int]) -> int: + """Bytes a whole fetch skips by taking its chunks apart, blocks against chunks. + + What the budget is charged to is what the extra round trip is charged to, + and where a transport carries many ranges per request that is the wave, + not the chunk. Block mode is two waves whatever a slice touches -- the + block offsets, then the blocks -- so its fixed cost is paid once per + fetch, while chunk mode pays for every chunk's bytes. Charging one + chunk for a round trip the whole fetch shares is what kept a + small-chunked dataset on the whole-chunk path however wide the slice. + + Measured against a Caterva2 server at 45 ms and 10 MB/s (which is + cat2.cloud from Europe), block mode against chunk mode: a dataset of + 193 KB chunks runs 0.4x on a point read and 1.6x on a slab touching 81 + of them, and one of 650 KB chunks 0.7x and 2.7x. Both were refused + outright before this, the second forfeiting 2.7x. Summing what the + fetch skips classifies all of it -- every measured loss below the + budget, every win above it -- and collapses to the old test at a slice + touching one chunk, since a wave of one is a chunk. + + None of this holds where every range is its own request: block mode is + then two requests per chunk against one, both sides scale with the + chunks touched, and a dataset of 193 KB chunks measured 0.70x against S3 + out to 121 of them. Hence the ``max_ranges`` gate above, which leaves + that path deciding exactly as it did. + + Blocks of a chunk are close enough in size to weigh what is wanted by + counting them, the same approximation :data:`BLOCK_MAX_FRACTION` makes, + so this needs no more read than the offsets already in hand. + """ + if self._wave_saved is not None and self._wave_saved[0] is wave: + return self._wave_saved[1] # one fetch asks once per chunk; count once + offsets, extents = self._frame_index() + nblocks = self.blocks_per_chunk + saved = 0 + for nchunk, nwanted in wave.items(): + # A chunk this would not take apart anyway saves nothing: it is + # fetched whole in either mode, so its bytes are not the wave's to spend + if int(offsets[nchunk]) < 0 or nwanted > nblocks * BLOCK_MAX_FRACTION: + continue + saved += int(extents[nchunk]) * (nblocks - nwanted) // nblocks + self._wave_saved = (wave, saved) + return saved def chunk_layout(self, nchunk: int) -> tuple[bytes, np.ndarray, np.ndarray] | None: """Read where the blocks of a chunk are: its header, bstarts and extents. @@ -1053,7 +1162,9 @@ def __init__(self, urlpath: str, max_concurrency: int = REMOTE_MAX_CONCURRENCY): super().__init__(urlpath, max_concurrency) def read_range(self, offset: int, size: int) -> bytes: - return self._fs.cat_file(self._path, start=offset, end=offset + size) + data = self._fs.cat_file(self._path, start=offset, end=offset + size) + self.traffic.charge(len(data)) + return data def convert_dtype(dt: str | DTypeLike): diff --git a/tests/ndarray/test_c2array_async.py b/tests/ndarray/test_c2array_async.py index 4556eaa5f..493325f01 100644 --- a/tests/ndarray/test_c2array_async.py +++ b/tests/ndarray/test_c2array_async.py @@ -18,6 +18,7 @@ class _FakeResponse: def __init__(self, json_data): self._json = json_data + self.headers = {} # a real response always has them; `info` reads Accept-Ranges def raise_for_status(self): pass diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index dc3c6071b..41a37e264 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -48,7 +48,11 @@ def __init__( fetch_failures=0, bad_parts=0, geometry=True, + accept_ranges=None, ): + # What `api/info` reports for byte ranges: "bytes", "none", or None for a + # server old enough to report nothing, which is what the client must survive + self.accept_ranges = accept_ranges self.fetch_failures = fetch_failures # answer this many fetches 503 first self.path = str(path) self.key = key # a leaf inside a .b2z container, rather than a file of its own @@ -105,6 +109,7 @@ def meta(self): "blocksize": schunk.blocksize, "vlmeta": {}, }, + **({} if self.accept_ranges is None else {"accept_ranges": self.accept_ranges}), } @@ -340,24 +345,117 @@ def test_a_computed_dataset_is_ruled_out_without_a_request(server, any_chunk_wan def test_small_chunks_are_fetched_whole(server): - # Below the threshold a chunk is one cheap request, so blocks would only add - # a round trip: nothing goes looking for the frame index, let alone a block + # A point read of a chunk this small saves fewer bytes than the round trip + # that would find them costs, so `wants_blocks` refuses it and the chunk + # comes whole -- the judgement the dataset no longer makes for every slice + # at once, only for the slice in hand data = _incompressible((200, 200)) array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") srv.log.clear() assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) + assert array.block_source() is not None # a stored frame is readable in ranges + assert not array.wants_blocks(0, 1) # ... and this one is not worth splitting + assert [kind for kind, _, _ in srv.log][-1] == "chunk" + + +def test_a_peer_dataset_is_ruled_out_by_what_api_info_says(server, any_chunk_wants_blocks): + """`Accept-Ranges: none` spares the request that would have found it out. + + A dataset the server mounts from a peer reports the peer's geometry, being + stored *there*, but is re-serialized here and so refuses a range. Nothing in + the payload tells it from a local one; the header does. + """ + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), accept_ranges="none") + srv.log.clear() + + assert not array.serves_blocks assert array.block_source() is None - assert [kind for kind, _, _ in srv.log] == ["chunk"] + assert not srv.log # ... and no request was spent learning it + +def test_a_server_that_names_nothing_is_asked_as_before(server, any_chunk_wants_blocks): + # An older server names no Accept-Ranges at all, and then the probe is the + # only way to know -- which is what was always done, and must keep working + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), accept_ranges=None) + srv.log.clear() + + assert array.serves_blocks + assert array.block_source() is not None + assert [kind for kind, _, _ in srv.log] == ["fetch"] # the probe, and only it + + +def test_a_server_that_says_bytes_is_believed(server, any_chunk_wants_blocks): + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), accept_ranges="bytes") + assert array.serves_blocks + assert array.block_source() is not None -def test_a_dataset_that_serves_no_blocks_keeps_the_chunkwise_bitmap(tmp_path, server): - # Nothing will ever ask this one for a block, so its cache records chunks: - # the bitmap an older blosc2 also reads, and none of the per-block - # bookkeeping that would be kept only to say `all of them` every time + +def test_traffic_counts_what_crossed_the_wire(server, any_chunk_wants_blocks): + """Bytes and requests, counted at the transport and not inferred. + + The point of counting them is that blocks and chunks cost about the same + wall time on a fast link and differ by the compression ratio in traffic, so + traffic is the only thing that shows the block path working at all. + """ + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + p = blosc2.Proxy(array, mode="w") + assert p.traffic is array.traffic # the array's tally, not a second one + + p.traffic.reset() + assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) + blocks = (p.traffic.requests, p.traffic.nbytes) + assert blocks[0] > 0 + assert blocks[1] > 0 + # What the server logged for the data endpoints is what was counted; the + # `api/info` that opened the handle is metadata and is deliberately not + served = [(kind, nbytes) for kind, _, nbytes in srv.log if kind != "info"] + assert blocks[0] == len(served) + assert blocks[1] <= sum(nbytes for _, nbytes in served) + + # The same slice again is served from the cache and costs nothing more + before = (p.traffic.requests, p.traffic.nbytes) + assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) + assert (p.traffic.requests, p.traffic.nbytes) == before + + +def test_traffic_shows_blocks_costing_fewer_bytes_than_chunks(tmp_path, server, monkeypatch): + """The comparison the counter exists to make, on one array and one slice.""" + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + + monkeypatch.setattr(blosc2.proxy_source, "BLOCK_MIN_CBYTES", 1 << 62) # chunks + whole = blosc2.Proxy(array, urlpath=str(tmp_path / "whole.b2nd"), mode="w") + whole.traffic.reset() + assert np.array_equal(whole[0:5, 0:10], data[0:5, 0:10]) + by_chunk = whole.traffic.nbytes + + monkeypatch.setattr(blosc2.proxy_source, "BLOCK_MIN_CBYTES", 0) # blocks + split = blosc2.Proxy( + blosc2.C2Array(array.path, urlbase=array.urlbase), + urlpath=str(tmp_path / "split.b2nd"), + mode="w", + ) + split.traffic.reset() + assert np.array_equal(split[0:5, 0:10], data[0:5, 0:10]) + assert split.traffic.nbytes < by_chunk # which is the whole point + + +def test_a_dataset_that_serves_no_blocks_keeps_the_chunkwise_bitmap(tmp_path, server, monkeypatch): + # A source that says it serves no blocks -- which a dataset the server + # computes says, having no frame to read ranges of -- gets a cache that + # records chunks: the bitmap an older blosc2 also reads, and none of the + # per-block bookkeeping that would be kept only to say `all of them` every + # time. Said here rather than served that way, since a computed dataset + # reports no partitioning at all and a proxy cannot be laid out over one. data = _incompressible((200, 200)) array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + monkeypatch.setattr(type(array), "serves_blocks", property(lambda self: False)) assert not array.serves_blocks # decided from api/info, without a request cache = str(tmp_path / "chunkwise-cache.b2nd") p = blosc2.Proxy(array, urlpath=cache, mode="w") diff --git a/tests/ndarray/test_c2array_writes.py b/tests/ndarray/test_c2array_writes.py index aa17f574d..01218c42f 100644 --- a/tests/ndarray/test_c2array_writes.py +++ b/tests/ndarray/test_c2array_writes.py @@ -494,15 +494,15 @@ def test_a_handle_that_writes_stamps_what_it_wrote(server): def test_asking_about_blocks_does_not_close_the_door_on_the_index(server): """Two questions, one source, and the answer to one must not answer the other. - `serves_blocks` weighs whether splitting a chunk into blocks would pay, which - a frame of small chunks fails; reading the frame's index is worth doing - anyway. Deciding that at the call rather than remembering it is what keeps - the block path from shutting the index path down. + `serves_blocks` asks only whether the server has a frame to read ranges of, + which a pre-sized array does before anything is written to it. Whether a + given chunk is worth splitting is `wants_blocks`, asked per fetch; neither + answer may stand in for the other, and neither may shut the index path down. """ array, srv = server - assert not array.serves_blocks # chunks here are far under BLOCK_MIN_CBYTES - assert array.max_ranges == 1 # the block path, asked first, and declining - assert array.block_source() is None + assert array.serves_blocks # a stored frame, however little it holds + assert array.block_source() is not None # the block path, asked first + assert not array.wants_blocks(0, 1) # ... and declining, for this chunk assert list(array.written_chunks()) == [False] * NCHUNKS # still answerable From df4d2125cdb927202211da1a381ba92c78c495c8 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 22 Aug 2026 18:28:18 +0200 Subject: [PATCH 03/27] Place a fancy index on the block grid instead of giving up on it A `Proxy` reduced a key to a box and intersected that with the block grid. An integer array is no box, so it fell through to every block of every chunk it touched -- and for scattered points that is the whole of each of them, which is the granularity blocks exist to avoid. It did not even get that far: `get_slice_nchunks` reads a key as slices, so a fancy one raised `AttributeError: 'numpy.ndarray' object has no attribute 'start'` before anything was fetched. Each selected coordinate lives in exactly one block, so a fancy key can be placed exactly. `process_key` has already broadcast the advanced indices against each other -- numpy's own rule for how they pair up -- so reading them elementwise is reading the coordinates selected; dimensions indexed plainly are crossed with those. N points then cost N blocks. Measured on a 900^3 array of 100^3 chunks, nine points in nine chunks: 236 KB in 19 requests, against 1.81 MB in 10 for the chunks holding them. The answer itself is not this code's to compute: `__getitem__` fetches and then hands the key to the cache, which is an `NDArray` and indexes as one. So the mapping is only ever allowed to be too generous, and a key it cannot place -- a boolean mask, anything `process_key` will not expand -- asks for whole chunks, which is a superset of any of them. A block that should have been fetched and was not would read as zeros, and nothing downstream could tell those from data. Integer arrays only, as numpy's own separated-array keys are not supported by the layer below either: a `Proxy` now raises what an `NDArray` raises for those, rather than something else. `slice_to_string` refuses an index it cannot express, instead of dropping it: a fancy key skipped that way asked `api/fetch` for the whole dataset and handed back all of it, which is neither what was asked for nor smaller. Co-Authored-By: Claude Opus 5 --- src/blosc2/c2array.py | 9 ++ src/blosc2/proxy.py | 146 ++++++++++++++++++++++++++- tests/ndarray/test_c2array_blocks.py | 52 ++++++++++ 3 files changed, 204 insertions(+), 3 deletions(-) diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 050700dab..588d95e50 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -304,6 +304,15 @@ def slice_to_string(slice_): raise IndexError("Only step=1 is supported") # step = index.step or '' slice_parts.append(f"{start}:{stop}") + else: + # Anything else has no spelling here, and dropping it would widen the + # request rather than narrow it: a fancy index skipped this way asks + # `api/fetch` for the whole dataset and hands back all of it, which + # is neither what was asked for nor a smaller answer + raise IndexError( + f"Cannot ask a Caterva2 server for {index!r}: only integers and " + "step-1 slices can be expressed in a fetch request" + ) return ", ".join(slice_parts) diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index bce2923a4..b0a691b61 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -403,8 +403,22 @@ def _wanted_chunks(self, item) -> list[int]: self._sync_evictions() if item == (): # full realization return list(range(self._schunk_cache.nchunks)) + cells = self._cells(item) + if cells is _UNMAPPABLE: + # Not a key that can be placed on the grid, and `get_slice_nchunks` + # reads it as slices and trips over it: every chunk, which is what + # the proxy fetched for anything it could not narrow down anyway + return list(range(self._schunk_cache.nchunks)) + if cells is not None: + return sorted(cells) return [int(n) for n in blosc2.get_slice_nchunks(self._cache, item)] + def _cells(self, item): + """:func:`_fancy_cells` for this proxy's grid; None where *item* is a box.""" + if not isinstance(self._cache, blosc2.NDArray): + return None + return _fancy_cells(item, self._cache.shape, self._cache.chunks, self._cache.blocks) + def _missing_chunks(self, item) -> list[int]: """The chunks *item* touches that the cache does not hold in full.""" bpc = self._blocks_per_chunk @@ -422,12 +436,18 @@ def _missing_blocks(self, item) -> dict[int, list[int]]: def _wanted_blocks(self, item) -> dict[int, Sequence[int]]: """{chunk: blocks} that *item* touches, by intersecting it with the block grid. - Anything this cannot reduce to a box -- fancy indexing, a step -- asks for - every block of the chunks it touches, which is the granularity the proxy - had before blocks and always a superset of the right answer. + An integer-array key is placed on the grid exactly, by + :func:`_fancy_cells`: each selected coordinate lives in one block, so + scattered points cost blocks and not the chunks holding them. Anything + left that this cannot reduce to a box -- a boolean mask, a step -- asks + for every block of the chunks it touches, which is the granularity the + proxy had before blocks and always a superset of the right answer. """ chunks, blocks = self._cache.chunks, self._cache.blocks every = range(self._blocks_per_chunk) + cells = self._cells(item) + if cells is not None and cells is not _UNMAPPABLE: + return {n: sorted(b) for n, b in cells.items()} spans = _item_spans(item, self._cache.shape) if spans is None: return dict.fromkeys(self._wanted_chunks(item), every) @@ -1009,6 +1029,126 @@ def fields(self) -> dict: return {key: ProxyNDField(self, key) for key in _fields} +_UNMAPPABLE = object() +"""A key that selects something, but nothing this can reduce to cells of the grid.""" + + +def _dim_cells(dim, lo, hi, chunks, blocks) -> set[tuple[int, int]]: + """The (chunk, block) pairs along *dim* that coordinates lo..hi inclusive fall in. + + Blocks partition a chunk and restart at every chunk boundary -- a chunk need + not be a whole number of blocks -- so a block is located by where it sits + inside its chunk, never by a running count across the array. + """ + cells = set() + for c in range(lo // chunks[dim], hi // chunks[dim] + 1): + first = max(lo - c * chunks[dim], 0) // blocks[dim] + last = min(hi - c * chunks[dim], chunks[dim] - 1) // blocks[dim] + cells |= {(c, b) for b in range(first, last + 1)} + return cells + + +def _sort_dims(key): + """The dimensions of *key* indexed by an array, and those indexed plainly. + + `_UNMAPPABLE` for anything else, which is what keeps this to integer arrays: + a boolean mask selects by a rule rather than by coordinates, and placing one + is a different job from placing these. + """ + advanced, basic = [], [] + for dim, k in enumerate(key): + if isinstance(k, np.ndarray): + if not np.issubdtype(k.dtype, np.integer): + return _UNMAPPABLE # a boolean mask is not one of these + advanced.append(dim) + elif isinstance(k, (slice, int, np.integer)): + basic.append(dim) + else: + return _UNMAPPABLE + return advanced, basic + + +def _cross_cells(paired, crossed, advanced, basic, shape, chunks, blocks): + """Every (chunk, block) the paired and crossed dimensions come to, as {chunk: {block}}.""" + return _cross_cells(paired, crossed, advanced, basic, shape, chunks, blocks) + + +def _fancy_cells(item, shape, chunks, blocks): + """{chunk: {blocks}} a fancy key touches, exactly, or None where it is a plain box. + + A slice reduces to a box and the caller intersects that with the block grid; + an integer array does not, and used to fall back to every block of every + chunk it touched -- which for scattered points is the whole of each of them, + the granularity blocks exist to avoid. Here each selected coordinate is + located in its own block, so N points cost N blocks and not N chunks. + + `process_key` has already broadcast the advanced indices against each other, + which is numpy's own rule for how they pair up, so the arrays arrive with one + shape and reading them elementwise is reading the coordinates selected. + Dimensions indexed by a slice are crossed with those, since every selected + coordinate is taken at every position of the slice. + + Integer arrays only. `_UNMAPPABLE` for a key that selects something this + cannot place -- a boolean mask, anything `process_key` will not expand -- and + the caller then asks for whole chunks, which is a superset and so always + safe. Never a smaller answer than the truth: a block that should have been + fetched and was not reads as zeros, which nothing downstream could tell from + data. + """ + from blosc2.utils import process_key + + try: + key, _ = process_key(item, shape) + except Exception: + return _UNMAPPABLE + sorted_dims = _sort_dims(key) + if sorted_dims is _UNMAPPABLE: + return _UNMAPPABLE + advanced, basic = sorted_dims + if not advanced: + return None # a box, which the caller has a cheaper way to intersect + + def cell(dim, coord): + chunk, offset = divmod(int(coord), chunks[dim]) + return chunk, offset // blocks[dim] + + # The advanced dimensions are read together: one coordinate each, per element + flat = [key[d].reshape(-1) for d in advanced] + if flat[0].size == 0: + return {} + paired = { + tuple(cell(d, v) for d, v in zip(advanced, vals, strict=True)) for vals in zip(*flat, strict=True) + } + + crossed = [] + for dim in basic: + k = key[dim] + if isinstance(k, (int, np.integer)): + crossed.append(_dim_cells(dim, int(k), int(k), chunks, blocks)) + continue + start, stop, step = k.indices(shape[dim]) + if stop <= start: + return {} + # A step is not followed: the span it lies in is a superset of it, and a + # superset is the one kind of wrong answer this may give + crossed.append(_dim_cells(dim, start, stop - 1, chunks, blocks)) + + chunk_grid = [math.ceil(s / c) for s, c in zip(shape, chunks, strict=True)] + blocks_in_chunk = [math.ceil(c / b) for c, b in zip(chunks, blocks, strict=True)] + out = {} + for pair in paired: + for combo in itertools.product(*crossed): + coords = [None] * len(shape) + for i, dim in enumerate(advanced): + coords[dim] = pair[i] + for i, dim in enumerate(basic): + coords[dim] = combo[i] + nchunk = int(np.ravel_multi_index([c for c, _ in coords], chunk_grid)) + nblock = int(np.ravel_multi_index([b for _, b in coords], blocks_in_chunk)) + out.setdefault(nchunk, set()).add(nblock) + return out + + def _item_spans(item, shape) -> list[tuple[int, int]] | None: """The (start, stop) of *item* along every dimension, or None if it is no box. diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 41a37e264..b17f93cf6 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -360,6 +360,58 @@ def test_small_chunks_are_fetched_whole(server): assert [kind for kind, _, _ in srv.log][-1] == "chunk" +def test_scattered_points_cost_blocks_and_not_chunks(tmp_path, server, any_chunk_wants_blocks): + """An integer-array key is placed on the block grid, one block per point. + + Falling back to whole chunks for it -- which is what anything that cannot be + reduced to a box got -- fetches the chunks the points live in, and a chunk is + what blocks exist to avoid fetching. + """ + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + p = blosc2.Proxy(array, urlpath=str(tmp_path / "points.b2nd"), mode="w") + p.traffic.reset() + + pts = [5, 105] # one in each chunk, and one block of ten in each + np.testing.assert_array_equal(p[pts, 7], data[pts, 7]) + blocks = p.traffic.nbytes + + whole = blosc2.Proxy( + blosc2.C2Array(array.path, urlbase=array.urlbase), + urlpath=str(tmp_path / "whole.b2nd"), + mode="w", + ) + whole.traffic.reset() + whole.fetch((slice(0, 200), slice(0, 200))) + assert blocks < whole.traffic.nbytes # the point of the exercise + + +def test_a_fancy_key_reads_what_numpy_reads(server, any_chunk_wants_blocks): + """Whatever is fetched, the answer is the array's own -- the cache decides it. + + Which is why the mapping may only ever be too generous: a block that should + have been fetched and was not reads as zeros, and nothing downstream could + tell those from data. + """ + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + p = blosc2.Proxy(array, mode="w") + for key in ([1, 5, 59], [0, -1], ([1, 5], [2, 7]), (np.array([[1, 2], [3, 4]]),), [5]): + np.testing.assert_array_equal(p[key], data[key]) + + +def test_a_key_that_cannot_be_placed_asks_for_everything(server, any_chunk_wants_blocks): + # A boolean mask selects by a rule rather than by coordinates, so it is not + # placed on the grid. The answer stays right -- whole chunks are a superset + # of any of them -- which is the property the fallback exists to keep + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + p = blosc2.Proxy(array, mode="w") + mask = np.zeros(60, dtype=bool) + mask[[3, 40]] = True + np.testing.assert_array_equal(p[mask], data[mask]) + + def test_a_peer_dataset_is_ruled_out_by_what_api_info_says(server, any_chunk_wants_blocks): """`Accept-Ranges: none` spares the request that would have found it out. From bd38b426a2c1193fb6462459051f73561a5e9660 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 22 Aug 2026 18:35:20 +0200 Subject: [PATCH 04/27] Say that a boolean mask is placed on the grid, because it already was `process_key` turns a mask into the coordinates it selects -- an integer array per dimension it spanned, paired the way a mask's own dimensions pair -- so by the time a key reaches `_fancy_cells` there is no mask left to recognize. Masks were therefore placed as exactly as lists from the start, and the comments saying they were not were describing an intention rather than the code. A mask picking two rows of a 60x70 array fetches 2 blocks, not the 6 chunks holding them. The test that claimed otherwise asserted only that the values came out right, which they do either way, so nothing caught it. It is replaced by one that measures what a mask costs, and by a sweep over the spellings a mask has -- one dimension or several, alone or beside an int or a slice -- since each pairs its coordinates differently and none may lose a block. `_fancy_cells` no longer swallows what `process_key` raises. A key it refuses is one the cache cannot index either, so the answer was never going to be returned; raising it where it happens beats reporting the key unplaceable, fetching whole chunks to build an answer from, and raising the same error at the end. Nothing fetched it in practice -- `_item_spans` raised first, on the way past -- but only by the order the two are called in, which is not something to leave holding the property up. Co-Authored-By: Claude Opus 5 --- src/blosc2/proxy.py | 43 ++++++++++++---------- tests/ndarray/test_c2array_blocks.py | 53 +++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 24 deletions(-) diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index b0a691b61..05ac7c874 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -436,12 +436,13 @@ def _missing_blocks(self, item) -> dict[int, list[int]]: def _wanted_blocks(self, item) -> dict[int, Sequence[int]]: """{chunk: blocks} that *item* touches, by intersecting it with the block grid. - An integer-array key is placed on the grid exactly, by - :func:`_fancy_cells`: each selected coordinate lives in one block, so + A key of integer arrays or boolean masks is placed on the grid exactly, + by :func:`_fancy_cells`: each selected coordinate lives in one block, so scattered points cost blocks and not the chunks holding them. Anything - left that this cannot reduce to a box -- a boolean mask, a step -- asks - for every block of the chunks it touches, which is the granularity the - proxy had before blocks and always a superset of the right answer. + left that this cannot reduce to a box -- a step, a key nobody has thought + about -- asks for every block of the chunks it touches, which is the + granularity the proxy had before blocks and always a superset of the + right answer. """ chunks, blocks = self._cache.chunks, self._cache.blocks every = range(self._blocks_per_chunk) @@ -1051,15 +1052,16 @@ def _dim_cells(dim, lo, hi, chunks, blocks) -> set[tuple[int, int]]: def _sort_dims(key): """The dimensions of *key* indexed by an array, and those indexed plainly. - `_UNMAPPABLE` for anything else, which is what keeps this to integer arrays: - a boolean mask selects by a rule rather than by coordinates, and placing one - is a different job from placing these. + `_UNMAPPABLE` for anything else. Everything `process_key` hands back is one + of these today -- a mask has already become an integer array by the time it + arrives -- so this is what keeps a key nobody has thought about from being + read as one that was. """ advanced, basic = [], [] for dim, k in enumerate(key): if isinstance(k, np.ndarray): if not np.issubdtype(k.dtype, np.integer): - return _UNMAPPABLE # a boolean mask is not one of these + return _UNMAPPABLE # coordinates, or this cannot place it advanced.append(dim) elif isinstance(k, (slice, int, np.integer)): basic.append(dim) @@ -1088,19 +1090,22 @@ def _fancy_cells(item, shape, chunks, blocks): Dimensions indexed by a slice are crossed with those, since every selected coordinate is taken at every position of the slice. - Integer arrays only. `_UNMAPPABLE` for a key that selects something this - cannot place -- a boolean mask, anything `process_key` will not expand -- and - the caller then asks for whole chunks, which is a superset and so always - safe. Never a smaller answer than the truth: a block that should have been - fetched and was not reads as zeros, which nothing downstream could tell from - data. + A boolean mask arrives here already an integer array, and one per dimension + it spanned: `process_key` turns it into the coordinates it selects, paired + the way a mask's own dimensions pair. So masks are placed as exactly as + lists are, and nothing here has to know which it was given. + + `_UNMAPPABLE` for a key that selects something this cannot place, and the + caller then asks for whole chunks, which is a superset and so always safe. + Never a smaller answer than the truth: a block that should have been fetched + and was not reads as zeros, which nothing downstream could tell from data. + A key `process_key` refuses is not one of those -- the cache cannot index it + either, so it raises here rather than fetching an array's worth of blocks for + an answer that is never going to be returned. """ from blosc2.utils import process_key - try: - key, _ = process_key(item, shape) - except Exception: - return _UNMAPPABLE + key, _ = process_key(item, shape) sorted_dims = _sort_dims(key) if sorted_dims is _UNMAPPABLE: return _UNMAPPABLE diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index b17f93cf6..daac3a316 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -400,16 +400,59 @@ def test_a_fancy_key_reads_what_numpy_reads(server, any_chunk_wants_blocks): np.testing.assert_array_equal(p[key], data[key]) -def test_a_key_that_cannot_be_placed_asks_for_everything(server, any_chunk_wants_blocks): - # A boolean mask selects by a rule rather than by coordinates, so it is not - # placed on the grid. The answer stays right -- whole chunks are a superset - # of any of them -- which is the property the fallback exists to keep +def test_a_boolean_mask_is_placed_as_exactly_as_a_list(tmp_path, server, any_chunk_wants_blocks): + """A mask reaches the grid as the coordinates it selects, not as a rule. + + `process_key` turns one into an integer array per dimension it spanned, so + nothing in the mapping has to know which it was given -- and a mask picking + two rows of an array costs those rows' blocks, not every chunk they lie in. + """ data = _incompressible((60, 70)) array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) - p = blosc2.Proxy(array, mode="w") + p = blosc2.Proxy(array, urlpath=str(tmp_path / "mask.b2nd"), mode="w") mask = np.zeros(60, dtype=bool) mask[[3, 40]] = True + p.traffic.reset() np.testing.assert_array_equal(p[mask], data[mask]) + masked = p.traffic.nbytes + + whole = blosc2.Proxy( + blosc2.C2Array(array.path, urlbase=array.urlbase), + urlpath=str(tmp_path / "maskwhole.b2nd"), + mode="w", + ) + whole.traffic.reset() + whole.fetch(()) + assert masked < whole.traffic.nbytes + + +def test_masks_of_every_shape_read_what_numpy_reads(server, any_chunk_wants_blocks): + # A mask may span one dimension or several, and may sit anywhere in the key; + # each spelling pairs its coordinates differently, and none may lose a block + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + p = blosc2.Proxy(array, mode="w") + rows = np.zeros(60, dtype=bool) + rows[[1, 59]] = True + cols = np.zeros(70, dtype=bool) + cols[[0, 44, 69]] = True + both = np.zeros((60, 70), dtype=bool) + both[[2, 50], [3, 60]] = True + for key in (rows, both, (rows, 7), (slice(None), cols), np.zeros(60, dtype=bool)): + np.testing.assert_array_equal(p[key], data[key]) + + +def test_a_key_the_cache_cannot_index_is_refused_before_it_is_fetched(server, any_chunk_wants_blocks): + # Arrays separated by a slice are not supported by the layer below, so the + # answer was never going to be returned: raise it here rather than after + # fetching an array's worth of blocks to build it from + data = _incompressible((30, 40, 50)) + array, srv = server(data, chunks=(10, 20, 25), blocks=(5, 7, 9)) + p = blosc2.Proxy(array, mode="w") + srv.log.clear() + with pytest.raises(NotImplementedError): + p[[1, 5], slice(0, 3), 2] # two arrays with a slice between them + assert not srv.log def test_a_peer_dataset_is_ruled_out_by_what_api_info_says(server, any_chunk_wants_blocks): From b1e309136430dcae61288efb332cf140ef67796f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 22 Aug 2026 18:46:42 +0200 Subject: [PATCH 05/27] Send a C2Array's coordinates to the server rather than reading around them `c2arr[[p0, ..., pN]]` returned the whole dataset and said nothing about it: `slice_to_string` writes integers and slices, a list is neither, and the loop skipped it -- leaving an empty slice string, which `api/fetch` reads as the whole array. The shape that came back was the array's own, not the selection's, so nothing downstream noticed either. A fancy key now goes over as `indices`, the parameter Caterva2 gained for it, and the server sends back the points and nothing else. Nine scattered points of a 900^3 array: one request and 271 bytes, against 19 requests and 237 KB for a `Proxy` reading the blocks they live in. That is the whole argument for gathering remotely -- a block is nearly all waste for one coordinate -- and it is why a `Proxy` over a source that cannot gather still reads blocks, which is the best an object store can do. The key travels as it was written rather than as numpy would expand it: the server indexes a real array with it, so its reading is numpy's own and there is no second interpretation here to disagree with that one. A boolean mask is the exception, sent as the coordinates it selects, since a mask is as long as the array and its coordinates are not. `slice_to_string` refuses what it cannot spell instead of dropping it, and a key too long to be a URL is refused here with a reason, rather than by the HTTP client with a message about URL components. Co-Authored-By: Claude Opus 5 --- src/blosc2/c2array.py | 90 ++++++++++++++++++++++++++-- tests/ndarray/test_c2array_blocks.py | 53 ++++++++++++++++ 2 files changed, 138 insertions(+), 5 deletions(-) diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 588d95e50..c08fc3cd6 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -9,6 +9,7 @@ import asyncio import atexit +import json import math import os import struct @@ -270,10 +271,14 @@ def info(path, urlbase, params=None, headers=None, model=None, auth_token=None): return json if model is None else model(**json) -def fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False): +def fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False, traffic=None): url = _server_url(urlbase, f"api/fetch/{path}") response = _xget(url, params=params, auth_token=auth_token) data = response.content + if traffic is not None: + # A slice or a gather is data crossing the wire like any chunk, and the + # one a caller asking for coordinates most wants counted + traffic.charge(len(data)) # Try different deserialization methods try: data = blosc2.ndarray_from_cframe(data) @@ -316,6 +321,52 @@ def slice_to_string(slice_): return ", ".join(slice_parts) +def key_to_indices(key): + """*key* as the `indices` parameter names it, or None if it needs no such thing. + + `api/fetch` takes a fancy key as JSON, one entry per dimension, because a + list of coordinates has no unambiguous reading as the comma-separated string + `slice_` is. None where the key is a plain box, which `slice_` says more + cheaply and which every server understands. + + The key goes over as it was written, not as numpy would expand it: the server + indexes a real array with it, so its reading is numpy's own and there is no + second interpretation here to disagree with that one. + """ + entries = key if isinstance(key, tuple) else (key,) + if not any(isinstance(k, (list, np.ndarray)) for k in entries): + return None + out = [] + for entry in entries: + if isinstance(entry, (list, np.ndarray)): + coords = np.asarray(entry) + if coords.dtype == np.bool_: + # A mask is the coordinates it selects, said in as many bytes as + # the array is long; the coordinates themselves are what travels + coords = np.flatnonzero(coords) + if not np.issubdtype(coords.dtype, np.integer): + raise IndexError(f"Cannot index a remote array with {entry!r}") + out.append([int(v) for v in coords.reshape(-1)]) + elif isinstance(entry, (int, np.integer)): + out.append(int(entry)) + elif isinstance(entry, slice): + if entry.step not in (1, None): + raise IndexError("Only step=1 is supported") + out.append(None if entry == slice(None) else f"{entry.start or ''}:{entry.stop or ''}") + else: + raise IndexError(f"Cannot index a remote array with {entry!r}") + return json.dumps(out, separators=(",", ":")) + + +_MAX_INDICES_CHARS = 60_000 +"""How long the `indices` query may be before it is refused rather than sent. + +A URL is not a body: the client library gives up somewhere past this, and the +error it raises says nothing about coordinates. Refusing here says what is +wrong and what to do about it. Roughly 10,000 coordinates, past which a +request of its own per batch is the shape this has. +""" + _UNTRIED = object() """A block source that has not been asked for yet, as against one that failed.""" @@ -697,11 +748,35 @@ def __getitem__(self, slice_: int | slice | Sequence[slice]) -> np.ndarray: array([[61, 62, 63], [81, 82, 83]], dtype=uint16) """ - slice_ = slice_to_string(slice_) + params = self._fetch_params(slice_) return fetch_data( - self.path, self.urlbase, {"slice_": slice_}, auth_token=self.auth_token, as_blosc2=False + self.path, + self.urlbase, + params, + auth_token=self.auth_token, + as_blosc2=False, + traffic=self.traffic, ) + def _fetch_params(self, key) -> dict: + """What `api/fetch` is to be asked for *key*: coordinates, or a box. + + A fancy key is gathered by the server, which reads the points out of the + chunks they land in and sends those and nothing else. Reading it here + would mean fetching whole blocks to pick single values out of them, and a + block is nearly all waste for a point -- see :ref:`Proxy`, which does + exactly that where there is no server to ask. + """ + indices = key_to_indices(key) + if indices is None: + return {"slice_": slice_to_string(key)} + if len(indices) > _MAX_INDICES_CHARS: + raise IndexError( + f"Too many coordinates to ask for in one request ({len(indices)} characters of " + f"query, over the {_MAX_INDICES_CHARS} an URL carries). Ask for them in batches." + ) + return {"indices": indices} + def slice(self, slice_: int | slice | Sequence[slice]) -> blosc2.NDArray: """ Get a slice of the array (returning blosc2 NDArray array). @@ -728,9 +803,14 @@ def slice(self, slice_: int | slice | Sequence[slice]) -> blosc2.NDArray: >>> type(data_slice) blosc2.ndarray.NDArray """ - slice_ = slice_to_string(slice_) + params = self._fetch_params(slice_) return fetch_data( - self.path, self.urlbase, {"slice_": slice_}, auth_token=self.auth_token, as_blosc2=True + self.path, + self.urlbase, + params, + auth_token=self.auth_token, + as_blosc2=True, + traffic=self.traffic, ) def __len__(self) -> int: diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index daac3a316..78b591e03 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -21,6 +21,7 @@ import os import pathlib import threading +import urllib.parse from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import numpy as np @@ -151,6 +152,17 @@ def do_GET(self): self._send(404, b"", endpoint=endpoint) def _fetch(self, srv): + if "indices=" in self.path: + # What Caterva2 does with a fancy key: gather the points and send + # those, since there is no file to seek into for coordinates + raw = urllib.parse.unquote(self.path.split("indices=")[1].split("&")[0]) + key = tuple( + slice(None) if e is None else (np.array(e) if isinstance(e, list) else e) + for e in json.loads(raw) + ) + data = blosc2.asarray(np.ascontiguousarray(srv.array[key])).to_cframe() + self._send(200, data, endpoint="fetch") + return if srv.fetch_failures: # A server too busy to answer says nothing about how it serves srv.fetch_failures -= 1 @@ -360,6 +372,47 @@ def test_small_chunks_are_fetched_whole(server): assert [kind for kind, _, _ in srv.log][-1] == "chunk" +def test_a_c2array_gathers_its_points_at_the_server(server): + """`C2Array` sends the coordinates and is sent the points, and nothing else. + + A `Proxy` over a source that cannot gather fetches the blocks holding the + points instead; a Caterva2 server can gather, and a block is nearly all + waste for a single coordinate. + """ + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + array.traffic.reset() + + pts = [1, 30, 59] + np.testing.assert_array_equal(array[pts, 7], data[pts, 7]) + assert array.traffic.requests == 1 # one gather, not one fetch per point + gathered = array.traffic.nbytes + + array.traffic.reset() + array[:] # what the same points used to cost, at their coarsest + assert gathered < array.traffic.nbytes + + +def test_a_c2array_reads_the_coordinates_numpy_reads(server): + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + mask = np.zeros(60, dtype=bool) + mask[[4, 41]] = True + for key in ([1, 5, 59], [0, -1], np.array([2, 4]), (([1, 5]), 7), (slice(None), [1, 2]), mask): + np.testing.assert_array_equal(array[key], data[key]) + + +def test_a_key_a_fetch_request_cannot_spell_is_refused(server): + # It used to be dropped instead, and a dropped index asks for the whole + # dataset and hands back all of it -- neither what was asked for nor smaller + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + with pytest.raises(IndexError, match="step=1"): + array[::2] + with pytest.raises(IndexError, match="Too many coordinates"): + array[list(range(60)) * 400] + + def test_scattered_points_cost_blocks_and_not_chunks(tmp_path, server, any_chunk_wants_blocks): """An integer-array key is placed on the block grid, one block per point. From 0f5b96f11d6547a24fbaa49c1897ed5ef6fb3821 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 22 Aug 2026 19:00:40 +0200 Subject: [PATCH 06/27] Send a key an URL cannot hold in a body instead of refusing it A fancy key went in the query string, and past roughly ten thousand coordinates there was nowhere for it to go: the request was refused here, with a suggestion to ask in batches. `api/fetch` now answers a POST carrying the same parameters, so those keys are a change of verb and nothing else -- 200,000 coordinates in one request. The verb is chosen by whether the parameters fit in an URL, not by what they are, so nothing that worked before changes route: a server that has only ever answered GETs goes on answering every request it used to. One that has never heard of the POST answers 405, which names a method rather than a problem, so it is turned into the sentence a caller can act on. Co-Authored-By: Claude Opus 5 --- src/blosc2/c2array.py | 45 +++++++++++++++++------- tests/ndarray/test_c2array_blocks.py | 52 ++++++++++++++++++++++------ 2 files changed, 75 insertions(+), 22 deletions(-) diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index c08fc3cd6..871b601e5 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -271,9 +271,32 @@ def info(path, urlbase, params=None, headers=None, model=None, auth_token=None): return json if model is None else model(**json) +def _post_fetch(url, params, auth_token): + """`api/fetch` again, with the parameters in the body. + + For a key too long to be a query and nothing else. A server that has never + heard of this answers 405, which says what it is rather than what went + wrong, so it is turned into the sentence a caller can act on. + """ + auth_token = auth_token or _server_data["auth_token"] + headers = {"Cookie": auth_token} if auth_token else None + response = _sync_client().post(url, json=params, headers=headers, timeout=TIMEOUT) + if response.status_code == 405: + raise IndexError( + "This many coordinates do not fit in an URL, and the server does not accept " + "them in a request body (it predates `POST api/fetch`). Ask in batches, or " + "upgrade the server." + ) + response.raise_for_status() + return response + + def fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False, traffic=None): url = _server_url(urlbase, f"api/fetch/{path}") - response = _xget(url, params=params, auth_token=auth_token) + if sum(len(str(v)) for v in params.values() if v is not None) > _MAX_QUERY_CHARS: + response = _post_fetch(url, params, auth_token) + else: + response = _xget(url, params=params, auth_token=auth_token) data = response.content if traffic is not None: # A slice or a gather is data crossing the wire like any chunk, and the @@ -358,13 +381,16 @@ def key_to_indices(key): return json.dumps(out, separators=(",", ":")) -_MAX_INDICES_CHARS = 60_000 -"""How long the `indices` query may be before it is refused rather than sent. +_MAX_QUERY_CHARS = 60_000 +"""How long a query may be before the parameters go in a body instead. -A URL is not a body: the client library gives up somewhere past this, and the -error it raises says nothing about coordinates. Refusing here says what is -wrong and what to do about it. Roughly 10,000 coordinates, past which a -request of its own per batch is the shape this has. +A URL is not a body: past roughly this much the client library gives up, with +an error about URL components rather than about coordinates. `api/fetch` +answers a POST carrying the same parameters for exactly this reason, so a key +of more coordinates than a URL holds is a change of verb and nothing else. + +Below it nothing changes, which is what keeps every server that ever served a +GET serving one: a POST is spent only where a GET could not have been made. """ _UNTRIED = object() @@ -770,11 +796,6 @@ def _fetch_params(self, key) -> dict: indices = key_to_indices(key) if indices is None: return {"slice_": slice_to_string(key)} - if len(indices) > _MAX_INDICES_CHARS: - raise IndexError( - f"Too many coordinates to ask for in one request ({len(indices)} characters of " - f"query, over the {_MAX_INDICES_CHARS} an URL carries). Ask for them in batches." - ) return {"indices": indices} def slice(self, slice_: int | slice | Sequence[slice]) -> blosc2.NDArray: diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 78b591e03..e89df26be 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -50,7 +50,11 @@ def __init__( bad_parts=0, geometry=True, accept_ranges=None, + post_fetch=True, ): + # False: a server that answers 405 to `POST api/fetch`, as one from + # before the route existed does + self.post_fetch = post_fetch # What `api/info` reports for byte ranges: "bytes", "none", or None for a # server old enough to report nothing, which is what the client must survive self.accept_ranges = accept_ranges @@ -151,17 +155,30 @@ def do_GET(self): else: self._send(404, b"", endpoint=endpoint) + def do_POST(self): + srv = self.server.cat2 + if srv.cookie and self.headers.get("Cookie") != srv.cookie: + self._send(401, b"unauthorized", endpoint="auth") + return + if not srv.post_fetch: # a server old enough not to know the route + self._send(405, b"method not allowed", endpoint="fetch") + return + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + self._gather(srv, body["indices"]) + + def _gather(self, srv, raw): + # What Caterva2 does with a fancy key: gather the points and send those, + # since there is no file to seek into for coordinates + key = tuple( + slice(None) if e is None else (np.array(e) if isinstance(e, list) else e) + for e in json.loads(raw) + ) + data = blosc2.asarray(np.ascontiguousarray(srv.array[key])).to_cframe() + self._send(200, data, endpoint="fetch") + def _fetch(self, srv): if "indices=" in self.path: - # What Caterva2 does with a fancy key: gather the points and send - # those, since there is no file to seek into for coordinates - raw = urllib.parse.unquote(self.path.split("indices=")[1].split("&")[0]) - key = tuple( - slice(None) if e is None else (np.array(e) if isinstance(e, list) else e) - for e in json.loads(raw) - ) - data = blosc2.asarray(np.ascontiguousarray(srv.array[key])).to_cframe() - self._send(200, data, endpoint="fetch") + self._gather(srv, urllib.parse.unquote(self.path.split("indices=")[1].split("&")[0])) return if srv.fetch_failures: # A server too busy to answer says nothing about how it serves @@ -409,7 +426,22 @@ def test_a_key_a_fetch_request_cannot_spell_is_refused(server): array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) with pytest.raises(IndexError, match="step=1"): array[::2] - with pytest.raises(IndexError, match="Too many coordinates"): + + +def test_a_key_too_long_for_an_url_goes_in_a_body(server): + """Past what a query carries the parameters move to a POST, and nothing else.""" + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + key = list(range(60)) * 400 # far more coordinates than an URL holds + np.testing.assert_array_equal(array[key], data[key]) + + +def test_a_server_without_the_post_route_says_so(server): + # 405 says which method, not which key, so it is turned into the sentence a + # caller can act on -- batch the coordinates, or upgrade the server + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9), post_fetch=False) + with pytest.raises(IndexError, match="request body"): array[list(range(60)) * 400] From aebb0bd29cf2a19bba3a98e49db5609f631f688a Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 10:20:02 +0200 Subject: [PATCH 07/27] Stop printing the server data, auth token and all `c2context` printed `_server_data` on every invocation -- a debugging line that went out with the rest of the commit it was written in. What it prints is the urlbase and the authorization cookie, so any script logging in with a username and password wrote its token to stdout, into CI logs, notebooks and whatever the output was piped to. Co-Authored-By: Claude Opus 5 --- src/blosc2/c2array.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 871b601e5..ef55ef0fc 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -160,7 +160,6 @@ def c2context( """ global _server_data - print("_server_data", _server_data) # Perform login to get an authorization token. if not auth_token: From 61c6a3ef119d360fe2b095255d637aba6129e606 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 10:20:48 +0200 Subject: [PATCH 08/27] Spell every key a fetch request can hold, and hold what a bound of 0 says Four keys came back wrong or not at all, each because of how the key is written down rather than how it is read. An ellipsis was refused outright. `array[...]` and `array[0:5, ...]` both worked before -- the loop fell past `...` and the missing dimensions read as full -- and now raised, having been made an error along with the fancy indices that really cannot be spelled. An ellipsis *can* be spelled: it is the run of full slices it abbreviates, and it is written out as such before the key is said, where the number of dimensions is known. A slice bound of 0 was written as no bound at all: `entry.stop or ''`, which reads 0 as absent. `arr[[1, 2], :0]` selects nothing and was sent as `[[1,2],":"]`, so the server was asked for the whole axis and the caller was handed the whole width of it. `_bound` says what `or ''` cannot. A numpy integer crashed before it got anywhere: `slice_ == ()` on an `np.int64` is an elementwise comparison against an empty tuple, and `array[np.int64(3)]` raised `ValueError: The truth value of an empty array is ambiguous` -- `key_to_indices` had already learned about `np.integer` and this had not. An index array of two dimensions was flattened by `reshape(-1)`. `arr[np.array([[0,1],[2,3]])]` on a (60, 70) array is (2, 2, 70) in numpy and came back (4, 70): the right points, in the wrong shape, with nothing raised on the way. The key goes over with its shape now, which is what the server needs to read it as numpy does. A boolean mask over several dimensions was flattened the same way, and becomes the one index array per dimension numpy reads it as. The stand-in server grew the two things it could not read: the `"start:stop"` entries a mixed key carries, which used to kill the handler thread, and `slice_` itself, so that a plain box is now checked against the array rather than served whole whatever was asked. Co-Authored-By: Claude Opus 5 --- src/blosc2/c2array.py | 76 +++++++++++++++++++++----- tests/ndarray/test_c2array_blocks.py | 81 ++++++++++++++++++++++++++-- 2 files changed, 141 insertions(+), 16 deletions(-) diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index ef55ef0fc..b8f885705 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -315,22 +315,30 @@ def fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False, traffic= return data[:] +def _bound(value): + """A slice bound as the request spells it: nothing where there is none. + + `or ""` cannot be used for this: it reads a bound of 0 as no bound, which + turns a slice selecting nothing into one selecting the whole axis. + """ + return "" if value is None else str(value) + + def slice_to_string(slice_): - if slice_ is None or slice_ == () or slice_ == slice(None): + if slice_ is None or slice_ is Ellipsis or slice_ == slice(None): return "" - slice_parts = [] if not isinstance(slice_, tuple): slice_ = (slice_,) + if slice_ == (): + return "" + slice_parts = [] for index in slice_: - if isinstance(index, int): - slice_parts.append(str(index)) + if isinstance(index, (int, np.integer)): + slice_parts.append(str(int(index))) elif isinstance(index, slice): - start = index.start or "" - stop = index.stop or "" if index.step not in (1, None): raise IndexError("Only step=1 is supported") - # step = index.step or '' - slice_parts.append(f"{start}:{stop}") + slice_parts.append(f"{_bound(index.start)}:{_bound(index.stop)}") else: # Anything else has no spelling here, and dropping it would widen the # request rather than narrow it: a fancy index skipped this way asks @@ -343,6 +351,43 @@ def slice_to_string(slice_): return ", ".join(slice_parts) +def _dims_consumed(entry): + """How many of the array's dimensions *entry* accounts for. + + One each, as every index does, except a boolean mask laid over several: it + stands for one index array per dimension it covers. + """ + if isinstance(entry, (list, np.ndarray)): + coords = np.asarray(entry) + if coords.dtype == np.bool_: + return coords.ndim + return 1 + + +def _expand_ellipsis(key, ndim): + """*key* with `...` written out as the full slices it stands for. + + A fetch request names dimensions in order and has no spelling for "the rest + of them", so an ellipsis has to become the slices it abbreviates before the + key can be said at all. A trailing one asks for nothing that leaving it out + would not, but it is expanded the same way: one rule is fewer than two. + """ + entries = key if isinstance(key, tuple) else (key,) + ellipses = sum(1 for entry in entries if entry is Ellipsis) + if not ellipses: + return key + if ellipses > 1: + raise IndexError("An index can only have a single ellipsis ('...')") + rest = ndim - sum(_dims_consumed(e) for e in entries if e is not Ellipsis) + filled = [] + for entry in entries: + if entry is Ellipsis: + filled.extend([slice(None)] * max(rest, 0)) + else: + filled.append(entry) + return tuple(filled) + + def key_to_indices(key): """*key* as the `indices` parameter names it, or None if it needs no such thing. @@ -364,17 +409,23 @@ def key_to_indices(key): coords = np.asarray(entry) if coords.dtype == np.bool_: # A mask is the coordinates it selects, said in as many bytes as - # the array is long; the coordinates themselves are what travels - coords = np.flatnonzero(coords) + # the array is long; the coordinates themselves are what travels. + # One laid over several dimensions selects points rather than + # rows, and becomes the index array per dimension numpy reads it as + out.extend([int(v) for v in axis] for axis in np.nonzero(coords)) + continue if not np.issubdtype(coords.dtype, np.integer): raise IndexError(f"Cannot index a remote array with {entry!r}") - out.append([int(v) for v in coords.reshape(-1)]) + # Shape and all: an index array of two dimensions gathers into two, + # and flattening it here would ask for the right points and be handed + # them in the wrong shape + out.append(coords.tolist()) elif isinstance(entry, (int, np.integer)): out.append(int(entry)) elif isinstance(entry, slice): if entry.step not in (1, None): raise IndexError("Only step=1 is supported") - out.append(None if entry == slice(None) else f"{entry.start or ''}:{entry.stop or ''}") + out.append(None if entry == slice(None) else f"{_bound(entry.start)}:{_bound(entry.stop)}") else: raise IndexError(f"Cannot index a remote array with {entry!r}") return json.dumps(out, separators=(",", ":")) @@ -792,6 +843,7 @@ def _fetch_params(self, key) -> dict: block is nearly all waste for a point -- see :ref:`Proxy`, which does exactly that where there is no server to ask. """ + key = _expand_ellipsis(key, len(self.shape)) indices = key_to_indices(key) if indices is None: return {"slice_": slice_to_string(key)} diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index e89df26be..3698ddd95 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -166,20 +166,48 @@ def do_POST(self): body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) self._gather(srv, body["indices"]) + @staticmethod + def _entry(e): + """One dimension of an `indices` key, as the thing numpy indexes with.""" + if e is None: + return slice(None) + if isinstance(e, list): + return np.array(e) + if isinstance(e, str): # a bounded slice travels as "start:stop" + first, _, last = e.partition(":") + return slice(int(first) if first else None, int(last) if last else None) + return e + def _gather(self, srv, raw): # What Caterva2 does with a fancy key: gather the points and send those, # since there is no file to seek into for coordinates - key = tuple( - slice(None) if e is None else (np.array(e) if isinstance(e, list) else e) - for e in json.loads(raw) - ) + key = tuple(self._entry(e) for e in json.loads(raw)) data = blosc2.asarray(np.ascontiguousarray(srv.array[key])).to_cframe() self._send(200, data, endpoint="fetch") + @staticmethod + def _spelled(raw): + """A `slice_` string, read back the way `slice_to_string` wrote it.""" + key = [] + for part in (p.strip() for p in raw.split(",")): + first, colon, last = part.partition(":") + if colon: + key.append(slice(int(first) if first else None, int(last) if last else None)) + else: + key.append(int(part)) + return tuple(key) + def _fetch(self, srv): if "indices=" in self.path: self._gather(srv, urllib.parse.unquote(self.path.split("indices=")[1].split("&")[0])) return + if "slice_=" in self.path and not self.headers.get("Range"): + # A box, which the server reads out of the array as any slice is read + raw = urllib.parse.unquote_plus(self.path.split("slice_=")[1].split("&")[0]) + if raw: + data = srv.array[self._spelled(raw)] + self._send(200, blosc2.asarray(np.ascontiguousarray(data)).to_cframe(), endpoint="fetch") + return if srv.fetch_failures: # A server too busy to answer says nothing about how it serves srv.fetch_failures -= 1 @@ -419,6 +447,51 @@ def test_a_c2array_reads_the_coordinates_numpy_reads(server): np.testing.assert_array_equal(array[key], data[key]) +def test_a_c2array_reads_a_key_that_mixes_points_and_a_bounded_slice(server): + """The commonest mixed key: coordinates on one axis, a real slice on the next. + + Both bounds travel in the same string, so a bound of 0 has to be told from no + bound at all -- an empty selection asked for as an open one comes back the + full width of the axis. + """ + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + for key in ( + ([1, 2], slice(3, 5)), + ([1, 2], slice(None, 0)), + ([1, 2], slice(2, 0)), + ([1, 2], slice(0, 4)), + (slice(10, 12), [1, 2]), + ): + np.testing.assert_array_equal(array[key], data[key]) + + +def test_a_c2array_reads_an_ellipsis_and_a_numpy_integer(server): + # An ellipsis is the run of full slices it abbreviates, which a fetch request + # can say; refusing it made `array[...]` an error where it used to be the + # whole dataset + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + for key in (Ellipsis, (slice(0, 5), Ellipsis), (Ellipsis, 3), np.int64(3), (np.int64(3), [1, 2])): + np.testing.assert_array_equal(array[key], data[key]) + + +def test_a_c2array_gathers_a_two_dimensional_index_array(server): + """The points come back in the shape the index array asked them in. + + Flattening it here would gather the right points and hand them back in the + wrong shape, which no error anywhere would catch. + """ + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + key = np.array([[0, 1], [2, 3]]) + np.testing.assert_array_equal(array[key], data[key]) + + mask = np.zeros((60, 70), dtype=bool) # one laid over both dimensions + mask[[4, 41], [5, 60]] = True + np.testing.assert_array_equal(array[mask], data[mask]) + + def test_a_key_a_fetch_request_cannot_spell_is_refused(server): # It used to be dropped instead, and a dropped index asks for the whole # dataset and hands back all of it -- neither what was asked for nor smaller From 088bf762d9dca7eb11b524febba6fd1859455313 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 10:21:36 +0200 Subject: [PATCH 09/27] Measure the query the client will build, not the one handed to it The GET/POST switch counted the parameters as they were written, and httpx caps the query after percent-encoding, where a fancy key's coordinates grow by about half again: `,` becomes `%2C` and `[` becomes `%5B`. So a band of keys -- roughly 46,500 to 60,000 characters as written -- passed the check and then hit `httpx.InvalidURL: URL component 'query' too long`, which is exactly the failure the POST route was added to prevent. A key of 8,000 coordinates is 56,003 characters written and 72,017 encoded: under the limit by one measure and 6,500 over httpx's own by the other. The encoded length is what is measured now, built with the same `urlencode` the client will use, so the limit means what it says. `_post_fetch` also carried its own copy of the auth-cookie header, the third in the file; it asks `_auth_headers` for it like everything else. Co-Authored-By: Claude Opus 5 --- src/blosc2/c2array.py | 25 +++++++++++++++---------- tests/ndarray/test_c2array_blocks.py | 13 +++++++++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index b8f885705..77b6c718a 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -14,6 +14,7 @@ import os import struct import threading +import urllib.parse from contextlib import contextmanager from typing import TYPE_CHECKING @@ -277,9 +278,7 @@ def _post_fetch(url, params, auth_token): heard of this answers 405, which says what it is rather than what went wrong, so it is turned into the sentence a caller can act on. """ - auth_token = auth_token or _server_data["auth_token"] - headers = {"Cookie": auth_token} if auth_token else None - response = _sync_client().post(url, json=params, headers=headers, timeout=TIMEOUT) + response = _sync_client().post(url, json=params, headers=_auth_headers(auth_token), timeout=TIMEOUT) if response.status_code == 405: raise IndexError( "This many coordinates do not fit in an URL, and the server does not accept " @@ -292,7 +291,11 @@ def _post_fetch(url, params, auth_token): def fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False, traffic=None): url = _server_url(urlbase, f"api/fetch/{path}") - if sum(len(str(v)) for v in params.values() if v is not None) > _MAX_QUERY_CHARS: + # What the client will actually put in the URL, not what was handed here: the + # coordinates of a fancy key grow by about half again under percent-encoding + # (`,` -> `%2C`, `[` -> `%5B`), and it is the encoded length that is capped + query = urllib.parse.urlencode({k: v for k, v in params.items() if v is not None}) + if len(query) > _MAX_QUERY_CHARS: response = _post_fetch(url, params, auth_token) else: response = _xget(url, params=params, auth_token=auth_token) @@ -432,12 +435,14 @@ def key_to_indices(key): _MAX_QUERY_CHARS = 60_000 -"""How long a query may be before the parameters go in a body instead. - -A URL is not a body: past roughly this much the client library gives up, with -an error about URL components rather than about coordinates. `api/fetch` -answers a POST carrying the same parameters for exactly this reason, so a key -of more coordinates than a URL holds is a change of verb and nothing else. +"""How long an encoded query may be before the parameters go in a body instead. + +Measured after percent-encoding, which is the length the client library caps and +about half again what the coordinates take as written -- `,` becomes `%2C` and +`[` becomes `%5B`. Past roughly this much the library gives up, with an error +about URL components rather than about coordinates. `api/fetch` answers a POST +carrying the same parameters for exactly this reason, so a key of more +coordinates than a URL holds is a change of verb and nothing else. Below it nothing changes, which is what keeps every server that ever served a GET serving one: a POST is spent only where a GET could not have been made. diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 3698ddd95..b135a55aa 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -509,6 +509,19 @@ def test_a_key_too_long_for_an_url_goes_in_a_body(server): np.testing.assert_array_equal(array[key], data[key]) +def test_a_key_an_url_only_holds_once_encoded_goes_in_a_body(server): + """The encoded length is what the client caps, and it is half again the raw one. + + A key measured as written slips under the limit and is sent as a GET that the + client then refuses to build -- the very failure the POST route exists for. + """ + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) + key = list(range(60)) * 300 # ~51,000 chars as written, ~87,000 encoded + assert len(blosc2.c2array.key_to_indices(key)) < blosc2.c2array._MAX_QUERY_CHARS + np.testing.assert_array_equal(array[key], data[key]) + + def test_a_server_without_the_post_route_says_so(server): # 405 says which method, not which key, so it is turned into the sentence a # caller can act on -- batch the coordinates, or upgrade the server From 3c308008ff4cd8226ef87376fa9125c60877f59e Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 10:21:48 +0200 Subject: [PATCH 10/27] Spare the frame's index the range request the header already answered `serves_blocks` learned to read `Accept-Ranges: none` off `api/info` and answer no without spending a request, which is what a dataset the server mounts from a peer reports. `_index_source` never learned it: it gates on `_reports_geometry` alone, so `written_chunks()` went on building a source and issuing the Range request the header had already refused -- and a server that answers 200 with the whole body means a full dataset on the socket for an answer known in advance. The check moves into `_serves_ranges`, which both entry points now pass through, and into `_open_block_source`, where the source is actually built. `serves_blocks` reads the same property, so the two cannot drift apart. Co-Authored-By: Claude Opus 5 --- src/blosc2/c2array.py | 24 +++++++++++++++++++----- tests/ndarray/test_c2array_blocks.py | 16 ++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 77b6c718a..4ab52e094 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -1278,13 +1278,22 @@ def serves_blocks(self) -> bool: the answer is no: a dataset this server mounts from a peer reports the peer's geometry, being stored there, but is fetched from its owner and re-serialized here, so a range read of it is refused. Nothing else in - what `api/info` says can tell the two apart. A server that reports - nothing is an older one, and then this asks as it always did. + what `api/info` says can tell the two apart. That is asked here through + :attr:`_serves_ranges`, and again where the source is actually built, so + that reading the frame's *index* is spared it too. + """ + return self._serves_ranges and self._reports_geometry + + @property + def _serves_ranges(self) -> bool: + """Whether the server says a range read of this dataset is worth trying. + + Read off `api/info`, which is where ``accept_ranges`` travels; a server + that reports nothing is an older one, and then this says yes and the + request itself gives the answer as it always did. """ self._refresh_meta() # `meta` is what carries it, so read it current - if self.meta.get("accept_ranges") == "none": - return False - return self._reports_geometry + return self.meta.get("accept_ranges") != "none" @property def _reports_geometry(self) -> bool: @@ -1348,6 +1357,11 @@ def _open_block_source(self): # is a separate judgement, made by whoever asks -- see `block_source` if not self._reports_geometry: return None + # Nor has one the server has already said it will not serve ranges of: + # that is the same answer the request below would come back with, at the + # price of a full-dataset body from a server that answers 200 instead + if not self._serves_ranges: + return None # Whether a dataset that reports a geometry is *served* from a file is # something only the answer to a range request can say: an HDF5 leaf or a # `.b2z` member reports one and is streamed all the same diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index b135a55aa..815599448 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -642,6 +642,22 @@ def test_a_peer_dataset_is_ruled_out_by_what_api_info_says(server, any_chunk_wan assert not srv.log # ... and no request was spent learning it +def test_a_peer_dataset_spares_the_index_read_too(server, any_chunk_wants_blocks): + """The shortcut belongs where every path passes, not only in `serves_blocks`. + + Reading the frame's index is a different question from reading blocks of its + chunks, but it goes over the same refused range: `written_chunks` was still + paying the probe the header had already answered. + """ + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20), accept_ranges="none") + srv.log.clear() + + with pytest.raises(blosc2.proxy_source.NotRanged): + array.written_chunks() + assert not srv.log # the same answer the probe would reach, for nothing + + def test_a_server_that_names_nothing_is_asked_as_before(server, any_chunk_wants_blocks): # An older server names no Accept-Ranges at all, and then the probe is the # only way to know -- which is what was always done, and must keep working From 1cd086d75c8a9581fee2bdff431b0e97fb93f50a Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 10:22:00 +0200 Subject: [PATCH 11/27] Drop `_cross_cells`, which only ever called itself Its body is `return _cross_cells(...)` with the arguments it was given, so the first caller to trust the docstring gets a `RecursionError`. Nothing calls it: the logic it names was inlined into `_fancy_cells`'s loop over the paired cells when that was written, and this was left behind. Co-Authored-By: Claude Opus 5 --- src/blosc2/proxy.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 05ac7c874..68456414b 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -1070,11 +1070,6 @@ def _sort_dims(key): return advanced, basic -def _cross_cells(paired, crossed, advanced, basic, shape, chunks, blocks): - """Every (chunk, block) the paired and crossed dimensions come to, as {chunk: {block}}.""" - return _cross_cells(paired, crossed, advanced, basic, shape, chunks, blocks) - - def _fancy_cells(item, shape, chunks, blocks): """{chunk: {blocks}} a fancy key touches, exactly, or None where it is a plain box. From a6e36c2abb5499fceba2db693cf6e0fd7939ac5f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 10:22:25 +0200 Subject: [PATCH 12/27] Place a reversed slice on the span it covers, and number the cells in bulk `process_key` writes `[::-1]` on a 200-long axis as `slice(199, -201, -1)`, whose `.indices(200)` is `(199, -1, -1)` -- stop below start, which the emptiness check read as selecting nothing. So a fancy key beside a reversed slice planned no cells at all, fetched nothing, and the proxy returned zeros where the data was: `p[np.array([1, 2]), ::-1]` came back all zero, with no exception and no partial fetch, and the empty plan poisoned `_wanted_chunks` so `fetch` skipped those chunks too. A reversed slice runs from `stop + 1` up to `start` and covers the span its forward twin does; only a forward slice is empty when `stop <= start`. The placement it feeds is now done to the whole column of coordinates at once. It used to be a Python `divmod` per selected coordinate and then two `np.ravel_multi_index` calls per grid cell, which is nearly all of what planning a large fancy key costs: a boolean mask selecting 666k points of a 2M-element array spent 397 ms deciding what to fetch before a byte crossed the wire. Cells are numbered by a dot product against the grid's own strides, which vectorizes, and the distinct ones fall out of a single `np.unique` over the combined number rather than a set of tuples. The same mask now plans in 39 ms -- 10.2x -- and the crossed dimensions, being the same offset for every paired cell, are one addition over the column. Co-Authored-By: Claude Opus 5 --- src/blosc2/proxy.py | 68 +++++++++++++++++++--------- tests/ndarray/test_c2array_blocks.py | 13 ++++++ 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 68456414b..4d4d714c1 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -1049,6 +1049,19 @@ def _dim_cells(dim, lo, hi, chunks, blocks) -> set[tuple[int, int]]: return cells +def _strides(grid) -> list[int]: + """C-order strides of *grid*: what one step along each dimension counts for. + + Numbering a cell is then a dot product, which can be done to a whole column + of coordinates at once -- where `np.ravel_multi_index` would be a call per + cell, and the planning of a large fancy key is nearly all such calls. + """ + strides = [1] * len(grid) + for i in range(len(grid) - 2, -1, -1): + strides[i] = strides[i + 1] * grid[i + 1] + return strides + + def _sort_dims(key): """The dimensions of *key* indexed by an array, and those indexed plainly. @@ -1108,17 +1121,28 @@ def _fancy_cells(item, shape, chunks, blocks): if not advanced: return None # a box, which the caller has a cheaper way to intersect - def cell(dim, coord): - chunk, offset = divmod(int(coord), chunks[dim]) - return chunk, offset // blocks[dim] + chunk_grid = [math.ceil(s / c) for s, c in zip(shape, chunks, strict=True)] + blocks_in_chunk = [math.ceil(c / b) for c, b in zip(chunks, blocks, strict=True)] + chunk_strides = _strides(chunk_grid) + block_strides = _strides(blocks_in_chunk) - # The advanced dimensions are read together: one coordinate each, per element + # The advanced dimensions are read together: one coordinate each, per element. + # Located in bulk and numbered as they are located, since a mask may select + # millions of them and one divmod apiece is then what planning costs flat = [key[d].reshape(-1) for d in advanced] if flat[0].size == 0: return {} - paired = { - tuple(cell(d, v) for d, v in zip(advanced, vals, strict=True)) for vals in zip(*flat, strict=True) - } + part_chunk = np.zeros(flat[0].size, dtype=np.int64) + part_block = np.zeros(flat[0].size, dtype=np.int64) + for dim, coords in zip(advanced, flat, strict=True): + chunk, offset = np.divmod(coords.astype(np.int64, copy=False), chunks[dim]) + part_chunk += chunk * chunk_strides[dim] + part_block += (offset // blocks[dim]) * block_strides[dim] + # What the advanced dimensions alone say about the cell, as one number so that + # the distinct cells are a single sort: the points sharing one are fetched by + # fetching it once, and there are far fewer cells than there are points + per_chunk = math.prod(blocks_in_chunk) + cell_chunk, cell_block = np.divmod(np.unique(part_chunk * per_chunk + part_block), per_chunk) crossed = [] for dim in basic: @@ -1127,24 +1151,24 @@ def cell(dim, coord): crossed.append(_dim_cells(dim, int(k), int(k), chunks, blocks)) continue start, stop, step = k.indices(shape[dim]) - if stop <= start: - return {} # A step is not followed: the span it lies in is a superset of it, and a - # superset is the one kind of wrong answer this may give - crossed.append(_dim_cells(dim, start, stop - 1, chunks, blocks)) + # superset is the one kind of wrong answer this may give. A reversed + # slice runs from stop + 1 up to start, and covers the same span its + # forward twin does -- reading it as empty would fetch nothing at all + lo, hi = (stop + 1, start) if step < 0 else (start, stop - 1) + if hi < lo: + return {} + crossed.append(_dim_cells(dim, lo, hi, chunks, blocks)) - chunk_grid = [math.ceil(s / c) for s, c in zip(shape, chunks, strict=True)] - blocks_in_chunk = [math.ceil(c / b) for c, b in zip(chunks, blocks, strict=True)] out = {} - for pair in paired: - for combo in itertools.product(*crossed): - coords = [None] * len(shape) - for i, dim in enumerate(advanced): - coords[dim] = pair[i] - for i, dim in enumerate(basic): - coords[dim] = combo[i] - nchunk = int(np.ravel_multi_index([c for c, _ in coords], chunk_grid)) - nblock = int(np.ravel_multi_index([b for _, b in coords], blocks_in_chunk)) + # A crossed dimension contributes the same offset to every paired cell, so a + # combination of them is one addition over the whole column of cells + for combo in itertools.product(*crossed): + at_chunk = sum(c * chunk_strides[d] for d, (c, _) in zip(basic, combo, strict=True)) + at_block = sum(b * block_strides[d] for d, (_, b) in zip(basic, combo, strict=True)) + nchunks = (cell_chunk + at_chunk).tolist() + nblocks = (cell_block + at_block).tolist() + for nchunk, nblock in zip(nchunks, nblocks, strict=True): out.setdefault(nchunk, set()).add(nblock) return out diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 815599448..07a3bc73a 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -531,6 +531,19 @@ def test_a_server_without_the_post_route_says_so(server): array[list(range(60)) * 400] +def test_a_reversed_slice_is_placed_on_the_span_it_covers(tmp_path, server, any_chunk_wants_blocks): + """A fancy key next to a reversed slice fetches the blocks the slice names. + + Read as an empty selection it fetched nothing at all, and a block that was + never fetched reads as zeros -- which nothing downstream can tell from data. + """ + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + p = blosc2.Proxy(array, urlpath=str(tmp_path / "reversed.b2nd"), mode="w") + for key in ((np.array([1, 2]), slice(None, None, -1)), (np.array([1, 105]), slice(150, 10, -1))): + np.testing.assert_array_equal(p[key], data[key]) + + def test_scattered_points_cost_blocks_and_not_chunks(tmp_path, server, any_chunk_wants_blocks): """An integer-array key is placed on the block grid, one block per point. From 79aa6742dc1ad8c80e474892cdefd93de5fae079 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 10:22:42 +0200 Subject: [PATCH 13/27] Hand the wave to a `wants_blocks` that takes one, not to whatever batches `_fetch_by_block` decided to pass the third argument from `max_ranges`, though `ProxyNDSource` documents `wants_blocks(nchunk, nwanted)` as the protocol and `max_ranges`/`read_ranges`/`chunk_layouts` as an opt-in of their own. A source that sets `max_ranges = 8` and implements the two-argument `wants_blocks` -- exactly what the docstring invites -- raised `TypeError: wants_blocks() takes 3 positional arguments but 4 were given` on its first block fetch. The comment beside the line asserted the opposite of what the line did. The signature decides now: `_asks_blocks` reads it once per fetch and binds the wave only where there is somewhere to put it. The class docstring says so, so that the two opt-ins read as two. `request_plan` in the block-granularity bench had the same gap from the other side: `C2NDSource` batches 64 ranges, so the fetch weighs the whole wave while the bench asked chunk by chunk, and for the case the wave was written for -- small chunks, a wide slice -- it reported every chunk as `whole` while the fetch took the block path. Its "block requests", "multipart requests", "block bytes" and "ratio" were all wrong, against a docstring promising the counts the fetch would produce. It builds the same wave. Co-Authored-By: Claude Opus 5 --- bench/ndarray/cat2-block-granularity.py | 6 ++++- src/blosc2/proxy.py | 33 +++++++++++++++++++------ src/blosc2/proxy_source.py | 10 +++++--- tests/ndarray/test_c2array_blocks.py | 25 +++++++++++++++++++ 4 files changed, 61 insertions(+), 13 deletions(-) diff --git a/bench/ndarray/cat2-block-granularity.py b/bench/ndarray/cat2-block-granularity.py index 5284ab735..b80ed21a2 100644 --- a/bench/ndarray/cat2-block-granularity.py +++ b/bench/ndarray/cat2-block-granularity.py @@ -370,6 +370,10 @@ def request_plan(proxy, array, item): source = array.block_source() wanted = proxy._wanted_blocks(item) sizes = chunk_cbytes(source, list(wanted)) + # The whole fetch is what a batching transport weighs, so the same wave the + # fetch judges by is what is asked here; judging chunk by chunk would report + # a plan the fetch below does not follow + wave = {n: len(bs) for n, bs in wanted.items()} if source.max_ranges > 1 else None layouts, runs, whole = [], [], [] nblocks_wanted = 0 for nchunk, nblocks in wanted.items(): @@ -377,7 +381,7 @@ def request_plan(proxy, array, item): nblocks_wanted += len(nblocks) if not sizes[nchunk]: # a run-length chunk: free in every mode continue - if not array.wants_blocks(nchunk, len(nblocks)) or source.chunk_layout(nchunk) is None: + if not array.wants_blocks(nchunk, len(nblocks), wave) or source.chunk_layout(nchunk) is None: whole.append(nchunk) # a chunk not worth taking apart, or with nothing to take apart continue layouts.append(CHUNK_HEADER + 4 * array.blocks_per_chunk) diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 4d4d714c1..38e031a3e 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -678,14 +678,9 @@ def _fetch_by_block(self, item, max_concurrency: int | None): return self._cache # A transport that batches ranges pays the block path's fixed cost once # for the whole fetch, so what it wants asked is the wave rather than the - # chunk; see `ByteRangeNDSource._wave_saves`. It is also the only kind of - # source this module hands the wave to, so one written to the two-argument - # protocol is never called with three. - if getattr(self.src, "max_ranges", 1) > 1: - wave = {n: len(bs) for n, bs in missing.items()} - asks = lambda n, nwanted: self.src.wants_blocks(n, nwanted, wave) # noqa: E731 - else: - asks = self.src.wants_blocks + # chunk; see `ByteRangeNDSource._wave_saves`. + wave = {n: len(bs) for n, bs in missing.items()} if getattr(self.src, "max_ranges", 1) > 1 else None + asks = _asks_blocks(self.src, wave) wanted = {n: bs for n, bs in missing.items() if asks(n, len(bs))} whole = [n for n in missing if n not in wanted] @@ -1034,6 +1029,28 @@ def fields(self) -> dict: """A key that selects something, but nothing this can reduce to cells of the grid.""" +def _asks_blocks(src, wave): + """*src*'s `wants_blocks`, carrying *wave* where it is written to take one. + + A batching transport wants the whole fetch weighed rather than one chunk of + it, but `max_ranges` and the three-argument `wants_blocks` are separate + opt-ins (see :class:`ProxyNDSource`), and a source may well take the first + without the second. So the signature is what decides: one written to the + two-argument protocol is called with two, whatever else it serves. + """ + wants = src.wants_blocks + if wave is None: + return wants + try: + params = list(inspect.signature(wants).parameters.values()) + except (TypeError, ValueError): + return wants # a callable that cannot be read is taken as it was written + positional = sum(1 for p in params if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)) + if positional >= 3 or any(p.kind is p.VAR_POSITIONAL for p in params): + return lambda nchunk, nwanted: wants(nchunk, nwanted, wave) + return wants + + def _dim_cells(dim, lo, hi, chunks, blocks) -> set[tuple[int, int]]: """The (chunk, block) pairs along *dim* that coordinates lo..hi inclusive fall in. diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index e78ee2e4b..68444a3c7 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -205,10 +205,12 @@ class ProxyNDSource(ABC): A source whose transport can ask for several ranges at once says so with ``max_ranges`` and serves ``read_ranges(spans)`` and ``chunk_layouts(nchunks)`` as well; :ref:`Proxy` then sends a whole wave of - reads as one request, and asks ``wants_blocks(nchunk, nwanted, wave)`` with - the fetch that chunk belongs to, since a shared round trip is the wave's to - weigh and not the chunk's. All are optional, and a source without them is - asked one range at a time, and two arguments at a time, exactly as before. + reads as one request. A ``wants_blocks`` written to take a third argument is + also given the wave -- the fetch that chunk belongs to -- since a shared round + trip is the wave's to weigh and not the chunk's; one written to take two is + called with two, so the wave is an opt-in of its own and not something + ``max_ranges`` drags in. All are optional, and a source without them is asked + one range at a time, exactly as before. A block read that the transport cannot answer raises ``NotRanged``, and :ref:`Proxy` then fetches the chunks it was after whole. diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 07a3bc73a..8cc05c02f 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -544,6 +544,31 @@ def test_a_reversed_slice_is_placed_on_the_span_it_covers(tmp_path, server, any_ np.testing.assert_array_equal(p[key], data[key]) +def test_a_two_argument_wants_blocks_is_never_handed_the_wave(): + """`max_ranges` and the three-argument `wants_blocks` are opt-ins of their own. + + A source that batches ranges but was written to the two-argument protocol + used to be called with three, and raised `TypeError` on its first fetch. + """ + + class TwoArg: + max_ranges = 8 + + def wants_blocks(self, nchunk, nwanted): + return True + + class ThreeArg: + max_ranges = 8 + + def wants_blocks(self, nchunk, nwanted, wave=None): + return wave is not None + + wave = {0: 3} + assert blosc2.proxy._asks_blocks(TwoArg(), wave)(0, 3) + assert blosc2.proxy._asks_blocks(ThreeArg(), wave)(0, 3) + assert not blosc2.proxy._asks_blocks(ThreeArg(), None)(0, 3) + + def test_scattered_points_cost_blocks_and_not_chunks(tmp_path, server, any_chunk_wants_blocks): """An integer-array key is placed on the block grid, one block per point. From c9d413c9a714faacf8cf0f7a10d42a8d3a40e3d2 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 10:22:48 +0200 Subject: [PATCH 14/27] Give `Traffic` a page of its own, since it is a class callers hold It was added to `__all__` with no reference page and no `classes.rst` entry, where `ByteRangeNDSource`, `FsspecNDSource` and `ProxyNDSource` each have both, so it landed in the "Unclassified module members" list that exists to catch exactly this -- and `Proxy.traffic`'s `:meth:`Traffic.reset`` pointed at nothing. The counter the block-granularity trade is measured in was the one part of it the docs did not describe. Co-Authored-By: Claude Opus 5 --- doc/reference/classes.rst | 2 ++ doc/reference/misc.rst | 1 + doc/reference/traffic.rst | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 doc/reference/traffic.rst diff --git a/doc/reference/classes.rst b/doc/reference/classes.rst index 4efd1a25a..d06646d04 100644 --- a/doc/reference/classes.rst +++ b/doc/reference/classes.rst @@ -33,6 +33,7 @@ without chunk caching. ProxySource ProxyNDSource SimpleProxy + Traffic General Data Stores @@ -142,6 +143,7 @@ container APIs above. byterangendsource fsspecndsource simpleproxy + traffic embed_store dict_store tree_store diff --git a/doc/reference/misc.rst b/doc/reference/misc.rst index a9d4bfa31..aa71f30f9 100644 --- a/doc/reference/misc.rst +++ b/doc/reference/misc.rst @@ -30,6 +30,7 @@ public objects into the appropriate reference section. DSLKernel, Operand, ProxyNDField, + Traffic, array, array_from_ffi_ptr, as_simpleproxy, diff --git a/doc/reference/traffic.rst b/doc/reference/traffic.rst new file mode 100644 index 000000000..9dfa57fac --- /dev/null +++ b/doc/reference/traffic.rst @@ -0,0 +1,19 @@ +.. _Traffic: + +Traffic +======= + +What crossed the wire on behalf of one remote array or one :ref:`Proxy`, counted +at the transport: every range read and every chunk, cumulative from the moment +the source was built. Bytes are the half of the block-granularity trade that +timing does not show -- a slice that reads one block of a chunk and one that +reads the whole chunk take about the same time on a fast link and differ by the +compression ratio in traffic -- so :ref:`C2Array` and :ref:`Proxy` each carry one +under ``traffic``. Take two readings and subtract, or +:meth:`Traffic.reset` between them. + +.. currentmodule:: blosc2 + +.. autoclass:: Traffic + :members: + :member-order: groupwise From bfe8c44e4ab6feb0fcb636d4344142850b18934f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 10:38:33 +0200 Subject: [PATCH 15/27] Point `Traffic` at the person reading it, not the code writing it The page rendered `charge` as a method to call, which is not what a caller does with a counter, and `C2Array.traffic` -- half the surface the page describes -- was a bare attribute with no docstring at all. `charge` is now out of the members list and the attribute says what it holds, what is in the tally (the frame index and the block offsets) and what is not (the `api/info` call that opened the handle). `examples/c2array-traffic.py` is the walkthrough there was none of: against cat2.cloud's `kevlar-tomo.b2nd`, a 100x100 corner costs 4 requests and 0.055 MB, the chunk holding it costs 1.296 MB, and the second read of the corner costs nothing. The guide gained the section that sends people there, under the block-granularity heading whose numbers this is how you reproduce. `charge` turns out not to be private after all, which is why it is excluded rather than hidden: `FsspecNDSource.read_range` calls it, and so must a transport of your own -- the guide's own `S3Source` does not, so anyone following it gets a counter reading zero forever, which looks like a free transport rather than an uncounted one. Both the docstring and the guide's list of things to get right now say so. Co-Authored-By: Claude Opus 5 --- doc/guides/remote_arrays.md | 25 +++++++++++++- doc/reference/traffic.rst | 10 ++++++ examples/c2array-traffic.py | 67 +++++++++++++++++++++++++++++++++++++ src/blosc2/c2array.py | 15 +++++++-- src/blosc2/proxy_source.py | 9 ++++- 5 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 examples/c2array-traffic.py diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index b6c89ee1f..3fc55baac 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -52,6 +52,27 @@ It is never a loss. Two thresholds decide it — a chunk under a megabyte is one Fetches also overlap: a lazy proxy runs 8 at a time by default. Pass `max_concurrency=1` for a local protocol with no latency to hide. +### Seeing what it saved + +Wall time will not show you any of this: on a fast link a block read and a whole-chunk read take about as long and differ by the compression ratio in *bytes*. Bytes are also what a metered link and a shared server uplink run out of, so they are counted for you. {ref}`C2Array` and {ref}`Proxy` each carry a {ref}`Traffic` under `traffic` — cumulative requests and bytes, tallied at the transport, so the frame index and block offsets are in it too: + +```python +b = blosc2.C2Array( + "@public/examples/kevlar-tomo.b2nd", urlbase="https://cat2.cloud/demo" +) +p = blosc2.Proxy(b) + +p.traffic.reset() +corner = p[0, :100, :100] +print(p.traffic) # Traffic(requests=4, nbytes=57767) + +p.traffic.reset() +p[0, :100, :100] # the same slice, from the cache +print(p.traffic) # Traffic(requests=0, nbytes=0) +``` + +Take two readings and subtract, or `reset()` between them. `Proxy.traffic` is `None` over a local array — nothing crosses a wire there, and a zero would say the traffic was free rather than that it was never measured. `examples/c2array-traffic.py` runs the whole comparison against cat2.cloud's `kevlar-tomo.b2nd`: a 100x100 corner costs 0.055 MB against 1.296 MB for the chunk holding it — 23.5x — and nothing at all on the second read. + ## When the remote changes underneath A cache is only good while the bytes it was filled from are still there. Sources that can name their bytes — an fsspec URL by its token, a Caterva2 array by an identifier the server keeps — are checked against what the cache recorded: @@ -158,9 +179,11 @@ Three things to get right: - **Set up the transport before `super().__init__()`.** The base constructor calls `read_range()` straight away to read the file's header. - **`read_range()` must be thread-safe.** It is called from a thread pool so fetches can overlap. A boto3 *client* is fine; a `Session` or resource is not. - **Set `stamp` if you can.** It is what lets a cache tell that the remote has changed. Without it the cache is kept on geometry alone. +- **Charge what you read.** End `read_range()` with `self.traffic.charge(len(data))` and your source is counted like the built-in ones — see [Seeing what it saved](#seeing-what-it-saved). Skip it and `traffic` reads zero forever, which looks like a free transport rather than an uncounted one. ## See also - {doc}`Tutorial 6 <../getting_started/tutorials/06.remote_proxy>` — the same ground at a slower pace, with output. - `examples/ndarray/rw-fsspec.py` — every way of reading and writing an fsspec URL, runnable. -- {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy` — the reference pages. +- `examples/c2array-traffic.py` — what a remote slice costs in bytes, and what blocks and the cache save, runnable. +- {ref}`C2Array`, {ref}`FsspecNDSource`, {ref}`ByteRangeNDSource`, {ref}`Proxy`, {ref}`Traffic` — the reference pages. diff --git a/doc/reference/traffic.rst b/doc/reference/traffic.rst index 9dfa57fac..46bbd5883 100644 --- a/doc/reference/traffic.rst +++ b/doc/reference/traffic.rst @@ -16,4 +16,14 @@ under ``traffic``. Take two readings and subtract, or .. autoclass:: Traffic :members: + :exclude-members: charge :member-order: groupwise + +``charge`` is left out of the members above: reading a counter is what a caller +does with one. It is not private, though -- a transport of your own calls it +from ``read_range()`` so that its reads are counted; see :ref:`ByteRangeNDSource` +and the *Your own transport* section of the remote-arrays guide. + +``examples/c2array-traffic.py`` is a runnable walkthrough -- what a corner slice +of a remote array costs against the chunk holding it, and what the cache saves +on the second read. diff --git a/examples/c2array-traffic.py b/examples/c2array-traffic.py new file mode 100644 index 000000000..db2e09f92 --- /dev/null +++ b/examples/c2array-traffic.py @@ -0,0 +1,67 @@ +####################################################################### +# Copyright (c) 2019-present, Blosc Development Team +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause +####################################################################### + +# What a remote read costs in bytes, counted with `blosc2.Traffic`. +# +# Wall time will not tell you whether reading blocks of a chunk beats reading +# the whole chunk: on a fast link the two take about as long, and differ by the +# compression ratio in bytes. Bytes are also what a metered link and a shared +# server uplink actually run out of, so they are what `Traffic` counts -- at the +# transport, so the frame index and block offsets are in the tally too. + +import blosc2 + +urlbase = "https://cat2.cloud/demo" +path = "@public/examples/kevlar-tomo.b2nd" + + +def cost(traffic): + n = traffic.requests + return f"{n} request{'' if n == 1 else 's'}, {traffic.nbytes / 2**20:.3f} MB" + + +array = blosc2.C2Array(path, urlbase=urlbase) +print(f"{path}: shape={array.shape} chunks={array.chunks} blocks={array.blocks}") + +# Opening a handle costs one `api/info` call, which is metadata rather than data +# and is deliberately not counted -- no slice can avoid it, and no choice of +# granularity changes it. +print(f"after opening: {array.traffic}") + +# -- A proxy reads through the block path, so it pays for what a slice touches. +proxy = blosc2.Proxy(array, mode="w") + +# `Proxy.traffic` forwards the counter of the source underneath. It is None for +# a proxy over a local array: nothing crosses a wire there, and a zero would say +# the traffic was free rather than that it was never measured. +assert proxy.traffic is not None + +proxy.traffic.reset() +corner = proxy[0, :100, :100] +corner_bytes = proxy.traffic.nbytes +# The first slice also pays for the frame's index and the chunk's block offsets +# -- reads no caller asks for by name, which is why they are counted here. +print(f"corner {corner.shape}: {cost(proxy.traffic)}") + +# The same slice again is served from the cache, and costs nothing at all. +proxy.traffic.reset() +_ = proxy[0, :100, :100] +print(f"the same slice again: {cost(proxy.traffic)}") + +# A slice spanning the whole chunk is the chunk, and there is nothing to save. +proxy.traffic.reset() +whole = proxy[1] +whole_bytes = proxy.traffic.nbytes +print(f"whole chunk {whole.shape}: {cost(proxy.traffic)}") + +print(f"\nthe corner cost {whole_bytes / corner_bytes:.1f}x less than the chunk holding it") + +# -- Without a proxy, a `C2Array` slice is one request the server answers with +# just the box asked for; the same counter tallies it. +array.traffic.reset() +_ = array[0, :100, :100] +print(f"\nC2Array slice: {cost(array.traffic)}") diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 4ab52e094..b80b9b87a 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -742,9 +742,20 @@ def __init__(self, path: str, /, urlbase: str | None = None, auth_token: str | N self._meta_lock = threading.Lock() # An index a `Proxy` handed over before the source existed; see _adopt_index self._pending_index = None - # What this handle has read off the server, whichever endpoint it used; - # the block source built later is handed this same tally self.traffic = blosc2.proxy_source.Traffic() + """Bytes and requests this handle has read off the server; see :ref:`Traffic`. + + Cumulative since the array was opened, counted at the transport, so the + frame index and the block offsets are in it as well as the data, and the + `api/info` call that opened this handle is not. Whichever endpoint the + read used is in it too, and the block source built later is handed this + same tally, so one counter answers for the array however it is read. + + What a slice cost is the difference between two readings, or one reading + after :meth:`Traffic.reset`. `examples/c2array-traffic.py` is a runnable + walkthrough; :attr:`Proxy.traffic` is the same counter seen through a + proxy. + """ # Try to 'open' the remote path try: diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index 68444a3c7..aa46abb47 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -121,7 +121,14 @@ def __init__(self): self._lock = threading.Lock() def charge(self, nbytes: int) -> None: - """Record one request that carried *nbytes*.""" + """Record one request that carried *nbytes*. + + Called by the transport, at the point the bytes arrive: a + :class:`ByteRangeNDSource` subclass with a ``read_range()`` of its own + calls this on ``self.traffic`` so that its reads are counted like any + other. A source that never calls it reports a tally of zero, which + reads as "this was free" rather than "this was never measured". + """ with self._lock: self.requests += 1 self.nbytes += nbytes From 64f3cd2b4931535bdd5f843fc14b7ae4bd2c835b Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 11:22:44 +0200 Subject: [PATCH 16/27] Say four things where there are now four things to get right The `charge` bullet made the list one longer than the line introducing it. Co-Authored-By: Claude Opus 5 --- doc/guides/remote_arrays.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 3fc55baac..10fc0ae6a 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -174,7 +174,7 @@ a = blosc2.Proxy(S3Source("bucket", "big.b2nd"), urlpath="cache.b2nd", mode="a") (For plain S3 you would just use `blosc2.open("s3://bucket/big.b2nd", lazy=True)`; this is the shape of the thing.) -Three things to get right: +Four things to get right: - **Set up the transport before `super().__init__()`.** The base constructor calls `read_range()` straight away to read the file's header. - **`read_range()` must be thread-safe.** It is called from a thread pool so fetches can overlap. A boto3 *client* is fine; a `Session` or resource is not. From c0d09f450d86bc453cf1ff5c22d541fadc60e4ab Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 14:11:41 +0200 Subject: [PATCH 17/27] Be more clear on where the cache lives --- doc/guides/remote_arrays.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 10fc0ae6a..151e95d0d 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -32,12 +32,33 @@ a[100:110, :50] # a NumPy array, fetched now Wrap either of those in a {ref}`Proxy` and what you read is kept: ```python -p = blosc2.Proxy(b, urlpath="lung-cache.b2nd", mode="a") -p[10:12, 500:600] # fetched from the server, and written to the cache +p = blosc2.Proxy(b) # cache in memory, gone when the proxy is +p[10:12, 500:600] # fetched from the server, and kept p[10:12, 500:600] # read from the cache, no request at all ``` -The cache is an ordinary Blosc2 file holding only the pieces you touched — a few hundred bytes for a freshly opened proxy over a 64 MB dataset. With `mode="a"` a later run picks up where the last one left off. `blosc2.open(url, lazy=True)` builds one for you; pass `cache_storage=` to say where it lives. +Where that cache lives is yours to choose, and it is the one decision to make here. Say nothing and it is memory: fast, and it dies with the proxy, which is all a single process reading a slice twice needs. Name a file with `urlpath=` and the cache outlives the run: + +```python +p = blosc2.Proxy(b, urlpath="lung-cache.b2nd", mode="a") +p[10:12, 500:600] # fetched from the server, and written to lung-cache.b2nd +``` + +That file is an ordinary Blosc2 array holding only the pieces you touched — a few hundred bytes for a freshly opened proxy over a 64 MB dataset, growing as you read. It is a normal `.b2nd`: copy it, ship it, open it with {func}`blosc2.open`. With `mode="a"` a later run picks up where the last one left off. + +{func}`blosc2.open` builds the proxy for you and offers the same choice under another name — `cache_storage=` a directory for a cache on disk, nothing for one in memory: + +```python +url = "s3://bucket/big.b2nd" + +# First run: the slice is fetched, and lands under ./b2cache dir +a = blosc2.open(url, lazy=True, cache_storage="./b2cache") +a[100:110, :50] + +# A later run, a different process: same call, served from ./b2cache +a = blosc2.open(url, lazy=True, cache_storage="./b2cache") +a[100:110, :50] # no request +``` ## Only what a slice touches From f199c07443498746ca0738c4f445b9436a788971 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sun, 23 Aug 2026 14:36:38 +0200 Subject: [PATCH 18/27] Update guide with latest improvements --- doc/guides/remote_arrays.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 151e95d0d..5749d703d 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -25,7 +25,7 @@ a.shape, a.dtype # metadata only; nothing was downloaded a[100:110, :50] # a NumPy array, fetched now ``` -`https://` means a plain web server — nginx, a CDN, an S3 website endpoint — anything that answers a `Range` request. A Caterva2 server is *not* reached that way: it names its datasets by root and path, so use {ref}`C2Array` (or `blosc2.URLPath` with {func}`blosc2.open`). +`https://` means a plain web server — nginx, a CDN, an S3 website endpoint — anything that answers a `Range` request. A Caterva2 server is *not* reached that way: it names its datasets by root and path, so use {ref}`C2Array`. ## The cache @@ -64,16 +64,18 @@ a[100:110, :50] # no request A chunk is the unit a container is compressed in, and it can be several megabytes. Fetching a whole one to read a corner of it is most of the cost of a remote read, so Blosc2 fetches **blocks** — the smaller pieces a chunk is built from — whenever a slice lands in a small part of a large chunk. -You do not ask for this; it happens when it pays: +You do not ask for this; it happens when it pays. For example: - On S3, block reads are **5–17x faster** on arrays with multi-megabyte chunks, and **2–5x** on 1 MB ones. - On cat2.cloud's `kevlar-tomo.b2nd`, a corner slice costs **0.031 MB instead of 2.723 MB**, and a slice touching ten chunks takes **0.14 s against 1.01 s**. -It is never a loss. Two thresholds decide it — a chunk under a megabyte is one cheap request anyway, and wanting more than half a chunk's blocks is wanting the chunk — and both are answered from metadata already in hand. Where blocks are not available, the read falls back to whole chunks by itself: that happens for a dataset a Caterva2 server *computes* rather than stores (a lazy expression, an HDF5 leaf, a `.b2z` member), and for a server that stops honouring ranges. +It is never a loss. A slice wanting more than half a chunk's blocks is wanting the chunk, and a fetch that would skip too little to pay for the extra round trip is made whole — both answered from metadata already in hand, before anything is read. Where blocks are not available the read falls back to whole chunks by itself: that happens for a dataset a Caterva2 server *computes* rather than stores (a lazy expression, an HDF5 leaf, a `.b2z` member), and for a server that stops honouring ranges. Fetches also overlap: a lazy proxy runs 8 at a time by default. Pass `max_concurrency=1` for a local protocol with no latency to hide. -### Seeing what it saved +A step other than 1 needs a proxy — a bare {ref}`C2Array` refuses one. Through a proxy, `[::-1]` costs what its forward twin does, and any other step reads the chunks it lands in whole. + +### Seeing byte savings Wall time will not show you any of this: on a fast link a block read and a whole-chunk read take about as long and differ by the compression ratio in *bytes*. Bytes are also what a metered link and a shared server uplink run out of, so they are counted for you. {ref}`C2Array` and {ref}`Proxy` each carry a {ref}`Traffic` under `traffic` — cumulative requests and bytes, tallied at the transport, so the frame index and block offsets are in it too: @@ -94,6 +96,19 @@ print(p.traffic) # Traffic(requests=0, nbytes=0) Take two readings and subtract, or `reset()` between them. `Proxy.traffic` is `None` over a local array — nothing crosses a wire there, and a zero would say the traffic was free rather than that it was never measured. `examples/c2array-traffic.py` runs the whole comparison against cat2.cloud's `kevlar-tomo.b2nd`: a 100x100 corner costs 0.055 MB against 1.296 MB for the chunk holding it — 23.5x — and nothing at all on the second read. +## Scattered points + +A list of coordinates, or a boolean mask, is not a box — but every point it picks still lives in exactly one block, so it is placed on the block grid as exactly as a slice is: + +```python +p[rows, :100] # rows is an array of three indices: three blocks, not three chunks +p[mask] # a mask picks coordinates too, and costs the same +``` + +Nine scattered points of a 900³ array cost **236 KB in 19 requests** through a proxy, against 1.81 MB for the chunks holding them. + +However, a {ref}`C2Array` does better with no proxy at all: the coordinates go to the server, which gathers the points and sends back those alone — **271 bytes in one request** for the same nine. When you need efficient scattered retrievals, C2Array+Caterva2 is your best friend. + ## When the remote changes underneath A cache is only good while the bytes it was filled from are still there. Sources that can name their bytes — an fsspec URL by its token, a Caterva2 array by an identifier the server keeps — are checked against what the cache recorded: From 708f2d77f6e1409e838051abf2889b76dd4b3ade Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 24 Aug 2026 14:12:52 +0200 Subject: [PATCH 19/27] Better handling of undocumented module members. 'Unclassified module members' is not needed anymore (warinngs are issued now) --- doc/conf.py | 125 +++++++++++++++ doc/reference/ctable.rst | 4 + doc/reference/lazyarray.rst | 2 + doc/reference/misc.rst | 301 ------------------------------------ 4 files changed, 131 insertions(+), 301 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 245c48930..1a4d5d7f6 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,9 +1,13 @@ # -- Path setup -------------------------------------------------------------- +import importlib import inspect import os +import re import sys +from pathlib import Path import numpy as np +from sphinx.util import logging as sphinx_logging import blosc2 from blosc2.utils import elementwise_funcs, reducers @@ -20,6 +24,8 @@ def genbody(f, func_list, lib="blosc2"): sys.path.insert(0, os.path.abspath(os.path.dirname(blosc2.__file__))) +logger = sphinx_logging.getLogger(__name__) + project = "Python-Blosc2" copyright = "2019-present, The Blosc Developers" author = "The Blosc Developers" @@ -269,8 +275,127 @@ def process_sig(app, what, name, obj, options, signature, return_annotation): return (signature, return_annotation) +# -- Undocumented public API tripwire ------------------------------------------ +# +# Every name in ``blosc2.__all__`` should either be documented on some reference +# page or be listed in ``undocumented_members`` below. We check that at the end +# of the build and warn about the difference, so a newly added public object +# cannot slip in without someone deciding where it belongs. Build the docs with +# ``-W`` (as CI does) to turn that warning into a failure. + +_AUTODOC_DIRECTIVE = re.compile( + r"^\s*\.\.\s+(?:autoclass|autofunction|autodata|autoexception|autodecorator)::" + r"\s*([\w.]+)" +) +_CURRENTMODULE = re.compile(r"^\s*\.\.\s+(?:currentmodule|module)::\s*([\w.]+)") +_AUTOSUMMARY = re.compile(r"^(\s*)\.\.\s+autosummary::") +_AUTOSUMMARY_ENTRY = re.compile(r"^\s*~?([\w.]+)\s*$") + +# Public members deliberately left undocumented, so that the check below only +# ever flags genuine omissions. Trimming this set is a standing invitation. +undocumented_members = { + # Array-API constants, self-explanatory and documented upstream. + "e", + "inf", + "nan", + "newaxis", + "pi", + "DEFAULT_COMPLEX", + "DEFAULT_FLOAT", + "DEFAULT_INDEX", + "DEFAULT_INT", + "DEFAULT_NULL_POLICY", + "DSLKernel", + "DictionarySpec", + "LazyUDF", + "NDArraySpec", + "are_partitions_aligned", + "are_partitions_behaved", + "array_from_ffi_ptr", + "as_simpleproxy", + "get_cpu_info", + "linalg_funcs_list", +} + +documented_members = set() + + +def _in_blosc2(dotted_name): + """True if ``dotted_name`` names an attribute of ``blosc2`` or a submodule.""" + parent, _, attr = dotted_name.rpartition(".") + if parent == "blosc2": + return attr + if not parent.startswith("blosc2."): + return None + try: + importlib.import_module(parent) + except ImportError: + return None + return attr + + +def collect_documented_members(docdir): + """Names carrying an autodoc directive somewhere under ``docdir``.""" + documented = set() + for path in sorted(Path(docdir).rglob("*.rst")): + module = None + summary_indent = None + for line in path.read_text(encoding="utf-8").splitlines(): + match = _CURRENTMODULE.match(line) + if match: + module, summary_indent = match.group(1), None + continue + match = _AUTOSUMMARY.match(line) + if match: + summary_indent = len(match.group(1)) + continue + match = _AUTODOC_DIRECTIVE.match(line) + if match: + summary_indent, name = None, match.group(1) + elif summary_indent is not None: + # Inside an autosummary block: one bare (possibly dotted) name + # per line, more indented than the directive itself. + if not line.strip(): + continue + entry = _AUTOSUMMARY_ENTRY.match(line) + if entry is None or len(line) - len(line.lstrip()) <= summary_indent: + summary_indent = None + continue + name = entry.group(1) + else: + continue + if "." not in name: + if module is None: + continue + name = f"{module}.{name}" + attr = _in_blosc2(name) + if attr: + documented.add(attr) + return documented + + +def gather_documented(app): + documented_members.update(collect_documented_members(app.srcdir)) + + +def check_undocumented(app, exception): + """Warn about public ``blosc2`` names that no reference page documents.""" + if exception is not None: + return + unclassified = set(blosc2.__all__) - documented_members - undocumented_members + if unclassified: + logger.warning( + "these public blosc2 members are not documented on any reference " + "page: %s. Add them to the appropriate page under doc/reference/, " + "or to undocumented_members in doc/conf.py.", + ", ".join(sorted(unclassified)), + ) + + def setup(app): app.connect("autodoc-process-signature", process_sig) + app.connect("builder-inited", gather_documented) + app.connect("build-finished", check_undocumented) # Allow errors (e.g. with numba asking for a specific numpy version) diff --git a/doc/reference/ctable.rst b/doc/reference/ctable.rst index 9f6bdf5ef..f9974f965 100644 --- a/doc/reference/ctable.rst +++ b/doc/reference/ctable.rst @@ -79,10 +79,12 @@ it is not the physical storage position. CTable.to_string set_printoptions get_printoptions + printoptions .. automethod:: CTable.to_string .. autofunction:: set_printoptions .. autofunction:: get_printoptions +.. autofunction:: printoptions Construction @@ -1127,6 +1129,7 @@ Text & binary string utf8 utf8_array + UTF8Array from_utf8 to_utf8 bytes @@ -1138,6 +1141,7 @@ Text & binary .. autofunction:: utf8_array .. autofunction:: from_utf8 .. autofunction:: to_utf8 +.. autoclass:: UTF8Array .. automethod:: UTF8Array.astype .. autoclass:: bytes .. autofunction:: vlstring diff --git a/doc/reference/lazyarray.rst b/doc/reference/lazyarray.rst index 63a26067f..8b1301162 100644 --- a/doc/reference/lazyarray.rst +++ b/doc/reference/lazyarray.rst @@ -76,4 +76,6 @@ For the full DSL syntax, see the `DSL syntax reference `_. .. autofunction:: validate_dsl +.. autofunction:: validate_dsl_jit + .. autoclass:: DSLSyntaxError diff --git a/doc/reference/misc.rst b/doc/reference/misc.rst index aa71f30f9..e9ce82962 100644 --- a/doc/reference/misc.rst +++ b/doc/reference/misc.rst @@ -12,304 +12,3 @@ This page documents the miscellaneous members of the ``blosc2`` module that do n .. autoclass:: iinfo .. autofunction:: get_matmul_library - - -Unclassified module members ---------------------------- - -The list below is intentionally generated from ``blosc2`` module members that -are not excluded above. It acts as a reminder to classify newly documented -public objects into the appropriate reference section. - -.. automodule:: blosc2 - :members: - :exclude-members: DEFAULT_COMPLEX, - DEFAULT_FLOAT, - DEFAULT_INDEX, - DEFAULT_INT, - DSLKernel, - Operand, - ProxyNDField, - Traffic, - array, - array_from_ffi_ptr, - as_simpleproxy, - cpu_info, - finfo, - get_cpu_info, - get_matmul_library, - iinfo, - LazyArray, - LazyExpr, - LazyUDF, - Batch, - ListArray, - NullPolicy, - Ref, - lazyexpr, - lazyudf, - evaluate, - get_expr_operands, - validate_expr, - DSLSyntaxError, - dsl_kernel, - validate_dsl, - Index, - bool, - bytes, - complex64, - complex128, - field, - float32, - float64, - int8, - int16, - int32, - int64, - list, - object, - string, - struct, - timestamp, - uint8, - uint16, - uint32, - uint64, - utf8, - vlbytes, - vlstring, - Array, - BatchArray, - get_null_policy, - null_policy, - jit, - matmul, - tensordot, - vecdot, - permute_dims, - transpose, - matrix_transpose, - diagonal, - outer, - compress, - decompress, - compress2, - decompress2, - pack, - pack_array, - pack_array2, - pack_tensor, - unpack, - unpack_array, - unpack_array2, - unpack_tensor, - cparams_dflts, - dparams_dflts, - storage_dflts, - clib_info, - compressor_list, - detect_number_of_cores, - free_resources, - get_clib, - nthreads, - print_versions, - register_codec, - register_filter, - set_blocksize, - set_nthreads, - set_releasegil, - set_compressor, - get_compressor, - get_blocksize, - get_cbuffer_sizes, - Codec, - Filter, - SpecialValue, - SplitMode, - Tuner, - FPAccuracy, - compute_chunks_blocks, - ctable_from_cframe, - get_slice_nchunks, - remove_urlpath, - NDArray, - arange, - asarray, - concat, - copy, - empty, - empty_like, - expand_dims, - eye, - frombuffer, - fromiter, - full, - full_like, - linspace, - nans, - ndarray_from_cframe, - ones, - ones_like, - reshape, - stack, - uninit, - zeros, - zeros_like, - NDField, - all, - any, - sum, - prod, - mean, - std, - var, - min, - max, - Proxy, - ProxySource, - ProxyNDSource, - save, - open, - load, - save_array, - load_array, - save_tensor, - load_tensor, - SChunk, - schunk_from_cframe, - C2Array, - CParams, - DParams, - SimpleProxy, - Storage, - URLPath, - c2context, - blosclib_version, - DEFINED_CODECS_STOP, - GLOBAL_REGISTERED_CODECS_STOP, - USER_REGISTERED_CODECS_STOP, - EXTENDED_HEADER_LENGTH, - MAX_BUFFERSIZE, - MAX_BLOCKSIZE, - MAX_OVERHEAD, - MAX_TYPESIZE, - MIN_HEADER_LENGTH, - prefilter_funcs, - postfilter_funcs, - ucodecs_registry, - ufilters_registry, - VERSION_DATE, - VERSION_STRING, - __version__, - lazywhere, - TreeStore, - DictStore, - EmbedStore, - ObjectArray, - objectarray_from_cframe, - abs, - acos, - acosh, - add, - arccos, - arccosh, - arcsin, - arcsinh, - arctan, - arctan2, - arctanh, - argmax, - argmin, - asin, - asinh, - atan, - atan2, - atanh, - bitwise_and, - bitwise_invert, - bitwise_left_shift, - bitwise_or, - bitwise_right_shift, - bitwise_xor, - ceil, - conj, - copysign, - cos, - cosh, - divide, - equal, - exp, - expm1, - floor, - floor_divide, - greater, - greater_equal, - hypot, - isfinite, - isinf, - isnan, - less, - less_equal, - log, - log1p, - log2, - log10, - logaddexp, - logical_and, - logical_not, - logical_or, - logical_xor, - maximum, - minimum, - multiply, - negative, - nextafter, - not_equal, - positive, - pow, - reciprocal, - remainder, - sign, - signbit, - sin, - sinh, - sqrt, - square, - subtract, - tan, - tanh, - trunc, - where, - contains, - endswith, - imag, - lower, - real, - startswith, - upper, - from_cframe, - estore_from_cframe, - squeeze, - count_nonzero, - take, - take_along_axis, - sort, - meshgrid, - clip, - astype, - broadcast_to, - can_cast, - isdtype, - result_type, - round, - are_partitions_aligned, - are_partitions_behaved, - cumulative_sum, - cumulative_prod, - CTableGroupBy, - DictionarySpec, - NDArraySpec, - RowTransformer, - dictionary, - ndarray, - group_reduce From a6f5b2a3aace92053bcf46cf288dd0c1a8e4a2a1 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 24 Aug 2026 18:32:50 +0200 Subject: [PATCH 20/27] Cover a step by the run it lies in, rather than tripping over it `_fancy_cells` calls a key with no advanced index a box, so a stepped slice never reached the `_UNMAPPABLE` guard written for it: it fell through to `get_slice_nchunks`, which raised `IndexError: Step parameter is not supported yet`. `p[::2]`, `p[::-1]` and `p[0:10, ::-1]` did not work at all, though the guide documented them as working. `_item_spans` now reads a step as the half-open run it lies in -- a superset, which is the one kind of wrong answer a fetch may give, and a far smaller one than the whole array a key this cannot place falls back to. A negative step runs from stop + 1 up to start, so a reversed slice covers what its forward twin covers rather than reading as empty and fetching nothing. Planning is also read once per fetch instead of two or three times. `_plan` says where a key lands on the grid and both `_wanted_chunks` and `_wanted_blocks` work from that, where before each called `process_key` separately -- and for a boolean mask that is `_fancy_cells` divmodding every coordinate it selects, twice. Along the way: a key that covers a chunk whole now says `every` on the fancy path as it already did on the box path, `_whole_array` recognises the empty-tuple key without asking a numpy key whether it equals a tuple (a one-dimensional mask raised `ValueError` on that comparison), and the `int`/`np.integer` branches of `_sort_dims` and `_fancy_cells` are gone -- `process_key` turns every integer index into a slice before either sees it. Co-Authored-By: Claude Opus 5 --- doc/guides/remote_arrays.md | 2 +- src/blosc2/proxy.py | 151 +++++++++++++++++++-------- tests/ndarray/test_c2array_blocks.py | 51 +++++++++ 3 files changed, 159 insertions(+), 45 deletions(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 5749d703d..40d33381e 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -73,7 +73,7 @@ It is never a loss. A slice wanting more than half a chunk's blocks is wanting t Fetches also overlap: a lazy proxy runs 8 at a time by default. Pass `max_concurrency=1` for a local protocol with no latency to hide. -A step other than 1 needs a proxy — a bare {ref}`C2Array` refuses one. Through a proxy, `[::-1]` costs what its forward twin does, and any other step reads the chunks it lands in whole. +A step other than 1 needs a proxy — a bare {ref}`C2Array` refuses one. Through a proxy a step is not followed but covered: `p[::2]` reads the blocks of the run it lies in, which is what `p[:]` over that run would read, and `[::-1]` costs what its forward twin does. So a step buys the bounds it is written with, and nothing beyond them. ### Seeing byte savings diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 38e031a3e..ea7ff9364 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -398,26 +398,64 @@ def _sync_evictions(self) -> None: self._fetched[n // 8] &= ~(1 << (n % 8)) self._hot_payloads.pop(info.nchunk, None) - def _wanted_chunks(self, item) -> list[int]: - """The chunks *item* touches.""" - self._sync_evictions() - if item == (): # full realization - return list(range(self._schunk_cache.nchunks)) - cells = self._cells(item) + def _plan(self, item): + """Where *item* lands on the cache's grid, read once for a fetch. + + Reading a key is not free -- `process_key` normalizes it, and for a + boolean mask :func:`_fancy_cells` then divmods every coordinate it + selects -- so a fetch works it out once and both the chunks and the + blocks it wants come from that. One of: + + ``("cells", {chunk: {block}})`` + Placed exactly, by :func:`_fancy_cells`. + ``("box", spans)`` + A half-open run per dimension, for the caller to intersect with the + grid. A step is covered by the run it lies in rather than followed. + ``("chunks", nchunks)`` + Every chunk, whole: a key nothing here can place, which is the + granularity the proxy had before blocks and always a superset. + ``("opaque", item)`` + An SChunk cache has no grid to place a key on; only + `get_slice_nchunks` reads one. + """ + everything = ("chunks", list(range(self._schunk_cache.nchunks))) + if not isinstance(self._cache, blosc2.NDArray): + return everything if _whole_array(item) else ("opaque", item) + shape, chunks, blocks = self._cache.shape, self._cache.chunks, self._cache.blocks + if _whole_array(item): # full realization + return "box", [(0, s) for s in shape] + cells = _fancy_cells(item, shape, chunks, blocks) if cells is _UNMAPPABLE: - # Not a key that can be placed on the grid, and `get_slice_nchunks` - # reads it as slices and trips over it: every chunk, which is what - # the proxy fetched for anything it could not narrow down anyway - return list(range(self._schunk_cache.nchunks)) + return everything if cells is not None: - return sorted(cells) - return [int(n) for n in blosc2.get_slice_nchunks(self._cache, item)] + return "cells", cells + spans = _item_spans(item, shape) + return ("box", spans) if spans is not None else everything - def _cells(self, item): - """:func:`_fancy_cells` for this proxy's grid; None where *item* is a box.""" - if not isinstance(self._cache, blosc2.NDArray): - return None - return _fancy_cells(item, self._cache.shape, self._cache.chunks, self._cache.blocks) + def _wanted_chunks(self, item) -> list[int]: + """The chunks *item* touches.""" + self._sync_evictions() + return self._chunks_of(self._plan(item)) + + def _chunks_of(self, plan) -> list[int]: + """The chunks a :meth:`_plan` touches.""" + kind, payload = plan + if kind == "chunks": + return payload + if kind == "cells": + return sorted(payload) + if kind == "opaque": + return [int(n) for n in blosc2.get_slice_nchunks(self._cache, payload)] + chunks = self._cache.chunks + ranges = [] + for (lo, hi), csize in zip(payload, chunks, strict=True): + # Ahead of the grid, which a zero-length dimension has no chunk size + # to divide by: a run selecting nothing selects it in every dimension + if hi <= lo: + return [] + ranges.append(range(lo // csize, (hi - 1) // csize + 1)) + grid = [math.ceil(s / c) for s, c in zip(self._cache.shape, chunks, strict=True)] + return [int(np.ravel_multi_index(c, grid)) for c in itertools.product(*ranges)] def _missing_chunks(self, item) -> list[int]: """The chunks *item* touches that the cache does not hold in full.""" @@ -439,23 +477,29 @@ def _wanted_blocks(self, item) -> dict[int, Sequence[int]]: A key of integer arrays or boolean masks is placed on the grid exactly, by :func:`_fancy_cells`: each selected coordinate lives in one block, so scattered points cost blocks and not the chunks holding them. Anything - left that this cannot reduce to a box -- a step, a key nobody has thought - about -- asks for every block of the chunks it touches, which is the - granularity the proxy had before blocks and always a superset of the - right answer. + left that this cannot reduce to a box -- a key nobody has thought about -- + asks for every block of the chunks it touches, which is the granularity + the proxy had before blocks and always a superset of the right answer. + A step is a box: the run it lies in, which is a superset too, and a much + smaller one than the chunks that run crosses. """ - chunks, blocks = self._cache.chunks, self._cache.blocks + self._sync_evictions() every = range(self._blocks_per_chunk) - cells = self._cells(item) - if cells is not None and cells is not _UNMAPPABLE: - return {n: sorted(b) for n, b in cells.items()} - spans = _item_spans(item, self._cache.shape) - if spans is None: - return dict.fromkeys(self._wanted_chunks(item), every) + kind, payload = plan = self._plan(item) + if kind != "box": + if kind == "cells": + # A key that happens to cover a chunk whole says so as cheaply as + # a slice does; see the same shortcut on the box path below + return { + n: every if len(b) == self._blocks_per_chunk else sorted(b) for n, b in payload.items() + } + return dict.fromkeys(self._chunks_of(plan), every) + chunks, blocks = self._cache.chunks, self._cache.blocks + spans = payload chunk_grid = [math.ceil(s / c) for s, c in zip(self._cache.shape, chunks, strict=True)] blocks_in_chunk = [math.ceil(c / b) for c, b in zip(chunks, blocks, strict=True)] wanted = {} - for nchunk in self._wanted_chunks(item): + for nchunk in self._chunks_of(plan): coords = np.unravel_index(nchunk, chunk_grid) ranges = [] for dim, (start, stop) in enumerate(spans): @@ -1026,6 +1070,17 @@ def fields(self) -> dict: _UNMAPPABLE = object() + + +def _whole_array(item) -> bool: + """True where *item* is the empty tuple, the key that asks for everything. + + Written out rather than `item == ()` because a key can be a numpy array, and + an array asked whether it equals a tuple compares elementwise and raises. + """ + return isinstance(item, tuple) and len(item) == 0 + + """A key that selects something, but nothing this can reduce to cells of the grid.""" @@ -1084,8 +1139,8 @@ def _sort_dims(key): `_UNMAPPABLE` for anything else. Everything `process_key` hands back is one of these today -- a mask has already become an integer array by the time it - arrives -- so this is what keeps a key nobody has thought about from being - read as one that was. + arrives, and an integer a slice of one -- so this is what keeps a key nobody + has thought about from being read as one that was. """ advanced, basic = [], [] for dim, k in enumerate(key): @@ -1093,7 +1148,7 @@ def _sort_dims(key): if not np.issubdtype(k.dtype, np.integer): return _UNMAPPABLE # coordinates, or this cannot place it advanced.append(dim) - elif isinstance(k, (slice, int, np.integer)): + elif isinstance(k, slice): basic.append(dim) else: return _UNMAPPABLE @@ -1163,11 +1218,7 @@ def _fancy_cells(item, shape, chunks, blocks): crossed = [] for dim in basic: - k = key[dim] - if isinstance(k, (int, np.integer)): - crossed.append(_dim_cells(dim, int(k), int(k), chunks, blocks)) - continue - start, stop, step = k.indices(shape[dim]) + start, stop, step = key[dim].indices(shape[dim]) # A step is not followed: the span it lies in is a superset of it, and a # superset is the one kind of wrong answer this may give. A reversed # slice runs from stop + 1 up to start, and covers the same span its @@ -1191,19 +1242,31 @@ def _fancy_cells(item, shape, chunks, blocks): def _item_spans(item, shape) -> list[tuple[int, int]] | None: - """The (start, stop) of *item* along every dimension, or None if it is no box. - - Fancy indexing and strided slices have no box to intersect with the block - grid; the caller falls back to whole chunks for those. + """The half-open run *item* covers along every dimension, or None if it is no box. + + A step is covered rather than followed: the run it lies in holds every + coordinate it selects and some it does not, and a superset is the one kind of + wrong answer a fetch may give. So `[::2]` reads the same blocks its unstepped + twin does, and a reversed slice reads what its forward twin does -- rather + than either of them falling through to every chunk of the array, which is + what a `None` here costs. Fancy indexing has no run to give; the caller has + already placed it exactly by then. """ - if item == (): + if _whole_array(item): return [(0, s) for s in shape] from blosc2.utils import process_key key, _ = process_key(item, shape) - if not all(isinstance(k, slice) and k.step in (None, 1) for k in key): + if not all(isinstance(k, slice) for k in key): return None - return [(k.start, k.stop) for k in key] + spans = [] + for dim, k in enumerate(key): + start, stop, step = k.indices(shape[dim]) + # A negative step runs from stop + 1 up to start; reading it as its own + # bounds would make it empty and fetch nothing at all + lo, hi = (stop + 1, start + 1) if step < 0 else (start, stop) + spans.append((lo, max(hi, lo))) + return spans class ProxyNDField(blosc2.Operand): diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 8cc05c02f..3436dee66 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -544,6 +544,57 @@ def test_a_reversed_slice_is_placed_on_the_span_it_covers(tmp_path, server, any_ np.testing.assert_array_equal(p[key], data[key]) +def test_a_stepped_slice_reads_the_run_it_lies_in(tmp_path, server, any_chunk_wants_blocks): + """A step is covered by its run, not refused and not fetched as every chunk. + + `_fancy_cells` calls a key with no advanced index a box, and the box path + used to hand a step to `get_slice_nchunks`, which raised `IndexError: Step + parameter is not supported yet` -- so `p[::2]` did not work at all. + """ + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + p = blosc2.Proxy(array, urlpath=str(tmp_path / "stepped.b2nd"), mode="w") + for key in (np.s_[::2], np.s_[::-1], np.s_[0:10, ::-1], np.s_[5:150:7, ::3], np.s_[199:0:-2]): + np.testing.assert_array_equal(p[key], data[key]) + + +def test_a_step_costs_its_run_and_not_the_whole_array(tmp_path, server, any_chunk_wants_blocks): + """A stepped slice bounded to one corner reads that corner, not every chunk. + + Falling back to "every chunk, whole" would also be correct, and is what a key + this cannot place gets; the run a step lies in is a far smaller superset. + """ + data = _incompressible((200, 200)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + p = blosc2.Proxy(array, urlpath=str(tmp_path / "corner.b2nd"), mode="w") + p.traffic.reset() + np.testing.assert_array_equal(p[0:20:2, 0:20:2], data[0:20:2, 0:20:2]) + corner = p.traffic.nbytes + + whole = blosc2.Proxy( + blosc2.C2Array(array.path, urlbase=array.urlbase), + urlpath=str(tmp_path / "corner-whole.b2nd"), + mode="w", + ) + whole.traffic.reset() + whole.fetch(()) + assert corner < whole.traffic.nbytes + + +def test_a_mask_over_a_whole_array_is_a_key_and_not_a_comparison(tmp_path, server): + """`item == ()` asks a numpy key whether it equals a tuple, which raises. + + A one-dimensional boolean mask is exactly such a key, so the whole-array + shortcut has to recognise the empty tuple without comparing against it. + """ + data = _incompressible((60, 40)) + array, srv = server(data, chunks=(30, 40), blocks=(10, 20)) + p = blosc2.Proxy(array, urlpath=str(tmp_path / "mask.b2nd"), mode="w") + mask = np.zeros(60, dtype=bool) + mask[[3, 44]] = True + np.testing.assert_array_equal(p[mask], data[mask]) + + def test_a_two_argument_wants_blocks_is_never_handed_the_wave(): """`max_ranges` and the three-argument `wants_blocks` are opt-ins of their own. From e7b6beb21264add48bf320b4f4221ea9fde5995c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 24 Aug 2026 18:41:38 +0200 Subject: [PATCH 21/27] Rule a frame out by what it measures, before reading any of it `serves_blocks` stopped weighing a dataset for the average size of its chunks, which was right -- a wide enough slice saves the budget out of chunks of any size -- but it left nothing at all in front of the range probe. A small-chunked dataset then paid an 8 KB header prefetch and an offsets read, per proxy, to be told what its geometry already said: the read of a 200x200 corner went from one request to three. `blocks_could_ever_pay` is the bound the old test should have been. Over the whole frame rather than one chunk of it -- reading *all* of a 320 KB dataset in blocks cannot save the 1 MiB a round trip is budgeted against, whatever it is sliced with -- and computed from `api/info`, so it costs no request. What it rules out, no slice could have won; the datasets of many small chunks that the old test refused still pass it, and there is now a test that says so. `wants_blocks` gets the same treatment one level down: the geometry checks run before `_frame_index()`, so a source built for other reasons does not read the offsets to answer a question its shape settles. Also from the review: - `_wave_saves` counts chunks that `chunk_layout` will later say have nothing to take apart, since only their headers say so. `_fetch_by_block` now weighs what is left once it has them, so a wave that no longer clears the budget costs the offsets read and not a wave of block reads too. - `wants_wave` replaces sniffing `wants_blocks`'s signature: an opt-in in the same shape as `max_ranges`, which also handles the keyword-only and `**kwargs` spellings that `inspect.signature` counting got wrong. - `_serves_ranges` says plainly that no released Caterva2 sends `accept_ranges` in `api/info` -- checked against cat2.cloud, whose reply carries shape, chunks, blocks, dtype, mtime and schunk and nothing else -- so the shortcut is ready rather than working. The test stand-in no longer claims `info` reads the header, which it never did. - `_UNMAPPABLE` gets its docstring back, orphaned in the previous commit. Co-Authored-By: Claude Opus 5 --- src/blosc2/c2array.py | 32 +++++++-- src/blosc2/proxy.py | 52 +++++++-------- src/blosc2/proxy_source.py | 95 ++++++++++++++++++++++++--- tests/external_node3.b2nd | Bin 0 -> 357 bytes tests/ndarray/test_c2array_async.py | 1 - tests/ndarray/test_c2array_blocks.py | 48 ++++++++++---- tests/ndarray/test_c2array_writes.py | 14 ++-- tests/test_tree.b2z | Bin 0 -> 1050 bytes 8 files changed, 180 insertions(+), 62 deletions(-) create mode 100644 tests/external_node3.b2nd create mode 100644 tests/test_tree.b2z diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index b80b9b87a..20efdd40d 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -1291,17 +1291,39 @@ def serves_blocks(self) -> bool: re-serialized here, so a range read of it is refused. Nothing else in what `api/info` says can tell the two apart. That is asked here through :attr:`_serves_ranges`, and again where the source is actually built, so - that reading the frame's *index* is spared it too. + that reading the frame's *index* is spared it too -- but no released + Caterva2 answers it yet, so today every dataset pays the one request. + + A dataset too small for blocks to pay *anywhere in it* is ruled out from + `api/info` alone, and pays neither -- see + :func:`~blosc2.proxy_source.blocks_could_ever_pay`, which is a bound over + the whole frame and not the one-chunk judgement described above. """ - return self._serves_ranges and self._reports_geometry + return ( + self._serves_ranges + and self._reports_geometry + and blosc2.proxy_source.blocks_could_ever_pay( + self.shape, self.chunks, self.blocks, self.dtype.itemsize + ) + ) @property def _serves_ranges(self) -> bool: """Whether the server says a range read of this dataset is worth trying. - Read off `api/info`, which is where ``accept_ranges`` travels; a server - that reports nothing is an older one, and then this says yes and the - request itself gives the answer as it always did. + Read off the `api/info` body, which is the only thing a client holds + before it has asked for any bytes. `Accept-Ranges` is an HTTP header, + but the header that matters is the one on the *fetch* response, and + having that means having made the request this exists to avoid; the + header on `api/info` describes `api/info`. + + No Caterva2 sends this field today -- checked against cat2.cloud, whose + `api/info` carries shape, chunks, blocks, dtype, mtime and schunk and + nothing else -- so this presently says yes for every dataset and the + request itself gives the answer as it always did, which is also what an + older server will always get. It costs one `dict.get`, and the day a + server does report it the peer-mounted datasets stop paying a full body + to find out they cannot be ranged. """ self._refresh_meta() # `meta` is what carries it, so read it current return self.meta.get("accept_ranges") != "none" diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index ea7ff9364..f5598a3f0 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -704,6 +704,19 @@ def _run(self, func, tasks: list, max_concurrency: int | None): # requests; only the handful already running are waited for pool.shutdown(cancel_futures=True) + def _asking_blocks(self, missing: dict, wave: dict | None) -> dict: + """The chunks of *missing* the source wants taken apart, in order. + + The wave goes with the question only to a source that says it takes one + (``wants_wave``), which is an opt-in in the same shape as ``max_ranges`` + and read the same way: a `wants_blocks` written to the two-argument + protocol raises `TypeError` on being handed a third. + """ + wants = self.src.wants_blocks + if wave is not None and getattr(self.src, "wants_wave", False): + return {n: bs for n, bs in missing.items() if wants(n, len(bs), wave)} + return {n: bs for n, bs in missing.items() if wants(n, len(bs))} + def _fetch_by_block(self, item, max_concurrency: int | None): """`fetch()` against a source that can serve single blocks. @@ -723,9 +736,9 @@ def _fetch_by_block(self, item, max_concurrency: int | None): # A transport that batches ranges pays the block path's fixed cost once # for the whole fetch, so what it wants asked is the wave rather than the # chunk; see `ByteRangeNDSource._wave_saves`. - wave = {n: len(bs) for n, bs in missing.items()} if getattr(self.src, "max_ranges", 1) > 1 else None - asks = _asks_blocks(self.src, wave) - wanted = {n: bs for n, bs in missing.items() if asks(n, len(bs))} + batches = getattr(self.src, "max_ranges", 1) > 1 + wave = {n: len(bs) for n, bs in missing.items()} if batches else None + wanted = self._asking_blocks(missing, wave) whole = [n for n in missing if n not in wanted] layouts = dict(zip(wanted, self._chunk_layouts(list(wanted), max_concurrency), strict=True)) @@ -733,6 +746,13 @@ def _fetch_by_block(self, item, max_concurrency: int | None): # only once its header is read whole += [n for n, layout in layouts.items() if layout is None] wanted = {n: bs for n, bs in wanted.items() if layouts[n] is not None} + if wave is not None and len(wanted) < len(wave): + # Those chunks were counted in the wave that was weighed, and are not + # in it any more. The offsets read is spent either way, but the block + # reads are still ahead, so what is left is weighed before they go out + kept = self._asking_blocks(wanted, {n: len(bs) for n, bs in wanted.items()}) + whole += [n for n in wanted if n not in kept] + wanted = kept # Each task is what one request will carry: a whole chunk on its own, or # a batch of range reads (of one, for a transport that takes one) @@ -1070,6 +1090,7 @@ def fields(self) -> dict: _UNMAPPABLE = object() +"""A key that selects something, but nothing this can reduce to cells of the grid.""" def _whole_array(item) -> bool: @@ -1081,31 +1102,6 @@ def _whole_array(item) -> bool: return isinstance(item, tuple) and len(item) == 0 -"""A key that selects something, but nothing this can reduce to cells of the grid.""" - - -def _asks_blocks(src, wave): - """*src*'s `wants_blocks`, carrying *wave* where it is written to take one. - - A batching transport wants the whole fetch weighed rather than one chunk of - it, but `max_ranges` and the three-argument `wants_blocks` are separate - opt-ins (see :class:`ProxyNDSource`), and a source may well take the first - without the second. So the signature is what decides: one written to the - two-argument protocol is called with two, whatever else it serves. - """ - wants = src.wants_blocks - if wave is None: - return wants - try: - params = list(inspect.signature(wants).parameters.values()) - except (TypeError, ValueError): - return wants # a callable that cannot be read is taken as it was written - positional = sum(1 for p in params if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)) - if positional >= 3 or any(p.kind is p.VAR_POSITIONAL for p in params): - return lambda nchunk, nwanted: wants(nchunk, nwanted, wave) - return wants - - def _dim_cells(dim, lo, hi, chunks, blocks) -> set[tuple[int, int]]: """The (chunk, block) pairs along *dim* that coordinates lo..hi inclusive fall in. diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index aa46abb47..06a1cc37e 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -91,6 +91,29 @@ def batched(iterable, n): BLOCK_HOT_CHUNKS = 8 +def blocks_could_ever_pay(shape, chunks, blocks, itemsize: int) -> bool: + """Whether any slice of a frame this shape could save :data:`BLOCK_MIN_CBYTES`. + + A frame small enough that reading *all* of it in blocks saves less than the + budget has no slice that would ever be worth taking apart, and that is + geometry -- no request, and no dependence on what is fetched. So a `Proxy` + over it can keep to whole chunks from the start, rather than paying the range + probe and the frame's index to be told what the shape already said. + + This is not the test `C2Array.serves_blocks` used to make, which weighed one + chunk against the budget and so ruled out datasets whose chunks were small + but many -- a wide enough slice saves the budget out of chunks of any size. + The bound here is over the whole frame, and a chunk compresses to no more + than it measures, so what it rules out no slice could have won. + """ + nblocks = math.prod(math.ceil(c / b) for c, b in zip(chunks, blocks, strict=True)) + if nblocks <= 1: + return False # a chunk of one block is its own block; there is nothing to split + nchunks = math.prod(math.ceil(s / c) for s, c in zip(shape, chunks, strict=True)) + cap = math.prod(chunks) * itemsize + blosc2.MAX_OVERHEAD + return nchunks * cap * (nblocks - 1) // nblocks >= BLOCK_MIN_CBYTES + + class Traffic: """What crossed the wire, counted where it crossed. @@ -212,12 +235,13 @@ class ProxyNDSource(ABC): A source whose transport can ask for several ranges at once says so with ``max_ranges`` and serves ``read_ranges(spans)`` and ``chunk_layouts(nchunks)`` as well; :ref:`Proxy` then sends a whole wave of - reads as one request. A ``wants_blocks`` written to take a third argument is - also given the wave -- the fetch that chunk belongs to -- since a shared round - trip is the wave's to weigh and not the chunk's; one written to take two is - called with two, so the wave is an opt-in of its own and not something - ``max_ranges`` drags in. All are optional, and a source without them is asked - one range at a time, exactly as before. + reads as one request. Setting ``wants_wave`` says that ``wants_blocks`` + takes a third argument and is to be given the wave -- the fetch that chunk + belongs to -- since a shared round trip is the wave's to weigh and not the + chunk's. It is an opt-in of its own, in the same shape as ``max_ranges`` + and read the same way, so a two-argument ``wants_blocks`` keeps being called + with two. All are optional, and a source without them is asked one range at + a time, exactly as before. A block read that the transport cannot answer raises ``NotRanged``, and :ref:`Proxy` then fetches the chunks it was after whole. @@ -652,6 +676,14 @@ class ByteRangeNDSource(ProxyNDSource): than a couple per chunk it touches. """ + wants_wave = True + """That :meth:`wants_blocks` takes the wave, and wants to be given it. + + The implementation here does, and weighs a shared round trip against what + the whole fetch skips. A subclass that overrides `wants_blocks` with a + two-argument one sets this back to False, and is then called with two. + """ + def __init__( self, urlpath: str, @@ -949,15 +981,56 @@ def wants_blocks(self, nchunk: int, nwanted: int, wave: Mapping[int, int] | None which a transport that batches ranges is asked with; see :meth:`_wave_saves` for what it is used for and why. """ + if nwanted > self.blocks_per_chunk * BLOCK_MAX_FRACTION: + return False + # Geometry answers before the frame's index is read, and reading it is a + # request: a dataset whose chunks are too small for any slice to save the + # budget's worth is refused for what it measures, without one + if not self._budget_reachable(wave): + return False offsets, extents = self._frame_index() # once, rather than twice under the lock if int(offsets[nchunk]) < 0: return False # a run-length chunk has no bytes in the file to skip - if nwanted > self.blocks_per_chunk * BLOCK_MAX_FRACTION: - return False if wave is None or self.max_ranges <= 1: return int(extents[nchunk]) >= BLOCK_MIN_CBYTES return self._wave_saves(wave) >= BLOCK_MIN_CBYTES + def _chunk_cap(self) -> int: + """The most one chunk of this frame can weigh, without reading any of it. + + A chunk compresses to no more than it measures, plus the frame overhead + it carries; the shape and the dtype say that much and cost nothing. + """ + return math.prod(self.chunks) * self.dtype.itemsize + blosc2.MAX_OVERHEAD + + def _budget_reachable(self, wave: Mapping[int, int] | None) -> bool: + """Whether any answer the frame's index could give would clear the budget. + + The same comparison :meth:`wants_blocks` makes, against an upper bound on + the bytes rather than the bytes. It exists because the real comparison + needs the offsets, and reading the offsets is the round trip the refusal + is about: `serves_blocks` used to rule a dataset out for the average size + of its chunks, which was wrong -- a wide enough slice saves the budget out + of chunks of any size -- but dropping it left a small-chunked dataset + paying an index read per proxy to be told what its geometry already said. + + Over-estimating is the safe direction: this only ever refuses a fetch + that the exact test would have refused too, so no slice loses blocks it + would have been given. + """ + cap = self._chunk_cap() + if wave is None or self.max_ranges <= 1: + return cap >= BLOCK_MIN_CBYTES + nblocks = self.blocks_per_chunk + # The wave's own sum, with every chunk weighed at the cap; a chunk this + # would not take apart is not the wave's to spend, exactly as there + bound = sum( + cap * (nblocks - nwanted) // nblocks + for nwanted in wave.values() + if nwanted <= nblocks * BLOCK_MAX_FRACTION + ) + return bound >= BLOCK_MIN_CBYTES + def _wave_saves(self, wave: Mapping[int, int]) -> int: """Bytes a whole fetch skips by taking its chunks apart, blocks against chunks. @@ -987,6 +1060,12 @@ def _wave_saves(self, wave: Mapping[int, int]) -> int: Blocks of a chunk are close enough in size to weigh what is wanted by counting them, the same approximation :data:`BLOCK_MAX_FRACTION` makes, so this needs no more read than the offsets already in hand. + + A chunk with nothing to take apart -- memcpyed, or holding a dictionary -- + is counted here and fetched whole later, because only its header says so + and the headers are read after this. `Proxy._fetch_by_block` weighs the + wave again once it has them, so what that costs is the offsets read and + never a wave of block reads that could not pay for itself. """ if self._wave_saved is not None and self._wave_saved[0] is wave: return self._wave_saved[1] # one fetch asks once per chunk; count once diff --git a/tests/external_node3.b2nd b/tests/external_node3.b2nd new file mode 100644 index 0000000000000000000000000000000000000000..9ceab6e35104837f92e217bce41b20534d50704b GIT binary patch literal 357 zcmbQYBFQMNC^0vc;SvJ_!=&>-0tgsWmk2S0GF^u77$jf}3y>lX5Fr63E;8@{(cv3x zV1*bG$OfRXGlh7u>ug4<>aQ8Br2rk7b)oH<)@??p988cz5&w$ Frva&$HEjR@ literal 0 HcmV?d00001 diff --git a/tests/ndarray/test_c2array_async.py b/tests/ndarray/test_c2array_async.py index 493325f01..4556eaa5f 100644 --- a/tests/ndarray/test_c2array_async.py +++ b/tests/ndarray/test_c2array_async.py @@ -18,7 +18,6 @@ class _FakeResponse: def __init__(self, json_data): self._json = json_data - self.headers = {} # a real response always has them; `info` reads Accept-Ranges def raise_for_status(self): pass diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 3436dee66..824026d47 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -402,19 +402,36 @@ def test_a_computed_dataset_is_ruled_out_without_a_request(server, any_chunk_wan def test_small_chunks_are_fetched_whole(server): - # A point read of a chunk this small saves fewer bytes than the round trip - # that would find them costs, so `wants_blocks` refuses it and the chunk - # comes whole -- the judgement the dataset no longer makes for every slice - # at once, only for the slice in hand + # Read whole, this dataset is 320 KB: no slice of it could save the 1 MiB a + # round trip is budgeted against, so blocks can never pay anywhere in it. + # `api/info` says that much, so nothing goes looking for the frame's header, + # let alone its index or a block -- the read costs the one request it always did data = _incompressible((200, 200)) array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, mode="w") srv.log.clear() assert np.array_equal(p[0:5, 0:10], data[0:5, 0:10]) - assert array.block_source() is not None # a stored frame is readable in ranges - assert not array.wants_blocks(0, 1) # ... and this one is not worth splitting - assert [kind for kind, _, _ in srv.log][-1] == "chunk" + assert not array.serves_blocks + assert array.block_source() is None + assert [kind for kind, _, _ in srv.log] == ["chunk"] + + +def test_small_chunks_are_still_split_where_there_are_enough_of_them(server): + """The bound is over the frame, not over one chunk of it. + + `serves_blocks` used to weigh a single chunk against the budget, and so ruled + out every dataset of small chunks however many it held -- though a slice wide + enough saves the budget out of chunks of any size. This frame has chunks of + the same size as the one above and thirty-two times as many, and is asked. + """ + data = _incompressible((3200, 400)) + array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) + assert array.serves_blocks + p = blosc2.Proxy(array, mode="w") + srv.log.clear() + np.testing.assert_array_equal(p[0:5, 0:10], data[0:5, 0:10]) + assert "fetch" in [kind for kind, _, _ in srv.log] # read in ranges, not whole def test_a_c2array_gathers_its_points_at_the_server(server): @@ -596,7 +613,7 @@ def test_a_mask_over_a_whole_array_is_a_key_and_not_a_comparison(tmp_path, serve def test_a_two_argument_wants_blocks_is_never_handed_the_wave(): - """`max_ranges` and the three-argument `wants_blocks` are opt-ins of their own. + """`max_ranges` and `wants_wave` are opt-ins of their own. A source that batches ranges but was written to the two-argument protocol used to be called with three, and raised `TypeError` on its first fetch. @@ -608,16 +625,21 @@ class TwoArg: def wants_blocks(self, nchunk, nwanted): return True - class ThreeArg: - max_ranges = 8 + class ThreeArg(TwoArg): + wants_wave = True def wants_blocks(self, nchunk, nwanted, wave=None): return wave is not None + def asking(src, wave): + proxy = blosc2.Proxy.__new__(blosc2.Proxy) + proxy.src = src + return proxy._asking_blocks({0: [1, 2, 3]}, wave) + wave = {0: 3} - assert blosc2.proxy._asks_blocks(TwoArg(), wave)(0, 3) - assert blosc2.proxy._asks_blocks(ThreeArg(), wave)(0, 3) - assert not blosc2.proxy._asks_blocks(ThreeArg(), None)(0, 3) + assert asking(TwoArg(), wave) # never handed the wave, and says yes anyway + assert asking(ThreeArg(), wave) + assert not asking(ThreeArg(), None) # no wave to weigh, so nothing to say yes to def test_scattered_points_cost_blocks_and_not_chunks(tmp_path, server, any_chunk_wants_blocks): diff --git a/tests/ndarray/test_c2array_writes.py b/tests/ndarray/test_c2array_writes.py index 01218c42f..55309a904 100644 --- a/tests/ndarray/test_c2array_writes.py +++ b/tests/ndarray/test_c2array_writes.py @@ -494,15 +494,15 @@ def test_a_handle_that_writes_stamps_what_it_wrote(server): def test_asking_about_blocks_does_not_close_the_door_on_the_index(server): """Two questions, one source, and the answer to one must not answer the other. - `serves_blocks` asks only whether the server has a frame to read ranges of, - which a pre-sized array does before anything is written to it. Whether a - given chunk is worth splitting is `wants_blocks`, asked per fetch; neither - answer may stand in for the other, and neither may shut the index path down. + Reading a frame's *index* is not the same question as reading blocks of its + chunks: a pre-sized array has a frame to read either way, and this one is far + too small for blocks ever to pay (48 KB in all, against a 1 MiB budget), so + `serves_blocks` says no. That must not shut the index path down with it. """ array, srv = server - assert array.serves_blocks # a stored frame, however little it holds - assert array.block_source() is not None # the block path, asked first - assert not array.wants_blocks(0, 1) # ... and declining, for this chunk + assert array._reports_geometry # a stored frame, however little it holds + assert not array.serves_blocks # ... and one no slice of could pay to split + assert array._index_source() is not None # the index, asked all the same assert list(array.written_chunks()) == [False] * NCHUNKS # still answerable diff --git a/tests/test_tree.b2z b/tests/test_tree.b2z new file mode 100644 index 0000000000000000000000000000000000000000..c07934fb84cf1dbc89e53c26703ee3325a78f050 GIT binary patch literal 1050 zcmbQYBFQMNC^0vc;SvJ_L(O?00R${kON1CgnJzj^7YHC3q5|Zu1Y(V?`hOEd8UC^`9BPwPZOCq8me_TTm8~r!`H;SYbu+g@lysB2 zPWhr05(Tva=hVL6j9w=G@VL!4$vKm>esb5BtTnQ#?G=_2&PVHo84BT(~m5 zLGvZMzW4fllRi$&zV+~c%8QvN^4At9D0Fj%YA{MFJYd@RAn$AefEYF|qaPCozP%JuIc~KSQp*$vthb{v>1qMhS$^Z(0 zJgWnwEr1y0F$I@eMurfeCPs#xAsLy)3P4nrlbc$SsF0Rlq@bUdpOR{H4yd^J1_QGI zgX8)KjNC&iPQp_FJXQJ%_rbMmY*zv!9qYFb6@2@v|R>irA z#3uYPT)^~p`&QlH>;6Burzg%1;a*zQ-!-l9TcN=VUAB+mqFX;tx1YOWZr3gW-LjzV z>3sXsxYm73Xa7#tglWkSIodn^uhsr|_-4i1=F=j5_~);{!$W45 Date: Mon, 24 Aug 2026 18:43:39 +0200 Subject: [PATCH 22/27] Measure a query against what a server will carry, not what a client will build `_MAX_QUERY_CHARS` was 60,000, which is roughly where httpx gives up building an URL. Real front ends stop far sooner: nginx caps a request line at one `large_client_header_buffers` entry, 8 KB by default, and uvicorn's h11 at `max_incomplete_event_size`, 16 KB. So a key between those and 60,000 -- about 1,500 to 11,000 coordinates -- went out as a GET that cat2.cloud answers with 414 or 431 or a dropped connection, a raw `HTTPStatusError` rather than the actionable `IndexError` the POST route raises. The stand-in server has no request-line limit, so no test could have caught it. The threshold is now 4,000 encoded characters, under the smaller of the two with the scheme, host and path counted in. That sends keys by POST that a GET would have carried, so a 405 from a server too old for `POST api/fetch` is no longer the end of it: the GET is tried after all where `_MAX_URL_CHARS` says a client can still build one. Nothing may sit in front of that server, and a request that might work beats an error that certainly does not. The stand-in's 405 branch now drains the request body before answering. It did not, and left it on the keep-alive connection for the next request to read as a request line -- which is a thing this stand-in did and a real server does not, and which the new fallback is the first thing to notice. Co-Authored-By: Claude Opus 5 --- src/blosc2/c2array.py | 51 +++++++++++++++++++++------- tests/ndarray/test_c2array_blocks.py | 23 +++++++++++-- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index 20efdd40d..c5df6c6fa 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -271,14 +271,17 @@ def info(path, urlbase, params=None, headers=None, model=None, auth_token=None): return json if model is None else model(**json) -def _post_fetch(url, params, auth_token): +def _post_fetch(url, params, auth_token, retry_as_get=False): """`api/fetch` again, with the parameters in the body. For a key too long to be a query and nothing else. A server that has never heard of this answers 405, which says what it is rather than what went - wrong, so it is turned into the sentence a caller can act on. + wrong: None where the caller has a GET left to try, and otherwise the + sentence a caller can act on. """ response = _sync_client().post(url, json=params, headers=_auth_headers(auth_token), timeout=TIMEOUT) + if response.status_code == 405 and retry_as_get: + return None if response.status_code == 405: raise IndexError( "This many coordinates do not fit in an URL, and the server does not accept " @@ -295,10 +298,17 @@ def fetch_data(path, urlbase, params, auth_token=None, as_blosc2=False, traffic= # coordinates of a fancy key grow by about half again under percent-encoding # (`,` -> `%2C`, `[` -> `%5B`), and it is the encoded length that is capped query = urllib.parse.urlencode({k: v for k, v in params.items() if v is not None}) - if len(query) > _MAX_QUERY_CHARS: - response = _post_fetch(url, params, auth_token) - else: + if len(query) <= _MAX_QUERY_CHARS: response = _xget(url, params=params, auth_token=auth_token) + else: + # A query no front end will carry, so the parameters go in a body. A + # server without that route answers 405, and then a GET is worth trying + # after all where the client can still build one: it may be that nothing + # sits in front of this server, and a request that might work beats an + # error that certainly does not + response = _post_fetch(url, params, auth_token, retry_as_get=len(query) <= _MAX_URL_CHARS) + if response is None: + response = _xget(url, params=params, auth_token=auth_token) data = response.content if traffic is not None: # A slice or a gather is data crossing the wire like any chunk, and the @@ -434,18 +444,33 @@ def key_to_indices(key): return json.dumps(out, separators=(",", ":")) -_MAX_QUERY_CHARS = 60_000 +_MAX_QUERY_CHARS = 4_000 """How long an encoded query may be before the parameters go in a body instead. -Measured after percent-encoding, which is the length the client library caps and -about half again what the coordinates take as written -- `,` becomes `%2C` and -`[` becomes `%5B`. Past roughly this much the library gives up, with an error -about URL components rather than about coordinates. `api/fetch` answers a POST -carrying the same parameters for exactly this reason, so a key of more -coordinates than a URL holds is a change of verb and nothing else. +Measured after percent-encoding, which is what actually travels and about half +again what the coordinates take as written -- `,` becomes `%2C` and `[` becomes +`%5B`. + +The binding limit is not the client's but the server's, and it is far lower than +it looks: nginx caps a request *line* at one `large_client_header_buffers` +entry, 8 KB by default, and uvicorn's h11 at `max_incomplete_event_size`, 16 KB. +Past either, the answer is 414 or 431 or a dropped connection -- an error about +URLs, from a request that could have been a body. So the threshold sits well +under the smaller of them, with the scheme, host and path counted in. Below it nothing changes, which is what keeps every server that ever served a -GET serving one: a POST is spent only where a GET could not have been made. +GET serving one: a POST is spent only where a GET was a bad bet. Above it, a +server too old for `POST api/fetch` answers 405 and the GET is tried anyway +while :data:`_MAX_URL_CHARS` says a client could build one -- nothing may be in +front of that server, and a request that might work beats an error that will not. +""" + +_MAX_URL_CHARS = 60_000 +"""How long an encoded query may be before no GET is worth attempting. + +Roughly where httpx gives up building the URL, with an error about URL +components rather than about coordinates. Past this a key has to travel in a +body, and a server without that route can only be told so. """ _UNTRIED = object() diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 824026d47..8bcdfb06f 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -160,10 +160,14 @@ def do_POST(self): if srv.cookie and self.headers.get("Cookie") != srv.cookie: self._send(401, b"unauthorized", endpoint="auth") return + raw = self.rfile.read(int(self.headers["Content-Length"])) # drained either way if not srv.post_fetch: # a server old enough not to know the route + # Answering without reading the body would leave it on the connection + # for the next request to read as a request line, which is a thing + # this stand-in does and a real server does not self._send(405, b"method not allowed", endpoint="fetch") return - body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + body = json.loads(raw) self._gather(srv, body["indices"]) @staticmethod @@ -534,7 +538,7 @@ def test_a_key_an_url_only_holds_once_encoded_goes_in_a_body(server): """ data = _incompressible((60, 70)) array, srv = server(data, chunks=(20, 25), blocks=(7, 9)) - key = list(range(60)) * 300 # ~51,000 chars as written, ~87,000 encoded + key = list(range(60)) * 16 # 2,723 chars as written, 4,656 encoded assert len(blosc2.c2array.key_to_indices(key)) < blosc2.c2array._MAX_QUERY_CHARS np.testing.assert_array_equal(array[key], data[key]) @@ -548,6 +552,21 @@ def test_a_server_without_the_post_route_says_so(server): array[list(range(60)) * 400] +def test_a_mid_sized_key_falls_back_to_a_get_where_there_is_no_post_route(server): + """Below what a client can build, a 405 is worth one GET rather than an error. + + The threshold is set by what a *server* will carry in a request line -- 8 KB + on nginx -- not by what httpx will build, which is far more. So keys now + take the POST route that a GET would have carried, and an older server + answering 405 must not turn those into a failure. + """ + data = _incompressible((60, 70)) + array, srv = server(data, chunks=(20, 25), blocks=(7, 9), post_fetch=False) + key = list(range(60)) * 16 # over the query threshold, under what httpx builds + assert len(blosc2.c2array.key_to_indices(key)) < blosc2.c2array._MAX_URL_CHARS + np.testing.assert_array_equal(array[key], data[key]) + + def test_a_reversed_slice_is_placed_on_the_span_it_covers(tmp_path, server, any_chunk_wants_blocks): """A fancy key next to a reversed slice fetches the blocks the slice names. From b15392fad10ac958f28e221742ef984a204bbcc2 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 24 Aug 2026 18:45:59 +0200 Subject: [PATCH 23/27] Say what `Traffic` counts, and make the samples do what they say From the review: - `Traffic.requests` and `nbytes` are `__slots__` with no docstrings, so `:members:` rendered nothing for the two attributes every example reads. A reader following `Traffic` from the guide -- which prints `Traffic(requests=4, nbytes=57767)` and says to take two readings and subtract -- found a page documenting `reset()` and nothing about the numbers, nor what unit they are in. Both slots now carry one, and the page renders them. - The guide's fourth bullet linked to `#seeing-what-it-saved`; the heading it meant is "Seeing byte savings". And the `S3Source` above it did not call `self.traffic.charge()`, so the canonical sample contradicted the rule the bullet next to it states. - `examples/c2array-traffic.py` divided by an unguarded `corner_bytes`, and compared the corner against `proxy[1]` -- a different chunk from the one holding it, and only "the chunk" at all if the dataset happens to be chunked one row at a time. It now reads the corner's own chunk, whole, through a proxy with an empty cache and by the array's own `chunks`. Checked against cat2.cloud: 0.055 MB against 1.309 MB, 23.8x. - `C2Array.__getitem__` and `slice` no longer advertise `Sequence[slice]`, which raises `IndexError` -- numpy stopped reading a list of slices as a tuple, and this follows it. The annotation now says what they do take, coordinates and masks included. Co-Authored-By: Claude Opus 5 --- doc/guides/remote_arrays.md | 6 ++++-- doc/reference/traffic.rst | 3 +++ examples/c2array-traffic.py | 17 +++++++++++------ src/blosc2/c2array.py | 18 ++++++++++++------ src/blosc2/proxy_source.py | 13 +++++++++++++ 5 files changed, 43 insertions(+), 14 deletions(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index 40d33381e..a4ac6c71f 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -202,7 +202,9 @@ class S3Source(blosc2.ByteRangeNDSource): Key=self._key, Range=f"bytes={offset}-{offset + size - 1}", ) - return answer["Body"].read() + data = answer["Body"].read() + self.traffic.charge(len(data)) + return data a = blosc2.Proxy(S3Source("bucket", "big.b2nd"), urlpath="cache.b2nd", mode="a") @@ -215,7 +217,7 @@ Four things to get right: - **Set up the transport before `super().__init__()`.** The base constructor calls `read_range()` straight away to read the file's header. - **`read_range()` must be thread-safe.** It is called from a thread pool so fetches can overlap. A boto3 *client* is fine; a `Session` or resource is not. - **Set `stamp` if you can.** It is what lets a cache tell that the remote has changed. Without it the cache is kept on geometry alone. -- **Charge what you read.** End `read_range()` with `self.traffic.charge(len(data))` and your source is counted like the built-in ones — see [Seeing what it saved](#seeing-what-it-saved). Skip it and `traffic` reads zero forever, which looks like a free transport rather than an uncounted one. +- **Charge what you read.** End `read_range()` with `self.traffic.charge(len(data))` and your source is counted like the built-in ones — see [Seeing byte savings](#seeing-byte-savings). Skip it and `traffic` reads zero forever, which looks like a free transport rather than an uncounted one. ## See also diff --git a/doc/reference/traffic.rst b/doc/reference/traffic.rst index 46bbd5883..f25e185f9 100644 --- a/doc/reference/traffic.rst +++ b/doc/reference/traffic.rst @@ -19,6 +19,9 @@ under ``traffic``. Take two readings and subtract, or :exclude-members: charge :member-order: groupwise + .. autoattribute:: requests + .. autoattribute:: nbytes + ``charge`` is left out of the members above: reading a counter is what a caller does with one. It is not private, though -- a transport of your own calls it from ``read_range()`` so that its reads are counted; see :ref:`ByteRangeNDSource` diff --git a/examples/c2array-traffic.py b/examples/c2array-traffic.py index db2e09f92..60b323b76 100644 --- a/examples/c2array-traffic.py +++ b/examples/c2array-traffic.py @@ -52,13 +52,18 @@ def cost(traffic): _ = proxy[0, :100, :100] print(f"the same slice again: {cost(proxy.traffic)}") -# A slice spanning the whole chunk is the chunk, and there is nothing to save. -proxy.traffic.reset() -whole = proxy[1] -whole_bytes = proxy.traffic.nbytes -print(f"whole chunk {whole.shape}: {cost(proxy.traffic)}") +# What that corner would have cost at chunk granularity: the very chunk holding +# it, read whole. By a proxy with an empty cache, and by the array's own +# `chunks` rather than a guess at them -- the corner's chunk is already partly +# in the cache above, and a dataset need not be chunked one row at a time. +fresh = blosc2.Proxy(blosc2.C2Array(path, urlbase=urlbase), mode="w") +fresh.traffic.reset() +whole = fresh[tuple(slice(0, c) for c in array.chunks)] +whole_bytes = fresh.traffic.nbytes +print(f"the chunk holding it {whole.shape}: {cost(fresh.traffic)}") -print(f"\nthe corner cost {whole_bytes / corner_bytes:.1f}x less than the chunk holding it") +if corner_bytes: # zero against a cache that already held it, and no ratio to give + print(f"\nthe corner cost {whole_bytes / corner_bytes:.1f}x less than the chunk holding it") # -- Without a proxy, a `C2Array` slice is one request the server answers with # just the box asked for; the same counter tallies it. diff --git a/src/blosc2/c2array.py b/src/blosc2/c2array.py index c5df6c6fa..9db9583eb 100644 --- a/src/blosc2/c2array.py +++ b/src/blosc2/c2array.py @@ -838,14 +838,17 @@ def save(self, urlpath: str, contiguous: bool = True, **kwargs) -> None: kwargs["mode"] = "w" self._to_b2object_carrier(**kwargs) - def __getitem__(self, slice_: int | slice | Sequence[slice]) -> np.ndarray: + def __getitem__(self, slice_: int | slice | tuple | Sequence[int] | np.ndarray) -> np.ndarray: """ Get a slice of the array (returning NumPy array). Parameters ---------- - slice_ : int, slice, tuple of ints and slices, or None - The slice to fetch. + slice_ : int, slice, tuple of ints and slices, sequence of ints, or ndarray + The slice to fetch. A sequence of integers or an integer or boolean + array gathers those coordinates, as numpy reads them. A *list of + slices* is not a key -- numpy stopped reading one as a tuple -- and + raises `IndexError` rather than being read as something else. Returns ------- @@ -890,14 +893,17 @@ def _fetch_params(self, key) -> dict: return {"slice_": slice_to_string(key)} return {"indices": indices} - def slice(self, slice_: int | slice | Sequence[slice]) -> blosc2.NDArray: + def slice(self, slice_: int | slice | tuple | Sequence[int] | np.ndarray) -> blosc2.NDArray: """ Get a slice of the array (returning blosc2 NDArray array). Parameters ---------- - slice_ : int, slice, tuple of ints and slices, or None - The slice to fetch. + slice_ : int, slice, tuple of ints and slices, sequence of ints, or ndarray + The slice to fetch. A sequence of integers or an integer or boolean + array gathers those coordinates, as numpy reads them. A *list of + slices* is not a key -- numpy stopped reading one as a tuple -- and + raises `IndexError` rather than being read as something else. Returns ------- diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index 06a1cc37e..398717502 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -136,6 +136,19 @@ class Traffic: __slots__ = ("_lock", "nbytes", "requests") + requests: int + """How many requests have carried data, cumulative. + + One per range read and one per chunk, including the frame's header, its + index and the block offsets -- everything the transport went out for. + """ + + nbytes: int + """How many bytes those requests carried, cumulative. + + Compressed bytes, as they crossed the wire, not what they decompress to. + """ + def __init__(self): self.requests = 0 self.nbytes = 0 From 26d95b5558faa9a70a497444cc87217e928e509c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 24 Aug 2026 18:46:06 +0200 Subject: [PATCH 24/27] Read an autosummary block past its options, not up to them `_AUTOSUMMARY_ENTRY` cannot match ` :toctree: autofiles/low_level/` -- a leading colon is not `\w` -- and the collector treated that as the end of the block, dropping every name listed under it. `reference/low_level.rst` uses `:toctree:`, so this was live rather than latent: `compress`, `pack`, `unpack_tensor` and the rest went uncollected, and only the committed stub files under `reference/autofiles/` -- which carry real `autofunction` directives -- kept the check from reporting them as undocumented. Verified by building with that directory stashed away: silent now, and it would have named all twelve before. Co-Authored-By: Claude Opus 5 --- doc/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/conf.py b/doc/conf.py index 1a4d5d7f6..f5a1aeee8 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -290,6 +290,7 @@ def process_sig(app, what, name, obj, options, signature, return_annotation): _CURRENTMODULE = re.compile(r"^\s*\.\.\s+(?:currentmodule|module)::\s*([\w.]+)") _AUTOSUMMARY = re.compile(r"^(\s*)\.\.\s+autosummary::") _AUTOSUMMARY_ENTRY = re.compile(r"^\s*~?([\w.]+)\s*$") +_AUTOSUMMARY_OPTION = re.compile(r"^\s*:[\w-]+:") # Public members deliberately left undocumented, so that the check below only # ever flags genuine omissions. Trimming this set is a standing invitation. @@ -355,7 +356,7 @@ def collect_documented_members(docdir): elif summary_indent is not None: # Inside an autosummary block: one bare (possibly dotted) name # per line, more indented than the directive itself. - if not line.strip(): + if not line.strip() or _AUTOSUMMARY_OPTION.match(line): continue entry = _AUTOSUMMARY_ENTRY.match(line) if entry is None or len(line) - len(line.lstrip()) <= summary_indent: From 8b175970d6da41bf6ad967ae40825d050fbc905d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 24 Aug 2026 19:40:40 +0200 Subject: [PATCH 25/27] Cross the cells of a fancy key by broadcasting, and group them by one sort `_fancy_cells` built its cross product with `itertools.product` over the basic dimensions and a `setdefault` per cell. Both are now numpy: the paired cells lie along one axis, each crossed dimension adds an axis of its own, and the sum of those is the cross product. `_group_cells` then turns the two flat columns into `{chunk: {block}}` with one `lexsort` and one `split`, so the grouping costs an entry per chunk where before it cost an operation per cell. Worth 1.1x to 1.4x on the keys measured -- a boolean mask over a 10k x 10k array with a full slice beside it goes from 1.098 ms to 0.805 ms for the same 10,000 cells. Modest, because the loop it replaces ran over combinations of the basic dimensions and vectorised the advanced column inside each, so there was less Python in it than it looked. The reason to make the change now is that the stepped-slice path wants exactly this machinery and has far more to gain from it: everything it names is crossed, with nothing vectorised inside. Verified identical to the previous implementation over 6,000 random keys -- paired coordinate arrays, boolean masks, advanced dimensions first, last and alone, stepped and reversed slices beside them, ragged chunk and block geometries -- of which 5,605 selected something and 395 selected nothing. Co-Authored-By: Claude Opus 5 --- src/blosc2/proxy.py | 47 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index f5598a3f0..2d7b424c2 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -1224,17 +1224,42 @@ def _fancy_cells(item, shape, chunks, blocks): return {} crossed.append(_dim_cells(dim, lo, hi, chunks, blocks)) - out = {} - # A crossed dimension contributes the same offset to every paired cell, so a - # combination of them is one addition over the whole column of cells - for combo in itertools.product(*crossed): - at_chunk = sum(c * chunk_strides[d] for d, (c, _) in zip(basic, combo, strict=True)) - at_block = sum(b * block_strides[d] for d, (_, b) in zip(basic, combo, strict=True)) - nchunks = (cell_chunk + at_chunk).tolist() - nblocks = (cell_block + at_block).tolist() - for nchunk, nblock in zip(nchunks, nblocks, strict=True): - out.setdefault(nchunk, set()).add(nblock) - return out + # A crossed dimension contributes the same offset to every paired cell, so the + # whole cross product is one broadcast sum rather than a loop over + # combinations: the paired cells lie along one axis and each crossed + # dimension adds an axis of its own, which is `itertools.product` done by + # numpy. For a mask over a large array the crossing, not the divmod above, + # is nearly all of what planning costs. + axes = [cell_chunk], [cell_block] + for i, dim in enumerate(basic): + at = np.asarray(sorted(crossed[i]), dtype=np.int64) + shape_i = [1] * (len(basic) + 1) + shape_i[i + 1] = -1 + axes[0].append((at[:, 0] * chunk_strides[dim]).reshape(shape_i)) + axes[1].append((at[:, 1] * block_strides[dim]).reshape(shape_i)) + grid = [1] * (len(basic) + 1) + grid[0] = -1 + nchunks = sum(axes[0][1:], axes[0][0].reshape(grid)).reshape(-1) + nblocks = sum(axes[1][1:], axes[1][0].reshape(grid)).reshape(-1) + + return _group_cells(nchunks, nblocks) + + +def _group_cells(nchunks: np.ndarray, nblocks: np.ndarray) -> dict[int, set]: + """``{chunk: {block}}`` out of two flat columns naming one cell each. + + One sort and one split, rather than a dict lookup per cell: the grouping then + costs an entry per *chunk*, where a large fancy key names cells in the + millions. Duplicates go the same way -- several coordinates of a mask share + a block, which is the whole reason blocks are worth naming. + """ + order = np.lexsort((nblocks, nchunks)) + nchunks, nblocks = nchunks[order], nblocks[order] + starts = np.flatnonzero(np.concatenate(([True], nchunks[1:] != nchunks[:-1]))) + return { + int(n): set(group.tolist()) + for n, group in zip(nchunks[starts], np.split(nblocks, starts[1:]), strict=True) + } def _item_spans(item, shape) -> list[tuple[int, int]] | None: From e094acda5dfaab0eca676eb97408280dfd08e03c Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Mon, 24 Aug 2026 19:43:09 +0200 Subject: [PATCH 26/27] Follow a step onto the blocks it selects, instead of covering its run A stepped slice was read as the run it lies in, which is a superset by the step's own factor wherever blocks are shallower than the step. That is not a corner case: the leading axis of an image stack, a time series or a tomography volume is routinely chunked one slab at a time, and then its block extent is 1 and every step over-fetches. On `kevlar-tomo.b2nd`'s geometry -- shape (10, 2167, 2070), chunks (1, 2167, 2070), blocks (1, 47, 2070) -- `p[::2]` read 470 blocks where 235 hold the data, `p[::5]` 470 where 94 do. `_dim_cells` now takes the step and the coordinate to anchor it on, and asks of each cell whether the lowest selected coordinate at or after its start falls at or before its end. Which needs the cell's extent, and that is the part worth reading twice: the last block of a chunk that is not a whole number of blocks is shorter than the rest, so a block ends where its chunk does when that comes first. Getting that wrong under-fetches, and a block that was never fetched reads as zeros, which nothing downstream can tell from data. Cells of a box are the cross product of what each dimension selects, since a box is: `_box_cells` crosses them by broadcasting and groups them with `_group_cells`, the machinery the previous commit built for `_fancy_cells`. A fancy key with a stepped slice beside it gets the same treatment, through `_fancy_cells`'s own crossed dimensions. Only a box that steps takes this path. A plain slice keeps the span intersection, whose `every` shortcut says a chunk is covered whole without counting its blocks out -- naming 10,000 cells to say `every` 100 times is the expensive way round, and a full read of a 10k x 10k array stays at 0.194 ms against 0.749 ms for the stepped path over the same grid. The threshold this seemed to want does not exist: gating on `step > block extent` would have left real over-fetch unclaimed in 32 of 144 measured cases below it, because blocks clipped by their chunk's extent or by the slice's own bounds are shorter than the nominal block. Exactness is also not free at the margin -- the superset was accidentally prefetching, so `p[::2]` followed by `p[1::2]` now pays twice where it used to pay once (334 KB against 335 KB in total, so level when the complement is read and 2x to 5x better when it is not). Fetching what nobody asked for is not a policy worth keeping inside the path that exists to stop doing it. Verified against numpy ground truth over 2,500 random stepped boxes, 1 to 3 dimensions, ragged chunk and block geometries, steps of both signs: no under-fetch, and no over-fetch either -- the placement is exact and not merely safe. Co-Authored-By: Claude Opus 5 --- doc/guides/remote_arrays.md | 2 +- src/blosc2/proxy.py | 80 ++++++++++++++++++++++++---- tests/ndarray/test_c2array_blocks.py | 36 +++++++++++-- 3 files changed, 104 insertions(+), 14 deletions(-) diff --git a/doc/guides/remote_arrays.md b/doc/guides/remote_arrays.md index a4ac6c71f..5b57dbfa6 100644 --- a/doc/guides/remote_arrays.md +++ b/doc/guides/remote_arrays.md @@ -73,7 +73,7 @@ It is never a loss. A slice wanting more than half a chunk's blocks is wanting t Fetches also overlap: a lazy proxy runs 8 at a time by default. Pass `max_concurrency=1` for a local protocol with no latency to hide. -A step other than 1 needs a proxy — a bare {ref}`C2Array` refuses one. Through a proxy a step is not followed but covered: `p[::2]` reads the blocks of the run it lies in, which is what `p[:]` over that run would read, and `[::-1]` costs what its forward twin does. So a step buys the bounds it is written with, and nothing beyond them. +A step other than 1 needs a proxy — a bare {ref}`C2Array` refuses one. Through a proxy it is placed on the block grid like any other key: `p[::2]` reads the blocks holding the coordinates it selects and no others, and `[::-1]` costs what its forward twin does. What that saves is `min(step, block extent along that axis)`, so it is nothing where blocks already span the axis whole — a step along the last dimension, usually — and the step's own factor where they do not. On `kevlar-tomo.b2nd`, whose blocks are one row deep, `[::2]` halves the read and `[::5]` cuts it fivefold. ### Seeing byte savings diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 2d7b424c2..1707983a9 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -410,7 +410,8 @@ def _plan(self, item): Placed exactly, by :func:`_fancy_cells`. ``("box", spans)`` A half-open run per dimension, for the caller to intersect with the - grid. A step is covered by the run it lies in rather than followed. + grid. Only for a key of plain slices: a step is placed exactly, as + cells, since the run it lies in holds blocks it selects nothing from. ``("chunks", nchunks)`` Every chunk, whole: a key nothing here can place, which is the granularity the proxy had before blocks and always a superset. @@ -429,8 +430,16 @@ def _plan(self, item): return everything if cells is not None: return "cells", cells + # A box, which the span path intersects with the grid more cheaply than + # naming its cells would -- a chunk it covers whole says `every` rather + # than counting its blocks out. Unless it steps: then the run is a + # superset by the step's own factor, and the cells are worth naming spans = _item_spans(item, shape) - return ("box", spans) if spans is not None else everything + if spans is None: + return everything + if _stepped(item, shape): + return "cells", _box_cells(item, shape, chunks, blocks) + return "box", spans def _wanted_chunks(self, item) -> list[int]: """The chunks *item* touches.""" @@ -1102,18 +1111,34 @@ def _whole_array(item) -> bool: return isinstance(item, tuple) and len(item) == 0 -def _dim_cells(dim, lo, hi, chunks, blocks) -> set[tuple[int, int]]: +def _dim_cells(dim, lo, hi, chunks, blocks, step=1, anchor=0) -> set[tuple[int, int]]: """The (chunk, block) pairs along *dim* that coordinates lo..hi inclusive fall in. Blocks partition a chunk and restart at every chunk boundary -- a chunk need not be a whole number of blocks -- so a block is located by where it sits inside its chunk, never by a running count across the array. + + With a *step*, only the cells actually holding a selected coordinate: those + are the ones congruent to *anchor* modulo *step*, whichever way the slice + runs, so a cell is wanted exactly when the lowest such coordinate at or after + its start falls at or before its end. Which needs the cell's *extent*, and + that is where this is easy to get wrong: the last block of a chunk that is + not a whole number of blocks is shorter than the rest, so a block ends where + its chunk does when that comes first. """ + exact = step > 1 cells = set() for c in range(lo // chunks[dim], hi // chunks[dim] + 1): first = max(lo - c * chunks[dim], 0) // blocks[dim] last = min(hi - c * chunks[dim], chunks[dim] - 1) // blocks[dim] - cells |= {(c, b) for b in range(first, last + 1)} + if not exact: + cells |= {(c, b) for b in range(first, last + 1)} + continue + for b in range(first, last + 1): + start = max(c * chunks[dim] + b * blocks[dim], lo) + end = min(c * chunks[dim] + min((b + 1) * blocks[dim], chunks[dim]) - 1, hi) + if anchor + -((anchor - start) // step) * step <= end: + cells.add((c, b)) return cells @@ -1215,14 +1240,12 @@ def _fancy_cells(item, shape, chunks, blocks): crossed = [] for dim in basic: start, stop, step = key[dim].indices(shape[dim]) - # A step is not followed: the span it lies in is a superset of it, and a - # superset is the one kind of wrong answer this may give. A reversed - # slice runs from stop + 1 up to start, and covers the same span its - # forward twin does -- reading it as empty would fetch nothing at all + # A reversed slice runs from stop + 1 up to start, and covers the same + # run its forward twin does -- reading it as empty would fetch nothing lo, hi = (stop + 1, start) if step < 0 else (start, stop - 1) if hi < lo: return {} - crossed.append(_dim_cells(dim, lo, hi, chunks, blocks)) + crossed.append(_dim_cells(dim, lo, hi, chunks, blocks, abs(step), start)) # A crossed dimension contributes the same offset to every paired cell, so the # whole cross product is one broadcast sum rather than a loop over @@ -1262,6 +1285,45 @@ def _group_cells(nchunks: np.ndarray, nblocks: np.ndarray) -> dict[int, set]: } +def _stepped(item, shape) -> bool: + """Whether *item* is a box that steps, and so is worth placing exactly.""" + from blosc2.utils import process_key + + key, _ = process_key(item, shape) + return any(isinstance(k, slice) and abs(k.indices(shape[d])[2]) > 1 for d, k in enumerate(key)) + + +def _box_cells(item, shape, chunks, blocks) -> dict[int, set]: + """``{chunk: {block}}`` a stepped box touches, exactly. + + The cells of a box are the cross product of what each dimension selects, + because a box is: a cell holds a selected coordinate exactly when every one + of its dimensions does. So this is :func:`_fancy_cells`'s crossing with + nothing paired to cross against, and :func:`_dim_cells` following the step + is what makes each dimension's answer exact rather than the run it lies in. + """ + from blosc2.utils import process_key + + key, _ = process_key(item, shape) + chunk_grid = [math.ceil(s / c) for s, c in zip(shape, chunks, strict=True)] + blocks_in_chunk = [math.ceil(c / b) for c, b in zip(chunks, blocks, strict=True)] + chunk_strides, block_strides = _strides(chunk_grid), _strides(blocks_in_chunk) + + at_chunk, at_block = [], [] + for dim, k in enumerate(key): + start, stop, step = k.indices(shape[dim]) + lo, hi = (stop + 1, start) if step < 0 else (start, stop - 1) + if hi < lo: + return {} + cells = np.array(sorted(_dim_cells(dim, lo, hi, chunks, blocks, abs(step), start)), dtype=np.int64) + axis = [1] * len(shape) + axis[dim] = -1 + at_chunk.append((cells[:, 0] * chunk_strides[dim]).reshape(axis)) + at_block.append((cells[:, 1] * block_strides[dim]).reshape(axis)) + + return _group_cells(sum(at_chunk).reshape(-1), sum(at_block).reshape(-1)) + + def _item_spans(item, shape) -> list[tuple[int, int]] | None: """The half-open run *item* covers along every dimension, or None if it is no box. diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 8bcdfb06f..8006595eb 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -594,12 +594,40 @@ def test_a_stepped_slice_reads_the_run_it_lies_in(tmp_path, server, any_chunk_wa np.testing.assert_array_equal(p[key], data[key]) -def test_a_step_costs_its_run_and_not_the_whole_array(tmp_path, server, any_chunk_wants_blocks): - """A stepped slice bounded to one corner reads that corner, not every chunk. +def test_a_step_is_placed_on_the_blocks_it_selects(tmp_path, server, any_chunk_wants_blocks): + """A step reads the blocks holding its coordinates, not the run they lie in. - Falling back to "every chunk, whole" would also be correct, and is what a key - this cannot place gets; the run a step lies in is a far smaller superset. + With blocks of extent 1 along the stepped axis -- which is how an image stack + or a tomography volume is chunked, `kevlar-tomo.b2nd` included -- the run a + step lies in is the whole array, and covering it over-fetches by the step. """ + data = _incompressible((60, 200)) + array, srv = server(data, chunks=(1, 200), blocks=(1, 20)) + blocks_per_chunk = 10 + p = blosc2.Proxy(array, urlpath=str(tmp_path / "stepped-exact.b2nd"), mode="w") + for step in (2, 3, 5): + key = np.s_[::step] + wanted = p._wanted_blocks(key) + assert sum(len(bs) for bs in wanted.values()) == len(range(0, 60, step)) * blocks_per_chunk + assert sorted(wanted) == list(range(0, 60, step)) # and nothing in between + np.testing.assert_array_equal(p[key], data[key]) + + +def test_an_unstepped_box_still_says_every_rather_than_counting(tmp_path, server): + """The exact path is for steps only; a plain slice keeps the cheaper one. + + Naming a covered chunk's blocks one by one is the expensive way to say + `every`, and that shortcut is what keeps a large box cheap to plan. + """ + data = _incompressible((60, 200)) + array, srv = server(data, chunks=(1, 200), blocks=(1, 20)) + p = blosc2.Proxy(array, urlpath=str(tmp_path / "unstepped.b2nd"), mode="w") + assert p._plan(np.s_[0:10])[0] == "box" + assert isinstance(next(iter(p._wanted_blocks(np.s_[0:10]).values())), range) + + +def test_a_step_costs_its_run_and_not_the_whole_array(tmp_path, server, any_chunk_wants_blocks): + """A stepped slice bounded to one corner reads that corner, not every chunk.""" data = _incompressible((200, 200)) array, srv = server(data, chunks=(100, 200), blocks=(10, 20)) p = blosc2.Proxy(array, urlpath=str(tmp_path / "corner.b2nd"), mode="w") From 2fe5a4f52291d11f7e601afdc345dbbe0895327d Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 25 Aug 2026 09:02:35 +0200 Subject: [PATCH 27/27] Price the requests a fragmented chunk costs a one-range-at-a-time transport Placing a step exactly makes a selection sparser, and where every range is its own request that is paid for in round trips: against S3 with 3.22 MB chunks and 16 blocks each, `p[::128]` on an axis of 64-deep blocks became 111 requests where whole chunks were 20. In-region (15 ms, 90 MB/s) that ran 0.54x against reading the chunks whole -- slower than before the previous commit, which covered such a step by its run and so wanted every block. From Europe (240 ms, 3.5 MB/s) the same key ran 1.35x, because there the halved bytes outweigh the round trips. No single answer is right for both networks; `_runs_pay` takes the one that is never worse than the whole-chunk read. The threshold is the module's own pricing rather than a constant fitted to those two measurements: a round trip costs `BLOCK_MIN_CBYTES`, so a chunk split into R ranges must be worth R of them. For a 3.22 MB chunk that allows 3, which is what a sweep of fixed caps also picked -- but this scales with the chunk instead of being a number that happened to fit. Only where `max_ranges <= 1`. A transport that batches ranges wants the split and always did: the same 111 ranges are 4 requests to Caterva2, against 20 for whole chunks, at half the bytes -- better on both counts, so it never comes here. Nothing that does not fragment is touched, which the measurements bear out: the control slice holds 3.38x from Europe and 1.34x in-region, and the stepped keys now sit at 1.00x and 1.01x rather than 0.54x. `wants_blocks` takes the run count as a fourth argument, under the `wants_wave` opt-in that already carries the wave; `_runs` counts runs of consecutive block indices, which is what `block_plan` coalesces, without reading the layout that would say exactly. A budget of zero -- which tests set to force the block path -- prices nothing and so forbids nothing. Co-Authored-By: Claude Opus 5 --- src/blosc2/proxy.py | 24 ++++++++++++------ src/blosc2/proxy_source.py | 37 +++++++++++++++++++++++++--- tests/ndarray/test_c2array_blocks.py | 4 +-- 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/src/blosc2/proxy.py b/src/blosc2/proxy.py index 1707983a9..5f7481c26 100644 --- a/src/blosc2/proxy.py +++ b/src/blosc2/proxy.py @@ -716,15 +716,15 @@ def _run(self, func, tasks: list, max_concurrency: int | None): def _asking_blocks(self, missing: dict, wave: dict | None) -> dict: """The chunks of *missing* the source wants taken apart, in order. - The wave goes with the question only to a source that says it takes one - (``wants_wave``), which is an opt-in in the same shape as ``max_ranges`` - and read the same way: a `wants_blocks` written to the two-argument - protocol raises `TypeError` on being handed a third. + The wave and the run count go with the question only to a source that + says it takes them (``wants_wave``), which is an opt-in in the same shape + as ``max_ranges`` and read the same way: a `wants_blocks` written to the + two-argument protocol raises `TypeError` on being handed more. """ wants = self.src.wants_blocks - if wave is not None and getattr(self.src, "wants_wave", False): - return {n: bs for n, bs in missing.items() if wants(n, len(bs), wave)} - return {n: bs for n, bs in missing.items() if wants(n, len(bs))} + if not getattr(self.src, "wants_wave", False): + return {n: bs for n, bs in missing.items() if wants(n, len(bs))} + return {n: bs for n, bs in missing.items() if wants(n, len(bs), wave, _runs(sorted(bs)))} def _fetch_by_block(self, item, max_concurrency: int | None): """`fetch()` against a source that can serve single blocks. @@ -1102,6 +1102,16 @@ def fields(self) -> dict: """A key that selects something, but nothing this can reduce to cells of the grid.""" +def _runs(nblocks: Sequence[int]) -> int: + """How many ranges *nblocks* will coalesce into, near enough to price them. + + Blocks land in the frame roughly in index order, so consecutive indices are + the runs `block_plan` merges; this counts them without reading the layout + that would say exactly. What they cost is `ByteRangeNDSource._runs_pay`. + """ + return 1 + sum(later != earlier + 1 for earlier, later in itertools.pairwise(nblocks)) + + def _whole_array(item) -> bool: """True where *item* is the empty tuple, the key that asks for everything. diff --git a/src/blosc2/proxy_source.py b/src/blosc2/proxy_source.py index 398717502..8f825d5fb 100644 --- a/src/blosc2/proxy_source.py +++ b/src/blosc2/proxy_source.py @@ -982,7 +982,13 @@ def read_ranges(self, spans: Sequence[tuple[int, int]]) -> list[bytes]: """ return [self.read_range(offset, size) for offset, size in spans] - def wants_blocks(self, nchunk: int, nwanted: int, wave: Mapping[int, int] | None = None) -> bool: + def wants_blocks( + self, + nchunk: int, + nwanted: int, + wave: Mapping[int, int] | None = None, + nruns: int | None = None, + ) -> bool: """Whether fetching *nwanted* blocks of a chunk beats fetching all of it. Answered without reading anything, so a chunk that says no costs exactly @@ -992,7 +998,9 @@ def wants_blocks(self, nchunk: int, nwanted: int, wave: Mapping[int, int] | None *wave* is the whole fetch this chunk belongs to, ``{nchunk: nwanted}``, which a transport that batches ranges is asked with; see - :meth:`_wave_saves` for what it is used for and why. + :meth:`_wave_saves` for what it is used for and why. *nruns* is how many + ranges those blocks will coalesce into, which is what they cost where + every range is its own request. """ if nwanted > self.blocks_per_chunk * BLOCK_MAX_FRACTION: return False @@ -1005,9 +1013,31 @@ def wants_blocks(self, nchunk: int, nwanted: int, wave: Mapping[int, int] | None if int(offsets[nchunk]) < 0: return False # a run-length chunk has no bytes in the file to skip if wave is None or self.max_ranges <= 1: + if not self._runs_pay(int(extents[nchunk]), nruns): + return False return int(extents[nchunk]) >= BLOCK_MIN_CBYTES return self._wave_saves(wave) >= BLOCK_MIN_CBYTES + def _runs_pay(self, cbytes: int, nruns: int | None) -> bool: + """Whether a chunk is worth splitting into *nruns* separate requests. + + Where each range is its own request, a chunk split into R of them pays R + round trips against one, so it has to be worth R times what one costs -- + the same :data:`BLOCK_MIN_CBYTES` the test below spends, once per + request rather than once per chunk. + + A scattered key fragments; a step past a block's extent fragments by the + step. Measured against S3 with 3.22 MB chunks, block mode ran 0.54x on + such a key in-region (15 ms, 90 MB/s) and 1.35x from Europe (240 ms, + 3.5 MB/s), so no single answer is right for both networks and this takes + the one that is never worse than reading the chunks whole. Transports + that carry many ranges per request never come here: Caterva2 collapses + the same 111 ranges into 4 requests, and wants the split. + """ + if nruns is None or self.max_ranges > 1 or BLOCK_MIN_CBYTES <= 0: + return True # a budget of zero prices nothing, and forbids nothing + return nruns <= max(1, cbytes // BLOCK_MIN_CBYTES) + def _chunk_cap(self) -> int: """The most one chunk of this frame can weigh, without reading any of it. @@ -1068,7 +1098,8 @@ def _wave_saves(self, wave: Mapping[int, int]) -> int: then two requests per chunk against one, both sides scale with the chunks touched, and a dataset of 193 KB chunks measured 0.70x against S3 out to 121 of them. Hence the ``max_ranges`` gate above, which leaves - that path deciding exactly as it did. + that path deciding exactly as it did -- and :meth:`_runs_pay`, which + prices the requests a fragmented chunk costs it. Blocks of a chunk are close enough in size to weigh what is wanted by counting them, the same approximation :data:`BLOCK_MAX_FRACTION` makes, diff --git a/tests/ndarray/test_c2array_blocks.py b/tests/ndarray/test_c2array_blocks.py index 8006595eb..4ca8deb40 100644 --- a/tests/ndarray/test_c2array_blocks.py +++ b/tests/ndarray/test_c2array_blocks.py @@ -663,7 +663,7 @@ def test_a_two_argument_wants_blocks_is_never_handed_the_wave(): """`max_ranges` and `wants_wave` are opt-ins of their own. A source that batches ranges but was written to the two-argument protocol - used to be called with three, and raised `TypeError` on its first fetch. + used to be called with more, and raised `TypeError` on its first fetch. """ class TwoArg: @@ -675,7 +675,7 @@ def wants_blocks(self, nchunk, nwanted): class ThreeArg(TwoArg): wants_wave = True - def wants_blocks(self, nchunk, nwanted, wave=None): + def wants_blocks(self, nchunk, nwanted, wave=None, nruns=None): return wave is not None def asking(src, wave):