obstore-based Store implementation (#1661)

Implementation of the Zarr Store API using obstore.

---------

Co-authored-by: Davis Bennett <[email protected]>
Co-authored-by: Joe Hamman <[email protected]>
Co-authored-by: Deepak Cherian <[email protected]>
Co-authored-by: Max Jones <[email protected]>
Co-authored-by: Deepak Cherian <[email protected]>
Co-authored-by: Tom Augspurger <[email protected]>

Author: 6 people

.pre-commit-config.yaml

@@ -8,9 +8,9 @@ repos:
   - repo: https://github.com./astral-sh/ruff-pre-commit
     rev: v0.9.9
     hooks:
-    - id: ruff
-      args: ["--fix", "--show-fixes"]
-    - id: ruff-format
+      - id: ruff
+        args: ["--fix", "--show-fixes"]
+      - id: ruff-format
   - repo: https://github.com./codespell-project/codespell
     rev: v2.4.1
     hooks:
@@ -19,8 +19,8 @@ repos:
   - repo: https://github.com./pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
-    - id: check-yaml
-    - id: trailing-whitespace
+      - id: check-yaml
+      - id: trailing-whitespace
   - repo: https://github.com./pre-commit/mirrors-mypy
     rev: v1.15.0
     hooks:
@@ -31,9 +31,10 @@ repos:
           - packaging
           - donfig
           - numcodecs[crc32c]
-          - numpy==2.1  # until https://github.com./numpy/numpy/issues/28034 is resolved
+          - numpy==2.1 # until https://github.com./numpy/numpy/issues/28034 is resolved
           - typing_extensions
           - universal-pathlib
+          - obstore>=0.5.1
           # Tests
           - pytest
   - repo: https://github.com./scientific-python/cookie

changes/1661.feature.rst

@@ -0,0 +1 @@
+Add experimental ObjectStore storage class based on obstore.

docs/conf.py

@@ -369,6 +369,7 @@ def setup(app: sphinx.application.Sphinx) -> None:
     "python": ("https://docs.python.org/3/", None),
     "numpy": ("https://numpy.org/doc/stable/", None),
     "numcodecs": ("https://numcodecs.readthedocs.io/en/stable/", None),
+    "obstore": ("https://developmentseed.org/obstore/latest/", None),
 }
 
 

docs/user-guide/storage.rst

@@ -47,7 +47,7 @@ Explicit Store Creation
 
 In some cases, it may be helpful to create a store instance directly. Zarr-Python offers four
 built-in store: :class:`zarr.storage.LocalStore`, :class:`zarr.storage.FsspecStore`,
-:class:`zarr.storage.ZipStore`, and :class:`zarr.storage.MemoryStore`.
+:class:`zarr.storage.ZipStore`, :class:`zarr.storage.MemoryStore`, and :class:`zarr.storage.ObjectStore`.
 
 Local Store
 ~~~~~~~~~~~
@@ -99,6 +99,42 @@ Zarr data (metadata and chunks) to a dictionary.:
    >>> zarr.create_array(store=store, shape=(2,), dtype='float64')
    <Array memory://... shape=(2,) dtype=float64>
 
+Object Store
+~~~~~~~~~~~~
+
+:class:`zarr.storage.ObjectStore` stores the contents of the Zarr hierarchy using any ObjectStore
+`storage implementation <https://developmentseed.org/obstore/latest/api/store/>`_, including AWS S3 (:class:`obstore.store.S3Store`), Google Cloud Storage (:class:`obstore.store.GCSStore`), and Azure Blob Storage (:class:`obstore.store.AzureStore`). This store is backed by `obstore <https://developmentseed.org/obstore/latest/>`_, which
+builds on the production quality Rust library `object_store <https://docs.rs/object_store/latest/object_store/>`_.
+
+
+   >>> from zarr.storage import ObjectStore
+   >>> from obstore.store import MemoryStore
+   >>>
+   >>> store = ObjectStore(MemoryStore())
+   >>> zarr.create_array(store=store, shape=(2,), dtype='float64')
+   <Array object_store://... shape=(2,) dtype=float64>
+
+Here's an example of using ObjectStore for accessing remote data:
+
+   >>> from zarr.storage import ObjectStore
+   >>> from obstore.store import S3Store
+   >>>
+   >>> s3_store = S3Store('noaa-nwm-retro-v2-zarr-pds', skip_signature=True, region="us-west-2")
+   >>> store = zarr.storage.ObjectStore(store=s3_store, read_only=True)
+   >>> group = zarr.open_group(store=store, mode='r')
+   >>> group.info
+   Name        :
+   Type        : Group
+   Zarr format : 2
+   Read-only   : True
+   Store type  : ObjectStore
+   No. members : 12
+   No. arrays  : 12
+   No. groups  : 0
+
+.. warning::
+   The :class:`zarr.storage.ObjectStore` class is experimental.
+
 .. _user-guide-custom-stores:
 
 Developing custom stores

pyproject.toml

@@ -64,6 +64,7 @@ keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"]
 # User extras
 remote = [
     "fsspec>=2023.10.0",
+    "obstore>=0.5.1",
 ]
 gpu = [
     "cupy-cuda12x",
@@ -114,6 +115,12 @@ Discussions = "https://github.com./zarr-developers/zarr-python/discussions"
 Documentation = "https://zarr.readthedocs.io/"
 Homepage = "https://github.com./zarr-developers/zarr-python"
 
+[dependency-groups]
+dev = [
+    "ipykernel>=6.29.5",
+    "pip>=25.0.1",
+]
+
 [tool.coverage.report]
 exclude_lines = [
     "pragma: no cover",
@@ -129,7 +136,9 @@ omit = [
 
 [tool.hatch]
 version.source = "vcs"
-build.hooks.vcs.version-file = "src/zarr/_version.py"
+
+[tool.hatch.build]
+hooks.vcs.version-file = "src/zarr/_version.py"
 
 [tool.hatch.envs.test]
 dependencies = [
@@ -165,7 +174,7 @@ run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tes
 list-env = "pip list"
 
 [tool.hatch.envs.doctest]
-features = ["test", "optional"]
+features = ["test", "optional", "remote"]
 description = "Test environment for doctests"
 
 [tool.hatch.envs.doctest.scripts]
@@ -211,6 +220,7 @@ dependencies = [
     'universal_pathlib @ git+https://github.com./fsspec/universal_pathlib',
     'typing_extensions @ git+https://github.com./python/typing_extensions',
     'donfig @ git+https://github.com./pytroll/donfig',
+    'obstore @ git+https://github.com./developmentseed/obstore@main#subdirectory=obstore',
     # test deps
     'zarr[test]',
 ]

src/zarr/storage/__init__.py

@@ -8,6 +8,7 @@
 from zarr.storage._local import LocalStore
 from zarr.storage._logging import LoggingStore
 from zarr.storage._memory import GpuMemoryStore, MemoryStore
+from zarr.storage._obstore import ObjectStore
 from zarr.storage._wrapper import WrapperStore
 from zarr.storage._zip import ZipStore
 
@@ -17,6 +18,7 @@
     "LocalStore",
     "LoggingStore",
     "MemoryStore",
+    "ObjectStore",
     "StoreLike",
     "StorePath",
     "WrapperStore",