diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 278c8fa877..7192b6d78a 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -31,9 +31,9 @@ jobs: with: python-version: "3.11" - name: Install Hatch - run: | - python -m pip install --upgrade pip - pip install hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + with: + version: '1.16.5' - name: Run the benchmarks uses: CodSpeedHQ/action@v4 with: diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml index 6011610681..337a2d2578 100644 --- a/.github/workflows/gpu_test.yml +++ b/.github/workflows/gpu_test.yml @@ -19,13 +19,12 @@ concurrency: jobs: test: - name: py=${{ matrix.python-version }}, deps=${{ matrix.dependency-set }} + name: py=${{ matrix.python-version }} runs-on: gpu-runner strategy: matrix: python-version: ['3.11'] - dependency-set: ["minimal"] steps: - uses: actions/checkout@v6 @@ -54,17 +53,17 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - - name: Install Hatch and CuPy - run: | - python -m pip install --upgrade pip - pip install hatch + - name: Install Hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + with: + version: '1.16.5' - name: Set Up Hatch Env run: | - hatch env create gputest.py${{ matrix.python-version }}-${{ matrix.dependency-set }} - hatch env run -e gputest.py${{ matrix.python-version }}-${{ matrix.dependency-set }} list-env + hatch env create gputest.py${{ matrix.python-version }} + hatch env run -e gputest.py${{ matrix.python-version }} list-env - name: Run Tests run: | - hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run-coverage-gpu + hatch env run --env gputest.py${{ matrix.python-version }} run-coverage - name: Upload coverage uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3 # v5.3.1 diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml index abf7c5f4ce..d8a0acdfd4 100644 --- a/.github/workflows/hypothesis.yaml +++ b/.github/workflows/hypothesis.yaml @@ -41,9 +41,9 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install Hatch - run: | - python -m pip install --upgrade pip - pip install hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + with: + version: '1.16.5' - name: Set Up Hatch Env run: | hatch env create test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} diff --git a/.github/workflows/nightly_wheels.yml b/.github/workflows/nightly_wheels.yml index dbbc0354e9..82a8676834 100644 --- a/.github/workflows/nightly_wheels.yml +++ b/.github/workflows/nightly_wheels.yml @@ -22,10 +22,10 @@ jobs: with: python-version: '3.13' - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - pip install hatch + - name: Install Hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + with: + version: '1.16.5' - name: Build wheel and sdist run: hatch build diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index bca01466bb..054bc00e45 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -21,10 +21,10 @@ jobs: with: python-version: '3.11' - - name: Install PyBuild - run: | - python -m pip install --upgrade pip - pip install hatch + - name: Install Hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + with: + version: '1.16.5' - name: Build wheel and sdist run: hatch build - uses: actions/upload-artifact@v6 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15fe94d3f9..bd6a3a91a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,9 +51,9 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install Hatch - run: | - python -m pip install --upgrade pip - pip install hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + with: + version: '1.16.5' - name: Set Up Hatch Env run: | hatch env create test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} @@ -93,9 +93,9 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' - name: Install Hatch - run: | - python -m pip install --upgrade pip - pip install hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + with: + version: '1.16.5' - name: Set Up Hatch Env run: | hatch env create ${{ matrix.dependency-set }} @@ -122,9 +122,9 @@ jobs: python-version: '3.13' cache: 'pip' - name: Install Hatch - run: | - python -m pip install --upgrade pip - pip install hatch + uses: pypa/hatch@257e27e51a6a5616ed08a39a408a21c35c9931bc + with: + version: '1.16.5' - name: Set Up Hatch Env run: | hatch run doctest:pip list diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 894778c5a4..1edd099ebd 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -5,6 +5,9 @@ build: tools: python: "3.12" jobs: + install: + - pip install --upgrade pip + - pip install .[remote] --group docs pre_build: - | if [ "$READTHEDOCS_VERSION_TYPE" != "tag" ]; @@ -16,11 +19,3 @@ build: - mkdocs build --strict --site-dir $READTHEDOCS_OUTPUT/html mkdocs: configuration: mkdocs.yml - -python: - install: - - method: pip - path: . - extra_requirements: - - docs - - remote diff --git a/changes/3728.misc.md b/changes/3728.misc.md new file mode 100644 index 0000000000..a3cbb8d3f0 --- /dev/null +++ b/changes/3728.misc.md @@ -0,0 +1 @@ +Move development dependencies (`test`, `remote_tests`, `docs`, `dev`) from optional dependencies to [dependency groups](https://packaging.python.org/en/latest/specifications/dependency-groups/). This may cause breakage for anyone who used e.g. `pip install zarr[test]` to get access to test dependencies. To install these dependency groups from a local checkout, use `pip install --group ` (pip 25.1+) or `uv run --group `. \ No newline at end of file diff --git a/docs/contributing.md b/docs/contributing.md index e4f341a8b3..a4bbaafbd5 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -64,7 +64,7 @@ hatch env show # list all available environments To verify that your development environment is working, you can run the unit tests for one of the test environments, e.g.: ```bash -hatch env run --env test.py3.12-2.2-optional run-pytest +hatch env run --env test.py3.12-optional run ``` ### Creating a branch @@ -102,7 +102,7 @@ Again, any conflicts need to be resolved before submitting a pull request. Zarr includes a suite of unit tests. The simplest way to run the unit tests is to activate your development environment (see [creating a development environment](#creating-a-development-environment) above) and invoke: ```bash -hatch env run --env test.py3.12-2.2-optional run-pytest +hatch env run --env test.py3.12-optional run ``` All tests are automatically run via GitHub Actions for every pull request and must pass before code can be accepted. Test coverage is also collected automatically via the Codecov service. @@ -164,7 +164,7 @@ If you would like to skip the failing checks and push the code for further discu Zarr strives to maintain 100% test coverage under the latest Python stable release. Both unit tests and docstring doctests are included when computing coverage. Running: ```bash -hatch env run --env test.py3.12-2.2-optional run-coverage +hatch env run --env test.py3.12-optional run-coverage ``` will automatically run the test suite with coverage and produce an XML coverage report. This should be 100% before code can be accepted into the main code base. @@ -172,7 +172,7 @@ will automatically run the test suite with coverage and produce an XML coverage You can also generate an HTML coverage report by running: ```bash -hatch env run --env test.py3.12-2.2-optional run-coverage-html +hatch env run --env test.py3.12-optional run-coverage-html ``` When submitting a pull request, coverage will also be collected across all supported Python versions via the Codecov service, and will be reported back within the pull request. Codecov coverage must also be 100% before code can be accepted. diff --git a/pyproject.toml b/pyproject.toml index 068caa1f0d..02ebdc4176 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling>=1.27.0", "hatch-vcs"] +requires = ["hatchling>=1.29.0", "hatch-vcs"] build-backend = "hatchling.build" [tool.hatch.build.targets.sdist] @@ -61,7 +61,7 @@ license-files = ["LICENSE.txt"] keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] [project.optional-dependencies] -# User extras +# User-facing extras (shipped in package metadata) remote = [ "fsspec>=2023.10.0", "obstore>=0.5.1", @@ -70,30 +70,43 @@ gpu = [ "cupy-cuda12x", ] cli = ["typer"] -# Testing extras +optional = ["rich", "universal-pathlib"] + +[project.scripts] +zarr = "zarr._cli.cli:app" + + +[project.urls] +issues = "https://github.com/zarr-developers/zarr-python/issues" +changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" +Discussions = "https://github.com/zarr-developers/zarr-python/discussions" +documentation = "https://zarr.readthedocs.io/" +homepage = "https://github.com/zarr-developers/zarr-python" + +[dependency-groups] test = [ "coverage>=7.10", "pytest", "pytest-asyncio", "pytest-cov", "pytest-accept", - 'numpydoc', + "numpydoc", "hypothesis", "pytest-xdist", "pytest-benchmark", "pytest-codspeed", - "packaging", "tomlkit", "uv", ] -remote_tests = [ - 'zarr[remote]', +remote-tests = [ + {include-group = "test"}, + "fsspec>=2023.10.0", + "obstore>=0.5.1", "botocore", "s3fs>=2023.10.0", "moto[s3,server]", "requests", ] -optional = ["rich", "universal-pathlib"] docs = [ # Doc building "mkdocs-material[imaging]>=9.6.14", @@ -106,40 +119,23 @@ docs = [ "griffe-inherited-docstrings", "ruff", # Changelog generation - 'towncrier', + "towncrier", # Optional dependencies to run examples - 'numcodecs[msgpack]', - 'rich', - 's3fs>=2023.10.0', - 'astroid<4', - 'pytest' + "numcodecs[msgpack]", + "rich", + "s3fs>=2023.10.0", + "astroid<4", + "pytest", ] -# All development extras dev = [ - "zarr[optional]", - "zarr[test]", - "zarr[remote_tests]", - "zarr[docs]", + {include-group = "test"}, + {include-group = "remote-tests"}, + {include-group = "docs"}, + "rich", + "universal-pathlib", "mypy", ] -[project.scripts] -zarr = "zarr._cli.cli:app" - - -[project.urls] -issues = "https://github.com/zarr-developers/zarr-python/issues" -changelog = "https://zarr.readthedocs.io/en/stable/release-notes.html" -Discussions = "https://github.com/zarr-developers/zarr-python/discussions" -documentation = "https://zarr.readthedocs.io/" -homepage = "https://github.com/zarr-developers/zarr-python" - -[dependency-groups] -dev = [ - "ipykernel>=6.29.5", - "pip>=25.0.1", -] - [tool.coverage.report] exclude_also = [ 'if TYPE_CHECKING:', @@ -158,7 +154,7 @@ version.source = "vcs" hooks.vcs.version-file = "src/zarr/_version.py" [tool.hatch.envs.test] -features = ["test"] +dependency-groups = ["test"] [tool.hatch.envs.test.env-vars] # Required to test with a pytest plugin; see https://pytest-cov.readthedocs.io/en/latest/plugins.html @@ -171,16 +167,19 @@ python = ["3.11", "3.12", "3.13"] deps = ["minimal", "optional"] [tool.hatch.envs.test.overrides] -matrix.deps.dependencies = [ - {value = "zarr[remote, remote_tests, test, optional, cli]", if = ["optional"]} +matrix.deps.features = [ + {value = "remote", if = ["optional"]}, + {value = "optional", if = ["optional"]}, + {value = "cli", if = ["optional"]}, +] +matrix.deps.dependency-groups = [ + {value = "remote-tests", if = ["optional"]}, ] [tool.hatch.envs.test.scripts] run-coverage = "pytest --cov-config=pyproject.toml --cov=src --cov-append --cov-report xml --junitxml=junit.xml -o junit_family=legacy" run-coverage-html = "pytest --cov-config=pyproject.toml --cov=src --cov-append --cov-report html" -run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=src --cov-append --cov-report xml --junitxml=junit.xml -o junit_family=legacy" run = "run-coverage --no-cov --ignore tests/benchmarks" -run-pytest = "run" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" run-hypothesis = "run-coverage -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" @@ -189,27 +188,22 @@ list-env = "pip list" [tool.hatch.envs.gputest] template = "test" -dependencies = [ +extra-dependencies = [ "universal_pathlib", ] -features = ["test", "gpu"] +features = ["gpu"] [[tool.hatch.envs.gputest.matrix]] python = ["3.11", "3.12", "3.13"] -version = ["minimal"] [tool.hatch.envs.gputest.scripts] -run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy --ignore tests/benchmarks" +run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=src --cov-report xml --junitxml=junit.xml -o junit_family=legacy --ignore tests/benchmarks" run = "run-coverage --no-cov" -run-verbose = "run-coverage --verbose" -run-mypy = "mypy src" -run-hypothesis = "run-coverage --hypothesis-profile ci --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*" -list-env = "pip list" [tool.hatch.envs.upstream] template = 'test' python = "3.13" -dependencies = [ +extra-dependencies = [ 'packaging @ git+https://github.com/pypa/packaging', 'numpy', # from scientific-python-nightly-wheels 'numcodecs @ git+https://github.com/zarr-developers/numcodecs', @@ -218,8 +212,6 @@ dependencies = [ 'typing_extensions @ git+https://github.com/python/typing_extensions', 'donfig @ git+https://github.com/pytroll/donfig', 'obstore @ git+https://github.com/developmentseed/obstore@main#subdirectory=obstore', - # test deps - 'zarr[test]', ] [tool.hatch.envs.upstream.env-vars] @@ -234,8 +226,9 @@ See Spec 0000 for details and drop schedule: https://scientific-python.org/specs """ template = "test" python = "3.11" -dependencies = [ - 'zarr[remote]', +features = ["remote"] +dependency-groups = ["remote-tests"] +extra-dependencies = [ 'packaging==22.*', 'numpy==2.0.*', 'numcodecs==0.14.*', # 0.14 needed for zarr3 codecs @@ -245,13 +238,11 @@ dependencies = [ 'typing_extensions==4.12.*', 'donfig==0.8.*', 'obstore==0.5.*', - # test deps - 'zarr[test]', - 'zarr[remote_tests]', ] [tool.hatch.envs.docs] -features = ['docs', 'remote'] +features = ['remote'] +dependency-groups = ['docs'] [tool.hatch.envs.docs.scripts] serve = "mkdocs serve --watch src" @@ -261,10 +252,10 @@ readthedocs = "rm -rf $READTHEDOCS_OUTPUT/html && cp -r site $READTHEDOCS_OUTPUT [tool.hatch.envs.doctest] description = "Test environment for validating executable code blocks in documentation" -features = ['test', 'remote'] # Include remote dependencies for s3fs -dependencies = [ +features = ['remote'] +dependency-groups = ['test'] +extra-dependencies = [ "s3fs>=2023.10.0", - "pytest", "pytest-examples", ] @@ -416,7 +407,11 @@ addopts = [ filterwarnings = [ "error", "ignore:Unclosed client session