diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6595cbe..56f6a44 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,11 +17,6 @@ repos: - id: python-no-log-warn - id: python-use-type-annotations - id: text-unicode-replacement-char -- repo: https://github.com/asottile/pyupgrade - rev: v3.3.1 - hooks: - - id: pyupgrade - args: [--py37-plus] - repo: https://github.com/asottile/reorder_python_imports rev: v3.9.0 hooks: @@ -32,7 +27,7 @@ repos: hooks: - id: setup-cfg-fmt - repo: https://github.com/PYCQA/docformatter - rev: v1.5.0 + rev: v1.5.1 hooks: - id: docformatter args: [--in-place, --wrap-summaries, "88", --wrap-descriptions, "88", --blank] @@ -40,27 +35,15 @@ repos: rev: 22.12.0 hooks: - id: black -- repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.215 hooks: - - id: flake8 - types: [python] - additional_dependencies: [ - flake8-alfred, - flake8-bugbear, - flake8-builtins, - flake8-comprehensions, - flake8-docstrings, - flake8-eradicate, - flake8-print, - flake8-pytest-style, - flake8-todo, - flake8-typing-imports, - flake8-unused-arguments, - pep8-naming, - pydocstyle, - Pygments, - ] + - id: ruff +- repo: https://github.com/dosisod/refurb + rev: v1.9.1 + hooks: + - id: refurb + args: [--ignore, FURB126] - repo: https://github.com/econchick/interrogate rev: 1.5.0 hooks: diff --git a/CHANGES.md b/CHANGES.md index 4c4d790..fcf8963 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask-parallel) and ## 0.3.0 - 2023-xx-xx - {pull}`50` deprecates INI configurations and aligns the package with pytask v0.3. +- {pull}`51` adds ruff and refurb. ## 0.2.1 - 2022-08-19 diff --git a/pyproject.toml b/pyproject.toml index ebdbddd..3393b3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,3 +22,41 @@ warn_unused_ignores = true module = "tests.*" disallow_untyped_defs = false ignore_errors = true + + +[tool.ruff] +target-version = "py37" +select = ["ALL"] +fix = true +extend-ignore = [ + # Numpy docstyle + "D107", + "D203", + "D212", + "D213", + "D402", + "D413", + "D415", + "D416", + "D417", + # Others. + "D404", # Do not start module docstring with "This". + "RET504", # unnecessary variable assignment before return. + "S101", # raise errors for asserts. + "B905", # strict parameter for zip that was implemented in py310. + "I", # ignore isort + "ANN101", # type annotating self + "ANN102", # type annotating cls + "FBT", # flake8-boolean-trap + "EM", # flake8-errmsg + "ANN401", # flake8-annotate typing.Any + "PD", # pandas-vet +] + + +[tool.ruff.per-file-ignores] +"tests/*" = ["D", "ANN"] + + +[tool.ruff.pydocstyle] +convention = "numpy" diff --git a/src/pytask_parallel/__init__.py b/src/pytask_parallel/__init__.py index f493641..9ddfe07 100644 --- a/src/pytask_parallel/__init__.py +++ b/src/pytask_parallel/__init__.py @@ -1,3 +1,4 @@ +"""This module is the main namespace of the package.""" from __future__ import annotations try: diff --git a/src/pytask_parallel/backends.py b/src/pytask_parallel/backends.py index 4397224..6d81c3e 100644 --- a/src/pytask_parallel/backends.py +++ b/src/pytask_parallel/backends.py @@ -1,9 +1,9 @@ """This module configures the available backends.""" from __future__ import annotations +import enum from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ThreadPoolExecutor -from enum import Enum try: @@ -11,7 +11,9 @@ except ImportError: - class ParallelBackendChoices(str, Enum): + class ParallelBackendChoices(enum.Enum): + """Choices for parallel backends.""" + PROCESSES = "processes" THREADS = "threads" @@ -24,7 +26,9 @@ class ParallelBackendChoices(str, Enum): else: - class ParallelBackendChoices(str, Enum): # type: ignore[no-redef] + class ParallelBackendChoices(enum.Enum): # type: ignore[no-redef] + """Choices for parallel backends.""" + PROCESSES = "processes" THREADS = "threads" LOKY = "loky" diff --git a/src/pytask_parallel/callbacks.py b/src/pytask_parallel/callbacks.py deleted file mode 100644 index 6061f4c..0000000 --- a/src/pytask_parallel/callbacks.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Validate command line inputs and configuration values.""" -from __future__ import annotations - -from typing import Any - -from pytask_parallel.backends import PARALLEL_BACKENDS - - -def n_workers_callback(value: Any) -> int: - """Validate the n-workers option.""" - if value == "auto": - pass - elif value in [None, "None", "none"]: - value = None - elif isinstance(value, int) and 1 <= value: - pass - elif isinstance(value, str) and value.isdigit(): - value = int(value) - else: - raise ValueError("n_workers can either be an integer >= 1, 'auto' or None.") - - return value - - -def parallel_backend_callback(value: Any) -> str | None: - """Validate the input for the parallel backend.""" - if value in [None, "None", "none"]: - value = None - elif value in PARALLEL_BACKENDS: - pass - else: - raise ValueError( - f"parallel_backend has to be one of {list(PARALLEL_BACKENDS)}." - ) - return value diff --git a/src/pytask_parallel/execute.py b/src/pytask_parallel/execute.py index 65d422f..c780833 100644 --- a/src/pytask_parallel/execute.py +++ b/src/pytask_parallel/execute.py @@ -69,10 +69,11 @@ def pytask_execute_build(session: Session) -> bool | None: newly_collected_reports = [] n_new_tasks = session.config["n_workers"] - len(running_tasks) - if n_new_tasks >= 1: - ready_tasks = list(session.scheduler.get_ready(n_new_tasks)) - else: - ready_tasks = [] + ready_tasks = ( + list(session.scheduler.get_ready(n_new_tasks)) + if n_new_tasks >= 1 + else [] + ) for task_name in ready_tasks: task = session.dag.nodes[task_name]["task"] @@ -83,7 +84,7 @@ def pytask_execute_build(session: Session) -> bool | None: session.hook.pytask_execute_task_setup( session=session, task=task ) - except Exception: + except Exception: # noqa: BLE001 report = ExecutionReport.from_task_and_exception( task, sys.exc_info() ) @@ -122,7 +123,7 @@ def pytask_execute_build(session: Session) -> bool | None: session.hook.pytask_execute_task_teardown( session=session, task=task ) - except Exception: + except Exception: # noqa: BLE001 report = ExecutionReport.from_task_and_exception( task, sys.exc_info() ) @@ -146,8 +147,8 @@ def pytask_execute_build(session: Session) -> bool | None: if session.should_stop: break - else: - sleeper.sleep() + sleeper.sleep() + except KeyboardInterrupt: break @@ -230,7 +231,7 @@ def _unserialize_and_execute_task( try: task.execute(**kwargs) - except Exception: + except Exception: # noqa: BLE001 exc_info = sys.exc_info() processed_exc_info = _process_exception( exc_info, show_locals, console_options @@ -282,8 +283,7 @@ def pytask_execute_task(session: Session, task: Task) -> Future[Any] | None: return session.config["_parallel_executor"].submit( _mock_processes_for_threads, func=task.execute, **kwargs ) - else: - return None + return None def _mock_processes_for_threads( @@ -299,7 +299,7 @@ def _mock_processes_for_threads( __tracebackhide__ = True try: func(**kwargs) - except Exception: + except Exception: # noqa: BLE001 exc_info = sys.exc_info() else: exc_info = None diff --git a/tests/test_callbacks.py b/tests/test_callbacks.py deleted file mode 100644 index 40555a7..0000000 --- a/tests/test_callbacks.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -from contextlib import ExitStack as does_not_raise # noqa: N813 - -import pytest -from pytask_parallel.backends import PARALLEL_BACKENDS -from pytask_parallel.callbacks import n_workers_callback -from pytask_parallel.callbacks import parallel_backend_callback - - -@pytest.mark.unit -@pytest.mark.parametrize( - "value, expectation", - [ - (0, pytest.raises(ValueError)), - (1, does_not_raise()), - (2, does_not_raise()), - ("auto", does_not_raise()), - ("asdad", pytest.raises(ValueError)), - (None, does_not_raise()), - ("None", does_not_raise()), - ("none", does_not_raise()), - ("1", does_not_raise()), - ("1.1", pytest.raises(ValueError)), - ], -) -def test_n_workers_callback(value, expectation): - with expectation: - n_workers_callback(value) - - -@pytest.mark.unit -@pytest.mark.parametrize( - "value, expectation", - [ - (1, pytest.raises(ValueError)), - ("asdad", pytest.raises(ValueError)), - (None, does_not_raise()), - ("None", does_not_raise()), - ("none", does_not_raise()), - ] - + [(i, does_not_raise()) for i in PARALLEL_BACKENDS], -) -def test_parallel_backend_callback(value, expectation): - with expectation: - parallel_backend_callback(value) diff --git a/tests/test_config.py b/tests/test_config.py index 95929a8..817713f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -9,9 +9,9 @@ from pytask_parallel.backends import ParallelBackendChoices -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize( - "pdb, n_workers, expected", + ("pdb", "n_workers", "expected"), [ (False, 1, 1), (True, 1, 1), @@ -25,9 +25,9 @@ def test_interplay_between_debugging_and_parallel(tmp_path, pdb, n_workers, expe assert session.config["n_workers"] == expected -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize( - "configuration_option, value, exit_code", + ("configuration_option", "value", "exit_code"), [ ("n_workers", "auto", ExitCode.OK), ("n_workers", 1, ExitCode.OK), diff --git a/tests/test_execute.py b/tests/test_execute.py index 1af4048..959838c 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -20,7 +20,7 @@ class Session: pass -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize("parallel_backend", PARALLEL_BACKENDS) def test_parallel_execution_speedup(tmp_path, parallel_backend): source = """ @@ -55,7 +55,7 @@ def task_2(produces): assert session.execution_end - session.execution_start < 10 -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize("parallel_backend", PARALLEL_BACKENDS) def test_parallel_execution_speedup_w_cli(runner, tmp_path, parallel_backend): source = """ @@ -102,7 +102,7 @@ def task_2(produces): assert end - start < 10 -@pytest.mark.integration +@pytest.mark.integration() @pytest.mark.parametrize("parallel_backend", PARALLEL_BACKENDS) def test_pytask_execute_task_w_processes(parallel_backend): # Local function which cannot be used with multiprocessing. @@ -142,7 +142,7 @@ def myfunc(): assert exception is None -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize("parallel_backend", PARALLEL_BACKENDS) def test_stop_execution_when_max_failures_is_reached(tmp_path, parallel_backend): source = """ @@ -171,7 +171,7 @@ def task_3(): time.sleep(3) assert len(session.execution_reports) == 2 -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize("parallel_backend", PARALLEL_BACKENDS) def test_task_priorities(tmp_path, parallel_backend): source = """ @@ -213,7 +213,7 @@ def task_5(): assert last_task_name.endswith("task_2") or last_task_name.endswith("task_5") -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize("parallel_backend", PARALLEL_BACKENDS) @pytest.mark.parametrize("show_locals", [True, False]) def test_rendering_of_tracebacks_with_rich( @@ -239,7 +239,7 @@ def task_raising_error(): assert ("[0, 1, 2, 3, 4]" in result.output) is show_locals -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize("parallel_backend", PARALLEL_BACKENDS) def test_generators_are_removed_from_depends_on_produces(tmp_path, parallel_backend): """Only works with pytask >=0.1.9.""" @@ -265,7 +265,7 @@ def task_example(produces): assert session.exit_code == ExitCode.OK -@pytest.mark.end_to_end +@pytest.mark.end_to_end() @pytest.mark.parametrize( "parallel_backend", # Capturing warnings is not thread-safe.