From 011215d793c0eedfe0b83bffa101cdfa03699b9a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 14 Oct 2024 19:13:42 +0500 Subject: [PATCH 1/4] Update Python versions. --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 4 ++-- .readthedocs.yml | 2 +- README.rst | 2 +- setup.py | 4 ++-- tox.ini | 2 +- zyte_api/_async.py | 4 ++-- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 240ffca..9fab21b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.x' + python-version: '3.13' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d0ce2d..2a28888 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v2 @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.11'] + python-version: ['3.13'] tox-job: ["mypy", "docs"] steps: diff --git a/.readthedocs.yml b/.readthedocs.yml index de19d2d..5332597 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -7,7 +7,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.11" # Keep in sync with .github/workflows/test.yml + python: "3.13" # Keep in sync with .github/workflows/test.yml python: install: - requirements: docs/requirements.txt diff --git a/README.rst b/README.rst index 4035e57..e52e951 100644 --- a/README.rst +++ b/README.rst @@ -35,7 +35,7 @@ Installation pip install zyte-api -.. note:: Python 3.8+ is required. +.. note:: Python 3.9+ is required. .. install-end diff --git a/setup.py b/setup.py index 310eb5e..612d6d4 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import os from setuptools import find_packages, setup @@ -41,9 +40,10 @@ def get_version(): "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], ) diff --git a/tox.ini b/tox.ini index 21023e8..c3320ef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38,py39,py310,py311,mypy,docs,twine +envlist = py39,py310,py311,py312,py313,mypy,docs,twine [testenv] deps = diff --git a/zyte_api/_async.py b/zyte_api/_async.py index 261277f..afa20ff 100644 --- a/zyte_api/_async.py +++ b/zyte_api/_async.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import time from asyncio import Future @@ -17,8 +19,6 @@ if TYPE_CHECKING: _ResponseFuture = Future[Dict[str, Any]] -else: - _ResponseFuture = Future # Python 3.8 support def _post_func(session): From c69fe9e4c1e8525152d8c285bdf695d9d2a302bd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 14 Oct 2024 19:21:19 +0500 Subject: [PATCH 2/4] Bump tool versions. --- .github/workflows/publish.yml | 4 ++-- .github/workflows/test.yml | 8 ++++---- .pre-commit-config.yaml | 10 +++++----- docs/conf.py | 9 +-------- pyproject.toml | 2 +- tests/test_main.py | 7 ++++--- tox.ini | 13 ++++++++----- 7 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9fab21b..820c29d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,9 +13,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: '3.13' - name: Install dependencies diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a28888..a844809 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,9 +19,9 @@ jobs: python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -46,9 +46,9 @@ jobs: tox-job: ["mypy", "docs"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5420cd..c2700c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,19 @@ repos: - repo: https://github.com/PyCQA/isort - rev: 5.13.1 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/psf/black - rev: 24.3.0 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 - repo: https://github.com/adamchainz/blacken-docs - rev: 1.16.0 + rev: 1.19.0 hooks: - id: blacken-docs additional_dependencies: - - black==24.3.0 + - black==24.10.0 diff --git a/docs/conf.py b/docs/conf.py index 8ed5574..a652286 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,8 +14,6 @@ import sys from pathlib import Path -import sphinx_rtd_theme - sys.path.insert(0, os.path.abspath("../")) @@ -58,7 +56,7 @@ # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = ".rst" +source_suffix = {".rst": "restructuredtext"} # The master toctree document. master_doc = "index" @@ -86,11 +84,6 @@ # html_theme = "sphinx_rtd_theme" -# Add any paths that contain custom themes here, relative to this directory. -# Add path to the RTD explicitly to robustify builds (otherwise might -# fail in a clean Debian build env) -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. diff --git a/pyproject.toml b/pyproject.toml index 830e253..c6b28cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,4 +3,4 @@ profile = "black" multi_line_output = 3 [tool.black] -target-version = ["py38", "py39", "py310", "py311", "py312"] +target-version = ["py39", "py310", "py311", "py312", "py313"] diff --git a/tests/test_main.py b/tests/test_main.py index 9fbb3bd..7e3c8d8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -109,9 +109,10 @@ async def test_run(queries, expected_response, store_errors, exception): async_client_mock.return_value.iter = request_parallel_mock # Patch the AsyncZyteAPI class in __main__ with the mock - with patch("zyte_api.__main__.AsyncZyteAPI", async_client_mock), patch( - "zyte_api.__main__.create_session" - ) as create_session_mock: + with ( + patch("zyte_api.__main__.AsyncZyteAPI", async_client_mock), + patch("zyte_api.__main__.create_session") as create_session_mock, + ): # Mock create_session to return an AsyncMock create_session_mock.return_value = AsyncMock() diff --git a/tox.ini b/tox.ini index c3320ef..33663b7 100644 --- a/tox.ini +++ b/tox.ini @@ -19,9 +19,12 @@ commands = [testenv:mypy] deps = - mypy==0.982 + mypy==1.12.0 + pytest==8.3.3 + Twisted==24.7.0 + types-tqdm==4.66.0.20240417 -commands = mypy --ignore-missing-imports --no-warn-no-return \ +commands = mypy --ignore-missing-imports \ zyte_api \ tests @@ -39,8 +42,8 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:twine] deps = - twine==4.0.2 - build==1.0.3 + twine==5.1.1 + build==1.2.2 commands = - python setup.py sdist + python -m build --sdist twine check dist/* From 4132cedae1cb402fc89b1f25c15801c067b857b2 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 15 Oct 2024 16:10:59 +0500 Subject: [PATCH 3/4] Roll back RTD Python --- .github/workflows/test.yml | 2 +- .readthedocs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a844809..b056a24 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.13'] + python-version: ['3.12'] # Keep in sync with .readthedocs.yml tox-job: ["mypy", "docs"] steps: diff --git a/.readthedocs.yml b/.readthedocs.yml index 5332597..f81f402 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -7,7 +7,7 @@ build: tools: # For available versions, see: # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.13" # Keep in sync with .github/workflows/test.yml + python: "3.12" # Keep in sync with .github/workflows/test.yml python: install: - requirements: docs/requirements.txt From b95d703fd9c4f607d7a424797fc17b056ef22945 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 16 Oct 2024 14:13:18 +0500 Subject: [PATCH 4/4] Update docs referebcesto docs.zyte.com. --- CHANGES.rst | 2 +- docs/use/api.rst | 14 +++++++------- docs/use/cli.rst | 10 +++++----- zyte_api/_errors.py | 4 ++-- zyte_api/_retry.py | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 55e4c49..fb5fff1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,7 +5,7 @@ Changes ------------------ * Improved how the :ref:`default retry policy ` handles - :ref:`temporary download errors `. + :ref:`temporary download errors `. Before, 3 HTTP 429 responses followed by a single HTTP 520 response would have prevented a retry. Now, unrelated responses and errors do not count towards the HTTP 520 retry limit. diff --git a/docs/use/api.rst b/docs/use/api.rst index 7d05a47..8e3d34e 100644 --- a/docs/use/api.rst +++ b/docs/use/api.rst @@ -134,15 +134,15 @@ The number of concurrent connections if enforced across all method calls, including different sessions of the same client. For guidelines on how to choose the optimal value for you, and other -optimization tips, see :ref:`zyte-api-optimize`. +optimization tips, see :ref:`zapi-optimize`. Errors and retries ================== Methods of :class:`ZyteAPI` and :class:`AsyncZyteAPI` automatically handle -retries for :ref:`rate-limiting ` and :ref:`unsuccessful -` responses, as well as network errors. +retries for :ref:`rate-limiting ` and :ref:`unsuccessful +` responses, as well as network errors. .. _retry-policy: .. _default-retry-policy: @@ -150,10 +150,10 @@ retries for :ref:`rate-limiting ` and :ref:`unsuccessful The default retry policy, :data:`~zyte_api.zyte_api_retrying`, does the following: -- Retries :ref:`rate-limiting responses ` forever. +- Retries :ref:`rate-limiting responses ` forever. - Retries :ref:`temporary download errors - ` up to 3 times. + ` up to 3 times. - Retries network errors until they have happened for 15 minutes straight. @@ -161,13 +161,13 @@ All retries are done with an exponential backoff algorithm. .. _aggressive-retry-policy: -If some :ref:`unsuccessful responses ` exceed +If some :ref:`unsuccessful responses ` exceed maximum retries with the default retry policy, try using :data:`~zyte_api.aggressive_retrying` instead, which modifies the default retry policy as follows: - Temporary download error are retried 7 times. :ref:`Permanent download - errors ` also count towards this retry + errors ` also count towards this retry limit. - Retries permanent download errors up to 3 times. diff --git a/docs/use/cli.rst b/docs/use/cli.rst index 49f4a9a..abf2479 100644 --- a/docs/use/cli.rst +++ b/docs/use/cli.rst @@ -33,7 +33,7 @@ The input file can be either of the following: :http:`request:browserHtml` set to ``True``. - A `JSON Lines `_ file with a object of :ref:`Zyte - API request parameters ` per line. For example: + API request parameters ` per line. For example: .. code-block:: json @@ -84,19 +84,19 @@ order and hence distribute the load somewhat evenly: zyte-api urls.txt --shuffle … For guidelines on how to choose the optimal ``--n-conn`` value for you, and -other optimization tips, see :ref:`zyte-api-optimize`. +other optimization tips, see :ref:`zapi-optimize`. Errors and retries ================== ``zyte-api`` automatically handles retries for :ref:`rate-limiting -` and :ref:`unsuccessful -` responses, as well as network errors, +` and :ref:`unsuccessful +` responses, as well as network errors, following the :ref:`default retry policy `. Use ``--dont-retry-errors`` to disable the retrying of error responses, and -retrying only :ref:`rate-limiting responses `: +retrying only :ref:`rate-limiting responses `: .. code-block:: shell diff --git a/zyte_api/_errors.py b/zyte_api/_errors.py index ea589ae..6476c39 100644 --- a/zyte_api/_errors.py +++ b/zyte_api/_errors.py @@ -10,8 +10,8 @@ class RequestError(ClientResponseError): """Exception raised upon receiving a :ref:`rate-limiting - ` or :ref:`unsuccessful - ` response from Zyte API.""" + ` or :ref:`unsuccessful + ` response from Zyte API.""" def __init__(self, *args, **kwargs): #: Query sent to Zyte API. diff --git a/zyte_api/_retry.py b/zyte_api/_retry.py index 04b16b1..3b01c0e 100644 --- a/zyte_api/_retry.py +++ b/zyte_api/_retry.py @@ -137,7 +137,7 @@ class RetryFactory: the corresponding :class:`tenacity.AsyncRetrying` object. For example, to double the number of attempts for :ref:`temporary - download errors ` and the time network + download errors ` and the time network errors are retried: .. code-block:: python