diff --git a/docs/02_concepts/14_standby_actors.mdx b/docs/02_concepts/14_standby_actors.mdx new file mode 100644 index 00000000..c30ffff9 --- /dev/null +++ b/docs/02_concepts/14_standby_actors.mdx @@ -0,0 +1,52 @@ +--- +id: standby-actors +title: Standby Actors +description: Send HTTP requests to Actors running in Standby mode. +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import CodeBlock from '@theme/CodeBlock'; +import ApiLink from '@theme/ApiLink'; + +import StandbyAsyncExample from '!!raw-loader!./code/14_standby_async.py'; +import StandbySyncExample from '!!raw-loader!./code/14_standby_sync.py'; + +An Actor in [Standby mode](https://docs.apify.com/platform/actors/running/standby) runs a web server. You call it over HTTP, and the platform keeps a run ready to answer, so each call skips the run startup. To send a request to such an Actor, use `ActorClient.standby_request`. It takes the path on the Actor's server, and optionally the HTTP method, query parameters, headers, and a body: + + + + + {StandbyAsyncExample} + + + + + {StandbySyncExample} + + + + +The method returns the server's response. To find out which endpoints an Actor serves, see its page in [Apify Store](https://apify.com/store) or its README. + +## How the request is sent + +Note that: + +- The client takes the server's URL from the Actor's `standby_url`. It fetches the Actor from the API on the first call and reuses the URL for later calls on the same `ActorClient`. If the Actor doesn't have Standby mode enabled, the method raises an `ApifyClientError`. +- The path is always resolved against that URL, so a path can't send the request to another host. +- The request body is sent uncompressed, since the Actor's server may not decode a compressed body. If the server decodes it, pass `compression`, for example `compression='gzip'`. The client then compresses large bodies the same way as for API calls. + +## Authentication + +The request carries the client's API token in the `Authorization` header. The platform removes the token before it forwards the request, so the token doesn't reach the Actor's code. + +If the Actor reads the `Authorization` header for its own authentication, pass that header in `headers`, and send your Apify token in the `X-Apify-Authorization` header instead. A header you pass in `headers` replaces the client's `Authorization` header for that request. + +For details, see [How do I authenticate my requests](https://docs.apify.com/platform/actors/running/standby#how-do-i-authenticate-my-requests) in the platform documentation. + +## Timeouts, retries, and errors + +When no Standby run is ready, the first request waits for the platform to start one, which can take a few minutes. The method uses the `long` timeout tier by default for that reason. You can pass a different `timeout`, the same way as for API calls. For details, see [Timeouts](./11_timeouts.mdx). + +An error status from the Actor's server raises an `ApifyApiError`, like an error from the API. Server errors and rate-limit responses are retried with the client's retry settings, so a request that the server failed to answer may reach it more than once. If your endpoint isn't safe to call twice, lower `max_retries` on the client. For details, see [Retries](./05_retries.mdx) and [Error handling](./04_error_handling.mdx). diff --git a/docs/02_concepts/code/14_standby_async.py b/docs/02_concepts/code/14_standby_async.py new file mode 100644 index 00000000..c3fa0ab3 --- /dev/null +++ b/docs/02_concepts/code/14_standby_async.py @@ -0,0 +1,20 @@ +from apify_client import ApifyClientAsync + +TOKEN = 'MY-APIFY-TOKEN' + + +async def main() -> None: + apify_client = ApifyClientAsync(TOKEN) + actor_client = apify_client.actor('username/actor-name') + + # Send a GET request to the /search endpoint of the Actor's Standby server. + response = await actor_client.standby_request('/search', params={'q': 'apify'}) + print(response.json()) + + # Send a POST request with a JSON body. + response = await actor_client.standby_request( + '/items', + method='POST', + json={'url': 'https://apify.com'}, + ) + print(response.status_code) diff --git a/docs/02_concepts/code/14_standby_sync.py b/docs/02_concepts/code/14_standby_sync.py new file mode 100644 index 00000000..5aca6151 --- /dev/null +++ b/docs/02_concepts/code/14_standby_sync.py @@ -0,0 +1,20 @@ +from apify_client import ApifyClient + +TOKEN = 'MY-APIFY-TOKEN' + + +def main() -> None: + apify_client = ApifyClient(TOKEN) + actor_client = apify_client.actor('username/actor-name') + + # Send a GET request to the /search endpoint of the Actor's Standby server. + response = actor_client.standby_request('/search', params={'q': 'apify'}) + print(response.json()) + + # Send a POST request with a JSON body. + response = actor_client.standby_request( + '/items', + method='POST', + json={'url': 'https://apify.com'}, + ) + print(response.status_code) diff --git a/scripts/_utils.py b/scripts/_utils.py index 48612c54..5be77090 100644 --- a/scripts/_utils.py +++ b/scripts/_utils.py @@ -20,7 +20,7 @@ """Path to the source code of the apify_client package.""" _SUBSTITUTIONS = [ - (re.compile(r'Client'), 'ClientAsync'), + (re.compile(r'Client(?!Error)'), 'ClientAsync'), (re.compile(r'\bsynchronously\b'), 'asynchronously'), (re.compile(r'\bSynchronously\b'), 'Asynchronously'), (re.compile(r'\bsynchronous\b'), 'asynchronous'), diff --git a/src/apify_client/_resource_clients/actor.py b/src/apify_client/_resource_clients/actor.py index 6bd06fbb..08fb9ecf 100644 --- a/src/apify_client/_resource_clients/actor.py +++ b/src/apify_client/_resource_clients/actor.py @@ -32,6 +32,8 @@ start_waiting_for_resources, start_waiting_for_resources_async, ) +from apify_client.errors import ApifyClientError +from apify_client.http_compressors._resolve import resolve_compressor if TYPE_CHECKING: from datetime import timedelta @@ -55,7 +57,9 @@ WebhookCollectionClient, WebhookCollectionClientAsync, ) - from apify_client.types import Timeout, WebhooksList + from apify_client.http_clients import HttpResponse + from apify_client.http_compressors import HttpCompressor + from apify_client.types import HttpCompressionAlgorithm, JsonSerializable, Timeout, WebhooksList _PricingInfo = ( PayPerEventActorPricingInfo @@ -86,6 +90,7 @@ def __init__( resource_path=resource_path, **kwargs, ) + self._standby_url: str | None = None def get(self, *, timeout: Timeout = 'short') -> ActorResource | None: """Retrieve the Actor. @@ -625,6 +630,74 @@ def validate_input( return True + def standby_request( + self, + path: str = '/', + *, + method: str = 'GET', + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + data: str | bytes | None = None, + json: JsonSerializable | None = None, + compression: HttpCompressionAlgorithm | HttpCompressor | None = None, + timeout: Timeout = 'long', + ) -> HttpResponse: + """Send an HTTP request to the Actor's Standby server. + + An Actor in Standby mode runs a web server that is called over HTTP, without starting a run for each call. The + server's URL is the Actor's `standby_url`, fetched from the API on the first call and reused by later calls on + this client. The request carries the client's token, which the Standby router consumes, so it never reaches + the Actor's code. The first request to an idle Actor waits for the platform to start a Standby run, which is + why the timeout defaults to `long`. + + A server error or a rate-limit response is retried like an API call, so the server may receive the same + request more than once. Request bodies are sent uncompressed unless `compression` is set, since the Actor's + server may not decode a compressed body. + + https://docs.apify.com/platform/actors/running/standby + + Args: + path: The path on the Actor's server, for example `/search`. It is always resolved against the Standby URL, + so the request, and the token with it, cannot be sent to another host. Pass the query in `params`. + method: HTTP method of the request. + params: Query parameters of the request, converted the same way as for API calls: booleans become `true` + or `false`, a list of strings becomes one comma-separated value, a datetime becomes an ISO 8601 string + in UTC, and `None` values are left out. + headers: Additional headers of the request. + data: Raw request body. Cannot be used together with `json`. + json: JSON-serializable request body. Cannot be used together with `data`. + compression: Compression algorithm for the request body, as a string literal or an `HttpCompressor` + instance. The body is compressed under the same conditions as for API calls. Set it only if the + Actor's server decodes the resulting `Content-Encoding`. + timeout: Timeout for the HTTP request. + + Returns: + The response of the Actor's server. + + Raises: + ApifyClientError: If the Actor does not have Standby mode enabled. + ApifyApiError: If the Actor does not exist, or the server responds with an error status. + ImportError: If `compression` needs an optional extra that is not installed. + ValueError: If both `data` and `json` are passed, or `compression` is not a supported algorithm. + """ + if self._standby_url is None: + response = self._http_client.call(url=self._build_url(), method='GET', timeout='short') + actor = ActorResponse.model_validate(response_to_dict(response)).data + if actor.standby_url is None: + raise ApifyClientError(f'Actor {self.resource_id} does not have Standby mode enabled.') + self._standby_url = actor.standby_url + + return self._http_client.call( + url=f'{self._standby_url.rstrip("/")}/{path.lstrip("/")}', + method=method, + params=params, + headers=headers, + data=data, + json=json, + timeout=timeout, + compression=resolve_compressor(compression) if compression is not None else False, + ) + @docs_group('Resource clients') class ActorClientAsync(ResourceClientAsync): @@ -646,6 +719,7 @@ def __init__( resource_path=resource_path, **kwargs, ) + self._standby_url: str | None = None async def get(self, *, timeout: Timeout = 'short') -> ActorResource | None: """Retrieve the Actor. @@ -1185,3 +1259,71 @@ async def validate_input( ) return True + + async def standby_request( + self, + path: str = '/', + *, + method: str = 'GET', + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + data: str | bytes | None = None, + json: JsonSerializable | None = None, + compression: HttpCompressionAlgorithm | HttpCompressor | None = None, + timeout: Timeout = 'long', + ) -> HttpResponse: + """Send an HTTP request to the Actor's Standby server. + + An Actor in Standby mode runs a web server that is called over HTTP, without starting a run for each call. The + server's URL is the Actor's `standby_url`, fetched from the API on the first call and reused by later calls on + this client. The request carries the client's token, which the Standby router consumes, so it never reaches + the Actor's code. The first request to an idle Actor waits for the platform to start a Standby run, which is + why the timeout defaults to `long`. + + A server error or a rate-limit response is retried like an API call, so the server may receive the same + request more than once. Request bodies are sent uncompressed unless `compression` is set, since the Actor's + server may not decode a compressed body. + + https://docs.apify.com/platform/actors/running/standby + + Args: + path: The path on the Actor's server, for example `/search`. It is always resolved against the Standby URL, + so the request, and the token with it, cannot be sent to another host. Pass the query in `params`. + method: HTTP method of the request. + params: Query parameters of the request, converted the same way as for API calls: booleans become `true` + or `false`, a list of strings becomes one comma-separated value, a datetime becomes an ISO 8601 string + in UTC, and `None` values are left out. + headers: Additional headers of the request. + data: Raw request body. Cannot be used together with `json`. + json: JSON-serializable request body. Cannot be used together with `data`. + compression: Compression algorithm for the request body, as a string literal or an `HttpCompressor` + instance. The body is compressed under the same conditions as for API calls. Set it only if the + Actor's server decodes the resulting `Content-Encoding`. + timeout: Timeout for the HTTP request. + + Returns: + The response of the Actor's server. + + Raises: + ApifyClientError: If the Actor does not have Standby mode enabled. + ApifyApiError: If the Actor does not exist, or the server responds with an error status. + ImportError: If `compression` needs an optional extra that is not installed. + ValueError: If both `data` and `json` are passed, or `compression` is not a supported algorithm. + """ + if self._standby_url is None: + response = await self._http_client.call(url=self._build_url(), method='GET', timeout='short') + actor = ActorResponse.model_validate(response_to_dict(response)).data + if actor.standby_url is None: + raise ApifyClientError(f'Actor {self.resource_id} does not have Standby mode enabled.') + self._standby_url = actor.standby_url + + return await self._http_client.call( + url=f'{self._standby_url.rstrip("/")}/{path.lstrip("/")}', + method=method, + params=params, + headers=headers, + data=data, + json=json, + timeout=timeout, + compression=resolve_compressor(compression) if compression is not None else False, + ) diff --git a/src/apify_client/http_clients/_base.py b/src/apify_client/http_clients/_base.py index b06d542c..9f8616f4 100644 --- a/src/apify_client/http_clients/_base.py +++ b/src/apify_client/http_clients/_base.py @@ -312,6 +312,7 @@ def _prepare_request_call( params: dict[str, Any] | None = None, data: str | bytes | bytearray | StreamedBodySource | None = None, json: JsonSerializable | None = None, + compression: HttpCompressor | bool = True, ) -> tuple[dict[str, str], dict[str, Any] | None, bytes | StreamedRequestBody | None]: """Prepare headers, params, and body for an HTTP request. @@ -319,11 +320,11 @@ def _prepare_request_call( JSON body. Header names are treated case-insensitively and per-request values win over the client defaults. For JSON bodies, a `Content-Type` header is set unless the caller supplied one. - The body is compressed unless a `Content-Encoding` header is already set, the body is smaller than + The body is compressed with the client's compressor, or with the one passed in `compression`, unless + `compression` is `False`, a `Content-Encoding` header is already set, the body is smaller than `MIN_COMPRESSION_SIZE`, or its content type says the payload is already compressed. A caller-supplied `Content-Encoding` is forwarded verbatim, which is how a pre-encoded body is uploaded - including one in - an encoding the client ships no compressor for. `Content-Encoding: identity` therefore opts a single - request out of compression. + an encoding the client ships no compressor for. A body streamed from a file-like object, an iterable of byte chunks, or a streamed response is wrapped in `StreamedRequestBody` and never compressed, since its chunks go out as they are produced. A caller-supplied @@ -353,13 +354,15 @@ def _prepare_request_call( # A caller-supplied encoding says the body arrives already encoded, so compressing it here would # both mislabel it and waste the work. + compressor = self._http_compressor if compression is True else compression if ( - self._get_header(headers, 'content-encoding') is None + compressor is not False + and self._get_header(headers, 'content-encoding') is None and len(content) >= MIN_COMPRESSION_SIZE and is_compressible_content_type(self._get_header(headers, 'content-type')) ): - content = self._http_compressor.compress(content) - headers = self._merge_headers(headers, {'Content-Encoding': self._http_compressor.content_encoding}) + content = compressor.compress(content) + headers = self._merge_headers(headers, {'Content-Encoding': compressor.content_encoding}) elif data is not None: # Without this the request would go out with no body at all, losing the payload without a word. @@ -509,6 +512,7 @@ def call( json: JsonSerializable | None = None, stream: bool | None = None, timeout: Timeout = 'medium', + compression: HttpCompressor | bool = True, ) -> HttpResponse: """Make an HTTP request with automatic retry and exponential backoff. @@ -526,6 +530,8 @@ def call( timeout: Timeout for the API HTTP request. Use `short`, `medium`, or `long` tier literals for preconfigured timeouts. A `timedelta` overrides it for this call (capped at `timeout_max`), and `no_timeout` disables the timeout entirely. + compression: How to compress the request body. `True` uses the client's compressor, an `HttpCompressor` + is used for this request only, and `False` sends the body uncompressed. Returns: The HTTP response object. @@ -545,6 +551,7 @@ def call( params=params, data=data, json=json, + compression=compression, ) return self._retry_with_exp_backoff( @@ -720,6 +727,7 @@ async def call( json: JsonSerializable | None = None, stream: bool | None = None, timeout: Timeout = 'medium', + compression: HttpCompressor | bool = True, ) -> HttpResponse: """Make an HTTP request with automatic retry and exponential backoff. @@ -737,6 +745,8 @@ async def call( timeout: Timeout for the API HTTP request. Use `short`, `medium`, or `long` tier literals for preconfigured timeouts. A `timedelta` overrides it for this call (capped at `timeout_max`), and `no_timeout` disables the timeout entirely. + compression: How to compress the request body. `True` uses the client's compressor, an `HttpCompressor` + is used for this request only, and `False` sends the body uncompressed. Returns: The HTTP response object. @@ -755,13 +765,14 @@ async def call( # offload preparation to a worker thread whenever there is something to compress. A body the # client sends as it is costs less to prepare inline than the hop itself. A `json` body always # hops, as its size is only known once serialized. - if json is not None or self._is_body_worth_compressing(data): + if json is not None or (compression is not False and self._is_body_worth_compressing(data)): prepared_headers, prepared_params, content = await asyncio.to_thread( self._prepare_request_call, headers=headers, params=params, data=data, json=json, + compression=compression, ) else: prepared_headers, prepared_params, content = self._prepare_request_call( @@ -769,6 +780,7 @@ async def call( params=params, data=data, json=json, + compression=compression, ) return await self._retry_with_exp_backoff( diff --git a/tests/unit/test_actor_standby_request.py b/tests/unit/test_actor_standby_request.py new file mode 100644 index 00000000..9c3ead09 --- /dev/null +++ b/tests/unit/test_actor_standby_request.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +import gzip +import json +from typing import TYPE_CHECKING, Any + +import pytest + +from apify_client.errors import ApifyClientError, NotFoundError +from apify_client.http_compressors import GzipHttpCompressor + +if TYPE_CHECKING: + from pytest_httpserver import HTTPServer + from werkzeug import Request + + from apify_client import ApifyClient, ApifyClientAsync + from apify_client.http_compressors import HttpCompressor + from apify_client.types import HttpCompressionAlgorithm + +pytestmark = pytest.mark.usefixtures('http_client_classes') + +_MOCKED_ACTOR_ID = 'test_actor_id' +_ACTOR_PATH = f'/v2/actors/{_MOCKED_ACTOR_ID}' +_STANDBY_PREFIX = '/standby' + + +def actor_response(standby_url: str | None) -> dict[str, Any]: + return { + 'data': { + 'id': _MOCKED_ACTOR_ID, + 'userId': 'test_user_id', + 'name': 'test-actor', + 'username': 'test-user', + 'isPublic': False, + 'createdAt': '2026-08-01T10:00:00.000Z', + 'modifiedAt': '2026-08-01T10:00:00.000Z', + 'stats': {}, + 'versions': [], + 'defaultRunOptions': {}, + 'standbyUrl': standby_url, + } + } + + +def expect_standby_actor(httpserver: HTTPServer) -> None: + standby_url = httpserver.url_for(_STANDBY_PREFIX) + httpserver.expect_request(_ACTOR_PATH, method='GET').respond_with_json(actor_response(standby_url)) + + +def standby_requests(httpserver: HTTPServer) -> list[Request]: + return [request for request, _ in httpserver.log if request.path.startswith(f'{_STANDBY_PREFIX}/')] + + +def test_standby_request_sends_request_to_standby_url(httpserver: HTTPServer, sync_client: ApifyClient) -> None: + """The request goes to the Standby URL with the method, path, query, body, and the client's token.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/search', method='POST').respond_with_json({'ok': True}) + + response = sync_client.actor(_MOCKED_ACTOR_ID).standby_request( + '/search', method='POST', params={'q': 'apify'}, json={'limit': 5} + ) + + assert response.status_code == 200 + assert response.json() == {'ok': True} + captured = standby_requests(httpserver) + assert len(captured) == 1 + assert captured[0].args.to_dict() == {'q': 'apify'} + assert json.loads(captured[0].get_data()) == {'limit': 5} + assert captured[0].headers['Authorization'] == 'Bearer test' + + +async def test_standby_request_sends_request_to_standby_url_async( + httpserver: HTTPServer, async_client: ApifyClientAsync +) -> None: + """Async variant of `test_standby_request_sends_request_to_standby_url`.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/search', method='POST').respond_with_json({'ok': True}) + + response = await async_client.actor(_MOCKED_ACTOR_ID).standby_request( + '/search', method='POST', params={'q': 'apify'}, json={'limit': 5} + ) + + assert response.status_code == 200 + assert response.json() == {'ok': True} + captured = standby_requests(httpserver) + assert len(captured) == 1 + assert captured[0].args.to_dict() == {'q': 'apify'} + assert json.loads(captured[0].get_data()) == {'limit': 5} + assert captured[0].headers['Authorization'] == 'Bearer test' + + +def test_standby_request_fetches_standby_url_once(httpserver: HTTPServer, sync_client: ApifyClient) -> None: + """Later calls on the same client reuse the Standby URL fetched by the first one.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/', method='GET').respond_with_json({'ok': True}) + actor_client = sync_client.actor(_MOCKED_ACTOR_ID) + + actor_client.standby_request() + actor_client.standby_request() + + actor_gets = [request for request, _ in httpserver.log if request.path == _ACTOR_PATH] + assert len(actor_gets) == 1 + + +async def test_standby_request_fetches_standby_url_once_async( + httpserver: HTTPServer, async_client: ApifyClientAsync +) -> None: + """Async variant of `test_standby_request_fetches_standby_url_once`.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/', method='GET').respond_with_json({'ok': True}) + actor_client = async_client.actor(_MOCKED_ACTOR_ID) + + await actor_client.standby_request() + await actor_client.standby_request() + + actor_gets = [request for request, _ in httpserver.log if request.path == _ACTOR_PATH] + assert len(actor_gets) == 1 + + +@pytest.mark.parametrize( + 'path', + [ + pytest.param('//evil.example/steal', id='protocol-relative'), + pytest.param('@evil.example/steal', id='userinfo'), + pytest.param('https://evil.example/steal', id='absolute url'), + ], +) +def test_standby_request_path_stays_on_standby_host( + httpserver: HTTPServer, sync_client: ApifyClient, path: str +) -> None: + """A path that looks like another host is still sent to the Standby server, as a path on it.""" + expect_standby_actor(httpserver) + httpserver.no_handler_status_code = 200 + + response = sync_client.actor(_MOCKED_ACTOR_ID).standby_request(path) + + assert response.status_code == 200 + assert len(standby_requests(httpserver)) == 1 + + +def test_standby_request_sends_body_uncompressed(httpserver: HTTPServer, sync_client: ApifyClient) -> None: + """A body large enough for the client to compress is sent as is, since Standby servers may not decode it.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/items', method='POST').respond_with_json({'ok': True}) + payload = {'items': ['x' * 100] * 100} + + sync_client.actor(_MOCKED_ACTOR_ID).standby_request('/items', method='POST', json=payload) + + captured = standby_requests(httpserver) + assert len(captured) == 1 + assert 'Content-Encoding' not in captured[0].headers + assert json.loads(captured[0].get_data()) == payload + + +async def test_standby_request_sends_body_uncompressed_async( + httpserver: HTTPServer, async_client: ApifyClientAsync +) -> None: + """Async variant of `test_standby_request_sends_body_uncompressed`.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/items', method='POST').respond_with_json({'ok': True}) + payload = {'items': ['x' * 100] * 100} + + await async_client.actor(_MOCKED_ACTOR_ID).standby_request('/items', method='POST', json=payload) + + captured = standby_requests(httpserver) + assert len(captured) == 1 + assert 'Content-Encoding' not in captured[0].headers + assert json.loads(captured[0].get_data()) == payload + + +@pytest.mark.parametrize( + 'compression', + [ + pytest.param('gzip', id='algorithm name'), + pytest.param(GzipHttpCompressor(quality=1), id='compressor instance'), + ], +) +def test_standby_request_compresses_body_when_asked( + httpserver: HTTPServer, sync_client: ApifyClient, compression: HttpCompressionAlgorithm | HttpCompressor +) -> None: + """With `compression` set, a body large enough to compress is sent compressed and labeled with its encoding.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/items', method='POST').respond_with_json({'ok': True}) + payload = {'items': ['x' * 100] * 100} + + sync_client.actor(_MOCKED_ACTOR_ID).standby_request('/items', method='POST', json=payload, compression=compression) + + captured = standby_requests(httpserver) + assert len(captured) == 1 + assert captured[0].headers.get('Content-Encoding') == 'gzip' + assert json.loads(gzip.decompress(captured[0].get_data())) == payload + + +async def test_standby_request_compresses_body_when_asked_async( + httpserver: HTTPServer, async_client: ApifyClientAsync +) -> None: + """Async variant of `test_standby_request_compresses_body_when_asked`.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/items', method='POST').respond_with_json({'ok': True}) + payload = {'items': ['x' * 100] * 100} + + await async_client.actor(_MOCKED_ACTOR_ID).standby_request( + '/items', method='POST', json=payload, compression='gzip' + ) + + captured = standby_requests(httpserver) + assert len(captured) == 1 + assert captured[0].headers.get('Content-Encoding') == 'gzip' + assert json.loads(gzip.decompress(captured[0].get_data())) == payload + + +def test_standby_request_rejects_unknown_compression(httpserver: HTTPServer, sync_client: ApifyClient) -> None: + """An unsupported compression algorithm raises a `ValueError` without sending a request to the Standby server.""" + expect_standby_actor(httpserver) + + with pytest.raises(ValueError, match='Unsupported compression algorithm'): + sync_client.actor(_MOCKED_ACTOR_ID).standby_request('/items', method='POST', data='x', compression='zip') # ty: ignore[invalid-argument-type] + + assert standby_requests(httpserver) == [] + + +def test_standby_request_keeps_caller_content_encoding(httpserver: HTTPServer, sync_client: ApifyClient) -> None: + """A `Content-Encoding` header passed by the caller is sent as given, with the body untouched.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/items', method='POST').respond_with_json({'ok': True}) + body = gzip.compress(b'x' * 2000) + + sync_client.actor(_MOCKED_ACTOR_ID).standby_request( + '/items', method='POST', headers={'content-encoding': 'gzip'}, data=body + ) + + captured = standby_requests(httpserver) + assert len(captured) == 1 + assert captured[0].headers.get('Content-Encoding') == 'gzip' + assert captured[0].get_data() == body + + +def test_standby_request_without_standby_mode_raises(httpserver: HTTPServer, sync_client: ApifyClient) -> None: + """An Actor without Standby mode enabled raises an error before any request to a Standby server.""" + httpserver.expect_request(_ACTOR_PATH, method='GET').respond_with_json(actor_response(None)) + + with pytest.raises(ApifyClientError, match='does not have Standby mode enabled'): + sync_client.actor(_MOCKED_ACTOR_ID).standby_request() + + +async def test_standby_request_without_standby_mode_raises_async( + httpserver: HTTPServer, async_client: ApifyClientAsync +) -> None: + """Async variant of `test_standby_request_without_standby_mode_raises`.""" + httpserver.expect_request(_ACTOR_PATH, method='GET').respond_with_json(actor_response(None)) + + with pytest.raises(ApifyClientError, match='does not have Standby mode enabled'): + await async_client.actor(_MOCKED_ACTOR_ID).standby_request() + + +def test_standby_request_missing_actor_raises_not_found(httpserver: HTTPServer, sync_client: ApifyClient) -> None: + """A missing Actor raises `NotFoundError`.""" + httpserver.expect_request(_ACTOR_PATH, method='GET').respond_with_json( + {'error': {'type': 'record-not-found', 'message': 'Actor was not found'}}, status=404 + ) + + with pytest.raises(NotFoundError): + sync_client.actor(_MOCKED_ACTOR_ID).standby_request() + + +def test_standby_request_error_status_raises(httpserver: HTTPServer, sync_client: ApifyClient) -> None: + """An error status from the Standby server raises the matching `ApifyApiError` subclass.""" + expect_standby_actor(httpserver) + httpserver.expect_request(f'{_STANDBY_PREFIX}/missing', method='GET').respond_with_data('Not here', status=404) + + with pytest.raises(NotFoundError, match='Not here'): + sync_client.actor(_MOCKED_ACTOR_ID).standby_request('/missing') diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index 00d07b9a..778651a5 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -783,6 +783,28 @@ def test_prepare_request_call_skips_compression_below_threshold(compressor_case: assert not any(key.lower() == 'content-encoding' for key in headers) +def test_prepare_request_call_skips_compression_when_disabled() -> None: + """With `compression=False`, a body above the threshold is sent verbatim with no `Content-Encoding`.""" + client = ConcreteHttpClient() + body = b'x' * MIN_COMPRESSION_SIZE + + headers, _params, data = client._prepare_request_call(data=body, compression=False) + + assert data == body + assert not any(key.lower() == 'content-encoding' for key in headers) + + +def test_prepare_request_call_uses_per_request_compressor() -> None: + """A compressor passed in `compression` is used for the request in place of the client's compressor.""" + client = ConcreteHttpClient(http_compressor=GzipHttpCompressor()) + body = b'x' * MIN_COMPRESSION_SIZE + + headers, _params, data = client._prepare_request_call(data=body, compression=BrotliHttpCompressor()) + + assert headers['Content-Encoding'] == 'br' + assert brotli.decompress(data) == body + + def test_prepare_request_call_compresses_bytearray_data(compressor_case: tuple) -> None: """A `bytearray` body above the threshold is compressed without error (regression: needs bytes conversion).""" compressor, content_encoding, decompress = compressor_case @@ -1219,6 +1241,22 @@ async def test_async_call_skips_thread_offload_for_a_body_it_cannot_compress( spy.assert_not_called() +async def test_async_call_skips_thread_offload_when_compression_is_disabled( + http_client_async_class: type[HttpClientAsync], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A raw body is prepared inline when compression is disabled, whatever its size.""" + client = http_client_async_class(token='test_token') + monkeypatch.setattr(client, 'send_request', AsyncMock(return_value=Mock(status_code=200))) + spy = _to_thread_spy(monkeypatch) + + await client.call( + method='PUT', url='https://api.test.com/endpoint', data=b'x' * MIN_COMPRESSION_SIZE, compression=False + ) + + spy.assert_not_called() + + class StreamingTransport(HttpClient): """A transport that drains every body it is handed and answers from a script of statuses and exceptions.""" diff --git a/tests/unit/test_pluggable_http_client.py b/tests/unit/test_pluggable_http_client.py index edbc6c0a..75387db4 100644 --- a/tests/unit/test_pluggable_http_client.py +++ b/tests/unit/test_pluggable_http_client.py @@ -35,6 +35,7 @@ from pytest_httpserver import HTTPServer + from apify_client.http_compressors import HttpCompressor from apify_client.types import StreamedBodySource, Timeout @@ -99,6 +100,7 @@ def call( json: Any = None, stream: bool | None = None, timeout: Timeout = 'medium', + compression: HttpCompressor | bool = True, ) -> HttpResponse: self.calls.append( { @@ -110,6 +112,7 @@ def call( 'json': json, 'stream': stream, 'timeout': timeout, + 'compression': compression, } ) return _make_fake_response() @@ -133,6 +136,7 @@ async def call( json: Any = None, stream: bool | None = None, timeout: Timeout = 'medium', + compression: HttpCompressor | bool = True, ) -> HttpResponse: self.calls.append( { @@ -144,6 +148,7 @@ async def call( 'json': json, 'stream': stream, 'timeout': timeout, + 'compression': compression, } ) return _make_fake_response()