diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80e33a8..8d19c8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - name: Checkout repository @@ -36,7 +36,7 @@ jobs: API_KEY: ${{ secrets.API_KEY }} - name: Run example tests - if: matrix.python-version == '3.14' + if: matrix.python-version == '3.15' run: pytest tests/example_search_*_test.py env: API_KEY: ${{ secrets.API_KEY }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 62943cc..7e37094 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.15"] steps: - uses: actions/checkout@v6 @@ -43,7 +43,7 @@ jobs: API_KEY: ${{ secrets.API_KEY }} - name: Run engine and documentation examples - if: matrix.python-version == '3.14' + if: matrix.python-version == '3.15' continue-on-error: ${{ inputs.allow_example_test_failures == true }} run: pytest --ignore=tests/test_docs_publishing.py -k "example" --require-docs-key env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e6e2303..180f214 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ Bug reports and pull requests are welcome on GitHub. Run commands from the repos ## Development setup -Use Python 3.13 to work on the package and its documentation. The SDK CI also tests Python 3.8 through 3.14. Create a virtual environment and install the package with its test and documentation dependencies: +The SDK CI tests Python 3.8 through 3.15. Create a virtual environment and install the package with its test and documentation dependencies: ```sh uv venv --python 3.13 @@ -31,10 +31,10 @@ The commands below use `python` from that environment. ### Checks that need no API key -These tests cover response parsing, image uploads with simulated responses, timeouts, exceptions, documentation syntax, and the example runner: +These tests cover response parsing, image uploads and extraction with simulated responses, timeouts, exceptions, documentation syntax, and the example runner: ```sh -python -m pytest tests/test_output_formats.py tests/test_image_upload.py tests/test_timeout.py tests/test_exceptions.py tests/test_docs_examples.py tests/test_docs_example_runner.py -k 'not live' -q +python -m pytest tests/test_output_formats.py tests/test_image_upload.py tests/test_extract.py tests/test_timeout.py tests/test_exceptions.py tests/test_docs_examples.py tests/test_docs_example_runner.py -k 'not live' -q ``` The file list limits this command to tests that make no external requests. Applying `-k 'not live'` to the entire suite is not enough to run offline: some SDK integration tests do not have `live` in their names. @@ -61,7 +61,7 @@ This includes live account, location, search, and pagination checks. To run ever python -m pytest -q ``` -Both commands make real SerpApi requests. The full suite can use more searches than a test run limited to the files you changed. PR CI also runs the standalone engine examples on Python 3.14. Documentation tests run in the package and documentation release workflows on release tags, and in manual documentation runs on `master`. +Both commands make real SerpApi requests. The full suite can use more searches than a test run limited to the files you changed. PR CI also runs the standalone engine examples on Python 3.15. Documentation tests run in the package and documentation release workflows on release tags, and in manual documentation runs on `master`. ### Testing documentation examples diff --git a/README.md b/README.md index 2af6912..f586dd8 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,48 @@ Environment variables are a secure, safe, and easy way to manage secrets. Set `export SERPAPI_KEY=` in your shell. Python accesses these variables from `os.environ["SERPAPI_KEY"]`. +### Extract content from a URL + +The [Extract API](https://serpapi.com/extract-api) supports single-page scraping, URL discovery, and crawling. Call `client.extract()` with a parameter dictionary or keyword arguments. The client sets `engine="extract"` automatically. + +Set `output="md"` to get the page's main content as a Markdown string: + +```python +import os +import serpapi + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=60) +markdown = client.extract(url="https://serpapi.com/", output="md") +print(markdown) +``` + +JSON is the default output format. It returns a dictionary-like `SerpResults` object with page information, structured data, links, and Markdown: + +```python +results = client.extract(url="https://serpapi.com/") +print(results["page_information"]["title"]) +print(results["markdown"]) +``` + +Use `output="html"` to get the retrieved HTML as a string: + +```python +html = client.extract(url="https://serpapi.com/", output="html") +``` + +Use `mode="map"` to discover URLs or `mode="crawl"` to extract multiple pages: + +```python +urls = client.extract(url="https://serpapi.com/", mode="map", include_sitemap=False) +pages = client.extract( + url="https://serpapi.com/", mode="crawl", max_pages=2, max_depth=1, timeout=180 +) +``` + +Crawl mode can consume one search credit per successfully extracted page, including the starting page. Set `max_pages` to limit the number of pages. + +See the [Extract API guide](https://github.com/serpapi/serpapi-python/blob/master/docs/user_guide/extract.md) for response examples and more options. + ### Error handling Unsuccessful requests raise `serpapi.HTTPError` or `serpapi.TimeoutError` exceptions. The returned status code will reflect the sort of error that occurred, please refer to [Status and Error Codes Documentation](https://serpapi.com/api-status-and-error-codes) for more details. @@ -267,15 +309,30 @@ results = client.search({ - API Documentation: [serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) ### Search Google Immersive Product + +Get a current `page_token` from Google Shopping before requesting product details: + ```python import os import serpapi client = serpapi.Client(api_key=os.getenv("API_KEY")) -results = client.search({ - 'engine': 'google_immersive_product', - 'page_token': 'eyJlaSI6Im5ZVmxaOXVVTDY2X3A4NFBqTnZELUFjIiwicHJvZHVjdGlkIjoiIiwiY2F0YWxvZ2lkIjoiNTE1NDU2NTc1NTc5MzcxMDY3NSIsImhlYWRsaW5lT2ZmZXJEb2NpZCI6IjI1MDkyMjcwMDUzMjk2NzQwODMiLCJpbWFnZURvY2lkIjoiMTYzOTg5MjU0MDcwMDU4MDA1NTQiLCJyZHMiOiJQQ18zNDg4MDE0MTg3ODgxNzc5NjU0fFBST0RfUENfMzQ4ODAxNDE4Nzg4MTc3OTY1NCIsInF1ZXJ5IjoibGcrdHYiLCJncGNpZCI6IjM0ODgwMTQxODc4ODE3Nzk2NTQiLCJtaWQiOiI1NzY0NjI3ODM3Nzc5MTUzMTMiLCJwdnQiOiJoZyIsInV1bGUiOm51bGx9=', -}) +shopping = client.search( + engine="google_shopping", + q="lg tv", + gl="us", + hl="en", + no_cache=True, +) +page_token = next( + product["immersive_product_page_token"] + for product in shopping["shopping_results"] + if product.get("immersive_product_page_token") +) +results = client.search( + engine="google_immersive_product", + page_token=page_token, +) ``` - API Documentation: [serpapi.com/google-immersive-product-api](https://serpapi.com/google-immersive-product-api) @@ -287,7 +344,7 @@ import serpapi client = serpapi.Client(api_key=os.getenv("API_KEY")) results = client.search({ 'engine': 'google_reverse_image', - 'image_url': 'https://i.imgur.com/5bGzZi7.jpg', + 'image_url': 'https://upload.wikimedia.org/wikipedia/commons/6/63/Wikipedia-logo.png', 'max_results': '1', }) ``` diff --git a/docs/index.md b/docs/index.md index f5c4537..f416f44 100644 --- a/docs/index.md +++ b/docs/index.md @@ -90,6 +90,7 @@ Use the same search parameter names as the [SerpApi API documentation](https://s - [Getting Started](user_guide/getting-started.md) covers installation and your first search. [Client Usage](user_guide/client-usage.md) explains how to make requests and read responses. - [AI Agents](ai-agents.md) covers search tools, Markdown results, and API references for agents. - [Output Formats](user_guide/output-formats.md) explains when to use JSON, Markdown, or HTML. +- [Extract API](user_guide/extract.md) covers page content in Markdown or JSON, URL discovery, and crawling. - [Migration Guide](user_guide/migrating-from-google-search-results.md) shows how to replace `google-search-results` with this package. - [Parameters and Engines](user_guide/parameters-and-engines.md) explains search parameters and how to test them in the [SerpApi Playground](https://serpapi.com/playground). - For scripts that collect results, see [pagination](user_guide/pagination.md), [timeouts and errors](user_guide/errors-and-timeouts.md), [request options](user_guide/request-options.md), and [account and locations](user_guide/account-and-locations.md). @@ -113,6 +114,7 @@ user_guide/client-usage User Guide Overview user_guide/parameters-and-engines user_guide/output-formats +user_guide/extract user_guide/pagination user_guide/errors-and-timeouts user_guide/account-and-locations diff --git a/docs/user_guide/client-usage.md b/docs/user_guide/client-usage.md index 8680c10..eccd43b 100644 --- a/docs/user_guide/client-usage.md +++ b/docs/user_guide/client-usage.md @@ -148,10 +148,13 @@ For one additional page, call `next_page()`: next_results = results.next_page() ``` -Use `yield_pages()` in a loop to read the current page and fetch more pages. This example stops after ten pages, or earlier if there is no next page: +Use `yield_pages()` in a loop to read the current page and fetch more pages. This example reads up to ten pages and stops if a page contains an `error` message: ```python for page_number, page in enumerate(results.yield_pages(max_pages=10), start=1): + if "error" in page: + print(page["error"]) + break current = page.get("serpapi_pagination", {}).get("current", page_number) print(current) ``` diff --git a/docs/user_guide/extract.md b/docs/user_guide/extract.md new file mode 100644 index 0000000..0fa6650 --- /dev/null +++ b/docs/user_guide/extract.md @@ -0,0 +1,107 @@ +--- +title: "Extract API" +description: "Extract page content, discover URLs, and crawl websites with the Python client." +--- + +# Extract API + +Use `client.extract()` to read content from a public URL. It supports single-page scraping, URL discovery, and crawling. Pass a parameter dictionary or keyword arguments, as you would with `client.search()`. The client sets `engine="extract"` automatically. + +See the [Extract API documentation](https://serpapi.com/extract-api) for all parameters and response fields. + +## Get Page Content as Markdown + +Set `output="md"` to get the page's main content as a Markdown string. + +After [setting your API key](getting-started.md#installation), run: + +```python +import os +import serpapi + +client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=60) +markdown = client.extract( + url="https://serpapi.com/", + output="md", +) + +print(markdown) +``` + +You can pass the returned string to an AI agent or save it to a file. + +## Get Structured Data as JSON + +JSON is the default output format. The SDK returns a dictionary-like `SerpResults` object with page information, meta tags, structured data, links, images, and Markdown. The sections returned depend on the target page. + +```python +results = client.extract(url="https://serpapi.com/") + +print(results["page_information"]["title"]) +print(results["markdown"]) +``` + +The `markdown` field contains the page's main content. Set `include_markdown=False` to omit it from the JSON response. + +The module-level `serpapi.extract(url=..., api_key=...)` function accepts the same parameters. + +## Get HTML + +Set `output="html"` to get the retrieved HTML as a string: + +```python +html = client.extract(url="https://serpapi.com/", output="html") +``` + +## Discover URLs + +Set `mode="map"` to discover URLs on a website: + +```python +results = client.extract( + url="https://serpapi.com/", + mode="map", + include_sitemap=False, + include_subdomains=False, +) + +for entry in results["urls"]: + print(entry["url"]) +``` + +Map mode includes sitemap URLs by default. Set `include_sitemap=False` to discover URLs from the page's links only. Set `include_subdomains=True` to include subdomains. + +## Crawl Multiple Pages + +Set `mode="crawl"` to follow links and extract pages: + +```python +results = client.extract( + url="https://serpapi.com/", + mode="crawl", + max_pages=2, + max_depth=1, + timeout=180, +) + +for page in results["pages"]: + print(page["url"]) + print(page["markdown"]) +``` + +Each entry in `results["pages"]` contains the extracted page's data, including its `markdown` field. The starting page has depth `0`. `max_pages` includes the starting page and defaults to `10`. Each successfully extracted page can consume one search credit. Set a page limit for your crawl and allow enough time for the HTTP request to finish. + +For requests you want to retrieve later, pass `"async": True` in a parameter dictionary and use the [Search Archive](async-search-archive.md). + +## Timeouts and Content Filters + +The `timeout=` keyword sets the SDK's HTTP timeout in seconds. Set it above the API fetch timeout, and allow time for all pages when crawling. Put the Extract API's fetch timeout in the parameter dictionary to send it in milliseconds: + +```python +results = client.extract( + {"url": "https://serpapi.com/", "timeout": 30000}, + timeout=60, +) +``` + +Use `include_tags` and `exclude_tags` to keep or remove content with CSS selectors, such as `include_tags="article, main"` or `exclude_tags="nav, footer"`. See the [Extract API parameters](https://serpapi.com/extract-api) for details and [Errors and Timeouts](errors-and-timeouts.md) for exception handling. diff --git a/docs/user_guide/index.md b/docs/user_guide/index.md index 9e1fd5b..4e58543 100644 --- a/docs/user_guide/index.md +++ b/docs/user_guide/index.md @@ -11,6 +11,7 @@ Start with [Getting Started](getting-started.md) to install the package and run - [Parameters and Engines](parameters-and-engines.md) - [Output Formats](output-formats.md) +- [Extract API](extract.md) - [Pagination](pagination.md) - [Errors and Timeouts](errors-and-timeouts.md) - [Account and Locations](account-and-locations.md) diff --git a/pyproject.toml b/pyproject.toml index 2c259a9..73c7799 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", "Programming Language :: Python :: Implementation :: CPython", "Natural Language :: English", "Topic :: Utilities", diff --git a/serpapi/core.py b/serpapi/core.py index f24e089..f00f7cd 100644 --- a/serpapi/core.py +++ b/serpapi/core.py @@ -55,6 +55,32 @@ def search(self, params: dict = None, **kwargs): return SerpResults.from_http_response(r, client=self) + def extract(self, params: dict = None, **kwargs): + """Extract content from a public URL using the Extract API. + + Accepts a parameter dictionary or keyword arguments, like :meth:`search`. + Sets ``engine=extract`` and returns a :class:`SerpResults ` object for JSON, or text for HTML and Markdown. + + .. code-block:: python + + results = client.extract(url="https://example.com/") + markdown = client.extract(url="https://example.com/", output="md") + results = client.extract({"url": "https://example.com/", "timeout": 30000}, timeout=60) + + :param params: Optional mapping of Extract API parameters, including the required ``url``. Put ``timeout`` here to set the API fetch timeout in milliseconds. + :param kwargs: Additional API parameters or HTTP request options, as in :meth:`search`. The ``timeout`` keyword controls the HTTP client timeout in seconds. + + ``mode`` accepts ``scrape`` (default), ``map``, or ``crawl``. ``output`` accepts ``json`` (default), ``html``, or ``md``. + JSON includes page content in the top-level ``markdown`` field by default, + or in each ``pages`` entry for crawls. Set ``include_markdown=False`` to omit it. + For a scraped page, ``output="md"`` returns the page Markdown without search metadata. + + Crawl mode can consume one search credit per successfully extracted page. Use ``max_pages`` to limit the number of pages. + """ + params = dict(params) if params is not None else {} + kwargs["engine"] = "extract" + return self.search(params, **kwargs) + def search_archive(self, params: dict = None, **kwargs): """Get a result from the SerpApi Search Archive API. @@ -190,6 +216,7 @@ def account(self, params: dict = None, **kwargs): # An un-authenticated client instance. _client = Client() search = _client.search +extract = _client.extract search_archive = _client.search_archive upload_image = _client.upload_image locations = _client.locations diff --git a/tests/docs_example_support.py b/tests/docs_example_support.py index 7d61d72..1d48679 100644 --- a/tests/docs_example_support.py +++ b/tests/docs_example_support.py @@ -48,7 +48,18 @@ def validate_response(response, path, params): if not isinstance(payload, (dict, list)) or not payload: raise AssertionError("Expected a nonempty JSON object or list") if isinstance(payload, dict): - if payload.get("error"): + # Google can report an empty final page as Success with an error message. + empty_final_page = ( + path in {"/search", "/search.json"} + and params.get("engine") == "google" + and payload.get("search_metadata", {}).get("status") == "Success" + and payload.get("search_information", {}).get("organic_results_state") == "Fully empty" + and payload.get("error") == "Google hasn't returned any results for this query." + and not payload.get("organic_results") + and not payload.get("serpapi_pagination", {}).get("next") + and int(params.get("start", 0)) > 0 + ) + if payload.get("error") and not empty_final_page: raise AssertionError(str(payload["error"])) if payload.get("search_metadata", {}).get("status") == "Error": raise AssertionError("Search metadata reports an error") diff --git a/tests/test_docs_example_runner.py b/tests/test_docs_example_runner.py index 507cddd..c0d5f8c 100644 --- a/tests/test_docs_example_runner.py +++ b/tests/test_docs_example_runner.py @@ -1,3 +1,4 @@ +import json import os import subprocess import sys @@ -111,6 +112,39 @@ def test_doc_response_check_rejects_json_for_text_output(output): validate_response(response, "/search", {"output": output}) +@pytest.mark.parametrize( + "start, overrides, accepted", + [ + pytest.param(70, {}, True, id="empty-final-page"), + pytest.param(0, {}, False, id="empty-first-page"), + pytest.param(70, {"search_metadata": {"status": "Error"}}, False, + id="failed-search"), + pytest.param(70, {"error": "Failed to fetch results"}, False, + id="different-error"), + pytest.param(70, {"serpapi_pagination": {"next": "https://serpapi.com/search?start=80"}}, + False, id="not-final-page"), + ], +) +def test_doc_response_check_handles_empty_google_pagination(start, overrides, accepted): + payload = { + "search_metadata": {"status": "Success"}, + "search_information": {"organic_results_state": "Fully empty"}, + "error": "Google hasn't returned any results for this query.", + } + payload.update(overrides) + response = requests.Response() + response.status_code = 200 + response.headers["Content-Type"] = "application/json" + response._content = json.dumps(payload).encode() + params = {"engine": "google", "start": str(start)} + + if accepted: + validate_response(response, "/search.json", params) + else: + with pytest.raises(AssertionError): + validate_response(response, "/search.json", params) + + def test_doc_gate_fails_without_a_key(): env = os.environ.copy() env.pop("SERPAPI_KEY", None) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 8ff68f8..02c0793 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -1,4 +1,5 @@ from unittest.mock import Mock +import pytest import requests import serpapi @@ -15,3 +16,18 @@ def test_http_error(): assert http_error.status_code == 401 assert http_error.error == "Invalid API key" assert http_error.response == mock_response + + +@pytest.mark.parametrize( + "cause, expected", + [ + (requests.exceptions.Timeout, serpapi.TimeoutError), + (requests.exceptions.ConnectionError, serpapi.HTTPConnectionError), + ], +) +def test_transport_errors_are_translated(monkeypatch, cause, expected): + client = serpapi.Client(api_key="test-key") + monkeypatch.setattr(client.session, "request", Mock(side_effect=cause("Request failed"))) + + with pytest.raises(expected): + client.search(engine="google", q="coffee") diff --git a/tests/test_extract.py b/tests/test_extract.py new file mode 100644 index 0000000..3c7121b --- /dev/null +++ b/tests/test_extract.py @@ -0,0 +1,155 @@ +import os +from unittest.mock import Mock +from urllib.parse import urlparse + +import pytest +import requests + +import serpapi + + +@pytest.fixture +def extract_request(monkeypatch): + response = requests.Response() + response.status_code = 200 + response.headers["Content-Type"] = "application/json" + response._content = b'{"markdown": "Example content."}' + request = Mock(return_value=response) + monkeypatch.setattr(requests.Session, "request", request) + return request + + +def test_extract_forwards_params_without_mutating_input(extract_request): + client = serpapi.Client(api_key="client-key", timeout=60) + params = { + "url": "https://example.com/?q=coffee&lang=en", + "engine": "google", + "mode": "map", + "include_markdown": False, + "timeout": 30000, + } + original = params.copy() + + result = client.extract(params, engine="bing", mode="crawl", max_pages=2) + + assert params == original + assert result["markdown"] == "Example content." + extract_request.assert_called_once() + _, sent = extract_request.call_args + assert sent["method"] == "GET" + assert sent["url"] == "https://serpapi.com/search" + assert sent["params"] == dict( + original, engine="extract", mode="crawl", max_pages=2, api_key="client-key" + ) + assert sent["timeout"] == 60 + + +def test_extract_separates_api_and_http_timeouts(extract_request): + client = serpapi.Client(api_key="client-key", timeout=60) + + client.extract( + {"url": "https://example.com/", "timeout": 30000}, + timeout=90, + ) + + _, sent = extract_request.call_args + assert sent["params"]["timeout"] == 30000 + assert sent["timeout"] == 90 + + +def test_module_extract_uses_request_key(extract_request): + result = serpapi.extract(url="https://example.com/", api_key="request-key") + + assert result["markdown"] == "Example content." + _, sent = extract_request.call_args + assert sent["params"] == { + "engine": "extract", + "url": "https://example.com/", + "api_key": "request-key", + } + + +@pytest.fixture +def extract_live_client(): + api_key = os.environ.get("API_KEY") or os.environ.get("SERPAPI_KEY") + if not api_key: + pytest.fail("Extract API live tests require API_KEY or SERPAPI_KEY") + client = serpapi.Client(api_key=api_key, timeout=180) + yield client + client.session.close() + + +def assert_success(result): + assert isinstance(result, serpapi.SerpResults) + assert "error" not in result + assert result["search_metadata"]["status"] == "Success" + + +def test_live_extract_scrape(extract_live_client): + result = extract_live_client.extract( + {"url": "https://serpapi.com/", "timeout": 30000}, + timeout=90, + ) + + assert_success(result) + assert "SerpApi" in result["page_information"]["title"] + assert result["page_information"]["final_url"] == "https://serpapi.com/" + assert isinstance(result["markdown"], str) + assert result["markdown"].strip() + + +def test_live_extract_map(extract_live_client): + result = extract_live_client.extract( + url="https://serpapi.com/", + mode="map", + include_sitemap=False, + include_subdomains=False, + ) + + assert_success(result) + urls = [entry["url"] for entry in result["urls"]] + assert urls + assert len(urls) == len(set(urls)) + assert all(urlparse(url).hostname == "serpapi.com" for url in urls) + assert all(urlparse(url).scheme in {"http", "https"} for url in urls) + + +def test_live_extract_crawl(extract_live_client): + result = extract_live_client.extract( + url="https://serpapi.com/", + mode="crawl", + max_pages=2, + max_depth=1, + ) + + assert_success(result) + pages = result["pages"] + assert len(pages) == 2 + assert pages[0]["url"] == "https://serpapi.com/" + assert pages[0]["depth"] == 0 + assert len({page["url"] for page in pages}) == 2 + for page in pages: + assert "error" not in page + assert page["markdown"] + assert page["depth"] <= 1 + + +@pytest.mark.parametrize("output", ["html", "md"]) +def test_live_extract_text(extract_live_client, output): + result = extract_live_client.extract(url="https://serpapi.com/", output=output) + + assert isinstance(result, str) + assert "SerpApi" in result + if output == "html": + assert "" in result.lower() + else: + assert "