diff --git a/.github/workflows/spec_drift.yaml b/.github/workflows/spec_drift.yaml new file mode 100644 index 00000000..f8c0510c --- /dev/null +++ b/.github/workflows/spec_drift.yaml @@ -0,0 +1,55 @@ +name: API spec drift + +on: + schedule: + - cron: "0 13 * * 1" + workflow_dispatch: + +permissions: + contents: read + issues: write + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v7 + - name: Set up Python 3.13 + uses: actions/setup-python@v6 + with: + python-version: "3.13" + - name: Install dependencies + env: + UV_LOCKED: "1" + UV_PYTHON: "3.13" + run: make install + - name: Check spec drift + id: drift + env: + UV_PYTHON: "3.13" + run: | + set -o pipefail + make check-spec-drift | tee drift.txt + - name: Report drift in an issue + if: failure() && steps.drift.outcome == 'failure' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if ! grep -qE '^(New drift|Stale baseline):' drift.txt; then + echo "The check failed without reporting drift; see the previous step." + exit 0 + fi + title="SDK models have drifted from the API spec" + body="$(printf '%s\n\n```\n%s\n```\n\nSee CONTRIBUTING.md, \"Keeping models in sync with the API\".' "$RUN_URL" "$(cat drift.txt)")" + number="$(gh issue list --state open --search "in:title \"$title\"" --json number --jq '.[0].number')" + if [ -n "$number" ]; then + gh issue comment "$number" --body "$body" + else + gh issue create --title "$title" --body "$body" + fi diff --git a/CHANGELOG.md b/CHANGELOG.md index 98cb8e57..1f610a70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## 0.47.0 + +### Features +* Add `skip_preflight` to `CreateWorkflow` and `UpdateWorkflow`. Set it to `True` to skip the job preflight check for a workflow's jobs; on update, leaving it unset keeps the current value. `WorkflowInformation.skip_preflight` reports the workflow's current setting. For a one-off job, set `"skip_preflight": true` in the `request_data` JSON passed to `jobs.create_job`. + +### Fixes +* Deserialize jobs the preflight check rejected. `JobStatus` and `JobProcessingStatus` gain `REJECTED`; previously `jobs.get_job` raised on a rejected job, and one rejected job failed the whole `jobs.list_jobs` page. `JobInformation.status` and `JobDetails.processing_status` now also accept a status this SDK version does not know, returning it as a plain `str` instead of raising. Known statuses are still returned as enum members, but the fields are typed `Union[JobStatus, str]` and `Union[JobProcessingStatus, str]`, so type-checked code that reads `.value` or `.name` from them needs to handle the `str` case. +* Expose why a job was rejected. `JobNodeDetails` gains `failure_category` (a `PreflightFailureCategory`, or a plain `str` for a category this SDK version does not know) and `failure_reason`, set on the node that failed preflight in `jobs.get_job_details`. Previously both were dropped during deserialization. + ## 0.46.6 ### Fixes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 817478ee..afa95c9e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,6 +17,19 @@ If you encounter any bugs or have suggestions for improvements, please open an i We will review and triage issues as quickly as possible. Our goal is to address bugs and incorporate improvements in the next release. +## Keeping models in sync with the API + +The models under `src/unstructured_client/models/` are maintained by hand. They are no longer regenerated from the API's OpenAPI spec, so a field or enum value added on the server does not reach this SDK until someone adds it. + +`make check-spec-drift` compares the public specs (`https://platform.unstructuredapp.io/openapi.json` and `https://api.unstructuredapp.io/general/openapi.json`) with the models in `unstructured_client.models.shared`. It reports every property and enum value the spec has that the matching model lacks. Gaps the SDK does not cover yet are recorded in `scripts/spec_drift_baseline.json`, so the check fails only on new drift, or on a baseline entry that no longer applies. + +The `API spec drift` workflow runs the check every Monday and opens or comments on an issue when it fails. To resolve it: + +1. Add new fields and enum values to the models, and update the matching page under `docs/models/`. + * A new enum value on a response model is the case that breaks callers. A closed enum raises on a value it does not know, and one bad item fails a whole list response. Response fields whose enum is expected to grow should accept unknown values. See `JobInformation.status` for the pattern: `Union[, UnrecognizedStr]` with `union_mode="left_to_right"`. +2. Acknowledge any gap you are deliberately not covering, and clear entries the SDK now covers, with `uv run python scripts/check_spec_drift.py --update-baseline`. +3. Add a `CHANGELOG.md` entry and bump `src/unstructured_client/_version.py`. + ## Contact If you have any questions or need further assistance, please feel free to reach out by opening an issue. diff --git a/Makefile b/Makefile index e78276c2..dd469234 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,10 @@ lint: uv run pylint --rcfile=pylintrc src uv run mypy src +.PHONY: check-spec-drift +check-spec-drift: + uv run python scripts/check_spec_drift.py + .PHONY: publish publish: ./scripts/publish.sh \ No newline at end of file diff --git a/_test_contract/platform_api/test_jobs.py b/_test_contract/platform_api/test_jobs.py index bfd39508..1e66959b 100644 --- a/_test_contract/platform_api/test_jobs.py +++ b/_test_contract/platform_api/test_jobs.py @@ -222,3 +222,118 @@ def test_create_job(httpx_mock, platform_client: UnstructuredClient, platform_ap assert job.status == "SCHEDULED" assert job.job_type == "template" assert job.created_at == datetime.fromisoformat("2025-06-22T11:37:21.648+00:00") + + +def test_list_jobs_tolerates_rejected_and_unrecognized_statuses( + httpx_mock, platform_client: UnstructuredClient, platform_api_url: str +): + url = f"{platform_api_url}/api/v1/jobs/" + + def job(job_id: str, status: str) -> dict: + return { + "created_at": "2025-06-22T11:37:21.648Z", + "id": job_id, + "status": status, + "workflow_id": "16b80fee-64dc-472d-8f26-1d7729b6423d", + "workflow_name": "test_workflow", + } + + httpx_mock.add_response( + method="GET", + headers={"Content-Type": "application/json"}, + json=[ + job("fcdc4994-eea5-425c-91fa-e03f2bd8030d", "REJECTED"), + job("a1b2c3d4-eea5-425c-91fa-e03f2bd8030d", "SOME_FUTURE_STATUS"), + job("b2c3d4e5-eea5-425c-91fa-e03f2bd8030d", "COMPLETED"), + ], + url=url, + ) + + jobs = platform_client.jobs.list_jobs(request=operations.ListJobsRequest()).response_list_jobs + + assert jobs[0].status is shared.JobStatus.REJECTED + assert type(jobs[1].status) is str + assert jobs[1].status == "SOME_FUTURE_STATUS" + assert jobs[2].status is shared.JobStatus.COMPLETED + + +@pytest.mark.parametrize( + ("processing_status", "expected"), + [ + ("REJECTED", shared.JobProcessingStatus.REJECTED), + ("SOME_FUTURE_STATUS", "SOME_FUTURE_STATUS"), + ], +) +def test_get_job_details_tolerates_rejected_and_unrecognized_statuses( + httpx_mock, + platform_client: UnstructuredClient, + platform_api_url: str, + processing_status: str, + expected, +): + url = f"{platform_api_url}/api/v1/jobs/fcdc4994-eea5-425c-91fa-e03f2bd8030d/details" + + httpx_mock.add_response( + method="GET", + headers={"Content-Type": "application/json"}, + json={ + "id": "fcdc4994-eea5-425c-91fa-e03f2bd8030d", + "node_stats": [], + "processing_status": processing_status, + }, + url=url, + ) + + details = platform_client.jobs.get_job_details( + request=operations.GetJobDetailsRequest(job_id="fcdc4994-eea5-425c-91fa-e03f2bd8030d") + ).job_details + + assert details.processing_status == expected + assert type(details.processing_status) is type(expected) + + +def test_get_job_details_exposes_preflight_failure( + httpx_mock, platform_client: UnstructuredClient, platform_api_url: str +): + url = f"{platform_api_url}/api/v1/jobs/fcdc4994-eea5-425c-91fa-e03f2bd8030d/details" + + def node(name: str, **failure) -> dict: + return { + "node_name": name, + "ready": 0, + "in_progress": 0, + "success": 0, + "failure": 0, + **failure, + } + + httpx_mock.add_response( + method="GET", + headers={"Content-Type": "application/json"}, + json={ + "id": "fcdc4994-eea5-425c-91fa-e03f2bd8030d", + "processing_status": "REJECTED", + "node_stats": [ + node( + "s3-source", + failure_category="AUTH_INVALID_CREDENTIAL", + failure_reason="The AWS Access Key Id you provided does not exist.", + ), + node("future-destination", failure_category="SOME_FUTURE_CATEGORY"), + node("partitioner", failure_category=None, failure_reason=None), + ], + }, + url=url, + ) + + details = platform_client.jobs.get_job_details( + request=operations.GetJobDetailsRequest(job_id="fcdc4994-eea5-425c-91fa-e03f2bd8030d") + ).job_details + + source, destination, partitioner = details.node_stats + assert source.failure_category is shared.PreflightFailureCategory.AUTH_INVALID_CREDENTIAL + assert source.failure_reason == "The AWS Access Key Id you provided does not exist." + assert type(destination.failure_category) is str + assert destination.failure_category == "SOME_FUTURE_CATEGORY" + assert partitioner.failure_category is None + assert partitioner.failure_reason is None diff --git a/_test_contract/platform_api/test_workflows.py b/_test_contract/platform_api/test_workflows.py index f6abe1a9..1b62a573 100644 --- a/_test_contract/platform_api/test_workflows.py +++ b/_test_contract/platform_api/test_workflows.py @@ -1,3 +1,4 @@ +import json from datetime import datetime import pytest @@ -256,4 +257,66 @@ def test_run_workflow(httpx_mock, platform_client: UnstructuredClient, platform_ new_job = run_workflow_response.job_information assert new_job.id == "fcdc4994-eea5-425c-91fa-e03f2bd8030d" assert new_job.workflow_name == "test_workflow" - assert new_job.status == "IN_PROGRESS" \ No newline at end of file + assert new_job.status == "IN_PROGRESS" + +WORKFLOW_RESPONSE = { + "created_at": "2025-06-22T11:37:21.648Z", + "destinations": [], + "id": "16b80fee-64dc-472d-8f26-1d7729b6423d", + "name": "test_workflow", + "sources": [], + "workflow_nodes": [], + "status": "active", +} + + +def test_create_workflow_sends_skip_preflight( + httpx_mock, platform_client: UnstructuredClient, platform_api_url: str +): + httpx_mock.add_response( + method="POST", + url=f"{platform_api_url}/api/v1/workflows/", + json={**WORKFLOW_RESPONSE, "skip_preflight": True}, + ) + + response = platform_client.workflows.create_workflow( + request=operations.CreateWorkflowRequest( + create_workflow=shared.CreateWorkflow( + name="test_workflow", + workflow_type="custom", + skip_preflight=True, + ) + ) + ) + + assert json.loads(httpx_mock.get_requests()[0].content)["skip_preflight"] is True + assert response.workflow_information.skip_preflight is True + + +@pytest.mark.parametrize( + ("update_workflow", "expected_body"), + [ + (shared.UpdateWorkflow(name="renamed"), {"name": "renamed"}), + (shared.UpdateWorkflow(skip_preflight=False), {"skip_preflight": False}), + (shared.UpdateWorkflow(skip_preflight=True), {"skip_preflight": True}), + ], +) +def test_update_workflow_sends_skip_preflight_only_when_set( + httpx_mock, + platform_client: UnstructuredClient, + platform_api_url: str, + update_workflow: shared.UpdateWorkflow, + expected_body: dict, +): + url = f"{platform_api_url}/api/v1/workflows/16b80fee-64dc-472d-8f26-1d7729b6423d" + httpx_mock.add_response(method="PUT", url=url, json=WORKFLOW_RESPONSE) + + response = platform_client.workflows.update_workflow( + request=operations.UpdateWorkflowRequest( + workflow_id="16b80fee-64dc-472d-8f26-1d7729b6423d", + update_workflow=update_workflow, + ) + ) + + assert json.loads(httpx_mock.get_requests()[0].content) == expected_body + assert response.workflow_information.skip_preflight is False diff --git a/_test_unstructured_client/unit/test_check_spec_drift.py b/_test_unstructured_client/unit/test_check_spec_drift.py new file mode 100644 index 00000000..a36a479a --- /dev/null +++ b/_test_unstructured_client/unit/test_check_spec_drift.py @@ -0,0 +1,52 @@ +from scripts.check_spec_drift import find_drift + + +def _spec(schemas: dict) -> dict: + return {"components": {"schemas": schemas}} + + +def test_matching_spec_reports_no_drift(): + spec = _spec( + { + "JobStatus": {"type": "string", "enum": ["SCHEDULED", "REJECTED"]}, + "CreateWorkflow": { + "properties": { + "name": {"type": "string"}, + "schedule": {"anyOf": [{"type": "string", "enum": ["daily"]}, {"type": "null"}]}, + } + }, + } + ) + assert find_drift([spec]) == {} + + +def test_reports_missing_enum_values_and_properties(): + spec = _spec( + { + "JobStatus": {"type": "string", "enum": ["SCHEDULED", "PAUSED"]}, + "CreateWorkflow": {"properties": {"name": {}, "brand_new_field": {}}}, + "SomeSchemaTheSdkDoesNotModel": {"properties": {"x": {}}}, + } + ) + assert find_drift([spec]) == { + "CreateWorkflow": ["brand_new_field"], + "JobStatus": ["PAUSED"], + } + + +def test_reports_missing_values_of_an_inline_property_enum(): + spec = _spec( + { + "CreateWorkflow": { + "properties": { + "schedule": { + "anyOf": [ + {"type": "string", "enum": ["daily", "every 30 minutes"]}, + {"type": "null"}, + ] + } + } + } + } + ) + assert find_drift([spec]) == {"CreateWorkflow.schedule": ["every 30 minutes"]} diff --git a/docs/models/shared/bodycreatejob.md b/docs/models/shared/bodycreatejob.md index 1632b184..0a04ae6a 100644 --- a/docs/models/shared/bodycreatejob.md +++ b/docs/models/shared/bodycreatejob.md @@ -6,4 +6,4 @@ | Field | Type | Required | Description | | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | | `input_files` | List[[shared.InputFiles](../../models/shared/inputfiles.md)] | :heavy_minus_sign: | N/A | -| `request_data` | *str* | :heavy_check_mark: | N/A | \ No newline at end of file +| `request_data` | *str* | :heavy_check_mark: | JSON-encoded object. Provide either `template_id` or `job_nodes`, not both. Set `"skip_preflight": true` to skip the job preflight check for this job. | \ No newline at end of file diff --git a/docs/models/shared/createworkflow.md b/docs/models/shared/createworkflow.md index a2f5f669..8a36269a 100644 --- a/docs/models/shared/createworkflow.md +++ b/docs/models/shared/createworkflow.md @@ -9,6 +9,7 @@ | `name` | *str* | :heavy_check_mark: | N/A | | `reprocess_all` | *OptionalNullable[bool]* | :heavy_minus_sign: | N/A | | `schedule` | [OptionalNullable[shared.Schedule]](../../models/shared/schedule.md) | :heavy_minus_sign: | N/A | +| `skip_preflight` | *OptionalNullable[bool]* | :heavy_minus_sign: | Skip the job preflight check for this workflow. | | `source_id` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | | `template_id` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | | `workflow_nodes` | List[[shared.WorkflowNode](../../models/shared/workflownode.md)] | :heavy_minus_sign: | N/A | diff --git a/docs/models/shared/jobdetails.md b/docs/models/shared/jobdetails.md index 232be238..89d42e47 100644 --- a/docs/models/shared/jobdetails.md +++ b/docs/models/shared/jobdetails.md @@ -8,4 +8,4 @@ | `id` | *str* | :heavy_check_mark: | N/A | | `message` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | | `node_stats` | List[[shared.JobNodeDetails](../../models/shared/jobnodedetails.md)] | :heavy_check_mark: | N/A | -| `processing_status` | [shared.JobProcessingStatus](../../models/shared/jobprocessingstatus.md) | :heavy_check_mark: | N/A | \ No newline at end of file +| `processing_status` | Union[[shared.JobProcessingStatus](../../models/shared/jobprocessingstatus.md), *str*] | :heavy_check_mark: | N/A | \ No newline at end of file diff --git a/docs/models/shared/jobinformation.md b/docs/models/shared/jobinformation.md index 2578774c..5bd9d456 100644 --- a/docs/models/shared/jobinformation.md +++ b/docs/models/shared/jobinformation.md @@ -11,6 +11,6 @@ | `job_type` | [Optional[shared.WorkflowJobType]](../../models/shared/workflowjobtype.md) | :heavy_minus_sign: | N/A | | `output_node_files` | List[[shared.NodeFileMetadata](../../models/shared/nodefilemetadata.md)] | :heavy_minus_sign: | N/A | | `runtime` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | -| `status` | [shared.JobStatus](../../models/shared/jobstatus.md) | :heavy_check_mark: | N/A | +| `status` | Union[[shared.JobStatus](../../models/shared/jobstatus.md), *str*] | :heavy_check_mark: | N/A | | `workflow_id` | *str* | :heavy_check_mark: | N/A | | `workflow_name` | *str* | :heavy_check_mark: | N/A | \ No newline at end of file diff --git a/docs/models/shared/jobnodedetails.md b/docs/models/shared/jobnodedetails.md index b20f0391..4330813a 100644 --- a/docs/models/shared/jobnodedetails.md +++ b/docs/models/shared/jobnodedetails.md @@ -3,12 +3,14 @@ ## Fields -| Field | Type | Required | Description | -| ----------------------- | ----------------------- | ----------------------- | ----------------------- | -| `failure` | *int* | :heavy_check_mark: | N/A | -| `in_progress` | *int* | :heavy_check_mark: | N/A | -| `node_name` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | -| `node_subtype` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | -| `node_type` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | -| `ready` | *int* | :heavy_check_mark: | N/A | -| `success` | *int* | :heavy_check_mark: | N/A | \ No newline at end of file +| Field | Type | Required | Description | +| ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `failure` | *int* | :heavy_check_mark: | N/A | +| `failure_category` | Union[[shared.PreflightFailureCategory](../../models/shared/preflightfailurecategory.md), *str*] | :heavy_minus_sign: | Why this node failed its preflight check. Set only on the failing node of a `REJECTED` job. | +| `failure_reason` | *OptionalNullable[str]* | :heavy_minus_sign: | Human-readable detail forwarded from the provider. Free-form and unstable: show it to a person, never parse it. | +| `in_progress` | *int* | :heavy_check_mark: | N/A | +| `node_name` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | +| `node_subtype` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | +| `node_type` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | +| `ready` | *int* | :heavy_check_mark: | N/A | +| `success` | *int* | :heavy_check_mark: | N/A | diff --git a/docs/models/shared/jobprocessingstatus.md b/docs/models/shared/jobprocessingstatus.md index 0e666dff..ba247c34 100644 --- a/docs/models/shared/jobprocessingstatus.md +++ b/docs/models/shared/jobprocessingstatus.md @@ -10,4 +10,7 @@ | `SUCCESS` | SUCCESS | | `COMPLETED_WITH_ERRORS` | COMPLETED_WITH_ERRORS | | `STOPPED` | STOPPED | -| `FAILED` | FAILED | \ No newline at end of file +| `FAILED` | FAILED | +| `REJECTED` | REJECTED | + +A status value this version of the SDK does not recognize is returned as a plain `str` rather than raising. diff --git a/docs/models/shared/jobstatus.md b/docs/models/shared/jobstatus.md index 8846b157..367fcf4f 100644 --- a/docs/models/shared/jobstatus.md +++ b/docs/models/shared/jobstatus.md @@ -9,4 +9,7 @@ | `IN_PROGRESS` | IN_PROGRESS | | `COMPLETED` | COMPLETED | | `STOPPED` | STOPPED | -| `FAILED` | FAILED | \ No newline at end of file +| `FAILED` | FAILED | +| `REJECTED` | REJECTED | + +A status value this version of the SDK does not recognize is returned as a plain `str` rather than raising. diff --git a/docs/models/shared/preflightfailurecategory.md b/docs/models/shared/preflightfailurecategory.md new file mode 100644 index 00000000..742c56e3 --- /dev/null +++ b/docs/models/shared/preflightfailurecategory.md @@ -0,0 +1,17 @@ +# PreflightFailureCategory + +Stable, machine-routable category for a node that failed its preflight check. + +## Values + +| Name | Value | +| --------------------------- | --------------------------- | +| `AUTH_INVALID_CREDENTIAL` | AUTH_INVALID_CREDENTIAL | +| `AUTH_PERMISSION_DENIED` | AUTH_PERMISSION_DENIED | +| `AUTH_NETWORK_UNREACHABLE` | AUTH_NETWORK_UNREACHABLE | +| `AUTH_PROVIDER_UNAVAILABLE` | AUTH_PROVIDER_UNAVAILABLE | +| `PROVIDER_RATE_LIMITED` | PROVIDER_RATE_LIMITED | +| `CONFIG_INVALID` | CONFIG_INVALID | +| `UNKNOWN` | UNKNOWN | + +A category this version of the SDK does not recognize is returned as a plain `str` rather than raising; treat it as `UNKNOWN`. diff --git a/docs/models/shared/updateworkflow.md b/docs/models/shared/updateworkflow.md index 5f3b2014..d3e80fdd 100644 --- a/docs/models/shared/updateworkflow.md +++ b/docs/models/shared/updateworkflow.md @@ -9,6 +9,7 @@ | `name` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | | `reprocess_all` | *OptionalNullable[bool]* | :heavy_minus_sign: | N/A | | `schedule` | [OptionalNullable[shared.UpdateWorkflowSchedule]](../../models/shared/updateworkflowschedule.md) | :heavy_minus_sign: | N/A | +| `skip_preflight` | *OptionalNullable[bool]* | :heavy_minus_sign: | Skip the job preflight check for this workflow. Omit to leave the current value unchanged. | | `source_id` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | | `template_id` | *OptionalNullable[str]* | :heavy_minus_sign: | N/A | | `workflow_nodes` | List[[shared.WorkflowNode](../../models/shared/workflownode.md)] | :heavy_minus_sign: | N/A | diff --git a/docs/models/shared/workflowinformation.md b/docs/models/shared/workflowinformation.md index f8c4b867..e121f749 100644 --- a/docs/models/shared/workflowinformation.md +++ b/docs/models/shared/workflowinformation.md @@ -11,6 +11,7 @@ | `name` | *str* | :heavy_check_mark: | N/A | | | `reprocess_all` | *Optional[bool]* | :heavy_minus_sign: | N/A | | | `schedule` | [OptionalNullable[shared.WorkflowSchedule]](../../models/shared/workflowschedule.md) | :heavy_minus_sign: | N/A | {
"crontab_entries": [
{
"cron_expression": "0 0 * * *"
}
]
} | +| `skip_preflight` | *Optional[bool]* | :heavy_minus_sign: | Whether the job preflight check is skipped for this workflow. | | | `sources` | List[*str*] | :heavy_check_mark: | N/A | | | `status` | [shared.WorkflowState](../../models/shared/workflowstate.md) | :heavy_check_mark: | N/A | | | `updated_at` | [date](https://docs.python.org/3/library/datetime.html#date-objects) | :heavy_minus_sign: | N/A | | diff --git a/scripts/check_spec_drift.py b/scripts/check_spec_drift.py new file mode 100644 index 00000000..4ff9194b --- /dev/null +++ b/scripts/check_spec_drift.py @@ -0,0 +1,149 @@ +"""Report API fields and enum values that the SDK's shared models do not cover. + +Each component schema in the public OpenAPI specs is matched by name to a model or +enum in ``unstructured_client.models.shared``. A property missing from a model, or a +value missing from an enum, is drift. An enum declared inline on a property is +compared with the enum its model field is typed with. + +Drift recorded in the baseline file is acknowledged; anything else, or a baseline +entry that no longer applies, fails the check. + + uv run python scripts/check_spec_drift.py + uv run python scripts/check_spec_drift.py --update-baseline +""" + +from __future__ import annotations + +import argparse +import enum +import json +import re +import sys +import urllib.request +from pathlib import Path +from typing import Any, Dict, List, Set, Type, Union, get_args + +import pydantic + +from unstructured_client.models import shared + +SPEC_URLS = [ + "https://platform.unstructuredapp.io/openapi.json", + "https://api.unstructuredapp.io/general/openapi.json", +] +BASELINE_PATH = Path(__file__).with_name("spec_drift_baseline.json") + +Drift = Dict[str, List[str]] +SdkType = Union[Type[pydantic.BaseModel], Type[enum.Enum]] + + +def _normalize(name: str) -> str: + return re.sub(r"[^a-z0-9]", "", name.lower()) + + +def _sdk_types() -> Dict[str, SdkType]: + types: Dict[str, SdkType] = {} + for name in dir(shared): + obj = getattr(shared, name) + if isinstance(obj, type) and issubclass(obj, (pydantic.BaseModel, enum.Enum)): + types.setdefault(_normalize(name), obj) + return types + + +def _inline_enum_values(schema: Any) -> Set[str]: + if not isinstance(schema, dict): + return set() + values = {str(value) for value in schema.get("enum", [])} + for key in ("anyOf", "oneOf", "allOf"): + for option in schema.get(key, []): + values |= _inline_enum_values(option) + return values | _inline_enum_values(schema.get("items")) + + +def _enum_values(annotation: Any) -> Set[str]: + if isinstance(annotation, type) and issubclass(annotation, enum.Enum): + return {str(member.value) for member in annotation} + values: Set[str] = set() + for arg in get_args(annotation): + values |= _enum_values(arg) + return values + + +def _fetch(url: str) -> dict: + with urllib.request.urlopen(url, timeout=30) as response: + return json.load(response) + + +def _schema_drift(name: str, schema: dict, sdk_type: SdkType) -> Dict[str, Set[str]]: + if issubclass(sdk_type, enum.Enum): + return {name: _inline_enum_values(schema) - _enum_values(sdk_type)} + + fields = {f.alias or n: f for n, f in sdk_type.model_fields.items()} + properties = schema.get("properties", {}) + drift = {name: set(properties) - set(fields)} + for prop_name, prop_schema in properties.items(): + field_enum = _enum_values(fields[prop_name].annotation) if prop_name in fields else set() + if field_enum: + drift[f"{name}.{prop_name}"] = _inline_enum_values(prop_schema) - field_enum + return drift + + +def find_drift(specs: List[dict]) -> Drift: + sdk_types = _sdk_types() + drift: Dict[str, Set[str]] = {} + for spec in specs: + for schema_name, schema in spec.get("components", {}).get("schemas", {}).items(): + sdk_type = sdk_types.get(_normalize(schema_name)) + if sdk_type is None: + continue + for name, missing in _schema_drift(schema_name, schema, sdk_type).items(): + drift.setdefault(name, set()).update(missing) + return {name: sorted(values) for name, values in sorted(drift.items()) if values} + + +def _subtract(drift: Drift, other: Drift) -> Drift: + diff: Drift = {} + for name, values in drift.items(): + extra = sorted(set(values) - set(other.get(name, []))) + if extra: + diff[name] = extra + return diff + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--update-baseline", + action="store_true", + help="Rewrite the baseline to acknowledge all current drift.", + ) + args = parser.parse_args() + + current = find_drift([_fetch(url) for url in SPEC_URLS]) + + if args.update_baseline: + BASELINE_PATH.write_text(json.dumps(current, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {BASELINE_PATH}") + return 0 + + baseline: Drift = json.loads(BASELINE_PATH.read_text(encoding="utf-8")) + new = _subtract(current, baseline) + stale = _subtract(baseline, current) + + for name, values in new.items(): + print(f"New drift: {name} is missing {', '.join(values)}") + for name, values in stale.items(): + print(f"Stale baseline: {name} is no longer missing {', '.join(values)}") + if new or stale: + print( + "\nCover new drift in the SDK models, then run with --update-baseline " + "to record any drift left uncovered and drop stale entries." + ) + return 1 + + print("No drift beyond the baseline.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/spec_drift_baseline.json b/scripts/spec_drift_baseline.json new file mode 100644 index 00000000..86ce1c79 --- /dev/null +++ b/scripts/spec_drift_baseline.json @@ -0,0 +1,84 @@ +{ + "Body_create_job": [ + "input_file_ids" + ], + "CreateDestinationConnector": [ + "key" + ], + "CreateSourceConnector": [ + "key" + ], + "CreateWorkflow": [ + "destination_ids", + "key", + "source_ids" + ], + "DestinationConnectorInformation": [ + "key" + ], + "DestinationConnectorType": [ + "teradata" + ], + "EncryptionType": [ + "runtime_config" + ], + "FailedFile": [ + "node" + ], + "JobDetails": [ + "completion_issue", + "progress", + "queued_behind_job_ids_snapshot", + "summary" + ], + "JobFailedFiles": [ + "limit", + "next_offset", + "offset", + "total" + ], + "JobInformation": [ + "completion_issue", + "created_by_id", + "failed_file_count", + "processing_status", + "reason", + "stop_requested" + ], + "JobNodeDetails": [ + "last_change_at", + "node_id", + "node_phase", + "node_phase_observed_at" + ], + "SourceConnectorInformation": [ + "key" + ], + "SourceConnectorType": [ + "teradata" + ], + "UpdateDestinationConnector": [ + "name" + ], + "UpdateSourceConnector": [ + "name" + ], + "UpdateWorkflow": [ + "state" + ], + "ValidationError": [ + "ctx", + "input" + ], + "WorkflowInformation": [ + "edit_lock", + "key" + ], + "partition_parameters": [ + "contextual_chunking_auth", + "contextual_chunking_service_name", + "isolate_table", + "skip_table_chunking", + "tracking_enabled" + ] +} diff --git a/src/unstructured_client/_version.py b/src/unstructured_client/_version.py index 5ab1a8bd..51a85b03 100644 --- a/src/unstructured_client/_version.py +++ b/src/unstructured_client/_version.py @@ -3,10 +3,10 @@ import importlib.metadata __title__: str = "unstructured-client" -__version__: str = "0.46.6" +__version__: str = "0.47.0" __openapi_doc_version__: str = "1.2.31" __gen_version__: str = "2.680.0" -__user_agent__: str = "speakeasy-sdk/python 0.46.6 2.680.0 1.2.31 unstructured-client" +__user_agent__: str = "speakeasy-sdk/python 0.47.0 2.680.0 1.2.31 unstructured-client" try: if __package__ is not None: diff --git a/src/unstructured_client/models/shared/__init__.py b/src/unstructured_client/models/shared/__init__.py index 422aeaa9..b69bd68d 100644 --- a/src/unstructured_client/models/shared/__init__.py +++ b/src/unstructured_client/models/shared/__init__.py @@ -61,6 +61,7 @@ Strategy, VLMModelProvider, ) + from .preflightfailurecategory import PreflightFailureCategory from .secretreference import SecretReference, SecretReferenceTypedDict from .security import Security, SecurityTypedDict from .sortdirection import SortDirection @@ -155,6 +156,7 @@ "OutputFormat", "PartitionParameters", "PartitionParametersTypedDict", + "PreflightFailureCategory", "Schedule", "SecretReference", "SecretReferenceTypedDict", @@ -250,6 +252,7 @@ "PartitionParametersTypedDict": ".partition_parameters", "Strategy": ".partition_parameters", "VLMModelProvider": ".partition_parameters", + "PreflightFailureCategory": ".preflightfailurecategory", "SecretReference": ".secretreference", "SecretReferenceTypedDict": ".secretreference", "Security": ".security", diff --git a/src/unstructured_client/models/shared/createworkflow.py b/src/unstructured_client/models/shared/createworkflow.py index 741f838d..5c78214a 100644 --- a/src/unstructured_client/models/shared/createworkflow.py +++ b/src/unstructured_client/models/shared/createworkflow.py @@ -36,6 +36,7 @@ class CreateWorkflowTypedDict(TypedDict): destination_id: NotRequired[Nullable[str]] reprocess_all: NotRequired[Nullable[bool]] schedule: NotRequired[Nullable[Schedule]] + skip_preflight: NotRequired[Nullable[bool]] source_id: NotRequired[Nullable[str]] template_id: NotRequired[Nullable[str]] workflow_nodes: NotRequired[Nullable[List[WorkflowNodeTypedDict]]] @@ -52,6 +53,8 @@ class CreateWorkflow(BaseModel): schedule: OptionalNullable[Schedule] = UNSET + skip_preflight: OptionalNullable[bool] = UNSET + source_id: OptionalNullable[str] = UNSET template_id: OptionalNullable[str] = UNSET @@ -64,6 +67,7 @@ def serialize_model(self, handler): "destination_id", "reprocess_all", "schedule", + "skip_preflight", "source_id", "template_id", "workflow_nodes", @@ -72,6 +76,7 @@ def serialize_model(self, handler): "destination_id", "reprocess_all", "schedule", + "skip_preflight", "source_id", "template_id", "workflow_nodes", diff --git a/src/unstructured_client/models/shared/jobdetails.py b/src/unstructured_client/models/shared/jobdetails.py index 6997b12a..9ba995fd 100644 --- a/src/unstructured_client/models/shared/jobdetails.py +++ b/src/unstructured_client/models/shared/jobdetails.py @@ -3,22 +3,23 @@ from __future__ import annotations from .jobnodedetails import JobNodeDetails, JobNodeDetailsTypedDict from .jobprocessingstatus import JobProcessingStatus -from pydantic import model_serializer -from typing import List -from typing_extensions import NotRequired, TypedDict +from pydantic import Field, model_serializer +from typing import List, Union +from typing_extensions import Annotated, NotRequired, TypedDict from unstructured_client.types import ( BaseModel, Nullable, OptionalNullable, UNSET, UNSET_SENTINEL, + UnrecognizedStr, ) class JobDetailsTypedDict(TypedDict): id: str node_stats: List[JobNodeDetailsTypedDict] - processing_status: JobProcessingStatus + processing_status: Union[JobProcessingStatus, UnrecognizedStr] message: NotRequired[Nullable[str]] @@ -27,7 +28,9 @@ class JobDetails(BaseModel): node_stats: List[JobNodeDetails] - processing_status: JobProcessingStatus + processing_status: Annotated[ + Union[JobProcessingStatus, UnrecognizedStr], Field(union_mode="left_to_right") + ] message: OptionalNullable[str] = UNSET diff --git a/src/unstructured_client/models/shared/jobinformation.py b/src/unstructured_client/models/shared/jobinformation.py index ee21007f..e75a03fa 100644 --- a/src/unstructured_client/models/shared/jobinformation.py +++ b/src/unstructured_client/models/shared/jobinformation.py @@ -5,22 +5,23 @@ from .nodefilemetadata import NodeFileMetadata, NodeFileMetadataTypedDict from .workflowjobtype import WorkflowJobType from datetime import datetime -from pydantic import model_serializer -from typing import List, Optional -from typing_extensions import NotRequired, TypedDict +from pydantic import Field, model_serializer +from typing import List, Optional, Union +from typing_extensions import Annotated, NotRequired, TypedDict from unstructured_client.types import ( BaseModel, Nullable, OptionalNullable, UNSET, UNSET_SENTINEL, + UnrecognizedStr, ) class JobInformationTypedDict(TypedDict): created_at: datetime id: str - status: JobStatus + status: Union[JobStatus, UnrecognizedStr] workflow_id: str workflow_name: str input_file_ids: NotRequired[Nullable[List[str]]] @@ -34,7 +35,9 @@ class JobInformation(BaseModel): id: str - status: JobStatus + status: Annotated[ + Union[JobStatus, UnrecognizedStr], Field(union_mode="left_to_right") + ] workflow_id: str diff --git a/src/unstructured_client/models/shared/jobnodedetails.py b/src/unstructured_client/models/shared/jobnodedetails.py index 5a9231c5..956130fd 100644 --- a/src/unstructured_client/models/shared/jobnodedetails.py +++ b/src/unstructured_client/models/shared/jobnodedetails.py @@ -1,14 +1,17 @@ """Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.""" from __future__ import annotations -from pydantic import model_serializer -from typing_extensions import NotRequired, TypedDict +from .preflightfailurecategory import PreflightFailureCategory +from pydantic import Field, model_serializer +from typing import Union +from typing_extensions import Annotated, NotRequired, TypedDict from unstructured_client.types import ( BaseModel, Nullable, OptionalNullable, UNSET, UNSET_SENTINEL, + UnrecognizedStr, ) @@ -17,6 +20,10 @@ class JobNodeDetailsTypedDict(TypedDict): in_progress: int ready: int success: int + failure_category: NotRequired[ + Nullable[Union[PreflightFailureCategory, UnrecognizedStr]] + ] + failure_reason: NotRequired[Nullable[str]] node_name: NotRequired[Nullable[str]] node_subtype: NotRequired[Nullable[str]] node_type: NotRequired[Nullable[str]] @@ -31,6 +38,17 @@ class JobNodeDetails(BaseModel): success: int + failure_category: OptionalNullable[ + Annotated[ + Union[PreflightFailureCategory, UnrecognizedStr], + Field(union_mode="left_to_right"), + ] + ] = UNSET + r"""Why this node failed its preflight check. Set only on the failing node of a `REJECTED` job.""" + + failure_reason: OptionalNullable[str] = UNSET + r"""Human-readable detail forwarded from the provider. Free-form and unstable: show it to a person, never parse it.""" + node_name: OptionalNullable[str] = UNSET node_subtype: OptionalNullable[str] = UNSET @@ -39,8 +57,20 @@ class JobNodeDetails(BaseModel): @model_serializer(mode="wrap") def serialize_model(self, handler): - optional_fields = ["node_name", "node_subtype", "node_type"] - nullable_fields = ["node_name", "node_subtype", "node_type"] + optional_fields = [ + "failure_category", + "failure_reason", + "node_name", + "node_subtype", + "node_type", + ] + nullable_fields = [ + "failure_category", + "failure_reason", + "node_name", + "node_subtype", + "node_type", + ] null_default_fields = [] serialized = handler(self) diff --git a/src/unstructured_client/models/shared/jobprocessingstatus.py b/src/unstructured_client/models/shared/jobprocessingstatus.py index 5e68d14c..d66fd3d1 100644 --- a/src/unstructured_client/models/shared/jobprocessingstatus.py +++ b/src/unstructured_client/models/shared/jobprocessingstatus.py @@ -11,3 +11,4 @@ class JobProcessingStatus(str, Enum): COMPLETED_WITH_ERRORS = "COMPLETED_WITH_ERRORS" STOPPED = "STOPPED" FAILED = "FAILED" + REJECTED = "REJECTED" diff --git a/src/unstructured_client/models/shared/jobstatus.py b/src/unstructured_client/models/shared/jobstatus.py index bf847dbb..7ff32348 100644 --- a/src/unstructured_client/models/shared/jobstatus.py +++ b/src/unstructured_client/models/shared/jobstatus.py @@ -10,3 +10,4 @@ class JobStatus(str, Enum): COMPLETED = "COMPLETED" STOPPED = "STOPPED" FAILED = "FAILED" + REJECTED = "REJECTED" diff --git a/src/unstructured_client/models/shared/preflightfailurecategory.py b/src/unstructured_client/models/shared/preflightfailurecategory.py new file mode 100644 index 00000000..151c3062 --- /dev/null +++ b/src/unstructured_client/models/shared/preflightfailurecategory.py @@ -0,0 +1,12 @@ +from __future__ import annotations +from enum import Enum + + +class PreflightFailureCategory(str, Enum): + AUTH_INVALID_CREDENTIAL = "AUTH_INVALID_CREDENTIAL" + AUTH_PERMISSION_DENIED = "AUTH_PERMISSION_DENIED" + AUTH_NETWORK_UNREACHABLE = "AUTH_NETWORK_UNREACHABLE" + AUTH_PROVIDER_UNAVAILABLE = "AUTH_PROVIDER_UNAVAILABLE" + PROVIDER_RATE_LIMITED = "PROVIDER_RATE_LIMITED" + CONFIG_INVALID = "CONFIG_INVALID" + UNKNOWN = "UNKNOWN" diff --git a/src/unstructured_client/models/shared/updateworkflow.py b/src/unstructured_client/models/shared/updateworkflow.py index 3f474988..73646422 100644 --- a/src/unstructured_client/models/shared/updateworkflow.py +++ b/src/unstructured_client/models/shared/updateworkflow.py @@ -35,6 +35,7 @@ class UpdateWorkflowTypedDict(TypedDict): name: NotRequired[Nullable[str]] reprocess_all: NotRequired[Nullable[bool]] schedule: NotRequired[Nullable[UpdateWorkflowSchedule]] + skip_preflight: NotRequired[Nullable[bool]] source_id: NotRequired[Nullable[str]] template_id: NotRequired[Nullable[str]] workflow_nodes: NotRequired[Nullable[List[WorkflowNodeTypedDict]]] @@ -50,6 +51,8 @@ class UpdateWorkflow(BaseModel): schedule: OptionalNullable[UpdateWorkflowSchedule] = UNSET + skip_preflight: OptionalNullable[bool] = UNSET + source_id: OptionalNullable[str] = UNSET template_id: OptionalNullable[str] = UNSET @@ -65,6 +68,7 @@ def serialize_model(self, handler): "name", "reprocess_all", "schedule", + "skip_preflight", "source_id", "template_id", "workflow_nodes", @@ -75,6 +79,7 @@ def serialize_model(self, handler): "name", "reprocess_all", "schedule", + "skip_preflight", "source_id", "template_id", "workflow_nodes", diff --git a/src/unstructured_client/models/shared/workflowinformation.py b/src/unstructured_client/models/shared/workflowinformation.py index 2025fa84..a649fa34 100644 --- a/src/unstructured_client/models/shared/workflowinformation.py +++ b/src/unstructured_client/models/shared/workflowinformation.py @@ -28,6 +28,7 @@ class WorkflowInformationTypedDict(TypedDict): workflow_nodes: List[WorkflowNodeTypedDict] reprocess_all: NotRequired[bool] schedule: NotRequired[Nullable[WorkflowScheduleTypedDict]] + skip_preflight: NotRequired[bool] updated_at: NotRequired[Nullable[datetime]] workflow_type: NotRequired[Nullable[WorkflowType]] @@ -51,13 +52,21 @@ class WorkflowInformation(BaseModel): schedule: OptionalNullable[WorkflowSchedule] = UNSET + skip_preflight: Optional[bool] = False + updated_at: OptionalNullable[datetime] = UNSET workflow_type: OptionalNullable[WorkflowType] = UNSET @model_serializer(mode="wrap") def serialize_model(self, handler): - optional_fields = ["reprocess_all", "schedule", "updated_at", "workflow_type"] + optional_fields = [ + "reprocess_all", + "schedule", + "skip_preflight", + "updated_at", + "workflow_type", + ] nullable_fields = ["schedule", "updated_at", "workflow_type"] null_default_fields = []