From 95c6553f6e605e18971b7b09e376e2eabd00222a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 09:23:01 -0700 Subject: [PATCH 01/79] Turned temporalio.common into a package. The Streams API moves into temporalio.common.streams, so the module becomes a package with the same contents. --- temporalio/{common.py => common/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename temporalio/{common.py => common/__init__.py} (100%) diff --git a/temporalio/common.py b/temporalio/common/__init__.py similarity index 100% rename from temporalio/common.py rename to temporalio/common/__init__.py From 8e0d7ced6dbcb5d93cd6e6298756deec078ebb3a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 09:26:19 -0700 Subject: [PATCH 02/79] Moved the Streams package to temporalio.common.streams. The first-class API lives next to the converter it registers with. The paths of the generators and the generated imports follow. --- pyproject.toml | 14 +- scripts/gen_protos.py | 8 +- scripts/gen_stream_service.py | 10 +- .../{contrib => common}/streams/__init__.py | 28 ++-- .../{contrib => common}/streams/_access.py | 16 +-- .../{contrib => common}/streams/_activity.py | 4 +- .../streams/_activity_stream.py | 12 +- .../{contrib => common}/streams/_backend.py | 10 +- .../{contrib => common}/streams/_buffered.py | 10 +- .../{contrib => common}/streams/_convert.py | 0 .../{contrib => common}/streams/_defn.py | 0 .../{contrib => common}/streams/_errors.py | 4 +- .../{contrib => common}/streams/_handle.py | 4 +- .../{contrib => common}/streams/_ids.py | 0 .../{contrib => common}/streams/_items.py | 2 +- .../{contrib => common}/streams/_plugin.py | 4 +- .../{contrib => common}/streams/_reader.py | 8 +- .../streams/_stream_client.py | 20 +-- .../{contrib => common}/streams/_writer.py | 8 +- .../{contrib => common}/streams/_writes.py | 8 +- .../streams/proto/__init__.py | 0 .../streams/proto/v1/__init__.py | 0 .../streams/proto/v1/extension_pb2.py | 24 ++-- .../streams/proto/v1/extension_pb2.pyi | 0 .../{contrib => common}/streams/redis.py | 20 +-- .../streams/system/__init__.py | 0 .../streams/system/_resources/__init__.py | 0 .../streams/system/_resources/stream.py | 0 .../system/_system_nexus_interceptor.py | 0 .../streams/system/client.py | 0 .../streams/system/models.py | 0 .../streams/system/operations/__init__.py | 0 .../system/operations/create_stream.py | 0 .../streams/system/services.py | 0 .../{contrib => common}/streams/testing.py | 4 +- .../{contrib => common}/streams/workflow.py | 10 +- .../streams/writer_system/__init__.py | 0 .../streams/writer_system/client.py | 0 .../streams/writer_system/models.py | 131 ++++++++---------- tests/contrib/streams/conftest.py | 6 +- .../examples/activity_streams_tokens.py | 4 +- .../streams/examples/agent_publishes_turns.py | 4 +- .../streams/examples/client_creates_stream.py | 4 +- .../streams/examples/resume_from_offset.py | 4 +- .../contrib/streams/examples/test_examples.py | 2 +- .../streams/examples/turn_scoped_stream.py | 4 +- .../streams/examples/ui_backend_reads.py | 4 +- .../contrib/streams/examples/worker_setup.py | 8 +- tests/contrib/streams/fake_extension.py | 2 +- .../contrib/streams/test_activity_handles.py | 7 +- tests/contrib/streams/test_bindings.py | 2 +- tests/contrib/streams/test_chain.py | 2 +- tests/contrib/streams/test_facade.py | 11 +- tests/contrib/streams/test_handles.py | 5 +- tests/contrib/streams/test_pydantic.py | 6 +- tests/contrib/streams/test_redis.py | 11 +- tests/contrib/streams/test_stream_client.py | 10 +- tests/contrib/streams/test_vectors.py | 4 +- tests/contrib/streams/test_workflow_module.py | 8 +- 59 files changed, 227 insertions(+), 240 deletions(-) rename temporalio/{contrib => common}/streams/__init__.py (71%) rename temporalio/{contrib => common}/streams/_access.py (93%) rename temporalio/{contrib => common}/streams/_activity.py (95%) rename temporalio/{contrib => common}/streams/_activity_stream.py (96%) rename temporalio/{contrib => common}/streams/_backend.py (96%) rename temporalio/{contrib => common}/streams/_buffered.py (96%) rename temporalio/{contrib => common}/streams/_convert.py (100%) rename temporalio/{contrib => common}/streams/_defn.py (100%) rename temporalio/{contrib => common}/streams/_errors.py (98%) rename temporalio/{contrib => common}/streams/_handle.py (99%) rename temporalio/{contrib => common}/streams/_ids.py (100%) rename temporalio/{contrib => common}/streams/_items.py (99%) rename temporalio/{contrib => common}/streams/_plugin.py (97%) rename temporalio/{contrib => common}/streams/_reader.py (97%) rename temporalio/{contrib => common}/streams/_stream_client.py (97%) rename temporalio/{contrib => common}/streams/_writer.py (94%) rename temporalio/{contrib => common}/streams/_writes.py (97%) rename temporalio/{contrib => common}/streams/proto/__init__.py (100%) rename temporalio/{contrib => common}/streams/proto/v1/__init__.py (100%) rename temporalio/{contrib => common}/streams/proto/v1/extension_pb2.py (90%) rename temporalio/{contrib => common}/streams/proto/v1/extension_pb2.pyi (100%) rename temporalio/{contrib => common}/streams/redis.py (97%) rename temporalio/{contrib => common}/streams/system/__init__.py (100%) rename temporalio/{contrib => common}/streams/system/_resources/__init__.py (100%) rename temporalio/{contrib => common}/streams/system/_resources/stream.py (100%) rename temporalio/{contrib => common}/streams/system/_system_nexus_interceptor.py (100%) rename temporalio/{contrib => common}/streams/system/client.py (100%) rename temporalio/{contrib => common}/streams/system/models.py (100%) rename temporalio/{contrib => common}/streams/system/operations/__init__.py (100%) rename temporalio/{contrib => common}/streams/system/operations/create_stream.py (100%) rename temporalio/{contrib => common}/streams/system/services.py (100%) rename temporalio/{contrib => common}/streams/testing.py (93%) rename temporalio/{contrib => common}/streams/workflow.py (98%) rename temporalio/{contrib => common}/streams/writer_system/__init__.py (100%) rename temporalio/{contrib => common}/streams/writer_system/client.py (100%) rename temporalio/{contrib => common}/streams/writer_system/models.py (82%) diff --git a/pyproject.toml b/pyproject.toml index ebbdc02bcf..af19d3f8f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -186,9 +186,9 @@ exclude = [ 'temporalio/api', 'temporalio/bridge/proto', 'temporalio/nexus/system/workflow_service', - 'temporalio/contrib/streams/proto', - 'temporalio/contrib/streams/system', - 'temporalio/contrib/streams/writer_system', + 'temporalio/common/streams/proto', + 'temporalio/common/streams/system', + 'temporalio/common/streams/writer_system', ] [[tool.mypy.overrides]] @@ -197,9 +197,9 @@ ignore_errors = true [[tool.mypy.overrides]] module = [ - "temporalio.contrib.streams.proto.*", - "temporalio.contrib.streams.system.*", - "temporalio.contrib.streams.writer_system.*", + "temporalio.common.streams.proto.*", + "temporalio.common.streams.system.*", + "temporalio.common.streams.writer_system.*", ] ignore_errors = true @@ -272,7 +272,7 @@ exclude = [ "temporalio/api", "temporalio/bridge/proto", "temporalio/bridge/_visitor.py", - "temporalio/contrib/streams/proto", + "temporalio/common/streams/proto", "tests/worker/workflow_sandbox/testmodules/proto", ] diff --git a/scripts/gen_protos.py b/scripts/gen_protos.py index 1ecfd31c6d..51b0a9ec44 100644 --- a/scripts/gen_protos.py +++ b/scripts/gen_protos.py @@ -33,7 +33,7 @@ api_out_dir = base_dir / "temporalio" / "api" sdk_out_dir = base_dir / "temporalio" / "bridge" / "proto" -streams_out_dir = base_dir / "temporalio" / "contrib" / "streams" / "proto" +streams_out_dir = base_dir / "temporalio" / "common" / "streams" / "proto" py_fixes = [ partial(re.compile(r"from temporal\.api\.").sub, r"from temporalio.api."), @@ -53,11 +53,11 @@ ), partial( re.compile(r"from temporal\.sdk\.streams\.").sub, - r"from temporalio.contrib.streams.proto.", + r"from temporalio.common.streams.proto.", ), partial( re.compile(r"'__module__' : 'temporal\.sdk\.streams\.").sub, - r"'__module__' : 'temporalio.contrib.streams.proto.", + r"'__module__' : 'temporalio.common.streams.proto.", ), partial( re.compile(r"'__module__' : 'temporal\.api\.").sub, @@ -82,7 +82,7 @@ partial(re.compile(r"temporal\.sdk\.core\.").sub, r"temporalio.bridge.proto."), partial( re.compile(r"temporal\.sdk\.streams\.").sub, - r"temporalio.contrib.streams.proto.", + r"temporalio.common.streams.proto.", ), ] diff --git a/scripts/gen_stream_service.py b/scripts/gen_stream_service.py index 473825f756..afe4f09347 100644 --- a/scripts/gen_stream_service.py +++ b/scripts/gen_stream_service.py @@ -1,4 +1,4 @@ -"""Generates the Stream service bindings of temporalio.contrib.streams. +"""Generates the Stream service bindings of temporalio.common.streams. The WIT and the protos are copies of the temporalio-streams crate's, in scripts/_wit and scripts/_proto. Run scripts/gen_protos.py first for the proto modules. No nexgen release has the @@ -30,9 +30,9 @@ / "v1" / "stream_service.proto" ) -output_dir = base_dir / "temporalio" / "contrib" / "streams" / "system" +output_dir = base_dir / "temporalio" / "common" / "streams" / "system" # The writer is client-only, so it has a caller and its resource class, and no Workflow code. -writer_output_dir = base_dir / "temporalio" / "contrib" / "streams" / "writer_system" +writer_output_dir = base_dir / "temporalio" / "common" / "streams" / "writer_system" NEX_GEN_REPOSITORY = "https://github.com/moetemp/nexgen" # The head of moe/brook/v3/n8-stream-resource. A branch moves, so the pin is the commit. @@ -81,13 +81,13 @@ def generate_stream_service() -> None: raise RuntimeError(f"missing generator input: {path}") with tempfile.TemporaryDirectory(dir=base_dir) as temp_dir: descriptor_path = Path(temp_dir) / "stream_service.bin" - # The extension messages live in temporalio.contrib.streams.proto, so the descriptor + # The extension messages live in temporalio.common.streams.proto, so the descriptor # names their file by that module path. extension_root = Path(temp_dir) / "extension" extension_path = ( extension_root / "temporalio" - / "contrib" + / "common" / "streams" / "proto" / "v1" diff --git a/temporalio/contrib/streams/__init__.py b/temporalio/common/streams/__init__.py similarity index 71% rename from temporalio/contrib/streams/__init__.py rename to temporalio/common/streams/__init__.py index 7f493ef9b0..c625a91e4b 100644 --- a/temporalio/contrib/streams/__init__.py +++ b/temporalio/common/streams/__init__.py @@ -2,14 +2,14 @@ A Stream definition, a class with ``@streams.defn``, gives the Stream name and the type of each Topic. Workflow code creates a Stream with -:py:func:`temporalio.contrib.streams.workflow.create_stream`. A client creates and gets Streams +:py:func:`temporalio.common.streams.workflow.create_stream`. A client creates and gets Streams with :py:class:`StreamClient`. A Stream is data: a Workflow passes it to an Activity, a child Workflow or the next run as an argument, and the parameter's type gives the handle class of that context, such as :py:class:`ActivityStream`. A Stream plugin carries the calls and registers a backend name with ``name=``, such as -:py:class:`temporalio.contrib.streams.redis.RedisStreamPlugin` (``redis``) and -:py:class:`temporalio.contrib.streams.testing.MemoryStreamPlugin` (``memory``). A handle +:py:class:`temporalio.common.streams.redis.RedisStreamPlugin` (``redis``) and +:py:class:`temporalio.common.streams.testing.MemoryStreamPlugin` (``memory``). A handle carries the name of its backend. A client may have two plugins, and exactly one of them has ``default=True``, the one that a create with no ``backend=`` uses. For now a Worker has one Stream plugin, so it serves one backend. @@ -18,23 +18,23 @@ This package is experimental. """ -from temporalio.contrib.streams import workflow -from temporalio.contrib.streams._activity_stream import ( +from temporalio.common.streams import workflow +from temporalio.common.streams._activity_stream import ( ActivityStream, ActivityTopic, SyncActivityStream, SyncActivityTopic, SyncSubscription, ) -from temporalio.contrib.streams._buffered import BufferedPublisher -from temporalio.contrib.streams._defn import ( +from temporalio.common.streams._buffered import BufferedPublisher +from temporalio.common.streams._defn import ( StreamDefinition, TopicDefinition, defn, topic, with_name, ) -from temporalio.contrib.streams._errors import ( +from temporalio.common.streams._errors import ( ReadOnlyContextError, StreamClosedError, StreamConfigError, @@ -48,22 +48,22 @@ StreamStorageError, StreamUnsupportedError, ) -from temporalio.contrib.streams._handle import StreamHandle -from temporalio.contrib.streams._items import ( +from temporalio.common.streams._handle import StreamHandle +from temporalio.common.streams._items import ( END, Closed, CloseStatus, Record, Superseded, ) -from temporalio.contrib.streams._plugin import StreamPlugin -from temporalio.contrib.streams._reader import Subscription -from temporalio.contrib.streams._stream_client import ( +from temporalio.common.streams._plugin import StreamPlugin +from temporalio.common.streams._reader import Subscription +from temporalio.common.streams._stream_client import ( ClientStream, ClientTopic, StreamClient, ) -from temporalio.contrib.streams.workflow import WorkflowStream, WorkflowTopic +from temporalio.common.streams.workflow import WorkflowStream, WorkflowTopic __all__ = [ "END", diff --git a/temporalio/contrib/streams/_access.py b/temporalio/common/streams/_access.py similarity index 93% rename from temporalio/contrib/streams/_access.py rename to temporalio/common/streams/_access.py index 814bc5c17e..1a7180ece2 100644 --- a/temporalio/contrib/streams/_access.py +++ b/temporalio/common/streams/_access.py @@ -8,14 +8,14 @@ from typing import Any, TypeVar, cast import temporalio.converter -from temporalio.contrib.streams import _convert, _reader -from temporalio.contrib.streams._backend import CallOptions, StreamBackend -from temporalio.contrib.streams._defn import TopicDefinition -from temporalio.contrib.streams._handle import StreamHandle, TopicBase -from temporalio.contrib.streams._items import End, close_status -from temporalio.contrib.streams._reader import Item, Position, Subscription -from temporalio.contrib.streams._writer import StreamWriter -from temporalio.contrib.streams.system import models +from temporalio.common.streams import _convert, _reader +from temporalio.common.streams._backend import CallOptions, StreamBackend +from temporalio.common.streams._defn import TopicDefinition +from temporalio.common.streams._handle import StreamHandle, TopicBase +from temporalio.common.streams._items import End, close_status +from temporalio.common.streams._reader import Item, Position, Subscription +from temporalio.common.streams._writer import StreamWriter +from temporalio.common.streams.system import models T = TypeVar("T") diff --git a/temporalio/contrib/streams/_activity.py b/temporalio/common/streams/_activity.py similarity index 95% rename from temporalio/contrib/streams/_activity.py rename to temporalio/common/streams/_activity.py index 09b2eafc4a..7c271331fe 100644 --- a/temporalio/contrib/streams/_activity.py +++ b/temporalio/common/streams/_activity.py @@ -9,8 +9,8 @@ import temporalio.api.streamservice.v1 as stream_pb import temporalio.converter -from temporalio.contrib.streams._backend import StreamBackend -from temporalio.contrib.streams._writer import StreamWriter +from temporalio.common.streams._backend import StreamBackend +from temporalio.common.streams._writer import StreamWriter @dataclass diff --git a/temporalio/contrib/streams/_activity_stream.py b/temporalio/common/streams/_activity_stream.py similarity index 96% rename from temporalio/contrib/streams/_activity_stream.py rename to temporalio/common/streams/_activity_stream.py index 9898332d24..86048dd96c 100644 --- a/temporalio/contrib/streams/_activity_stream.py +++ b/temporalio/common/streams/_activity_stream.py @@ -9,18 +9,18 @@ from typing import Any, Generic, TypeVar, cast, overload import temporalio.api.streamservice.v1 as stream_pb -from temporalio.contrib.streams import _activity, _errors -from temporalio.contrib.streams._access import ( +from temporalio.common.streams import _activity, _errors +from temporalio.common.streams._access import ( Access, SyncTopicBase, _AsyncTopic, read_spec, subscription, ) -from temporalio.contrib.streams._defn import TopicDefinition -from temporalio.contrib.streams._handle import StreamHandle -from temporalio.contrib.streams._items import Closed, End, Record, Superseded -from temporalio.contrib.streams._reader import Subscription +from temporalio.common.streams._defn import TopicDefinition +from temporalio.common.streams._handle import StreamHandle +from temporalio.common.streams._items import Closed, End, Record, Superseded +from temporalio.common.streams._reader import Subscription E = TypeVar("E") T = TypeVar("T") diff --git a/temporalio/contrib/streams/_backend.py b/temporalio/common/streams/_backend.py similarity index 96% rename from temporalio/contrib/streams/_backend.py rename to temporalio/common/streams/_backend.py index 197ee7d5fd..4380910a1d 100644 --- a/temporalio/contrib/streams/_backend.py +++ b/temporalio/common/streams/_backend.py @@ -17,13 +17,13 @@ import temporalio.bridge.streams import temporalio.converter import temporalio.service -from temporalio.contrib.streams import _errors -from temporalio.contrib.streams.proto import v1 as ext_pb -from temporalio.contrib.streams.system import client as service_client +from temporalio.common.streams import _errors +from temporalio.common.streams.proto import v1 as ext_pb +from temporalio.common.streams.system import client as service_client if TYPE_CHECKING: # The writer bindings import this package's protos, so they load after it. - from temporalio.contrib.streams.writer_system import client as writer_client + from temporalio.common.streams.writer_system import client as writer_client _M = TypeVar("_M", bound=google.protobuf.message.Message) @@ -164,7 +164,7 @@ def writers( self, namespace: str, data_converter: temporalio.converter.DataConverter ) -> writer_client.StreamWriterClient: """The generated writer client over the runtime.""" - from temporalio.contrib.streams.writer_system import client + from temporalio.common.streams.writer_system import client return client.StreamWriterClient( _ExtensionTransport(self, namespace), data_converter=data_converter diff --git a/temporalio/contrib/streams/_buffered.py b/temporalio/common/streams/_buffered.py similarity index 96% rename from temporalio/contrib/streams/_buffered.py rename to temporalio/common/streams/_buffered.py index 3156336461..09064a77ba 100644 --- a/temporalio/contrib/streams/_buffered.py +++ b/temporalio/common/streams/_buffered.py @@ -7,11 +7,11 @@ from types import TracebackType from typing import Any, Generic, TypeVar, overload -from temporalio.contrib.streams import _activity -from temporalio.contrib.streams._activity_stream import ActivityStream, ActivityTopic -from temporalio.contrib.streams._defn import TopicDefinition -from temporalio.contrib.streams._handle import TopicBase -from temporalio.contrib.streams._stream_client import ClientStream, ClientTopic +from temporalio.common.streams import _activity +from temporalio.common.streams._activity_stream import ActivityStream, ActivityTopic +from temporalio.common.streams._defn import TopicDefinition +from temporalio.common.streams._handle import TopicBase +from temporalio.common.streams._stream_client import ClientStream, ClientTopic T = TypeVar("T") U = TypeVar("U") diff --git a/temporalio/contrib/streams/_convert.py b/temporalio/common/streams/_convert.py similarity index 100% rename from temporalio/contrib/streams/_convert.py rename to temporalio/common/streams/_convert.py diff --git a/temporalio/contrib/streams/_defn.py b/temporalio/common/streams/_defn.py similarity index 100% rename from temporalio/contrib/streams/_defn.py rename to temporalio/common/streams/_defn.py diff --git a/temporalio/contrib/streams/_errors.py b/temporalio/common/streams/_errors.py similarity index 98% rename from temporalio/contrib/streams/_errors.py rename to temporalio/common/streams/_errors.py index 4b40bceb91..a6c36b5f8c 100644 --- a/temporalio/contrib/streams/_errors.py +++ b/temporalio/common/streams/_errors.py @@ -1,10 +1,10 @@ -"""Errors of temporalio.contrib.streams.""" +"""Errors of temporalio.common.streams.""" from __future__ import annotations import temporalio.exceptions import temporalio.service -from temporalio.contrib.streams.proto.v1 import StreamErrorDetails +from temporalio.common.streams.proto.v1 import StreamErrorDetails class StreamError(temporalio.exceptions.ApplicationError): diff --git a/temporalio/contrib/streams/_handle.py b/temporalio/common/streams/_handle.py similarity index 99% rename from temporalio/contrib/streams/_handle.py rename to temporalio/common/streams/_handle.py index f0f03e1d6e..bc4691e4a8 100644 --- a/temporalio/contrib/streams/_handle.py +++ b/temporalio/common/streams/_handle.py @@ -12,8 +12,8 @@ import temporalio.converter import temporalio.exceptions -from temporalio.contrib.streams import _errors -from temporalio.contrib.streams._defn import TopicDefinition, definition_of +from temporalio.common.streams import _errors +from temporalio.common.streams._defn import TopicDefinition, definition_of E = TypeVar("E") T = TypeVar("T") diff --git a/temporalio/contrib/streams/_ids.py b/temporalio/common/streams/_ids.py similarity index 100% rename from temporalio/contrib/streams/_ids.py rename to temporalio/common/streams/_ids.py diff --git a/temporalio/contrib/streams/_items.py b/temporalio/common/streams/_items.py similarity index 99% rename from temporalio/contrib/streams/_items.py rename to temporalio/common/streams/_items.py index 9148e8157e..ca0c3d5b06 100644 --- a/temporalio/contrib/streams/_items.py +++ b/temporalio/common/streams/_items.py @@ -9,7 +9,7 @@ from temporalio.api.streamservice.v1 import CloseStatus as _CloseStatus from temporalio.api.streamservice.v1 import ProducerKind -from temporalio.contrib.streams.system import models +from temporalio.common.streams.system import models from temporalio.exceptions import CancelledError T = TypeVar("T") diff --git a/temporalio/contrib/streams/_plugin.py b/temporalio/common/streams/_plugin.py similarity index 97% rename from temporalio/contrib/streams/_plugin.py rename to temporalio/common/streams/_plugin.py index 708fa139a3..920f97cc3c 100644 --- a/temporalio/contrib/streams/_plugin.py +++ b/temporalio/common/streams/_plugin.py @@ -7,8 +7,8 @@ import temporalio.client import temporalio.worker -from temporalio.contrib.streams import _errors -from temporalio.contrib.streams._backend import StreamBackend +from temporalio.common.streams import _errors +from temporalio.common.streams._backend import StreamBackend from temporalio.plugin import SimplePlugin diff --git a/temporalio/contrib/streams/_reader.py b/temporalio/common/streams/_reader.py similarity index 97% rename from temporalio/contrib/streams/_reader.py rename to temporalio/common/streams/_reader.py index a2b6f48353..425062c90d 100644 --- a/temporalio/contrib/streams/_reader.py +++ b/temporalio/common/streams/_reader.py @@ -10,8 +10,8 @@ import temporalio.converter from temporalio.api.streamservice.v1 import ReadStart, StreamRecordKind -from temporalio.contrib.streams import _backend, _convert, _errors -from temporalio.contrib.streams._items import ( +from temporalio.common.streams import _backend, _convert, _errors +from temporalio.common.streams._items import ( Closed, CloseStatus, End, @@ -20,8 +20,8 @@ producer_key, withdraws, ) -from temporalio.contrib.streams.system import client as service_client -from temporalio.contrib.streams.system import models +from temporalio.common.streams.system import client as service_client +from temporalio.common.streams.system import models T = TypeVar("T") diff --git a/temporalio/contrib/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py similarity index 97% rename from temporalio/contrib/streams/_stream_client.py rename to temporalio/common/streams/_stream_client.py index f5ad67d7b7..3a2513c2db 100644 --- a/temporalio/contrib/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -13,22 +13,22 @@ import temporalio.api.streamservice.v1 as stream_pb import temporalio.client -from temporalio.contrib.streams import _backend, _errors, _ids -from temporalio.contrib.streams._access import ( +from temporalio.common.streams import _backend, _errors, _ids +from temporalio.common.streams._access import ( Access, ReadSpec, _AsyncTopic, read_spec, subscription, ) -from temporalio.contrib.streams._backend import StreamBackend -from temporalio.contrib.streams._defn import TopicDefinition, definition_of -from temporalio.contrib.streams._handle import Creating, StreamHandle, exit_failure -from temporalio.contrib.streams._items import End -from temporalio.contrib.streams._plugin import client_backend -from temporalio.contrib.streams._reader import Item, Position, Subscription -from temporalio.contrib.streams._writer import StreamWriter -from temporalio.contrib.streams.system import models +from temporalio.common.streams._backend import StreamBackend +from temporalio.common.streams._defn import TopicDefinition, definition_of +from temporalio.common.streams._handle import Creating, StreamHandle, exit_failure +from temporalio.common.streams._items import End +from temporalio.common.streams._plugin import client_backend +from temporalio.common.streams._reader import Item, Position, Subscription +from temporalio.common.streams._writer import StreamWriter +from temporalio.common.streams.system import models E = TypeVar("E") T = TypeVar("T") diff --git a/temporalio/contrib/streams/_writer.py b/temporalio/common/streams/_writer.py similarity index 94% rename from temporalio/contrib/streams/_writer.py rename to temporalio/common/streams/_writer.py index a02f419129..670817997b 100644 --- a/temporalio/contrib/streams/_writer.py +++ b/temporalio/common/streams/_writer.py @@ -8,10 +8,10 @@ from typing import TYPE_CHECKING, Any import temporalio.api.streamservice.v1 as stream_pb -from temporalio.contrib.streams import _errors +from temporalio.common.streams import _errors if TYPE_CHECKING: - from temporalio.contrib.streams.writer_system import client as writer_client + from temporalio.common.streams.writer_system import client as writer_client class StreamWriter: @@ -62,7 +62,7 @@ async def open(self) -> writer_client.Writer: """Open the crate's writer once. The open of a new attempt withdraws the earlier one's Records, even when it writes nothing. """ - from temporalio.contrib.streams.writer_system import models + from temporalio.common.streams.writer_system import models async with self._open_lock: if self._writer is None: @@ -94,7 +94,7 @@ async def write(self, records: Sequence[tuple[str, Any]]) -> int: """Write ``(topic, value)`` pairs as Records now, in order. Returns the last Record's offset. """ - from temporalio.contrib.streams.writer_system import models + from temporalio.common.streams.writer_system import models if self._closed: raise _errors.StreamClosedError( diff --git a/temporalio/contrib/streams/_writes.py b/temporalio/common/streams/_writes.py similarity index 97% rename from temporalio/contrib/streams/_writes.py rename to temporalio/common/streams/_writes.py index b59c9aea5e..15b7f43a98 100644 --- a/temporalio/contrib/streams/_writes.py +++ b/temporalio/common/streams/_writes.py @@ -13,10 +13,10 @@ import temporalio.activity import temporalio.api.streamservice.v1 as stream_pb import temporalio.common -from temporalio.contrib.streams import _activity, _errors -from temporalio.contrib.streams.system import client as service_client -from temporalio.contrib.streams.system import models -from temporalio.contrib.streams.workflow import ( +from temporalio.common.streams import _activity, _errors +from temporalio.common.streams.system import client as service_client +from temporalio.common.streams.system import models +from temporalio.common.streams.workflow import ( APPEND_ACTIVITY, CLOSE_ACTIVITY, CREATE_ACTIVITY, diff --git a/temporalio/contrib/streams/proto/__init__.py b/temporalio/common/streams/proto/__init__.py similarity index 100% rename from temporalio/contrib/streams/proto/__init__.py rename to temporalio/common/streams/proto/__init__.py diff --git a/temporalio/contrib/streams/proto/v1/__init__.py b/temporalio/common/streams/proto/v1/__init__.py similarity index 100% rename from temporalio/contrib/streams/proto/v1/__init__.py rename to temporalio/common/streams/proto/v1/__init__.py diff --git a/temporalio/contrib/streams/proto/v1/extension_pb2.py b/temporalio/common/streams/proto/v1/extension_pb2.py similarity index 90% rename from temporalio/contrib/streams/proto/v1/extension_pb2.py rename to temporalio/common/streams/proto/v1/extension_pb2.py index b35975f103..3596df46f8 100644 --- a/temporalio/contrib/streams/proto/v1/extension_pb2.py +++ b/temporalio/common/streams/proto/v1/extension_pb2.py @@ -42,7 +42,7 @@ (_message.Message,), { "DESCRIPTOR": _REDISSTREAMCONFIG, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.RedisStreamConfig) }, ) @@ -53,7 +53,7 @@ (_message.Message,), { "DESCRIPTOR": _OPENWRITERREQUEST, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.OpenWriterRequest) }, ) @@ -64,7 +64,7 @@ (_message.Message,), { "DESCRIPTOR": _OPENWRITERRESPONSE, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.OpenWriterResponse) }, ) @@ -75,7 +75,7 @@ (_message.Message,), { "DESCRIPTOR": _WRITERECORDSREQUEST, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.WriteRecordsRequest) }, ) @@ -86,7 +86,7 @@ (_message.Message,), { "DESCRIPTOR": _WRITERECORDSRESPONSE, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.WriteRecordsResponse) }, ) @@ -97,7 +97,7 @@ (_message.Message,), { "DESCRIPTOR": _CLOSEWRITERREQUEST, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.CloseWriterRequest) }, ) @@ -108,7 +108,7 @@ (_message.Message,), { "DESCRIPTOR": _CLOSEWRITERRESPONSE, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.CloseWriterResponse) }, ) @@ -119,7 +119,7 @@ (_message.Message,), { "DESCRIPTOR": _FREEWRITERREQUEST, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.FreeWriterRequest) }, ) @@ -130,7 +130,7 @@ (_message.Message,), { "DESCRIPTOR": _FREEWRITERRESPONSE, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.FreeWriterResponse) }, ) @@ -141,7 +141,7 @@ (_message.Message,), { "DESCRIPTOR": _CLOSEWRITERSREQUEST, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.CloseWritersRequest) }, ) @@ -152,7 +152,7 @@ (_message.Message,), { "DESCRIPTOR": _CLOSEWRITERSRESPONSE, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.CloseWritersResponse) }, ) @@ -163,7 +163,7 @@ (_message.Message,), { "DESCRIPTOR": _STREAMERRORDETAILS, - "__module__": "temporalio.contrib.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common.streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.StreamErrorDetails) }, ) diff --git a/temporalio/contrib/streams/proto/v1/extension_pb2.pyi b/temporalio/common/streams/proto/v1/extension_pb2.pyi similarity index 100% rename from temporalio/contrib/streams/proto/v1/extension_pb2.pyi rename to temporalio/common/streams/proto/v1/extension_pb2.pyi diff --git a/temporalio/contrib/streams/redis.py b/temporalio/common/streams/redis.py similarity index 97% rename from temporalio/contrib/streams/redis.py rename to temporalio/common/streams/redis.py index c3cdcad1e3..4c7f3c1e7f 100644 --- a/temporalio/contrib/streams/redis.py +++ b/temporalio/common/streams/redis.py @@ -28,15 +28,15 @@ import temporalio.service import temporalio.worker import temporalio.workflow -from temporalio.contrib.streams import _activity, _ids, _writes -from temporalio.contrib.streams import workflow as streams_workflow -from temporalio.contrib.streams._backend import StreamBackend, _ExtensionStreamBackend -from temporalio.contrib.streams._errors import StreamConfigError -from temporalio.contrib.streams._handle import StreamHandle -from temporalio.contrib.streams._items import close_status -from temporalio.contrib.streams._plugin import StreamPlugin -from temporalio.contrib.streams.proto import v1 as ext_pb -from temporalio.contrib.streams.system import models +from temporalio.common.streams import _activity, _ids, _writes +from temporalio.common.streams import workflow as streams_workflow +from temporalio.common.streams._backend import StreamBackend, _ExtensionStreamBackend +from temporalio.common.streams._errors import StreamConfigError +from temporalio.common.streams._handle import StreamHandle +from temporalio.common.streams._items import close_status +from temporalio.common.streams._plugin import StreamPlugin +from temporalio.common.streams.proto import v1 as ext_pb +from temporalio.common.streams.system import models from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner logger = logging.getLogger(__name__) @@ -164,7 +164,7 @@ def configure_worker( config["workflow_runner"] = dataclasses.replace( runner, restrictions=runner.restrictions.with_passthrough_modules( - "temporalio.contrib.streams" + "temporalio.common.streams" ), ) return config diff --git a/temporalio/contrib/streams/system/__init__.py b/temporalio/common/streams/system/__init__.py similarity index 100% rename from temporalio/contrib/streams/system/__init__.py rename to temporalio/common/streams/system/__init__.py diff --git a/temporalio/contrib/streams/system/_resources/__init__.py b/temporalio/common/streams/system/_resources/__init__.py similarity index 100% rename from temporalio/contrib/streams/system/_resources/__init__.py rename to temporalio/common/streams/system/_resources/__init__.py diff --git a/temporalio/contrib/streams/system/_resources/stream.py b/temporalio/common/streams/system/_resources/stream.py similarity index 100% rename from temporalio/contrib/streams/system/_resources/stream.py rename to temporalio/common/streams/system/_resources/stream.py diff --git a/temporalio/contrib/streams/system/_system_nexus_interceptor.py b/temporalio/common/streams/system/_system_nexus_interceptor.py similarity index 100% rename from temporalio/contrib/streams/system/_system_nexus_interceptor.py rename to temporalio/common/streams/system/_system_nexus_interceptor.py diff --git a/temporalio/contrib/streams/system/client.py b/temporalio/common/streams/system/client.py similarity index 100% rename from temporalio/contrib/streams/system/client.py rename to temporalio/common/streams/system/client.py diff --git a/temporalio/contrib/streams/system/models.py b/temporalio/common/streams/system/models.py similarity index 100% rename from temporalio/contrib/streams/system/models.py rename to temporalio/common/streams/system/models.py diff --git a/temporalio/contrib/streams/system/operations/__init__.py b/temporalio/common/streams/system/operations/__init__.py similarity index 100% rename from temporalio/contrib/streams/system/operations/__init__.py rename to temporalio/common/streams/system/operations/__init__.py diff --git a/temporalio/contrib/streams/system/operations/create_stream.py b/temporalio/common/streams/system/operations/create_stream.py similarity index 100% rename from temporalio/contrib/streams/system/operations/create_stream.py rename to temporalio/common/streams/system/operations/create_stream.py diff --git a/temporalio/contrib/streams/system/services.py b/temporalio/common/streams/system/services.py similarity index 100% rename from temporalio/contrib/streams/system/services.py rename to temporalio/common/streams/system/services.py diff --git a/temporalio/contrib/streams/testing.py b/temporalio/common/streams/testing.py similarity index 93% rename from temporalio/contrib/streams/testing.py rename to temporalio/common/streams/testing.py index 5988c28668..64dbc95943 100644 --- a/temporalio/contrib/streams/testing.py +++ b/temporalio/common/streams/testing.py @@ -9,8 +9,8 @@ from datetime import timedelta import temporalio.runtime -from temporalio.contrib.streams.proto import v1 as ext_pb -from temporalio.contrib.streams.redis import _ExtensionStreamPlugin +from temporalio.common.streams.proto import v1 as ext_pb +from temporalio.common.streams.redis import _ExtensionStreamPlugin class MemoryStreamPlugin(_ExtensionStreamPlugin): diff --git a/temporalio/contrib/streams/workflow.py b/temporalio/common/streams/workflow.py similarity index 98% rename from temporalio/contrib/streams/workflow.py rename to temporalio/common/streams/workflow.py index b22671361a..d8182171a9 100644 --- a/temporalio/contrib/streams/workflow.py +++ b/temporalio/common/streams/workflow.py @@ -27,16 +27,16 @@ import temporalio.converter import temporalio.exceptions import temporalio.workflow -from temporalio.contrib.streams import _errors, _ids -from temporalio.contrib.streams._defn import TopicDefinition, definition_of -from temporalio.contrib.streams._handle import ( +from temporalio.common.streams import _errors, _ids +from temporalio.common.streams._defn import TopicDefinition, definition_of +from temporalio.common.streams._handle import ( Creating, StreamHandle, TopicBase, exit_failure, ) -from temporalio.contrib.streams._items import close_status -from temporalio.contrib.streams.system import models +from temporalio.common.streams._items import close_status +from temporalio.common.streams.system import models E = TypeVar("E") T = TypeVar("T") diff --git a/temporalio/contrib/streams/writer_system/__init__.py b/temporalio/common/streams/writer_system/__init__.py similarity index 100% rename from temporalio/contrib/streams/writer_system/__init__.py rename to temporalio/common/streams/writer_system/__init__.py diff --git a/temporalio/contrib/streams/writer_system/client.py b/temporalio/common/streams/writer_system/client.py similarity index 100% rename from temporalio/contrib/streams/writer_system/client.py rename to temporalio/common/streams/writer_system/client.py diff --git a/temporalio/contrib/streams/writer_system/models.py b/temporalio/common/streams/writer_system/models.py similarity index 82% rename from temporalio/contrib/streams/writer_system/models.py rename to temporalio/common/streams/writer_system/models.py index 189ca8840c..0542ae5ded 100644 --- a/temporalio/contrib/streams/writer_system/models.py +++ b/temporalio/common/streams/writer_system/models.py @@ -10,7 +10,7 @@ import typing_extensions import temporalio.api.streamservice.v1.stream_service_pb2 -import temporalio.contrib.streams.proto.v1.extension_pb2 +import temporalio.common.streams.proto.v1.extension_pb2 import temporalio.converter import temporalio.nexus.system._support as _support @@ -30,17 +30,17 @@ class OpenWriterRequest: class _OpenWriterRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ OpenWriterRequest, - temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterRequest, + temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterRequest] | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterRequest + type[temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest] | None + ) = temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterRequest, + value: temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest, type_hint: type[OpenWriterRequest], ) -> OpenWriterRequest: return OpenWriterRequest( @@ -70,8 +70,8 @@ def from_transfer_type( def to_transfer_type( self, value: OpenWriterRequest, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterRequest: - message = temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterRequest() + ) -> temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest: + message = temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest() if value.stream is not None: message.stream.CopyFrom( _StreamRefTransferTypeConverter().to_transfer_type(value.stream) @@ -416,18 +416,17 @@ class WriterResult: class _WriterResultTransferTypeConverter( temporalio.converter.TransferTypeConverter[ WriterResult, - temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterResponse, + temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterResponse] - | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterResponse + type[temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse] | None + ) = temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterResponse, + value: temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse, type_hint: type[WriterResult], ) -> WriterResult: return WriterResult( @@ -438,8 +437,8 @@ def from_transfer_type( def to_transfer_type( self, value: WriterResult, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterResponse: - message = temporalio.contrib.streams.proto.v1.extension_pb2.OpenWriterResponse() + ) -> temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse: + message = temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse() if value.writer_id is not None: message.writer_id = value.writer_id return message @@ -459,18 +458,18 @@ class WriteRecordsRequest: class _WriteRecordsRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ WriteRecordsRequest, - temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsRequest, + temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsRequest] + type[temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest] | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsRequest + ) = temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsRequest, + value: temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest, type_hint: type[WriteRecordsRequest], ) -> WriteRecordsRequest: return WriteRecordsRequest( @@ -487,10 +486,8 @@ def from_transfer_type( def to_transfer_type( self, value: WriteRecordsRequest, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsRequest: - message = ( - temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsRequest() - ) + ) -> temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest: + message = temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest() if value.writer_id is not None: message.writer_id = value.writer_id if value.records: @@ -566,18 +563,18 @@ class WriteRecordsResponse: class _WriteRecordsResponseTransferTypeConverter( temporalio.converter.TransferTypeConverter[ WriteRecordsResponse, - temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsResponse, + temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsResponse] + type[temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse] | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsResponse + ) = temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsResponse, + value: temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse, type_hint: type[WriteRecordsResponse], ) -> WriteRecordsResponse: return WriteRecordsResponse( @@ -589,9 +586,9 @@ def from_transfer_type( def to_transfer_type( self, value: WriteRecordsResponse, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsResponse: + ) -> temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse: message = ( - temporalio.contrib.streams.proto.v1.extension_pb2.WriteRecordsResponse() + temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse() ) if value.first_offset is not None: message.first_offset = value.first_offset @@ -613,18 +610,17 @@ class CloseWriterRequest: class _CloseWriterRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ CloseWriterRequest, - temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterRequest, + temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterRequest] - | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterRequest + type[temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest] | None + ) = temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterRequest, + value: temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest, type_hint: type[CloseWriterRequest], ) -> CloseWriterRequest: return CloseWriterRequest( @@ -635,8 +631,8 @@ def from_transfer_type( def to_transfer_type( self, value: CloseWriterRequest, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterRequest: - message = temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterRequest() + ) -> temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest: + message = temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest() if value.writer_id is not None: message.writer_id = value.writer_id return message @@ -655,18 +651,18 @@ class CloseWriterResponse: class _CloseWriterResponseTransferTypeConverter( temporalio.converter.TransferTypeConverter[ CloseWriterResponse, - temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterResponse, + temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterResponse] + type[temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse] | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterResponse + ) = temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterResponse, + value: temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse, type_hint: type[CloseWriterResponse], ) -> CloseWriterResponse: return CloseWriterResponse( @@ -677,10 +673,8 @@ def from_transfer_type( def to_transfer_type( self, value: CloseWriterResponse, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterResponse: - message = ( - temporalio.contrib.streams.proto.v1.extension_pb2.CloseWriterResponse() - ) + ) -> temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse: + message = temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse() if value.last_offset is not None: message.last_offset = value.last_offset return message @@ -699,17 +693,17 @@ class FreeWriterRequest: class _FreeWriterRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ FreeWriterRequest, - temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterRequest, + temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterRequest] | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterRequest + type[temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest] | None + ) = temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterRequest, + value: temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest, type_hint: type[FreeWriterRequest], ) -> FreeWriterRequest: return FreeWriterRequest( @@ -720,8 +714,8 @@ def from_transfer_type( def to_transfer_type( self, value: FreeWriterRequest, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterRequest: - message = temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterRequest() + ) -> temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest: + message = temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest() if value.writer_id is not None: message.writer_id = value.writer_id return message @@ -740,18 +734,17 @@ class FreeWriterResponse: class _FreeWriterResponseTransferTypeConverter( temporalio.converter.TransferTypeConverter[ FreeWriterResponse, - temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterResponse, + temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterResponse] - | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterResponse + type[temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse] | None + ) = temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterResponse, + value: temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse, type_hint: type[FreeWriterResponse], ) -> FreeWriterResponse: return FreeWriterResponse() @@ -760,8 +753,8 @@ def from_transfer_type( def to_transfer_type( self, value: FreeWriterResponse, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterResponse: - message = temporalio.contrib.streams.proto.v1.extension_pb2.FreeWriterResponse() + ) -> temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse: + message = temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse() return message @@ -778,18 +771,18 @@ class CloseWritersRequest: class _CloseWritersRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ CloseWritersRequest, - temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersRequest, + temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersRequest] + type[temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest] | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersRequest + ) = temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersRequest, + value: temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest, type_hint: type[CloseWritersRequest], ) -> CloseWritersRequest: return CloseWritersRequest() @@ -798,10 +791,8 @@ def from_transfer_type( def to_transfer_type( self, value: CloseWritersRequest, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersRequest: - message = ( - temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersRequest() - ) + ) -> temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest: + message = temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest() return message @@ -818,18 +809,18 @@ class CloseWritersResponse: class _CloseWritersResponseTransferTypeConverter( temporalio.converter.TransferTypeConverter[ CloseWritersResponse, - temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersResponse, + temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse, ] ): transfer_type: ( - type[temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersResponse] + type[temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse] | None - ) = temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersResponse + ) = temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersResponse, + value: temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse, type_hint: type[CloseWritersResponse], ) -> CloseWritersResponse: return CloseWritersResponse( @@ -840,9 +831,9 @@ def from_transfer_type( def to_transfer_type( self, value: CloseWritersResponse, - ) -> temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersResponse: + ) -> temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse: message = ( - temporalio.contrib.streams.proto.v1.extension_pb2.CloseWritersResponse() + temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse() ) if value.failed is not None: message.failed = value.failed diff --git a/tests/contrib/streams/conftest.py b/tests/contrib/streams/conftest.py index c2cde1e5e1..8853732587 100644 --- a/tests/contrib/streams/conftest.py +++ b/tests/contrib/streams/conftest.py @@ -9,9 +9,9 @@ import pytest from temporalio.client import Client -from temporalio.contrib.streams import StreamPlugin -from temporalio.contrib.streams.redis import RedisStreamPlugin -from temporalio.contrib.streams.testing import MemoryStreamPlugin +from temporalio.common.streams import StreamPlugin +from temporalio.common.streams.redis import RedisStreamPlugin +from temporalio.common.streams.testing import MemoryStreamPlugin # Ports that other local services own. _RESERVED = {7333, 7861, 7931, 6431, 7101, 7102, 7103} diff --git a/tests/contrib/streams/examples/activity_streams_tokens.py b/tests/contrib/streams/examples/activity_streams_tokens.py index 49303c502a..87ab07c12c 100644 --- a/tests/contrib/streams/examples/activity_streams_tokens.py +++ b/tests/contrib/streams/examples/activity_streams_tokens.py @@ -14,8 +14,8 @@ from temporalio import activity, workflow from temporalio.client import Client -from temporalio.contrib import streams -from temporalio.contrib.streams import workflow as stream_workflow +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker diff --git a/tests/contrib/streams/examples/agent_publishes_turns.py b/tests/contrib/streams/examples/agent_publishes_turns.py index cd50961991..2995c86c3a 100644 --- a/tests/contrib/streams/examples/agent_publishes_turns.py +++ b/tests/contrib/streams/examples/agent_publishes_turns.py @@ -13,8 +13,8 @@ from temporalio import activity, workflow from temporalio.client import Client -from temporalio.contrib import streams -from temporalio.contrib.streams import workflow as stream_workflow +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker diff --git a/tests/contrib/streams/examples/client_creates_stream.py b/tests/contrib/streams/examples/client_creates_stream.py index 5894524d77..c47b8b8638 100644 --- a/tests/contrib/streams/examples/client_creates_stream.py +++ b/tests/contrib/streams/examples/client_creates_stream.py @@ -12,8 +12,8 @@ from temporalio import workflow from temporalio.client import Client -from temporalio.contrib import streams -from temporalio.contrib.streams import workflow as stream_workflow +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker diff --git a/tests/contrib/streams/examples/resume_from_offset.py b/tests/contrib/streams/examples/resume_from_offset.py index 57468479f9..0da3156643 100644 --- a/tests/contrib/streams/examples/resume_from_offset.py +++ b/tests/contrib/streams/examples/resume_from_offset.py @@ -12,8 +12,8 @@ from temporalio import workflow from temporalio.client import Client, WorkflowHandle -from temporalio.contrib import streams -from temporalio.contrib.streams import workflow as stream_workflow +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker diff --git a/tests/contrib/streams/examples/test_examples.py b/tests/contrib/streams/examples/test_examples.py index 4b0a6d69cf..f7c72eb190 100644 --- a/tests/contrib/streams/examples/test_examples.py +++ b/tests/contrib/streams/examples/test_examples.py @@ -3,7 +3,7 @@ import uuid from temporalio.client import Client -from temporalio.contrib import streams +from temporalio.common import streams from tests.contrib.streams.examples import ( activity_streams_tokens, agent_publishes_turns, diff --git a/tests/contrib/streams/examples/turn_scoped_stream.py b/tests/contrib/streams/examples/turn_scoped_stream.py index 826df20aa6..c3c3e3b1d5 100644 --- a/tests/contrib/streams/examples/turn_scoped_stream.py +++ b/tests/contrib/streams/examples/turn_scoped_stream.py @@ -13,8 +13,8 @@ from temporalio import activity, workflow from temporalio.client import Client -from temporalio.contrib import streams -from temporalio.contrib.streams import workflow as stream_workflow +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker diff --git a/tests/contrib/streams/examples/ui_backend_reads.py b/tests/contrib/streams/examples/ui_backend_reads.py index 006cea2651..b2558238a7 100644 --- a/tests/contrib/streams/examples/ui_backend_reads.py +++ b/tests/contrib/streams/examples/ui_backend_reads.py @@ -13,8 +13,8 @@ from temporalio import activity, workflow from temporalio.client import Client -from temporalio.contrib import streams -from temporalio.contrib.streams import workflow as stream_workflow +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker diff --git a/tests/contrib/streams/examples/worker_setup.py b/tests/contrib/streams/examples/worker_setup.py index e29bda15f0..841587f333 100644 --- a/tests/contrib/streams/examples/worker_setup.py +++ b/tests/contrib/streams/examples/worker_setup.py @@ -11,10 +11,10 @@ from temporalio import workflow from temporalio.client import Client -from temporalio.contrib import streams -from temporalio.contrib.streams import workflow as stream_workflow -from temporalio.contrib.streams.redis import RedisStreamPlugin -from temporalio.contrib.streams.testing import MemoryStreamPlugin +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow +from temporalio.common.streams.redis import RedisStreamPlugin +from temporalio.common.streams.testing import MemoryStreamPlugin from temporalio.worker import Worker diff --git a/tests/contrib/streams/fake_extension.py b/tests/contrib/streams/fake_extension.py index 715fa04466..727b52ca87 100644 --- a/tests/contrib/streams/fake_extension.py +++ b/tests/contrib/streams/fake_extension.py @@ -6,7 +6,7 @@ import temporalio.api.streamservice.v1 as stream_pb from temporalio.bridge.streams import StreamCallError -from temporalio.contrib.streams.proto import v1 as ext_pb +from temporalio.common.streams.proto import v1 as ext_pb from temporalio.service import RPCStatusCode diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index b53a144a6d..38cadfaa9d 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -13,10 +13,9 @@ from temporalio import activity, workflow from temporalio.client import Client, WorkflowHandle -from temporalio.common import RetryPolicy -from temporalio.contrib import streams -from temporalio.contrib.streams import _activity -from temporalio.contrib.streams import workflow as stream_workflow +from temporalio.common import RetryPolicy, streams +from temporalio.common.streams import _activity +from temporalio.common.streams import workflow as stream_workflow from temporalio.exceptions import CancelledError from tests.helpers import new_worker diff --git a/tests/contrib/streams/test_bindings.py b/tests/contrib/streams/test_bindings.py index 2780c8123e..95e5d0dcb4 100644 --- a/tests/contrib/streams/test_bindings.py +++ b/tests/contrib/streams/test_bindings.py @@ -4,7 +4,7 @@ import temporalio.api.streamservice.v1 as stream_pb import temporalio.converter import temporalio.nexus.system as nexus_system -from temporalio.contrib.streams.system import client, models +from temporalio.common.streams.system import client, models class FakeTransport: diff --git a/tests/contrib/streams/test_chain.py b/tests/contrib/streams/test_chain.py index 06b439eddb..041324442e 100644 --- a/tests/contrib/streams/test_chain.py +++ b/tests/contrib/streams/test_chain.py @@ -5,7 +5,7 @@ import pytest from temporalio.common import RetryPolicy -from temporalio.contrib.streams.redis import chain_continues +from temporalio.common.streams.redis import chain_continues from temporalio.exceptions import ApplicationError START = datetime(2026, 10, 21, tzinfo=timezone.utc) diff --git a/tests/contrib/streams/test_facade.py b/tests/contrib/streams/test_facade.py index 3421cb56ec..4089582909 100644 --- a/tests/contrib/streams/test_facade.py +++ b/tests/contrib/streams/test_facade.py @@ -10,12 +10,11 @@ import temporalio.api.streamservice.v1 as stream_pb from temporalio.bridge.streams import StreamCallError from temporalio.client import Client -from temporalio.common import RawValue -from temporalio.contrib import streams -from temporalio.contrib.streams import _errors -from temporalio.contrib.streams._backend import _ExtensionStreamBackend -from temporalio.contrib.streams.proto import v1 as ext_pb -from temporalio.contrib.streams.system import models +from temporalio.common import RawValue, streams +from temporalio.common.streams import _errors +from temporalio.common.streams._backend import _ExtensionStreamBackend +from temporalio.common.streams.proto import v1 as ext_pb +from temporalio.common.streams.system import models from temporalio.exceptions import ApplicationError from temporalio.service import RPCError, RPCStatusCode from tests.contrib.streams.fake_extension import FakeExtension diff --git a/tests/contrib/streams/test_handles.py b/tests/contrib/streams/test_handles.py index 2d8d6267a9..b00d3d66f4 100644 --- a/tests/contrib/streams/test_handles.py +++ b/tests/contrib/streams/test_handles.py @@ -5,10 +5,9 @@ import pydantic import pytest -from temporalio.common import RawValue -from temporalio.contrib import streams +from temporalio.common import RawValue, streams +from temporalio.common.streams._defn import definition_of from temporalio.contrib.pydantic import pydantic_data_converter -from temporalio.contrib.streams._defn import definition_of from temporalio.converter import DataConverter E = TypeVar("E") diff --git a/tests/contrib/streams/test_pydantic.py b/tests/contrib/streams/test_pydantic.py index 29c11e422b..53ebe89e1b 100644 --- a/tests/contrib/streams/test_pydantic.py +++ b/tests/contrib/streams/test_pydantic.py @@ -5,10 +5,10 @@ from temporalio import activity, workflow from temporalio.client import Client -from temporalio.contrib import streams +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow +from temporalio.common.streams.testing import MemoryStreamPlugin from temporalio.contrib.pydantic import pydantic_data_converter -from temporalio.contrib.streams import workflow as stream_workflow -from temporalio.contrib.streams.testing import MemoryStreamPlugin from tests.helpers import new_worker diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index eedf145456..87f1e379a1 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -13,16 +13,15 @@ from temporalio import activity, workflow from temporalio.api.common.v1 import Payload from temporalio.client import Client, Interceptor, WorkflowFailureError, WorkflowHandle -from temporalio.common import RawValue, RetryPolicy -from temporalio.contrib import streams -from temporalio.contrib.streams import StreamPlugin -from temporalio.contrib.streams import workflow as stream_workflow -from temporalio.contrib.streams.redis import ( +from temporalio.common import RawValue, RetryPolicy, streams +from temporalio.common.streams import StreamPlugin +from temporalio.common.streams import workflow as stream_workflow +from temporalio.common.streams.redis import ( RedisStreamPlugin, _ExtensionStreamPlugin, _WorkflowInbound, ) -from temporalio.contrib.streams.testing import MemoryStreamPlugin +from temporalio.common.streams.testing import MemoryStreamPlugin from temporalio.converter import DataConverter, PayloadCodec from temporalio.exceptions import ApplicationError from temporalio.worker import ( diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index a189031d03..acc768940a 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -9,11 +9,11 @@ from temporalio import workflow from temporalio.client import Client -from temporalio.contrib import streams -from temporalio.contrib.streams import workflow as stream_workflow -from temporalio.contrib.streams._backend import _ExtensionStreamBackend -from temporalio.contrib.streams.redis import RedisStreamPlugin -from temporalio.contrib.streams.testing import MemoryStreamPlugin +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow +from temporalio.common.streams._backend import _ExtensionStreamBackend +from temporalio.common.streams.redis import RedisStreamPlugin +from temporalio.common.streams.testing import MemoryStreamPlugin from temporalio.converter import DataConverter from tests.contrib.streams.fake_extension import FakeExtension from tests.helpers import new_worker diff --git a/tests/contrib/streams/test_vectors.py b/tests/contrib/streams/test_vectors.py index a8ae5dfa45..6580f10d56 100644 --- a/tests/contrib/streams/test_vectors.py +++ b/tests/contrib/streams/test_vectors.py @@ -4,8 +4,8 @@ import pytest import temporalio.api.streamservice.v1 as stream_pb -from temporalio.contrib.streams import _ids, _items -from temporalio.contrib.streams.system import models +from temporalio.common.streams import _ids, _items +from temporalio.common.streams.system import models from tests.contrib.streams._support import crate_file, vectors _KINDS = { diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index cbb78e93c3..eba3416d7d 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -17,10 +17,10 @@ WorkflowFailureError, WorkflowHandle, ) -from temporalio.contrib import streams -from temporalio.contrib.streams import workflow as stream_workflow -from temporalio.contrib.streams._backend import _ExtensionStreamBackend -from temporalio.contrib.streams._defn import definition_of +from temporalio.common import streams +from temporalio.common.streams import workflow as stream_workflow +from temporalio.common.streams._backend import _ExtensionStreamBackend +from temporalio.common.streams._defn import definition_of from temporalio.converter import DataConverter, PayloadCodec from temporalio.exceptions import ApplicationError from temporalio.worker import Replayer From e55fb79127c4d5f6295c84a5edc483c0d0002682 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 09:32:58 -0700 Subject: [PATCH 03/79] Removed auto_close, close_with_run and the run-end close on Redis. A Stream's lifecycle is its owner's to manage, so a run end closes nothing. drain() still lets unawaited writes finish before the run ends. --- temporalio/common/streams/_stream_client.py | 16 +- temporalio/common/streams/_writes.py | 3 - temporalio/common/streams/redis.py | 217 ++---------------- temporalio/common/streams/workflow.py | 31 +-- .../examples/activity_streams_tokens.py | 1 + .../streams/examples/agent_publishes_turns.py | 8 +- .../streams/examples/client_creates_stream.py | 3 +- .../streams/examples/turn_scoped_stream.py | 3 +- .../streams/examples/ui_backend_reads.py | 2 + .../contrib/streams/examples/worker_setup.py | 1 + .../contrib/streams/test_activity_handles.py | 9 +- tests/contrib/streams/test_chain.py | 80 ------- tests/contrib/streams/test_pydantic.py | 1 + tests/contrib/streams/test_redis.py | 82 +++---- tests/contrib/streams/test_stream_client.py | 14 +- tests/contrib/streams/test_workflow_module.py | 67 ++---- 16 files changed, 99 insertions(+), 439 deletions(-) delete mode 100644 tests/contrib/streams/test_chain.py diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index 3a2513c2db..8a5bf35300 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -460,23 +460,19 @@ def create( *, id: str, retention: timedelta | None = None, - auto_close: timedelta | None = None, backend: str | None = None, rpc_metadata: Mapping[str, str | bytes] = {}, rpc_timeout: timedelta | None = None, ) -> Creating[ClientStream[E]]: """Create a Stream that this client owns, so only a client closes it. - It is idempotent for one id, one Retention and one ``auto_close``. Await the call for - the handle, or use it in ``async with``, which closes the Stream at the end of the - block. + It is idempotent for one id and one Retention. Await the call for the handle, or use + it in ``async with``, which closes the Stream at the end of the block. Args: definition: The Stream definition class. id: The Stream id, as a Workflow takes a Workflow id. retention: How long the Records stay after the close. Defaults to the backend's. - auto_close: Close the Stream with no result when no Record was appended for this - long, so a Stream that nobody closes does not stay open. Defaults to never. backend: The name of the backend. Defaults to the client's default backend. rpc_metadata: Headers of the call. See :py:meth:`ClientStream.get_offset`. rpc_timeout: The deadline of the call. @@ -484,8 +480,8 @@ def create( Raises: StreamConfigError: The client has no backend of that name. No call is made. StreamClosedError: A Stream of that id was created and closed before. - StreamError: The Stream exists with another Retention or ``auto_close``, with - the reason ``stream_already_exists``. + StreamError: The Stream exists with another Retention, with the reason + ``stream_already_exists``. """ async def create() -> ClientStream[E]: @@ -494,9 +490,7 @@ async def create() -> ClientStream[E]: service = stream_backend.service(self._client.namespace) with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): response = await service.create_stream( - models.CreateStreamRequest( - stream_id=id, retention=retention, auto_close=auto_close - ) + models.CreateStreamRequest(stream_id=id, retention=retention) ) if response.status == stream_pb.StreamStatus.STREAM_STATUS_CLOSED: raise _errors.StreamClosedError( diff --git a/temporalio/common/streams/_writes.py b/temporalio/common/streams/_writes.py index 15b7f43a98..f45455362c 100644 --- a/temporalio/common/streams/_writes.py +++ b/temporalio/common/streams/_writes.py @@ -85,9 +85,6 @@ async def create_stream(request: stream_pb.CreateStreamRequest) -> int: retention=request.retention.ToTimedelta() if request.HasField("retention") else None, - auto_close=request.auto_close.ToTimedelta() - if request.HasField("auto_close") - else None, withdraw_run=request.withdraw_run or None, owner=models.Producer( kind=request.owner.kind, diff --git a/temporalio/common/streams/redis.py b/temporalio/common/streams/redis.py index 4c7f3c1e7f..2ff8a49308 100644 --- a/temporalio/common/streams/redis.py +++ b/temporalio/common/streams/redis.py @@ -10,20 +10,16 @@ import dataclasses import logging import weakref -from collections.abc import Awaitable, Callable, Mapping, Sequence -from dataclasses import dataclass -from datetime import datetime, timedelta +from collections.abc import Mapping, Sequence +from datetime import timedelta from typing import Any, NoReturn import temporalio.activity import temporalio.api.common.v1 -import temporalio.api.failure.v1 import temporalio.api.streamservice.v1 as stream_pb import temporalio.bridge.streams import temporalio.client -import temporalio.common import temporalio.converter -import temporalio.exceptions import temporalio.runtime import temporalio.service import temporalio.worker @@ -33,7 +29,6 @@ from temporalio.common.streams._backend import StreamBackend, _ExtensionStreamBackend from temporalio.common.streams._errors import StreamConfigError from temporalio.common.streams._handle import StreamHandle -from temporalio.common.streams._items import close_status from temporalio.common.streams._plugin import StreamPlugin from temporalio.common.streams.proto import v1 as ext_pb from temporalio.common.streams.system import models @@ -68,10 +63,6 @@ def __init__( config.backend = backend_name self._config = config.SerializeToString() self._backends: weakref.WeakSet[_ExtensionStreamBackend] = weakref.WeakSet() - # The closes of the runs that ended on this plugin's Workers, by run id, until the - # Worker evicts the run. - self._run_closes: dict[str, _RunClose] = {} - self._closing: set[asyncio.Task[None]] = set() # Each client gets its own runtime object on Redis. The memory Store has one for the # plugin, so every client and Worker that shares the plugin shares its Streams. self._memory: _ExtensionStreamBackend | None = None @@ -180,29 +171,6 @@ def configure_replayer( ] return config - def on_workflow_evicted( - self, input: temporalio.worker.WorkflowEvictedInput - ) -> None: - """Write the closes of a run that ended, once the server accepted its completion.""" - close = self._run_closes.pop(input.run_id, None) - if close is None or not input.execution_ending: - return - task = asyncio.get_running_loop().create_task(_write_closes(close)) - self._closing.add(task) - task.add_done_callback(self._closing.discard) - - async def run_worker( - self, - worker: temporalio.worker.Worker, - next: Callable[[temporalio.worker.Worker], Awaitable[None]], - ) -> None: - """Run the Worker, then wait for the closes of the runs that ended on it.""" - try: - await super().run_worker(worker, next) - finally: - if self._closing: - await asyncio.gather(*list(self._closing), return_exceptions=True) - async def shutdown(self) -> None: """Drop every open writer, including the client's. Call it once the process no longer writes Streams. @@ -301,15 +269,11 @@ def __init__( def workflow_interceptor_class( self, input: temporalio.worker.WorkflowInterceptorClassInput ) -> type[temporalio.worker.WorkflowInboundInterceptor] | None: - # Workflow code learns the Worker's backend name from the interceptor class, and the - # run gives its closes to the interceptor. + # Workflow code learns the Worker's backend name from the interceptor class. return type( "_WorkflowInbound", (_WorkflowInbound,), - { - "_backend_name": self.plugin.backend_name, - "_interceptor": self if self.backend is not None else None, - }, + {"_backend_name": self.plugin.backend_name}, ) def intercept_activity( @@ -318,22 +282,8 @@ def intercept_activity( return _ActivityInbound(next, self) -@dataclass -class _RunClose: - # The closes of a run's close-with-run Streams, which the worker writes once the server - # accepted the run's completion. - interceptor: _StreamInterceptor - namespace: str - run_id: str - streams: list[models.StreamRef] - result: temporalio.api.common.v1.Payload | None - failure: temporalio.api.failure.v1.Failure | None - status: stream_pb.CloseStatus.ValueType | None - - class _WorkflowInbound(temporalio.worker.WorkflowInboundInterceptor): _backend_name = "" - _interceptor: _StreamInterceptor | None = None def init(self, outbound: temporalio.worker.WorkflowOutboundInterceptor) -> None: super().init(_WorkflowOutbound(outbound, self._backend_name)) @@ -344,152 +294,24 @@ async def execute_workflow( _take_owned_streams(input.headers) try: result = await super().execute_workflow(input) - await streams_workflow.drain() - except temporalio.workflow.ContinueAsNewError: - # The next run continues the run chain's Streams. - await streams_workflow.drain() - raise except BaseException as err: - if not isinstance(err, (Exception, asyncio.CancelledError)): - # A GeneratorExit or another eviction exception closes the coroutine outside - # the Workflow's loop. The run did not end, so there is nothing to close. - raise - if ( - isinstance(err, asyncio.CancelledError) - and temporalio.workflow.cancellation_reason() is not None - ): - # The run ends cancelled. - await streams_workflow.drain() - self._close_with_run( - failure=temporalio.exceptions.CancelledError("Workflow cancelled") - ) - elif _fails_run(err) and not chain_continues( - temporalio.workflow.info(), temporalio.workflow.now(), err - ): + if _ends_run(err): await streams_workflow.drain() - self._close_with_run(failure=err) raise - # A cron run that completes still goes on with the chain. - if not chain_continues( - temporalio.workflow.info(), temporalio.workflow.now(), None - ): - self._close_with_run(result=result) + await streams_workflow.drain() return result - def _close_with_run( - self, *, result: Any | None = None, failure: BaseException | None = None - ) -> None: - # The closes wait until the server accepts the run's completion. A last Workflow Task - # that fails then leaves the Streams open, so its retry still writes to them. - interceptor = self._interceptor - if interceptor is None or temporalio.workflow.unsafe.is_replaying(): - return - streams = [ - run_stream.ref - for run_stream in streams_workflow._state().streams.values() - if run_stream.close_with_run and not run_stream.closed - ] - if not streams: - return - converter = temporalio.workflow.payload_converter() - failure_proto = None - if failure is not None: - failure_proto = temporalio.api.failure.v1.Failure() - temporalio.converter.DataConverter.default.failure_converter.to_failure( - failure, converter, failure_proto - ) - info = temporalio.workflow.info() - interceptor.plugin._run_closes[info.run_id] = _RunClose( - interceptor=interceptor, - namespace=info.namespace, - run_id=info.run_id, - streams=streams, - result=converter.to_payload(result) if result is not None else None, - failure=failure_proto, - status=close_status(failure), - ) - - -def _fails_run(err: BaseException) -> bool: - # Any other exception fails the Workflow Task, and the run goes on. - return temporalio.workflow._Runtime.current().workflow_is_failure_exception(err) - -def chain_continues( - info: temporalio.workflow.Info, now: datetime, failure: BaseException | None -) -> bool: - """Whether the server starts another run of the chain after this run ends. - - Then the next run goes on with the run chain's Streams, so the runtime skips this run's - closes. A cron run always continues. A failed run continues when its retry policy has - attempts and time left for the failure. - """ - if info.cron_schedule: +def _ends_run(err: BaseException) -> bool: + # The run's writes that nobody awaits go out before its end. Any other exception fails the + # Workflow Task, and an eviction closes the coroutine outside the Workflow's loop. + if isinstance(err, temporalio.workflow.ContinueAsNewError): return True - policy = info.retry_policy - if failure is None or policy is None: - return False - if policy.maximum_attempts != 0 and info.attempt >= policy.maximum_attempts: - return False - if _non_retryable(failure, policy.non_retryable_error_types): - return False - if info.execution_timeout is not None: - backoff = policy.initial_interval * ( - policy.backoff_coefficient ** (info.attempt - 1) - ) - if policy.maximum_interval is not None: - backoff = min(backoff, policy.maximum_interval) - # The execution timeout covers the whole chain, so this run's start is not its start. - expiration = info.execution_expiration_time or ( - info.workflow_start_time + info.execution_timeout - ) - if now + backoff >= expiration: - return False - return True - - -def _non_retryable(err: BaseException, types: Sequence[str] | None) -> bool: - if isinstance(err, temporalio.exceptions.ApplicationError): - return err.non_retryable or (err.type or "") in (types or []) - return type(err).__name__ in (types or []) - - -async def _write_closes(close: _RunClose) -> None: - backend, converter = close.interceptor.backend, close.interceptor.converter - assert backend is not None - service = backend.service(close.namespace, converter) - failure = ( - converter.failure_converter.from_failure( - close.failure, converter.payload_converter - ) - if close.failure is not None - else None - ) - for ref in close.streams: - try: - await service.close_stream( - models.CloseStreamRequest( - stream=ref, - # The run itself closes its Streams, as their owner. - producer=models.Producer( - kind=stream_pb.ProducerKind.PRODUCER_KIND_WORKFLOW, - id=_ids.workflow_producer_id(close.run_id), - run_id=close.run_id, - ), - result=temporalio.common.RawValue(close.result) - if close.result is not None - else None, - failure=failure, - status=close.status, - ) - ) - except Exception: - logger.warning( - "Failed closing the Stream %s with the run %s", - ref.id, - close.run_id, - exc_info=True, - ) + if isinstance(err, asyncio.CancelledError): + return temporalio.workflow.cancellation_reason() is not None + return isinstance( + err, Exception + ) and temporalio.workflow._Runtime.current().workflow_is_failure_exception(err) def _owned_ref(stream_id: str) -> models.StreamRef: @@ -510,11 +332,8 @@ def _take_owned_streams( return state = streams_workflow._state() converter = temporalio.converter.PayloadConverter.default - for stream_id, backend, close_with_run in converter.from_payload( - payload, list[Any] - ): - run_stream = state.stream(_owned_ref(stream_id)) - run_stream.backend, run_stream.close_with_run = backend, close_with_run + for stream_id, backend, *_ in converter.from_payload(payload, list[Any]): + state.stream(_owned_ref(stream_id)).backend = backend class _WorkflowOutbound(temporalio.worker.WorkflowOutboundInterceptor): @@ -540,7 +359,7 @@ def _activity_headers(self) -> dict[str, temporalio.api.common.v1.Payload]: def continue_as_new(self, input: temporalio.worker.ContinueAsNewInput) -> NoReturn: # The next run needs each owned Stream's backend for stream_handle. owned = [ - [stream_id, run_stream.backend, run_stream.close_with_run] + [stream_id, run_stream.backend] for stream_id, run_stream in streams_workflow._state().streams.items() if run_stream.backend is not None and not run_stream.closed ] diff --git a/temporalio/common/streams/workflow.py b/temporalio/common/streams/workflow.py index d8182171a9..6e3469349e 100644 --- a/temporalio/common/streams/workflow.py +++ b/temporalio/common/streams/workflow.py @@ -66,7 +66,6 @@ class _RunStream: # What this run knows about one Stream. Many handles may name it, so it is kept by id. ref: models.StreamRef - close_with_run: bool = False closed: bool = False # Set for a Stream that the run chain created, which continue-as-new passes on. backend: str | None = None @@ -482,14 +481,14 @@ def create_stream( definition: type[E], *, retention: timedelta | None = None, - auto_close: timedelta | None = None, - close_with_run: bool = True, backend: str | None = None, ) -> Creating[WorkflowStream[E]]: """Create a Stream that this run chain owns. Its id is ``::``, so a client finds it from the Workflow - handle with no call. The create is one Local Activity, and its error raises here. + handle with no call. The create is one Local Activity, and its error raises here. Only the + run chain closes the Stream. On Redis, the end of the run chain closes nothing, so a Stream + that nobody closes stays open until its ``safety_ttl``. Await the call for the handle, or use it in ``async with``, which closes the Stream at the end of the block:: @@ -501,10 +500,6 @@ def create_stream( definition: The Stream definition class. Its name is the Stream name. For one Stream per turn, for example, pass ``streams.with_name(Events, f"turn-{n}")``. retention: How long the Records stay after the close. Defaults to the backend's. - auto_close: Close the Stream with no result when no Record was appended for this - long, so a Stream that nobody closes does not stay open. Defaults to never. - close_with_run: Close the Stream when the run chain closes, with the result or the - failure of its last run. A run that continues as new passes it on. backend: The name of the backend. Defaults to the Worker's. The default is part of the Workflow's code, since a replay must create on the same backend. To move a Workflow to another backend, gate it with a patch, for example @@ -515,14 +510,10 @@ def create_stream( ReadOnlyContextError: The code must not change state, such as a query handler. StreamConfigError: No Stream plugin is on the Worker, or it has no backend of that name. It fails the Workflow Task, not the run. - ValueError: ``close_with_run=False`` for a Stream that already closes with the run - chain. """ async def create() -> WorkflowStream[E]: - stream, task = _create( - definition, retention, auto_close, close_with_run, backend - ) + stream, task = _create(definition, retention, backend) await task return stream @@ -533,8 +524,6 @@ def create_stream_nowait( definition: type[E], *, retention: timedelta | None = None, - auto_close: timedelta | None = None, - close_with_run: bool = True, backend: str | None = None, ) -> WorkflowStream[E]: """Create a Stream that this run chain owns, with no await. @@ -543,7 +532,7 @@ def create_stream_nowait( is for code that can't await ``put``. It takes the options of :py:func:`create_stream`. A backend error fails the Workflow Task. """ - stream, task = _create(definition, retention, auto_close, close_with_run, backend) + stream, task = _create(definition, retention, backend) _write_nowait(task) return stream @@ -551,17 +540,10 @@ def create_stream_nowait( def _create( definition: type[E], retention: timedelta | None, - auto_close: timedelta | None, - close_with_run: bool, backend: str | None, ) -> tuple[WorkflowStream[E], asyncio.Future[int]]: stream = _own_handle(definition, _check_backend(backend, "create")) run_stream = stream._check("create", backend_checked=True) - if run_stream.close_with_run and not close_with_run: - raise ValueError( - f"The Stream {stream.id} already closes with the run chain, and " - "close_with_run=False can't turn that off" - ) run_id = temporalio.workflow.info().run_id # The owner makes the Stream the run chain's, and names a retry run's withdrawal. # The ref names the run chain, which the Store keeps for its owner check of a close. @@ -576,9 +558,6 @@ def _create( ) if retention is not None: request.retention.FromTimedelta(retention) - if auto_close is not None: - request.auto_close.FromTimedelta(auto_close) - run_stream.close_with_run = close_with_run run_stream.backend = stream.backend return stream, _write(stream.id, CREATE_ACTIVITY, request) diff --git a/tests/contrib/streams/examples/activity_streams_tokens.py b/tests/contrib/streams/examples/activity_streams_tokens.py index 87ab07c12c..ebdf40a260 100644 --- a/tests/contrib/streams/examples/activity_streams_tokens.py +++ b/tests/contrib/streams/examples/activity_streams_tokens.py @@ -71,6 +71,7 @@ async def run(self, prompt: str) -> list[int]: SummarizeInput(prompt, streams.SyncActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(minutes=5), ) + await stream.close() return [first, summary] diff --git a/tests/contrib/streams/examples/agent_publishes_turns.py b/tests/contrib/streams/examples/agent_publishes_turns.py index 2995c86c3a..c2318a91e6 100644 --- a/tests/contrib/streams/examples/agent_publishes_turns.py +++ b/tests/contrib/streams/examples/agent_publishes_turns.py @@ -1,8 +1,8 @@ """Brook API proposal, "An agent Workflow publishes its turns". The Workflow creates its Stream, publishes each turn with an awaited ``publish``, and passes the -Stream to the model Activity in its one input object. ``close_with_run`` closes the Stream with -the result of the run chain. +Stream to the model Activity in its one input object. It closes the Stream with its result +before it returns, since a run end closes nothing. """ from __future__ import annotations @@ -60,7 +60,7 @@ class AgentTurns: @workflow.run async def run(self, messages: list[str]) -> str: stream = await stream_workflow.create_stream( - Events, retention=timedelta(days=1), auto_close=timedelta(hours=1) + Events, retention=timedelta(days=1) ) turns = stream.topic(Events.turns) while True: @@ -73,6 +73,8 @@ async def run(self, messages: list[str]) -> str: ) if not reply.tool_calls: await turns.publish(Turn("done", reply.text)) + # A run end closes nothing, so the Workflow closes its Stream. + await stream.close(result=reply.text) return reply.text for call in reply.tool_calls: messages.append( diff --git a/tests/contrib/streams/examples/client_creates_stream.py b/tests/contrib/streams/examples/client_creates_stream.py index c47b8b8638..17e6c9daaf 100644 --- a/tests/contrib/streams/examples/client_creates_stream.py +++ b/tests/contrib/streams/examples/client_creates_stream.py @@ -32,7 +32,7 @@ class Events: class Chat: @workflow.run async def run(self, stream: stream_workflow.WorkflowStream[Events]) -> None: - # The Workflow only writes. The client, or auto-close, closes the Stream. + # The Workflow only writes. The client closes the Stream. await stream.topic(Events.turns).publish( Turn("model", "hi"), Turn("last", "bye") ) @@ -46,7 +46,6 @@ async def main(client: Client, task_queue: str) -> list[Turn]: Events, id=f"chat-{uuid.uuid4()}", retention=timedelta(days=1), - auto_close=timedelta(hours=1), ) as chat_stream: await client.start_workflow( Chat.run, diff --git a/tests/contrib/streams/examples/turn_scoped_stream.py b/tests/contrib/streams/examples/turn_scoped_stream.py index c3c3e3b1d5..d2d4306259 100644 --- a/tests/contrib/streams/examples/turn_scoped_stream.py +++ b/tests/contrib/streams/examples/turn_scoped_stream.py @@ -67,7 +67,6 @@ async def run(self, input: ChatTurnsInput) -> None: async with stream_workflow.create_stream( streams.with_name(Events, f"turn-{input.next + i}"), retention=timedelta(days=1), - close_with_run=False, ) as turn: # A Stream is data, so the index Record carries the turn's Stream. await started.publish( @@ -82,6 +81,8 @@ async def run(self, input: ChatTurnsInput) -> None: workflow.continue_as_new( ChatTurnsInput(input.prompts[2:], input.next + len(batch), index) ) + # The last run closes the index, since a run end closes nothing. + await index.close() async def main(client: Client, task_queue: str) -> dict[int, list[str]]: diff --git a/tests/contrib/streams/examples/ui_backend_reads.py b/tests/contrib/streams/examples/ui_backend_reads.py index b2558238a7..8d6ff90b4c 100644 --- a/tests/contrib/streams/examples/ui_backend_reads.py +++ b/tests/contrib/streams/examples/ui_backend_reads.py @@ -57,6 +57,8 @@ async def run(self, prompt: str) -> str: start_to_close_timeout=timedelta(minutes=5), ) await turns.publish(Turn("done", reply)) + # A run end closes nothing, so the Workflow closes its Stream before it returns. + await stream.close(result=reply) return reply diff --git a/tests/contrib/streams/examples/worker_setup.py b/tests/contrib/streams/examples/worker_setup.py index 841587f333..bf72452b35 100644 --- a/tests/contrib/streams/examples/worker_setup.py +++ b/tests/contrib/streams/examples/worker_setup.py @@ -34,6 +34,7 @@ def __init__(self, name: str) -> None: @workflow.run async def run(self, name: str) -> str: await self.stream.topic(Events.tokens).publish(f"bye {name}") + await self.stream.close() return name diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index 38cadfaa9d..ae6269ba79 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -50,7 +50,7 @@ async def read( stream = streams.StreamClient(client).workflow_stream( handle, streams.with_name(Events, name) ) - return [item async for item in stream.subscribe(create_wait=timedelta(seconds=20))] + return [item async for item in stream.subscribe(wait_new_records=False)] def records(items: Sequence[object]) -> list[streams.Record[Any]]: @@ -107,6 +107,7 @@ async def run(self, prompt: str) -> list[Any]: streams.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) + await stream.close() return [offsets, last, read] @@ -210,6 +211,7 @@ async def run(self, key: str) -> None: start_to_close_timeout=timedelta(seconds=30), retry_policy=RetryPolicy(initial_interval=timedelta(milliseconds=10)), ) + await stream.close() async def test_reader_withdraws_a_shown_record_after_a_reconnect( @@ -277,7 +279,7 @@ async def close_it(stream: streams.ActivityStream[Events]) -> str: class Closing: @workflow.run async def run(self) -> list[str]: - stream = await stream_workflow.create_stream(Events, close_with_run=False) + stream = await stream_workflow.create_stream(Events) backend = await workflow.execute_activity( wrong_backend, streams.ActivityStream(id=stream.id, backend="elsewhere"), @@ -316,7 +318,7 @@ async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): class CreatesAndLeavesOpen: @workflow.run async def run(self) -> str: - stream = await stream_workflow.create_stream(Events, close_with_run=False) + stream = await stream_workflow.create_stream(Events) return stream.id @@ -441,6 +443,7 @@ async def run(self) -> list[Any]: start_to_close_timeout=timedelta(seconds=30), ) await tokens.publish("after") + await stream.close() return seen diff --git a/tests/contrib/streams/test_chain.py b/tests/contrib/streams/test_chain.py deleted file mode 100644 index 041324442e..0000000000 --- a/tests/contrib/streams/test_chain.py +++ /dev/null @@ -1,80 +0,0 @@ -from datetime import datetime, timedelta, timezone -from types import SimpleNamespace -from typing import Any - -import pytest - -from temporalio.common import RetryPolicy -from temporalio.common.streams.redis import chain_continues -from temporalio.exceptions import ApplicationError - -START = datetime(2026, 10, 21, tzinfo=timezone.utc) - - -def info(**fields: Any) -> Any: - base: dict[str, Any] = { - "cron_schedule": None, - "retry_policy": None, - "attempt": 1, - "execution_timeout": None, - "workflow_start_time": START, - "execution_expiration_time": None, - } - return SimpleNamespace(**{**base, **fields}) - - -RETRY = RetryPolicy(initial_interval=timedelta(seconds=1), maximum_attempts=3) - - -@pytest.mark.parametrize( - "case, run, failure, expected", - [ - ("success", info(), None, False), - ("cron success", info(cron_schedule="* * * * *"), None, True), - ("cron failure", info(cron_schedule="* * * * *"), RuntimeError("x"), True), - ("no retry policy", info(), RuntimeError("x"), False), - ("attempts left", info(retry_policy=RETRY), RuntimeError("x"), True), - ("last attempt", info(retry_policy=RETRY, attempt=3), RuntimeError("x"), False), - ( - "unlimited attempts", - info(retry_policy=RetryPolicy(maximum_attempts=0), attempt=50), - RuntimeError("x"), - True, - ), - ( - "non-retryable error", - info(retry_policy=RETRY), - ApplicationError("x", non_retryable=True), - False, - ), - ( - "non-retryable type", - info( - retry_policy=RetryPolicy(non_retryable_error_types=["Bad"]), - ), - ApplicationError("x", type="Bad"), - False, - ), - ( - "no time left", - info(retry_policy=RETRY, execution_timeout=timedelta(seconds=10)), - RuntimeError("x"), - False, - ), - ( - "no time left in a later run of the chain", - info( - retry_policy=RETRY, - execution_timeout=timedelta(seconds=10), - workflow_start_time=START + timedelta(seconds=9), - execution_expiration_time=START + timedelta(seconds=10), - ), - RuntimeError("x"), - False, - ), - ], -) -def test_chain_continues(case: str, run: Any, failure: Any, expected: bool): - # "no time left" runs 9.5 s into a 10 s chain, so a 1 s backoff passes the expiration. - now = START + timedelta(seconds=9.5) - assert chain_continues(run, now, failure) == expected, case diff --git a/tests/contrib/streams/test_pydantic.py b/tests/contrib/streams/test_pydantic.py index 53ebe89e1b..100201a3b3 100644 --- a/tests/contrib/streams/test_pydantic.py +++ b/tests/contrib/streams/test_pydantic.py @@ -45,6 +45,7 @@ async def run(self, prompt: str) -> TurnResult: ), start_to_close_timeout=timedelta(seconds=30), ) + await stream.close(result=prompt.upper()) return TurnResult( text=prompt.upper(), stream=streams.ClientStream.from_handle(stream) ) diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 87f1e379a1..8a0c5c821d 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -94,6 +94,7 @@ async def run(self, prompt: str) -> str: start_to_close_timeout=timedelta(seconds=30), ) await turns.publish(Turn("done", reply)) + await stream.close(result=reply) return reply @@ -129,13 +130,15 @@ async def test_workflow_and_activity_write_one_stream(stream_client: Client): class PublishThenFailTask: @workflow.run async def run(self, key: str) -> None: - turns = (await stream_workflow.create_stream(Events)).topic(Events.tokens) + stream = await stream_workflow.create_stream(Events) + turns = stream.topic(Events.tokens) await turns.publish("before") await workflow.sleep(timedelta(milliseconds=10)) await turns.publish("after") if attempts.setdefault(key, 0) == 0: attempts[key] = 1 raise RuntimeError("fail this Workflow Task once") + await stream.close() async def test_failed_workflow_task_is_withdrawn(stream_client: Client): @@ -239,10 +242,11 @@ async def run(self, write: bool) -> str: stream = await stream_workflow.create_stream(Events) await stream.topic(Events.tokens).publish("first run") workflow.continue_as_new(False) + await stream_workflow.stream_handle(Events).close(result="second run") return "second run" -async def test_last_run_closes_a_stream_it_did_not_write(stream_client: Client): +async def test_later_run_closes_a_stream_it_did_not_write(stream_client: Client): async with new_worker(stream_client, WritesThenContinues) as worker: handle = await stream_client.start_workflow( WritesThenContinues.run, @@ -262,7 +266,7 @@ class CancelledBlock: @workflow.run async def run(self) -> None: async with await stream_workflow.create_stream( - streams.with_name(Events, "turn"), close_with_run=False + streams.with_name(Events, "turn") ) as turn: await turn.topic(Events.tokens).publish("started") await workflow.wait_condition(lambda: False) @@ -297,9 +301,10 @@ async def run(self) -> str: stream = await stream_workflow.create_stream(Plain) await stream.topic(Plain.t).publish("plain") async with await stream_workflow.create_stream( - streams.with_name(Plain, "scoped"), close_with_run=False + streams.with_name(Plain, "scoped") ) as scoped: await scoped.topic(Plain.t).publish("scoped") + await stream.close(result="done") return "done" @@ -343,6 +348,7 @@ async def run(self) -> None: CallModelInput("0 1", streams.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), ) + await stream.close() async def test_local_activity_writes_with_the_task_epoch(stream_client: Client): @@ -377,6 +383,7 @@ async def run(self, key: str) -> None: if first: attempts[key] = 1 raise RuntimeError("fail this Workflow Task once") + await stream.close() async def test_rerun_local_activity_withdraws_its_failed_task(stream_client: Client): @@ -422,6 +429,7 @@ async def run(self, key: str) -> None: ) raise RuntimeError("fail this Workflow Task once") await stream.topic(Events.tokens).publish("other") + await stream.close() async def test_later_write_withdraws_a_skipped_local_activity(stream_client: Client): @@ -452,11 +460,12 @@ async def test_later_write_withdraws_a_skipped_local_activity(stream_client: Cli class FailingRun: @workflow.run async def run(self) -> str: - turns = (await stream_workflow.create_stream(Events)).topic(Events.tokens) + stream = await stream_workflow.create_stream(Events) info = workflow.info() - await turns.publish(f"run-{info.attempt}") + await stream.topic(Events.tokens).publish(f"run-{info.attempt}") if info.attempt == 1: raise ApplicationError("first run fails") + await stream.close(result="done") return "done" @@ -477,7 +486,6 @@ async def test_retry_run_withdraws_the_failed_run(stream_client: Client): assert [record.data for record in written] == ["run-1", "run-2"] assert len(superseded) == 1 assert [r.data for r in written if superseded[0].withdraws(r)] == ["run-1"] - # The failed run had retries left, so only the last run closes the Stream. closed = items[-1] assert isinstance(closed, streams.Closed) and closed.result == "done" @@ -559,14 +567,16 @@ def __init__(self) -> None: @workflow.run async def run(self) -> str: - turns = (await stream_workflow.create_stream(Events)).topic(Events.tokens) + stream = await stream_workflow.create_stream(Events) + turns = stream.topic(Events.tokens) await turns.publish("before restart") async with await stream_workflow.create_stream( - streams.with_name(Events, "scoped"), close_with_run=False + streams.with_name(Events, "scoped") ) as scoped: await scoped.topic(Events.tokens).publish("scoped") await workflow.wait_condition(lambda: self.go) await turns.publish("after restart") + await stream.close() return "done" @workflow.signal @@ -617,31 +627,6 @@ async def test_plugin_replays_a_history_made_without_it( await replayer.replay_workflow(history) -@workflow.defn -class QuickWriter: - @workflow.run - async def run(self) -> str: - stream = await stream_workflow.create_stream(Events) - await stream.topic(Events.tokens).publish("quick") - return "done" - - -async def test_worker_shutdown_drains_closes(stream_client: Client): - async with new_worker(stream_client, QuickWriter) as worker: - handle = await stream_client.start_workflow( - QuickWriter.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue - ) - await handle.result() - # Calls after the Worker exits still work, and the drain waited for the run's close. - items = [ - item - async for item in workflow_stream(stream_client, handle).subscribe( - wait_new_records=False - ) - ] - assert isinstance(items[-1], streams.Closed) and items[-1].result == "done" - - async def test_plugin_shutdown_after_client_writes(stream_client: Client): [plugin] = [ p @@ -748,20 +733,18 @@ async def test_final_failure_under_a_retry_policy( ) with pytest.raises(WorkflowFailureError): await handle.result() - items = await read_all(workflow_stream(stream_client, handle)) + stream = workflow_stream(stream_client, handle) + items = [item async for item in stream.subscribe(wait_new_records=False)] written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] - closes = [item for item in items if isinstance(item, streams.Closed)] - # Only the run that ends the chain closes the Stream, with its failure, and it is last. - assert len(closes) == 1 and items[-1] is closes[0] - assert closes[0].failure is not None + # The end of the chain closes nothing. + assert not any(isinstance(item, streams.Closed) for item in items) if non_retryable: - # A non-retryable failure ends the chain on attempt 1, so it closes at once. + # A non-retryable failure ends the chain on attempt 1. assert [r.data for r in written] == ["run-1"] assert superseded == [] return - # Attempts 1 and 2 left the Stream open, so attempts 2 and 3 could write. Each retry run - # withdraws the run before it. + # Each retry run withdraws the run before it. assert [r.data for r in written] == ["run-1", "run-2", "run-3"] assert len(superseded) == 2 assert [r.data for r in written if superseded[0].withdraws(r)] == ["run-1"] @@ -826,7 +809,7 @@ async def test_two_backends_on_one_client(client: Client, redis_url: str): class _FailsTheLastTaskOnce(Interceptor, WorkerInterceptor): # On the client before the Stream plugin, so it is the outer Workflow interceptor and - # fails the task after the plugin's part of the run end. + # fails the task after the plugin drained the run's writes. def workflow_interceptor_class( self, input: WorkflowInterceptorClassInput ) -> type[WorkflowInboundInterceptor]: @@ -853,11 +836,9 @@ async def run(self) -> str: @pytest.mark.parametrize("cache", [0, 10]) -async def test_failed_last_task_leaves_the_close_to_its_retry( +async def test_failed_last_task_is_withdrawn_by_its_retry( stream_client: Client, cache: int ): - # The close with the run goes out once the server accepted the run's completion. A close - # in the failed task would make the retry's create raise StreamClosedError. config = stream_client.config() config["interceptors"] = [_FailsTheLastTaskOnce()] client = Client(**config) @@ -873,14 +854,12 @@ async def test_failed_last_task_leaves_the_close_to_its_retry( task_queue=worker.task_queue, ) assert await handle.result() == "done" - items = await asyncio.wait_for(read_all(workflow_stream(client, handle)), 20) + stream = workflow_stream(client, handle) + items = [item async for item in stream.subscribe(wait_new_records=False)] written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] visible = [r for r in written if not any(s.withdraws(r) for s in superseded)] assert [r.data for r in visible] == ["token"] - closes = [item for item in items if isinstance(item, streams.Closed)] - assert len(closes) == 1 and closes[0].result == "done" - assert closes[0].status == streams.CloseStatus.COMPLETED @workflow.defn @@ -969,10 +948,11 @@ async def run(self, stream: stream_workflow.WorkflowStream[Events] | None) -> st await stream.topic(Events.tokens).publish(f"attempt-{info.attempt}") if info.attempt == 1: raise ApplicationError("the continued run fails once") + await stream.close(result="done") return "done" -async def test_retry_of_a_continued_run_keeps_the_close_with_run(stream_client: Client): +async def test_retry_of_a_continued_run_writes_the_same_stream(stream_client: Client): async with new_worker(stream_client, ContinuesThenRetries) as worker: handle = await stream_client.start_workflow( ContinuesThenRetries.run, diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index acc768940a..8c18e6beab 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -161,6 +161,7 @@ async def run(self, prompt: str) -> str: await workflow.sleep(timedelta(milliseconds=200)) stream = await stream_workflow.create_stream(Events) await stream.topic(Events.tokens).publish(*prompt.split()) + await stream.close(result=prompt.upper()) return prompt.upper() @@ -320,19 +321,6 @@ async def test_named_producer_fences_its_earlier_epoch(stream_client: Client): stream.producer("ingest", epoch=0) -async def test_auto_close_of_a_client_stream(stream_client: Client): - stream = await streams.StreamClient(stream_client).create( - Events, id=f"quiet-{uuid.uuid4()}", auto_close=timedelta(seconds=1) - ) - await stream.topic(Events.tokens).publish("only") - await asyncio.sleep(1.5) - items = [item async for item in stream.subscribe(wait_new_records=False)] - assert [item.data for item in items if isinstance(item, streams.Record)] == ["only"] - assert isinstance(items[-1], streams.Closed) and items[-1].result is None - with pytest.raises(streams.StreamClosedError): - await stream.topic(Events.tokens).publish("late") - - async def test_unresolved_workflow_stream_compares_and_prints(client: Client): client, _ = fake_client(client) stream_client = streams.StreamClient(client) diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index eba3416d7d..92a6ced70f 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -45,7 +45,7 @@ async def read( stream = streams.StreamClient(client).workflow_stream( handle, streams.with_name(Events, name) ) - return [item async for item in stream.subscribe(create_wait=timedelta(seconds=20))] + return [item async for item in stream.subscribe(wait_new_records=False)] def data(items: Sequence[object]) -> list[object]: @@ -60,10 +60,12 @@ async def run(self, prompt: str) -> str: turns = stream.topic(Events.turns) await turns.publish(Turn("user", prompt), Turn("model")) await stream.topic("tokens").publish(*prompt.split()) + # A run end closes nothing, so the run closes its Stream before it returns. + await stream.close(result=prompt.upper()) return prompt.upper() -async def test_create_publish_and_close_with_run(stream_client: Client): +async def test_create_publish_and_close(stream_client: Client): async with new_worker(stream_client, Agent) as worker: handle = await stream_client.start_workflow( Agent.run, "hi there", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue @@ -82,7 +84,7 @@ async def run(self, fail_at: int) -> None: for n in range(2): try: async with stream_workflow.create_stream( - streams.with_name(Events, f"turn-{n}"), close_with_run=False + streams.with_name(Events, f"turn-{n}") ) as turn: await turn.topic(Events.tokens).publish(f"token-{n}") if n == fail_at: @@ -118,6 +120,7 @@ async def run(self, stream: stream_workflow.WorkflowStream[Events] | None) -> st assert stream.definition is Events assert stream == stream_workflow.stream_handle(Events) await stream.topic(Events.tokens).publish("run 2") + await stream.close(result="done") return "done" @@ -132,6 +135,7 @@ async def run(self, first: bool) -> list[str]: workflow.continue_as_new(False) turn = stream_workflow.stream_handle(renamed) await turn.topic(Events.tokens).publish("run 2") + await turn.close() return [stream_workflow.stream_handle(Events).id, turn.id] @@ -176,7 +180,7 @@ def __init__(self) -> None: @workflow.run async def run(self) -> list[str]: errors = [] - stream = await stream_workflow.create_stream(Events, close_with_run=False) + stream = await stream_workflow.create_stream(Events) self._stream = stream try: stream.topic("unknown") @@ -260,7 +264,7 @@ async def run(self) -> None: raise ApplicationError("run failed", non_retryable=True) -async def test_close_with_run_takes_the_failure(stream_client: Client): +async def test_run_end_closes_nothing(stream_client: Client): async with new_worker(stream_client, Failing) as worker: handle = await stream_client.start_workflow( Failing.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue @@ -268,43 +272,16 @@ async def test_close_with_run_takes_the_failure(stream_client: Client): with pytest.raises(WorkflowFailureError): await handle.result() items = await read(stream_client, handle) + # The Stream stays open, so a later reader still waits for a close. assert data(items) == ["before"] - assert isinstance(items[-1], streams.Closed) - assert str(items[-1].failure) == "run failed" - assert items[-1].status == streams.CloseStatus.FAILED - - -@workflow.defn -class LeavesOpen: - @workflow.run - async def run(self) -> None: - stream = await stream_workflow.create_stream( - Events, auto_close=timedelta(seconds=1), close_with_run=False - ) - await stream.topic(Events.tokens).publish("only") - - -async def test_auto_close_closes_a_quiet_stream(stream_client: Client): - async with new_worker(stream_client, LeavesOpen) as worker: - handle = await stream_client.start_workflow( - LeavesOpen.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue - ) - await handle.result() - await asyncio.sleep(1.5) - stream = streams.StreamClient(stream_client).workflow_stream(handle, Events) - items = [item async for item in stream.subscribe(wait_new_records=False)] - assert data(items) == ["only"] - assert isinstance(items[-1], streams.Closed) - assert items[-1].result is None and items[-1].failure is None + assert not any(isinstance(item, streams.Closed) for item in items) @workflow.defn class CreatesOn: @workflow.run - async def run(self, backend: str | None, again_without_close: bool) -> None: + async def run(self, backend: str | None) -> None: await stream_workflow.create_stream(Events, backend=backend) - if again_without_close: - await stream_workflow.create_stream(Events, close_with_run=False) async def test_worker_config_errors_fail_the_task( @@ -312,15 +289,14 @@ async def test_worker_config_errors_fail_the_task( ): # A Worker with no plugin, or with another backend, fails the Workflow Task, so a Worker # with the right config can take the run on. - for task_client, args, message in ( - (client, [None, False], "No Stream plugin"), - (stream_client, ["elsewhere", False], "backend elsewhere"), - (stream_client, [None, True], "close_with_run=False"), + for task_client, backend, message in ( + (client, None, "No Stream plugin"), + (stream_client, "elsewhere", "backend elsewhere"), ): async with new_worker(task_client, CreatesOn) as worker: handle = await task_client.start_workflow( CreatesOn.run, - args=args, + backend, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -338,8 +314,7 @@ async def run(self) -> str: async def test_cron_runs_share_an_open_stream(stream_client: Client): - # A cron chain goes on after each run, so no run closes the chain's Stream, and each run - # writes to it again. + # No run end closes the chain's Stream, so each cron run writes to it again. async with new_worker(stream_client, CronTurn) as worker: handle = await stream_client.start_workflow( CronTurn.run, @@ -364,9 +339,7 @@ class SyncHandlers: @workflow.init def __init__(self, prompt: str) -> None: # The create goes first, so the signal's writes find the Stream. - self._stream = stream_workflow.create_stream_nowait( - Events, close_with_run=False - ) + self._stream = stream_workflow.create_stream_nowait(Events) self._prompt = prompt self._done = False @@ -414,6 +387,7 @@ def __init__(self, prompt: str) -> None: @workflow.run async def run(self, prompt: str) -> str: await self._stream.topic(Events.tokens).publish(prompt.upper()) + await self._stream.close(result="done") return "done" @@ -464,8 +438,7 @@ async def test_concurrent_publishes_keep_their_order(stream_client: Client): for e in history.events if e.event_type == EventType.EVENT_TYPE_MARKER_RECORDED ] - # The create, the first write and the four that waited. The worker writes the close with - # the run after the run, so it has no marker. + # The create, the first write and the four that waited. assert len(markers) == 3 # Replay answers from the markers and writes nothing. plugins = [ From 0aa2dd80c5c2be31e0a354f1a515af47521967b4 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 09:32:58 -0700 Subject: [PATCH 04/79] Removed the Worker eviction hook and the plugin shutdown wait. Only the run-end close on Redis used them, along with workflow.Info.execution_expiration_time. --- temporalio/worker/__init__.py | 3 +- temporalio/worker/_plugin.py | 37 --------------- temporalio/worker/_worker.py | 71 ++-------------------------- temporalio/worker/_workflow.py | 5 -- temporalio/workflow/_context.py | 4 -- tests/worker/test_worker_exit.py | 80 -------------------------------- 6 files changed, 6 insertions(+), 194 deletions(-) delete mode 100644 tests/worker/test_worker_exit.py diff --git a/temporalio/worker/__init__.py b/temporalio/worker/__init__.py index c322971e51..4f6efe68c3 100644 --- a/temporalio/worker/__init__.py +++ b/temporalio/worker/__init__.py @@ -25,7 +25,7 @@ WorkflowInterceptorClassInput, WorkflowOutboundInterceptor, ) -from ._plugin import Plugin, WorkflowEvictedInput +from ._plugin import Plugin from ._replayer import ( Replayer, ReplayerConfig, @@ -87,7 +87,6 @@ "WorkflowOutboundInterceptor", "NexusOperationInboundInterceptor", "Plugin", - "WorkflowEvictedInput", # Interceptor input "ContinueAsNewInput", "ExecuteActivityInput", diff --git a/temporalio/worker/_plugin.py b/temporalio/worker/_plugin.py index 6447682a67..c222f836c2 100644 --- a/temporalio/worker/_plugin.py +++ b/temporalio/worker/_plugin.py @@ -3,7 +3,6 @@ import abc from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import AbstractAsyncContextManager -from dataclasses import dataclass from typing import TYPE_CHECKING from temporalio.client import WorkflowHistory @@ -18,22 +17,6 @@ ) -@dataclass(frozen=True) -class WorkflowEvictedInput: - """Input for :py:meth:`Plugin.on_workflow_evicted`. - - .. warning:: - This API is experimental. - """ - - run_id: str - """Run ID of the Workflow.""" - - execution_ending: bool - """Whether the server accepted the run's last completion, so the run ended. False for any - other eviction, such as a full cache, a failed Workflow Task or a Worker shutdown.""" - - class Plugin(abc.ABC): """Base class for worker plugins that can intercept and modify worker behavior. @@ -81,26 +64,6 @@ async def run_worker( next: Callable to continue the worker execution. """ - def on_workflow_evicted( - self, - input: WorkflowEvictedInput, # pyright: ignore[reportUnusedParameter] - ) -> None: - """Hook called after the Worker evicts a Workflow run from its cache. - - The Worker calls it outside the sandbox, on its event loop, after Core reports the - eviction. So a plugin can act on a run that ended only once the server accepted its - completion. The hook must not block. A plugin starts its async work here and waits for - it in :py:meth:`run_worker`. The Worker logs an exception and goes on. The - :py:class:`Replayer` never calls it. The default does nothing. - - .. warning:: - This API is experimental. - - Args: - input: The run and why it was evicted. - """ - pass - @abc.abstractmethod def configure_replayer(self, config: ReplayerConfig) -> ReplayerConfig: """Hook called when creating a replayer to allow modification of configuration. diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index d687faf06f..3c6effd80e 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -19,7 +19,6 @@ from typing_extensions import TypedDict -import temporalio.bridge.proto.workflow_activation import temporalio.bridge.worker import temporalio.client import temporalio.common @@ -34,7 +33,7 @@ from ._activity import SharedStateManager, _ActivityWorker from ._interceptor import Interceptor from ._nexus import _NexusWorker -from ._plugin import Plugin, WorkflowEvictedInput +from ._plugin import Plugin from ._tuning import WorkerTuner from ._workflow import ( _DEFAULT_WORKFLOW_TASK_EXTERNAL_STORAGE_CONCURRENCY, @@ -133,7 +132,6 @@ def __init__( max_task_queue_activities_per_second: float | None = None, max_eager_activity_reservations_per_workflow_task: int = 3, graceful_shutdown_timeout: timedelta = timedelta(), - plugin_shutdown_timeout: timedelta | None = timedelta(seconds=60), workflow_failure_exception_types: Sequence[type[BaseException]] = [], shared_state_manager: SharedStateManager | None = None, debug_mode: bool = False, @@ -280,11 +278,6 @@ def __init__( graceful_shutdown_timeout: Amount of time after shutdown is called that activities are given to complete before their tasks are cancelled. - plugin_shutdown_timeout: Amount of time that exiting - ``async with Worker(...)`` waits, after shutdown, for the code - that plugins run after the Worker stops in ``run_worker``. That - code is cancelled after this time. ``None`` waits with no limit. - The default is 60 seconds. workflow_failure_exception_types: The types of exceptions that, if a workflow-thrown exception extends, will cause the workflow/update to fail instead of suspending the workflow via @@ -386,7 +379,6 @@ def __init__( max_task_queue_activities_per_second=max_task_queue_activities_per_second, max_eager_activity_reservations_per_workflow_task=max_eager_activity_reservations_per_workflow_task, graceful_shutdown_timeout=graceful_shutdown_timeout, - plugin_shutdown_timeout=plugin_shutdown_timeout, workflow_failure_exception_types=workflow_failure_exception_types, shared_state_manager=shared_state_manager, debug_mode=debug_mode, @@ -496,7 +488,6 @@ def _init_from_config(self, client: temporalio.client.Client, config: WorkerConf self._async_context_inner_task: asyncio.Task | None = None self._async_context_run_task: asyncio.Task | None = None self._async_context_run_exception: BaseException | None = None - self._async_context_exiting = False self._activity_worker: _ActivityWorker | None = None self._runtime = ( @@ -577,7 +568,7 @@ def check_activity(activity: str): "disable_eager_activity_execution" ], # type: ignore[reportTypedDictNotRequiredAccess] metric_meter=self._runtime.metric_meter, - on_eviction_hook=self._eviction_hook(), + on_eviction_hook=None, disable_safe_eviction=config["disable_safe_workflow_eviction"], # type: ignore[reportTypedDictNotRequiredAccess] should_enforce_versioning_behavior=should_enforce_versioning_behavior, assert_local_activity_valid=check_activity, @@ -719,42 +710,6 @@ def check_activity(activity: str): ), ) - def _eviction_hook( - self, - ) -> ( - Callable[ - [str, temporalio.bridge.proto.workflow_activation.RemoveFromCache], None - ] - | None - ): - plugins = [ - plugin - for plugin in self._plugins - if type(plugin).on_workflow_evicted is not Plugin.on_workflow_evicted - ] - if not plugins: - return None - ending = temporalio.bridge.proto.workflow_activation.RemoveFromCache.EvictionReason.WORKFLOW_EXECUTION_ENDING - - def on_evicted( - run_id: str, - job: temporalio.bridge.proto.workflow_activation.RemoveFromCache, - ) -> None: - input = WorkflowEvictedInput( - run_id=run_id, execution_ending=job.reason == ending - ) - for plugin in plugins: - try: - plugin.on_workflow_evicted(input) - except Exception: - logger.exception( - "Plugin %s failed on the eviction of run %s", - plugin.name(), - run_id, - ) - - return on_evicted - def config(self, *, active_config: bool = False) -> WorkerConfig: """Config, as a dictionary, used to create this worker. @@ -1005,10 +960,7 @@ async def run(): await self.run() except BaseException as err: self._async_context_run_exception = err - # After __aexit__ starts, the caller is past the block, so a cancel would hit - # its later awaits. - if not self._async_context_exiting: - self._async_context_inner_task.cancel() # type: ignore[union-attr] + self._async_context_inner_task.cancel() # type: ignore[union-attr] self._async_context_run_task = asyncio.create_task(run()) return self @@ -1023,20 +975,8 @@ async def __aexit__(self, exc_type: type[BaseException] | None, *args: Any) -> N if not self._async_context_run_task: raise RuntimeError("Never started") await self.shutdown() - # Plugins can run cleanup code after shutdown. Wait for that code instead of cancelling it. - self._async_context_exiting = True - run_task = self._async_context_run_task - timeout = self._config.get("plugin_shutdown_timeout", timedelta(seconds=60)) - try: - # wait() doesn't raise for the task's own outcome, only when this call is cancelled. - await asyncio.wait( - [run_task], timeout=timeout.total_seconds() if timeout else None - ) - except asyncio.CancelledError: - run_task.cancel() - raise - if not run_task.done(): - run_task.cancel() + # Cancel our run task + self._async_context_run_task.cancel() # Only re-raise our exception if present and exc_type is cancel if exc_type is asyncio.CancelledError and self._async_context_run_exception: raise self._async_context_run_exception @@ -1076,7 +1016,6 @@ class WorkerConfig(TypedDict, total=False): max_task_queue_activities_per_second: float | None max_eager_activity_reservations_per_workflow_task: int graceful_shutdown_timeout: timedelta - plugin_shutdown_timeout: timedelta | None workflow_failure_exception_types: Sequence[type[BaseException]] shared_state_manager: SharedStateManager | None debug_mode: bool diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index bfff26cd91..f50955c4cf 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -720,11 +720,6 @@ def _create_workflow_instance( workflow_id=init.workflow_id, workflow_type=init.workflow_type, priority=temporalio.common.Priority._from_proto(init.priority), - execution_expiration_time=init.workflow_execution_expiration_time.ToDatetime().replace( - tzinfo=timezone.utc - ) - if init.HasField("workflow_execution_expiration_time") - else None, ) last_failure = ( diff --git a/temporalio/workflow/_context.py b/temporalio/workflow/_context.py index c86620eea9..a33759c37a 100644 --- a/temporalio/workflow/_context.py +++ b/temporalio/workflow/_context.py @@ -136,10 +136,6 @@ class Info: workflow_type: str - execution_expiration_time: datetime | None = None - """When the execution timeout of the run chain passes, across retries and continue-as-new. - None when the chain has no execution timeout.""" - def _logger_details(self) -> Mapping[str, Any]: return { # TODO(cretz): worker ID? diff --git a/tests/worker/test_worker_exit.py b/tests/worker/test_worker_exit.py deleted file mode 100644 index 737639f475..0000000000 --- a/tests/worker/test_worker_exit.py +++ /dev/null @@ -1,80 +0,0 @@ -import asyncio -import uuid -from collections.abc import Awaitable, Callable -from datetime import timedelta - -from temporalio import workflow -from temporalio.client import Client -from temporalio.plugin import SimplePlugin -from temporalio.worker import Worker - - -@workflow.defn -class NothingWorkflow: - @workflow.run - async def run(self) -> None: - pass - - -class CleanupPlugin(SimplePlugin): - def __init__(self, client: Client) -> None: - super().__init__("test-cleanup") - self.client = client - self.cleanup_result: int | None = None - - async def run_worker( - self, worker: Worker, next: Callable[[Worker], Awaitable[None]] - ) -> None: - try: - await super().run_worker(worker, next) - finally: - # A call after the Worker stops, as a plugin's cleanup makes. - self.cleanup_result = (await self.client.count_workflows()).count - - -async def test_plugin_cleanup_awaits_after_worker_exit(client: Client): - plugin = CleanupPlugin(client) - async with Worker( - client, - task_queue=f"tq-{uuid.uuid4()}", - workflows=[NothingWorkflow], - plugins=[plugin], - ): - pass - assert plugin.cleanup_result is not None - # The caller can still await after the block. - await client.count_workflows() - - -class StuckCleanupPlugin(SimplePlugin): - def __init__(self) -> None: - super().__init__("test-stuck-cleanup") - self.cancelled = asyncio.Event() - - async def run_worker( - self, worker: Worker, next: Callable[[Worker], Awaitable[None]] - ) -> None: - try: - await super().run_worker(worker, next) - finally: - try: - await asyncio.Event().wait() - except asyncio.CancelledError: - self.cancelled.set() - raise - - -async def test_plugin_cleanup_is_cancelled_after_plugin_shutdown_timeout( - client: Client, -): - plugin = StuckCleanupPlugin() - async with Worker( - client, - task_queue=f"tq-{uuid.uuid4()}", - workflows=[NothingWorkflow], - plugins=[plugin], - plugin_shutdown_timeout=timedelta(milliseconds=100), - ): - pass - # The cancel reaches the cleanup code on a later loop iteration, so wait for it. - await asyncio.wait_for(plugin.cancelled.wait(), timeout=10) From 8aaff835daa53cb5008f58654d2030a47f842824 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 09:45:04 -0700 Subject: [PATCH 05/79] Registered Stream backends on the client, and added the entry points. Backends go in stream_backends= of Client.connect, a Worker takes its client's, and the Replayer and WorkflowEnvironment take them too. temporalio.workflow, Client.stream and WorkflowHandle.stream replace StreamClient, and Redis moves to temporalio.contrib.stream. --- temporalio/client/_client.py | 57 ++++ temporalio/client/_workflow.py | 30 ++ temporalio/common/streams/__init__.py | 40 +-- temporalio/common/streams/_access.py | 5 +- temporalio/common/streams/_activity.py | 8 +- temporalio/common/streams/_activity_stream.py | 7 +- temporalio/common/streams/_backend.py | 8 +- temporalio/common/streams/_buffered.py | 1 - temporalio/common/streams/_errors.py | 17 +- .../streams/{redis.py => _extension.py} | 155 ++++----- temporalio/common/streams/_handle.py | 4 - temporalio/common/streams/_plugin.py | 200 ++++++++---- temporalio/common/streams/_stream_client.py | 309 +++++++----------- .../streams/{workflow.py => _workflow.py} | 126 +++---- temporalio/common/streams/_writes.py | 8 +- temporalio/contrib/stream/__init__.py | 159 +++++++++ .../streams => contrib/stream}/testing.py | 26 +- temporalio/contrib/stream/workflow.py | 99 ++++++ temporalio/testing/_workflow.py | 16 + temporalio/worker/_replayer.py | 15 +- temporalio/worker/_worker.py | 8 + temporalio/workflow/__init__.py | 8 + temporalio/workflow/_streams.py | 104 ++++++ tests/contrib/streams/conftest.py | 18 +- .../examples/activity_streams_tokens.py | 5 +- .../streams/examples/agent_publishes_turns.py | 7 +- .../streams/examples/client_creates_stream.py | 9 +- .../streams/examples/resume_from_offset.py | 17 +- .../streams/examples/turn_scoped_stream.py | 12 +- .../streams/examples/ui_backend_reads.py | 5 +- .../contrib/streams/examples/worker_setup.py | 37 ++- tests/contrib/streams/fake_extension.py | 22 ++ .../contrib/streams/test_activity_handles.py | 43 ++- tests/contrib/streams/test_facade.py | 31 +- tests/contrib/streams/test_pydantic.py | 9 +- tests/contrib/streams/test_redis.py | 164 +++++----- tests/contrib/streams/test_stream_client.py | 226 +++++++------ tests/contrib/streams/test_workflow_module.py | 143 +++++--- 38 files changed, 1336 insertions(+), 822 deletions(-) rename temporalio/common/streams/{redis.py => _extension.py} (76%) rename temporalio/common/streams/{workflow.py => _workflow.py} (84%) create mode 100644 temporalio/contrib/stream/__init__.py rename temporalio/{common/streams => contrib/stream}/testing.py (56%) create mode 100644 temporalio/contrib/stream/workflow.py create mode 100644 temporalio/workflow/_streams.py diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index efef5cb528..176243bb55 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -13,6 +13,7 @@ TYPE_CHECKING, Any, Concatenate, + TypeVar, cast, overload, ) @@ -110,9 +111,13 @@ ) if TYPE_CHECKING: + from temporalio.common.streams import ClientStream, StreamBackend, StreamHandle + from ._interceptor import Interceptor from ._plugin import Plugin +_StreamDefinitionT = TypeVar("_StreamDefinitionT") + class Client: """Client for accessing Temporal. @@ -156,6 +161,7 @@ async def connect( grpc_compression: GrpcCompression = GrpcCompression.GZIP, payload_limits: PayloadLimitsConfig = PayloadLimitsConfig(), header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, + stream_backends: Sequence[StreamBackend] = (), ) -> Self: """Connect to a Temporal server. @@ -225,6 +231,13 @@ async def connect( payload_limits: Warning thresholds for outbound payload/memo sizes. Over-threshold fields are logged but still sent. Set a threshold to 0 to disable it. header_codec_behavior: Encoding behavior for headers sent by the client. + stream_backends: The Stream backends of this client and its Workers, such as + :py:class:`temporalio.contrib.stream.RedisStreamBackend`. Names are unique. + With more than one, exactly one has ``default=True``. See + :py:class:`temporalio.common.streams.StreamBackend`. + + .. warning:: + This API is experimental. """ connect_config = temporalio.service.ConnectConfig( target_host=target_host, @@ -262,6 +275,7 @@ def make_lambda( default_workflow_query_reject_condition=default_workflow_query_reject_condition, header_codec_behavior=header_codec_behavior, plugins=plugins, + stream_backends=stream_backends, ) def __init__( @@ -275,6 +289,7 @@ def __init__( default_workflow_query_reject_condition: None | (temporalio.common.QueryRejectCondition) = None, header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, + stream_backends: Sequence[StreamBackend] = (), ): """Create a Temporal client from a service client. @@ -289,11 +304,17 @@ def __init__( interceptors=interceptors, default_workflow_query_reject_condition=default_workflow_query_reject_condition, header_codec_behavior=header_codec_behavior, + stream_backends=stream_backends, ) self._initial_config = config.copy() for plugin in plugins: config = plugin.configure_client(config) + if config.get("stream_backends"): + # Only a client with Stream backends loads the Streams package. + from temporalio.common.streams import _plugin as _stream_plugin + + config = _stream_plugin.configure_client(config) self._init_from_config(config) @@ -894,6 +915,40 @@ def get_workflow_handle( result_type=result_type, ) + def stream( + self, + definition: type[_StreamDefinitionT], + *, + id: str | None = None, + stream: StreamHandle[Any] | None = None, + backend: str | None = None, + ) -> ClientStream[_StreamDefinitionT]: + """A Stream by its id, or a handle that came as data, such as in a Record. + + It makes no call. To create a Stream that the client owns, use the create of a + backend, such as :py:func:`temporalio.contrib.stream.create`. + + .. warning:: + This API is experimental. + + Args: + definition: The Stream definition class. + id: The Stream id. + stream: A handle of the Stream, such as one in a Record. Its id, owner and + backend stay. + backend: The name of the backend. Defaults to the handle's, and else to the + registered backend with ``default=True``. + + Raises: + StreamConfigError: The client has no backend of that name, or none has + ``default=True`` for a call with no backend. + """ + from temporalio.common.streams import _stream_client + + return _stream_client.client_stream( + self, definition, id=id, stream=stream, backend=backend + ) + def get_workflow_handle_for( self, workflow: ( @@ -3029,6 +3084,7 @@ class ClientConnectConfig(TypedDict, total=False): grpc_compression: GrpcCompression payload_limits: PayloadLimitsConfig header_codec_behavior: HeaderCodecBehavior + stream_backends: Sequence[StreamBackend] class ClientConfig(TypedDict, total=False): @@ -3043,3 +3099,4 @@ class ClientConfig(TypedDict, total=False): temporalio.common.QueryRejectCondition | None ] header_codec_behavior: Required[HeaderCodecBehavior] + stream_backends: Required[Sequence[StreamBackend]] diff --git a/temporalio/client/_workflow.py b/temporalio/client/_workflow.py index 94bc2bb23d..82735f0761 100644 --- a/temporalio/client/_workflow.py +++ b/temporalio/client/_workflow.py @@ -21,6 +21,7 @@ Any, Concatenate, Generic, + TypeVar, cast, overload, ) @@ -78,9 +79,13 @@ ) if TYPE_CHECKING: + from temporalio.common.streams import ClientStream + from ._client import Client from ._interceptor import ListWorkflowsInput +_StreamDefinitionT = TypeVar("_StreamDefinitionT") + class WorkflowHistoryEventFilterType(IntEnum): """Type of history events to get for a workflow. @@ -187,6 +192,31 @@ def first_execution_run_id(self) -> str | None: """ return self._first_execution_run_id + def stream( + self, definition: type[_StreamDefinitionT], *, backend: str | None = None + ) -> ClientStream[_StreamDefinitionT]: + """A Stream that this Workflow created, by its definition. + + It builds the Stream id from the Workflow id and the first run id, so it makes no + call. A handle with no first run id makes one Describe call on the first use of its + Streams, and keeps the answer. + + .. warning:: + This API is experimental. + + Args: + definition: The Stream definition class, renamed as the Workflow's create. + backend: The name of the backend. Defaults to the registered backend with + ``default=True``. + + Raises: + StreamConfigError: The client has no backend of that name, or none has + ``default=True`` for a call with no backend. + """ + from temporalio.common.streams import _stream_client + + return _stream_client.workflow_stream(self, definition, backend=backend) + async def result( self, *, diff --git a/temporalio/common/streams/__init__.py b/temporalio/common/streams/__init__.py index c625a91e4b..214b376368 100644 --- a/temporalio/common/streams/__init__.py +++ b/temporalio/common/streams/__init__.py @@ -1,24 +1,23 @@ """Streams of Records that Workflows, Activities and clients write and read. A Stream definition, a class with ``@streams.defn``, gives the Stream name and the type of each -Topic. Workflow code creates a Stream with -:py:func:`temporalio.common.streams.workflow.create_stream`. A client creates and gets Streams -with :py:class:`StreamClient`. A Stream is data: a Workflow passes it to an Activity, a child -Workflow or the next run as an argument, and the parameter's type gives the handle class of -that context, such as :py:class:`ActivityStream`. +Topic. Workflow code creates a Stream with :py:func:`temporalio.workflow.create_stream`. A +client gets a Workflow's Stream with :py:meth:`temporalio.client.WorkflowHandle.stream`, and +any Stream with :py:meth:`temporalio.client.Client.stream`. A Stream is data: a Workflow +passes it to an Activity, a child Workflow or the next run as an argument, and the +parameter's type gives the handle class of that context, such as :py:class:`ActivityStream`. -A Stream plugin carries the calls and registers a backend name with ``name=``, such as -:py:class:`temporalio.common.streams.redis.RedisStreamPlugin` (``redis``) and -:py:class:`temporalio.common.streams.testing.MemoryStreamPlugin` (``memory``). A handle -carries the name of its backend. A client may have two plugins, and exactly one of them has -``default=True``, the one that a create with no ``backend=`` uses. For now a Worker has one -Stream plugin, so it serves one backend. +A :py:class:`StreamBackend` carries the calls and registers a backend name, such as +:py:class:`temporalio.contrib.stream.RedisStreamBackend` (``redis``) and +:py:class:`temporalio.contrib.stream.testing.MemoryStreamBackend` (``memory``). Register +backends with ``Client.connect(..., stream_backends=[...])``. A handle carries the name of its +backend. With more than one registered backend, exactly one has ``default=True``, the one +that a create with no ``backend=`` uses. For now a Worker serves one backend. .. warning:: This package is experimental. """ -from temporalio.common.streams import workflow from temporalio.common.streams._activity_stream import ( ActivityStream, ActivityTopic, @@ -48,7 +47,7 @@ StreamStorageError, StreamUnsupportedError, ) -from temporalio.common.streams._handle import StreamHandle +from temporalio.common.streams._handle import Creating, StreamHandle from temporalio.common.streams._items import ( END, Closed, @@ -56,14 +55,10 @@ Record, Superseded, ) -from temporalio.common.streams._plugin import StreamPlugin +from temporalio.common.streams._plugin import StreamBackend from temporalio.common.streams._reader import Subscription -from temporalio.common.streams._stream_client import ( - ClientStream, - ClientTopic, - StreamClient, -) -from temporalio.common.streams.workflow import WorkflowStream, WorkflowTopic +from temporalio.common.streams._stream_client import ClientStream, ClientTopic +from temporalio.common.streams._workflow import WorkflowStream, WorkflowTopic __all__ = [ "END", @@ -74,12 +69,13 @@ "ClientTopic", "CloseStatus", "Closed", + "Creating", "ReadOnlyContextError", "Record", - "StreamClient", "StreamClosedError", "StreamConfigError", "StreamDefinition", + "StreamBackend", "StreamError", "StreamExpiredError", "StreamFencedError", @@ -87,7 +83,6 @@ "StreamNotFoundError", "StreamNotOwnerError", "StreamOffsetError", - "StreamPlugin", "StreamSeqDivergentError", "StreamStorageError", "StreamUnsupportedError", @@ -102,5 +97,4 @@ "defn", "topic", "with_name", - "workflow", ] diff --git a/temporalio/common/streams/_access.py b/temporalio/common/streams/_access.py index 1a7180ece2..2416b375f1 100644 --- a/temporalio/common/streams/_access.py +++ b/temporalio/common/streams/_access.py @@ -9,7 +9,7 @@ import temporalio.converter from temporalio.common.streams import _convert, _reader -from temporalio.common.streams._backend import CallOptions, StreamBackend +from temporalio.common.streams._backend import BackendCalls, CallOptions from temporalio.common.streams._defn import TopicDefinition from temporalio.common.streams._handle import StreamHandle, TopicBase from temporalio.common.streams._items import End, close_status @@ -24,7 +24,7 @@ class Access: """The calls of one Stream handle, bound to a backend.""" - backend: StreamBackend + backend: BackendCalls converter: temporalio.converter.DataConverter namespace: str stream_id: Callable[[], Awaitable[str]] @@ -188,7 +188,6 @@ async def publish(self, *values: T) -> int: A list is one value. """ - await self._stream._ready() return await self._stream._access().publish(self._name, values) def subscribe( diff --git a/temporalio/common/streams/_activity.py b/temporalio/common/streams/_activity.py index 7c271331fe..9b6f15491b 100644 --- a/temporalio/common/streams/_activity.py +++ b/temporalio/common/streams/_activity.py @@ -9,20 +9,22 @@ import temporalio.api.streamservice.v1 as stream_pb import temporalio.converter -from temporalio.common.streams._backend import StreamBackend +from temporalio.common.streams._backend import BackendCalls from temporalio.common.streams._writer import StreamWriter @dataclass class ActivityStreams: - """What a Stream plugin gives the Activity handles for one Activity attempt.""" + """What a Stream backend gives the Activity handles for one Activity attempt.""" - backend: StreamBackend + backend: BackendCalls converter: temporalio.converter.DataConverter namespace: str producer: stream_pb.Producer loop: asyncio.AbstractEventLoop backend_name: str = "" + # Whether the Worker's backend is the default, which a handle with no backend name uses. + default: bool = False # The Workflow of the attempt, so a close checks that the Stream is its run chain's. workflow_id: str | None = None # A Local Activity's run history, which lets its writes withdraw the run's orphaned Records. diff --git a/temporalio/common/streams/_activity_stream.py b/temporalio/common/streams/_activity_stream.py index 86048dd96c..29ad96ccc8 100644 --- a/temporalio/common/streams/_activity_stream.py +++ b/temporalio/common/streams/_activity_stream.py @@ -34,13 +34,18 @@ def _bound(handle: StreamHandle[Any]) -> _activity.ActivityStreams: streams = _activity.current.get() if streams is None: raise _errors.StreamConfigError( - "No Stream plugin runs this Activity, such as RedisStreamPlugin" + "No Stream backend runs this Activity. Register one on the Worker's client " + "with stream_backends=." ) if handle.backend and handle.backend != streams.backend_name: raise _errors.StreamConfigError( f"The Stream {handle.id} is of the backend {handle.backend}. The Worker has only " f"{streams.backend_name}." ) + if not handle.backend and not streams.default: + raise _errors.no_default_backend( + "Worker", [streams.backend_name], "a handle that carries its backend name" + ) return streams diff --git a/temporalio/common/streams/_backend.py b/temporalio/common/streams/_backend.py index 4380910a1d..24f02b2a48 100644 --- a/temporalio/common/streams/_backend.py +++ b/temporalio/common/streams/_backend.py @@ -102,8 +102,8 @@ def longest_wait(longest: timedelta) -> timedelta: return min(longest, max(timeout - timedelta(seconds=1), timeout / 2)) -class StreamBackend(Protocol): - """What a Stream plugin gives the handles. Not public. Each backend has its own class.""" +class BackendCalls(Protocol): + """The calls that a Stream backend gives the handles of one client.""" def service( self, @@ -136,7 +136,7 @@ def free_writer(self, writer_id: int) -> None: ... -class _ExtensionStreamBackend: +class _ExtensionCalls: """Carries Stream calls to the crate's runtime object, which serves one client. The five Stream service operations go through the generated client, and the writer @@ -200,7 +200,7 @@ async def close_writers(self) -> int: class _ExtensionTransport: # The generated client takes a transport, so the five operations reach the runtime as they # would reach a server later. - def __init__(self, backend: _ExtensionStreamBackend, namespace: str) -> None: + def __init__(self, backend: _ExtensionCalls, namespace: str) -> None: self._backend = backend self._namespace = namespace diff --git a/temporalio/common/streams/_buffered.py b/temporalio/common/streams/_buffered.py index 09064a77ba..26f2f55071 100644 --- a/temporalio/common/streams/_buffered.py +++ b/temporalio/common/streams/_buffered.py @@ -219,7 +219,6 @@ async def _send(self, batch: list[tuple[str, Any]]) -> int: if self._topic is not None: return await self._topic.publish(*(value for _, value in batch)) assert self._stream is not None - await self._stream._ready() return await self._stream._access().publish_records(batch) @property diff --git a/temporalio/common/streams/_errors.py b/temporalio/common/streams/_errors.py index a6c36b5f8c..48ac3fde49 100644 --- a/temporalio/common/streams/_errors.py +++ b/temporalio/common/streams/_errors.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Sequence + import temporalio.exceptions import temporalio.service from temporalio.common.streams.proto.v1 import StreamErrorDetails @@ -12,7 +14,7 @@ class StreamError(temporalio.exceptions.ApplicationError): It is an application error that is not retryable, except :py:class:`StreamStorageError`. So in Workflow code, where only - :py:class:`StreamClosedError` and the plugin errors are raised, an uncaught one fails the + :py:class:`StreamClosedError` and the backend errors are raised, an uncaught one fails the Workflow and does not fail the Workflow Task again and again. .. warning:: @@ -98,12 +100,23 @@ class StreamSeqDivergentError(StreamError): class StreamConfigError(RuntimeError): - """No Stream plugin, or no backend of a given name. It is config, not a Store error. + """No Stream backend, or no backend of a given name. It is config, not a Store error. ``except StreamError`` does not catch it. It fails a Workflow Task, and an Activity retries. """ +def no_default_backend( + owner: str, names: Sequence[str], hint: str +) -> StreamConfigError: + """The error of a call with no backend name when no backend has ``default=True``.""" + return StreamConfigError( + f"No Stream backend of the {owner} ({', '.join(names)}) has default=True, so the " + "default is native, which this SDK does not have. Mark a backend with " + f"default=True, or name one, such as with {hint}." + ) + + class ReadOnlyContextError(StreamError): """Workflow code that must not change state, such as a query handler, wrote to a Stream.""" diff --git a/temporalio/common/streams/redis.py b/temporalio/common/streams/_extension.py similarity index 76% rename from temporalio/common/streams/redis.py rename to temporalio/common/streams/_extension.py index 2ff8a49308..23384ce19c 100644 --- a/temporalio/common/streams/redis.py +++ b/temporalio/common/streams/_extension.py @@ -1,8 +1,4 @@ -"""The Redis Stream plugin. - -.. warning:: - This module is experimental. -""" +"""The Stream backends whose calls go to the temporalio-streams runtime in the bridge.""" from __future__ import annotations @@ -11,7 +7,6 @@ import logging import weakref from collections.abc import Mapping, Sequence -from datetime import timedelta from typing import Any, NoReturn import temporalio.activity @@ -25,26 +20,30 @@ import temporalio.worker import temporalio.workflow from temporalio.common.streams import _activity, _ids, _writes -from temporalio.common.streams import workflow as streams_workflow -from temporalio.common.streams._backend import StreamBackend, _ExtensionStreamBackend +from temporalio.common.streams import _workflow as streams_workflow +from temporalio.common.streams._backend import BackendCalls, _ExtensionCalls from temporalio.common.streams._errors import StreamConfigError from temporalio.common.streams._handle import StreamHandle -from temporalio.common.streams._plugin import StreamPlugin +from temporalio.common.streams._plugin import StreamBackend from temporalio.common.streams.proto import v1 as ext_pb from temporalio.common.streams.system import models from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner logger = logging.getLogger(__name__) -# The plugin puts this key into the headers of each Activity and Local Activity that a Workflow +# The backend puts this key into the headers of each Activity and Local Activity that a Workflow # schedules. It keeps the Producer id the same across retries. _KEY_HEADER = "__temporal_stream_key" # Continue-as-new passes the run chain's open Streams to the next run in this header. _OWNED_STREAMS_HEADER = "__temporal_owned_streams" -class _ExtensionStreamPlugin(StreamPlugin): - # The Redis and memory plugins: the Stream calls go to the temporalio-streams runtime. +class ExtensionStreamBackend(StreamBackend): + """Base of the Redis and memory backends, whose calls go to the temporalio-streams runtime. + + .. warning:: + This API is experimental. + """ def __init__( self, @@ -55,17 +54,17 @@ def __init__( backend_name: str, default: bool, ) -> None: - """Create the plugin over the bridge's Stream runtime, on Redis or in memory. + """Create the backend over the bridge's Stream runtime, on Redis or in memory. ``backend_name`` is the name that Stream handles carry for this backend. ``default`` - makes it the client's default backend when the client has more than one. + makes it the backend of a create with no backend name. """ config.backend = backend_name self._config = config.SerializeToString() - self._backends: weakref.WeakSet[_ExtensionStreamBackend] = weakref.WeakSet() + self._backends: weakref.WeakSet[_ExtensionCalls] = weakref.WeakSet() # Each client gets its own runtime object on Redis. The memory Store has one for the - # plugin, so every client and Worker that shares the plugin shares its Streams. - self._memory: _ExtensionStreamBackend | None = None + # backend, so every client and Worker that shares the backend shares its Streams. + self._memory: _ExtensionCalls | None = None if memory: self._memory = self._new_backend( runtime or temporalio.runtime.Runtime.default(), True @@ -79,10 +78,10 @@ def __init__( def _new_backend( self, runtime: temporalio.runtime.Runtime, memory: bool = False - ) -> _ExtensionStreamBackend: + ) -> _ExtensionCalls: if self._memory is not None and not memory: return self._memory - backend = _ExtensionStreamBackend( + backend = _ExtensionCalls( temporalio.bridge.streams.StreamRuntime( runtime._core_runtime, memory, self._config ) @@ -90,11 +89,11 @@ def _new_backend( self._backends.add(backend) return backend - def configure_client( + def _configure_client( self, config: temporalio.client.ClientConfig ) -> temporalio.client.ClientConfig: - """Check the default backend, and give the client its own Stream runtime.""" - config = super().configure_client(config) + # Each client gets its own Stream runtime. + config = super()._configure_client(config) backend = self._new_backend(_runtime_of(config["service_client"])) config["interceptors"] = [ *config.get("interceptors", []), @@ -102,14 +101,15 @@ def configure_client( ] return config - def backend_for(self, client: temporalio.client.Client) -> StreamBackend: - """The Stream runtime that this plugin gave ``client``.""" + def backend_for(self, client: temporalio.client.Client) -> BackendCalls: + """The Stream runtime that this backend gave ``client``.""" interceptor = self._interceptor_of( client.config(active_config=True)["interceptors"] ) if interceptor is None or interceptor.backend is None: raise StreamConfigError( - f"The client has no {type(self).__name__}. Put the plugin on the client." + f"The client has no {type(self).__name__}. Register it with " + "stream_backends=." ) return interceptor.backend @@ -122,11 +122,11 @@ def _interceptor_of(self, interceptors: Sequence[Any]) -> _StreamInterceptor | N return interceptor return None - def configure_worker( + def _configure_worker( self, config: temporalio.worker.WorkerConfig ) -> temporalio.worker.WorkerConfig: - """Add the plugin's Activities, and its interceptor when the client has none.""" - config = super().configure_worker(config) + # Add the backend's Activities, and its interceptor when the client has none. + config = super()._configure_worker(config) client = config["client"] # type: ignore[reportTypedDictNotRequiredAccess] interceptors = [ *client.config(active_config=True)["interceptors"], @@ -136,8 +136,9 @@ def configure_worker( isinstance(other, _StreamInterceptor) and other.plugin is not self for other in interceptors ): - raise ValueError( - "The Worker has another Stream plugin. For now a Worker serves one backend." + raise StreamConfigError( + "The Worker's client has another Stream backend. For now a Worker serves " + "one backend." ) # Records get no serialization context, so Workflows, Activities and clients encode # them alike. @@ -160,11 +161,11 @@ def configure_worker( ) return config - def configure_replayer( + def _configure_replayer( self, config: temporalio.worker.ReplayerConfig ) -> temporalio.worker.ReplayerConfig: - """Add the Workflow interceptor. A replay runs no Activity, so it needs no runtime.""" - config = super().configure_replayer(config) + # A replay runs no Activity, so it needs the Workflow interceptor and no runtime. + config = super()._configure_replayer(config) config["interceptors"] = [ *config.get("interceptors", []), _StreamInterceptor(self, None, None), @@ -193,72 +194,13 @@ def _runtime_of( return runtime or temporalio.runtime.Runtime.default() -class RedisStreamPlugin(_ExtensionStreamPlugin): - """Carries Stream calls to Redis, through the temporalio-streams runtime in the bridge. - - Put it on the client. A Worker takes its client's plugins, so one plugin serves Workflow - code, Activities and the client. Each client of the plugin gets its own runtime object, and - the client's Workers share it. It registers the backend name ``redis``, which the Stream - handles it creates carry. For now a Worker takes one Stream plugin, so it serves one - backend. A client may have more than one, and ``backend=`` picks one. - - .. warning:: - This API is experimental. - """ - - def __init__( - self, - url: str | Sequence[str], - *, - cluster: bool = False, - key_prefix: str | None = None, - retention: timedelta | None = None, - safety_ttl: timedelta | None = None, - response_timeout: timedelta | None = None, - name: str = "redis", - default: bool = False, - ) -> None: - """Create the plugin. Each client of the plugin connects to Redis on its first Stream - call, on the client's runtime. - - Args: - url: A Redis URL, or the seed URLs of a cluster. Credentials and TLS come from the - URL. - cluster: Whether the URLs are of a Redis cluster. - key_prefix: The prefix of every key. Defaults to ``temporal-streams``. - retention: The default Retention of a new Stream. Defaults to 7 days. - safety_ttl: How long an open Workflow-owned Stream stays after its last write. - Defaults to 90 days. - response_timeout: How long one Redis call takes at most. Defaults to 5 s. - name: The backend name that Stream handles carry, so every process routes a - handle it receives. - default: Make this the client's default backend. A client with more than one - Stream plugin marks exactly one, else it raises ``StreamConfigError``. - """ - config = ext_pb.RedisStreamConfig( - urls=[url] if isinstance(url, str) else list(url), - cluster=cluster, - key_prefix=key_prefix or "", - ) - for field, value in ( - ("retention", retention), - ("safety_ttl", safety_ttl), - ("response_timeout", response_timeout), - ): - if value is not None: - getattr(config, field).FromTimedelta(value) - super().__init__( - "temporalio.RedisStreamPlugin", False, config, None, name, default - ) - - class _StreamInterceptor(temporalio.client.Interceptor, temporalio.worker.Interceptor): - # One for each client of the plugin, with that client's Stream runtime. The client's + # One for each client of the backend, with that client's Stream runtime. The client's # Workers take it from the client. def __init__( self, - plugin: _ExtensionStreamPlugin, - backend: _ExtensionStreamBackend | None, + plugin: ExtensionStreamBackend, + backend: _ExtensionCalls | None, converter: temporalio.converter.DataConverter | None, ) -> None: super().__init__() @@ -269,11 +211,15 @@ def __init__( def workflow_interceptor_class( self, input: temporalio.worker.WorkflowInterceptorClassInput ) -> type[temporalio.worker.WorkflowInboundInterceptor] | None: - # Workflow code learns the Worker's backend name from the interceptor class. + # Workflow code learns the Worker's backend from the interceptor class. return type( "_WorkflowInbound", (_WorkflowInbound,), - {"_backend_name": self.plugin.backend_name}, + { + "_backend_name": self.plugin.backend_name, + "_default": self.plugin.default, + "_kind": self.plugin._kind, + }, ) def intercept_activity( @@ -284,9 +230,13 @@ def intercept_activity( class _WorkflowInbound(temporalio.worker.WorkflowInboundInterceptor): _backend_name = "" + _default = False + _kind = "" def init(self, outbound: temporalio.worker.WorkflowOutboundInterceptor) -> None: - super().init(_WorkflowOutbound(outbound, self._backend_name)) + super().init( + _WorkflowOutbound(outbound, self._backend_name, self._default, self._kind) + ) async def execute_workflow( self, input: temporalio.worker.ExecuteWorkflowInput @@ -340,10 +290,16 @@ class _WorkflowOutbound(temporalio.worker.WorkflowOutboundInterceptor): _temporal_stream_transport = True def __init__( - self, next: temporalio.worker.WorkflowOutboundInterceptor, backend_name: str + self, + next: temporalio.worker.WorkflowOutboundInterceptor, + backend_name: str, + default: bool, + kind: str, ) -> None: super().__init__(next) self._temporal_stream_backend_name = backend_name + self._temporal_stream_default = default + self._temporal_stream_kind = kind # Numbers each Activity call of the run in order, so the key is the same on replay and # a retry keeps it. A counter, not workflow.uuid4(), leaves the user's random values # alone. @@ -519,6 +475,7 @@ def _activity_streams( producer=producer, loop=asyncio.get_running_loop(), backend_name=interceptor.plugin.backend_name, + default=interceptor.plugin.default, workflow_id=info.workflow_id, previous_completed_task=previous, failed_tasks=failed, diff --git a/temporalio/common/streams/_handle.py b/temporalio/common/streams/_handle.py index bc4691e4a8..7d5c321ec6 100644 --- a/temporalio/common/streams/_handle.py +++ b/temporalio/common/streams/_handle.py @@ -272,10 +272,6 @@ def _known_types(self) -> dict[str, Any]: topics = definition_of(self._definition).topics return {name: topic.type for name, topic in topics.items()} - async def _ready(self) -> None: - # A handle that must look up its backend first does it here. - pass - def _fields(self) -> dict[str, str]: # The fields that name the Stream. They need no call, so a handle whose id is not known # yet still compares, hashes and prints. diff --git a/temporalio/common/streams/_plugin.py b/temporalio/common/streams/_plugin.py index 920f97cc3c..129e8cebf5 100644 --- a/temporalio/common/streams/_plugin.py +++ b/temporalio/common/streams/_plugin.py @@ -1,44 +1,57 @@ -"""The base of the Stream plugins.""" +"""The base of the Stream backends, and the rules of their registration.""" from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Sequence from typing import Any import temporalio.client import temporalio.worker from temporalio.common.streams import _errors -from temporalio.common.streams._backend import StreamBackend +from temporalio.common.streams._backend import BackendCalls from temporalio.plugin import SimplePlugin +# The default backend when no registered backend has default=True. This SDK has no native +# backend, so a call that needs the default then raises StreamConfigError. +NATIVE = "native" -class StreamPlugin(SimplePlugin): - """Base of the Stream plugins. Each registers one backend under its name. - Put a Stream plugin on the client. A Worker takes its client's plugins. With more than one - Stream plugin, exactly one has ``default=True``. +class StreamBackend(SimplePlugin): + """Base of the Stream backends. Each registers one backend under its name. + + Register backends on the client with ``Client.connect(..., stream_backends=[...])``. A + Worker takes its client's backends, and a ``Replayer`` takes ``stream_backends=`` too. + Names are unique. With more than one registered backend, exactly one has ``default=True``. + With none marked, the default is ``native``, which this SDK does not have, so a create with + no backend raises ``StreamConfigError``. Mark a backend with ``default=True``, or use the + calls of its package, such as ``temporalio.contrib.stream.workflow.create_stream``. + + A backend is not a plugin, so passing it in ``plugins=`` raises ``StreamConfigError``. .. warning:: This API is experimental. """ + # The kind of Store, such as "redis", so the calls of a backend's package find the + # registered backend of their kind. + _kind = "" + def __init__( self, name: str, *, backend_name: str, default: bool, - stream_backend: StreamBackend | None = None, + calls: BackendCalls | None = None, **options: Any, ) -> None: - """Create the plugin. ``options`` go to :py:class:`temporalio.plugin.SimplePlugin`. + """Create the backend. ``options`` go to :py:class:`temporalio.plugin.SimplePlugin`. - A plugin with no ``stream_backend`` gives each client its own in - :py:meth:`backend_for`. + A backend with no ``calls`` gives each client its own in :py:meth:`backend_for`. """ self._backend_name = backend_name self._default = default - self._stream_backend = stream_backend + self._calls = calls super().__init__(name, **options) @property @@ -48,91 +61,156 @@ def backend_name(self) -> str: @property def default(self) -> bool: - """Whether a client with more than one Stream plugin uses this one by default.""" + """Whether a create with no backend name uses this backend.""" return self._default def backend_for( self, client: temporalio.client.Client, # pyright: ignore[reportUnusedParameter] - ) -> StreamBackend: - """The backend that carries the calls of ``client`` and of its Workers.""" - if self._stream_backend is None: - raise NotImplementedError(f"{type(self).__name__} gives no backend") - return self._stream_backend + ) -> BackendCalls: + """The calls of this backend for ``client`` and its Workers.""" + if self._calls is None: + raise NotImplementedError(f"{type(self).__name__} gives no calls") + return self._calls def configure_client( self, config: temporalio.client.ClientConfig ) -> temporalio.client.ClientConfig: - """Check the default backend, then add the plugin's parts to the client.""" - check_default(config.get("plugins", []), "client") - return super().configure_client(config) + """Refuse a backend in ``plugins=``.""" + raise _not_a_plugin(self) def configure_worker( self, config: temporalio.worker.WorkerConfig ) -> temporalio.worker.WorkerConfig: - """Check the default backend, then add the plugin's parts to the Worker.""" - client = config.get("client") - client_plugins = client.config()["plugins"] if client is not None else [] - check_default([*client_plugins, *config.get("plugins", [])], "Worker") + """Refuse a backend in ``plugins=``.""" + raise _not_a_plugin(self) + + def configure_replayer( + self, config: temporalio.worker.ReplayerConfig + ) -> temporalio.worker.ReplayerConfig: + """Refuse a backend in ``plugins=``.""" + raise _not_a_plugin(self) + + # The registration calls these, so the plugin hooks above only see a backend that went + # into plugins= by mistake. + + def _configure_client( + self, config: temporalio.client.ClientConfig + ) -> temporalio.client.ClientConfig: + return super().configure_client(config) + + def _configure_worker( + self, config: temporalio.worker.WorkerConfig + ) -> temporalio.worker.WorkerConfig: return super().configure_worker(config) + def _configure_replayer( + self, config: temporalio.worker.ReplayerConfig + ) -> temporalio.worker.ReplayerConfig: + return super().configure_replayer(config) + + +def _not_a_plugin(backend: StreamBackend) -> _errors.StreamConfigError: + return _errors.StreamConfigError( + f"{type(backend).__name__} is a Stream backend, not a plugin. Pass it in " + "stream_backends= of Client.connect, or of the Replayer." + ) -def check_default(plugins: Iterable[object], owner: str) -> None: - """Check that one Stream plugin is the default. + +def check_backends(backends: Sequence[object], owner: str) -> list[StreamBackend]: + """Check the registered backends, and return them. The default backend is part of Workflow code, since a replayed create uses it. So the - default must not follow the order of the plugin list. + default must not follow the order of the list. Raises: - StreamConfigError: More than one Stream plugin, and not exactly one with - ``default=True``. + TypeError: An entry is not a ``StreamBackend``. + StreamConfigError: Two backends have one name, or more than one backend and not + exactly one with ``default=True``. """ - stream_plugins = list( - {id(p): p for p in plugins if isinstance(p, StreamPlugin)}.values() - ) - if len(stream_plugins) < 2: - return - defaults = [p.backend_name for p in stream_plugins if p.default] + checked: list[StreamBackend] = [] + for backend in backends: + if not isinstance(backend, StreamBackend): + raise TypeError(f"{backend!r} is not a StreamBackend") + if any(other.backend_name == backend.backend_name for other in checked): + raise _errors.StreamConfigError( + f"The {owner} has two Stream backends named {backend.backend_name}" + ) + checked.append(backend) + if len(checked) < 2: + return checked + defaults = [b.backend_name for b in checked if b.default] if len(defaults) != 1: - names = ", ".join(p.backend_name for p in stream_plugins) + names = ", ".join(b.backend_name for b in checked) marked = f"{', '.join(defaults)} have" if defaults else "none has" raise _errors.StreamConfigError( - f"The {owner} has the Stream plugins {names}, and {marked} default=True. " + f"The {owner} has the Stream backends {names}, and {marked} default=True. " "Mark exactly one with default=True." ) + return checked + + +def default_name(backends: Sequence[StreamBackend]) -> str: + """The name of the default backend: the one with ``default=True``, else ``native``.""" + return next((b.backend_name for b in backends if b.default), NATIVE) + + +def configure_client( + config: temporalio.client.ClientConfig, +) -> temporalio.client.ClientConfig: + """Check the client's backends, and add the parts of each to the client.""" + for backend in check_backends(config.get("stream_backends", ()), "client"): + config = backend._configure_client(config) + return config + + +def configure_worker( + backends: Sequence[StreamBackend], config: temporalio.worker.WorkerConfig +) -> temporalio.worker.WorkerConfig: + """Add the parts of the client's backends to a Worker.""" + for backend in backends: + config = backend._configure_worker(config) + return config + + +def configure_replayer( + config: temporalio.worker.ReplayerConfig, +) -> temporalio.worker.ReplayerConfig: + """Check the Replayer's backends, and add the parts of each to it.""" + for backend in check_backends(config.get("stream_backends", ()), "Replayer"): + config = backend._configure_replayer(config) + return config + + +def client_backends(client: temporalio.client.Client) -> list[StreamBackend]: + """The backends registered on a client.""" + return list(client.config(active_config=True).get("stream_backends", ())) def client_backend( - client: temporalio.client.Client, name: str | None, *, only: bool = False -) -> tuple[str, StreamBackend]: - """The name and the backend of a client's Stream plugin. None picks the default plugin, - which is the only one or the one with ``default=True``. With ``only``, None needs a client - with one backend, since a handle with no backend name may be of any of them. + client: temporalio.client.Client, name: str | None, *, hint: str = "backend=" +) -> tuple[str, BackendCalls]: + """The name and the calls of a client's backend. None picks the default backend. Raises: - StreamConfigError: The client has no Stream plugin, none of that name, or more than - one where ``only`` needs one. + StreamConfigError: The client has no backend of that name, or none has + ``default=True`` for a call with no name. """ - backends = [ - (plugin.backend_name, plugin.backend_for(client), plugin.default) - for plugin in client.config()["plugins"] - if isinstance(plugin, StreamPlugin) - ] + backends = client_backends(client) + wanted = name or default_name(backends) + for backend in backends: + if backend.backend_name == wanted: + return wanted, backend.backend_for(client) if not backends: raise _errors.StreamConfigError( - "The client has no Stream plugin, such as RedisStreamPlugin" - ) - if not name and only and len(backends) > 1: - raise _errors.StreamConfigError( - "The handle names no backend, and the client has more than one" + "The client has no Stream backend. Register one with " + "Client.connect(..., stream_backends=[...])." ) if not name: - defaults = [entry for entry in backends if entry[2]] or backends - return defaults[0][0], defaults[0][1] - for backend_name, backend, _ in backends: - if backend_name == name: - return backend_name, backend + raise _errors.no_default_backend( + "client", [b.backend_name for b in backends], hint + ) raise _errors.StreamConfigError( f"The client has no Stream backend {name}. It has " - f"{', '.join(entry[0] for entry in backends)}." + f"{', '.join(b.backend_name for b in backends)}." ) diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index 8a5bf35300..dd84655d03 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -21,7 +21,7 @@ read_spec, subscription, ) -from temporalio.common.streams._backend import StreamBackend +from temporalio.common.streams._backend import BackendCalls from temporalio.common.streams._defn import TopicDefinition, definition_of from temporalio.common.streams._handle import Creating, StreamHandle, exit_failure from temporalio.common.streams._items import End @@ -115,7 +115,11 @@ def subscribe( class ClientStream(StreamHandle[E]): - """A Stream that a client writes, reads and closes. Get one from :py:class:`StreamClient`. + """A Stream that a client writes, reads and closes. + + Get one from :py:meth:`temporalio.client.Client.stream`, + :py:meth:`temporalio.client.WorkflowHandle.stream` or a backend's create, such as + :py:func:`temporalio.contrib.stream.create`. Each publish writes before it returns. Use :py:class:`BufferedPublisher` to batch. The handles of one client and Stream write as one Producer, and :py:meth:`producer` gives a @@ -135,7 +139,7 @@ def __init__( backend: str | None = None, definition: type[E] | None = None, ) -> None: - """Create a handle. Use :py:class:`StreamClient` instead.""" + """Create a handle. Use :py:meth:`temporalio.client.Client.stream` instead.""" super().__init__( id=id, workflow_id=workflow_id, @@ -144,14 +148,12 @@ def __init__( definition=definition, ) self._client: temporalio.client.Client | None = None - self._stream_backend: StreamBackend | None = None + self._stream_backend: BackendCalls | None = None self._find_first_run: Callable[[], Awaitable[str]] | None = None self._writer: StreamWriter | None = None # The Stream name of a Workflow's Stream whose first run id is not known yet. self._stream_name = "" self._finding: asyncio.Future[str] | None = None - # The backends to look in, in order, until one has the Stream. - self._probe: list[tuple[str, StreamBackend]] = [] self._closed = False # The name and the epoch of a named Producer. self._named: tuple[str, int] | None = None @@ -159,40 +161,14 @@ def __init__( def _bind( self, client: temporalio.client.Client, - backend: StreamBackend, + backend: BackendCalls, find_first_run: Callable[[], Awaitable[str]] | None = None, - probe: list[tuple[str, StreamBackend]] | None = None, ) -> ClientStream[E]: self._client = client self._stream_backend = backend self._find_first_run = find_first_run - self._probe = probe or [] return self - @property - def backend(self) -> str | None: - """Name of the backend. A handle that looks in many backends knows it after a call.""" - return None if self._probe else self._backend - - async def _ready(self) -> None: - # The first backend that has the Stream keeps it. Two first calls may both look. - for name, backend in self._probe: - assert self._client is not None - request = models.DescribeStreamRequest( - stream=models.StreamRef(id=await self._stream_id()) - ) - try: - await backend.service(self._client.namespace).describe_stream(request) - except _errors.StreamNotFoundError: - continue - self._backend, self._stream_backend, self._probe = name, backend, [] - return - if self._probe: - raise _errors.StreamNotFoundError( - f"No backend of {[name for name, _ in self._probe]} has the Stream {self._id}", - reason="stream_not_found", - ) - @property def id(self) -> str: """The Stream id. @@ -234,7 +210,7 @@ def _access(self) -> Access: if client is None or backend is None: raise _errors.StreamConfigError( "This ClientStream came from a payload, so it has no client. " - "Use StreamClient.get_handle." + "Use Client.stream(definition, stream=handle)." ) def writer() -> StreamWriter: @@ -280,9 +256,7 @@ def producer(self, name: str, epoch: int) -> ClientStream[E]: handle._stream_name = self._stream_name handle._named = (name, epoch) if self._client is not None and self._stream_backend is not None: - handle._bind( - self._client, self._stream_backend, self._find_first_run, self._probe - ) + handle._bind(self._client, self._stream_backend, self._find_first_run) return handle @overload @@ -365,7 +339,6 @@ def subscribe( async def _items( self, read: ReadSpec, position: Position ) -> AsyncGenerator[Item, None]: - await self._ready() items = self._access().items(read, position) try: async for item in items: @@ -388,7 +361,6 @@ async def get_offset( ``DEADLINE_EXCEEDED``. """ with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): - await self._ready() return await self._access().get_offset() async def close( @@ -418,7 +390,6 @@ async def close( ) self._closed = True with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): - await self._ready() await self._access().close(result, failure) async def __aenter__(self) -> ClientStream[E]: @@ -443,160 +414,120 @@ async def __aexit__( await self.close(failure=failure) -class StreamClient: - """Creates and gets Streams over a Temporal client with a Stream plugin. - - .. warning:: - This API is experimental. +def create( + client: temporalio.client.Client, + definition: type[E], + *, + id: str, + retention: timedelta | None, + backend: str, +) -> Creating[ClientStream[E]]: + """Create a Stream that this client owns on a registered backend. See + :py:func:`temporalio.contrib.stream.create`. """ - def __init__(self, client: temporalio.client.Client) -> None: - """Create a Stream client. It makes no call.""" - self._client = client - - def create( - self, - definition: type[E], - *, - id: str, - retention: timedelta | None = None, - backend: str | None = None, - rpc_metadata: Mapping[str, str | bytes] = {}, - rpc_timeout: timedelta | None = None, - ) -> Creating[ClientStream[E]]: - """Create a Stream that this client owns, so only a client closes it. - - It is idempotent for one id and one Retention. Await the call for the handle, or use - it in ``async with``, which closes the Stream at the end of the block. - - Args: - definition: The Stream definition class. - id: The Stream id, as a Workflow takes a Workflow id. - retention: How long the Records stay after the close. Defaults to the backend's. - backend: The name of the backend. Defaults to the client's default backend. - rpc_metadata: Headers of the call. See :py:meth:`ClientStream.get_offset`. - rpc_timeout: The deadline of the call. - - Raises: - StreamConfigError: The client has no backend of that name. No call is made. - StreamClosedError: A Stream of that id was created and closed before. - StreamError: The Stream exists with another Retention, with the reason - ``stream_already_exists``. - """ - - async def create() -> ClientStream[E]: - definition_of(definition) - backend_name, stream_backend = client_backend(self._client, backend) - service = stream_backend.service(self._client.namespace) - with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): - response = await service.create_stream( - models.CreateStreamRequest(stream_id=id, retention=retention) - ) - if response.status == stream_pb.StreamStatus.STREAM_STATUS_CLOSED: - raise _errors.StreamClosedError( - f"The Stream {id} was created and closed before", - reason="stream_closed", - ) - stream = ClientStream( - id=id, backend=backend_name or None, definition=definition - ) - return stream._bind(self._client, stream_backend) - - return Creating(create) - - def get_handle( - self, - definition: type[E], - *, - id: str | None = None, - stream: StreamHandle[Any] | None = None, - backend: str | tuple[str, ...] | None = None, - ) -> ClientStream[E]: - """A Stream by its id, or a handle that came as data, such as in a Record. No call. - - Args: - definition: The Stream definition class. - id: The Stream id. - stream: A handle of the Stream, such as one in a Record. Its id, owner and - backend stay. - backend: The name of the backend. Defaults to the client's default backend. A - tuple, such as ``("native", "redis")`` during a move, looks in each in order - on first use and keeps the first that has the Stream. - """ + async def create() -> ClientStream[E]: definition_of(definition) - if stream is not None: - # An empty name is of any backend, so it needs a client with one. - backend_name, stream_backend = client_backend( - self._client, stream.backend, only=True - ) - probe: list[tuple[str, StreamBackend]] = [] - elif id is not None: - backend_name, stream_backend, probe = self._backends(backend) - else: - raise TypeError("get_handle takes id= or stream=") - handle = ClientStream( - id=stream.id if stream else id or "", - workflow_id=stream.workflow_id if stream else None, - first_execution_run_id=stream.first_execution_run_id if stream else None, - backend=backend_name or None, - definition=definition, + backend_name, calls = client_backend(client, backend) + response = await calls.service(client.namespace).create_stream( + models.CreateStreamRequest(stream_id=id, retention=retention) ) - return handle._bind(self._client, stream_backend, probe=probe) - - def _backends( - self, backend: str | tuple[str, ...] | None - ) -> tuple[str, StreamBackend, list[tuple[str, StreamBackend]]]: - if not isinstance(backend, tuple): - return (*client_backend(self._client, backend), []) - probe = [client_backend(self._client, name) for name in backend] - return (*probe[0], probe if len(probe) > 1 else []) + if response.status == stream_pb.StreamStatus.STREAM_STATUS_CLOSED: + raise _errors.StreamClosedError( + f"The Stream {id} was created and closed before", + reason="stream_closed", + ) + stream = ClientStream(id=id, backend=backend_name, definition=definition) + return stream._bind(client, calls) + + return Creating(create) + + +def client_stream( + client: temporalio.client.Client, + definition: type[E], + *, + id: str | None, + stream: StreamHandle[Any] | None, + backend: str | None, +) -> ClientStream[E]: + """A Stream by its id, or a handle that came as data. See + :py:meth:`temporalio.client.Client.stream`. + """ + definition_of(definition) + if (id is None) == (stream is None): + raise TypeError("stream takes one of id= and stream=") + if stream is not None: + if backend and stream.backend and backend != stream.backend: + raise ValueError( + f"The handle is of the backend {stream.backend}, not {backend}" + ) + backend = backend or stream.backend + backend_name, calls = client_backend(client, backend) + handle = ClientStream( + id=stream.id if stream else id or "", + workflow_id=stream.workflow_id if stream else None, + first_execution_run_id=stream.first_execution_run_id if stream else None, + backend=backend_name, + definition=definition, + ) + return handle._bind(client, calls) - def workflow_stream( - self, - handle: temporalio.client.WorkflowHandle[Any, Any], - definition: type[E], - *, - backend: str | tuple[str, ...] | None = None, - ) -> ClientStream[E]: - """A Stream that a Workflow created, by the Workflow handle and the definition. - It builds the id from the Workflow id and the first run id, so it makes no call. A - handle with no first run id makes one Describe call on the Stream's first use. +# The first run id that one Describe call found for a Workflow handle, so the Streams of one +# handle share the call. +_FIRST_RUNS: weakref.WeakKeyDictionary[ + temporalio.client.WorkflowHandle[Any, Any], asyncio.Future[str] +] = weakref.WeakKeyDictionary() - Args: - handle: A handle of the Workflow, best the one that ``start_workflow`` returned. - definition: The Stream definition class, renamed as the Workflow's create. - backend: The name of the backend, or a tuple of names to look in, as for - :py:meth:`get_handle`. Defaults to the client's default backend. - """ - stream_name = definition_of(definition).name - backend_name, stream_backend, probe = self._backends(backend) - first_run = handle.first_execution_run_id - stream = ClientStream( - id=_ids.scoped_stream_id(handle.id, first_run, stream_name) - if first_run - else "", - workflow_id=handle.id, - first_execution_run_id=first_run, - backend=backend_name or None, - definition=definition, - ) - if first_run: - return stream._bind(self._client, stream_backend, probe=probe) - - async def find_first_run() -> str: - # The first call that needs the id gives its own call options. - options = _backend.call_options.get() - description = await handle.describe( - rpc_metadata=options.metadata if options else {}, - rpc_timeout=options.timeout if options else None, - ) - first = description.raw_description.workflow_execution_info.first_run_id - if not first: - raise ValueError( - "The server gave no first run id, so the Stream id is unknown" - ) - return first - stream._stream_name = stream_name - return stream._bind(self._client, stream_backend, find_first_run, probe) +async def _describe_first_run( + handle: temporalio.client.WorkflowHandle[Any, Any], +) -> str: + # The first call that needs the id gives its own call options. + options = _backend.call_options.get() + description = await handle.describe( + rpc_metadata=options.metadata if options else {}, + rpc_timeout=options.timeout if options else None, + ) + first = description.raw_description.workflow_execution_info.first_run_id + if not first: + raise ValueError("The server gave no first run id, so the Stream id is unknown") + return first + + +async def _first_run(handle: temporalio.client.WorkflowHandle[Any, Any]) -> str: + finding = _FIRST_RUNS.get(handle) + if finding is None or (finding.done() and finding.exception() is not None): + # A failed call does not stay, so a later call tries again. + finding = asyncio.ensure_future(_describe_first_run(handle)) + _FIRST_RUNS[handle] = finding + return await asyncio.shield(finding) + + +def workflow_stream( + handle: temporalio.client.WorkflowHandle[Any, Any], + definition: type[E], + *, + backend: str | None, +) -> ClientStream[E]: + """A Stream that a Workflow created. See + :py:meth:`temporalio.client.WorkflowHandle.stream`. + """ + stream_name = definition_of(definition).name + client = handle._client + backend_name, calls = client_backend(client, backend) + first_run = handle.first_execution_run_id + stream = ClientStream( + id=_ids.scoped_stream_id(handle.id, first_run, stream_name) + if first_run + else "", + workflow_id=handle.id, + first_execution_run_id=first_run, + backend=backend_name, + definition=definition, + ) + if first_run: + return stream._bind(client, calls) + stream._stream_name = stream_name + return stream._bind(client, calls, lambda: _first_run(handle)) diff --git a/temporalio/common/streams/workflow.py b/temporalio/common/streams/_workflow.py similarity index 84% rename from temporalio/common/streams/workflow.py rename to temporalio/common/streams/_workflow.py index 6e3469349e..b2f7a86e28 100644 --- a/temporalio/common/streams/workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -1,16 +1,14 @@ """Streams in Workflow code. -Workflow code creates a Stream with :py:func:`create_stream` and gets one again with -:py:func:`stream_handle`. A Stream that the Workflow receives as an argument, such as a -:py:class:`WorkflowStream` that a run passes on through continue-as-new, works the same. +Workflow code creates a Stream with :py:func:`temporalio.workflow.create_stream` and gets one +again with :py:func:`temporalio.workflow.stream_handle`. A Stream that the Workflow receives as +an argument, such as a :py:class:`WorkflowStream` that a run passes on through continue-as-new, +works the same. Each create, publish and close is a Local Activity, so it has a place in the run's commands, as an Activity call has. To add, remove or move one in a Workflow that has open runs, gate the change with ``workflow.patched``, else the replay of those runs fails as nondeterministic. Publishes that wait behind a write to the same Stream go as one Local Activity. - -.. warning:: - This module is experimental. """ from __future__ import annotations @@ -19,7 +17,7 @@ from dataclasses import dataclass, field from datetime import timedelta from types import TracebackType -from typing import Any, TypeVar, overload +from typing import Any, NamedTuple, TypeVar, overload import google.protobuf.message @@ -41,7 +39,7 @@ E = TypeVar("E") T = TypeVar("T") -# The plugin's Local Activities that write for Workflow code. The marker keeps the answer only, +# The backend's Local Activities that write for Workflow code. The marker keeps the answer only, # since a Local Activity's input is not in History. CREATE_ACTIVITY = "__temporal_streams_create" APPEND_ACTIVITY = "__temporal_streams_append" @@ -53,14 +51,6 @@ _MERGE_RECORDS = 1000 _MERGE_BYTES = 2 * 1024 * 1024 -__all__ = [ - "WorkflowStream", - "WorkflowTopic", - "create_stream", - "create_stream_nowait", - "stream_handle", -] - @dataclass class _RunStream: @@ -130,31 +120,56 @@ def _state() -> _RunState: return state -def _backend_name() -> str | None: - # A Stream plugin marks its outbound interceptor with its backend name. With none, the - # call would go to the server as a System Nexus command, which only native Streams serve. +class WorkerBackend(NamedTuple): + """The Stream backend of the Worker that runs the Workflow.""" + + name: str + default: bool + kind: str + + +def worker_backend() -> WorkerBackend | None: + """The Worker's backend, which marks the run's outbound interceptor, or None.""" outbound = getattr(temporalio.workflow._Runtime.current(), "_outbound", None) while outbound is not None: if getattr(outbound, "_temporal_stream_transport", False): - return getattr(outbound, "_temporal_stream_backend_name", "") + return WorkerBackend( + getattr(outbound, "_temporal_stream_backend_name", ""), + getattr(outbound, "_temporal_stream_default", False), + getattr(outbound, "_temporal_stream_kind", ""), + ) outbound = getattr(outbound, "next", None) return None +def no_worker_backend() -> _errors.StreamConfigError: + """The error of Workflow code on a Worker with no Stream backend.""" + return _errors.StreamConfigError( + "No Stream backend is on the Worker. Register one on its client with " + "stream_backends=." + ) + + def _check_backend(backend: str | None, action: str) -> str: # The Worker's config gives these errors, not the history, so they fail the Workflow Task # and a Worker with the right config retries it. An ApplicationError would fail the run. - worker_backend = _backend_name() - if worker_backend is None: - raise _errors.StreamConfigError( - "No Stream plugin is on the Worker, such as RedisStreamPlugin" - ) - if backend and backend != worker_backend: + worker = worker_backend() + if worker is None: + raise no_worker_backend() + if not backend: + if not worker.default: + raise _errors.no_default_backend( + "Worker", + [worker.name], + "temporalio.contrib.stream.workflow.create_stream", + ) + return worker.name + if backend != worker.name: raise _errors.StreamConfigError( f"Can't {action} a Stream of the backend {backend}. The Worker has only " - f"{worker_backend}." + f"{worker.name}." ) - return worker_backend + return worker.name def _records(request: google.protobuf.message.Message) -> int: @@ -273,7 +288,7 @@ async def drain() -> None: def _payload(value: Any) -> Any: - # The Workflow's converter runs here, in the sandbox. The plugin runs only the codec. + # The Workflow's converter runs here, in the sandbox. The backend runs only the codec. return temporalio.workflow.payload_converter().to_payload(value) @@ -297,7 +312,7 @@ async def publish(self, *values: T) -> int: Stream that something else closed. StreamClosedError: This run closed the Stream. ReadOnlyContextError: The code must not change state, such as a query handler. - StreamConfigError: No Stream plugin is on the Worker, or the Stream is of a + StreamConfigError: No Stream backend is on the Worker, or the Stream is of a backend that the Worker does not have. It fails the Workflow Task, not the run. """ return await self._stream._append(self._name, values) @@ -483,33 +498,8 @@ def create_stream( retention: timedelta | None = None, backend: str | None = None, ) -> Creating[WorkflowStream[E]]: - """Create a Stream that this run chain owns. - - Its id is ``::``, so a client finds it from the Workflow - handle with no call. The create is one Local Activity, and its error raises here. Only the - run chain closes the Stream. On Redis, the end of the run chain closes nothing, so a Stream - that nobody closes stays open until its ``safety_ttl``. - - Await the call for the handle, or use it in ``async with``, which closes the Stream at the - end of the block:: - - async with create_stream(streams.with_name(Events, "turn-3")) as turn: - await turn.topic(Events.tokens).publish("hi") - - Args: - definition: The Stream definition class. Its name is the Stream name. For one Stream - per turn, for example, pass ``streams.with_name(Events, f"turn-{n}")``. - retention: How long the Records stay after the close. Defaults to the backend's. - backend: The name of the backend. Defaults to the Worker's. The default is part of - the Workflow's code, since a replay must create on the same backend. To move a - Workflow to another backend, gate it with a patch, for example - ``backend="native" if workflow.patched("streams-native") else "redis"``. - - Raises: - StreamError: The backend refused the create, such as for another Retention. - ReadOnlyContextError: The code must not change state, such as a query handler. - StreamConfigError: No Stream plugin is on the Worker, or it has no backend of that name. - It fails the Workflow Task, not the run. + """Create a Stream that this run chain owns. See + :py:func:`temporalio.workflow.create_stream`. """ async def create() -> WorkflowStream[E]: @@ -526,11 +516,8 @@ def create_stream_nowait( retention: timedelta | None = None, backend: str | None = None, ) -> WorkflowStream[E]: - """Create a Stream that this run chain owns, with no await. - - It is for sync Workflow code, such as ``@workflow.init``, as ``asyncio.Queue.put_nowait`` - is for code that can't await ``put``. It takes the options of :py:func:`create_stream`. A - backend error fails the Workflow Task. + """Create a Stream that this run chain owns, with no await. See + :py:func:`temporalio.workflow.create_stream_nowait`. """ stream, task = _create(definition, retention, backend) _write_nowait(task) @@ -565,18 +552,13 @@ def _create( def stream_handle( definition: type[E], *, backend: str | None = None ) -> WorkflowStream[E]: - """A Stream that this run chain created, by its definition. It makes no call. - - A later run of the chain, such as one after continue-as-new, uses it to write the Stream - again. - - Args: - definition: The Stream definition class, renamed as for the create. - backend: The backend of the Stream. Defaults to the one that the run chain created it - on, and else to the Worker's. + """A Stream that this run chain created, by its definition. See + :py:func:`temporalio.workflow.stream_handle`. """ stream = _own_handle(definition, backend) if backend is None: known = _state().streams.get(stream.id) - stream._backend = (known.backend if known else None) or _backend_name() or None + stream._backend = (known.backend if known else None) or _check_backend( + None, "get" + ) return stream diff --git a/temporalio/common/streams/_writes.py b/temporalio/common/streams/_writes.py index f45455362c..c0a1fc8dcf 100644 --- a/temporalio/common/streams/_writes.py +++ b/temporalio/common/streams/_writes.py @@ -14,19 +14,19 @@ import temporalio.api.streamservice.v1 as stream_pb import temporalio.common from temporalio.common.streams import _activity, _errors -from temporalio.common.streams.system import client as service_client -from temporalio.common.streams.system import models -from temporalio.common.streams.workflow import ( +from temporalio.common.streams._workflow import ( APPEND_ACTIVITY, CLOSE_ACTIVITY, CREATE_ACTIVITY, ) +from temporalio.common.streams.system import client as service_client +from temporalio.common.streams.system import models def _context() -> tuple[_activity.ActivityStreams, service_client.StreamServiceClient]: streams = _activity.current.get() if streams is None: - raise _errors.StreamConfigError("No Stream plugin runs this Local Activity") + raise _errors.StreamConfigError("No Stream backend runs this Local Activity") return streams, streams.backend.service(streams.namespace, streams.converter) diff --git a/temporalio/contrib/stream/__init__.py b/temporalio/contrib/stream/__init__.py new file mode 100644 index 0000000000..4dd740c693 --- /dev/null +++ b/temporalio/contrib/stream/__init__.py @@ -0,0 +1,159 @@ +"""The Redis Stream backend. + +Register it on the client. :py:func:`create` makes a Stream that the client owns, and +:py:mod:`temporalio.contrib.stream.workflow` has the Workflow calls that pick this backend:: + + from temporalio.contrib import stream as redis + + client = await Client.connect( + "localhost:7233", stream_backends=[redis.RedisStreamBackend("redis://localhost")] + ) + +.. warning:: + This package is experimental. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import timedelta +from typing import TypeVar + +import temporalio.client +from temporalio.common.streams import _plugin, _stream_client +from temporalio.common.streams._errors import StreamConfigError +from temporalio.common.streams._extension import ExtensionStreamBackend +from temporalio.common.streams._handle import Creating +from temporalio.common.streams._stream_client import ClientStream +from temporalio.common.streams.proto import v1 as ext_pb +from temporalio.contrib.stream import workflow + +D = TypeVar("D") + + +class RedisStreamBackend(ExtensionStreamBackend): + """Carries Stream calls to Redis, through the temporalio-streams runtime in the bridge. + + Register it on the client with ``stream_backends=``. A Worker takes its client's backends, + so one backend serves Workflow code, Activities and the client. Each client of the backend + gets its own runtime object, and the client's Workers share it. It registers the backend + name ``redis``, which the Stream handles it creates carry. For now a Worker serves one + backend. A client may have more than one, and ``backend=`` picks one. + + .. warning:: + This API is experimental. + """ + + _kind = "redis" + + def __init__( + self, + url: str | Sequence[str], + *, + cluster: bool = False, + key_prefix: str | None = None, + retention: timedelta | None = None, + safety_ttl: timedelta | None = None, + response_timeout: timedelta | None = None, + name: str = "redis", + default: bool = False, + ) -> None: + """Create the backend. Each client of the backend connects to Redis on its first + Stream call, on the client's runtime. + + Args: + url: A Redis URL, or the seed URLs of a cluster. Credentials and TLS come from the + URL. + cluster: Whether the URLs are of a Redis cluster. + key_prefix: The prefix of every key. Defaults to ``temporal-streams``. + retention: The default Retention of a new Stream. Defaults to 7 days. + safety_ttl: How long an open Workflow-owned Stream stays after its last write. + Defaults to 90 days. + response_timeout: How long one Redis call takes at most. Defaults to 5 s. + name: The backend name that Stream handles carry, so every process routes a + handle it receives. + default: Make this the backend of a create with no backend name. With more than + one registered backend, exactly one has it, else the client raises + ``StreamConfigError``. + """ + config = ext_pb.RedisStreamConfig( + urls=[url] if isinstance(url, str) else list(url), + cluster=cluster, + key_prefix=key_prefix or "", + ) + for field, value in ( + ("retention", retention), + ("safety_ttl", safety_ttl), + ("response_timeout", response_timeout), + ): + if value is not None: + getattr(config, field).FromTimedelta(value) + super().__init__( + "temporalio.RedisStreamBackend", False, config, None, name, default + ) + + +def create( + client: temporalio.client.Client, + definition: type[D], + *, + id: str, + retention: timedelta | None = None, + backend: str | None = None, +) -> Creating[ClientStream[D]]: + """Create a Stream that this client owns, so only a client closes it. + + It is idempotent for one id and one Retention. Await the call for the handle, or use it in + ``async with``, which closes the Stream at the end of the block:: + + async with redis.create(client, Events, id="chat-1") as chat: + await chat.topic(Events.turns).publish(Turn("user", "hi")) + + .. warning:: + This API is experimental. + + Args: + client: The client, with a registered Redis backend. + definition: The Stream definition class. + id: The Stream id, as a Workflow takes a Workflow id. + retention: How long the Records stay after the close. Defaults to the backend's. + backend: The name of the Redis backend. Defaults to the client's only Redis backend, + or the one with ``default=True``. + + Raises: + StreamConfigError: The client has no Redis backend of that name. No call is made. + StreamClosedError: A Stream of that id was created and closed before. + StreamError: The Stream exists with another Retention, with the reason + ``stream_already_exists``. + """ + + async def create() -> ClientStream[D]: + name = _redis_backend(_plugin.client_backends(client), backend) + return await _stream_client.create( + client, definition, id=id, retention=retention, backend=name + ) + + return Creating(create) + + +def _redis_backend(backends: Sequence[_plugin.StreamBackend], name: str | None) -> str: + # The memory backend keeps Redis's rules, so it stands in for Redis in tests. + redis = [backend for backend in backends if backend._kind == "redis"] + if name: + if not any(backend.backend_name == name for backend in redis): + raise StreamConfigError(f"The client has no Redis Stream backend {name}") + return name + if not redis: + raise StreamConfigError( + "The client has no Redis Stream backend. Register a RedisStreamBackend with " + "stream_backends=." + ) + defaults = [backend for backend in redis if backend.default] + if len(redis) > 1 and len(defaults) != 1: + raise StreamConfigError( + "The client has more than one Redis Stream backend, so pass backend=" + ) + return (defaults if len(redis) > 1 else redis)[0].backend_name + + +__all__ = ["RedisStreamBackend", "create", "workflow"] diff --git a/temporalio/common/streams/testing.py b/temporalio/contrib/stream/testing.py similarity index 56% rename from temporalio/common/streams/testing.py rename to temporalio/contrib/stream/testing.py index 64dbc95943..b0ea5eec44 100644 --- a/temporalio/common/streams/testing.py +++ b/temporalio/contrib/stream/testing.py @@ -1,4 +1,4 @@ -"""A Stream plugin for tests, with no Redis. +"""A Stream backend for tests, with no Redis. .. warning:: This module is experimental. @@ -9,21 +9,25 @@ from datetime import timedelta import temporalio.runtime +from temporalio.common.streams._extension import ExtensionStreamBackend from temporalio.common.streams.proto import v1 as ext_pb -from temporalio.common.streams.redis import _ExtensionStreamPlugin -class MemoryStreamPlugin(_ExtensionStreamPlugin): +class MemoryStreamBackend(ExtensionStreamBackend): """Keeps Streams in this process, with the same rules as on Redis, for tests. - Put it on the client, as with ``RedisStreamPlugin``. Every Worker and client that shares one - plugin shares its Streams, and they go when the process ends. It registers the backend - name ``memory``, so tests swap the Store with no change to Workflow code. + Register it on the client, as with ``RedisStreamBackend``. Every Worker and client that + shares one backend shares its Streams, and they go when the process ends. It registers the + backend name ``memory``. It keeps Redis's rules, so the calls of + :py:mod:`temporalio.contrib.stream` take it as a Redis backend, and tests swap the Store + with no change to Workflow code. .. warning:: This API is experimental. """ + _kind = "redis" + def __init__( self, *, @@ -33,17 +37,17 @@ def __init__( name: str = "memory", default: bool = False, ) -> None: - """Create the plugin. + """Create the backend. Args: retention: The default Retention of a new Stream. Defaults to 7 days. safety_ttl: How long an open Workflow-owned Stream stays after its last write. Defaults to 90 days. - runtime: The runtime of the plugin's memory Store. Defaults to the default + runtime: The runtime of the backend's memory Store. Defaults to the default runtime. name: The backend name that Stream handles carry. - default: Make this the client's default backend. A client with more than one - Stream plugin marks exactly one. + default: Make this the backend of a create with no backend name. With more than + one registered backend, exactly one has it. """ config = ext_pb.RedisStreamConfig() if retention is not None: @@ -51,5 +55,5 @@ def __init__( if safety_ttl is not None: config.safety_ttl.FromTimedelta(safety_ttl) super().__init__( - "temporalio.MemoryStreamPlugin", True, config, runtime, name, default + "temporalio.MemoryStreamBackend", True, config, runtime, name, default ) diff --git a/temporalio/contrib/stream/workflow.py b/temporalio/contrib/stream/workflow.py new file mode 100644 index 0000000000..b9316efdd5 --- /dev/null +++ b/temporalio/contrib/stream/workflow.py @@ -0,0 +1,99 @@ +"""Streams in Workflow code on the registered Redis backend. + +These calls work as :py:func:`temporalio.workflow.create_stream`, +:py:func:`temporalio.workflow.create_stream_nowait` and +:py:func:`temporalio.workflow.stream_handle`, with the Worker's Redis backend in place of the +default one. A ``MemoryStreamBackend`` keeps Redis's rules, so tests use it here with no change +to Workflow code. + +.. warning:: + This module is experimental. +""" + +from __future__ import annotations + +from datetime import timedelta +from typing import TypeVar + +from temporalio.common.streams import _workflow +from temporalio.common.streams._errors import StreamConfigError +from temporalio.common.streams._handle import Creating +from temporalio.common.streams._workflow import WorkflowStream + +D = TypeVar("D") + + +def _backend(backend: str | None) -> str: + # The Worker's config gives this error, so it fails the Workflow Task and not the run. + worker = _workflow.worker_backend() + if worker is None: + raise _workflow.no_worker_backend() + if worker.kind != "redis": + raise StreamConfigError( + f"The Worker's Stream backend {worker.name} is not a Redis backend" + ) + return backend or worker.name + + +def create_stream( + definition: type[D], + *, + retention: timedelta | None = None, + backend: str | None = None, +) -> Creating[WorkflowStream[D]]: + """Create a Stream that this run chain owns, on the Worker's Redis backend. + + See :py:func:`temporalio.workflow.create_stream`. On Redis, the end of the run chain closes + nothing, so close the Stream before the run returns. + + .. warning:: + This API is experimental. + + Args: + definition: The Stream definition class. Its name is the Stream name. + retention: How long the Records stay after the close. Defaults to the backend's. + backend: The name of the Redis backend. Defaults to the Worker's. + + Raises: + StreamError: The backend refused the create, such as for another Retention. + StreamConfigError: The Worker has no Redis backend, or none of that name. It fails + the Workflow Task, not the run. + """ + + async def create() -> WorkflowStream[D]: + return await _workflow.create_stream( + definition, retention=retention, backend=_backend(backend) + ) + + return Creating(create) + + +def create_stream_nowait( + definition: type[D], + *, + retention: timedelta | None = None, + backend: str | None = None, +) -> WorkflowStream[D]: + """Create a Stream on the Worker's Redis backend, with no await. + + See :py:func:`temporalio.workflow.create_stream_nowait`. + + .. warning:: + This API is experimental. + """ + return _workflow.create_stream_nowait( + definition, retention=retention, backend=_backend(backend) + ) + + +def stream_handle( + definition: type[D], *, backend: str | None = None +) -> WorkflowStream[D]: + """A Stream that this run chain created on the Worker's Redis backend. It makes no call. + + See :py:func:`temporalio.workflow.stream_handle`. + + .. warning:: + This API is experimental. + """ + return _workflow.stream_handle(definition, backend=_backend(backend)) diff --git a/temporalio/testing/_workflow.py b/temporalio/testing/_workflow.py index 6c8354162c..3d363a9215 100644 --- a/temporalio/testing/_workflow.py +++ b/temporalio/testing/_workflow.py @@ -8,6 +8,7 @@ from contextlib import asynccontextmanager, contextmanager from datetime import datetime, timedelta, timezone from typing import ( + TYPE_CHECKING, Any, cast, ) @@ -31,6 +32,9 @@ TimeSkippingConfig, ) +if TYPE_CHECKING: + from temporalio.common.streams import StreamBackend + logger = logging.getLogger(__name__) @@ -102,6 +106,7 @@ async def start_local( dev_server_download_ttl: timedelta | None = None, dev_server_startup_timeout: timedelta | None = None, ui_port: int | None = None, + stream_backends: Sequence[StreamBackend] = (), ) -> WorkflowEnvironment: """Start a full Temporal server locally, downloading if necessary. @@ -165,6 +170,8 @@ async def start_local( become ready after spawning. Defaults to 5 seconds and does not include download time. ui_port: UI port to use if UI is enabled. + stream_backends: See parameter of the same name on + :py:meth:`temporalio.client.Client.connect`. Returns: The started CLI dev server workflow environment. @@ -194,6 +201,7 @@ async def start_local( dev_server_download_ttl=dev_server_download_ttl, dev_server_startup_timeout=dev_server_startup_timeout, ui_port=ui_port, + stream_backends=stream_backends, ) @classmethod @@ -216,6 +224,7 @@ async def start_time_skipping( test_server_extra_args: Sequence[str] = [], test_server_download_ttl: timedelta | None = None, test_server_startup_timeout: timedelta | None = None, + stream_backends: Sequence[StreamBackend] = (), ) -> WorkflowEnvironment: """Start a V1 time skipping workflow environment. @@ -276,6 +285,8 @@ async def start_time_skipping( test_server_startup_timeout: Maximum time to wait for the test server to become ready after spawning. Defaults to 5 seconds and does not include download time. + stream_backends: See parameter of the same name on + :py:meth:`temporalio.client.Client.connect`. Returns: The started workflow environment with time skipping. @@ -296,6 +307,7 @@ async def start_time_skipping( test_server_extra_args=test_server_extra_args, test_server_download_ttl=test_server_download_ttl, test_server_startup_timeout=test_server_startup_timeout, + stream_backends=stream_backends, ) @classmethod @@ -582,6 +594,7 @@ async def _bootstrap_dev_server( dev_server_download_ttl: timedelta | None = None, dev_server_startup_timeout: timedelta | None = None, ui_port: int | None = None, + stream_backends: Sequence[StreamBackend] = (), ) -> tuple[ temporalio.bridge.testing.EphemeralServer, temporalio.client.Client, @@ -649,6 +662,7 @@ async def _bootstrap_dev_server( rpc_metadata=rpc_metadata, identity=identity, runtime=runtime, + stream_backends=stream_backends, ) except: try: @@ -719,6 +733,7 @@ async def _create( test_server_extra_args: Sequence[str], test_server_download_ttl: timedelta | None, test_server_startup_timeout: timedelta | None, + stream_backends: Sequence[StreamBackend], ) -> _V1WorkflowEnvironment: runtime = runtime or temporalio.runtime.Runtime.default() download_ttl_ms = None @@ -753,6 +768,7 @@ async def _create( rpc_metadata=rpc_metadata, identity=identity, runtime=runtime, + stream_backends=stream_backends, ) except: try: diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index b3eb1a4d12..92a05909ca 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -8,6 +8,7 @@ from collections.abc import AsyncIterator, Mapping, Sequence from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass +from typing import TYPE_CHECKING from typing_extensions import TypedDict @@ -32,6 +33,9 @@ ) from .workflow_sandbox import SandboxedWorkflowRunner +if TYPE_CHECKING: + from temporalio.common.streams import StreamBackend + logger = logging.getLogger(__name__) @@ -56,13 +60,16 @@ def __init__( runtime: temporalio.runtime.Runtime | None = None, disable_safe_workflow_eviction: bool = False, header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, + stream_backends: Sequence[StreamBackend] = (), ) -> None: """Create a replayer to replay workflows from history. See :py:meth:`temporalio.worker.Worker.__init__` for a description of most of the arguments. Most of the same arguments need to be passed to the replayer that were passed to the worker when the workflow originally - ran. + ran. ``stream_backends`` are the Stream backends of the Worker's client, + such as :py:class:`temporalio.contrib.stream.RedisStreamBackend`. A + replay makes no Stream call, so they need not reach their Store. Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that @@ -86,6 +93,7 @@ def __init__( runtime=runtime, disable_safe_workflow_eviction=disable_safe_workflow_eviction, header_codec_behavior=header_codec_behavior, + stream_backends=stream_backends, ) self._initial_config = self._config.copy() self._default_workflow_logic_flags = set(_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS) @@ -94,6 +102,10 @@ def __init__( self.plugins = plugins for plugin in plugins: self._config = plugin.configure_replayer(self._config) + if self._config.get("stream_backends"): + from temporalio.common.streams import _plugin as _stream_plugin + + self._config = _stream_plugin.configure_replayer(self._config) # Validate workflows after plugin configuration if not self._config.get("workflows"): @@ -433,6 +445,7 @@ class ReplayerConfig(TypedDict, total=False): runtime: temporalio.runtime.Runtime | None disable_safe_workflow_eviction: bool header_codec_behavior: HeaderCodecBehavior + stream_backends: Sequence[StreamBackend] @dataclass(frozen=True) diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 3c6effd80e..8b30d9fc42 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -407,6 +407,14 @@ def __init__( plugins = plugins_from_client + list(plugins) self._initial_config = config.copy() + # The Worker takes its client's Stream backends, before the plugins as a client's + # plugins come first. + stream_backends = client.config(active_config=True).get("stream_backends") + if stream_backends: + from temporalio.common.streams import _plugin as _stream_plugin + + config = _stream_plugin.configure_worker(stream_backends, config) + self._plugins = plugins for plugin in plugins: config = plugin.configure_worker(config) diff --git a/temporalio/workflow/__init__.py b/temporalio/workflow/__init__.py index c631fd88e8..ef1c20a37c 100644 --- a/temporalio/workflow/__init__.py +++ b/temporalio/workflow/__init__.py @@ -151,6 +151,11 @@ logger, unsafe, ) +from ._streams import ( + create_stream, + create_stream_nowait, + stream_handle, +) from ._workflow_ops import ( ChildWorkflowCancellationType, ChildWorkflowConfig, @@ -258,6 +263,9 @@ "SandboxImportNotificationPolicy", "logger", "unsafe", + "create_stream", + "create_stream_nowait", + "stream_handle", "ChildWorkflowCancellationType", "ChildWorkflowConfig", "ChildWorkflowHandle", diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py new file mode 100644 index 0000000000..301e2a69c2 --- /dev/null +++ b/temporalio/workflow/_streams.py @@ -0,0 +1,104 @@ +"""Streams in Workflow code. + +The calls load :py:mod:`temporalio.common.streams` on first use, so a Workflow with no Stream +never loads it. +""" + +from __future__ import annotations + +from datetime import timedelta +from typing import TYPE_CHECKING, TypeVar + +if TYPE_CHECKING: + from temporalio.common.streams import Creating, WorkflowStream + +D = TypeVar("D") + + +def create_stream( + definition: type[D], + *, + retention: timedelta | None = None, + backend: str | None = None, +) -> Creating[WorkflowStream[D]]: + """Create a Stream that this run chain owns. + + Its id is ``::``, so a client finds it from the Workflow + handle with no call. The create is one Local Activity, and its error raises here. Only the + run chain closes the Stream. On Redis, the end of the run chain closes nothing, so a Stream + that nobody closes stays open until its ``safety_ttl``. + + Await the call for the handle, or use it in ``async with``, which closes the Stream at the + end of the block:: + + async with workflow.create_stream(streams.with_name(Events, "turn-3")) as turn: + await turn.topic(Events.tokens).publish("hi") + + .. warning:: + This API is experimental. + + Args: + definition: The Stream definition class. Its name is the Stream name. For one Stream + per turn, for example, pass ``streams.with_name(Events, f"turn-{n}")``. + retention: How long the Records stay after the close. Defaults to the backend's. + backend: The name of the backend. Defaults to the registered backend with + ``default=True``. The default is part of the Workflow's code, since a replay must + create on the same backend. To move a Workflow to another backend, gate it with a + patch, for example ``backend="redis-b" if workflow.patched("move") else "redis"``. + + Raises: + StreamError: The backend refused the create, such as for another Retention. + ReadOnlyContextError: The code must not change state, such as a query handler. + StreamConfigError: No Stream backend is on the Worker, none of that name, or none + has ``default=True`` for a create with no backend. It fails the Workflow Task, + not the run. + """ + from temporalio.common.streams import _workflow + + return _workflow.create_stream(definition, retention=retention, backend=backend) + + +def create_stream_nowait( + definition: type[D], + *, + retention: timedelta | None = None, + backend: str | None = None, +) -> WorkflowStream[D]: + """Create a Stream that this run chain owns, with no await. + + It is for sync Workflow code, such as ``@workflow.init``, as ``asyncio.Queue.put_nowait`` + is for code that can't await ``put``. It takes the options of :py:func:`create_stream`. A + backend error fails the Workflow Task. + + .. warning:: + This API is experimental. + """ + from temporalio.common.streams import _workflow + + return _workflow.create_stream_nowait( + definition, retention=retention, backend=backend + ) + + +def stream_handle( + definition: type[D], *, backend: str | None = None +) -> WorkflowStream[D]: + """A Stream that this run chain created, by its definition. It makes no call. + + A later run of the chain, such as one after continue-as-new, uses it to write or close the + Stream again. + + .. warning:: + This API is experimental. + + Args: + definition: The Stream definition class, renamed as for the create. + backend: The backend of the Stream. Defaults to the one that the run chain created it + on, and else to the registered backend with ``default=True``. + + Raises: + StreamConfigError: No backend is given, known or marked as the default. + """ + from temporalio.common.streams import _workflow + + return _workflow.stream_handle(definition, backend=backend) diff --git a/tests/contrib/streams/conftest.py b/tests/contrib/streams/conftest.py index 8853732587..bf93a3526d 100644 --- a/tests/contrib/streams/conftest.py +++ b/tests/contrib/streams/conftest.py @@ -9,9 +9,9 @@ import pytest from temporalio.client import Client -from temporalio.common.streams import StreamPlugin -from temporalio.common.streams.redis import RedisStreamPlugin -from temporalio.common.streams.testing import MemoryStreamPlugin +from temporalio.common.streams import StreamBackend +from temporalio.contrib.stream import RedisStreamBackend +from temporalio.contrib.stream.testing import MemoryStreamBackend # Ports that other local services own. _RESERVED = {7333, 7861, 7931, 6431, 7101, 7102, 7103} @@ -60,13 +60,15 @@ async def stream_client( client: Client, request: pytest.FixtureRequest ) -> AsyncIterator[Client]: # Every live test runs on the memory Store and on Redis, since both must keep one set of - # rules. - plugin: StreamPlugin + # rules. The backend is the default, so a create with no backend name uses it. + backend: StreamBackend if request.param == "memory": - plugin = MemoryStreamPlugin() + backend = MemoryStreamBackend(default=True) else: redis_url = request.getfixturevalue("redis_url") - plugin = RedisStreamPlugin(redis_url, key_prefix=f"t-{uuid.uuid4()}") + backend = RedisStreamBackend( + redis_url, key_prefix=f"t-{uuid.uuid4()}", default=True + ) config = client.config() - config["plugins"] = [plugin] + config["stream_backends"] = [backend] yield Client(**config) diff --git a/tests/contrib/streams/examples/activity_streams_tokens.py b/tests/contrib/streams/examples/activity_streams_tokens.py index ebdf40a260..cc4bcfc5f8 100644 --- a/tests/contrib/streams/examples/activity_streams_tokens.py +++ b/tests/contrib/streams/examples/activity_streams_tokens.py @@ -15,7 +15,6 @@ from temporalio import activity, workflow from temporalio.client import Client from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker @@ -60,7 +59,7 @@ def summarize(input: SummarizeInput) -> int: class Model: @workflow.run async def run(self, prompt: str) -> list[int]: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) first = await workflow.execute_activity( call_model, CallModelInput(prompt, streams.ActivityStream.from_handle(stream)), @@ -93,7 +92,7 @@ async def main( task_queue=task_queue, ) offsets = await handle.result() - stream = streams.StreamClient(client).workflow_stream(handle, Events) + stream = handle.stream(Events) records = [ item async for item in stream.subscribe() if isinstance(item, streams.Record) ] diff --git a/tests/contrib/streams/examples/agent_publishes_turns.py b/tests/contrib/streams/examples/agent_publishes_turns.py index c2318a91e6..b869d61276 100644 --- a/tests/contrib/streams/examples/agent_publishes_turns.py +++ b/tests/contrib/streams/examples/agent_publishes_turns.py @@ -14,7 +14,6 @@ from temporalio import activity, workflow from temporalio.client import Client from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker @@ -59,9 +58,7 @@ async def run_tool(call: str) -> str: class AgentTurns: @workflow.run async def run(self, messages: list[str]) -> str: - stream = await stream_workflow.create_stream( - Events, retention=timedelta(days=1) - ) + stream = await workflow.create_stream(Events, retention=timedelta(days=1)) turns = stream.topic(Events.turns) while True: # The publish is one Local Activity. History keeps its offset, not the Turn. @@ -100,7 +97,7 @@ async def main( task_queue=task_queue, ) await handle.result() - stream = streams.StreamClient(client).workflow_stream(handle, Events) + stream = handle.stream(Events) return [ item async for item in stream.subscribe() diff --git a/tests/contrib/streams/examples/client_creates_stream.py b/tests/contrib/streams/examples/client_creates_stream.py index 17e6c9daaf..e42a0775cf 100644 --- a/tests/contrib/streams/examples/client_creates_stream.py +++ b/tests/contrib/streams/examples/client_creates_stream.py @@ -13,7 +13,7 @@ from temporalio import workflow from temporalio.client import Client from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow +from temporalio.contrib import stream as redis from temporalio.worker import Worker @@ -31,7 +31,7 @@ class Events: @workflow.defn class Chat: @workflow.run - async def run(self, stream: stream_workflow.WorkflowStream[Events]) -> None: + async def run(self, stream: streams.WorkflowStream[Events]) -> None: # The Workflow only writes. The client closes the Stream. await stream.topic(Events.turns).publish( Turn("model", "hi"), Turn("last", "bye") @@ -42,7 +42,8 @@ async def main(client: Client, task_queue: str) -> list[Turn]: async with Worker(client, task_queue=task_queue, workflows=[Chat]): turns = [] # The block closes the Stream at its end, or with the error that ends it. - async with streams.StreamClient(client).create( + async with redis.create( + client, Events, id=f"chat-{uuid.uuid4()}", retention=timedelta(days=1), @@ -50,7 +51,7 @@ async def main(client: Client, task_queue: str) -> list[Turn]: await client.start_workflow( Chat.run, # A Stream is data, so the Workflow gets it as an argument. - stream_workflow.WorkflowStream.from_handle(chat_stream), + streams.WorkflowStream.from_handle(chat_stream), id=f"chat-{uuid.uuid4()}", task_queue=task_queue, ) diff --git a/tests/contrib/streams/examples/resume_from_offset.py b/tests/contrib/streams/examples/resume_from_offset.py index 0da3156643..3c34dba7ad 100644 --- a/tests/contrib/streams/examples/resume_from_offset.py +++ b/tests/contrib/streams/examples/resume_from_offset.py @@ -13,7 +13,6 @@ from temporalio import workflow from temporalio.client import Client, WorkflowHandle from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker @@ -33,7 +32,7 @@ def __init__(self) -> None: @workflow.run async def run(self, first: list[str]) -> None: - turn = await stream_workflow.create_stream(Turn3) + turn = await workflow.create_stream(Turn3) tokens = turn.topic(Events.tokens) await tokens.publish(*first) await workflow.wait_condition(lambda: bool(self.more) or self.done) @@ -58,9 +57,9 @@ def save(self, key: str, offset: int) -> None: async def read_some( - client: Client, handle: WorkflowHandle[Turns, None], store: OffsetStore, count: int + handle: WorkflowHandle[Turns, None], store: OffsetStore, count: int ) -> list[str]: - turn3 = streams.StreamClient(client).workflow_stream(handle, Turn3) + turn3 = handle.stream(Turn3) tokens = turn3.topic(Events.tokens) seen = [] read = tokens.subscribe( @@ -80,9 +79,9 @@ async def read_some( return seen -async def read_new(client: Client, handle: WorkflowHandle[Turns, None]) -> list[str]: +async def read_new(handle: WorkflowHandle[Turns, None]) -> list[str]: # A view that shows only what comes next starts after the newest Record. - turn3 = streams.StreamClient(client).workflow_stream(handle, Turn3) + turn3 = handle.stream(Turn3) return [ item.data async for item in turn3.topic(Events.tokens).subscribe( @@ -100,9 +99,9 @@ async def main( Turns.run, ["a", "b"], id=f"turns-{uuid.uuid4()}", task_queue=task_queue ) store = OffsetStore() - before = await read_some(client, handle, store, 2) - new = await read_new(client, handle) + before = await read_some(handle, store, 2) + new = await read_new(handle) await handle.signal(Turns.add, ["c", "d"]) - after = await read_some(client, handle, store, 2) + after = await read_some(handle, store, 2) await handle.result() return before, new, after diff --git a/tests/contrib/streams/examples/turn_scoped_stream.py b/tests/contrib/streams/examples/turn_scoped_stream.py index d2d4306259..ab79a9a4d0 100644 --- a/tests/contrib/streams/examples/turn_scoped_stream.py +++ b/tests/contrib/streams/examples/turn_scoped_stream.py @@ -14,7 +14,6 @@ from temporalio import activity, workflow from temporalio.client import Client from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker @@ -51,20 +50,20 @@ class ChatTurnsInput: # The first turn number of this run. A Stream name must be unique in the run chain. next: int = 0 # From the run before continue-as-new. - index: stream_workflow.WorkflowStream[TurnIndex] | None = None + index: streams.WorkflowStream[TurnIndex] | None = None @workflow.defn class ChatTurns: @workflow.run async def run(self, input: ChatTurnsInput) -> None: - index = input.index or await stream_workflow.create_stream( + index = input.index or await workflow.create_stream( TurnIndex, retention=timedelta(days=1) ) started = index.topic(TurnIndex.started) batch = input.prompts[:2] for i, prompt in enumerate(batch): - async with stream_workflow.create_stream( + async with workflow.create_stream( streams.with_name(Events, f"turn-{input.next + i}"), retention=timedelta(days=1), ) as turn: @@ -99,14 +98,13 @@ async def main(client: Client, task_queue: str) -> dict[int, list[str]]: task_queue=task_queue, ) await handle.result() - stream_client = streams.StreamClient(client) - index = stream_client.workflow_stream(handle, TurnIndex) + index = handle.stream(TurnIndex) turns = {} async for item in index.subscribe(): if not isinstance(item, streams.Record): continue # A Stream from a payload has no client, so the reader binds it to this one. - turn = stream_client.get_handle(Events, stream=item.data.stream) + turn = client.stream(Events, stream=item.data.stream) turns[item.data.n] = [ token.data async for token in turn.subscribe() diff --git a/tests/contrib/streams/examples/ui_backend_reads.py b/tests/contrib/streams/examples/ui_backend_reads.py index 8d6ff90b4c..ce54b92621 100644 --- a/tests/contrib/streams/examples/ui_backend_reads.py +++ b/tests/contrib/streams/examples/ui_backend_reads.py @@ -14,7 +14,6 @@ from temporalio import activity, workflow from temporalio.client import Client from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow from temporalio.worker import Worker @@ -48,7 +47,7 @@ async def call_model(input: CallModelInput) -> str: class Agent: @workflow.run async def run(self, prompt: str) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) turns = stream.topic(Events.turns) await turns.publish(Turn("model")) reply = await workflow.execute_activity( @@ -85,7 +84,7 @@ async def main(client: Client, task_queue: str) -> Screen: handle = await client.start_workflow( Agent.run, "hello there", id=f"agent-{uuid.uuid4()}", task_queue=task_queue ) - stream = streams.StreamClient(client).workflow_stream(handle, Events) + stream = handle.stream(Events) screen = Screen() # The Workflow may not have created the Stream yet, so the read waits for it. async for item in stream.subscribe( diff --git a/tests/contrib/streams/examples/worker_setup.py b/tests/contrib/streams/examples/worker_setup.py index bf72452b35..3c12967b34 100644 --- a/tests/contrib/streams/examples/worker_setup.py +++ b/tests/contrib/streams/examples/worker_setup.py @@ -1,7 +1,8 @@ """Brook API proposal, "Worker setup". -The Stream plugin goes on the client, and the Worker takes the client's plugins. Each plugin -registers a backend name. Tests use ``MemoryStreamPlugin`` with no change to Workflow code. +The Stream backends go on the client with ``stream_backends=``, and the Worker takes the +client's backends. Each backend registers a name. A create with no backend name goes to the +one with ``default=True``. Tests use ``MemoryStreamBackend`` with no change to Workflow code. Sync Workflow code, such as ``@workflow.init``, uses the ``_nowait`` calls. """ @@ -12,9 +13,8 @@ from temporalio import workflow from temporalio.client import Client from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow -from temporalio.common.streams.redis import RedisStreamPlugin -from temporalio.common.streams.testing import MemoryStreamPlugin +from temporalio.contrib import stream as redis +from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.worker import Worker @@ -28,7 +28,7 @@ class Greeter: @workflow.init def __init__(self, name: str) -> None: # The init can't await, so it creates and publishes with the _nowait calls. - self.stream = stream_workflow.create_stream_nowait(Events) + self.stream = workflow.create_stream_nowait(Events) self.stream.topic(Events.tokens).publish_nowait(f"hello {name}") @workflow.run @@ -39,26 +39,33 @@ async def run(self, name: str) -> str: async def connect_with_redis(target_host: str, redis_url: str) -> Client: - return await Client.connect(target_host, plugins=[RedisStreamPlugin(redis_url)]) + # With no default=True, the default is native, and Workflow code would name Redis with + # redis.workflow.create_stream. + return await Client.connect( + target_host, + stream_backends=[redis.RedisStreamBackend(redis_url, default=True)], + ) async def connect_for_tests(target_host: str) -> Client: - return await Client.connect(target_host, plugins=[MemoryStreamPlugin()]) + return await Client.connect( + target_host, stream_backends=[MemoryStreamBackend(default=True)] + ) async def connect_while_migrating(target_host: str, url_a: str, url_b: str) -> Client: # Two backends during a move. Creates with no backend= go to the default one. return await Client.connect( target_host, - plugins=[ - RedisStreamPlugin(url_a, name="redis-a", default=True), - RedisStreamPlugin(url_b, name="redis-b"), + stream_backends=[ + redis.RedisStreamBackend(url_a, name="redis-a", default=True), + redis.RedisStreamBackend(url_b, name="redis-b"), ], ) def run_worker(client: Client, task_queue: str) -> Worker: - # The Worker takes the client's plugins, so it needs no Stream setup of its own. + # The Worker takes the client's backends, so it needs no Stream setup of its own. return Worker(client, task_queue=task_queue, workflows=[Greeter]) @@ -68,7 +75,7 @@ async def main(client: Client, task_queue: str) -> list[str]: Greeter.run, "ada", id=f"greeter-{uuid.uuid4()}", task_queue=task_queue ) await handle.result() - stream = streams.StreamClient(client).workflow_stream(handle, Events) + stream = handle.stream(Events) return [ item.data async for item in stream.subscribe() @@ -77,7 +84,7 @@ async def main(client: Client, task_queue: str) -> list[str]: async def create_on_second_backend(client: Client) -> str | None: - stream = await streams.StreamClient(client).create( - Events, id=f"chat-{uuid.uuid4()}", backend="redis-b" + stream = await redis.create( + client, Events, id=f"chat-{uuid.uuid4()}", backend="redis-b" ) return stream.backend diff --git a/tests/contrib/streams/fake_extension.py b/tests/contrib/streams/fake_extension.py index 727b52ca87..bf49aafadf 100644 --- a/tests/contrib/streams/fake_extension.py +++ b/tests/contrib/streams/fake_extension.py @@ -6,6 +6,9 @@ import temporalio.api.streamservice.v1 as stream_pb from temporalio.bridge.streams import StreamCallError +from temporalio.client import Client +from temporalio.common.streams import StreamBackend +from temporalio.common.streams._backend import _ExtensionCalls from temporalio.common.streams.proto import v1 as ext_pb from temporalio.service import RPCStatusCode @@ -146,3 +149,22 @@ async def _read( await asyncio.wait_for(stream.changed.wait(), left) except asyncio.TimeoutError: pass + + +class _FakeBackend(StreamBackend): + # The fake keeps Redis's rules, so the Redis calls take it. + _kind = "redis" + + +def fake_client(client: Client) -> tuple[Client, FakeExtension]: + """A copy of ``client`` whose one Stream backend, ``fake``, is a fake runtime.""" + extension = FakeExtension() + backend = _FakeBackend( + "fake-streams", + backend_name="fake", + default=True, + calls=_ExtensionCalls(extension), + ) + config = client.config() + config["stream_backends"] = [backend] + return Client(**config), extension diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index ae6269ba79..67c02ef199 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -15,7 +15,6 @@ from temporalio.client import Client, WorkflowHandle from temporalio.common import RetryPolicy, streams from temporalio.common.streams import _activity -from temporalio.common.streams import workflow as stream_workflow from temporalio.exceptions import CancelledError from tests.helpers import new_worker @@ -44,12 +43,8 @@ class SyncInput: stream: streams.SyncActivityStream[Events] -async def read( - client: Client, handle: WorkflowHandle[Any, Any], name: str = "events" -) -> list[object]: - stream = streams.StreamClient(client).workflow_stream( - handle, streams.with_name(Events, name) - ) +async def read(handle: WorkflowHandle[Any, Any], name: str = "events") -> list[object]: + stream = handle.stream(streams.with_name(Events, name)) return [item async for item in stream.subscribe(wait_new_records=False)] @@ -91,7 +86,7 @@ async def read_back(stream: streams.ActivityStream[Events]) -> list[Any]: class Agent: @workflow.run async def run(self, prompt: str) -> list[Any]: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) offsets = await workflow.execute_activity( call_model, CallModelInput(prompt, streams.ActivityStream.from_handle(stream)), @@ -122,7 +117,7 @@ async def test_activity_publishes_and_returns_offsets(stream_client: Client): Agent.run, "hi there", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) offsets, last, read_tokens = await handle.result() - items = await read(stream_client, handle) + items = await read(handle) data = records(items) assert [record.data for record in data] == [ "hi", @@ -160,7 +155,7 @@ async def flaky(input: CallModelInput) -> None: class Retrying: @workflow.run async def run(self, key: str) -> None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("start") await workflow.execute_activity( flaky, @@ -177,7 +172,7 @@ async def test_retry_withdraws_a_received_stream(stream_client: Client): Retrying.run, key, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) await handle.result() - items = await read(stream_client, handle) + items = await read(handle) superseded = [item for item in items if isinstance(item, streams.Superseded)] # The second attempt writes nothing, and its start still withdraws the first one's Record. assert len(superseded) == 1 @@ -204,7 +199,7 @@ async def fails_after_it_is_shown(input: CallModelInput) -> None: class ShownThenWithdrawn: @workflow.run async def run(self, key: str) -> None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await workflow.execute_activity( fails_after_it_is_shown, CallModelInput(key, streams.ActivityStream.from_handle(stream)), @@ -228,7 +223,7 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) - stream = streams.StreamClient(stream_client).workflow_stream(handle, Events) + stream = handle.stream(Events) live = stream.subscribe(Events.tokens, create_wait=timedelta(seconds=20)) reconnects = stream.subscribe(Events.tokens, create_wait=timedelta(seconds=20)) first = await anext(live) @@ -279,7 +274,7 @@ async def close_it(stream: streams.ActivityStream[Events]) -> str: class Closing: @workflow.run async def run(self) -> list[str]: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) backend = await workflow.execute_activity( wrong_backend, streams.ActivityStream(id=stream.id, backend="elsewhere"), @@ -308,7 +303,7 @@ async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): ) # An Activity writes to the Workflow's Stream but can't end it for every reader. assert await handle.result() == ["config", "stream_not_owner", "closed"] - items = await read(stream_client, handle) + items = await read(handle) assert [r.data for r in records(items)] == ["last", "last"] assert isinstance(items[-1], streams.Closed) assert items[-1].result == "closed by activity" @@ -318,7 +313,7 @@ async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): class CreatesAndLeavesOpen: @workflow.run async def run(self) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) return stream.id @@ -356,7 +351,7 @@ async def test_store_refuses_another_chains_close(stream_client: Client): ) == "stream_not_owner" ) - stream = streams.StreamClient(stream_client).get_handle(Events, id=stream_id) + stream = stream_client.stream(Events, id=stream_id) items = [item async for item in stream.subscribe(wait_new_records=False)] assert [type(item) for item in items] == [streams.Record] @@ -434,7 +429,7 @@ async def local_step(stream: streams.ActivityStream[Events]) -> list[Any]: class LocalOrder: @workflow.run async def run(self) -> list[Any]: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) tokens = stream.topic(Events.tokens) await tokens.publish("before") seen = await workflow.execute_local_activity( @@ -453,7 +448,7 @@ async def test_workflow_writes_go_out_before_a_local_activity(stream_client: Cli LocalOrder.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await handle.result() == ["before"] - items = await read(stream_client, handle) + items = await read(handle) written = records(items) assert [record.data for record in written] == ["before", "local", "after"] producer = written[1].producer @@ -514,8 +509,8 @@ def sync_read(stream: streams.SyncActivityStream[Events]) -> list[Any]: class Leftovers: @workflow.run async def run(self, key: str) -> list[Any]: - stream = await stream_workflow.create_stream(Events) - other = await stream_workflow.create_stream(streams.with_name(Events, "other")) + stream = await workflow.create_stream(Events) + other = await workflow.create_stream(streams.with_name(Events, "other")) await workflow.execute_activity( never_flushes, CallModelInput("a b", streams.ActivityStream.from_handle(stream)), @@ -553,8 +548,8 @@ async def test_attempt_end_flushes_and_retry_finds_nested_streams( ) # The publisher was never flushed, so the end of the attempt wrote its values. assert await handle.result() == ["a", "b", "attempt-1"] - items = await read(stream_client, handle) - other = await read(stream_client, handle, "other") + items = await read(handle) + other = await read(handle, "other") for read_items in (items, other): superseded = [i for i in read_items if isinstance(i, streams.Superseded)] withdrawn = [r.data for r in records(read_items) if superseded[0].withdraws(r)] @@ -577,7 +572,7 @@ async def long_poll() -> None: stream: streams.SyncActivityStream[Any] = streams.SyncActivityStream(id="s") token = _activity.current.set( - cast(Any, SimpleNamespace(loop=loop, backend_name="")) + cast(Any, SimpleNamespace(loop=loop, backend_name="", default=True)) ) errors: list[BaseException] = [] diff --git a/tests/contrib/streams/test_facade.py b/tests/contrib/streams/test_facade.py index 4089582909..f249793de2 100644 --- a/tests/contrib/streams/test_facade.py +++ b/tests/contrib/streams/test_facade.py @@ -12,12 +12,12 @@ from temporalio.client import Client from temporalio.common import RawValue, streams from temporalio.common.streams import _errors -from temporalio.common.streams._backend import _ExtensionStreamBackend from temporalio.common.streams.proto import v1 as ext_pb from temporalio.common.streams.system import models +from temporalio.contrib import stream as redis from temporalio.exceptions import ApplicationError from temporalio.service import RPCError, RPCStatusCode -from tests.contrib.streams.fake_extension import FakeExtension +from tests.contrib.streams.fake_extension import fake_client @dataclass @@ -26,19 +26,6 @@ class Turn: text: str = "" -def fake_client(client: Client) -> tuple[Client, FakeExtension]: - extension = FakeExtension() - plugin = streams.StreamPlugin( - "fake-streams", - backend_name="fake", - default=False, - stream_backend=_ExtensionStreamBackend(extension), - ) - config = client.config() - config["plugins"] = [plugin] - return Client(**config), extension - - def test_runtime_errors_map_to_stream_errors(): details = ext_pb.StreamErrorDetails(reason="stream_fenced").SerializeToString() err = _errors.from_call_error( @@ -83,7 +70,7 @@ class Events: async def test_client_writes_reads_and_closes(client: Client): client, extension = fake_client(client) - stream = await streams.StreamClient(client).create(Events, id="chat-42") + stream = await redis.create(client, Events, id="chat-42") first = await stream.topic(Events.turns).publish(Turn("model")) second = await stream.topic("tokens").publish("hello") assert second == first + 1 @@ -111,15 +98,14 @@ async def test_client_writes_reads_and_closes(client: Client): async def test_read_waits_for_create(client: Client): client, _ = fake_client(client) - stream_client = streams.StreamClient(client) handle = client.get_workflow_handle("wf-1", first_execution_run_id="run-1") - stream = stream_client.workflow_stream(handle, Events) + stream = handle.stream(Events) with pytest.raises(streams.StreamNotFoundError): async for _ in stream.subscribe(): pass async def create_later() -> None: - created = await stream_client.create(Events, id=stream.id) + created = await redis.create(client, Events, id=stream.id) await created.topic(Events.tokens).publish("five") await created.close() @@ -144,9 +130,8 @@ def test_stream_errors_fail_the_workflow_not_the_task(): async def test_handles_of_one_client_share_a_producer_and_free_it(client: Client): client, extension = fake_client(client) - stream_client = streams.StreamClient(client) - stream = await stream_client.create(Events, id="two-writers") - other = stream_client.get_handle(Events, id="two-writers") + stream = await redis.create(client, Events, id="two-writers") + other = client.stream(Events, id="two-writers") await stream.topic(Events.tokens).publish("1") await other.topic(Events.tokens).publish("2") items = [item async for item in stream.subscribe(wait_new_records=False)] @@ -165,7 +150,7 @@ async def test_handles_of_one_client_share_a_producer_and_free_it(client: Client async def test_rpc_timeout_bounds_each_call_and_not_the_read(client: Client): client, extension = fake_client(client) - stream = await streams.StreamClient(client).create(Events, id="slow") + stream = await redis.create(client, Events, id="slow") call = extension.call async def slow_describe(method: str, input: bytes) -> bytes: diff --git a/tests/contrib/streams/test_pydantic.py b/tests/contrib/streams/test_pydantic.py index 100201a3b3..89ead4949d 100644 --- a/tests/contrib/streams/test_pydantic.py +++ b/tests/contrib/streams/test_pydantic.py @@ -6,9 +6,8 @@ from temporalio import activity, workflow from temporalio.client import Client from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow -from temporalio.common.streams.testing import MemoryStreamPlugin from temporalio.contrib.pydantic import pydantic_data_converter +from temporalio.contrib.stream.testing import MemoryStreamBackend from tests.helpers import new_worker @@ -37,7 +36,7 @@ async def call_model(input: CallModelInput) -> int: class TurnWorkflow: @workflow.run async def run(self, prompt: str) -> TurnResult: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await workflow.execute_activity( call_model, CallModelInput( @@ -53,7 +52,7 @@ async def run(self, prompt: str) -> TurnResult: async def test_handles_in_models_round_trip_on_the_memory_store(client: Client): config = client.config() - config["plugins"] = [MemoryStreamPlugin()] + config["stream_backends"] = [MemoryStreamBackend(default=True)] config["data_converter"] = pydantic_data_converter client = Client(**config) async with new_worker(client, TurnWorkflow, activities=[call_model]) as worker: @@ -68,7 +67,7 @@ async def test_handles_in_models_round_trip_on_the_memory_store(client: Client): assert result.stream.workflow_id == handle.id assert result.stream.first_execution_run_id == handle.first_execution_run_id assert result.stream.backend == "memory" - stream = streams.StreamClient(client).get_handle(Events, id=result.stream.id) + stream = client.stream(Events, id=result.stream.id) items = [item async for item in stream.subscribe()] assert [i.data for i in items if isinstance(i, streams.Record)] == ["hi", "there"] assert isinstance(items[-1], streams.Closed) and items[-1].result is not None diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 8a0c5c821d..7fb99a9a02 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -14,14 +14,13 @@ from temporalio.api.common.v1 import Payload from temporalio.client import Client, Interceptor, WorkflowFailureError, WorkflowHandle from temporalio.common import RawValue, RetryPolicy, streams -from temporalio.common.streams import StreamPlugin -from temporalio.common.streams import workflow as stream_workflow -from temporalio.common.streams.redis import ( - RedisStreamPlugin, - _ExtensionStreamPlugin, +from temporalio.common.streams._extension import ( + ExtensionStreamBackend, _WorkflowInbound, ) -from temporalio.common.streams.testing import MemoryStreamPlugin +from temporalio.contrib import stream as redis +from temporalio.contrib.stream import RedisStreamBackend +from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.converter import DataConverter, PayloadCodec from temporalio.exceptions import ApplicationError from temporalio.worker import ( @@ -58,11 +57,9 @@ class CallModelInput: def workflow_stream( - client: Client, handle: WorkflowHandle[Any, Any], name: str | None = None + handle: WorkflowHandle[Any, Any], name: str | None = None ) -> streams.ClientStream[Events]: - return streams.StreamClient(client).workflow_stream( - handle, streams.with_name(Events, name) if name else Events - ) + return handle.stream(streams.with_name(Events, name) if name else Events) async def read_all(stream: streams.ClientStream[Any]) -> list[Any]: @@ -85,7 +82,7 @@ async def call_model(input: CallModelInput) -> str: class Agent: @workflow.run async def run(self, prompt: str) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) turns = stream.topic(Events.turns) await turns.publish(Turn("model")) reply = await workflow.execute_activity( @@ -103,7 +100,7 @@ async def test_workflow_and_activity_write_one_stream(stream_client: Client): handle = await stream_client.start_workflow( Agent.run, "hi there", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) - stream = workflow_stream(stream_client, handle) + stream = workflow_stream(handle) items = await read_all(stream) assert await handle.result() == "HI THERE" data = [(item.topic, item.data) for item in records(items)] @@ -130,7 +127,7 @@ async def test_workflow_and_activity_write_one_stream(stream_client: Client): class PublishThenFailTask: @workflow.run async def run(self, key: str) -> None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) turns = stream.topic(Events.tokens) await turns.publish("before") await workflow.sleep(timedelta(milliseconds=10)) @@ -156,7 +153,7 @@ async def test_failed_workflow_task_is_withdrawn(stream_client: Client): task_queue=worker.task_queue, ) await handle.result() - items = await read_all(workflow_stream(stream_client, handle)) + items = await read_all(workflow_stream(handle)) # The retried task writes "after" again, and withdraws the copy of the failed task. written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] @@ -180,7 +177,7 @@ async def test_resume_after_a_superseded_item_keeps_its_record(stream_client: Cl task_queue=worker.task_queue, ) await handle.result() - stream = workflow_stream(stream_client, handle) + stream = workflow_stream(handle) items = [item async for item in stream.subscribe(wait_new_records=False)] # The retried task's batch has its Superseded item and its first Record in one entry. record_offsets = {r.offset for r in records(items)} @@ -223,7 +220,7 @@ async def run(self) -> bool: return same -async def test_plugin_leaves_the_workflow_random_alone(stream_client: Client): +async def test_backend_leaves_the_workflow_random_alone(stream_client: Client): async with new_worker( stream_client, RandomAroundActivity, activities=[noop] ) as worker: @@ -239,10 +236,10 @@ class WritesThenContinues: @workflow.run async def run(self, write: bool) -> str: if write: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("first run") workflow.continue_as_new(False) - await stream_workflow.stream_handle(Events).close(result="second run") + await workflow.stream_handle(Events).close(result="second run") return "second run" @@ -255,7 +252,7 @@ async def test_later_run_closes_a_stream_it_did_not_write(stream_client: Client) task_queue=worker.task_queue, ) assert await handle.result() == "second run" - items = await read_all(workflow_stream(stream_client, handle)) + items = await read_all(workflow_stream(handle)) assert [i.data for i in records(items)] == ["first run"] closed = items[-1] assert isinstance(closed, streams.Closed) and closed.result == "second run" @@ -265,7 +262,7 @@ async def test_later_run_closes_a_stream_it_did_not_write(stream_client: Client) class CancelledBlock: @workflow.run async def run(self) -> None: - async with await stream_workflow.create_stream( + async with await workflow.create_stream( streams.with_name(Events, "turn") ) as turn: await turn.topic(Events.tokens).publish("started") @@ -277,7 +274,7 @@ async def test_cancelled_block_closes_with_a_failure(stream_client: Client): handle = await stream_client.start_workflow( CancelledBlock.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) - turn = workflow_stream(stream_client, handle, "turn") + turn = workflow_stream(handle, "turn") reader = turn.subscribe(create_wait=timedelta(seconds=20)) assert isinstance(await anext(reader), streams.Record) await reader.aclose() @@ -298,9 +295,9 @@ class Plain: class PlainWriter: @workflow.run async def run(self) -> str: - stream = await stream_workflow.create_stream(Plain) + stream = await workflow.create_stream(Plain) await stream.topic(Plain.t).publish("plain") - async with await stream_workflow.create_stream( + async with await workflow.create_stream( streams.with_name(Plain, "scoped") ) as scoped: await scoped.topic(Plain.t).publish("scoped") @@ -322,11 +319,8 @@ async def test_temporal_only_passthrough_keeps_records(stream_client: Client): PlainWriter.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) await handle.result() - plain = streams.StreamClient(stream_client) - items = await read_all(plain.workflow_stream(handle, Plain)) - scoped = await read_all( - plain.workflow_stream(handle, streams.with_name(Plain, "scoped")) - ) + items = await read_all(handle.stream(Plain)) + scoped = await read_all(handle.stream(streams.with_name(Plain, "scoped"))) assert [i.data for i in records(items)] == ["plain"] assert isinstance(items[-1], streams.Closed) and items[-1].result == "done" assert [i.data for i in records(scoped)] == ["scoped"] @@ -342,7 +336,7 @@ async def local_tokens(input: CallModelInput) -> None: class LocalWriter: @workflow.run async def run(self) -> None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await workflow.execute_local_activity( local_tokens, CallModelInput("0 1", streams.ActivityStream.from_handle(stream)), @@ -357,7 +351,7 @@ async def test_local_activity_writes_with_the_task_epoch(stream_client: Client): LocalWriter.run, id=f"wf-{uuid.uuid4()}", task_queue=w.task_queue ) await handle.result() - items = await read_all(workflow_stream(stream_client, handle)) + items = await read_all(workflow_stream(handle)) written = records(items) assert [record.data for record in written] == ["0", "1"] producer = written[0].producer @@ -371,7 +365,7 @@ async def test_local_activity_writes_with_the_task_epoch(stream_client: Client): class LocalThenFailTask: @workflow.run async def run(self, key: str) -> None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) first = attempts.setdefault(key, 0) == 0 await workflow.execute_local_activity( local_tokens, @@ -400,7 +394,7 @@ async def test_rerun_local_activity_withdraws_its_failed_task(stream_client: Cli task_queue=worker.task_queue, ) await handle.result() - items = await read_all(workflow_stream(stream_client, handle)) + items = await read_all(workflow_stream(handle)) written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] visible = [r for r in written if not any(i.withdraws(r) for i in superseded)] @@ -419,7 +413,7 @@ async def test_rerun_local_activity_withdraws_its_failed_task(stream_client: Cli class LocalOnlyInFailedTask: @workflow.run async def run(self, key: str) -> None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) if attempts.setdefault(key, 0) == 0: attempts[key] = 1 await workflow.execute_local_activity( @@ -448,7 +442,7 @@ async def test_later_write_withdraws_a_skipped_local_activity(stream_client: Cli task_queue=worker.task_queue, ) await handle.result() - items = await read_all(workflow_stream(stream_client, handle)) + items = await read_all(workflow_stream(handle)) written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] visible = [r for r in written if not any(s.withdraws(r) for s in superseded)] @@ -460,7 +454,7 @@ async def test_later_write_withdraws_a_skipped_local_activity(stream_client: Cli class FailingRun: @workflow.run async def run(self) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) info = workflow.info() await stream.topic(Events.tokens).publish(f"run-{info.attempt}") if info.attempt == 1: @@ -480,7 +474,7 @@ async def test_retry_run_withdraws_the_failed_run(stream_client: Client): ), ) assert await handle.result() == "done" - items = await read_all(workflow_stream(stream_client, handle)) + items = await read_all(workflow_stream(handle)) written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] assert [record.data for record in written] == ["run-1", "run-2"] @@ -522,13 +516,12 @@ async def test_records_go_through_the_payload_codec(stream_client: Client): Agent.run, "a b", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) await handle.result() - stream = workflow_stream(codec_client, handle) + stream = workflow_stream(handle) decoded = await read_all(stream) # A Stream with no definition reads the raw Payloads. raw = [ item - async for item in streams.StreamClient(codec_client) - .get_handle(Plain, id=stream.id) + async for item in codec_client.stream(Plain, id=stream.id) .topic("tokens", type=RawValue) .subscribe(wait_new_records=False) ] @@ -548,10 +541,10 @@ async def test_client_close_goes_through_the_payload_codec(stream_client: Client config["data_converter"] = dataclasses.replace( DataConverter.default, payload_codec=_Rot13Codec() ) - stream_client_ = streams.StreamClient(Client(**config)) - done = await stream_client_.create(Events, id=f"done-{uuid.uuid4()}") + codec_client = Client(**config) + done = await redis.create(codec_client, Events, id=f"done-{uuid.uuid4()}") await done.close(result=Turn("done")) - failed = await stream_client_.create(Events, id=f"failed-{uuid.uuid4()}") + failed = await redis.create(codec_client, Events, id=f"failed-{uuid.uuid4()}") await failed.close(failure=ApplicationError("bad turn")) [closed] = [item async for item in done.subscribe()] assert isinstance(closed, streams.Closed) @@ -567,10 +560,10 @@ def __init__(self) -> None: @workflow.run async def run(self) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) turns = stream.topic(Events.tokens) await turns.publish("before restart") - async with await stream_workflow.create_stream( + async with await workflow.create_stream( streams.with_name(Events, "scoped") ) as scoped: await scoped.topic(Events.tokens).publish("scoped") @@ -590,7 +583,7 @@ async def test_new_worker_replays_and_writes_each_record_once(stream_client: Cli handle = await stream_client.start_workflow( WaitsForSignal.run, id=f"wf-{uuid.uuid4()}", task_queue=task_queue ) - first = workflow_stream(stream_client, handle) + first = workflow_stream(handle) reader = first.subscribe(create_wait=timedelta(seconds=20)) assert isinstance(await anext(reader), streams.Record) await reader.aclose() @@ -600,14 +593,14 @@ async def test_new_worker_replays_and_writes_each_record_once(stream_client: Cli await handle.signal(WaitsForSignal.proceed) assert await handle.result() == "done" items = await read_all(first) - scoped = await read_all(workflow_stream(stream_client, handle, "scoped")) + scoped = await read_all(workflow_stream(handle, "scoped")) assert [i.data for i in records(items)] == ["before restart", "after restart"] assert isinstance(items[-1], streams.Closed) assert [i.data for i in records(scoped)] == ["scoped"] assert [type(i) for i in scoped].count(streams.Closed) == 1 -async def test_plugin_replays_a_history_made_without_it( +async def test_backend_replays_a_history_made_without_it( client: Client, stream_client: Client ): async with new_worker(client, RandomAroundActivity, activities=[noop]) as worker: @@ -620,25 +613,18 @@ async def test_plugin_replays_a_history_made_without_it( history = await handle.fetch_history() replayer = Replayer( workflows=[RandomAroundActivity], - plugins=[ - p for p in stream_client.config()["plugins"] if isinstance(p, StreamPlugin) - ], + stream_backends=stream_client.config()["stream_backends"], ) await replayer.replay_workflow(history) -async def test_plugin_shutdown_after_client_writes(stream_client: Client): - [plugin] = [ - p - for p in stream_client.config()["plugins"] - if isinstance(p, _ExtensionStreamPlugin) - ] - stream = await streams.StreamClient(stream_client).create( - Events, id=f"shutdown-{uuid.uuid4()}" - ) +async def test_backend_shutdown_after_client_writes(stream_client: Client): + [backend] = stream_client.config()["stream_backends"] + assert isinstance(backend, ExtensionStreamBackend) + stream = await redis.create(stream_client, Events, id=f"shutdown-{uuid.uuid4()}") async with streams.BufferedPublisher(stream.topic(Events.tokens)) as tokens: tokens.publish("buffered") - await plugin.shutdown() + await backend.shutdown() written = [ item.data async for item in stream.subscribe(wait_new_records=False) @@ -651,7 +637,7 @@ async def test_plugin_shutdown_after_client_writes(stream_client: Client): class WaitsForever: @workflow.run async def run(self) -> None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("waiting") await workflow.wait_condition(lambda: False) @@ -696,9 +682,7 @@ async def test_eviction_prints_nothing( handle = await stream_client.start_workflow( WaitsForever.run, id=f"wf-{uuid.uuid4()}", task_queue=w.task_queue ) - reader = workflow_stream(stream_client, handle).subscribe( - create_wait=timedelta(seconds=20) - ) + reader = workflow_stream(handle).subscribe(create_wait=timedelta(seconds=20)) assert isinstance(await anext(reader), streams.Record) await reader.aclose() await handle.terminate() @@ -712,7 +696,7 @@ class AlwaysFails: @workflow.run async def run(self, non_retryable: bool) -> None: attempt = workflow.info().attempt - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish(f"run-{attempt}") raise ApplicationError(f"attempt {attempt} fails", non_retryable=non_retryable) @@ -733,7 +717,7 @@ async def test_final_failure_under_a_retry_policy( ) with pytest.raises(WorkflowFailureError): await handle.result() - stream = workflow_stream(stream_client, handle) + stream = workflow_stream(handle) items = [item async for item in stream.subscribe(wait_new_records=False)] written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] @@ -753,9 +737,7 @@ async def test_final_failure_under_a_retry_policy( async def test_reader_stops_on_its_own_timer_and_resumes(stream_client: Client): - stream = await streams.StreamClient(stream_client).create( - Events, id=f"timer-{uuid.uuid4()}" - ) + stream = await redis.create(stream_client, Events, id=f"timer-{uuid.uuid4()}") first = await stream.topic(Events.tokens).publish("one") seen: list[streams.Record[Any]] = [] @@ -780,22 +762,24 @@ async def read_until_quiet() -> None: async def test_two_backends_on_one_client(client: Client, redis_url: str): - # A client may carry two Stream plugins, such as during a move from one backend to + # A client may carry two Stream backends, such as during a move from one backend to # another. Each handle carries its backend, so a call goes to the backend that has it. config = client.config() - config["plugins"] = [ - RedisStreamPlugin(redis_url, key_prefix=f"t-{uuid.uuid4()}", default=True), - MemoryStreamPlugin(), + config["stream_backends"] = [ + RedisStreamBackend(redis_url, key_prefix=f"t-{uuid.uuid4()}", default=True), + MemoryStreamBackend(), ] - stream_client = streams.StreamClient(Client(**config)) + stream_client = Client(**config) stream_id = f"two-{uuid.uuid4()}" - on_redis = await stream_client.create(Events, id=stream_id) - on_memory = await stream_client.create(Events, id=stream_id, backend="memory") + on_redis = await redis.create(stream_client, Events, id=stream_id) + on_memory = await redis.create( + stream_client, Events, id=stream_id, backend="memory" + ) assert (on_redis.backend, on_memory.backend) == ("redis", "memory") await on_redis.topic(Events.tokens).publish("redis") await on_memory.topic(Events.tokens).publish("memory") for backend, expected in (("redis", "redis"), ("memory", "memory")): - again = stream_client.get_handle(Events, id=stream_id, backend=backend) + again = stream_client.stream(Events, id=stream_id, backend=backend) data = [ item.data async for item in again.subscribe(wait_new_records=False) @@ -808,8 +792,8 @@ async def test_two_backends_on_one_client(client: Client, redis_url: str): class _FailsTheLastTaskOnce(Interceptor, WorkerInterceptor): - # On the client before the Stream plugin, so it is the outer Workflow interceptor and - # fails the task after the plugin drained the run's writes. + # On the client before the Stream backend's interceptor, so it is the outer Workflow + # interceptor and fails the task after the backend drained the run's writes. def workflow_interceptor_class( self, input: WorkflowInterceptorClassInput ) -> type[WorkflowInboundInterceptor]: @@ -830,7 +814,7 @@ async def execute_workflow(self, input: ExecuteWorkflowInput) -> Any: class PublishesAndReturns: @workflow.run async def run(self) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("token") return "done" @@ -854,7 +838,7 @@ async def test_failed_last_task_is_withdrawn_by_its_retry( task_queue=worker.task_queue, ) assert await handle.result() == "done" - stream = workflow_stream(client, handle) + stream = workflow_stream(handle) items = [item async for item in stream.subscribe(wait_new_records=False)] written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] @@ -865,7 +849,7 @@ async def test_failed_last_task_is_withdrawn_by_its_retry( @workflow.defn class WritesToStream: @workflow.run - async def run(self, stream: stream_workflow.WorkflowStream[Events]) -> str: + async def run(self, stream: streams.WorkflowStream[Events]) -> str: try: await stream.topic(Events.tokens).publish("from workflow") except streams.StreamError as err: @@ -878,7 +862,7 @@ async def test_write_to_a_missing_stream_raises_at_the_await(stream_client: Clie assert ( await stream_client.execute_workflow( WritesToStream.run, - stream_workflow.WorkflowStream(id=f"missing-{uuid.uuid4()}"), + streams.WorkflowStream(id=f"missing-{uuid.uuid4()}"), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -887,15 +871,13 @@ async def test_write_to_a_missing_stream_raises_at_the_await(stream_client: Clie async def test_write_to_a_closed_stream_raises_at_the_await(stream_client: Client): - stream = await streams.StreamClient(stream_client).create( - Events, id=f"closed-{uuid.uuid4()}" - ) + stream = await redis.create(stream_client, Events, id=f"closed-{uuid.uuid4()}") await stream.close() async with new_worker(stream_client, WritesToStream) as worker: assert ( await stream_client.execute_workflow( WritesToStream.run, - stream_workflow.WorkflowStream.from_handle(stream), + streams.WorkflowStream.from_handle(stream), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -918,7 +900,7 @@ async def unknown_topic(input: CallModelInput) -> str: class UnknownTopic: @workflow.run async def run(self) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) return await workflow.execute_activity( unknown_topic, CallModelInput("", streams.ActivityStream.from_handle(stream)), @@ -939,9 +921,9 @@ async def test_activity_refuses_a_topic_outside_the_definition(stream_client: Cl @workflow.defn class ContinuesThenRetries: @workflow.run - async def run(self, stream: stream_workflow.WorkflowStream[Events] | None) -> str: + async def run(self, stream: streams.WorkflowStream[Events] | None) -> str: if stream is None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("first") workflow.continue_as_new(stream) info = workflow.info() @@ -964,7 +946,7 @@ async def test_retry_of_a_continued_run_writes_the_same_stream(stream_client: Cl ), ) assert await handle.result() == "done" - items = await read_all(workflow_stream(stream_client, handle)) + items = await read_all(workflow_stream(handle)) written = records(items) assert [r.data for r in written] == ["first", "attempt-1", "attempt-2"] superseded = [item for item in items if isinstance(item, streams.Superseded)] diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 8c18e6beab..af9f022f6d 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -10,12 +10,13 @@ from temporalio import workflow from temporalio.client import Client from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow -from temporalio.common.streams._backend import _ExtensionStreamBackend -from temporalio.common.streams.redis import RedisStreamPlugin -from temporalio.common.streams.testing import MemoryStreamPlugin +from temporalio.contrib import stream as redis +from temporalio.contrib.stream import RedisStreamBackend +from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.converter import DataConverter -from tests.contrib.streams.fake_extension import FakeExtension +from temporalio.testing import WorkflowEnvironment +from temporalio.worker import Replayer +from tests.contrib.streams.fake_extension import fake_client from tests.helpers import new_worker @@ -31,45 +32,35 @@ class Events: tokens = streams.topic(str) -def fake_client(client: Client) -> tuple[Client, FakeExtension]: - extension = FakeExtension() - plugin = streams.StreamPlugin( - "fake-streams", - backend_name="fake", - default=False, - stream_backend=_ExtensionStreamBackend(extension), - ) - config = client.config() - config["plugins"] = [plugin] - return Client(**config), extension - - async def test_unknown_backend_raises_before_any_call(client: Client): client, extension = fake_client(client) - stream_client = streams.StreamClient(client) + with pytest.raises( + streams.StreamConfigError, match="no Redis Stream backend redis" + ): + await redis.create(client, Events, id="chat-1", backend="redis") with pytest.raises(streams.StreamConfigError, match="It has fake"): - await stream_client.create(Events, id="chat-1", backend="redis") - with pytest.raises(streams.StreamConfigError): - stream_client.get_handle(Events, id="chat-1", backend="redis") + client.stream(Events, id="chat-1", backend="redis") assert not extension.streams and not extension.namespaces - stream = await stream_client.create(Events, id="chat-1") + stream = await redis.create(client, Events, id="chat-1") assert stream.backend == "fake" -async def test_client_without_plugin_has_no_transport(client: Client): - with pytest.raises(streams.StreamConfigError): - streams.StreamClient(client).get_handle(Events, id="chat-1") +async def test_client_without_backend_has_no_transport(client: Client): + with pytest.raises(streams.StreamConfigError, match="no Stream backend"): + client.stream(Events, id="chat-1") + with pytest.raises(streams.StreamConfigError, match="no Redis Stream backend"): + await redis.create(client, Events, id="chat-1") async def test_decoded_client_stream_has_no_client(client: Client): client, _ = fake_client(client) - stream = await streams.StreamClient(client).create(Events, id="chat-2") + stream = await redis.create(client, Events, id="chat-2") converter = DataConverter.default.payload_converter decoded = converter.from_payload( converter.to_payload(stream), streams.ClientStream[Events] ) assert decoded == stream - with pytest.raises(streams.StreamConfigError, match="get_handle"): + with pytest.raises(streams.StreamConfigError, match="Client.stream"): await decoded.get_offset() @@ -93,28 +84,29 @@ class Description: return Description() setattr(handle, "describe", describe) - stream_client = streams.StreamClient(client) - await stream_client.create(Events, id="agent-1:first:n") - stream = stream_client.workflow_stream(handle, streams.with_name(Events, "n")) + await redis.create(client, Events, id="agent-1:first:n") + stream = handle.stream(streams.with_name(Events, "n")) with pytest.raises(RuntimeError, match="not known yet"): stream.id assert await stream.get_offset() == 0 assert stream.id == "agent-1:first:n" assert stream.first_execution_run_id == "first" assert await stream.get_offset() == 0 + # The handle keeps the first run id, so its other Streams make no call. + other = handle.stream(streams.with_name(Events, "n")) + assert await other.get_offset() == 0 assert calls == [1] @workflow.defn class WriteToStream: @workflow.run - async def run(self, stream: stream_workflow.WorkflowStream[Events]) -> None: + async def run(self, stream: streams.WorkflowStream[Events]) -> None: await stream.topic(Events.turns).publish(Turn("workflow")) async def test_client_stream_takes_workflow_writes(stream_client: Client): - client = streams.StreamClient(stream_client) - stream = await client.create(Events, id=f"chat-{uuid.uuid4()}") + stream = await redis.create(stream_client, Events, id=f"chat-{uuid.uuid4()}") assert await stream.get_offset() == 0 first = await stream.topic(Events.turns).publish(Turn("client")) last = await stream.topic(Events.tokens).publish("a", "b") @@ -122,7 +114,7 @@ async def test_client_stream_takes_workflow_writes(stream_client: Client): async with new_worker(stream_client, WriteToStream) as worker: await stream_client.execute_workflow( WriteToStream.run, - stream_workflow.WorkflowStream.from_handle(stream), + streams.WorkflowStream.from_handle(stream), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -133,7 +125,7 @@ async def test_client_stream_takes_workflow_writes(stream_client: Client): assert isinstance(items[-1], streams.Closed) and items[-1].result == "bye" # Another handle of the same Stream reads it, from an offset and by Topic. - again = client.get_handle(Events, id=stream.id) + again = stream_client.stream(Events, id=stream.id) resumed = [ item.data async for item in again.subscribe(Events.tokens, from_offset=first + 1) @@ -159,7 +151,7 @@ class Agent: @workflow.run async def run(self, prompt: str) -> str: await workflow.sleep(timedelta(milliseconds=200)) - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish(*prompt.split()) await stream.close(result=prompt.upper()) return prompt.upper() @@ -170,7 +162,7 @@ async def test_client_reads_a_workflow_stream(stream_client: Client): handle = await stream_client.start_workflow( Agent.run, "hi there", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) - stream = streams.StreamClient(stream_client).workflow_stream(handle, Events) + stream = handle.stream(Events) # The Workflow creates its Stream later, so the read waits for the create. items = [ item async for item in stream.subscribe(create_wait=timedelta(seconds=20)) @@ -186,18 +178,19 @@ async def test_client_reads_a_workflow_stream(stream_client: Client): async def test_async_with_closes_a_client_stream(stream_client: Client): - client = streams.StreamClient(stream_client) - async with client.create(Events, id=f"block-{uuid.uuid4()}") as done: + async with redis.create(stream_client, Events, id=f"block-{uuid.uuid4()}") as done: await done.topic(Events.tokens).publish("a") - failed = client.get_handle(Events, id=f"block-{uuid.uuid4()}") + failed = stream_client.stream(Events, id=f"block-{uuid.uuid4()}") with pytest.raises(ValueError): - async with client.create(Events, id=failed.id): + async with redis.create(stream_client, Events, id=failed.id): raise ValueError("turn failed") - async with await client.create(Events, id=f"block-{uuid.uuid4()}") as early: + async with await redis.create( + stream_client, Events, id=f"block-{uuid.uuid4()}" + ) as early: await early.close(result="early") - cancelled = client.get_handle(Events, id=f"block-{uuid.uuid4()}") + cancelled = stream_client.stream(Events, id=f"block-{uuid.uuid4()}") with pytest.raises(asyncio.CancelledError): - async with client.create(Events, id=cancelled.id): + async with redis.create(stream_client, Events, id=cancelled.id): raise asyncio.CancelledError() for stream, result, failure, status in ( (done, None, None, streams.CloseStatus.COMPLETED), @@ -214,9 +207,7 @@ async def test_async_with_closes_a_client_stream(stream_client: Client): async def test_next_offset_resumes_after_the_last_item(stream_client: Client): - stream = await streams.StreamClient(stream_client).create( - Events, id=f"resume-{uuid.uuid4()}" - ) + stream = await redis.create(stream_client, Events, id=f"resume-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("a") turn = await stream.topic(Events.turns).publish(Turn("model")) tokens = stream.topic(Events.tokens).subscribe(wait_new_records=False) @@ -238,9 +229,7 @@ async def test_next_offset_resumes_after_the_last_item(stream_client: Client): async def test_last_reads_the_newest_records_then_follows(stream_client: Client): - stream = await streams.StreamClient(stream_client).create( - Events, id=f"last-{uuid.uuid4()}" - ) + stream = await redis.create(stream_client, Events, id=f"last-{uuid.uuid4()}") tokens = stream.topic(Events.tokens) await tokens.publish("a", "b", "c") await stream.topic(Events.turns).publish(Turn("model")) @@ -265,9 +254,7 @@ async def test_last_reads_the_newest_records_then_follows(stream_client: Client) async def test_read_type_decodes_topics_outside_the_definition(stream_client: Client): - stream = await streams.StreamClient(stream_client).create( - Events, id=f"typed-{uuid.uuid4()}" - ) + stream = await redis.create(stream_client, Events, id=f"typed-{uuid.uuid4()}") await stream.topic("extra", type=Turn).publish(Turn("extra")) await stream.topic(Events.tokens).publish("a") @@ -280,9 +267,7 @@ async def read(**options: Any) -> list[object]: async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: Client): - stream = await streams.StreamClient(stream_client).create( - Events, id=f"buffered-{uuid.uuid4()}" - ) + stream = await redis.create(stream_client, Events, id=f"buffered-{uuid.uuid4()}") async with streams.BufferedPublisher(stream) as publisher: publisher.publish(Events.turns, Turn("user")) publisher.publish(Events.tokens, "a", "b") @@ -303,9 +288,7 @@ async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: C async def test_named_producer_fences_its_earlier_epoch(stream_client: Client): - stream = await streams.StreamClient(stream_client).create( - Events, id=f"named-{uuid.uuid4()}" - ) + stream = await redis.create(stream_client, Events, id=f"named-{uuid.uuid4()}") # A server that restarts writes as one name with a higher epoch, so the zombie it # replaces can't write after it. before = stream.producer("ingest", epoch=1) @@ -323,11 +306,10 @@ async def test_named_producer_fences_its_earlier_epoch(stream_client: Client): async def test_unresolved_workflow_stream_compares_and_prints(client: Client): client, _ = fake_client(client) - stream_client = streams.StreamClient(client) handle = client.get_workflow_handle("agent-2") - first = stream_client.workflow_stream(handle, Events) - same = stream_client.workflow_stream(handle, Events) - other = stream_client.workflow_stream(handle, streams.with_name(Events, "other")) + first = handle.stream(Events) + same = handle.stream(Events) + other = handle.stream(streams.with_name(Events, "other")) key = hash(first) assert first == same and first != other describes = [] @@ -341,7 +323,7 @@ async def describe(**_: Any) -> Any: ) setattr(handle, "describe", describe) - await stream_client.create(Events, id="agent-2:run-2:events") + await redis.create(client, Events, id="agent-2:run-2:events") # Calls that start together share one Describe call. assert await asyncio.gather(first.get_offset(), first.get_offset()) == [0, 0] assert describes == [1] @@ -349,42 +331,100 @@ async def describe(**_: Any) -> Any: assert hash(first) == key -async def test_default_plugin_is_the_default_backend(client: Client): +async def test_registration_rules(client: Client): config = client.config() - config["plugins"] = [ - MemoryStreamPlugin(), - MemoryStreamPlugin(name="b", default=True), + config["stream_backends"] = [ + MemoryStreamBackend(), + MemoryStreamBackend(name="b", default=True), ] - stream = await streams.StreamClient(Client(**config)).create(Events, id="d-1") - assert stream.backend == "b" + two = Client(**config) + assert (await redis.create(two, Events, id="d-1")).backend == "b" + assert two.stream(Events, id="d-1").backend == "b" # The default is part of Workflow code, so it must not follow the order of the list. - for plugins in ( - [MemoryStreamPlugin(), MemoryStreamPlugin(name="b")], - [MemoryStreamPlugin(default=True), MemoryStreamPlugin(name="b", default=True)], + for backends in ( + [MemoryStreamBackend(), MemoryStreamBackend(name="b")], + [ + MemoryStreamBackend(default=True), + MemoryStreamBackend(name="b", default=True), + ], ): - config["plugins"] = plugins + config["stream_backends"] = backends with pytest.raises(streams.StreamConfigError, match="exactly one"): Client(**config) - config["plugins"] = [MemoryStreamPlugin()] - with pytest.raises(streams.StreamConfigError, match="Worker has"): - new_worker(Client(**config), Agent, plugins=[RedisStreamPlugin("redis://-")]) + config["stream_backends"] = [MemoryStreamBackend(), MemoryStreamBackend()] + with pytest.raises(streams.StreamConfigError, match="two Stream backends"): + Client(**config) + # For now a Worker serves one backend. + config["stream_backends"] = [ + MemoryStreamBackend(), + MemoryStreamBackend(name="b", default=True), + ] + with pytest.raises(streams.StreamConfigError, match="one backend"): + new_worker(Client(**config), Agent) + + +async def test_backend_is_not_a_plugin(client: Client): + config = client.config() + config["plugins"] = [MemoryStreamBackend()] + with pytest.raises(streams.StreamConfigError, match="not a plugin"): + Client(**config) + with pytest.raises(streams.StreamConfigError, match="not a plugin"): + new_worker(client, Agent, plugins=[RedisStreamBackend("redis://-")]) + with pytest.raises(streams.StreamConfigError, match="not a plugin"): + Replayer(workflows=[Agent], plugins=[MemoryStreamBackend()]) -async def test_handle_looks_in_each_backend_during_a_move(client: Client): +async def test_default_is_native_with_no_default_backend(client: Client): + config = client.config() + config["stream_backends"] = [MemoryStreamBackend()] + one = Client(**config) + with pytest.raises(streams.StreamConfigError, match="default is native"): + one.stream(Events, id="d-2") + with pytest.raises(streams.StreamConfigError, match="default is native"): + one.get_workflow_handle("agent-3").stream(Events) + assert one.stream(Events, id="d-2", backend="memory").backend == "memory" + # The Redis create picks the client's only Redis backend, and memory stands in for it. + assert (await redis.create(one, Events, id="d-2")).backend == "memory" + + +async def test_stream_takes_the_handle_backend_or_the_default(client: Client): config = client.config() - config["plugins"] = [ - MemoryStreamPlugin(name="a", default=True), - MemoryStreamPlugin(name="b"), + config["stream_backends"] = [ + MemoryStreamBackend(name="a", default=True), + MemoryStreamBackend(name="b"), ] - stream_client = streams.StreamClient(Client(**config)) - created = await stream_client.create(Events, id="moving", backend="b") + two = Client(**config) + created = await redis.create(two, Events, id="moving", backend="b") await created.topic(Events.tokens).publish("on b") - moving = stream_client.get_handle(Events, id="moving", backend=("a", "b")) - assert await moving.get_offset() > 0 - assert moving.backend == "b" - nowhere = stream_client.get_handle(Events, id="nowhere", backend=("a", "b")) - with pytest.raises(streams.StreamNotFoundError): - await nowhere.get_offset() - # A handle with no backend name is ambiguous when the client has two backends. - with pytest.raises(streams.StreamConfigError): - stream_client.get_handle(Events, stream=streams.ClientStream(id="moving")) + assert await two.stream(Events, stream=created).get_offset() > 0 + # A handle with no backend name is of the default backend. + assert two.stream(Events, stream=streams.ClientStream(id="moving")).backend == "a" + with pytest.raises(ValueError): + two.stream(Events, stream=created, backend="a") + with pytest.raises(TypeError): + two.stream(Events) + + +@workflow.defn +class CreatesAndCloses: + @workflow.run + async def run(self) -> None: + stream = await workflow.create_stream(Events) + await stream.topic(Events.tokens).publish("from the environment") + await stream.close() + + +async def test_workflow_environment_takes_stream_backends(): + async with await WorkflowEnvironment.start_local( + stream_backends=[MemoryStreamBackend(default=True)] + ) as env: + async with new_worker(env.client, CreatesAndCloses) as worker: + handle = await env.client.start_workflow( + CreatesAndCloses.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await handle.result() + items = [item async for item in handle.stream(Events).subscribe()] + data = [item.data for item in items if isinstance(item, streams.Record)] + assert data == ["from the environment"] diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 92a6ced70f..baaeafee92 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -18,9 +18,10 @@ WorkflowHandle, ) from temporalio.common import streams -from temporalio.common.streams import workflow as stream_workflow -from temporalio.common.streams._backend import _ExtensionStreamBackend +from temporalio.common.streams._backend import _ExtensionCalls from temporalio.common.streams._defn import definition_of +from temporalio.contrib import stream as redis +from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.converter import DataConverter, PayloadCodec from temporalio.exceptions import ApplicationError from temporalio.worker import Replayer @@ -39,12 +40,8 @@ class Events: tokens = streams.topic(str) -async def read( - client: Client, handle: WorkflowHandle[Any, Any], name: str = "events" -) -> list[object]: - stream = streams.StreamClient(client).workflow_stream( - handle, streams.with_name(Events, name) - ) +async def read(handle: WorkflowHandle[Any, Any], name: str = "events") -> list[object]: + stream = handle.stream(streams.with_name(Events, name)) return [item async for item in stream.subscribe(wait_new_records=False)] @@ -56,7 +53,7 @@ def data(items: Sequence[object]) -> list[object]: class Agent: @workflow.run async def run(self, prompt: str) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) turns = stream.topic(Events.turns) await turns.publish(Turn("user", prompt), Turn("model")) await stream.topic("tokens").publish(*prompt.split()) @@ -71,7 +68,7 @@ async def test_create_publish_and_close(stream_client: Client): Agent.run, "hi there", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await handle.result() == "HI THERE" - items = await read(stream_client, handle) + items = await read(handle) assert data(items) == [Turn("user", "hi there"), Turn("model"), "hi", "there"] closed = items[-1] assert isinstance(closed, streams.Closed) and closed.result == "HI THERE" @@ -83,7 +80,7 @@ class Turns: async def run(self, fail_at: int) -> None: for n in range(2): try: - async with stream_workflow.create_stream( + async with workflow.create_stream( streams.with_name(Events, f"turn-{n}") ) as turn: await turn.topic(Events.tokens).publish(f"token-{n}") @@ -100,8 +97,8 @@ async def test_async_with_closes_the_stream(stream_client: Client): Turns.run, 1, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) await handle.result() - first = await read(stream_client, handle, "turn-0") - second = await read(stream_client, handle, "turn-1") + first = await read(handle, "turn-0") + second = await read(handle, "turn-1") assert data(first) == ["token-0"] assert isinstance(first[-1], streams.Closed) and first[-1].failure is None assert data(second) == ["token-1"] @@ -112,13 +109,13 @@ async def test_async_with_closes_the_stream(stream_client: Client): @workflow.defn class Continuing: @workflow.run - async def run(self, stream: stream_workflow.WorkflowStream[Events] | None) -> str: + async def run(self, stream: streams.WorkflowStream[Events] | None) -> str: if stream is None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("run 1") workflow.continue_as_new(stream) assert stream.definition is Events - assert stream == stream_workflow.stream_handle(Events) + assert stream == workflow.stream_handle(Events) await stream.topic(Events.tokens).publish("run 2") await stream.close(result="done") return "done" @@ -130,13 +127,13 @@ class RenamedAcrossRuns: async def run(self, first: bool) -> list[str]: renamed = streams.with_name(Events, "turn-1") if first: - turn = await stream_workflow.create_stream(renamed) + turn = await workflow.create_stream(renamed) await turn.topic(Events.tokens).publish("run 1") workflow.continue_as_new(False) - turn = stream_workflow.stream_handle(renamed) + turn = workflow.stream_handle(renamed) await turn.topic(Events.tokens).publish("run 2") await turn.close() - return [stream_workflow.stream_handle(Events).id, turn.id] + return [workflow.stream_handle(Events).id, turn.id] async def test_renamed_definition_is_another_stream(stream_client: Client): @@ -148,7 +145,7 @@ async def test_renamed_definition_is_another_stream(stream_client: Client): task_queue=worker.task_queue, ) plain_id, turn_id = await handle.result() - turn = await read(stream_client, handle, "turn-1") + turn = await read(handle, "turn-1") assert plain_id.endswith(":events") and turn_id.endswith(":turn-1") assert data(turn) == ["run 1", "run 2"] @@ -166,7 +163,7 @@ async def test_continue_as_new_passes_the_stream(stream_client: Client): Continuing.run, None, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await handle.result() == "done" - items = await read(stream_client, handle) + items = await read(handle) assert data(items) == ["run 1", "run 2"] closed = [item for item in items if isinstance(item, streams.Closed)] assert len(closed) == 1 and closed[0].result == "done" @@ -175,12 +172,12 @@ async def test_continue_as_new_passes_the_stream(stream_client: Client): @workflow.defn class Mistakes: def __init__(self) -> None: - self._stream: stream_workflow.WorkflowStream[Events] | None = None + self._stream: streams.WorkflowStream[Events] | None = None @workflow.run async def run(self) -> list[str]: errors = [] - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) self._stream = stream try: stream.topic("unknown") @@ -192,7 +189,7 @@ async def run(self) -> list[str]: except streams.StreamClosedError as err: errors.append(err.reason) try: - await stream_workflow.create_stream(Events) + await workflow.create_stream(Events) except streams.StreamClosedError as err: errors.append(err.reason) return errors @@ -220,14 +217,14 @@ async def test_replay_reproducible_errors_raise_in_workflow_code( "stream_closed", ] assert await handle.query(Mistakes.publish_in_query) == "ReadOnlyContextError" - items = await read(stream_client, handle) + items = await read(handle) assert isinstance(items[-1], streams.Closed) and items[-1].result == "early" @workflow.defn class ClosesAReceivedStream: @workflow.run - async def run(self, stream: stream_workflow.WorkflowStream[Events]) -> str: + async def run(self, stream: streams.WorkflowStream[Events]) -> str: await stream.topic(Events.tokens).publish("from workflow") try: await stream.close(result="ended by a writer") @@ -237,14 +234,12 @@ async def run(self, stream: stream_workflow.WorkflowStream[Events]) -> str: async def test_workflow_closes_only_its_own_streams(stream_client: Client): - stream = await streams.StreamClient(stream_client).create( - Events, id=f"owned-{uuid.uuid4()}" - ) + stream = await redis.create(stream_client, Events, id=f"owned-{uuid.uuid4()}") async with new_worker(stream_client, ClosesAReceivedStream) as worker: assert ( await stream_client.execute_workflow( ClosesAReceivedStream.run, - stream_workflow.WorkflowStream.from_handle(stream), + streams.WorkflowStream.from_handle(stream), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -259,7 +254,7 @@ async def test_workflow_closes_only_its_own_streams(stream_client: Client): class Failing: @workflow.run async def run(self) -> None: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("before") raise ApplicationError("run failed", non_retryable=True) @@ -271,7 +266,7 @@ async def test_run_end_closes_nothing(stream_client: Client): ) with pytest.raises(WorkflowFailureError): await handle.result() - items = await read(stream_client, handle) + items = await read(handle) # The Stream stays open, so a later reader still waits for a close. assert data(items) == ["before"] assert not any(isinstance(item, streams.Closed) for item in items) @@ -281,7 +276,7 @@ async def test_run_end_closes_nothing(stream_client: Client): class CreatesOn: @workflow.run async def run(self, backend: str | None) -> None: - await stream_workflow.create_stream(Events, backend=backend) + await workflow.create_stream(Events, backend=backend) async def test_worker_config_errors_fail_the_task( @@ -290,7 +285,7 @@ async def test_worker_config_errors_fail_the_task( # A Worker with no plugin, or with another backend, fails the Workflow Task, so a Worker # with the right config can take the run on. for task_client, backend, message in ( - (client, None, "No Stream plugin"), + (client, None, "No Stream backend"), (stream_client, "elsewhere", "backend elsewhere"), ): async with new_worker(task_client, CreatesOn) as worker: @@ -308,7 +303,7 @@ async def test_worker_config_errors_fail_the_task( class CronTurn: @workflow.run async def run(self) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("turn") return "done" @@ -322,7 +317,7 @@ async def test_cron_runs_share_an_open_stream(stream_client: Client): task_queue=worker.task_queue, cron_schedule="@every 1s", ) - stream = streams.StreamClient(stream_client).workflow_stream(handle, Events) + stream = handle.stream(Events) reader = stream.subscribe(create_wait=timedelta(seconds=20)) first = await reader.__anext__() second = await reader.__anext__() @@ -339,7 +334,7 @@ class SyncHandlers: @workflow.init def __init__(self, prompt: str) -> None: # The create goes first, so the signal's writes find the Stream. - self._stream = stream_workflow.create_stream_nowait(Events) + self._stream = workflow.create_stream_nowait(Events) self._prompt = prompt self._done = False @@ -372,7 +367,7 @@ async def test_sync_handlers_publish_and_close_with_no_await(stream_client: Clie await handle.signal(SyncHandlers.token, "a") assert await handle.execute_update(SyncHandlers.finish) == "stream_closed" await handle.result() - items = await read(stream_client, handle) + items = await read(handle) assert data(items) == ["a", "p"] assert isinstance(items[-1], streams.Closed) and items[-1].result == "finished" @@ -381,7 +376,7 @@ async def test_sync_handlers_publish_and_close_with_no_await(stream_client: Clie class CreatesInInit: @workflow.init def __init__(self, prompt: str) -> None: - self._stream = stream_workflow.create_stream_nowait(Events) + self._stream = workflow.create_stream_nowait(Events) self._stream.topic(Events.tokens).publish_nowait(prompt) @workflow.run @@ -400,7 +395,7 @@ async def test_init_creates_and_publishes_with_no_await(stream_client: Client): task_queue=worker.task_queue, ) assert await handle.result() == "done" - items = await read(stream_client, handle) + items = await read(handle) assert data(items) == ["hi", "HI"] assert isinstance(items[-1], streams.Closed) and items[-1].result == "done" @@ -409,7 +404,7 @@ async def test_init_creates_and_publishes_with_no_await(stream_client: Client): class Gathers: @workflow.run async def run(self) -> list[int]: - tokens = (await stream_workflow.create_stream(Events)).topic(Events.tokens) + tokens = (await workflow.create_stream(Events)).topic(Events.tokens) tokens.publish_nowait("0") # The first write starts, so the next ones wait behind it. await asyncio.sleep(0) @@ -424,7 +419,7 @@ async def test_concurrent_publishes_keep_their_order(stream_client: Client): Gathers.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) offsets = await handle.result() - items = await read(stream_client, handle) + items = await read(handle) assert data(items) == ["0", "1", "2", "3", "4"] # The four writes that waited went as one Local Activity, and each call got the offset of # its own Record. @@ -441,13 +436,10 @@ async def test_concurrent_publishes_keep_their_order(stream_client: Client): # The create, the first write and the four that waited. assert len(markers) == 3 # Replay answers from the markers and writes nothing. - plugins = [ - p - for p in stream_client.config()["plugins"] - if isinstance(p, streams.StreamPlugin) - ] - await Replayer(workflows=[Gathers], plugins=plugins).replay_workflow(history) - assert data(await read(stream_client, handle)) == ["0", "1", "2", "3", "4"] + await Replayer( + workflows=[Gathers], stream_backends=stream_client.config()["stream_backends"] + ).replay_workflow(history) + assert data(await read(handle)) == ["0", "1", "2", "3", "4"] class _RandomCodec(PayloadCodec): @@ -512,7 +504,7 @@ def with_codec(client: Client, codec: PayloadCodec) -> Client: async def test_retried_write_goes_in_once( stream_client: Client, monkeypatch: pytest.MonkeyPatch ): - original = _ExtensionStreamBackend.call_async + original = _ExtensionCalls.call_async sent: list[list[bytes]] = [] async def loses_the_first_answer( @@ -525,14 +517,14 @@ async def loses_the_first_answer( raise RuntimeError("the answer is lost") return answer - monkeypatch.setattr(_ExtensionStreamBackend, "call_async", loses_the_first_answer) + monkeypatch.setattr(_ExtensionCalls, "call_async", loses_the_first_answer) client = with_codec(stream_client, _RandomCodec()) async with new_worker(client, Gathers) as worker: handle = await client.start_workflow( Gathers.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) await handle.result() - items = await read(client, handle) + items = await read(handle) assert data(items) == ["0", "1", "2", "3", "4"] # The retry sends the bytes of the first attempt, so the Store's dedupe takes it as the # same batch, though the codec adds random bytes. @@ -543,7 +535,7 @@ async def loses_the_first_answer( class PublishesLargeValues: @workflow.run async def run(self, size: int) -> list[int]: - tokens = (await stream_workflow.create_stream(Events)).topic(Events.tokens) + tokens = (await workflow.create_stream(Events)).topic(Events.tokens) return list( await asyncio.gather( tokens.publish(*["a" * size] * 3), tokens.publish(*["b" * size] * 3) @@ -563,7 +555,50 @@ async def test_merge_that_the_codec_grows_past_the_batch_limit(stream_client: Cl task_queue=worker.task_queue, ) offsets = await handle.result() - items = await read(client, handle) + items = await read(handle) written = [item for item in items if isinstance(item, streams.Record)] assert [len(r.data) for r in written] == [300_000] * 6 assert offsets == [written[2].offset, written[5].offset] + + +@workflow.defn +class CreatesOnTheDefault: + @workflow.run + async def run(self) -> None: + await workflow.create_stream(Events) + + +@workflow.defn +class CreatesOnRedis: + @workflow.run + async def run(self) -> str: + stream = await redis.workflow.create_stream(Events) + await stream.topic(Events.tokens).publish("on redis") + await redis.workflow.stream_handle(Events).close(result="closed") + return stream.backend or "" + + +async def test_default_is_native_with_no_default_backend(client: Client): + # With no backend marked default=True, the default is native, which this SDK lacks. The + # Redis calls still pick the registered Redis backend, and the memory one stands in for it. + config = client.config() + config["stream_backends"] = [MemoryStreamBackend()] + memory_client = Client(**config) + async with new_worker(memory_client, CreatesOnTheDefault, CreatesOnRedis) as worker: + handle = await memory_client.start_workflow( + CreatesOnTheDefault.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await assert_task_fail_eventually(handle, message_contains="default is native") + await handle.terminate() + on_redis = await memory_client.start_workflow( + CreatesOnRedis.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue + ) + assert await on_redis.result() == "memory" + with pytest.raises(streams.StreamConfigError, match="default=True"): + on_redis.stream(Events) + stream = on_redis.stream(Events, backend="memory") + items = [item async for item in stream.subscribe()] + assert data(items) == ["on redis"] + assert isinstance(items[-1], streams.Closed) and items[-1].result == "closed" From 5eb5bce871c1c51f2505ea3e7ced515e8232e918 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 09:48:03 -0700 Subject: [PATCH 06/79] Renamed the read options to offset and topic_prefixes. A read is stream-level only, so Topic objects lose subscribe and pass to the Stream's subscribe as topics. The proto field names stay. --- temporalio/common/streams/_access.py | 52 ++++++------------ temporalio/common/streams/_activity_stream.py | 46 ++++------------ temporalio/common/streams/_items.py | 2 +- temporalio/common/streams/_reader.py | 2 +- temporalio/common/streams/_stream_client.py | 53 +++++-------------- .../streams/examples/resume_from_offset.py | 11 ++-- .../contrib/streams/test_activity_handles.py | 3 +- tests/contrib/streams/test_facade.py | 6 +-- tests/contrib/streams/test_redis.py | 15 +++--- tests/contrib/streams/test_stream_client.py | 18 +++---- 10 files changed, 65 insertions(+), 143 deletions(-) diff --git a/temporalio/common/streams/_access.py b/temporalio/common/streams/_access.py index 2416b375f1..cd007be620 100644 --- a/temporalio/common/streams/_access.py +++ b/temporalio/common/streams/_access.py @@ -5,7 +5,7 @@ from collections.abc import AsyncGenerator, Awaitable, Callable, Sequence from dataclasses import dataclass from datetime import timedelta -from typing import Any, TypeVar, cast +from typing import Any, TypeVar import temporalio.converter from temporalio.common.streams import _convert, _reader @@ -55,10 +55,10 @@ async def items( await self.stream_id(), position=position, topics=read.topics, - topic_prefixes=read.prefixes, + topic_prefixes=read.topic_prefixes, types=read.types, default_type=read.default_type, - from_offset=read.from_offset, + from_offset=read.offset, last=read.last, wait_new_records=read.wait_new_records, create_wait=read.create_wait, @@ -72,7 +72,7 @@ async def items( async def get_offset(self) -> int: """The next offset of the Stream, the newest offset plus one. - It is 0 for a Stream with no Record, and ``from_offset=0`` reads from the start. + It is 0 for a Stream with no Record, and ``offset=0`` reads from the start. Offsets are sparse and above zero, so 0 is never a Record's offset. """ response = await self.backend.service(self.namespace).describe_stream( @@ -110,10 +110,10 @@ class ReadSpec: """What one read asks for.""" topics: list[str] | None - prefixes: Sequence[str] + topic_prefixes: Sequence[str] types: dict[str, Any] default_type: Any - from_offset: int | End | None + offset: int | End | None last: int | None wait_new_records: bool create_wait: timedelta | None @@ -125,9 +125,9 @@ def read_spec( str | TopicDefinition[Any] | _AsyncTopic[Any] | SyncTopicBase[Any] ], *, - prefixes: Sequence[str], + topic_prefixes: Sequence[str], default_type: Any, - from_offset: int | End | None, + offset: int | End | None, last: int | None, wait_new_records: bool, create_wait: timedelta | None, @@ -135,10 +135,10 @@ def read_spec( """The Topic names of a read, the decode type of each known Topic, and its options. Raises: - ValueError: ``last`` is not above zero, or it comes with ``from_offset``. + ValueError: ``last`` is not above zero, or it comes with ``offset``. """ - if last is not None and (last < 1 or from_offset is not None): - raise ValueError("last takes a count above zero and no from_offset") + if last is not None and (last < 1 or offset is not None): + raise ValueError("last takes a count above zero and no offset") types = handle._known_types() names = [] for topic in topics: @@ -155,10 +155,10 @@ def read_spec( types[name] = topic_type return ReadSpec( topics=names or None, - prefixes=prefixes, + topic_prefixes=topic_prefixes, types=types, default_type=default_type, - from_offset=from_offset, + offset=offset, last=last, wait_new_records=wait_new_records, create_wait=create_wait, @@ -173,10 +173,10 @@ def subscription( """A read over the items that ``items`` gives.""" # Before its first page, a read from the start resumes at 0, and a read from END or of the # last Records has no offset yet. - if read.last is not None or read.from_offset is End.END: + if read.last is not None or read.offset is End.END: position = Position(None) else: - position = Position(read.from_offset or 0) + position = Position(read.offset or 0) return Subscription(items(position), position, options) @@ -190,28 +190,6 @@ async def publish(self, *values: T) -> int: """ return await self._stream._access().publish(self._name, values) - def subscribe( - self, - *, - from_offset: int | End | None = None, - last: int | None = None, - wait_new_records: bool = True, - create_wait: timedelta | None = None, - ) -> Subscription[T]: - """Read this Topic only. Its Records have the Topic's type. See the Stream's - ``subscribe``. - """ - return cast( - Subscription[T], - self._stream.subscribe( - self, - from_offset=from_offset, - last=last, - wait_new_records=wait_new_records, - create_wait=create_wait, - ), - ) - class SyncTopicBase(TopicBase[T]): """Base of a Topic whose calls block, for sync Activities.""" diff --git a/temporalio/common/streams/_activity_stream.py b/temporalio/common/streams/_activity_stream.py index 29ad96ccc8..cf25b08e1f 100644 --- a/temporalio/common/streams/_activity_stream.py +++ b/temporalio/common/streams/_activity_stream.py @@ -6,7 +6,7 @@ import concurrent.futures from collections.abc import Coroutine, Generator, Sequence from datetime import timedelta -from typing import Any, Generic, TypeVar, cast, overload +from typing import Any, Generic, TypeVar, overload import temporalio.api.streamservice.v1 as stream_pb from temporalio.common.streams import _activity, _errors @@ -123,10 +123,10 @@ def topic( def subscribe( self, *topics: str | TopicDefinition[Any] | ActivityTopic[Any], - from_offset: int | End | None = None, + offset: int | End | None = None, last: int | None = None, wait_new_records: bool = True, - prefixes: Sequence[str] = (), + topic_prefixes: Sequence[str] = (), create_wait: timedelta | None = None, type: Any = None, ) -> Subscription[Any]: @@ -134,9 +134,9 @@ def subscribe( read = read_spec( self, topics, - prefixes=prefixes, + topic_prefixes=topic_prefixes, default_type=type, - from_offset=from_offset, + offset=offset, last=last, wait_new_records=wait_new_records, create_wait=create_wait, @@ -147,7 +147,7 @@ def subscribe( async def get_offset(self) -> int: """The next offset of the Stream, the newest offset plus one. - It is 0 for a Stream with no Record, and ``from_offset=0`` reads from the start. + It is 0 for a Stream with no Record, and ``offset=0`` reads from the start. Offsets are sparse and above zero, so 0 is never a Record's offset. """ return await self._access().get_offset() @@ -184,30 +184,6 @@ def publish(self, *values: T) -> int: assert isinstance(stream, SyncActivityStream) return stream._run(stream._access().publish(self._name, values)) - def subscribe( - self, - *, - from_offset: int | End | None = None, - last: int | None = None, - wait_new_records: bool = True, - create_wait: timedelta | None = None, - ) -> SyncSubscription[T]: - """Read this Topic only. Its Records have the Topic's type. See the Stream's - ``subscribe``. - """ - stream = self._stream - assert isinstance(stream, SyncActivityStream) - return cast( - SyncSubscription[T], - stream.subscribe( - self, - from_offset=from_offset, - last=last, - wait_new_records=wait_new_records, - create_wait=create_wait, - ), - ) - class SyncSubscription(Generic[T]): """The items of one read of a Stream in a sync Activity, in order. Use it in ``for``. @@ -300,10 +276,10 @@ def topic( def subscribe( self, *topics: str | TopicDefinition[Any] | SyncActivityTopic[Any], - from_offset: int | End | None = None, + offset: int | End | None = None, last: int | None = None, wait_new_records: bool = True, - prefixes: Sequence[str] = (), + topic_prefixes: Sequence[str] = (), create_wait: timedelta | None = None, type: Any = None, ) -> SyncSubscription[Any]: @@ -311,9 +287,9 @@ def subscribe( read = read_spec( self, topics, - prefixes=prefixes, + topic_prefixes=topic_prefixes, default_type=type, - from_offset=from_offset, + offset=offset, last=last, wait_new_records=wait_new_records, create_wait=create_wait, @@ -339,7 +315,7 @@ def _items( def get_offset(self) -> int: """The next offset of the Stream, the newest offset plus one. - It is 0 for a Stream with no Record, and ``from_offset=0`` reads from the start. + It is 0 for a Stream with no Record, and ``offset=0`` reads from the start. Offsets are sparse and above zero, so 0 is never a Record's offset. """ return self._run(self._access().get_offset()) diff --git a/temporalio/common/streams/_items.py b/temporalio/common/streams/_items.py index ca0c3d5b06..ade0ea70c4 100644 --- a/temporalio/common/streams/_items.py +++ b/temporalio/common/streams/_items.py @@ -16,7 +16,7 @@ class End(enum.Enum): - """A ``from_offset`` after the newest Record, for new Records only. Use ``END``.""" + """An ``offset`` after the newest Record, for new Records only. Use ``END``.""" END = "END" diff --git a/temporalio/common/streams/_reader.py b/temporalio/common/streams/_reader.py index 425062c90d..af2b526339 100644 --- a/temporalio/common/streams/_reader.py +++ b/temporalio/common/streams/_reader.py @@ -69,7 +69,7 @@ def next_offset(self) -> int | None: After a ``Superseded`` item it is that item's own offset, since the first Record of its batch has the same offset and comes next. A read from there gives the ``Superseded`` - item again, which withdraws nothing new. Pass it as ``from_offset`` to a later read. It + item again, which withdraws nothing new. Pass it as ``offset`` to a later read. It is None only before the first page of a read from ``END``. """ return self._position.next_offset diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index dd84655d03..1f688b0366 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -9,7 +9,7 @@ from collections.abc import AsyncGenerator, Awaitable, Callable, Mapping, Sequence from datetime import timedelta from types import TracebackType -from typing import Any, TypeVar, cast, overload +from typing import Any, TypeVar, overload import temporalio.api.streamservice.v1 as stream_pb import temporalio.client @@ -85,34 +85,6 @@ async def publish( with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): return await super().publish(*values) - def subscribe( - self, - *, - from_offset: int | End | None = None, - last: int | None = None, - wait_new_records: bool = True, - create_wait: timedelta | None = None, - rpc_metadata: Mapping[str, str | bytes] = {}, - rpc_timeout: timedelta | None = None, - ) -> Subscription[T]: - """Read this Topic only. Its Records have the Topic's type. See - :py:meth:`ClientStream.subscribe`. - """ - stream = self._stream - assert isinstance(stream, ClientStream) - return cast( - Subscription[T], - stream.subscribe( - self, - from_offset=from_offset, - last=last, - wait_new_records=wait_new_records, - create_wait=create_wait, - rpc_metadata=rpc_metadata, - rpc_timeout=rpc_timeout, - ), - ) - class ClientStream(StreamHandle[E]): """A Stream that a client writes, reads and closes. @@ -285,10 +257,10 @@ def topic( def subscribe( self, *topics: str | TopicDefinition[Any] | ClientTopic[Any], - from_offset: int | End | None = None, + offset: int | End | None = None, last: int | None = None, wait_new_records: bool = True, - prefixes: Sequence[str] = (), + topic_prefixes: Sequence[str] = (), create_wait: timedelta | None = None, type: Any = None, rpc_metadata: Mapping[str, str | bytes] = {}, @@ -298,24 +270,25 @@ def subscribe( The read ends after a ``Closed`` item, or at the newest Record when ``wait_new_records`` is False. A reader that wants to stop after a quiet time sets its - own timer, and resumes later with ``from_offset=`` the read's ``next_offset``. A + own timer, and resumes later with ``offset=`` the read's ``next_offset``. A ``Superseded`` item withdraws earlier Records of a failed attempt, and it passes every Topic filter. Args: - topics: Read these Topics. With no Topics and no prefixes, read every Topic. - from_offset: Start at this offset, inclusive, or at ``streams.END`` for new + topics: Read these Topics, by Topic of the definition, name or Topic object. + With no Topics and no ``topic_prefixes``, read every Topic. + offset: Start at this offset, inclusive, or at ``streams.END`` for new Records only. Defaults to the first Record. last: Start with the newest this many Records that pass the filter, oldest - first, such as the first view of a chat. Not with ``from_offset``. A Record + first, such as the first view of a chat. Not with ``offset``. A Record that a later ``Superseded`` item withdraws doesn't count. At most 1000. wait_new_records: Wait for new Records at the end of the Stream. - prefixes: Also read the Topics whose names start with one of these. + topic_prefixes: Also read the Topics whose names start with one of these. create_wait: How long to wait for a Stream that nobody created yet, such as the Stream of a Workflow that just started. Defaults to no wait. type: The type of the values of Topics outside the definition, such as Topics that - ``prefixes`` match. Topics of the definition keep their own types. Defaults to - no type, so the values decode as the converter gives them. + ``topic_prefixes`` match. Topics of the definition keep their own types. + Defaults to no type, so the values decode as the converter gives them. rpc_metadata: Headers of each call of the read. See :py:meth:`get_offset`. rpc_timeout: The deadline of each call of the read, not of the whole read. A call that waits for new Records answers before it. @@ -323,9 +296,9 @@ def subscribe( read = read_spec( self, topics, - prefixes=prefixes, + topic_prefixes=topic_prefixes, default_type=type, - from_offset=from_offset, + offset=offset, last=last, wait_new_records=wait_new_records, create_wait=create_wait, diff --git a/tests/contrib/streams/examples/resume_from_offset.py b/tests/contrib/streams/examples/resume_from_offset.py index 3c34dba7ad..95d3091293 100644 --- a/tests/contrib/streams/examples/resume_from_offset.py +++ b/tests/contrib/streams/examples/resume_from_offset.py @@ -2,7 +2,7 @@ The Workflow writes one Stream per turn, a renamed definition from ``streams.with_name``. The Reader saves the read's ``next_offset`` after each Record, stops, and later resumes from it with -``from_offset``. A view that wants only new Records starts at ``streams.END``. +``offset``. A view that wants only new Records starts at ``streams.END``. """ from __future__ import annotations @@ -60,10 +60,9 @@ async def read_some( handle: WorkflowHandle[Turns, None], store: OffsetStore, count: int ) -> list[str]: turn3 = handle.stream(Turn3) - tokens = turn3.topic(Events.tokens) seen = [] - read = tokens.subscribe( - from_offset=store.load("turn-3"), create_wait=timedelta(seconds=10) + read = turn3.subscribe( + Events.tokens, offset=store.load("turn-3"), create_wait=timedelta(seconds=10) ) async for item in read: if isinstance(item, streams.Closed): @@ -84,8 +83,8 @@ async def read_new(handle: WorkflowHandle[Turns, None]) -> list[str]: turn3 = handle.stream(Turn3) return [ item.data - async for item in turn3.topic(Events.tokens).subscribe( - from_offset=streams.END, wait_new_records=False + async for item in turn3.subscribe( + Events.tokens, offset=streams.END, wait_new_records=False ) if isinstance(item, streams.Record) ] diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index 67c02ef199..ee7191a475 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -235,8 +235,7 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( shown[key].set() live_items = [first, *[item async for item in live]] resumed = [ - item - async for item in stream.subscribe(Events.tokens, from_offset=resume_at) + item async for item in stream.subscribe(Events.tokens, offset=resume_at) ] assert isinstance(first, streams.Record) and first.data == "attempt-1" assert isinstance(before, streams.Record) and before.offset == first.offset diff --git a/tests/contrib/streams/test_facade.py b/tests/contrib/streams/test_facade.py index f249793de2..d0894825f7 100644 --- a/tests/contrib/streams/test_facade.py +++ b/tests/contrib/streams/test_facade.py @@ -92,7 +92,7 @@ async def test_client_writes_reads_and_closes(client: Client): raw = stream.topic("raw", type=RawValue) assert raw.type is RawValue - resumed = [item async for item in stream.subscribe(from_offset=token.offset)] + resumed = [item async for item in stream.subscribe(offset=token.offset)] assert [item.offset for item in resumed] == [token.offset, closed.offset] @@ -168,9 +168,7 @@ async def publish_later() -> None: await stream.topic(Events.tokens).publish("late") # Each call of a read that waits answers before the deadline, so the read goes on. - reader = stream.topic(Events.tokens).subscribe( - rpc_timeout=timedelta(milliseconds=300) - ) + reader = stream.subscribe(Events.tokens, rpc_timeout=timedelta(milliseconds=300)) publisher = asyncio.create_task(publish_later()) item = await anext(reader) await reader.aclose() diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 7fb99a9a02..58e77f2042 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -195,9 +195,7 @@ async def test_resume_after_a_superseded_item_keeps_its_record(stream_client: Cl assert resume_at == shared.offset resumed = [ item - async for item in stream.subscribe( - from_offset=resume_at, wait_new_records=False - ) + async for item in stream.subscribe(offset=resume_at, wait_new_records=False) ] assert [r.data for r in records(resumed) if r.offset == shared.offset] == ["after"] @@ -518,12 +516,13 @@ async def test_records_go_through_the_payload_codec(stream_client: Client): await handle.result() stream = workflow_stream(handle) decoded = await read_all(stream) - # A Stream with no definition reads the raw Payloads. + # A Topic outside the definition reads the raw Payloads. + plain = codec_client.stream(Plain, id=stream.id) raw = [ item - async for item in codec_client.stream(Plain, id=stream.id) - .topic("tokens", type=RawValue) - .subscribe(wait_new_records=False) + async for item in plain.subscribe( + plain.topic("tokens", type=RawValue), wait_new_records=False + ) ] assert [item.data for item in records(decoded)] == [ Turn("model"), @@ -754,7 +753,7 @@ async def read_until_quiet() -> None: resumed = [ item.data async for item in stream.subscribe( - from_offset=seen[-1].offset + 1, wait_new_records=False + offset=seen[-1].offset + 1, wait_new_records=False ) if isinstance(item, streams.Record) ] diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index af9f022f6d..caf80fdbef 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -128,17 +128,17 @@ async def test_client_stream_takes_workflow_writes(stream_client: Client): again = stream_client.stream(Events, id=stream.id) resumed = [ item.data - async for item in again.subscribe(Events.tokens, from_offset=first + 1) + async for item in again.subscribe(Events.tokens, offset=first + 1) if isinstance(item, streams.Record) ] assert resumed == ["a", "b"] by_prefix = [ item.data - async for item in again.subscribe(prefixes=["tok"]) + async for item in again.subscribe(topic_prefixes=["tok"]) if isinstance(item, streams.Record) ] assert by_prefix == ["a", "b"] - only_turns = [item async for item in again.topic(Events.turns).subscribe()] + only_turns = [item async for item in again.subscribe(again.topic(Events.turns))] assert [type(item) for item in only_turns] == [ streams.Record, streams.Record, @@ -210,19 +210,19 @@ async def test_next_offset_resumes_after_the_last_item(stream_client: Client): stream = await redis.create(stream_client, Events, id=f"resume-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("a") turn = await stream.topic(Events.turns).publish(Turn("model")) - tokens = stream.topic(Events.tokens).subscribe(wait_new_records=False) + tokens = stream.subscribe(Events.tokens, wait_new_records=False) assert tokens.next_offset == 0 assert [item.data async for item in tokens if isinstance(item, streams.Record)] == [ "a" ] # The page ends past the Record that the Topic filter left out. assert tokens.next_offset == turn + 1 - new = stream.subscribe(from_offset=streams.END, wait_new_records=False) + new = stream.subscribe(offset=streams.END, wait_new_records=False) assert new.next_offset is None assert [item async for item in new] == [] assert new.next_offset == turn + 1 await stream.topic(Events.tokens).publish("b") - resumed = stream.subscribe(from_offset=tokens.next_offset, wait_new_records=False) + resumed = stream.subscribe(offset=tokens.next_offset, wait_new_records=False) assert [ item.data async for item in resumed if isinstance(item, streams.Record) ] == ["b"] @@ -234,7 +234,7 @@ async def test_last_reads_the_newest_records_then_follows(stream_client: Client) await tokens.publish("a", "b", "c") await stream.topic(Events.turns).publish(Turn("model")) # A chat view shows the last few Records first, then follows. - tail = tokens.subscribe(last=2) + tail = stream.subscribe(tokens, last=2) assert tail.next_offset is None shown = [await anext(tail), await anext(tail)] await tokens.publish("d") @@ -249,8 +249,8 @@ async def test_last_reads_the_newest_records_then_follows(stream_client: Client) assert [item.data async for item in newest if isinstance(item, streams.Record)] == [ "d" ] - with pytest.raises(ValueError, match="no from_offset"): - stream.subscribe(last=2, from_offset=0) + with pytest.raises(ValueError, match="no offset"): + stream.subscribe(last=2, offset=0) async def test_read_type_decodes_topics_outside_the_definition(stream_client: Client): From 4d67eec88c1ada25fc2bf66c42289a7fa379b2a7 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 09:52:40 -0700 Subject: [PATCH 07/79] Vendored the s23 Stream protos and WIT, and regenerated the bindings. The crate reserves the auto_close fields, so the pb2 and the system bindings lose them. The generators write into the new package. --- .../api/streamservice/v1/stream_service.proto | 14 ++-- scripts/_wit/stream-service.wit | 8 +-- scripts/gen_protos.py | 2 +- .../streamservice/v1/stream_service_pb2.py | 64 +++++++++---------- .../streamservice/v1/stream_service_pb2.pyi | 36 +---------- .../streams/system/_resources/stream.py | 2 +- temporalio/common/streams/system/client.py | 2 +- temporalio/common/streams/system/models.py | 14 +--- .../system/operations/create_stream.py | 4 -- 9 files changed, 46 insertions(+), 100 deletions(-) diff --git a/scripts/_proto/temporal/api/streamservice/v1/stream_service.proto b/scripts/_proto/temporal/api/streamservice/v1/stream_service.proto index 4a9e1468ca..9f047f7270 100644 --- a/scripts/_proto/temporal/api/streamservice/v1/stream_service.proto +++ b/scripts/_proto/temporal/api/streamservice/v1/stream_service.proto @@ -131,8 +131,7 @@ message StreamRecord { CloseStatus close_status = 9; } -// Idempotent for one id, one Retention and one auto_close. ALREADY_EXISTS for another -// Retention or another auto_close. +// Idempotent for one id and one Retention. ALREADY_EXISTS for another Retention. message CreateStreamRequest { string namespace = 1; // Required from Workflow code. @@ -141,11 +140,8 @@ message CreateStreamRequest { google.protobuf.Duration retention = 3; // Workflow only. A retry run's first create: the previous run, whose Records it withdraws. string withdraw_run = 4; - // Close the Stream with no result when no Record was appended for this long, counted from - // the create. Empty never closes on its own. It is separate from Retention and the safety - // TTL. The Store applies it on the next access, so no call sees the Stream open after the - // period. UNIMPLEMENTED (detail stream_unsupported) on a Store that can't. - google.protobuf.Duration auto_close = 5; + reserved 5; + reserved "auto_close"; // The Workflow run that owns the Stream. Workflow kind, with its run_id and epoch. Producer owner = 6; // Close the Stream with the result or the failure of the owner's run chain when the chain @@ -282,8 +278,8 @@ message DescribeStreamResponse { // Zero when the Stream has no Record. int64 last_offset = 2; google.protobuf.Duration retention = 3; - // The auto_close of the create. Empty when the Stream has none. - google.protobuf.Duration auto_close = 4; + reserved 4; + reserved "auto_close"; } // The details of a failed Stream call, in the gRPC status details and in the details of the diff --git a/scripts/_wit/stream-service.wit b/scripts/_wit/stream-service.wit index 188860c9ae..6c307a43da 100644 --- a/scripts/_wit/stream-service.wit +++ b/scripts/_wit/stream-service.wit @@ -169,7 +169,7 @@ interface stream-service { close-status: option, } - /// @nexus.doc "Create a Stream idempotently. ALREADY_EXISTS for another Retention or auto-close." + /// @nexus.doc "Create a Stream idempotently. ALREADY_EXISTS for another Retention." /// @nexus.proto "temporal.api.streamservice.v1.CreateStreamRequest" record create-stream-request { /// @nexus.doc "Required from Workflow code. Empty elsewhere asks the service for an id." @@ -178,8 +178,6 @@ interface stream-service { retention: option, /// @nexus.doc "Retry run only. The previous run, whose Records this withdraws." withdraw-run: option, - /// @nexus.doc "Close with no result after this long with no append. Empty never auto-closes." - auto-close: option, /// @nexus.doc "The Workflow run that owns the Stream. The Workflow transport sets it." owner: option, /// @nexus.doc "Close with the owner run chain's result when it closes. Native only." @@ -330,8 +328,6 @@ interface stream-service { /// @nexus.offset "stream-offset" last-offset: option, retention: option, - /// @nexus.doc "The auto-close of the create. Empty when the Stream has none." - auto-close: option, } // Its stream-ref is its serialized form, so a handle passes through Workflow input, Activity @@ -375,7 +371,7 @@ interface stream-service { create-wait: option, ) -> read-records-response; - /// @nexus.doc "Describe the Stream: its status, newest offset, Retention and auto-close." + /// @nexus.doc "Describe the Stream: its status, newest offset and Retention." /// @nexus.operation "describe-stream" describe: func() -> describe-stream-response; } diff --git a/scripts/gen_protos.py b/scripts/gen_protos.py index 51b0a9ec44..a3330d8e87 100644 --- a/scripts/gen_protos.py +++ b/scripts/gen_protos.py @@ -231,7 +231,7 @@ def generate_protos(output_dir: Path): for p in (output_dir / "temporal" / "sdk" / "core").iterdir(): shutil.rmtree(sdk_out_dir / p.name, ignore_errors=True) p.replace(sdk_out_dir / p.name) - # The stream extension's own protos go with the contrib package that uses them + # The stream extension's own protos go with the Streams package that uses them streams_out_dir.mkdir(parents=True, exist_ok=True) for p in (output_dir / "temporal" / "sdk" / "streams").iterdir(): shutil.rmtree(streams_out_dir / p.name, ignore_errors=True) diff --git a/temporalio/api/streamservice/v1/stream_service_pb2.py b/temporalio/api/streamservice/v1/stream_service_pb2.py index 2c3894ef0c..19295a86d4 100644 --- a/temporalio/api/streamservice/v1/stream_service_pb2.py +++ b/temporalio/api/streamservice/v1/stream_service_pb2.py @@ -26,7 +26,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n2temporal/api/streamservice/v1/stream_service.proto\x12\x1dtemporal.api.streamservice.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto"\x8b\x01\n\tStreamRef\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x03 \x01(\t\x12\x0f\n\x07\x62\x61\x63kend\x18\x04 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x05 \x01(\t\x12\x17\n\x0f\x61\x63tivity_run_id\x18\x06 \x01(\t"\xc8\x01\n\rProducerEpoch\x12\x1d\n\x15task_started_event_id\x18\x01 \x01(\x03\x12\x14\n\x0ctask_attempt\x18\x02 \x01(\x05\x12\x35\n\x11task_started_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16\x61ttempt_scheduled_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x05"\x9e\x01\n\x08Producer\x12\x39\n\x04kind\x18\x01 \x01(\x0e\x32+.temporal.api.streamservice.v1.ProducerKind\x12\n\n\x02id\x18\x02 \x01(\t\x12;\n\x05\x65poch\x18\x03 \x01(\x0b\x32,.temporal.api.streamservice.v1.ProducerEpoch\x12\x0e\n\x06run_id\x18\x04 \x01(\t"p\n\x11WorkflowTaskEpoch\x12\x18\n\x10started_event_id\x18\x01 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05\x12\x30\n\x0cstarted_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"L\n\x0c\x41ppendRecord\x12\r\n\x05topic\x18\x01 \x01(\t\x12-\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"\xb3\x02\n\x0cSupersession\x12\x42\n\rproducer_kind\x18\x01 \x01(\x0e\x32+.temporal.api.streamservice.v1.ProducerKind\x12\x13\n\x0bproducer_id\x18\x02 \x01(\t\x12;\n\x05\x65poch\x18\x03 \x01(\x0b\x32,.temporal.api.streamservice.v1.ProducerEpoch\x12Q\n\x17previous_completed_task\x18\x04 \x01(\x0b\x32\x30.temporal.api.streamservice.v1.WorkflowTaskEpoch\x12\x14\n\x0c\x66\x61iled_tasks\x18\x05 \x03(\x03\x12\x14\n\x0cwithdraw_run\x18\x06 \x01(\t\x12\x0e\n\x06run_id\x18\x07 \x01(\t"\xcb\x03\n\x0cStreamRecord\x12=\n\x04kind\x18\x01 \x01(\x0e\x32/.temporal.api.streamservice.v1.StreamRecordKind\x12\x0e\n\x06offset\x18\x02 \x01(\x03\x12\r\n\x05topic\x18\x03 \x01(\t\x12\x39\n\x08producer\x18\x04 \x01(\x0b\x32\'.temporal.api.streamservice.v1.Producer\x12-\n\x04\x62ody\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x41\n\x0csupersession\x18\x06 \x01(\x0b\x32+.temporal.api.streamservice.v1.Supersession\x12\x35\n\x0c\x63lose_result\x18\x07 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\rclose_failure\x18\x08 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x0c\x63lose_status\x18\t \x01(\x0e\x32*.temporal.api.streamservice.v1.CloseStatus"\xc9\x02\n\x13\x43reateStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12,\n\tretention\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0cwithdraw_run\x18\x04 \x01(\t\x12-\n\nauto_close\x18\x05 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x36\n\x05owner\x18\x06 \x01(\x0b\x32\'.temporal.api.streamservice.v1.Producer\x12\x16\n\x0e\x63lose_with_run\x18\x07 \x01(\x08\x12\x35\n\x03ref\x18\x08 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x12\n\ntask_token\x18\t \x01(\x0c"\x9e\x01\n\x14\x43reateStreamResponse\x12\x38\n\x06stream\x18\x01 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x0f\n\x07\x63reated\x18\x02 \x01(\x08\x12;\n\x06status\x18\x03 \x01(\x0e\x32+.temporal.api.streamservice.v1.StreamStatus"\x9b\x03\n\x14\x41ppendRecordsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\x06stream\x18\x02 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x39\n\x08producer\x18\x03 \x01(\x0b\x32\'.temporal.api.streamservice.v1.Producer\x12\x10\n\x08sequence\x18\x04 \x01(\x03\x12<\n\x07records\x18\x05 \x03(\x0b\x32+.temporal.api.streamservice.v1.AppendRecord\x12Q\n\x17previous_completed_task\x18\x06 \x01(\x0b\x32\x30.temporal.api.streamservice.v1.WorkflowTaskEpoch\x12\x14\n\x0c\x66\x61iled_tasks\x18\x07 \x03(\x03\x12\x14\n\x0cwithdraw_run\x18\x08 \x01(\t\x12\x18\n\x10\x64\x65\x66\x65r_withdrawal\x18\t \x01(\x08\x12\x12\n\ntask_token\x18\n \x01(\x0c"B\n\x15\x41ppendRecordsResponse\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03"\xcf\x03\n\x12\x43loseStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\x06stream\x18\x02 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x39\n\x08producer\x18\x03 \x01(\x0b\x32\'.temporal.api.streamservice.v1.Producer\x12/\n\x06result\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12Q\n\x17previous_completed_task\x18\x06 \x01(\x0b\x32\x30.temporal.api.streamservice.v1.WorkflowTaskEpoch\x12\x14\n\x0c\x66\x61iled_tasks\x18\x07 \x03(\x03\x12\x14\n\x0cwithdraw_run\x18\x08 \x01(\t\x12:\n\x06status\x18\t \x01(\x0e\x32*.temporal.api.streamservice.v1.CloseStatus\x12\x12\n\ntask_token\x18\n \x01(\x0c"\x15\n\x13\x43loseStreamResponse"\xb4\x03\n\x12ReadRecordsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\x06stream\x18\x02 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x03\x12\x37\n\x05start\x18\x04 \x01(\x0e\x32(.temporal.api.streamservice.v1.ReadStart\x12\x0e\n\x06topics\x18\x05 \x03(\t\x12\x13\n\x0bmax_records\x18\x06 \x01(\x05\x12\'\n\x04wait\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12.\n\x0b\x63reate_wait\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x16\n\x0etopic_prefixes\x18\n \x03(\t\x12\x11\n\tto_offset\x18\x0b \x01(\x03\x12\x11\n\twake_only\x18\x0c \x01(\x08\x12\x0c\n\x04last\x18\r \x01(\x05\x12\x11\n\tmax_bytes\x18\x0e \x01(\x03\x12\x12\n\nfloor_from\x18\x0f \x01(\x03J\x04\x08\t\x10\nR\x0cidle_timeout"\x99\x01\n\x13ReadRecordsResponse\x12<\n\x07records\x18\x01 \x03(\x0b\x32+.temporal.api.streamservice.v1.StreamRecord\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12!\n\x19\x63ontinue_as_new_suggested\x18\x04 \x01(\x08"d\n\x15\x44\x65scribeStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\x06stream\x18\x02 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef"\xc7\x01\n\x16\x44\x65scribeStreamResponse\x12;\n\x06status\x18\x01 \x01(\x0e\x32+.temporal.api.streamservice.v1.StreamStatus\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03\x12,\n\tretention\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12-\n\nauto_close\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration"$\n\x12StreamErrorDetails\x12\x0e\n\x06reason\x18\x01 \x01(\t*\xc8\x01\n\x0cProducerKind\x12\x1d\n\x19PRODUCER_KIND_UNSPECIFIED\x10\x00\x12\x1a\n\x16PRODUCER_KIND_WORKFLOW\x10\x01\x12 \n\x1cPRODUCER_KIND_LOCAL_ACTIVITY\x10\x02\x12\x1a\n\x16PRODUCER_KIND_ACTIVITY\x10\x03\x12\x18\n\x14PRODUCER_KIND_CLIENT\x10\x04\x12%\n!PRODUCER_KIND_STANDALONE_ACTIVITY\x10\x05*\xb3\x01\n\x10StreamRecordKind\x12"\n\x1eSTREAM_RECORD_KIND_UNSPECIFIED\x10\x00\x12\x1b\n\x17STREAM_RECORD_KIND_DATA\x10\x01\x12!\n\x1dSTREAM_RECORD_KIND_SUPERSEDED\x10\x02\x12\x1c\n\x18STREAM_RECORD_KIND_CLOSE\x10\x03"\x04\x08\x04\x10\x04*\x17STREAM_RECORD_KIND_IDLE*\xb4\x01\n\x0b\x43loseStatus\x12\x1c\n\x18\x43LOSE_STATUS_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43LOSE_STATUS_COMPLETED\x10\x01\x12\x17\n\x13\x43LOSE_STATUS_FAILED\x10\x02\x12\x19\n\x15\x43LOSE_STATUS_CANCELED\x10\x03\x12\x1b\n\x17\x43LOSE_STATUS_TERMINATED\x10\x04\x12\x1a\n\x16\x43LOSE_STATUS_TIMED_OUT\x10\x05*z\n\x0cStreamStatus\x12\x1d\n\x19STREAM_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12STREAM_STATUS_OPEN\x10\x01\x12\x18\n\x14STREAM_STATUS_CLOSED\x10\x02\x12\x19\n\x15STREAM_STATUS_EXPIRED\x10\x03*U\n\tReadStart\x12\x1a\n\x16READ_START_UNSPECIFIED\x10\x00\x12\x18\n\x14READ_START_BEGINNING\x10\x01\x12\x12\n\x0eREAD_START_END\x10\x02\x42\xb2\x01\n io.temporal.api.streamservice.v1B\x12StreamServiceProtoP\x01Z1go.temporal.io/api/streamservice/v1;streamservice\xaa\x02\x1fTemporalio.Api.StreamService.V1\xea\x02"Temporalio::Api::StreamService::V1b\x06proto3' + b'\n2temporal/api/streamservice/v1/stream_service.proto\x12\x1dtemporal.api.streamservice.v1\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto"\x8b\x01\n\tStreamRef\x12\n\n\x02id\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x03 \x01(\t\x12\x0f\n\x07\x62\x61\x63kend\x18\x04 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x05 \x01(\t\x12\x17\n\x0f\x61\x63tivity_run_id\x18\x06 \x01(\t"\xc8\x01\n\rProducerEpoch\x12\x1d\n\x15task_started_event_id\x18\x01 \x01(\x03\x12\x14\n\x0ctask_attempt\x18\x02 \x01(\x05\x12\x35\n\x11task_started_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16\x61ttempt_scheduled_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x05"\x9e\x01\n\x08Producer\x12\x39\n\x04kind\x18\x01 \x01(\x0e\x32+.temporal.api.streamservice.v1.ProducerKind\x12\n\n\x02id\x18\x02 \x01(\t\x12;\n\x05\x65poch\x18\x03 \x01(\x0b\x32,.temporal.api.streamservice.v1.ProducerEpoch\x12\x0e\n\x06run_id\x18\x04 \x01(\t"p\n\x11WorkflowTaskEpoch\x12\x18\n\x10started_event_id\x18\x01 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05\x12\x30\n\x0cstarted_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"L\n\x0c\x41ppendRecord\x12\r\n\x05topic\x18\x01 \x01(\t\x12-\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"\xb3\x02\n\x0cSupersession\x12\x42\n\rproducer_kind\x18\x01 \x01(\x0e\x32+.temporal.api.streamservice.v1.ProducerKind\x12\x13\n\x0bproducer_id\x18\x02 \x01(\t\x12;\n\x05\x65poch\x18\x03 \x01(\x0b\x32,.temporal.api.streamservice.v1.ProducerEpoch\x12Q\n\x17previous_completed_task\x18\x04 \x01(\x0b\x32\x30.temporal.api.streamservice.v1.WorkflowTaskEpoch\x12\x14\n\x0c\x66\x61iled_tasks\x18\x05 \x03(\x03\x12\x14\n\x0cwithdraw_run\x18\x06 \x01(\t\x12\x0e\n\x06run_id\x18\x07 \x01(\t"\xcb\x03\n\x0cStreamRecord\x12=\n\x04kind\x18\x01 \x01(\x0e\x32/.temporal.api.streamservice.v1.StreamRecordKind\x12\x0e\n\x06offset\x18\x02 \x01(\x03\x12\r\n\x05topic\x18\x03 \x01(\t\x12\x39\n\x08producer\x18\x04 \x01(\x0b\x32\'.temporal.api.streamservice.v1.Producer\x12-\n\x04\x62ody\x18\x05 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x41\n\x0csupersession\x18\x06 \x01(\x0b\x32+.temporal.api.streamservice.v1.Supersession\x12\x35\n\x0c\x63lose_result\x18\x07 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x37\n\rclose_failure\x18\x08 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12@\n\x0c\x63lose_status\x18\t \x01(\x0e\x32*.temporal.api.streamservice.v1.CloseStatus"\xac\x02\n\x13\x43reateStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tstream_id\x18\x02 \x01(\t\x12,\n\tretention\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0cwithdraw_run\x18\x04 \x01(\t\x12\x36\n\x05owner\x18\x06 \x01(\x0b\x32\'.temporal.api.streamservice.v1.Producer\x12\x16\n\x0e\x63lose_with_run\x18\x07 \x01(\x08\x12\x35\n\x03ref\x18\x08 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x12\n\ntask_token\x18\t \x01(\x0cJ\x04\x08\x05\x10\x06R\nauto_close"\x9e\x01\n\x14\x43reateStreamResponse\x12\x38\n\x06stream\x18\x01 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x0f\n\x07\x63reated\x18\x02 \x01(\x08\x12;\n\x06status\x18\x03 \x01(\x0e\x32+.temporal.api.streamservice.v1.StreamStatus"\x9b\x03\n\x14\x41ppendRecordsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\x06stream\x18\x02 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x39\n\x08producer\x18\x03 \x01(\x0b\x32\'.temporal.api.streamservice.v1.Producer\x12\x10\n\x08sequence\x18\x04 \x01(\x03\x12<\n\x07records\x18\x05 \x03(\x0b\x32+.temporal.api.streamservice.v1.AppendRecord\x12Q\n\x17previous_completed_task\x18\x06 \x01(\x0b\x32\x30.temporal.api.streamservice.v1.WorkflowTaskEpoch\x12\x14\n\x0c\x66\x61iled_tasks\x18\x07 \x03(\x03\x12\x14\n\x0cwithdraw_run\x18\x08 \x01(\t\x12\x18\n\x10\x64\x65\x66\x65r_withdrawal\x18\t \x01(\x08\x12\x12\n\ntask_token\x18\n \x01(\x0c"B\n\x15\x41ppendRecordsResponse\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03"\xcf\x03\n\x12\x43loseStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\x06stream\x18\x02 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x39\n\x08producer\x18\x03 \x01(\x0b\x32\'.temporal.api.streamservice.v1.Producer\x12/\n\x06result\x18\x04 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x31\n\x07\x66\x61ilure\x18\x05 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12Q\n\x17previous_completed_task\x18\x06 \x01(\x0b\x32\x30.temporal.api.streamservice.v1.WorkflowTaskEpoch\x12\x14\n\x0c\x66\x61iled_tasks\x18\x07 \x03(\x03\x12\x14\n\x0cwithdraw_run\x18\x08 \x01(\t\x12:\n\x06status\x18\t \x01(\x0e\x32*.temporal.api.streamservice.v1.CloseStatus\x12\x12\n\ntask_token\x18\n \x01(\x0c"\x15\n\x13\x43loseStreamResponse"\xb4\x03\n\x12ReadRecordsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\x06stream\x18\x02 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x03\x12\x37\n\x05start\x18\x04 \x01(\x0e\x32(.temporal.api.streamservice.v1.ReadStart\x12\x0e\n\x06topics\x18\x05 \x03(\t\x12\x13\n\x0bmax_records\x18\x06 \x01(\x05\x12\'\n\x04wait\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12.\n\x0b\x63reate_wait\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x16\n\x0etopic_prefixes\x18\n \x03(\t\x12\x11\n\tto_offset\x18\x0b \x01(\x03\x12\x11\n\twake_only\x18\x0c \x01(\x08\x12\x0c\n\x04last\x18\r \x01(\x05\x12\x11\n\tmax_bytes\x18\x0e \x01(\x03\x12\x12\n\nfloor_from\x18\x0f \x01(\x03J\x04\x08\t\x10\nR\x0cidle_timeout"\x99\x01\n\x13ReadRecordsResponse\x12<\n\x07records\x18\x01 \x03(\x0b\x32+.temporal.api.streamservice.v1.StreamRecord\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12!\n\x19\x63ontinue_as_new_suggested\x18\x04 \x01(\x08"d\n\x15\x44\x65scribeStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x38\n\x06stream\x18\x02 \x01(\x0b\x32(.temporal.api.streamservice.v1.StreamRef"\xaa\x01\n\x16\x44\x65scribeStreamResponse\x12;\n\x06status\x18\x01 \x01(\x0e\x32+.temporal.api.streamservice.v1.StreamStatus\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03\x12,\n\tretention\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationJ\x04\x08\x04\x10\x05R\nauto_close"$\n\x12StreamErrorDetails\x12\x0e\n\x06reason\x18\x01 \x01(\t*\xc8\x01\n\x0cProducerKind\x12\x1d\n\x19PRODUCER_KIND_UNSPECIFIED\x10\x00\x12\x1a\n\x16PRODUCER_KIND_WORKFLOW\x10\x01\x12 \n\x1cPRODUCER_KIND_LOCAL_ACTIVITY\x10\x02\x12\x1a\n\x16PRODUCER_KIND_ACTIVITY\x10\x03\x12\x18\n\x14PRODUCER_KIND_CLIENT\x10\x04\x12%\n!PRODUCER_KIND_STANDALONE_ACTIVITY\x10\x05*\xb3\x01\n\x10StreamRecordKind\x12"\n\x1eSTREAM_RECORD_KIND_UNSPECIFIED\x10\x00\x12\x1b\n\x17STREAM_RECORD_KIND_DATA\x10\x01\x12!\n\x1dSTREAM_RECORD_KIND_SUPERSEDED\x10\x02\x12\x1c\n\x18STREAM_RECORD_KIND_CLOSE\x10\x03"\x04\x08\x04\x10\x04*\x17STREAM_RECORD_KIND_IDLE*\xb4\x01\n\x0b\x43loseStatus\x12\x1c\n\x18\x43LOSE_STATUS_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x43LOSE_STATUS_COMPLETED\x10\x01\x12\x17\n\x13\x43LOSE_STATUS_FAILED\x10\x02\x12\x19\n\x15\x43LOSE_STATUS_CANCELED\x10\x03\x12\x1b\n\x17\x43LOSE_STATUS_TERMINATED\x10\x04\x12\x1a\n\x16\x43LOSE_STATUS_TIMED_OUT\x10\x05*z\n\x0cStreamStatus\x12\x1d\n\x19STREAM_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12STREAM_STATUS_OPEN\x10\x01\x12\x18\n\x14STREAM_STATUS_CLOSED\x10\x02\x12\x19\n\x15STREAM_STATUS_EXPIRED\x10\x03*U\n\tReadStart\x12\x1a\n\x16READ_START_UNSPECIFIED\x10\x00\x12\x18\n\x14READ_START_BEGINNING\x10\x01\x12\x12\n\x0eREAD_START_END\x10\x02\x42\xb2\x01\n io.temporal.api.streamservice.v1B\x12StreamServiceProtoP\x01Z1go.temporal.io/api/streamservice/v1;streamservice\xaa\x02\x1fTemporalio.Api.StreamService.V1\xea\x02"Temporalio::Api::StreamService::V1b\x06proto3' ) _PRODUCERKIND = DESCRIPTOR.enum_types_by_name["ProducerKind"] @@ -283,16 +283,16 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n io.temporal.api.streamservice.v1B\022StreamServiceProtoP\001Z1go.temporal.io/api/streamservice/v1;streamservice\252\002\037Temporalio.Api.StreamService.V1\352\002"Temporalio::Api::StreamService::V1' - _PRODUCERKIND._serialized_start = 4099 - _PRODUCERKIND._serialized_end = 4299 - _STREAMRECORDKIND._serialized_start = 4302 - _STREAMRECORDKIND._serialized_end = 4481 - _CLOSESTATUS._serialized_start = 4484 - _CLOSESTATUS._serialized_end = 4664 - _STREAMSTATUS._serialized_start = 4666 - _STREAMSTATUS._serialized_end = 4788 - _READSTART._serialized_start = 4790 - _READSTART._serialized_end = 4875 + _PRODUCERKIND._serialized_start = 4041 + _PRODUCERKIND._serialized_end = 4241 + _STREAMRECORDKIND._serialized_start = 4244 + _STREAMRECORDKIND._serialized_end = 4423 + _CLOSESTATUS._serialized_start = 4426 + _CLOSESTATUS._serialized_end = 4606 + _STREAMSTATUS._serialized_start = 4608 + _STREAMSTATUS._serialized_end = 4730 + _READSTART._serialized_start = 4732 + _READSTART._serialized_end = 4817 _STREAMREF._serialized_start = 228 _STREAMREF._serialized_end = 367 _PRODUCEREPOCH._serialized_start = 370 @@ -308,25 +308,25 @@ _STREAMRECORD._serialized_start = 1236 _STREAMRECORD._serialized_end = 1695 _CREATESTREAMREQUEST._serialized_start = 1698 - _CREATESTREAMREQUEST._serialized_end = 2027 - _CREATESTREAMRESPONSE._serialized_start = 2030 - _CREATESTREAMRESPONSE._serialized_end = 2188 - _APPENDRECORDSREQUEST._serialized_start = 2191 - _APPENDRECORDSREQUEST._serialized_end = 2602 - _APPENDRECORDSRESPONSE._serialized_start = 2604 - _APPENDRECORDSRESPONSE._serialized_end = 2670 - _CLOSESTREAMREQUEST._serialized_start = 2673 - _CLOSESTREAMREQUEST._serialized_end = 3136 - _CLOSESTREAMRESPONSE._serialized_start = 3138 - _CLOSESTREAMRESPONSE._serialized_end = 3159 - _READRECORDSREQUEST._serialized_start = 3162 - _READRECORDSREQUEST._serialized_end = 3598 - _READRECORDSRESPONSE._serialized_start = 3601 - _READRECORDSRESPONSE._serialized_end = 3754 - _DESCRIBESTREAMREQUEST._serialized_start = 3756 - _DESCRIBESTREAMREQUEST._serialized_end = 3856 - _DESCRIBESTREAMRESPONSE._serialized_start = 3859 - _DESCRIBESTREAMRESPONSE._serialized_end = 4058 - _STREAMERRORDETAILS._serialized_start = 4060 - _STREAMERRORDETAILS._serialized_end = 4096 + _CREATESTREAMREQUEST._serialized_end = 1998 + _CREATESTREAMRESPONSE._serialized_start = 2001 + _CREATESTREAMRESPONSE._serialized_end = 2159 + _APPENDRECORDSREQUEST._serialized_start = 2162 + _APPENDRECORDSREQUEST._serialized_end = 2573 + _APPENDRECORDSRESPONSE._serialized_start = 2575 + _APPENDRECORDSRESPONSE._serialized_end = 2641 + _CLOSESTREAMREQUEST._serialized_start = 2644 + _CLOSESTREAMREQUEST._serialized_end = 3107 + _CLOSESTREAMRESPONSE._serialized_start = 3109 + _CLOSESTREAMRESPONSE._serialized_end = 3130 + _READRECORDSREQUEST._serialized_start = 3133 + _READRECORDSREQUEST._serialized_end = 3569 + _READRECORDSRESPONSE._serialized_start = 3572 + _READRECORDSRESPONSE._serialized_end = 3725 + _DESCRIBESTREAMREQUEST._serialized_start = 3727 + _DESCRIBESTREAMREQUEST._serialized_end = 3827 + _DESCRIBESTREAMRESPONSE._serialized_start = 3830 + _DESCRIBESTREAMRESPONSE._serialized_end = 4000 + _STREAMERRORDETAILS._serialized_start = 4002 + _STREAMERRORDETAILS._serialized_end = 4038 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/streamservice/v1/stream_service_pb2.pyi b/temporalio/api/streamservice/v1/stream_service_pb2.pyi index d664aba13f..5066333fe4 100644 --- a/temporalio/api/streamservice/v1/stream_service_pb2.pyi +++ b/temporalio/api/streamservice/v1/stream_service_pb2.pyi @@ -517,9 +517,7 @@ class StreamRecord(google.protobuf.message.Message): global___StreamRecord = StreamRecord class CreateStreamRequest(google.protobuf.message.Message): - """Idempotent for one id, one Retention and one auto_close. ALREADY_EXISTS for another - Retention or another auto_close. - """ + """Idempotent for one id and one Retention. ALREADY_EXISTS for another Retention.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -527,7 +525,6 @@ class CreateStreamRequest(google.protobuf.message.Message): STREAM_ID_FIELD_NUMBER: builtins.int RETENTION_FIELD_NUMBER: builtins.int WITHDRAW_RUN_FIELD_NUMBER: builtins.int - AUTO_CLOSE_FIELD_NUMBER: builtins.int OWNER_FIELD_NUMBER: builtins.int CLOSE_WITH_RUN_FIELD_NUMBER: builtins.int REF_FIELD_NUMBER: builtins.int @@ -541,13 +538,6 @@ class CreateStreamRequest(google.protobuf.message.Message): withdraw_run: builtins.str """Workflow only. A retry run's first create: the previous run, whose Records it withdraws.""" @property - def auto_close(self) -> google.protobuf.duration_pb2.Duration: - """Close the Stream with no result when no Record was appended for this long, counted from - the create. Empty never closes on its own. It is separate from Retention and the safety - TTL. The Store applies it on the next access, so no call sees the Stream open after the - period. UNIMPLEMENTED (detail stream_unsupported) on a Store that can't. - """ - @property def owner(self) -> global___Producer: """The Workflow run that owns the Stream. Workflow kind, with its run_id and epoch.""" close_with_run: builtins.bool @@ -570,7 +560,6 @@ class CreateStreamRequest(google.protobuf.message.Message): stream_id: builtins.str = ..., retention: google.protobuf.duration_pb2.Duration | None = ..., withdraw_run: builtins.str = ..., - auto_close: google.protobuf.duration_pb2.Duration | None = ..., owner: global___Producer | None = ..., close_with_run: builtins.bool = ..., ref: global___StreamRef | None = ..., @@ -579,21 +568,12 @@ class CreateStreamRequest(google.protobuf.message.Message): def HasField( self, field_name: typing_extensions.Literal[ - "auto_close", - b"auto_close", - "owner", - b"owner", - "ref", - b"ref", - "retention", - b"retention", + "owner", b"owner", "ref", b"ref", "retention", b"retention" ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ - "auto_close", - b"auto_close", "close_with_run", b"close_with_run", "namespace", @@ -1072,34 +1052,24 @@ class DescribeStreamResponse(google.protobuf.message.Message): STATUS_FIELD_NUMBER: builtins.int LAST_OFFSET_FIELD_NUMBER: builtins.int RETENTION_FIELD_NUMBER: builtins.int - AUTO_CLOSE_FIELD_NUMBER: builtins.int status: global___StreamStatus.ValueType last_offset: builtins.int """Zero when the Stream has no Record.""" @property def retention(self) -> google.protobuf.duration_pb2.Duration: ... - @property - def auto_close(self) -> google.protobuf.duration_pb2.Duration: - """The auto_close of the create. Empty when the Stream has none.""" def __init__( self, *, status: global___StreamStatus.ValueType = ..., last_offset: builtins.int = ..., retention: google.protobuf.duration_pb2.Duration | None = ..., - auto_close: google.protobuf.duration_pb2.Duration | None = ..., ) -> None: ... def HasField( - self, - field_name: typing_extensions.Literal[ - "auto_close", b"auto_close", "retention", b"retention" - ], + self, field_name: typing_extensions.Literal["retention", b"retention"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ - "auto_close", - b"auto_close", "last_offset", b"last_offset", "retention", diff --git a/temporalio/common/streams/system/_resources/stream.py b/temporalio/common/streams/system/_resources/stream.py index bba5cfaeb8..1e268b96fc 100644 --- a/temporalio/common/streams/system/_resources/stream.py +++ b/temporalio/common/streams/system/_resources/stream.py @@ -166,7 +166,7 @@ async def iter_read_records( async def describe( self, ) -> DescribeStreamResponse: - """Describe the Stream: its status, newest offset, Retention and auto-close.""" + """Describe the Stream: its status, newest offset and Retention.""" request = DescribeStreamRequest( stream=self.stream, ) diff --git a/temporalio/common/streams/system/client.py b/temporalio/common/streams/system/client.py index c667a86b74..3e717241c8 100644 --- a/temporalio/common/streams/system/client.py +++ b/temporalio/common/streams/system/client.py @@ -483,7 +483,7 @@ async def iter_read_records( async def describe( self, ) -> DescribeStreamResponse: - """Describe the Stream: its status, newest offset, Retention and auto-close.""" + """Describe the Stream: its status, newest offset and Retention.""" request = DescribeStreamRequest( stream=self.stream, ) diff --git a/temporalio/common/streams/system/models.py b/temporalio/common/streams/system/models.py index fb7101d8f8..9b8504ab8d 100644 --- a/temporalio/common/streams/system/models.py +++ b/temporalio/common/streams/system/models.py @@ -23,12 +23,11 @@ @dataclasses.dataclass(slots=True) class CreateStreamRequest: - """Create a Stream idempotently. ALREADY_EXISTS for another Retention or auto-close.""" + """Create a Stream idempotently. ALREADY_EXISTS for another Retention.""" stream_id: str retention: datetime.timedelta | None = None withdraw_run: str | None = None - auto_close: datetime.timedelta | None = None owner: Producer | None = None close_with_run: bool | None = None ref: StreamRef | None = None @@ -60,9 +59,6 @@ def from_transfer_type( if value.HasField("retention") else None, withdraw_run=value.withdraw_run if bool(value.withdraw_run) else None, - auto_close=_support.duration_from_proto(value.auto_close) - if value.HasField("auto_close") - else None, owner=_ProducerTransferTypeConverter().from_transfer_type( value.owner, Producer ) @@ -89,8 +85,6 @@ def to_transfer_type( message.retention.CopyFrom(_support.duration_to_proto(value.retention)) if value.withdraw_run is not None: message.withdraw_run = value.withdraw_run - if value.auto_close is not None: - message.auto_close.CopyFrom(_support.duration_to_proto(value.auto_close)) if value.owner is not None: message.owner.CopyFrom( _ProducerTransferTypeConverter().to_transfer_type(value.owner) @@ -1245,7 +1239,6 @@ class DescribeStreamResponse: ) = None last_offset: int | None = None retention: datetime.timedelta | None = None - auto_close: datetime.timedelta | None = None class _DescribeStreamResponseTransferTypeConverter( @@ -1275,9 +1268,6 @@ def from_transfer_type( retention=_support.duration_from_proto(value.retention) if value.HasField("retention") else None, - auto_close=_support.duration_from_proto(value.auto_close) - if value.HasField("auto_close") - else None, ) @typing_extensions.override @@ -1296,8 +1286,6 @@ def to_transfer_type( message.last_offset = value.last_offset if value.retention is not None: message.retention.CopyFrom(_support.duration_to_proto(value.retention)) - if value.auto_close is not None: - message.auto_close.CopyFrom(_support.duration_to_proto(value.auto_close)) return message diff --git a/temporalio/common/streams/system/operations/create_stream.py b/temporalio/common/streams/system/operations/create_stream.py index f2f30d8324..947b5d3608 100644 --- a/temporalio/common/streams/system/operations/create_stream.py +++ b/temporalio/common/streams/system/operations/create_stream.py @@ -47,7 +47,6 @@ async def create_stream( stream_id: str, retention: datetime.timedelta | None = None, withdraw_run: str | None = None, - auto_close: datetime.timedelta | None = None, owner: Producer | None = None, close_with_run: bool | None = None, ref: StreamRef | None = None, @@ -58,8 +57,6 @@ async def create_stream( id. retention: The age after which a Record is dropped. Empty is the Store default. withdraw_run: Retry run only. The previous run, whose Records this withdraws. - auto_close: Close with no result after this long with no append. Empty never - auto-closes. owner: The Workflow run that owns the Stream. The Workflow transport sets it. close_with_run: Close with the owner run chain's result when it closes. Native only. @@ -69,7 +66,6 @@ async def create_stream( stream_id=stream_id, retention=retention, withdraw_run=withdraw_run, - auto_close=auto_close, owner=owner, close_with_run=close_with_run, ref=ref, From 602b1b6ff08051c02f1d3c03043fe463f276d3f8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 09:54:37 -0700 Subject: [PATCH 08/79] Pinned the streams crate to s23 and sdk-core to c8. s23 drops auto_close from the Stores and the protos, and c8 drops the cache-off eviction reason that only the run-end close used. --- temporalio/bridge/Cargo.lock | 2 +- temporalio/bridge/Cargo.toml | 2 +- temporalio/bridge/sdk-core | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 67939bd6cd..d09168f0e1 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -2938,7 +2938,7 @@ dependencies = [ [[package]] name = "temporalio-streams" version = "0.1.0" -source = "git+https://github.com/moetemp/sdk-streams?rev=879563a2f47212c10052423dbdfb15f44bcbeef8#879563a2f47212c10052423dbdfb15f44bcbeef8" +source = "git+https://github.com/moetemp/sdk-streams?rev=2866bc971b0419e8861108d4b392319dccd0472c#2866bc971b0419e8861108d4b392319dccd0472c" dependencies = [ "async-trait", "aws-lc-rs", diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index 22a5ce272c..683d6af213 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -94,7 +94,7 @@ url = "2.2" # crates, so the bridge sees one type of each. [dependencies.temporalio-streams] git = "https://github.com/moetemp/sdk-streams" -rev = "879563a2f47212c10052423dbdfb15f44bcbeef8" +rev = "2866bc971b0419e8861108d4b392319dccd0472c" default-features = false optional = true diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index b565c43354..320e725ff5 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit b565c433547496f2c14a911549c288a1076e72b6 +Subproject commit 320e725ff5e0bfd838089e293dba48a8f46954b3 From 66832cf9b920f2a2c13b876cf472b3d8c0f593a0 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 10:36:37 -0700 Subject: [PATCH 09/79] Made native the default Stream backend, with no registration. Every client, Worker and Replayer has it, and a registered NativeStreamBackend only configures it. A client builds its native runtime at its first Stream call, and a Worker loads Streams only when it has Workflows or Activities. --- temporalio/client/_client.py | 12 +-- temporalio/client/_workflow.py | 5 +- temporalio/common/streams/__init__.py | 14 +-- temporalio/common/streams/_activity_stream.py | 4 - temporalio/common/streams/_errors.py | 13 --- temporalio/common/streams/_native.py | 87 ++++++++++------- temporalio/common/streams/_plugin.py | 96 +++++++++++++------ temporalio/common/streams/_workflow.py | 9 +- temporalio/worker/_replayer.py | 22 +++-- temporalio/worker/_worker.py | 25 +++-- temporalio/workflow/_streams.py | 21 ++-- tests/contrib/streams/test_stream_client.py | 22 +++-- tests/contrib/streams/test_workflow_module.py | 51 +++++----- 13 files changed, 218 insertions(+), 163 deletions(-) diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index 176243bb55..0f8dbaf240 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -232,8 +232,9 @@ async def connect( fields are logged but still sent. Set a threshold to 0 to disable it. header_codec_behavior: Encoding behavior for headers sent by the client. stream_backends: The Stream backends of this client and its Workers, such as - :py:class:`temporalio.contrib.stream.RedisStreamBackend`. Names are unique. - With more than one, exactly one has ``default=True``. See + :py:class:`temporalio.contrib.stream.RedisStreamBackend`. The native backend + is there with no registration. Names are unique. With more than one, exactly + one has ``default=True``. With none marked, the default is ``native``. See :py:class:`temporalio.common.streams.StreamBackend`. .. warning:: @@ -936,12 +937,11 @@ def stream( id: The Stream id. stream: A handle of the Stream, such as one in a Record. Its id, owner and backend stay. - backend: The name of the backend. Defaults to the handle's, and else to the - registered backend with ``default=True``. + backend: The name of the backend. Defaults to the handle's, else to the + registered backend with ``default=True``, else to ``native``. Raises: - StreamConfigError: The client has no backend of that name, or none has - ``default=True`` for a call with no backend. + StreamConfigError: The client has no backend of that name. """ from temporalio.common.streams import _stream_client diff --git a/temporalio/client/_workflow.py b/temporalio/client/_workflow.py index 82735f0761..3280cd078c 100644 --- a/temporalio/client/_workflow.py +++ b/temporalio/client/_workflow.py @@ -207,11 +207,10 @@ def stream( Args: definition: The Stream definition class, renamed as the Workflow's create. backend: The name of the backend. Defaults to the registered backend with - ``default=True``. + ``default=True``, else to ``native``. Raises: - StreamConfigError: The client has no backend of that name, or none has - ``default=True`` for a call with no backend. + StreamConfigError: The client has no backend of that name. """ from temporalio.common.streams import _stream_client diff --git a/temporalio/common/streams/__init__.py b/temporalio/common/streams/__init__.py index 7664358f58..bbba04cc42 100644 --- a/temporalio/common/streams/__init__.py +++ b/temporalio/common/streams/__init__.py @@ -7,13 +7,15 @@ passes it to an Activity, a child Workflow or the next run as an argument, and the parameter's type gives the handle class of that context, such as :py:class:`ActivityStream`. -A :py:class:`StreamBackend` carries the calls and registers a backend name, such as -:py:class:`NativeStreamBackend` (``native``), whose Streams the Temporal server keeps, +A :py:class:`StreamBackend` carries the calls and registers a backend name. Every client, +Worker and Replayer has :py:class:`NativeStreamBackend` (``native``), whose Streams the +Temporal server keeps, with no registration. Register others with +``Client.connect(..., stream_backends=[...])``, such as :py:class:`temporalio.contrib.stream.RedisStreamBackend` (``redis``) and -:py:class:`temporalio.contrib.stream.testing.MemoryStreamBackend` (``memory``). Register -backends with ``Client.connect(..., stream_backends=[...])``. A handle carries the name of its -backend. With more than one registered backend, exactly one has ``default=True``, the one -that a create with no ``backend=`` uses. A Worker serves every backend of its client. +:py:class:`temporalio.contrib.stream.testing.MemoryStreamBackend` (``memory``). A handle +carries the name of its backend. With more than one registered backend, exactly one has +``default=True``. A create with no ``backend=`` uses the one with ``default=True``, else +``native``. A Worker serves every backend of its client. .. warning:: This package is experimental. diff --git a/temporalio/common/streams/_activity_stream.py b/temporalio/common/streams/_activity_stream.py index f3eb283211..6b7582944d 100644 --- a/temporalio/common/streams/_activity_stream.py +++ b/temporalio/common/streams/_activity_stream.py @@ -41,10 +41,6 @@ def _bound(handle: StreamHandle[Any]) -> _activity.ActivityStreams: "with stream_backends=." ) streams = attempt.get(handle.backend) - if streams is None and not handle.backend: - raise _errors.no_default_backend( - "Worker", list(attempt.backends), "a handle that carries its backend name" - ) if streams is None: raise _errors.StreamConfigError( f"The Stream {handle.id} is of the backend {handle.backend}. The Worker has " diff --git a/temporalio/common/streams/_errors.py b/temporalio/common/streams/_errors.py index 060ca022a3..1ca99b7f4d 100644 --- a/temporalio/common/streams/_errors.py +++ b/temporalio/common/streams/_errors.py @@ -2,8 +2,6 @@ from __future__ import annotations -from collections.abc import Sequence - import temporalio.exceptions import temporalio.service from temporalio.common.streams.proto.v1 import StreamErrorDetails @@ -106,17 +104,6 @@ class StreamConfigError(RuntimeError): """ -def no_default_backend( - owner: str, names: Sequence[str], hint: str -) -> StreamConfigError: - """The error of a call with no backend name when no backend has ``default=True``.""" - return StreamConfigError( - f"No Stream backend of the {owner} ({', '.join(names)}) has default=True, so the " - f"default is native, which the {owner} does not have. Mark a backend with " - f"default=True, or name one, such as with {hint}." - ) - - class ReadOnlyContextError(StreamError): """Workflow code that must not change state, such as a query handler, wrote to a Stream.""" diff --git a/temporalio/common/streams/_native.py b/temporalio/common/streams/_native.py index a794796e4d..b9eb97fcc3 100644 --- a/temporalio/common/streams/_native.py +++ b/temporalio/common/streams/_native.py @@ -4,6 +4,7 @@ import dataclasses import hashlib +import weakref from collections import OrderedDict from collections.abc import Callable, Sequence from datetime import timedelta @@ -289,13 +290,17 @@ async def _temporal_stream_describe( class NativeStreamBackend(ExtensionStreamBackend): """Carries Stream calls to the Temporal server, which keeps the Streams. + Every client, Worker and Replayer has it with no registration, under the name + ``native``, and it is the default backend unless a registered backend has + ``default=True``. Register one only to configure it, such as for its Retention, or to + make it the default next to another backend. + Workflow writes run as Local Activities, as on Redis, and the client's Stream runtime writes them to the server's ``temporal.api.streamservice.v1.StreamService``. Activity and - client calls go to the same service. Each client of the backend gets its own runtime on - its own connection, and the client's Workers share it. A page of a Workflow read is a - direct read in a Local Activity whose marker holds only its positions, and a replay reads - the page again. It registers the backend name ``native``. Register it next to - ``RedisStreamBackend`` to keep old Streams on Redis and create new ones on the server. + client calls go to the same service. Each client gets its own runtime on its own + connection, built at the first call, and the client's Workers share it. A page of a + Workflow read is a direct read in a Local Activity whose marker holds only its positions, + and a replay reads the page again on the connection of a client of the backend. .. warning:: This API is experimental. @@ -328,6 +333,11 @@ def __init__( config = ext_pb.NativeStreamConfig() if retention is not None: config.retention.FromTimedelta(retention) + # The calls of each connection. A client and its Workers share them, and a Replayer + # reads again on any of them. + self._connections: weakref.WeakKeyDictionary[ + temporalio.service.ServiceClient, _ExtensionCalls + ] = weakref.WeakKeyDictionary() super().__init__( "temporalio.NativeStreamBackend", False, config, None, name, default ) @@ -335,19 +345,28 @@ def __init__( def _new_backend( self, service: temporalio.service.ServiceClient ) -> _ExtensionCalls: - # Each client gets its own runtime on its own connection, so a call goes on the - # connection of the client that makes it. - backend = _ExtensionCalls(_ConnectionRuntime(service, self._config)) - self._backends.add(backend) + # Each connection gets its own runtime, so a call goes on the connection of the + # client that makes it. The runtime comes at the first call. + backend = self._connections.get(service) + if backend is None: + backend = _ExtensionCalls(_ConnectionRuntime(service, self._config)) + self._connections[service] = backend + self._backends.add(backend) return backend + def backend_for(self, client: temporalio.client.Client) -> _ExtensionCalls: + """The Stream runtime of ``client``'s connection.""" + return self._new_backend(client.service_client) + def _any_backend(self) -> _ExtensionCalls: - # A Replayer has no client, so a replay reads again on a client of the backend. - for backend in self._backends: + # A Replayer has no client, so a replay reads again on a connection of the backend. + for backend in list(self._connections.values()): return backend raise StreamConfigError( - "A native replay reads the Stream again, so register the backend on a client " - "that connected" + "A native replay reads each page of a Workflow read again, so it needs a client " + "of the native backend in this process. Register the Replayer's " + "NativeStreamBackend on a client too, or run a Worker or a Stream call of a " + "client first." ) async def _read_result( @@ -392,29 +411,9 @@ def _activities(self) -> list[Any]: def _configure_client( self, config: temporalio.client.ClientConfig ) -> temporalio.client.ClientConfig: - # Wrap the client's codec, so a Workflow read's positions turn back into its page. - config = super()._configure_client(config) - interceptor = self._interceptor_of(config.get("interceptors", [])) - backend = interceptor.backend if interceptor is not None else None - converter = ( - config.get("data_converter") or temporalio.converter.DataConverter.default - ) - config["data_converter"] = _wrap( - (lambda: backend) if backend is not None else self._any_backend, - converter, - ) - return config - - def _configure_replayer( - self, config: temporalio.worker.ReplayerConfig - ) -> temporalio.worker.ReplayerConfig: - # Wrap the Replayer's codec, as the client's. A replay reads again on the connection - # of a client of the backend. - config = super()._configure_replayer(config) - converter = ( - config.get("data_converter") or temporalio.converter.DataConverter.default - ) - config["data_converter"] = _wrap(self._any_backend, converter) + # The calls come from the connection, not from an interceptor. Knowing the connection + # now lets a Replayer with this backend read on it. + self._new_backend(config["service_client"]) return config @@ -438,13 +437,27 @@ async def describe( return describe -def _wrap( +def wrap( backend: Callable[[], _ExtensionCalls], converter: temporalio.converter.DataConverter, ) -> temporalio.converter.DataConverter: + """The converter with a codec step that turns a native read's positions back into its + page, reading it again with ``backend`` when the page is not in the cache. + """ if isinstance(converter.payload_codec, _DirectReadCodec): return converter codec = converter.payload_codec return dataclasses.replace( converter, payload_codec=_DirectReadCodec(backend, codec, codec) ) + + +_implicit: NativeStreamBackend | None = None + + +def implicit() -> NativeStreamBackend: + """The native backend of the clients, Workers and Replayers that register none.""" + global _implicit + if _implicit is None: + _implicit = NativeStreamBackend() + return _implicit diff --git a/temporalio/common/streams/_plugin.py b/temporalio/common/streams/_plugin.py index 72f5333d4f..02ee344331 100644 --- a/temporalio/common/streams/_plugin.py +++ b/temporalio/common/streams/_plugin.py @@ -6,12 +6,14 @@ from typing import Any import temporalio.client +import temporalio.converter import temporalio.worker from temporalio.common.streams import _errors, _ids from temporalio.common.streams._backend import BackendCalls from temporalio.plugin import SimplePlugin -# The default backend when no registered backend has default=True. +# The default backend when no registered backend has default=True. Every client, Worker and +# Replayer has it with no registration. NATIVE = _ids.NATIVE_BACKEND @@ -20,11 +22,11 @@ class StreamBackend(SimplePlugin): Register backends on the client with ``Client.connect(..., stream_backends=[...])``. A Worker takes its client's backends, and a ``Replayer`` takes ``stream_backends=`` too. - Names are unique. With more than one registered backend, exactly one has ``default=True``. - With none marked, the default is ``native``, so a create with no backend needs a - registered ``NativeStreamBackend``, else it raises ``StreamConfigError``. Mark a backend - with ``default=True``, or use the calls of its package, such as - ``temporalio.contrib.stream.workflow.create_stream``. + Every client, Worker and Replayer also has the native backend, named ``native``, with no + registration. Register a ``NativeStreamBackend`` only to configure it, such as for its + Retention. Names are unique, and only the native backend is named ``native``. With more + than one registered backend, exactly one has ``default=True``. With none marked, the + default is ``native``. A backend is not a plugin, so passing it in ``plugins=`` raises ``StreamConfigError``. @@ -133,8 +135,9 @@ def check_backends(backends: Sequence[object], owner: str) -> list[StreamBackend Raises: TypeError: An entry is not a ``StreamBackend``. - StreamConfigError: Two backends have one name, or more than one backend and not - exactly one with ``default=True``. + StreamConfigError: Two backends have one name, a backend that is not native has the + name ``native``, or more than one backend and not exactly one with + ``default=True``. """ checked: list[StreamBackend] = [] for backend in backends: @@ -144,6 +147,11 @@ def check_backends(backends: Sequence[object], owner: str) -> list[StreamBackend raise _errors.StreamConfigError( f"The {owner} has two Stream backends named {backend.backend_name}" ) + if backend.backend_name == NATIVE and backend._kind != NATIVE: + raise _errors.StreamConfigError( + f"The name {NATIVE} is the native backend's, so {type(backend).__name__} " + "takes another name" + ) checked.append(backend) if len(checked) < 2: return checked @@ -163,6 +171,15 @@ def default_name(backends: Sequence[StreamBackend]) -> str: return next((b.backend_name for b in backends if b.default), NATIVE) +def with_native(backends: Sequence[StreamBackend]) -> list[StreamBackend]: + """The registered backends, and the native one when none of them is named ``native``.""" + from temporalio.common.streams import _native + + if any(b.backend_name == NATIVE for b in backends): + return list(backends) + return [*backends, _native.implicit()] + + def configure_client( config: temporalio.client.ClientConfig, ) -> temporalio.client.ClientConfig: @@ -173,10 +190,11 @@ def configure_client( def configure_worker( - backends: Sequence[StreamBackend], config: temporalio.worker.WorkerConfig + config: temporalio.worker.WorkerConfig, ) -> temporalio.worker.WorkerConfig: - """Add the parts of the client's backends to a Worker.""" - for backend in backends: + """Add the parts of the client's backends and of the native one to a Worker.""" + client = config["client"] # type: ignore[reportTypedDictNotRequiredAccess] + for backend in with_native(client_backends(client)): config = backend._configure_worker(config) return config @@ -184,41 +202,65 @@ def configure_worker( def configure_replayer( config: temporalio.worker.ReplayerConfig, ) -> temporalio.worker.ReplayerConfig: - """Check the Replayer's backends, and add the parts of each to it.""" - for backend in check_backends(config.get("stream_backends", ()), "Replayer"): + """Check the Replayer's backends, and add the parts of each and of the native one.""" + backends = check_backends(config.get("stream_backends", ()), "Replayer") + for backend in with_native(backends): config = backend._configure_replayer(config) return config +def worker_converter( + client: temporalio.client.Client, converter: temporalio.converter.DataConverter +) -> temporalio.converter.DataConverter: + """The converter of a Worker's Workflows, whose codec step turns a native read's + positions back into its page. + """ + from temporalio.common.streams import _native + + native = next( + b for b in with_native(client_backends(client)) if b.backend_name == NATIVE + ) + assert isinstance(native, _native.NativeStreamBackend) + return _native.wrap(lambda: native.backend_for(client), converter) + + +def replayer_converter( + config: temporalio.worker.ReplayerConfig, + converter: temporalio.converter.DataConverter, +) -> temporalio.converter.DataConverter: + """The converter of a Replayer's Workflows, as :py:func:`worker_converter`. A replay + reads a native page again on a connection that the native backend served. + """ + from temporalio.common.streams import _native + + native = next( + b + for b in with_native(config.get("stream_backends", ())) + if b.backend_name == NATIVE + ) + assert isinstance(native, _native.NativeStreamBackend) + return _native.wrap(native._any_backend, converter) + + def client_backends(client: temporalio.client.Client) -> list[StreamBackend]: """The backends registered on a client.""" return list(client.config(active_config=True).get("stream_backends", ())) def client_backend( - client: temporalio.client.Client, name: str | None, *, hint: str = "backend=" + client: temporalio.client.Client, name: str | None ) -> tuple[str, BackendCalls]: """The name and the calls of a client's backend. None picks the default backend. Raises: - StreamConfigError: The client has no backend of that name, or none has - ``default=True`` for a call with no name. + StreamConfigError: The client has no backend of that name. """ - backends = client_backends(client) + backends = with_native(client_backends(client)) wanted = name or default_name(backends) for backend in backends: if backend.backend_name == wanted: return wanted, backend.backend_for(client) - if not backends: - raise _errors.StreamConfigError( - "The client has no Stream backend. Register one with " - "Client.connect(..., stream_backends=[...])." - ) - if not name: - raise _errors.no_default_backend( - "client", [b.backend_name for b in backends], hint - ) raise _errors.StreamConfigError( - f"The client has no Stream backend {name}. It has " + f"The client has no Stream backend {wanted}. It has " f"{', '.join(b.backend_name for b in backends)}." ) diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index a5e4b0be72..9af59be4f4 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -167,13 +167,8 @@ def _check_backend(backend: str | None, action: str) -> str: raise no_worker_backend() names = [b.name for b in backends] if not backend: - # The default is the backend with default=True, else native. - backend = next((b.name for b in backends if b.default), _ids.NATIVE_BACKEND) - if backend not in names: - raise _errors.no_default_backend( - "Worker", names, "temporalio.contrib.stream.workflow.create_stream" - ) - return backend + # The default is the backend with default=True, else native, which every Worker has. + return next((b.name for b in backends if b.default), _ids.NATIVE_BACKEND) if backend not in names: raise _errors.StreamConfigError( f"Can't {action} a Stream of the backend {backend}. The Worker has " diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 92a05909ca..349887dc77 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -68,8 +68,10 @@ def __init__( most of the arguments. Most of the same arguments need to be passed to the replayer that were passed to the worker when the workflow originally ran. ``stream_backends`` are the Stream backends of the Worker's client, - such as :py:class:`temporalio.contrib.stream.RedisStreamBackend`. A - replay makes no Stream call, so they need not reach their Store. + such as :py:class:`temporalio.contrib.stream.RedisStreamBackend`. The + native backend is there with no registration. A replay makes no Stream + call, except that it reads each page of a native Workflow read again on + the connection of a client of the native backend in this process. Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that @@ -102,10 +104,11 @@ def __init__( self.plugins = plugins for plugin in plugins: self._config = plugin.configure_replayer(self._config) - if self._config.get("stream_backends"): - from temporalio.common.streams import _plugin as _stream_plugin + # A replay of Workflow code that uses Streams needs the native backend and the + # others of the Worker's client. + from temporalio.common.streams import _plugin as _stream_plugin - self._config = _stream_plugin.configure_replayer(self._config) + self._config = _stream_plugin.configure_replayer(self._config) # Validate workflows after plugin configuration if not self._config.get("workflows"): @@ -259,9 +262,14 @@ def on_eviction_hook( runtime = ( self._config.get("runtime") or temporalio.runtime.Runtime.default() ) - data_converter = ( + from temporalio.common.streams import _plugin as _stream_plugin + + # A native Stream read's marker holds positions, which the codec step turns back + # into the page. + data_converter = _stream_plugin.replayer_converter( + self._config, self._config.get("data_converter") - or temporalio.converter.DataConverter.default + or temporalio.converter.DataConverter.default, ) workflow_worker = _WorkflowWorker( bridge_worker=lambda: bridge_worker, diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 8b30d9fc42..9dd0846fa7 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -407,18 +407,18 @@ def __init__( plugins = plugins_from_client + list(plugins) self._initial_config = config.copy() - # The Worker takes its client's Stream backends, before the plugins as a client's - # plugins come first. - stream_backends = client.config(active_config=True).get("stream_backends") - if stream_backends: - from temporalio.common.streams import _plugin as _stream_plugin - - config = _stream_plugin.configure_worker(stream_backends, config) - self._plugins = plugins for plugin in plugins: config = plugin.configure_worker(config) + # Workflows and Activities may use Streams with no registration, on the native backend, + # so the Worker takes the native backend and its client's others. A Worker of only + # Nexus services loads nothing of Streams. + if config.get("workflows") or config.get("activities"): + from temporalio.common.streams import _plugin as _stream_plugin + + config = _stream_plugin.configure_worker(config) + self._init_from_config(client, config) def _init_from_config(self, client: temporalio.client.Client, config: WorkerConfig): @@ -554,6 +554,8 @@ def check_activity(activity: str): ) self._activity_worker.assert_activity_valid(activity) + from temporalio.common.streams import _plugin as _stream_plugin + self._workflow_worker = _WorkflowWorker( bridge_worker=lambda: self._bridge_worker, namespace=config["client"].namespace, # type: ignore[reportTypedDictNotRequiredAccess] @@ -565,7 +567,12 @@ def check_activity(activity: str): ), workflow_runner=config["workflow_runner"], # type: ignore[reportTypedDictNotRequiredAccess] unsandboxed_workflow_runner=config["unsandboxed_workflow_runner"], # type: ignore[reportTypedDictNotRequiredAccess] - data_converter=client_config["data_converter"], + # A native Stream read's marker holds positions, which the codec step turns + # back into the page. + data_converter=_stream_plugin.worker_converter( + config["client"], # type: ignore[reportTypedDictNotRequiredAccess] + client_config["data_converter"], + ), interceptors=interceptors, workflow_failure_exception_types=config[ "workflow_failure_exception_types" diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 301e2a69c2..771da50a0d 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -25,8 +25,9 @@ def create_stream( Its id is ``::``, so a client finds it from the Workflow handle with no call. The create is one Local Activity, and its error raises here. Only the - run chain closes the Stream. On Redis, the end of the run chain closes nothing, so a Stream - that nobody closes stays open until its ``safety_ttl``. + run chain closes the Stream. The native server closes it when the run chain ends, with the + result or the failure of its last run. On Redis, the end of the run chain closes nothing, + so a Stream that nobody closes stays open until its ``safety_ttl``. Await the call for the handle, or use it in ``async with``, which closes the Stream at the end of the block:: @@ -42,16 +43,16 @@ def create_stream( per turn, for example, pass ``streams.with_name(Events, f"turn-{n}")``. retention: How long the Records stay after the close. Defaults to the backend's. backend: The name of the backend. Defaults to the registered backend with - ``default=True``. The default is part of the Workflow's code, since a replay must - create on the same backend. To move a Workflow to another backend, gate it with a - patch, for example ``backend="redis-b" if workflow.patched("move") else "redis"``. + ``default=True``, else to ``native``. The default is part of the Workflow's code, + since a replay must create on the same backend. To move a Workflow to another + backend, gate it with a patch, for example + ``backend="native" if workflow.patched("move") else "redis"``. Raises: StreamError: The backend refused the create, such as for another Retention. ReadOnlyContextError: The code must not change state, such as a query handler. - StreamConfigError: No Stream backend is on the Worker, none of that name, or none - has ``default=True`` for a create with no backend. It fails the Workflow Task, - not the run. + StreamConfigError: The Worker has no backend of that name. It fails the Workflow + Task, not the run. """ from temporalio.common.streams import _workflow @@ -94,10 +95,10 @@ def stream_handle( Args: definition: The Stream definition class, renamed as for the create. backend: The backend of the Stream. Defaults to the one that the run chain created it - on, and else to the registered backend with ``default=True``. + on, else to the registered backend with ``default=True``, else to ``native``. Raises: - StreamConfigError: No backend is given, known or marked as the default. + StreamConfigError: The Worker has no backend of that name. """ from temporalio.common.streams import _workflow diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index f4c84e3785..d5f25a66f3 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -45,9 +45,8 @@ async def test_unknown_backend_raises_before_any_call(client: Client): assert stream.backend == "fake" -async def test_client_without_backend_has_no_transport(client: Client): - with pytest.raises(streams.StreamConfigError, match="no Stream backend"): - client.stream(Events, id="chat-1") +async def test_client_without_backend_has_only_native(client: Client): + assert client.stream(Events, id="chat-1").backend == "native" with pytest.raises(streams.StreamConfigError, match="no Redis Stream backend"): await redis.create(client, Events, id="chat-1") @@ -374,16 +373,25 @@ async def test_backend_is_not_a_plugin(client: Client): async def test_default_is_native_with_no_default_backend(client: Client): + # Every client has the native backend with no registration, and makes no call for it. + assert client.stream(Events, id="d-2").backend == "native" config = client.config() config["stream_backends"] = [MemoryStreamBackend()] one = Client(**config) - with pytest.raises(streams.StreamConfigError, match="default is native"): - one.stream(Events, id="d-2") - with pytest.raises(streams.StreamConfigError, match="default is native"): - one.get_workflow_handle("agent-3").stream(Events) + assert one.stream(Events, id="d-2").backend == "native" + assert one.get_workflow_handle("agent-3").stream(Events).backend == "native" assert one.stream(Events, id="d-2", backend="memory").backend == "memory" # The Redis create picks the client's only Redis backend, and memory stands in for it. assert (await redis.create(one, Events, id="d-2")).backend == "memory" + # A registered NativeStreamBackend configures native, and only it takes that name. + config["stream_backends"] = [ + streams.NativeStreamBackend(default=True), + MemoryStreamBackend(), + ] + assert Client(**config).stream(Events, id="d-2").backend == "native" + config["stream_backends"] = [MemoryStreamBackend(name="native")] + with pytest.raises(streams.StreamConfigError, match="native backend's"): + Client(**config) async def test_stream_takes_the_handle_backend_or_the_default(client: Client): diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index af307dfae7..2bb2c76a11 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -283,21 +283,17 @@ async def run(self, backend: str | None) -> None: async def test_worker_config_errors_fail_the_task( client: Client, stream_client: Client ): - # A Worker with no plugin, or with another backend, fails the Workflow Task, so a Worker - # with the right config can take the run on. - for task_client, backend, message in ( - (client, None, "No Stream backend"), - (stream_client, "elsewhere", "backend elsewhere"), - ): - async with new_worker(task_client, CreatesOn) as worker: - handle = await task_client.start_workflow( - CreatesOn.run, - backend, - id=f"wf-{uuid.uuid4()}", - task_queue=worker.task_queue, - ) - await assert_task_fail_eventually(handle, message_contains=message) - await handle.terminate() + # Every Worker has the native backend, so only a backend that the Worker lacks fails the + # Workflow Task, and a Worker with the right config can take the run on. + async with new_worker(stream_client, CreatesOn) as worker: + handle = await stream_client.start_workflow( + CreatesOn.run, + "elsewhere", + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await assert_task_fail_eventually(handle, message_contains="backend elsewhere") + await handle.terminate() @workflow.defn @@ -568,8 +564,8 @@ async def test_merge_that_the_codec_grows_past_the_batch_limit(stream_client: Cl @workflow.defn class CreatesOnTheDefault: @workflow.run - async def run(self) -> None: - await workflow.create_stream(Events) + async def run(self) -> str: + return workflow.stream_handle(Events).backend or "" @workflow.defn @@ -583,25 +579,26 @@ async def run(self) -> str: async def test_default_is_native_with_no_default_backend(client: Client): - # With no backend marked default=True, the default is native, which this SDK lacks. The - # Redis calls still pick the registered Redis backend, and the memory one stands in for it. + # With no backend marked default=True, the default is native, which every Worker has. + # The Redis calls still pick the registered Redis backend, and the memory one stands in + # for it. config = client.config() config["stream_backends"] = [MemoryStreamBackend()] memory_client = Client(**config) async with new_worker(memory_client, CreatesOnTheDefault, CreatesOnRedis) as worker: - handle = await memory_client.start_workflow( - CreatesOnTheDefault.run, - id=f"wf-{uuid.uuid4()}", - task_queue=worker.task_queue, + assert ( + await memory_client.execute_workflow( + CreatesOnTheDefault.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + == "native" ) - await assert_task_fail_eventually(handle, message_contains="default is native") - await handle.terminate() on_redis = await memory_client.start_workflow( CreatesOnRedis.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await on_redis.result() == "memory" - with pytest.raises(streams.StreamConfigError, match="default=True"): - on_redis.stream(Events) + assert on_redis.stream(Events).backend == "native" stream = on_redis.stream(Events, backend="memory") items = [item async for item in stream.subscribe()] assert data(items) == ["on redis"] From 95da634c1f3772c3e730e4d098bc31c682fdf599 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 10:38:04 -0700 Subject: [PATCH 10/79] Sent close_with_run on each Workflow create, for the native server. The server closes a native Stream when its run chain ends, and Redis ignores the field. The owner check of an empty page is now the backend's, so the close-with-run flag goes. --- temporalio/common/streams/_extension.py | 16 ++++++------ temporalio/common/streams/_native.py | 8 +++++- temporalio/common/streams/_workflow.py | 2 ++ tests/contrib/streams/test_native_server.py | 27 +++++++++++++++++++++ 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index c3a5fd24ab..12407ea64a 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -68,8 +68,6 @@ class ExtensionStreamBackend(StreamBackend): """ _outbound_class: type[_WorkflowOutbound] - # Whether the Store closes a Stream with its owner's run chain on the server. - _server_closes_with_run = False def __init__( self, @@ -116,6 +114,11 @@ async def _read_result( payload.metadata[_OWNER_ENDED_KEY] = b"1" return temporalio.common.RawValue(payload) + async def _owner_ended(self, ref: stream_pb.StreamRef) -> bool: + # A reader at the end of a Stream whose owner ended with no close would wait until + # the Stream expires, so an empty page tells it. + return await _owner_chain_ended(ref) + def _activities(self) -> list[Callable[..., Any]]: # The Local Activities that Workflow reads run, so the Worker must have them. return [self._read_activity] @@ -256,14 +259,11 @@ async def read_page(request: stream_pb.ReadRecordsRequest) -> Any: ) except _errors.StreamError as err: raise _activity_error(err) from None - # A reader at the end of a Stream whose owner ended with no close would wait until - # the Stream expires, so the page tells it. owner_ended = ( - not backend._server_closes_with_run - and not page.records + not page.records and not page.done and bool(request.stream.workflow_id) - and await _owner_ended(request.stream) + and await backend._owner_ended(request.stream) ) return await backend._read_result( run_id, request, page, owner_ended, streams.converter @@ -282,7 +282,7 @@ def _activity_error(err: _errors.StreamError) -> temporalio.exceptions.Applicati ) -async def _owner_ended(ref: stream_pb.StreamRef) -> bool: +async def _owner_chain_ended(ref: stream_pb.StreamRef) -> bool: # The run chain ended when the Workflow id's newest run is closed, or is of another chain. try: description = ( diff --git a/temporalio/common/streams/_native.py b/temporalio/common/streams/_native.py index b9eb97fcc3..4d67bdb7d1 100644 --- a/temporalio/common/streams/_native.py +++ b/temporalio/common/streams/_native.py @@ -308,7 +308,6 @@ class NativeStreamBackend(ExtensionStreamBackend): _kind = "native" _outbound_class = _NativeWorkflowOutbound - _server_closes_with_run = True _activity_streams = True # The server's `stream.maxBatchBytes` default, below the crate's 4 MiB. _max_batch_bytes = 2 * 1024 * 1024 @@ -405,6 +404,13 @@ async def _read_result( _pages.popitem(last=False) return temporalio.common.RawValue(positions) + async def _owner_ended( + self, + ref: stream_pb.StreamRef, # pyright: ignore[reportUnusedParameter] + ) -> bool: + # The server closes a Workflow's Stream when its run chain ends. + return False + def _activities(self) -> list[Any]: return [*super()._activities(), _describe_activity(self)] diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 9af59be4f4..7540a7d958 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -638,6 +638,8 @@ def _create( ) if retention is not None: request.retention.FromTimedelta(retention) + # The native server closes the Stream when the run chain ends. Redis ignores it. + request.close_with_run = True run_stream.backend = stream.backend return stream, _write(stream.id, CREATE_ACTIVITY, request, backend_name) diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index 97498246d5..1ee4ed6d0f 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -154,6 +154,33 @@ async def test_redis_and_native_on_a_native_server( assert items[-1].result == ["redis", "native"] +@workflow.defn +class LeavesItsStreamOpen: + @workflow.run + async def run(self) -> str: + stream = await workflow.create_stream(Events) + await stream.topic(Events.tokens).publish("a") + return "done" + + +async def test_the_server_closes_a_stream_when_its_run_chain_ends( + native_client: Client, +): + # A Workflow's create asks the server to close the Stream when the run chain ends, with + # the result of its last run. + async with new_worker(native_client, LeavesItsStreamOpen) as worker: + handle = await native_client.start_workflow( + LeavesItsStreamOpen.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await handle.result() == "done" + items = [item async for item in handle.stream(Events).subscribe()] + assert [item.data for item in items if isinstance(item, streams.Record)] == ["a"] + assert isinstance(items[-1], streams.Closed) and items[-1].result == "done" + assert items[-1].status == streams.CloseStatus.COMPLETED + + @workflow.defn class WritesByBareId: @workflow.run From ea642184d29d149c0bc71d93f0f94c54b49e126f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 10:39:55 -0700 Subject: [PATCH 11/79] Added temporalio.activity.create_stream and ActivityHandle.stream. A Standalone Activity owns its Stream on the native backend, which closes it when the Activity ends. A client binds it from the Activity handle with no call. --- temporalio/activity.py | 44 +++++++++++++++++++ temporalio/client/_activity.py | 28 ++++++++++++ temporalio/common/streams/_activity_stream.py | 8 ++-- temporalio/common/streams/_plugin.py | 20 ++++++--- temporalio/common/streams/_stream_client.py | 35 ++++++++++++++- .../contrib/streams/test_activity_handles.py | 31 +++++++++++++ tests/contrib/streams/test_stream_client.py | 18 ++++++++ 7 files changed, 174 insertions(+), 10 deletions(-) diff --git a/temporalio/activity.py b/temporalio/activity.py index 434f707bdf..09a0db712f 100644 --- a/temporalio/activity.py +++ b/temporalio/activity.py @@ -21,6 +21,7 @@ TYPE_CHECKING, Any, NoReturn, + TypeVar, overload, ) @@ -37,6 +38,9 @@ if TYPE_CHECKING: from temporalio.client import Client + from temporalio.common.streams import ActivityStream, Creating + +_StreamDefinitionT = TypeVar("_StreamDefinitionT") @overload @@ -515,6 +519,46 @@ def metric_meter() -> temporalio.common.MetricMeter: return _Context.current().metric_meter +def create_stream( + definition: type[_StreamDefinitionT], + *, + retention: timedelta | None = None, + backend: str | None = None, +) -> Creating[ActivityStream[_StreamDefinitionT]]: + """Create a Stream that this Standalone Activity owns, on the native backend. + + Its id is ``::``, so a client finds it from the + Activity handle with :py:meth:`temporalio.client.ActivityHandle.stream` and no call. The + Stream goes on across the Activity's retries, and the server closes it when the Activity + ends, with its status and result. Await the call for the handle, or use it in + ``async with``, which closes the Stream at the end of the block:: + + async with activity.create_stream(Events) as stream: + await stream.topic(Events.tokens).publish("hi") + + Only the native backend keeps the Stream of a Standalone Activity, since nothing else + would close it when the Activity ends. + + .. warning:: + This API is experimental. + + Args: + definition: The Stream definition class. Its name is the Stream name. + retention: How long the Records stay after the close. Defaults to the backend's. + backend: The name of a native backend. Defaults to ``native``. + + Raises: + StreamConfigError: The Activity runs for a Workflow, which owns its Streams, or the + backend is not native. No call is made. + StreamClosedError: The Stream was created and closed before. + """ + from temporalio.common.streams import _activity_stream + + return _activity_stream.create_stream( + definition, retention=retention, backend=backend + ) + + class LoggerAdapter(logging.LoggerAdapter): """Adapter that adds details to the log about the running activity. diff --git a/temporalio/client/_activity.py b/temporalio/client/_activity.py index df7cd5c283..86824a5bb2 100644 --- a/temporalio/client/_activity.py +++ b/temporalio/client/_activity.py @@ -57,9 +57,13 @@ ) if TYPE_CHECKING: + from temporalio.common.streams import ClientStream + from ._client import Client from ._interceptor import ListActivitiesInput +_StreamDefinitionT = TypeVar("_StreamDefinitionT") + class ActivityExecutionAsyncIterator: """Asynchronous iterator for activity execution values. @@ -1003,6 +1007,30 @@ def run_id(self) -> str | None: """Run ID of the activity.""" return self._run_id + def stream( + self, definition: type[_StreamDefinitionT], *, backend: str | None = None + ) -> ClientStream[_StreamDefinitionT]: + """A Stream that this Standalone Activity created, by its definition. + + It builds the Stream id from the Activity id and run id, so it makes no call. Only the + native backend keeps the Stream of a Standalone Activity, which + :py:func:`temporalio.activity.create_stream` creates. + + .. warning:: + This API is experimental. + + Args: + definition: The Stream definition class, renamed as the Activity's create. + backend: The name of a native backend. Defaults to ``native``. + + Raises: + StreamConfigError: The client has no backend of that name, or it is not native. + ValueError: The handle has no run id. + """ + from temporalio.common.streams import _stream_client + + return _stream_client.activity_stream(self, definition, backend=backend) + async def result( self, *, diff --git a/temporalio/common/streams/_activity_stream.py b/temporalio/common/streams/_activity_stream.py index 6b7582944d..f1600d34bf 100644 --- a/temporalio/common/streams/_activity_stream.py +++ b/temporalio/common/streams/_activity_stream.py @@ -385,11 +385,11 @@ def create_stream( "the Streams that it receives." ) attempt = _activity.current.get() - streams = attempt.get(backend) if attempt is not None else None + # Only a native backend keeps the Stream, so the default rule doesn't apply. + name = backend or _ids.NATIVE_BACKEND + streams = attempt.backends.get(name) if attempt is not None else None if streams is None: - raise _errors.StreamConfigError( - f"No Stream backend {backend or 'by default'} runs this Activity" - ) + raise _errors.StreamConfigError(f"No Stream backend {name} runs this Activity") if not streams.activity_streams: raise _errors.StreamConfigError( f"The backend {streams.backend_name} keeps no Stream of a Standalone Activity, " diff --git a/temporalio/common/streams/_plugin.py b/temporalio/common/streams/_plugin.py index 02ee344331..ea88542646 100644 --- a/temporalio/common/streams/_plugin.py +++ b/temporalio/common/streams/_plugin.py @@ -247,10 +247,8 @@ def client_backends(client: temporalio.client.Client) -> list[StreamBackend]: return list(client.config(active_config=True).get("stream_backends", ())) -def client_backend( - client: temporalio.client.Client, name: str | None -) -> tuple[str, BackendCalls]: - """The name and the calls of a client's backend. None picks the default backend. +def backend_of(client: temporalio.client.Client, name: str | None) -> StreamBackend: + """A client's backend. None picks the default backend. Raises: StreamConfigError: The client has no backend of that name. @@ -259,8 +257,20 @@ def client_backend( wanted = name or default_name(backends) for backend in backends: if backend.backend_name == wanted: - return wanted, backend.backend_for(client) + return backend raise _errors.StreamConfigError( f"The client has no Stream backend {wanted}. It has " f"{', '.join(b.backend_name for b in backends)}." ) + + +def client_backend( + client: temporalio.client.Client, name: str | None +) -> tuple[str, BackendCalls]: + """The name and the calls of a client's backend. None picks the default backend. + + Raises: + StreamConfigError: The client has no backend of that name. + """ + backend = backend_of(client, name) + return backend.backend_name, backend.backend_for(client) diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index 31642487b9..db5ea4fa85 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -25,7 +25,7 @@ from temporalio.common.streams._defn import TopicDefinition, definition_of from temporalio.common.streams._handle import Creating, StreamHandle, exit_failure from temporalio.common.streams._items import End -from temporalio.common.streams._plugin import client_backend +from temporalio.common.streams._plugin import backend_of, client_backend from temporalio.common.streams._reader import Item, Position, Subscription from temporalio.common.streams._writer import StreamWriter from temporalio.common.streams.system import models @@ -517,3 +517,36 @@ def workflow_stream( return stream._bind(client, calls) stream._stream_name = stream_name return stream._bind(client, calls, lambda: _first_run(handle)) + + +def activity_stream( + handle: temporalio.client.ActivityHandle[Any], + definition: type[E], + *, + backend: str | None, +) -> ClientStream[E]: + """A Stream that a Standalone Activity created. See + :py:meth:`temporalio.client.ActivityHandle.stream`. + """ + stream_name = definition_of(definition).name + client = handle._client + # Only a native backend keeps the Stream, so the default rule doesn't apply. + stream_backend = backend_of(client, backend or _ids.NATIVE_BACKEND) + if not stream_backend._activity_streams: + raise _errors.StreamConfigError( + f"The backend {stream_backend.backend_name} keeps no Stream of a Standalone " + "Activity. Use the native backend." + ) + if not handle.run_id: + raise ValueError( + "The Activity handle has no run id, so the Stream id is unknown. Use the " + "handle that start_activity returns." + ) + stream = ClientStream( + id=_ids.scoped_stream_id(handle.id, handle.run_id, stream_name), + backend=stream_backend.backend_name, + definition=definition, + activity_id=handle.id, + activity_run_id=handle.run_id, + ) + return stream._bind(client, stream_backend.backend_for(client)) diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index c16ceedcb6..9e6691de77 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -607,3 +607,34 @@ def call() -> None: assert stopped.wait(5) assert errors and isinstance(errors[0], CancelledError) loop.call_soon_threadsafe(loop.stop) + + +@activity.defn +async def creates_its_own() -> str: + try: + await activity.create_stream(Events) + except streams.StreamConfigError as err: + return str(err) + return "created" + + +@workflow.defn +class RunsCreatesItsOwn: + @workflow.run + async def run(self) -> str: + return await workflow.execute_activity( + creates_its_own, start_to_close_timeout=timedelta(seconds=10) + ) + + +async def test_only_a_standalone_activity_creates_a_stream(stream_client: Client): + # A Workflow owns the Streams of its Activities, so its Activity creates none. + async with new_worker( + stream_client, RunsCreatesItsOwn, activities=[creates_its_own] + ) as worker: + message = await stream_client.execute_workflow( + RunsCreatesItsOwn.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert "Only a Standalone Activity" in message diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index d5f25a66f3..9e1df3e7ed 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -10,6 +10,7 @@ from temporalio import workflow from temporalio.client import Client from temporalio.common import streams +from temporalio.common.streams import _plugin from temporalio.contrib import stream as redis from temporalio.contrib.stream import RedisStreamBackend from temporalio.contrib.stream.testing import MemoryStreamBackend @@ -435,3 +436,20 @@ async def test_workflow_environment_takes_stream_backends(): items = [item async for item in handle.stream(Events).subscribe()] data = [item.data for item in items if isinstance(item, streams.Record)] assert data == ["from the environment"] + + +async def test_only_the_native_backend_keeps_a_standalone_activitys_stream( + stream_client: Client, +): + # Nothing closes a Redis Stream when its Activity ends, so it would stay open. So an + # Activity's Stream is native unless the call names another native backend. + handle = stream_client.get_activity_handle("activity", run_id="run") + stream = handle.stream(Events) + assert stream.backend == "native" + assert (stream.activity_id, stream.activity_run_id) == ("activity", "run") + lane, _ = _plugin.client_backend(stream_client, None) + if lane != "native": + with pytest.raises(streams.StreamConfigError, match="native backend"): + handle.stream(Events, backend=lane) + with pytest.raises(ValueError, match="no run id"): + stream_client.get_activity_handle("activity").stream(Events) From ab3904e766780aba3b3d02f3a25a8523dfaf15d5 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 10:43:07 -0700 Subject: [PATCH 12/79] Pinned the streams crate to k3 and sdk-core to c10. k3 merges s23 into the native crate, and c10 merges c8 into the native sdk-rust. The vendored protos and WIT already equal k3's, and the bindings regenerate with no change. --- temporalio/bridge/Cargo.lock | 2 +- temporalio/bridge/Cargo.toml | 2 +- temporalio/bridge/sdk-core | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 66d89d7570..396244f8ab 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -2938,7 +2938,7 @@ dependencies = [ [[package]] name = "temporalio-streams" version = "0.1.0" -source = "git+https://github.com/moetemp/sdk-streams?rev=014211e7d4565df3431d5421c65bab83c1abff76#014211e7d4565df3431d5421c65bab83c1abff76" +source = "git+https://github.com/moetemp/sdk-streams?rev=a0a287ea97c3478d632b34239f341b9f202a0bc7#a0a287ea97c3478d632b34239f341b9f202a0bc7" dependencies = [ "async-trait", "aws-lc-rs", diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index 56a0308953..5582639bdc 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -94,7 +94,7 @@ url = "2.2" # crates, so the bridge sees one type of each. [dependencies.temporalio-streams] git = "https://github.com/moetemp/sdk-streams" -rev = "014211e7d4565df3431d5421c65bab83c1abff76" +rev = "a0a287ea97c3478d632b34239f341b9f202a0bc7" default-features = false optional = true diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 04312dfd0c..191c8b8b64 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 04312dfd0cb54f62e3df2f4379fae4b589f9f59f +Subproject commit 191c8b8b64e08b4f81d17d3204b893186295d023 From 9a6cc2176f8dce01852643de7384acd3cb311ec4 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 10:51:31 -0700 Subject: [PATCH 13/79] Replayed native reads on the connection of the Replayer's stream_client. A replay reads each page of a native Workflow read again, and only the caller knows which server keeps the Streams. The Replayer takes the client's backends when it has none. --- temporalio/common/streams/_native.py | 26 +++++---------------- temporalio/common/streams/_plugin.py | 34 ++++++++++++++++++++-------- temporalio/worker/_replayer.py | 9 ++++++-- 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/temporalio/common/streams/_native.py b/temporalio/common/streams/_native.py index 4d67bdb7d1..c17360d860 100644 --- a/temporalio/common/streams/_native.py +++ b/temporalio/common/streams/_native.py @@ -25,7 +25,6 @@ from temporalio.api.common.v1 import Payload, Payloads from temporalio.common.streams import _errors, _workflow_reader from temporalio.common.streams._backend import _ExtensionCalls -from temporalio.common.streams._errors import StreamConfigError from temporalio.common.streams._extension import ( ExtensionStreamBackend, _activity_error, @@ -92,8 +91,8 @@ def __init__( inner: temporalio.converter.PayloadCodec | None, records: temporalio.converter.PayloadCodec | None, ) -> None: - # The runtime that a replay reads again on: the client's, or for a Replayer, one of - # a client of the backend. + # The runtime that a replay reads again on: the Worker's client's, or the Replayer's + # stream_client's. self._backend = backend self._inner = inner # Writers encode Records with no serialization context, so a page decodes with the @@ -300,7 +299,7 @@ class NativeStreamBackend(ExtensionStreamBackend): client calls go to the same service. Each client gets its own runtime on its own connection, built at the first call, and the client's Workers share it. A page of a Workflow read is a direct read in a Local Activity whose marker holds only its positions, - and a replay reads the page again on the connection of a client of the backend. + and a replay reads the page again, on the connection of the Replayer's ``stream_client``. .. warning:: This API is experimental. @@ -332,8 +331,7 @@ def __init__( config = ext_pb.NativeStreamConfig() if retention is not None: config.retention.FromTimedelta(retention) - # The calls of each connection. A client and its Workers share them, and a Replayer - # reads again on any of them. + # The calls of each connection, which a client and its Workers share. self._connections: weakref.WeakKeyDictionary[ temporalio.service.ServiceClient, _ExtensionCalls ] = weakref.WeakKeyDictionary() @@ -357,17 +355,6 @@ def backend_for(self, client: temporalio.client.Client) -> _ExtensionCalls: """The Stream runtime of ``client``'s connection.""" return self._new_backend(client.service_client) - def _any_backend(self) -> _ExtensionCalls: - # A Replayer has no client, so a replay reads again on a connection of the backend. - for backend in list(self._connections.values()): - return backend - raise StreamConfigError( - "A native replay reads each page of a Workflow read again, so it needs a client " - "of the native backend in this process. Register the Replayer's " - "NativeStreamBackend on a client too, or run a Worker or a Stream call of a " - "client first." - ) - async def _read_result( self, run_id: str, @@ -417,9 +404,8 @@ def _activities(self) -> list[Any]: def _configure_client( self, config: temporalio.client.ClientConfig ) -> temporalio.client.ClientConfig: - # The calls come from the connection, not from an interceptor. Knowing the connection - # now lets a Replayer with this backend read on it. - self._new_backend(config["service_client"]) + # The calls come from the client's connection at the first call, so the client needs + # no interceptor. return config diff --git a/temporalio/common/streams/_plugin.py b/temporalio/common/streams/_plugin.py index ea88542646..a79bae5c15 100644 --- a/temporalio/common/streams/_plugin.py +++ b/temporalio/common/streams/_plugin.py @@ -9,7 +9,7 @@ import temporalio.converter import temporalio.worker from temporalio.common.streams import _errors, _ids -from temporalio.common.streams._backend import BackendCalls +from temporalio.common.streams._backend import BackendCalls, _ExtensionCalls from temporalio.plugin import SimplePlugin # The default backend when no registered backend has default=True. Every client, Worker and @@ -199,12 +199,20 @@ def configure_worker( return config +def replayer_backends(config: temporalio.worker.ReplayerConfig) -> list[StreamBackend]: + """The Replayer's backends, else those of its ``stream_client``, and the native one.""" + backends = config.get("stream_backends") or () + client = config.get("stream_client") + if not backends and client is not None: + backends = client_backends(client) + return with_native(check_backends(backends, "Replayer")) + + def configure_replayer( config: temporalio.worker.ReplayerConfig, ) -> temporalio.worker.ReplayerConfig: """Check the Replayer's backends, and add the parts of each and of the native one.""" - backends = check_backends(config.get("stream_backends", ()), "Replayer") - for backend in with_native(backends): + for backend in replayer_backends(config): config = backend._configure_replayer(config) return config @@ -229,17 +237,23 @@ def replayer_converter( converter: temporalio.converter.DataConverter, ) -> temporalio.converter.DataConverter: """The converter of a Replayer's Workflows, as :py:func:`worker_converter`. A replay - reads a native page again on a connection that the native backend served. + reads a native page again on the connection of the Replayer's ``stream_client``. """ from temporalio.common.streams import _native - native = next( - b - for b in with_native(config.get("stream_backends", ())) - if b.backend_name == NATIVE - ) + native = next(b for b in replayer_backends(config) if b.backend_name == NATIVE) assert isinstance(native, _native.NativeStreamBackend) - return _native.wrap(native._any_backend, converter) + client = config.get("stream_client") + + def backend() -> _ExtensionCalls: + if client is None: + raise _errors.StreamConfigError( + "A replay reads each page of a native Workflow read again, so pass the " + "Replayer a stream_client= of the server that keeps the Streams" + ) + return native.backend_for(client) + + return _native.wrap(backend, converter) def client_backends(client: temporalio.client.Client) -> list[StreamBackend]: diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 349887dc77..e7d1bce51b 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -61,6 +61,7 @@ def __init__( disable_safe_workflow_eviction: bool = False, header_codec_behavior: HeaderCodecBehavior = HeaderCodecBehavior.NO_CODEC, stream_backends: Sequence[StreamBackend] = (), + stream_client: temporalio.client.Client | None = None, ) -> None: """Create a replayer to replay workflows from history. @@ -70,8 +71,10 @@ def __init__( ran. ``stream_backends`` are the Stream backends of the Worker's client, such as :py:class:`temporalio.contrib.stream.RedisStreamBackend`. The native backend is there with no registration. A replay makes no Stream - call, except that it reads each page of a native Workflow read again on - the connection of a client of the native backend in this process. + call, except that it reads each page of a native Workflow read again. + It reads on the connection of ``stream_client``, a client of the server + that keeps the Streams, and takes that client's backends when + ``stream_backends`` is empty. Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that @@ -96,6 +99,7 @@ def __init__( disable_safe_workflow_eviction=disable_safe_workflow_eviction, header_codec_behavior=header_codec_behavior, stream_backends=stream_backends, + stream_client=stream_client, ) self._initial_config = self._config.copy() self._default_workflow_logic_flags = set(_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS) @@ -454,6 +458,7 @@ class ReplayerConfig(TypedDict, total=False): disable_safe_workflow_eviction: bool header_codec_behavior: HeaderCodecBehavior stream_backends: Sequence[StreamBackend] + stream_client: temporalio.client.Client | None @dataclass(frozen=True) From 9132dc68634aed0c763618ab8b249ec2aa780279 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 10:51:31 -0700 Subject: [PATCH 14/79] Moved the Jetty examples and the test lanes to the native default. The native lane registers nothing, and the examples follow the Jetty HLD. Examples that need the native server run with Redis or a memory Store named redis in place of Redis. --- tests/contrib/streams/_support.py | 12 ++- tests/contrib/streams/conftest.py | 49 +++++----- tests/contrib/streams/examples/README.md | 3 +- .../examples/migrate_redis_to_native.py | 72 +++++++-------- .../examples/standalone_activity_stream.py | 50 ++++++++++ .../contrib/streams/examples/test_examples.py | 92 +++++++++---------- .../contrib/streams/examples/two_backends.py | 38 ++++---- .../streams/examples/workflow_reader.py | 19 ++-- .../contrib/streams/test_activity_handles.py | 5 +- tests/contrib/streams/test_native_server.py | 18 ++-- tests/contrib/streams/test_redis.py | 20 ++-- tests/contrib/streams/test_stream_client.py | 21 +++-- tests/contrib/streams/test_workflow_module.py | 16 +++- tests/contrib/streams/test_workflow_reads.py | 6 +- 14 files changed, 243 insertions(+), 178 deletions(-) create mode 100644 tests/contrib/streams/examples/standalone_activity_stream.py diff --git a/tests/contrib/streams/_support.py b/tests/contrib/streams/_support.py index 1411ab608e..2481990f6a 100644 --- a/tests/contrib/streams/_support.py +++ b/tests/contrib/streams/_support.py @@ -52,17 +52,21 @@ def on_native(client: Any) -> bool: return name == "native" -async def create_owned( +def create_owned( client: Any, definition: Any, *, id: str, retention: Any = None ) -> Any: - """A Stream that the client owns, on the client's default backend. + """Create a Stream that the client owns, on the client's default backend. ``temporalio.contrib.stream.create`` makes one on Redis. The native server keeps one too, - with no public create, so a native lane makes it through the internal call. + with no public create, so a native lane makes it through the internal call. Await the + result, or use it in ``async with``. """ from temporalio.common.streams import _plugin, _stream_client + from temporalio.contrib import stream as redis name, _ = _plugin.client_backend(client, None) - return await _stream_client.create( + if name != "native": + return redis.create(client, definition, id=id, retention=retention) + return _stream_client.create( client, definition, id=id, retention=retention, backend=name ) diff --git a/tests/contrib/streams/conftest.py b/tests/contrib/streams/conftest.py index ac1ea7763d..48a503e6f7 100644 --- a/tests/contrib/streams/conftest.py +++ b/tests/contrib/streams/conftest.py @@ -4,12 +4,12 @@ import subprocess import time import uuid -from collections.abc import AsyncIterator, Iterator +from collections.abc import Iterator import pytest from temporalio.client import Client -from temporalio.common.streams import NativeStreamBackend, StreamBackend +from temporalio.common.streams import StreamBackend from temporalio.contrib.stream import RedisStreamBackend from temporalio.contrib.stream.testing import MemoryStreamBackend @@ -61,22 +61,12 @@ def redis_url() -> Iterator[str]: NATIVE_NAMESPACE = os.environ.get("TEMPORAL_STREAMS_NATIVE_NAMESPACE", "default") -@pytest.fixture(params=["memory", "redis", *(["native"] if NATIVE_TARGET else [])]) -async def stream_client( - client: Client, request: pytest.FixtureRequest -) -> AsyncIterator[Client]: - # Every live test runs on the memory Store and on Redis, since both must keep one set of - # rules, and on the native server when one is given. The backend is the default, so a - # create with no backend name uses it. - if request.param == "native": - yield await Client.connect( - NATIVE_TARGET, - namespace=NATIVE_NAMESPACE, - stream_backends=[NativeStreamBackend()], - ) - return +async def _brook_client( + client: Client, request: pytest.FixtureRequest, lane: str +) -> Client: + # The memory Store or Redis, as the default, so a create with no backend name uses it. backend: StreamBackend - if request.param == "memory": + if lane == "memory": backend = MemoryStreamBackend(default=True) else: redis_url = request.getfixturevalue("redis_url") @@ -85,7 +75,24 @@ async def stream_client( ) config = client.config() config["stream_backends"] = [backend] - yield Client(**config) + return Client(**config) + + +@pytest.fixture(params=["memory", "redis", *(["native"] if NATIVE_TARGET else [])]) +async def stream_client(client: Client, request: pytest.FixtureRequest) -> Client: + # Every live test runs on the memory Store and on Redis, since both must keep one set of + # rules, and on the native server when one is given. The native backend is every + # client's with no registration, and the default when no backend has default=True. + if request.param == "native": + return await Client.connect(NATIVE_TARGET, namespace=NATIVE_NAMESPACE) + return await _brook_client(client, request, request.param) + + +@pytest.fixture(params=["memory", "redis"]) +async def redis_client(client: Client, request: pytest.FixtureRequest) -> Client: + # A test of the Redis calls, such as temporalio.contrib.stream.create, which the memory + # Store keeps the rules of. + return await _brook_client(client, request, request.param) @pytest.fixture @@ -94,8 +101,4 @@ async def native_client() -> Client: pytest.skip( "TEMPORAL_STREAMS_NATIVE_TARGET names no server with the Stream service" ) - return await Client.connect( - NATIVE_TARGET, - namespace=NATIVE_NAMESPACE, - stream_backends=[NativeStreamBackend()], - ) + return await Client.connect(NATIVE_TARGET, namespace=NATIVE_NAMESPACE) diff --git a/tests/contrib/streams/examples/README.md b/tests/contrib/streams/examples/README.md index 1b001933da..3b60b8f2d3 100644 --- a/tests/contrib/streams/examples/README.md +++ b/tests/contrib/streams/examples/README.md @@ -16,10 +16,11 @@ The proposal's Nexus examples are a later stage, so they have no file here. ## Jetty v2 -These add Workflow reads, the native backend, and two backends in one Worker to the same API. `test_examples.py` runs each one on three lanes. The memory and Redis lanes use a memory Store with the backend name `native` in place of the server. The native lane uses `NativeStreamPlugin` against a server with the Stream service, and it runs only when `TEMPORAL_STREAMS_NATIVE_TARGET` is set. +These add Workflow reads, the native backend, two backends in one Worker, and Standalone Activity Streams to the same API. Every client has the native backend with no registration, and it is the default unless a registered backend has `default=True`. `test_examples.py` runs the Workflow reader on the memory Store, on Redis and on the native server. The others need the native server, with Redis or a memory Store named `redis` in place of Redis. The native runs happen only when `TEMPORAL_STREAMS_NATIVE_TARGET` is set. | Section | File | |---|---| | One Worker with Redis and native, one Stream on each | [`two_backends.py`](two_backends.py) | | A Workflow reads another Workflow's Stream | [`workflow_reader.py`](workflow_reader.py) | | Old Streams stay on Redis, new ones go to native | [`migrate_redis_to_native.py`](migrate_redis_to_native.py) | +| A Standalone Activity streams its progress | [`standalone_activity_stream.py`](standalone_activity_stream.py) | diff --git a/tests/contrib/streams/examples/migrate_redis_to_native.py b/tests/contrib/streams/examples/migrate_redis_to_native.py index 51dfa12077..b977a2cfdc 100644 --- a/tests/contrib/streams/examples/migrate_redis_to_native.py +++ b/tests/contrib/streams/examples/migrate_redis_to_native.py @@ -1,11 +1,11 @@ """Jetty v2, "Move from Redis to native Streams". -Old Streams stay on Redis, and new ones go to the server. Step 1 runs the old code on a Worker -with only ``RedisStreamPlugin``. Step 2 deploys new code on a Worker with both plugins. The new -code picks ``native`` behind ``workflow.patched``, so a run that started on the old code replays -the same and goes on with its Redis Stream. Each handle carries its backend, so the client reads -each Stream where it is. Redis stays the default, since a change of the default changes the -commands of a replayed create. +Old Streams stay on Redis, and new ones go to the server. The client registers +``RedisStreamBackend`` with ``default=True``, so Workflow code that names no backend keeps +creating on Redis, and the native backend is there with no registration. Step 1 runs the old +code. Step 2 deploys new code that picks ``native`` behind ``workflow.patched``, so a run that +started on the old code replays the same and goes on with its Redis Stream. Redis stays the +default, since a change of the default changes the commands of a replayed create. """ from __future__ import annotations @@ -14,13 +14,10 @@ from datetime import timedelta from typing import Any -from temporalio.contrib.streams import workflow as stream_workflow -from temporalio.contrib.streams.native import NativeStreamPlugin -from temporalio.contrib.streams.redis import RedisStreamPlugin - from temporalio import workflow from temporalio.client import Client, WorkflowHandle -from temporalio.contrib import streams +from temporalio.common import streams +from temporalio.contrib.stream import RedisStreamBackend from temporalio.worker import Worker @@ -37,7 +34,7 @@ def __init__(self) -> None: def more(self, text: str) -> None: self._more.append(text) - async def chat(self, stream: stream_workflow.WorkflowStream[Events]) -> None: + async def chat(self, stream: streams.WorkflowStream[Events]) -> None: await stream.topic(Events.tokens).publish("hello") await workflow.wait_condition(lambda: bool(self._more)) await stream.topic(Events.tokens).publish(*self._more) @@ -47,7 +44,7 @@ async def chat(self, stream: stream_workflow.WorkflowStream[Events]) -> None: class ChatBeforeTheMove(_Chat): @workflow.run async def run(self) -> None: - await self.chat(await stream_workflow.create_stream(Events)) + await self.chat(await workflow.create_stream(Events)) @workflow.defn(name="Chat") @@ -56,49 +53,48 @@ class Chat(_Chat): async def run(self) -> None: # A run from before the move has no patch marker, so it keeps its Redis Stream. backend = "native" if workflow.patched("streams-on-native") else None - await self.chat(await stream_workflow.create_stream(Events, backend=backend)) - - -async def connect_before(target_host: str, redis_url: str) -> Client: - return await Client.connect(target_host, plugins=[RedisStreamPlugin(redis_url)]) + await self.chat(await workflow.create_stream(Events, backend=backend)) -async def connect_during(target_host: str, redis_url: str) -> Client: +async def connect(target_host: str, redis_url: str) -> Client: return await Client.connect( - target_host, - plugins=[RedisStreamPlugin(redis_url, default=True), NativeStreamPlugin()], + target_host, stream_backends=[RedisStreamBackend(redis_url, default=True)] ) -async def read(client: Client, handle: WorkflowHandle[Any, None]) -> list[str]: +async def read(handle: WorkflowHandle[Any, None]) -> list[str]: await handle.result() # A UI that reads chats from both sides of the move looks on native first, then Redis. - stream = streams.StreamClient(client).workflow_stream( - handle, Events, backend=("native", "redis") + for backend in ("native", "redis"): + stream = handle.stream(Events, backend=backend) + try: + tokens = [ + item.data + async for item in stream.subscribe(wait_new_records=False) + if isinstance(item, streams.Record) + ] + except streams.StreamNotFoundError: + continue + return [f"{backend}: {token}" for token in tokens] + raise streams.StreamNotFoundError( + f"The chat {handle.id} has no Stream", reason="stream_not_found" ) - tokens = [ - item.data - async for item in stream.subscribe() - if isinstance(item, streams.Record) - ] - return [f"{stream.backend}: {token}" for token in tokens] -async def main(before: Client, during: Client, task_queue: str) -> list[str]: +async def main(client: Client, task_queue: str) -> list[str]: # Step 1: the old code starts a chat on Redis. - async with Worker(before, task_queue=task_queue, workflows=[ChatBeforeTheMove]): - old = await before.start_workflow( + async with Worker(client, task_queue=task_queue, workflows=[ChatBeforeTheMove]): + old = await client.start_workflow( ChatBeforeTheMove.run, id=f"chat-{uuid.uuid4()}", task_queue=task_queue ) - stream = streams.StreamClient(before).workflow_stream(old, Events) - async for _ in stream.subscribe(create_wait=timedelta(seconds=30)): + async for _ in old.stream(Events).subscribe(create_wait=timedelta(seconds=30)): # The old chat wrote its first Record, so it runs on the old code. break # Step 2: the new code goes on with the old chat, and starts a new one on the server. - async with Worker(during, task_queue=task_queue, workflows=[Chat]): + async with Worker(client, task_queue=task_queue, workflows=[Chat]): await old.signal(ChatBeforeTheMove.more, "old chat goes on") - new = await during.start_workflow( + new = await client.start_workflow( Chat.run, id=f"chat-{uuid.uuid4()}", task_queue=task_queue ) await new.signal(Chat.more, "new chat") - return [*await read(during, old), *await read(during, new)] + return [*await read(old), *await read(new)] diff --git a/tests/contrib/streams/examples/standalone_activity_stream.py b/tests/contrib/streams/examples/standalone_activity_stream.py new file mode 100644 index 0000000000..bf5eed9ba1 --- /dev/null +++ b/tests/contrib/streams/examples/standalone_activity_stream.py @@ -0,0 +1,50 @@ +"""Jetty v2, "A Standalone Activity streams its progress". + +A Standalone Activity, one that no Workflow runs, owns a Stream on the native backend with +``activity.create_stream``. A client reads it from the Activity handle with no call. The server +closes the Stream when the Activity ends, with its result, so the Activity need not close it. +""" + +from __future__ import annotations + +import uuid +from datetime import timedelta + +from temporalio import activity +from temporalio.client import Client +from temporalio.common import streams +from temporalio.worker import Worker + + +@streams.defn(name="events") +class Events: + tokens = streams.topic(str) + + +@activity.defn +async def answer(question: str) -> str: + stream = await activity.create_stream(Events) + for word in question.split(): + await stream.topic(Events.tokens).publish(word) + return question.upper() + + +async def main(client: Client, task_queue: str) -> list[str]: + async with Worker(client, task_queue=task_queue, activities=[answer]): + handle = await client.start_activity( + answer, + "find the answer", + id=f"answer-{uuid.uuid4()}", + task_queue=task_queue, + start_to_close_timeout=timedelta(seconds=30), + ) + seen = [] + # The handle has the run id, so the client builds the Stream with no call. + async for item in handle.stream(Events).subscribe( + create_wait=timedelta(seconds=30) + ): + if isinstance(item, streams.Record): + seen.append(item.data) + elif isinstance(item, streams.Closed): + seen.append(f"activity said {item.result}") + return seen diff --git a/tests/contrib/streams/examples/test_examples.py b/tests/contrib/streams/examples/test_examples.py index a515fd8570..f297058d8f 100644 --- a/tests/contrib/streams/examples/test_examples.py +++ b/tests/contrib/streams/examples/test_examples.py @@ -1,4 +1,8 @@ -"""Runs each example end to end on the memory Store and on Redis, so none goes stale.""" +"""Runs each example end to end, so none goes stale. + +The Brook examples run on the memory Store, on Redis, and on the native server when +TEMPORAL_STREAMS_NATIVE_TARGET is set. +""" import uuid @@ -15,6 +19,7 @@ client_creates_stream, migrate_redis_to_native, resume_from_offset, + standalone_activity_stream, turn_scoped_stream, two_backends, ui_backend_reads, @@ -46,8 +51,8 @@ async def test_resume_from_offset(stream_client: Client): ) -async def test_client_creates_stream(stream_client: Client): - turns = await client_creates_stream.main(stream_client, task_queue()) +async def test_client_creates_stream(redis_client: Client): + turns = await client_creates_stream.main(redis_client, task_queue()) assert [turn.kind for turn in turns] == ["model", "last"] @@ -100,62 +105,49 @@ async def test_worker_setup_on_redis(client: Client, redis_url: str): assert await worker_setup.create_on_second_backend(migrating) == "redis-b" -# The Jetty v2 examples run with a native stand-in, a memory Store with the backend name -# `native`, and on a server with the Stream service when TEMPORAL_STREAMS_NATIVE_TARGET is set. -LANES = ["memory", "redis", "native"] - - -def lane_plugins( - lane: str, request: pytest.FixtureRequest -) -> tuple[streams.StreamBackend, streams.StreamBackend]: - """The plugins of a lane: one named `redis` and one named `native`.""" - if lane == "memory": - return MemoryStreamBackend(name="redis", default=True), MemoryStreamBackend( - name="native" - ) - if lane == "native" and not NATIVE_TARGET: - pytest.skip( - "TEMPORAL_STREAMS_NATIVE_TARGET names no server with the Stream service" - ) - redis_url = request.getfixturevalue("redis_url") - redis = RedisStreamBackend(redis_url, key_prefix=f"t-{uuid.uuid4()}", default=True) - if lane == "redis": - return redis, MemoryStreamBackend(name="native") - return redis, streams.NativeStreamBackend() - - -async def lane_client( - lane: str, client: Client, plugins: list[streams.StreamBackend] -) -> Client: - if lane != "native": - config = client.config() - config["stream_backends"] = plugins - return Client(**config) +# The Jetty v2 examples. The Workflow reader runs on one backend: the memory Store, Redis or +# the native server. The others need the native server, with Redis, or a memory Store named +# `redis` in its place, so they run only when TEMPORAL_STREAMS_NATIVE_TARGET is set. +async def native_server(client: Client, *backends: streams.StreamBackend) -> Client: if not NATIVE_TARGET: pytest.skip( "TEMPORAL_STREAMS_NATIVE_TARGET names no server with the Stream service" ) return await Client.connect( - NATIVE_TARGET, namespace=NATIVE_NAMESPACE, stream_backends=plugins + NATIVE_TARGET, namespace=NATIVE_NAMESPACE, stream_backends=backends ) -@pytest.mark.parametrize("lane", LANES) +def redis_stand_in( + lane: str, request: pytest.FixtureRequest, *, default: bool = False +) -> streams.StreamBackend: + if lane == "memory": + return MemoryStreamBackend(name="redis", default=default) + redis_url = request.getfixturevalue("redis_url") + return RedisStreamBackend( + redis_url, key_prefix=f"t-{uuid.uuid4()}", default=default + ) + + +@pytest.mark.parametrize("lane", ["memory", "redis"]) async def test_two_backends(lane: str, client: Client, request: pytest.FixtureRequest): - two = await lane_client(lane, client, list(lane_plugins(lane, request))) + two = await native_server(client, redis_stand_in(lane, request)) assert await two_backends.main(two, task_queue()) == { "audit": ["started", "summarized"], "live": ["hello", "there", "summary"], } -@pytest.mark.parametrize("lane", LANES) +@pytest.mark.parametrize("lane", ["memory", "redis", "native"]) async def test_workflow_reader( lane: str, client: Client, request: pytest.FixtureRequest ): - # Each lane reads from one backend: the memory Store, Redis or the server. - plugin = {"memory": 0, "redis": 0, "native": 1}[lane] - one = await lane_client(lane, client, [lane_plugins(lane, request)[plugin]]) + if lane == "native": + one = await native_server(client) + else: + config = client.config() + config["stream_backends"] = [redis_stand_in(lane, request, default=True)] + one = Client(**config) assert await workflow_reader.main(one, task_queue()) == [ "find", "the", @@ -164,16 +156,24 @@ async def test_workflow_reader( ] -@pytest.mark.parametrize("lane", LANES) +@pytest.mark.parametrize("lane", ["memory", "redis"]) async def test_migrate_redis_to_native( lane: str, client: Client, request: pytest.FixtureRequest ): - redis, native = lane_plugins(lane, request) - before = await lane_client(lane, client, [redis]) - during = await lane_client(lane, client, [redis, native]) - assert await migrate_redis_to_native.main(before, during, task_queue()) == [ + moving = await native_server(client, redis_stand_in(lane, request, default=True)) + assert await migrate_redis_to_native.main(moving, task_queue()) == [ "redis: hello", "redis: old chat goes on", "native: hello", "native: new chat", ] + + +async def test_standalone_activity_stream(client: Client): + native = await native_server(client) + assert await standalone_activity_stream.main(native, task_queue()) == [ + "find", + "the", + "answer", + "activity said FIND THE ANSWER", + ] diff --git a/tests/contrib/streams/examples/two_backends.py b/tests/contrib/streams/examples/two_backends.py index 9afbee73c1..97e1d96549 100644 --- a/tests/contrib/streams/examples/two_backends.py +++ b/tests/contrib/streams/examples/two_backends.py @@ -1,9 +1,10 @@ """Jetty v2, "Two backends in one Worker". -The client has ``RedisStreamPlugin`` and ``NativeStreamPlugin``, and the Worker takes both from -it. Each Stream handle carries its backend name, so every call goes to the backend that keeps -the Stream. A create with no ``backend=`` goes to the default one, here Redis. Tests swap a -plugin for ``MemoryStreamPlugin(name="native")`` with no change to Workflow code. +The client registers ``RedisStreamBackend``, and has the native backend with no registration. +The Worker takes both from it. Each Stream handle carries its backend name, so every call goes +to the backend that keeps the Stream. A create with no ``backend=`` goes to the default one, +here native, since the Redis backend has no ``default=True``. Tests swap Redis for a +``MemoryStreamBackend(name="redis")`` with no change to Workflow code. """ from __future__ import annotations @@ -12,13 +13,10 @@ from dataclasses import dataclass from datetime import timedelta -from temporalio.contrib.streams import workflow as stream_workflow -from temporalio.contrib.streams.native import NativeStreamPlugin -from temporalio.contrib.streams.redis import RedisStreamPlugin - from temporalio import activity, workflow from temporalio.client import Client -from temporalio.contrib import streams +from temporalio.common import streams +from temporalio.contrib.stream import RedisStreamBackend from temporalio.worker import Worker @@ -50,12 +48,12 @@ async def summarize(input: SummarizeInput) -> str: class TwoStreams: @workflow.run async def run(self) -> str: - # The audit log stays on the default backend, Redis. The live view is on the server. - audit = await stream_workflow.create_stream(Audit) - live = await stream_workflow.create_stream(Live, backend="native") + # The audit log is on the server, the default backend. The live view is on Redis. + audit = await workflow.create_stream(Audit) + live = await workflow.create_stream(Live, backend="redis") await audit.topic(Events.tokens).publish("started") await live.topic(Events.tokens).publish("hello", "there") - return await workflow.execute_activity( + result = await workflow.execute_activity( summarize, SummarizeInput( streams.ActivityStream.from_handle(audit), @@ -63,12 +61,15 @@ async def run(self) -> str: ), start_to_close_timeout=timedelta(seconds=30), ) + # The server closes the audit log when the run chain ends. The end of a run closes + # nothing on Redis, so the run closes the live view. + await live.close(result=result) + return result async def connect(target_host: str, redis_url: str) -> Client: return await Client.connect( - target_host, - plugins=[RedisStreamPlugin(redis_url, default=True), NativeStreamPlugin()], + target_host, stream_backends=[RedisStreamBackend(redis_url)] ) @@ -80,13 +81,12 @@ async def main(client: Client, task_queue: str) -> dict[str, list[str]]: TwoStreams.run, id=f"two-{uuid.uuid4()}", task_queue=task_queue ) await handle.result() - stream_client = streams.StreamClient(client) read: dict[str, list[str]] = {} for name, definition, backend in ( - ("audit", Audit, "redis"), - ("live", Live, "native"), + ("audit", Audit, None), + ("live", Live, "redis"), ): - stream = stream_client.workflow_stream(handle, definition, backend=backend) + stream = handle.stream(definition, backend=backend) read[name] = [ item.data async for item in stream.subscribe() diff --git a/tests/contrib/streams/examples/workflow_reader.py b/tests/contrib/streams/examples/workflow_reader.py index fef93c73d8..9c42c1dbfe 100644 --- a/tests/contrib/streams/examples/workflow_reader.py +++ b/tests/contrib/streams/examples/workflow_reader.py @@ -12,11 +12,9 @@ import uuid from datetime import timedelta -from temporalio.contrib.streams import workflow as stream_workflow - from temporalio import workflow from temporalio.client import Client -from temporalio.contrib import streams +from temporalio.common import streams from temporalio.worker import Worker @@ -29,24 +27,27 @@ class Events: class Agent: @workflow.run async def run(self, prompt: str) -> str: - stream = await stream_workflow.create_stream(Events) + stream = await workflow.create_stream(Events) for word in prompt.split(): # Each word goes out in its own Workflow Task, so the follower waits for some. await workflow.sleep(timedelta(milliseconds=200)) await stream.topic(Events.tokens).publish(word) - return prompt.upper() + # The server would close a native Stream when the run chain ends. A Redis Stream + # stays open, so the agent closes it with its answer. + answer = prompt.upper() + await stream.close(result=answer) + return answer @workflow.defn class Follower: @workflow.run - async def run(self, agent: stream_workflow.WorkflowStream[Events]) -> list[str]: + async def run(self, agent: streams.WorkflowStream[Events]) -> list[str]: seen = [] async for item in agent.subscribe(create_wait=timedelta(seconds=30)): if isinstance(item, streams.Record): seen.append(item.data) elif isinstance(item, streams.Closed): - # The agent's Stream closes with the agent's result. seen.append(f"agent said {item.result}") return seen @@ -60,10 +61,10 @@ async def main(client: Client, task_queue: str) -> list[str]: task_queue=task_queue, ) # The handle has the first run id, so the client builds the Stream with no call. - stream = streams.StreamClient(client).workflow_stream(agent, Events) + stream = agent.stream(Events) return await client.execute_workflow( Follower.run, - stream_workflow.WorkflowStream.from_handle(stream), + streams.WorkflowStream.from_handle(stream), id=f"follower-{uuid.uuid4()}", task_queue=task_queue, ) diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index 9e6691de77..4fbd2fb83c 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -363,7 +363,10 @@ async def test_store_refuses_another_chains_close(stream_client: Client): ) == ("stream_not_found" if native else "stream_not_owner") stream = owner.stream(Events) items = [item async for item in stream.subscribe(wait_new_records=False)] - assert [type(item) for item in items] == ([] if native else [streams.Record]) + # The server closed the native Stream when its run chain ended. + assert [type(item) for item in items] == ( + [streams.Closed] if native else [streams.Record] + ) @dataclass diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index 1ee4ed6d0f..75029f8ece 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -14,7 +14,7 @@ from temporalio import activity, workflow from temporalio.client import Client, WorkflowHandle from temporalio.common import streams -from temporalio.common.streams import _native +from temporalio.common.streams import _native, _plugin from temporalio.common.streams._native import POSITIONS_TYPE from temporalio.contrib.stream import RedisStreamBackend from temporalio.worker import Replayer, UnsandboxedWorkflowRunner @@ -107,12 +107,9 @@ async def test_native_workflow_reads_and_replays(native_client: Client): if event.HasField("nexus_operation_completed_event_attributes") ] assert wakes and not any(wake.records for wake in wakes) - # A replay reads over the connection of a client of the native backend. It reads each page - # again once the cache is gone. - replayer = Replayer( - workflows=[Writer, Reader], - stream_backends=native_client.config()["stream_backends"], - ) + # A replay reads each page again on the connection of its stream_client, once the cache + # is gone. + replayer = Replayer(workflows=[Writer, Reader], stream_client=native_client) await replayer.replay_workflow(await writer.fetch_history()) await replayer.replay_workflow(history) _native._pages.clear() @@ -122,10 +119,10 @@ async def test_native_workflow_reads_and_replays(native_client: Client): async def test_redis_and_native_on_a_native_server( native_client: Client, redis_url: str ): + # The client registers only Redis, and has the native backend with no registration. config = native_client.config() config["stream_backends"] = [ RedisStreamBackend(redis_url, key_prefix=f"t-{uuid.uuid4()}", default=True), - streams.NativeStreamBackend(), ] two = Client(**config) async with new_worker( @@ -419,3 +416,8 @@ async def test_each_client_of_a_native_backend_calls_on_its_own_connection(): first.service_client, second.service_client, ] + # The native backend that no client registers works the same, and makes no call. + plain = await Client.connect("127.0.0.1:3", lazy=True) + _, calls = _plugin.client_backend(plain, None) + assert cast(Any, calls)._runtime._service is plain.service_client + assert _plugin.client_backend(plain, None)[1] is calls diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 38a9edc527..e479826c38 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -14,6 +14,7 @@ from temporalio.api.common.v1 import Payload from temporalio.client import Client, Interceptor, WorkflowFailureError, WorkflowHandle from temporalio.common import RawValue, RetryPolicy, streams +from temporalio.common.streams import _plugin from temporalio.common.streams._extension import ( ExtensionStreamBackend, _WorkflowInbound, @@ -35,7 +36,7 @@ SandboxedWorkflowRunner, SandboxRestrictions, ) -from tests.contrib.streams._support import on_native +from tests.contrib.streams._support import create_owned, on_native from tests.helpers import new_worker @@ -597,9 +598,9 @@ async def test_client_close_goes_through_the_payload_codec(stream_client: Client DataConverter.default, payload_codec=_Rot13Codec() ) codec_client = Client(**config) - done = await redis.create(codec_client, Events, id=f"done-{uuid.uuid4()}") + done = await create_owned(codec_client, Events, id=f"done-{uuid.uuid4()}") await done.close(result=Turn("done")) - failed = await redis.create(codec_client, Events, id=f"failed-{uuid.uuid4()}") + failed = await create_owned(codec_client, Events, id=f"failed-{uuid.uuid4()}") await failed.close(failure=ApplicationError("bad turn")) [closed] = [item async for item in done.subscribe()] assert isinstance(closed, streams.Closed) @@ -674,9 +675,9 @@ async def test_backend_replays_a_history_made_without_it( async def test_backend_shutdown_after_client_writes(stream_client: Client): - [backend] = stream_client.config()["stream_backends"] + backend = _plugin.backend_of(stream_client, None) assert isinstance(backend, ExtensionStreamBackend) - stream = await redis.create(stream_client, Events, id=f"shutdown-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"shutdown-{uuid.uuid4()}") async with streams.BufferedPublisher(stream.topic(Events.tokens)) as tokens: tokens.publish("buffered") await backend.shutdown() @@ -776,8 +777,9 @@ async def test_final_failure_under_a_retry_policy( items = [item async for item in stream.subscribe(wait_new_records=False)] written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] - # The end of the chain closes nothing. - assert not any(isinstance(item, streams.Closed) for item in items) + # The end of the chain closes only a native Stream, with the last run's failure. + closes = [item for item in items if isinstance(item, streams.Closed)] + assert len(closes) == (1 if on_native(stream_client) else 0) if non_retryable: # A non-retryable failure ends the chain on attempt 1. assert [r.data for r in written] == ["run-1"] @@ -792,7 +794,7 @@ async def test_final_failure_under_a_retry_policy( async def test_reader_stops_on_its_own_timer_and_resumes(stream_client: Client): - stream = await redis.create(stream_client, Events, id=f"timer-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"timer-{uuid.uuid4()}") first = await stream.topic(Events.tokens).publish("one") seen: list[streams.Record[Any]] = [] @@ -926,7 +928,7 @@ async def test_write_to_a_missing_stream_raises_at_the_await(stream_client: Clie async def test_write_to_a_closed_stream_raises_at_the_await(stream_client: Client): - stream = await redis.create(stream_client, Events, id=f"closed-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"closed-{uuid.uuid4()}") await stream.close() async with new_worker(stream_client, WritesToStream) as worker: assert ( diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 9e1df3e7ed..9a40e6441b 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -17,6 +17,7 @@ from temporalio.converter import DataConverter from temporalio.testing import WorkflowEnvironment from temporalio.worker import Replayer +from tests.contrib.streams._support import create_owned from tests.contrib.streams.fake_extension import fake_client from tests.helpers import new_worker @@ -106,7 +107,7 @@ async def run(self, stream: streams.WorkflowStream[Events]) -> None: async def test_client_stream_takes_workflow_writes(stream_client: Client): - stream = await redis.create(stream_client, Events, id=f"chat-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"chat-{uuid.uuid4()}") assert await stream.get_offset() == 0 first = await stream.topic(Events.turns).publish(Turn("client")) last = await stream.topic(Events.tokens).publish("a", "b") @@ -178,19 +179,19 @@ async def test_client_reads_a_workflow_stream(stream_client: Client): async def test_async_with_closes_a_client_stream(stream_client: Client): - async with redis.create(stream_client, Events, id=f"block-{uuid.uuid4()}") as done: + async with create_owned(stream_client, Events, id=f"block-{uuid.uuid4()}") as done: await done.topic(Events.tokens).publish("a") failed = stream_client.stream(Events, id=f"block-{uuid.uuid4()}") with pytest.raises(ValueError): - async with redis.create(stream_client, Events, id=failed.id): + async with create_owned(stream_client, Events, id=failed.id): raise ValueError("turn failed") - async with await redis.create( + async with await create_owned( stream_client, Events, id=f"block-{uuid.uuid4()}" ) as early: await early.close(result="early") cancelled = stream_client.stream(Events, id=f"block-{uuid.uuid4()}") with pytest.raises(asyncio.CancelledError): - async with redis.create(stream_client, Events, id=cancelled.id): + async with create_owned(stream_client, Events, id=cancelled.id): raise asyncio.CancelledError() for stream, result, failure, status in ( (done, None, None, streams.CloseStatus.COMPLETED), @@ -207,7 +208,7 @@ async def test_async_with_closes_a_client_stream(stream_client: Client): async def test_next_offset_resumes_after_the_last_item(stream_client: Client): - stream = await redis.create(stream_client, Events, id=f"resume-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"resume-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("a") turn = await stream.topic(Events.turns).publish(Turn("model")) tokens = stream.subscribe(Events.tokens, wait_new_records=False) @@ -229,7 +230,7 @@ async def test_next_offset_resumes_after_the_last_item(stream_client: Client): async def test_last_reads_the_newest_records_then_follows(stream_client: Client): - stream = await redis.create(stream_client, Events, id=f"last-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"last-{uuid.uuid4()}") tokens = stream.topic(Events.tokens) await tokens.publish("a", "b", "c") await stream.topic(Events.turns).publish(Turn("model")) @@ -254,7 +255,7 @@ async def test_last_reads_the_newest_records_then_follows(stream_client: Client) async def test_read_type_decodes_topics_outside_the_definition(stream_client: Client): - stream = await redis.create(stream_client, Events, id=f"typed-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"typed-{uuid.uuid4()}") await stream.topic("extra", type=Turn).publish(Turn("extra")) await stream.topic(Events.tokens).publish("a") @@ -267,7 +268,7 @@ async def read(**options: Any) -> list[object]: async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: Client): - stream = await redis.create(stream_client, Events, id=f"buffered-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"buffered-{uuid.uuid4()}") async with streams.BufferedPublisher(stream) as publisher: publisher.publish(Events.turns, Turn("user")) publisher.publish(Events.tokens, "a", "b") @@ -288,7 +289,7 @@ async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: C async def test_named_producer_fences_its_earlier_epoch(stream_client: Client): - stream = await redis.create(stream_client, Events, id=f"named-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"named-{uuid.uuid4()}") # A server that restarts writes as one name with a higher epoch, so the zombie it # replaces can't write after it. before = stream.producer("ingest", epoch=1) diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 2bb2c76a11..c2b5bb1304 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -25,7 +25,7 @@ from temporalio.converter import DataConverter, PayloadCodec from temporalio.exceptions import ApplicationError from temporalio.worker import Replayer -from tests.contrib.streams._support import on_native +from tests.contrib.streams._support import create_owned, on_native from tests.helpers import assert_task_fail_eventually, new_worker @@ -235,7 +235,7 @@ async def run(self, stream: streams.WorkflowStream[Events]) -> str: async def test_workflow_closes_only_its_own_streams(stream_client: Client): - stream = await redis.create(stream_client, Events, id=f"owned-{uuid.uuid4()}") + stream = await create_owned(stream_client, Events, id=f"owned-{uuid.uuid4()}") async with new_worker(stream_client, ClosesAReceivedStream) as worker: assert ( await stream_client.execute_workflow( @@ -260,7 +260,7 @@ async def run(self) -> None: raise ApplicationError("run failed", non_retryable=True) -async def test_run_end_closes_nothing(stream_client: Client): +async def test_run_end_closes_only_a_native_stream(stream_client: Client): async with new_worker(stream_client, Failing) as worker: handle = await stream_client.start_workflow( Failing.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue @@ -268,9 +268,15 @@ async def test_run_end_closes_nothing(stream_client: Client): with pytest.raises(WorkflowFailureError): await handle.result() items = await read(handle) - # The Stream stays open, so a later reader still waits for a close. assert data(items) == ["before"] - assert not any(isinstance(item, streams.Closed) for item in items) + if not on_native(stream_client): + # The Stream stays open, so a later reader still waits for a close. + assert not any(isinstance(item, streams.Closed) for item in items) + return + # The server closes the Stream when the run chain ends, with the run's failure. + closed = items[-1] + assert isinstance(closed, streams.Closed) and closed.failure is not None + assert closed.status == streams.CloseStatus.FAILED @workflow.defn diff --git a/tests/contrib/streams/test_workflow_reads.py b/tests/contrib/streams/test_workflow_reads.py index d7db700368..fc4d74f479 100644 --- a/tests/contrib/streams/test_workflow_reads.py +++ b/tests/contrib/streams/test_workflow_reads.py @@ -84,10 +84,6 @@ async def run(self, input: ReadInput) -> list[Any]: return seen -def backends_of(client: Client) -> list[Any]: - return list(client.config()["stream_backends"]) - - async def write_and_read( client: Client, backend: str | None, **read_options: Any ) -> tuple[list[Any], Any]: @@ -114,7 +110,7 @@ async def test_workflow_reads_another_workflows_stream(stream_client: Client): seen, reader = await write_and_read(stream_client, None) assert seen == ["turn user hi", "a", "b", "c", "closed written"] # Each page is in History, so a replay reads the same items. - replayer = Replayer(workflows=[Reader], stream_backends=backends_of(stream_client)) + replayer = Replayer(workflows=[Reader], stream_client=stream_client) await replayer.replay_workflow(await reader.fetch_history()) From b4e2332bb456d4a42f4b4e6c598fc1cc9d08bedd Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 10:55:32 -0700 Subject: [PATCH 15/79] Registered the Stream Local Activities as the Worker's own. A Worker with Workflows runs them, and they enable no polling of the task queue's Activities and stay out of the errors about the user's Activities. So a Worker that uses no Streams polls and fails as before. --- temporalio/common/streams/_extension.py | 11 ++--------- temporalio/common/streams/_plugin.py | 22 ++++++++++++++++++++-- temporalio/worker/_activity.py | 18 ++++++++++++++++-- temporalio/worker/_worker.py | 13 +++++++++++-- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index 12407ea64a..243359d90a 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -27,7 +27,6 @@ _errors, _ids, _workflow_reader, - _writes, ) from temporalio.common.streams import _workflow as streams_workflow from temporalio.common.streams._backend import BackendCalls, _ExtensionCalls @@ -178,14 +177,8 @@ def _interceptor_of(self, interceptors: Sequence[Any]) -> _StreamInterceptor | N def _configure_worker( self, config: temporalio.worker.WorkerConfig ) -> temporalio.worker.WorkerConfig: - # Add the backend's Activities, and its interceptor when the client has none. + # Add the backend's interceptor when the client has none. config = super()._configure_worker(config) - # The write Local Activities route by the backend in each request, so every backend of - # the Worker shares one set. - activities: list[Callable[..., Any]] = list(config.get("activities", [])) - writes = cast(list[Callable[..., Any]], _writes.ACTIVITIES) - activities.extend(write for write in writes if write not in activities) - config["activities"] = [*activities, *self._activities()] client = config["client"] # type: ignore[reportTypedDictNotRequiredAccess] interceptors = [ *client.config(active_config=True)["interceptors"], @@ -201,7 +194,7 @@ def _configure_worker( config["interceptors"] = [*config.get("interceptors", []), mine] mine.converter = client.data_converter runner = config.get("workflow_runner") - if isinstance(runner, SandboxedWorkflowRunner): + if config.get("workflows") and isinstance(runner, SandboxedWorkflowRunner): # The interceptor and the facade must share the run's Stream state, also when the # restrictions don't pass temporalio through. config["workflow_runner"] = dataclasses.replace( diff --git a/temporalio/common/streams/_plugin.py b/temporalio/common/streams/_plugin.py index a79bae5c15..ba40ac29f9 100644 --- a/temporalio/common/streams/_plugin.py +++ b/temporalio/common/streams/_plugin.py @@ -2,8 +2,8 @@ from __future__ import annotations -from collections.abc import Sequence -from typing import Any +from collections.abc import Callable, Sequence +from typing import Any, cast import temporalio.client import temporalio.converter @@ -199,6 +199,24 @@ def configure_worker( return config +def local_activities( + config: temporalio.worker.WorkerConfig, +) -> list[Callable[..., Any]]: + """The Local Activities that a Worker's Workflows run for Streams. + + The write Local Activities route by the backend in each request, so every backend of the + Worker shares one set. Each backend reads with its own. + """ + from temporalio.common.streams import _extension, _writes + + client = config["client"] # type: ignore[reportTypedDictNotRequiredAccess] + activities = list(cast(list[Callable[..., Any]], _writes.ACTIVITIES)) + for backend in with_native(client_backends(client)): + if isinstance(backend, _extension.ExtensionStreamBackend): + activities.extend(backend._activities()) + return activities + + def replayer_backends(config: temporalio.worker.ReplayerConfig) -> list[StreamBackend]: """The Replayer's backends, else those of its ``stream_client``, and the native one.""" backends = config.get("stream_backends") or () diff --git a/temporalio/worker/_activity.py b/temporalio/worker/_activity.py index ea7c773fad..f3967e19fd 100644 --- a/temporalio/worker/_activity.py +++ b/temporalio/worker/_activity.py @@ -63,6 +63,7 @@ def __init__( metric_meter: temporalio.common.MetricMeter, client: temporalio.client.Client, encode_headers: bool, + internal_activities: Sequence[Callable] = (), ) -> None: self._bridge_worker = bridge_worker self._task_queue = task_queue @@ -82,7 +83,13 @@ def __init__( # Validate and build activity dict self._activities: dict[str, temporalio.activity._Definition] = {} self._dynamic_activity: temporalio.activity._Definition | None = None - for activity in activities: + # The SDK's own Local Activities, such as those of Streams, which the errors about + # the user's activities leave out. + self._internal_activities = { + temporalio.activity._Definition.must_from_callable(activity).name + for activity in internal_activities + } + for activity in [*activities, *internal_activities]: # Get definition defn = temporalio.activity._Definition.must_from_callable(activity) # Confirm name unique @@ -727,7 +734,14 @@ def assert_activity_valid(self, activity: str) -> None: return activity_def = self._activities.get(activity) if not activity_def: - activity_names = ", ".join(sorted(self._activities.keys())) + activity_names = ", ".join( + sorted(set(self._activities) - self._internal_activities) + ) + if not activity_names: + raise ValueError( + f"Activity function {activity} " + f"is not registered on this worker, no available activities.", + ) raise ValueError( f"Activity function {activity} " f"is not registered on this worker, available activities: {activity_names}", diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 9dd0846fa7..ae65e7034e 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -504,12 +504,21 @@ def _init_from_config(self, client: temporalio.client.Client, config: WorkerConf or temporalio.runtime.Runtime.default() ) activities = config.get("activities") + # The Local Activities that the Worker's Workflows run for Streams. They are the SDK's + # own, so they enable no polling of the task queue's Activities. + internal_activities: Sequence[Callable[..., Any]] = [] + if config.get("workflows"): + from temporalio.common.streams import _plugin as _stream_plugin + + internal_activities = _stream_plugin.local_activities(config) if activities: _warn_if_activity_executor_max_workers_is_inconsistent(config) + if activities or internal_activities: self._activity_worker = _ActivityWorker( bridge_worker=lambda: self._bridge_worker, task_queue=config["task_queue"], # type: ignore[reportTypedDictNotRequiredAccess] - activities=activities, + activities=activities or [], + internal_activities=internal_activities, activity_executor=config.get("activity_executor"), shared_state_manager=config.get("shared_state_manager"), data_converter=client_config["data_converter"], @@ -677,7 +686,7 @@ def check_activity(activity: str): enable_workflows=self._workflow_worker is not None, enable_local_activities=self._activity_worker is not None and self._workflow_worker is not None, - enable_remote_activities=self._activity_worker is not None + enable_remote_activities=bool(config.get("activities")) and not config.get("no_remote_activities"), enable_nexus=self._nexus_worker is not None, ), From 80ea7ede42157bd32846367c9f63ab344ecf94e8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 10:55:32 -0700 Subject: [PATCH 16/79] Named the async Topic base AsyncTopicBase, and fixed the test warnings. read_spec takes any Topic object, so the base is no longer used in its own module, as SyncTopicBase isn't. basedpyright and mypy are clean. --- temporalio/common/streams/_access.py | 4 ++-- temporalio/common/streams/_activity_stream.py | 4 ++-- temporalio/common/streams/_stream_client.py | 4 ++-- .../contrib/streams/examples/test_examples.py | 18 ++++++++---------- tests/contrib/streams/test_activity_handles.py | 5 ++--- tests/contrib/streams/test_backends.py | 12 ++++++------ tests/contrib/streams/test_workflow_module.py | 4 +--- 7 files changed, 23 insertions(+), 28 deletions(-) diff --git a/temporalio/common/streams/_access.py b/temporalio/common/streams/_access.py index 6ddc6e5e7d..948272bcec 100644 --- a/temporalio/common/streams/_access.py +++ b/temporalio/common/streams/_access.py @@ -189,8 +189,8 @@ def subscription( return Subscription(items(position), position, options) -class _AsyncTopic(TopicBase[T]): - # A Topic of an Activity or client handle. Each publish writes before it returns. +class AsyncTopicBase(TopicBase[T]): + """Base of a Topic of an Activity or client handle. A publish writes before it returns.""" async def publish(self, *values: T) -> int: """Write one Record for each value, in order, and return the last Record's offset. diff --git a/temporalio/common/streams/_activity_stream.py b/temporalio/common/streams/_activity_stream.py index f1600d34bf..09db973687 100644 --- a/temporalio/common/streams/_activity_stream.py +++ b/temporalio/common/streams/_activity_stream.py @@ -14,8 +14,8 @@ from temporalio.common.streams import _activity, _backend, _errors, _ids from temporalio.common.streams._access import ( Access, + AsyncTopicBase, SyncTopicBase, - _AsyncTopic, read_spec, subscription, ) @@ -98,7 +98,7 @@ async def ref() -> models.StreamRef: ) -class ActivityTopic(_AsyncTopic[T]): +class ActivityTopic(AsyncTopicBase[T]): """One Topic of an :py:class:`ActivityStream`. All Topics of a Stream share one order. .. warning:: diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index db5ea4fa85..b414fb505f 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -16,8 +16,8 @@ from temporalio.common.streams import _backend, _errors, _ids from temporalio.common.streams._access import ( Access, + AsyncTopicBase, ReadSpec, - _AsyncTopic, read_spec, subscription, ) @@ -64,7 +64,7 @@ def _client_producer( ) -class ClientTopic(_AsyncTopic[T]): +class ClientTopic(AsyncTopicBase[T]): """One Topic of a :py:class:`ClientStream`. All Topics of a Stream share one order. .. warning:: diff --git a/tests/contrib/streams/examples/test_examples.py b/tests/contrib/streams/examples/test_examples.py index f297058d8f..4a12df48bc 100644 --- a/tests/contrib/streams/examples/test_examples.py +++ b/tests/contrib/streams/examples/test_examples.py @@ -108,7 +108,7 @@ async def test_worker_setup_on_redis(client: Client, redis_url: str): # The Jetty v2 examples. The Workflow reader runs on one backend: the memory Store, Redis or # the native server. The others need the native server, with Redis, or a memory Store named # `redis` in its place, so they run only when TEMPORAL_STREAMS_NATIVE_TARGET is set. -async def native_server(client: Client, *backends: streams.StreamBackend) -> Client: +async def native_server(*backends: streams.StreamBackend) -> Client: if not NATIVE_TARGET: pytest.skip( "TEMPORAL_STREAMS_NATIVE_TARGET names no server with the Stream service" @@ -130,8 +130,8 @@ def redis_stand_in( @pytest.mark.parametrize("lane", ["memory", "redis"]) -async def test_two_backends(lane: str, client: Client, request: pytest.FixtureRequest): - two = await native_server(client, redis_stand_in(lane, request)) +async def test_two_backends(lane: str, request: pytest.FixtureRequest): + two = await native_server(redis_stand_in(lane, request)) assert await two_backends.main(two, task_queue()) == { "audit": ["started", "summarized"], "live": ["hello", "there", "summary"], @@ -143,7 +143,7 @@ async def test_workflow_reader( lane: str, client: Client, request: pytest.FixtureRequest ): if lane == "native": - one = await native_server(client) + one = await native_server() else: config = client.config() config["stream_backends"] = [redis_stand_in(lane, request, default=True)] @@ -157,10 +157,8 @@ async def test_workflow_reader( @pytest.mark.parametrize("lane", ["memory", "redis"]) -async def test_migrate_redis_to_native( - lane: str, client: Client, request: pytest.FixtureRequest -): - moving = await native_server(client, redis_stand_in(lane, request, default=True)) +async def test_migrate_redis_to_native(lane: str, request: pytest.FixtureRequest): + moving = await native_server(redis_stand_in(lane, request, default=True)) assert await migrate_redis_to_native.main(moving, task_queue()) == [ "redis: hello", "redis: old chat goes on", @@ -169,8 +167,8 @@ async def test_migrate_redis_to_native( ] -async def test_standalone_activity_stream(client: Client): - native = await native_server(client) +async def test_standalone_activity_stream(): + native = await native_server() assert await standalone_activity_stream.main(native, task_queue()) == [ "find", "the", diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index 4fbd2fb83c..ee3c516667 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -364,9 +364,8 @@ async def test_store_refuses_another_chains_close(stream_client: Client): stream = owner.stream(Events) items = [item async for item in stream.subscribe(wait_new_records=False)] # The server closed the native Stream when its run chain ended. - assert [type(item) for item in items] == ( - [streams.Closed] if native else [streams.Record] - ) + expected: list[type] = [streams.Closed] if native else [streams.Record] + assert [type(item) for item in items] == expected @dataclass diff --git a/tests/contrib/streams/test_backends.py b/tests/contrib/streams/test_backends.py index 9ad63feef6..2981b5e523 100644 --- a/tests/contrib/streams/test_backends.py +++ b/tests/contrib/streams/test_backends.py @@ -87,7 +87,7 @@ async def run(self) -> list[str | None]: async def read_all( - client: Client, handle: WorkflowHandle[Any, Any], name: str, backend: str + handle: WorkflowHandle[Any, Any], name: str, backend: str ) -> list[object]: stream = handle.stream(streams.with_name(Events, name), backend=backend) return [item async for item in stream.subscribe(wait_new_records=False)] @@ -104,8 +104,8 @@ async def test_one_worker_serves_two_backends(client: Client, redis_url: str): WritesBoth.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await handle.result() == ["redis", "memory"] - on_redis = await read_all(two, handle, "a", "redis") - on_memory = await read_all(two, handle, "b", "memory") + on_redis = await read_all(handle, "a", "redis") + on_memory = await read_all(handle, "b", "memory") assert any(isinstance(item, streams.Superseded) for item in on_redis) assert any(isinstance(item, streams.Superseded) for item in on_memory) assert data(on_redis)[-1] == "redis activity 2" @@ -118,7 +118,7 @@ async def test_one_worker_serves_two_backends(client: Client, redis_url: str): assert closed.result == ["redis", "memory"] # Each Stream is only in its own backend. with pytest.raises(streams.StreamNotFoundError): - await read_all(two, handle, "b", "redis") + await read_all(handle, "b", "redis") @workflow.defn @@ -137,7 +137,7 @@ async def test_default_marks_the_worker_default(client: Client, redis_url: str): CreatesDefault.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await handle.result() == "memory" - assert data(await read_all(two, handle, "events", "memory")) == ["default"] + assert data(await read_all(handle, "events", "memory")) == ["default"] @activity.defn @@ -214,6 +214,6 @@ async def test_continued_run_closes_each_backend(client: Client, redis_url: str) ) assert await handle.result() == "last run" for name, backend in (("a", "redis"), ("b", "memory")): - items = await read_all(two, handle, name, backend) + items = await read_all(handle, name, backend) assert isinstance(items[-1], streams.Closed) assert items[-1].result == "last run" diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index c2b5bb1304..fd18ff1084 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -286,9 +286,7 @@ async def run(self, backend: str | None) -> None: await workflow.create_stream(Events, backend=backend) -async def test_worker_config_errors_fail_the_task( - client: Client, stream_client: Client -): +async def test_worker_config_errors_fail_the_task(stream_client: Client): # Every Worker has the native backend, so only a backend that the Worker lacks fails the # Workflow Task, and a Worker with the right config can take the run on. async with new_worker(stream_client, CreatesOn) as worker: From a66eab7e367953c68e6cf964ab406c635a548030 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:16:35 -0700 Subject: [PATCH 17/79] Closed a Workflow Stream with an Activity committed with its task. A Local Activity close outlived a Workflow Task that failed after it, so the retry's create failed the run. The block end now leaves the Stream open in a cron run and on a failure that a retry run follows. --- temporalio/common/streams/_workflow.py | 77 +++++++++--- temporalio/common/streams/_writes.py | 26 ++-- temporalio/workflow/_streams.py | 13 +- tests/contrib/streams/test_redis.py | 49 ++++++-- tests/contrib/streams/test_workflow_module.py | 111 +++++++++++++++++- 5 files changed, 236 insertions(+), 40 deletions(-) diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index b2f7a86e28..58f72f3eb6 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -5,10 +5,12 @@ an argument, such as a :py:class:`WorkflowStream` that a run passes on through continue-as-new, works the same. -Each create, publish and close is a Local Activity, so it has a place in the run's commands, -as an Activity call has. To add, remove or move one in a Workflow that has open runs, gate the -change with ``workflow.patched``, else the replay of those runs fails as nondeterministic. -Publishes that wait behind a write to the same Stream go as one Local Activity. +Each create and publish is a Local Activity, and each close is a regular Activity, so each has +a place in the run's commands. To add, remove or move one in a Workflow that has open runs, gate +the change with ``workflow.patched``, else the replay of those runs fails as nondeterministic. +Publishes that wait behind a write to the same Stream go as one Local Activity. A close is +committed with the Workflow Task that schedules it, so a task that fails, or a crash, before +then leaves the Stream open for the task's retry. """ from __future__ import annotations @@ -22,6 +24,7 @@ import google.protobuf.message import temporalio.api.streamservice.v1 as stream_pb +import temporalio.common import temporalio.converter import temporalio.exceptions import temporalio.workflow @@ -39,12 +42,17 @@ E = TypeVar("E") T = TypeVar("T") -# The backend's Local Activities that write for Workflow code. The marker keeps the answer only, -# since a Local Activity's input is not in History. +# The backend's Activities that write for Workflow code. The create and the append are Local +# Activities, whose marker keeps the answer only, since a Local Activity's input is not in +# History. The close is a regular Activity on the Workflow's task queue. CREATE_ACTIVITY = "__temporal_streams_create" APPEND_ACTIVITY = "__temporal_streams_append" CLOSE_ACTIVITY = "__temporal_streams_close" _WRITE_TIMEOUT = timedelta(seconds=30) +# A Store error is retryable and every other Stream error is not, as each one says itself. +_CLOSE_RETRY = temporalio.common.RetryPolicy( + initial_interval=timedelta(seconds=1), maximum_interval=timedelta(seconds=30) +) # The most that one merged append takes before the codec. The Local Activity checks the size # after the codec, and sends each call as its own batch when the codec grew the merge past the # Store's 4 MiB batch limit. Half of it leaves room for most codecs. @@ -229,6 +237,19 @@ async def run() -> None: result_type=list[Any], start_to_close_timeout=_WRITE_TIMEOUT, ) + elif activity == CLOSE_ACTIVITY: + # A close that a failed Workflow Task scheduled never runs, so the task's retry + # still writes to the Stream. A Local Activity would close it before the task + # completes. + answers = [ + await temporalio.workflow.execute_activity( + activity, + write.request, + result_type=int, + start_to_close_timeout=_WRITE_TIMEOUT, + retry_policy=_CLOSE_RETRY, + ) + ] else: answers = [ await temporalio.workflow.execute_local_activity( @@ -278,7 +299,7 @@ def done(future: asyncio.Future[int]) -> None: async def drain() -> None: - """Wait for the run's writes that nobody awaits, before the run ends.""" + """Wait for the run's writes and closes that nobody awaits, before the run ends.""" for task in list(_state().tails.values()): try: await task @@ -331,7 +352,8 @@ class WorkflowStream(StreamHandle[E]): """A Stream in Workflow code. Workflow code writes and closes it, and can't read it. Pass it to an Activity, a child Workflow or the next run of continue-as-new as an - argument. Use it in ``async with`` to close it at the end of the block. + argument. Use it in ``async with`` to close it at the end of the block. A run end closes + nothing, so a Stream that the run chain never closes stays open until its ``safety_ttl``. .. warning:: This API is experimental. @@ -409,11 +431,13 @@ def _append(self, topic: str, values: tuple[Any, ...]) -> asyncio.Future[int]: async def close( self, *, result: Any | None = None, failure: BaseException | None = None ) -> None: - """Close the Stream with one Local Activity, after the writes before it. + """Close the Stream with one regular Activity, after the writes before it. - Readers get the Records before it, then a ``Closed`` item with ``result`` or - ``failure``. A ``CancelledError`` failure closes it as canceled. Later writes in this run - raise :py:class:`StreamClosedError`. + The Activity runs once the Workflow Task that schedules it completes, so a task that + fails before then, or a crash, leaves the Stream open, and the task's retry writes to it + again. It closes as the run chain, the Stream's owner. Readers get the Records before + it, then a ``Closed`` item with ``result`` or ``failure``. A ``CancelledError`` failure + closes it as canceled. Later writes in this run raise :py:class:`StreamClosedError`. Raises: StreamNotOwnerError: This run chain didn't create the Stream. @@ -425,8 +449,9 @@ def close_nowait( ) -> None: """Close the Stream with no await, for sync Workflow code. - It is to :py:meth:`close` what ``asyncio.Queue.put_nowait`` is to ``put``. A backend - error fails the Workflow Task. + It is to :py:meth:`close` what ``asyncio.Queue.put_nowait`` is to ``put``. It schedules + the close Activity, and the end of the run waits for it. A backend error fails the + Workflow Task. """ _write_nowait(self._close(result, failure)) @@ -437,7 +462,7 @@ def _close(self, result: Any, failure: BaseException | None) -> asyncio.Future[i info.first_execution_run_id, ): # Only the run chain that created a Stream ends it for every reader. The backend - # refuses it too, and this error comes before a Local Activity is spent. + # refuses it too, and this error comes before an Activity is spent. raise _errors.StreamNotOwnerError( f"Can't close the Stream {self.id}, since this run chain didn't create it", reason="stream_not_owner", @@ -467,16 +492,32 @@ async def __aexit__( ) -> None: """Close the Stream, with the exception as its failure if there is one. - A cancel closes it with a ``CancelledError`` failure. Another ``BaseException``, such as - an eviction, leaves it open, since the block did not end in Workflow code. + A cancel closes it with a ``CancelledError`` failure. The block end leaves the Stream + open in a cron run, since the Stream spans the cron chain and the next run creates it + again. It also leaves it open on an exception that fails the run when the run has a + retry policy, since a retry run writes on to the same Stream, and on an exception that + fails only the Workflow Task, since the task's retry runs the block again. Another + ``BaseException``, such as an eviction, leaves it open, since the block did not end in + Workflow code. An explicit :py:meth:`close` always closes. """ if _state().stream(self._model_ref()).closed: return close, failure = exit_failure(exc) - if close: + if close and _closes_at_block_end(exc): await self.close(failure=failure) +def _closes_at_block_end(exc: BaseException | None) -> bool: + info = temporalio.workflow.info() + if info.cron_schedule: + return False + if exc is None or isinstance(exc, asyncio.CancelledError): + return True + if not temporalio.workflow._Runtime.current().workflow_is_failure_exception(exc): + return False + return info.retry_policy is None + + def _own_handle(definition: type[E], backend: str | None) -> WorkflowStream[E]: info = temporalio.workflow.info() return WorkflowStream( diff --git a/temporalio/common/streams/_writes.py b/temporalio/common/streams/_writes.py index c0a1fc8dcf..a9b91f91a7 100644 --- a/temporalio/common/streams/_writes.py +++ b/temporalio/common/streams/_writes.py @@ -1,7 +1,8 @@ -"""The Local Activities that write for Workflow code. +"""The Activities that write for Workflow code. -Each Workflow ``create_stream`` and ``close`` is one of them, and the publishes that wait -behind a write to the same Stream share one. History has the answer and not the values. +Each Workflow ``create_stream`` is a Local Activity, and the publishes that wait behind a write +to the same Stream share one. History has the answer and not the values. Each Workflow +``close`` is a regular Activity, so it runs only once its Workflow Task completed. """ from __future__ import annotations @@ -13,7 +14,7 @@ import temporalio.activity import temporalio.api.streamservice.v1 as stream_pb import temporalio.common -from temporalio.common.streams import _activity, _errors +from temporalio.common.streams import _activity, _errors, _ids from temporalio.common.streams._workflow import ( APPEND_ACTIVITY, CLOSE_ACTIVITY, @@ -228,8 +229,16 @@ async def append_records( @temporalio.activity.defn(name=CLOSE_ACTIVITY) async def close_stream(request: stream_pb.CloseStreamRequest) -> int: - """Close a Workflow's Stream.""" + """Close a Workflow's Stream, as the run chain that owns it.""" streams, service = _context() + # The Store refuses an Activity Producer's close of a Workflow's Stream, so the Activity + # writes the close as the run that scheduled it, and only for that run's chain. + if request.stream.workflow_id != streams.workflow_id: + raise _errors.StreamNotOwnerError( + f"Can't close the Stream {request.stream.id} for another Workflow", + reason="stream_not_owner", + ) + run_id = streams.producer.run_id failure: Any = None if request.HasField("failure"): failure = streams.converter.failure_converter.from_failure( @@ -238,13 +247,16 @@ async def close_stream(request: stream_pb.CloseStreamRequest) -> int: await service.close_stream( models.CloseStreamRequest( stream=_ref(request.stream), - producer=_producer(streams), + producer=models.Producer( + kind=stream_pb.ProducerKind.PRODUCER_KIND_WORKFLOW, + id=_ids.workflow_producer_id(run_id), + run_id=run_id, + ), result=temporalio.common.RawValue(request.result) if request.HasField("result") else None, failure=failure, status=request.status or None, - **_history(streams), ) ) return 0 diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 301e2a69c2..c7bc042c5b 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -24,9 +24,11 @@ def create_stream( """Create a Stream that this run chain owns. Its id is ``::``, so a client finds it from the Workflow - handle with no call. The create is one Local Activity, and its error raises here. Only the - run chain closes the Stream. On Redis, the end of the run chain closes nothing, so a Stream - that nobody closes stays open until its ``safety_ttl``. + handle with no call. The create and each publish are Local Activities, and the create's + error raises here. Only the run chain closes the Stream, with a regular Activity that runs + once its Workflow Task completed, so a task that fails before then leaves the Stream open + for its retry. On Redis, the end of the run chain closes nothing, so a Stream that nobody + closes stays open until its ``safety_ttl``. Await the call for the handle, or use it in ``async with``, which closes the Stream at the end of the block:: @@ -34,6 +36,11 @@ def create_stream( async with workflow.create_stream(streams.with_name(Events, "turn-3")) as turn: await turn.topic(Events.tokens).publish("hi") + The block end leaves the Stream open in a cron run, on an exception that fails the run when + the run has a retry policy, and on an exception that fails only the Workflow Task, since a + later run or the task's retry writes on to the same Stream. See + :py:meth:`temporalio.common.streams.WorkflowStream.__aexit__`. + .. warning:: This API is experimental. diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 58e77f2042..39c6a29a07 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -12,6 +12,7 @@ from temporalio import activity, workflow from temporalio.api.common.v1 import Payload +from temporalio.api.enums.v1 import EventType from temporalio.client import Client, Interceptor, WorkflowFailureError, WorkflowHandle from temporalio.common import RawValue, RetryPolicy, streams from temporalio.common.streams._extension import ( @@ -810,39 +811,67 @@ async def execute_workflow(self, input: ExecuteWorkflowInput) -> Any: @workflow.defn -class PublishesAndReturns: +class PublishesClosesAndReturns: @workflow.run async def run(self) -> str: stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("token") + await stream.close(result="done") return "done" +async def close_activity_events( + handle: WorkflowHandle[Any, Any], +) -> list[EventType.ValueType]: + # The events of the close Activity, from its schedule on. + history = await handle.fetch_history() + scheduled = { + event.event_id + for event in history.events + if event.activity_task_scheduled_event_attributes.activity_type.name + == "__temporal_streams_close" + } + return [ + event.event_type + for event in history.events + if event.event_id in scheduled + or event.activity_task_started_event_attributes.scheduled_event_id in scheduled + or event.activity_task_completed_event_attributes.scheduled_event_id + in scheduled + ] + + @pytest.mark.parametrize("cache", [0, 10]) -async def test_failed_last_task_is_withdrawn_by_its_retry( - stream_client: Client, cache: int -): +async def test_close_survives_a_failed_last_task(stream_client: Client, cache: int): + # The close is an Activity, committed with the Workflow Task that scheduled it. The last + # task fails after it, and its retry replays the close and writes nothing more. config = stream_client.config() config["interceptors"] = [_FailsTheLastTaskOnce()] client = Client(**config) async with new_worker( client, - PublishesAndReturns, + PublishesClosesAndReturns, max_cached_workflows=cache, workflow_runner=UnsandboxedWorkflowRunner(), ) as worker: handle = await client.start_workflow( - PublishesAndReturns.run, + PublishesClosesAndReturns.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) assert await handle.result() == "done" stream = workflow_stream(handle) items = [item async for item in stream.subscribe(wait_new_records=False)] - written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] - visible = [r for r in written if not any(s.withdraws(r) for s in superseded)] - assert [r.data for r in visible] == ["token"] + assert [r.data for r in records(items)] == ["token"] + closes = [item for item in items if isinstance(item, streams.Closed)] + assert len(closes) == 1 and items[-1] is closes[0] + assert closes[0].result == "done" + assert closes[0].status == streams.CloseStatus.COMPLETED + assert await close_activity_events(handle) == [ + EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED, + EventType.EVENT_TYPE_ACTIVITY_TASK_STARTED, + EventType.EVENT_TYPE_ACTIVITY_TASK_COMPLETED, + ] @workflow.defn diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index baaeafee92..ad6c05dc6a 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -17,14 +17,14 @@ WorkflowFailureError, WorkflowHandle, ) -from temporalio.common import streams +from temporalio.common import RetryPolicy, streams from temporalio.common.streams._backend import _ExtensionCalls from temporalio.common.streams._defn import definition_of from temporalio.contrib import stream as redis from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.converter import DataConverter, PayloadCodec from temporalio.exceptions import ApplicationError -from temporalio.worker import Replayer +from temporalio.worker import Replayer, UnsandboxedWorkflowRunner from tests.helpers import assert_task_fail_eventually, new_worker @@ -272,6 +272,113 @@ async def test_run_end_closes_nothing(stream_client: Client): assert not any(isinstance(item, streams.Closed) for item in items) +@workflow.defn +class FailsInItsBlockOnce: + @workflow.run + async def run(self) -> str: + attempt = workflow.info().attempt + async with workflow.create_stream(Events) as stream: + await stream.topic(Events.tokens).publish(f"run-{attempt}") + if attempt == 1: + raise ApplicationError("the first run fails") + return f"done on attempt {attempt}" + + +async def test_block_end_leaves_the_stream_open_for_a_retry_run(stream_client: Client): + async with new_worker(stream_client, FailsInItsBlockOnce) as worker: + handle = await stream_client.start_workflow( + FailsInItsBlockOnce.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + retry_policy=RetryPolicy( + initial_interval=timedelta(milliseconds=100), maximum_attempts=3 + ), + ) + assert await handle.result() == "done on attempt 2" + items = await read(handle) + written = [item for item in items if isinstance(item, streams.Record)] + superseded = [item for item in items if isinstance(item, streams.Superseded)] + assert [r.data for r in written] == ["run-1", "run-2"] + assert [r.data for r in written if superseded[0].withdraws(r)] == ["run-1"] + # Only the block end of the run that succeeds closes the Stream. + closes = [item for item in items if isinstance(item, streams.Closed)] + assert len(closes) == 1 and items[-1] is closes[0] + assert closes[0].failure is None + + +@workflow.defn +class CronBlock: + @workflow.run + async def run(self) -> str: + async with workflow.create_stream(Events) as stream: + await stream.topic(Events.tokens).publish("turn") + return "done" + + +async def test_block_end_leaves_a_cron_stream_open(stream_client: Client): + # The Stream spans the cron chain, so a close would fail the next run's create. + async with new_worker(stream_client, CronBlock) as worker: + handle = await stream_client.start_workflow( + CronBlock.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + cron_schedule="@every 1s", + ) + reader = handle.stream(Events).subscribe(create_wait=timedelta(seconds=20)) + first = await asyncio.wait_for(anext(reader), 20) + second = await asyncio.wait_for(anext(reader), 20) + await reader.aclose() + await handle.terminate() + assert isinstance(first, streams.Record) and isinstance(second, streams.Record) + items = await read(handle) + assert not any(isinstance(item, streams.Closed) for item in items) + + +failed_close_tasks: set[str] = set() + + +@workflow.defn +class SchedulesACloseThenFailsItsTask: + @workflow.run + async def run(self) -> None: + stream = await workflow.create_stream(Events) + await stream.topic(Events.tokens).publish("token") + stream.close_nowait(result="closed") + workflow_id = workflow.info().workflow_id + if workflow_id not in failed_close_tasks: + failed_close_tasks.add(workflow_id) + raise RuntimeError("fail the Workflow Task that schedules the close") + + +async def test_failed_task_writes_no_close(stream_client: Client): + # The test's set must survive the retry of the task, so no sandbox. + async with new_worker( + stream_client, + SchedulesACloseThenFailsItsTask, + workflow_runner=UnsandboxedWorkflowRunner(), + ) as worker: + handle = await stream_client.start_workflow( + SchedulesACloseThenFailsItsTask.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await handle.result() + items = await read(handle) + closes = [item for item in items if isinstance(item, streams.Closed)] + assert len(closes) == 1 and closes[0].result == "closed" + history = await handle.fetch_history() + # The failed task's close never went out, so History has one close Activity. + scheduled = [ + event + for event in history.events + if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED + ] + assert [ + event.activity_task_scheduled_event_attributes.activity_type.name + for event in scheduled + ] == ["__temporal_streams_close"] + + @workflow.defn class CreatesOn: @workflow.run From c9323c1df63029f2f217f9b7da2517b9d421c13a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:20:12 -0700 Subject: [PATCH 18/79] Tried a failed first-run lookup of a Workflow's Stream again. A ClientStream kept a failed or cancelled Describe forever, and the handle map kept a failed one with its handle. A found id now stays as text, and a failed call goes. --- temporalio/common/streams/_stream_client.py | 45 ++++++++++++---- tests/contrib/streams/test_stream_client.py | 58 +++++++++++++++++++++ 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index 1f688b0366..8cdb0f74d3 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import functools import itertools import uuid import weakref @@ -157,8 +158,9 @@ def id(self) -> str: async def _stream_id(self) -> str: if not self._id: - # Calls that start together share one Describe call. - if self._finding is None: + # Calls that start together share one Describe call. A failed call does not + # stay, so a later call tries again. + if self._finding is None or _failed(self._finding): assert self._find_first_run is not None self._finding = asyncio.ensure_future(self._find_first_run()) first_run = await asyncio.shield(self._finding) @@ -447,13 +449,17 @@ def client_stream( return handle._bind(client, calls) -# The first run id that one Describe call found for a Workflow handle, so the Streams of one -# handle share the call. +# The first run id of a Workflow handle, or the Describe call that looks for it, so the +# Streams of one handle share the call. _FIRST_RUNS: weakref.WeakKeyDictionary[ - temporalio.client.WorkflowHandle[Any, Any], asyncio.Future[str] + temporalio.client.WorkflowHandle[Any, Any], asyncio.Future[str] | str ] = weakref.WeakKeyDictionary() +def _failed(future: asyncio.Future[Any]) -> bool: + return future.done() and (future.cancelled() or future.exception() is not None) + + async def _describe_first_run( handle: temporalio.client.WorkflowHandle[Any, Any], ) -> str: @@ -470,12 +476,29 @@ async def _describe_first_run( async def _first_run(handle: temporalio.client.WorkflowHandle[Any, Any]) -> str: - finding = _FIRST_RUNS.get(handle) - if finding is None or (finding.done() and finding.exception() is not None): - # A failed call does not stay, so a later call tries again. - finding = asyncio.ensure_future(_describe_first_run(handle)) - _FIRST_RUNS[handle] = finding - return await asyncio.shield(finding) + known = _FIRST_RUNS.get(handle) + if isinstance(known, str): + return known + if known is None: + known = asyncio.ensure_future(_describe_first_run(handle)) + _FIRST_RUNS[handle] = known + known.add_done_callback(functools.partial(_settle, weakref.ref(handle))) + return await asyncio.shield(known) + + +def _settle( + handle_ref: weakref.ref[temporalio.client.WorkflowHandle[Any, Any]], + finding: asyncio.Future[str], +) -> None: + # A found id stays as text. A failed call goes, so a later call tries again, and the + # failure's traceback does not keep the handle in the map. + handle = handle_ref() + if handle is None or _FIRST_RUNS.get(handle) is not finding: + return + if _failed(finding): + del _FIRST_RUNS[handle] + else: + _FIRST_RUNS[handle] = finding.result() def workflow_stream( diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index caf80fdbef..8db63162e1 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -10,6 +10,7 @@ from temporalio import workflow from temporalio.client import Client from temporalio.common import streams +from temporalio.common.streams import _stream_client from temporalio.contrib import stream as redis from temporalio.contrib.stream import RedisStreamBackend from temporalio.contrib.stream.testing import MemoryStreamBackend @@ -98,6 +99,63 @@ class Description: assert calls == [1] +async def test_failed_first_run_lookup_is_tried_again(client: Client): + client, _ = fake_client(client) + handle = client.get_workflow_handle("agent-4") + describes = [] + + async def describe(**_: Any) -> Any: + describes.append(1) + if len(describes) == 1: + raise RuntimeError("the Workflow has not started yet") + info = SimpleNamespace(first_run_id="run-4") + return SimpleNamespace( + raw_description=SimpleNamespace(workflow_execution_info=info) + ) + + setattr(handle, "describe", describe) + await redis.create(client, Events, id="agent-4:run-4:events") + stream = handle.stream(Events) + with pytest.raises(RuntimeError, match="not started"): + await stream.get_offset() + assert await stream.get_offset() == 0 + assert stream.id == "agent-4:run-4:events" + # The handle keeps the id it found, so its other Streams make no call. + assert await handle.stream(Events).get_offset() == 0 + assert describes == [1, 1] + + +async def test_cancelled_first_run_lookup_is_tried_again(client: Client): + client, _ = fake_client(client) + handle = client.get_workflow_handle("agent-5") + started = asyncio.Event() + + async def hangs(**_: Any) -> Any: + started.set() + await asyncio.Event().wait() + + setattr(handle, "describe", hangs) + stream = handle.stream(Events) + offset = asyncio.ensure_future(stream.get_offset()) + await started.wait() + # The Describe call is cancelled, such as when the loop that ran it stops. + lookup = _stream_client._FIRST_RUNS[handle] + assert isinstance(lookup, asyncio.Future) + lookup.cancel() + with pytest.raises(asyncio.CancelledError): + await offset + + async def describe(**_: Any) -> Any: + info = SimpleNamespace(first_run_id="run-5") + return SimpleNamespace( + raw_description=SimpleNamespace(workflow_execution_info=info) + ) + + setattr(handle, "describe", describe) + await redis.create(client, Events, id="agent-5:run-5:events") + assert await stream.get_offset() == 0 + + @workflow.defn class WriteToStream: @workflow.run From e7271cdf5c97e5ba828ddc87bab748e48209468d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:22:17 -0700 Subject: [PATCH 19/79] Tightened the Stream backend registration. A Replayer serves one backend as a Worker does, a Worker configures its backends after its plugins, and a generator of backends stays. A backend in plugins= fails before the connect, and the per-client calls are private. --- temporalio/client/_client.py | 2 +- temporalio/common/streams/_extension.py | 24 ++++++++----- temporalio/common/streams/_plugin.py | 39 +++++++++++++-------- temporalio/worker/_replayer.py | 2 +- temporalio/worker/_worker.py | 11 +++--- tests/contrib/streams/fake_extension.py | 17 +++++---- tests/contrib/streams/test_stream_client.py | 32 +++++++++++++++-- 7 files changed, 88 insertions(+), 39 deletions(-) diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index 176243bb55..e3f823078c 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -304,7 +304,7 @@ def __init__( interceptors=interceptors, default_workflow_query_reject_condition=default_workflow_query_reject_condition, header_codec_behavior=header_codec_behavior, - stream_backends=stream_backends, + stream_backends=list(stream_backends), ) self._initial_config = config.copy() diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index 23384ce19c..c542d09ad5 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -101,8 +101,8 @@ def _configure_client( ] return config - def backend_for(self, client: temporalio.client.Client) -> BackendCalls: - """The Stream runtime that this backend gave ``client``.""" + def _backend_for(self, client: temporalio.client.Client) -> BackendCalls: + # The Stream runtime that this backend gave the client. interceptor = self._interceptor_of( client.config(active_config=True)["interceptors"] ) @@ -125,7 +125,7 @@ def _interceptor_of(self, interceptors: Sequence[Any]) -> _StreamInterceptor | N def _configure_worker( self, config: temporalio.worker.WorkerConfig ) -> temporalio.worker.WorkerConfig: - # Add the backend's Activities, and its interceptor when the client has none. + # Add the backend's Activities. The Worker takes the interceptor from its client. config = super()._configure_worker(config) client = config["client"] # type: ignore[reportTypedDictNotRequiredAccess] interceptors = [ @@ -140,14 +140,14 @@ def _configure_worker( "The Worker's client has another Stream backend. For now a Worker serves " "one backend." ) - # Records get no serialization context, so Workflows, Activities and clients encode - # them alike. mine = self._interceptor_of(interceptors) if mine is None: - mine = _StreamInterceptor( - self, self._new_backend(_runtime_of(client.service_client)), None + raise StreamConfigError( + f"The Worker's client has no {type(self).__name__}. Register it with " + "stream_backends=." ) - config["interceptors"] = [*config.get("interceptors", []), mine] + # Records get no serialization context, so Workflows, Activities and clients encode + # them alike. mine.converter = client.data_converter runner = config.get("workflow_runner") if isinstance(runner, SandboxedWorkflowRunner): @@ -166,6 +166,14 @@ def _configure_replayer( ) -> temporalio.worker.ReplayerConfig: # A replay runs no Activity, so it needs the Workflow interceptor and no runtime. config = super()._configure_replayer(config) + if any( + isinstance(other, _StreamInterceptor) and other.plugin is not self + for other in config.get("interceptors", []) + ): + raise StreamConfigError( + "The Replayer has another Stream backend. For now a Replayer serves one " + "backend, as a Worker does." + ) config["interceptors"] = [ *config.get("interceptors", []), _StreamInterceptor(self, None, None), diff --git a/temporalio/common/streams/_plugin.py b/temporalio/common/streams/_plugin.py index 129e8cebf5..cf71ce70b1 100644 --- a/temporalio/common/streams/_plugin.py +++ b/temporalio/common/streams/_plugin.py @@ -2,10 +2,11 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Awaitable, Callable, Sequence from typing import Any import temporalio.client +import temporalio.service import temporalio.worker from temporalio.common.streams import _errors from temporalio.common.streams._backend import BackendCalls @@ -42,16 +43,11 @@ def __init__( *, backend_name: str, default: bool, - calls: BackendCalls | None = None, **options: Any, ) -> None: - """Create the backend. ``options`` go to :py:class:`temporalio.plugin.SimplePlugin`. - - A backend with no ``calls`` gives each client its own in :py:meth:`backend_for`. - """ + """Create the backend. ``options`` go to :py:class:`temporalio.plugin.SimplePlugin`.""" self._backend_name = backend_name self._default = default - self._calls = calls super().__init__(name, **options) @property @@ -64,14 +60,23 @@ def default(self) -> bool: """Whether a create with no backend name uses this backend.""" return self._default - def backend_for( + def _backend_for( self, client: temporalio.client.Client, # pyright: ignore[reportUnusedParameter] ) -> BackendCalls: - """The calls of this backend for ``client`` and its Workers.""" - if self._calls is None: - raise NotImplementedError(f"{type(self).__name__} gives no calls") - return self._calls + # The calls of this backend for the client and its Workers. Each backend gives its own. + raise NotImplementedError(f"{type(self).__name__} gives no calls") + + async def connect_service_client( + self, + config: temporalio.service.ConnectConfig, + next: Callable[ + [temporalio.service.ConnectConfig], + Awaitable[temporalio.service.ServiceClient], + ], + ) -> temporalio.service.ServiceClient: + """Refuse a backend in ``plugins=``, before the client connects.""" + raise _not_a_plugin(self) def configure_client( self, config: temporalio.client.ClientConfig @@ -159,7 +164,9 @@ def configure_client( config: temporalio.client.ClientConfig, ) -> temporalio.client.ClientConfig: """Check the client's backends, and add the parts of each to the client.""" - for backend in check_backends(config.get("stream_backends", ()), "client"): + backends = check_backends(config.get("stream_backends", ()), "client") + config["stream_backends"] = backends + for backend in backends: config = backend._configure_client(config) return config @@ -177,7 +184,9 @@ def configure_replayer( config: temporalio.worker.ReplayerConfig, ) -> temporalio.worker.ReplayerConfig: """Check the Replayer's backends, and add the parts of each to it.""" - for backend in check_backends(config.get("stream_backends", ()), "Replayer"): + backends = check_backends(config.get("stream_backends", ()), "Replayer") + config["stream_backends"] = backends + for backend in backends: config = backend._configure_replayer(config) return config @@ -200,7 +209,7 @@ def client_backend( wanted = name or default_name(backends) for backend in backends: if backend.backend_name == wanted: - return wanted, backend.backend_for(client) + return wanted, backend._backend_for(client) if not backends: raise _errors.StreamConfigError( "The client has no Stream backend. Register one with " diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 92a05909ca..528db5c77a 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -93,7 +93,7 @@ def __init__( runtime=runtime, disable_safe_workflow_eviction=disable_safe_workflow_eviction, header_codec_behavior=header_codec_behavior, - stream_backends=stream_backends, + stream_backends=list(stream_backends), ) self._initial_config = self._config.copy() self._default_workflow_logic_flags = set(_DEFAULT_ENABLED_WORKFLOW_LOGIC_FLAGS) diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 8b30d9fc42..fba239d25d 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -407,18 +407,17 @@ def __init__( plugins = plugins_from_client + list(plugins) self._initial_config = config.copy() - # The Worker takes its client's Stream backends, before the plugins as a client's - # plugins come first. + self._plugins = plugins + for plugin in plugins: + config = plugin.configure_worker(config) + # The Worker takes its client's Stream backends after the plugins, as the client does, + # so a plugin that replaces the Workflow runner keeps the Streams passthrough. stream_backends = client.config(active_config=True).get("stream_backends") if stream_backends: from temporalio.common.streams import _plugin as _stream_plugin config = _stream_plugin.configure_worker(stream_backends, config) - self._plugins = plugins - for plugin in plugins: - config = plugin.configure_worker(config) - self._init_from_config(client, config) def _init_from_config(self, client: temporalio.client.Client, config: WorkerConfig): diff --git a/tests/contrib/streams/fake_extension.py b/tests/contrib/streams/fake_extension.py index bf49aafadf..b2cdd59a05 100644 --- a/tests/contrib/streams/fake_extension.py +++ b/tests/contrib/streams/fake_extension.py @@ -155,16 +155,21 @@ class _FakeBackend(StreamBackend): # The fake keeps Redis's rules, so the Redis calls take it. _kind = "redis" + def __init__(self, calls: _ExtensionCalls) -> None: + super().__init__("fake-streams", backend_name="fake", default=True) + self._calls = calls + + def _backend_for( + self, + client: Client, # pyright: ignore[reportUnusedParameter] + ) -> _ExtensionCalls: + return self._calls + def fake_client(client: Client) -> tuple[Client, FakeExtension]: """A copy of ``client`` whose one Stream backend, ``fake``, is a fake runtime.""" extension = FakeExtension() - backend = _FakeBackend( - "fake-streams", - backend_name="fake", - default=True, - calls=_ExtensionCalls(extension), - ) + backend = _FakeBackend(_ExtensionCalls(extension)) config = client.config() config["stream_backends"] = [backend] return Client(**config), extension diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 8db63162e1..611e436e5e 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import timedelta from types import SimpleNamespace -from typing import Any +from typing import Any, cast import pytest @@ -15,8 +15,10 @@ from temporalio.contrib.stream import RedisStreamBackend from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.converter import DataConverter +from temporalio.plugin import SimplePlugin from temporalio.testing import WorkflowEnvironment from temporalio.worker import Replayer +from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner from tests.contrib.streams.fake_extension import fake_client from tests.helpers import new_worker @@ -412,16 +414,30 @@ async def test_registration_rules(client: Client): config["stream_backends"] = [MemoryStreamBackend(), MemoryStreamBackend()] with pytest.raises(streams.StreamConfigError, match="two Stream backends"): Client(**config) - # For now a Worker serves one backend. + # For now a Worker serves one backend, and so does a Replayer. config["stream_backends"] = [ MemoryStreamBackend(), MemoryStreamBackend(name="b", default=True), ] with pytest.raises(streams.StreamConfigError, match="one backend"): new_worker(Client(**config), Agent) + with pytest.raises(streams.StreamConfigError, match="one backend"): + Replayer(workflows=[Agent], stream_backends=config["stream_backends"]) + # A generator of backends works too, and a copy of the client keeps them. + config["stream_backends"] = cast( + Any, (b for b in [MemoryStreamBackend(default=True)]) + ) + from_generator = Client(**config) + assert from_generator.stream(Events, id="d-3").backend == "memory" + assert ( + Client(**from_generator.config()).stream(Events, id="d-3").backend == "memory" + ) async def test_backend_is_not_a_plugin(client: Client): + # The connect refuses it before it connects, so the unreachable target doesn't matter. + with pytest.raises(streams.StreamConfigError, match="not a plugin"): + await Client.connect("localhost:1", plugins=[MemoryStreamBackend()]) config = client.config() config["plugins"] = [MemoryStreamBackend()] with pytest.raises(streams.StreamConfigError, match="not a plugin"): @@ -432,6 +448,18 @@ async def test_backend_is_not_a_plugin(client: Client): Replayer(workflows=[Agent], plugins=[MemoryStreamBackend()]) +async def test_worker_plugin_keeps_the_streams_passthrough(client: Client): + # The backends configure the Worker after its plugins, so a plugin that replaces the + # Workflow runner doesn't drop the passthrough. + config = client.config() + config["stream_backends"] = [MemoryStreamBackend(default=True)] + plugin = SimplePlugin("replaces-runner", workflow_runner=SandboxedWorkflowRunner()) + worker = new_worker(Client(**config), Agent, plugins=[plugin]) + runner = worker.config(active_config=True).get("workflow_runner") + assert isinstance(runner, SandboxedWorkflowRunner) + assert "temporalio.common.streams" in runner.restrictions.passthrough_modules + + async def test_default_is_native_with_no_default_backend(client: Client): config = client.config() config["stream_backends"] = [MemoryStreamBackend()] From df4c0d9ddb2b6e1e8c244110f5145c21b289870a Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:22:40 -0700 Subject: [PATCH 20/79] Corrected the Workflow Stream docs on backends and on init. A Worker serves one backend, so the backend-move recipe can't run. A continued run reads its Streams' backends after @workflow.init. --- temporalio/workflow/_streams.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index c7bc042c5b..69bec68721 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -50,8 +50,8 @@ def create_stream( retention: How long the Records stay after the close. Defaults to the backend's. backend: The name of the backend. Defaults to the registered backend with ``default=True``. The default is part of the Workflow's code, since a replay must - create on the same backend. To move a Workflow to another backend, gate it with a - patch, for example ``backend="redis-b" if workflow.patched("move") else "redis"``. + create on the same backend. For now a Worker serves one backend, so the name is + that of the Worker's backend. Raises: StreamError: The backend refused the create, such as for another Retention. @@ -93,7 +93,9 @@ def stream_handle( """A Stream that this run chain created, by its definition. It makes no call. A later run of the chain, such as one after continue-as-new, uses it to write or close the - Stream again. + Stream again. In ``@workflow.init`` of a run that continued as new, the run has not read + the backends of the chain's Streams yet, so pass ``backend=`` there, or call it in the run + method. .. warning:: This API is experimental. From 74b759cf755caa471eacfd9bd52f92439fb1ef00 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:41:11 -0700 Subject: [PATCH 21/79] Put the Workflow close behind a backend method. Redis keeps its regular Activity close, and a backend that closes another way overrides _close_in_workflow. Workers that share a task queue all register the Redis backend, since it polls Activities there. --- temporalio/common/streams/_extension.py | 29 ++++++- temporalio/common/streams/_workflow.py | 83 ++++++++++--------- temporalio/contrib/stream/__init__.py | 4 + temporalio/contrib/stream/testing.py | 3 +- temporalio/workflow/_streams.py | 8 +- tests/contrib/streams/test_workflow_module.py | 21 +++++ 6 files changed, 101 insertions(+), 47 deletions(-) diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index c542d09ad5..32683193a2 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -7,6 +7,7 @@ import logging import weakref from collections.abc import Mapping, Sequence +from datetime import timedelta from typing import Any, NoReturn import temporalio.activity @@ -14,6 +15,7 @@ import temporalio.api.streamservice.v1 as stream_pb import temporalio.bridge.streams import temporalio.client +import temporalio.common import temporalio.converter import temporalio.runtime import temporalio.service @@ -36,6 +38,11 @@ _KEY_HEADER = "__temporal_stream_key" # Continue-as-new passes the run chain's open Streams to the next run in this header. _OWNED_STREAMS_HEADER = "__temporal_owned_streams" +_CLOSE_TIMEOUT = timedelta(seconds=30) +# A Store error is retryable and every other Stream error is not, as each one says itself. +_CLOSE_RETRY = temporalio.common.RetryPolicy( + initial_interval=timedelta(seconds=1), maximum_interval=timedelta(seconds=30) +) class ExtensionStreamBackend(StreamBackend): @@ -180,6 +187,19 @@ def _configure_replayer( ] return config + async def _close_in_workflow(self, request: stream_pb.CloseStreamRequest) -> int: + # Workflow code closes a Workflow's Stream here. The close must run only once the + # Workflow Task commits, so a task that fails before then leaves the Stream open for its + # retry. A regular Activity on the Workflow's task queue does that, and it closes as the + # run chain, the Stream's owner. A backend that closes another way overrides this. + return await temporalio.workflow.execute_activity( + streams_workflow.CLOSE_ACTIVITY, + request, + result_type=int, + start_to_close_timeout=_CLOSE_TIMEOUT, + retry_policy=_CLOSE_RETRY, + ) + async def shutdown(self) -> None: """Drop every open writer, including the client's. Call it once the process no longer writes Streams. @@ -224,6 +244,7 @@ def workflow_interceptor_class( "_WorkflowInbound", (_WorkflowInbound,), { + "_backend": self.plugin, "_backend_name": self.plugin.backend_name, "_default": self.plugin.default, "_kind": self.plugin._kind, @@ -237,13 +258,17 @@ def intercept_activity( class _WorkflowInbound(temporalio.worker.WorkflowInboundInterceptor): + _backend: ExtensionStreamBackend | None = None _backend_name = "" _default = False _kind = "" def init(self, outbound: temporalio.worker.WorkflowOutboundInterceptor) -> None: + assert self._backend is not None super().init( - _WorkflowOutbound(outbound, self._backend_name, self._default, self._kind) + _WorkflowOutbound( + outbound, self._backend, self._backend_name, self._default, self._kind + ) ) async def execute_workflow( @@ -300,11 +325,13 @@ class _WorkflowOutbound(temporalio.worker.WorkflowOutboundInterceptor): def __init__( self, next: temporalio.worker.WorkflowOutboundInterceptor, + backend: ExtensionStreamBackend, backend_name: str, default: bool, kind: str, ) -> None: super().__init__(next) + self._temporal_stream_backend = backend self._temporal_stream_backend_name = backend_name self._temporal_stream_default = default self._temporal_stream_kind = kind diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 58f72f3eb6..d6213fb682 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -5,12 +5,12 @@ an argument, such as a :py:class:`WorkflowStream` that a run passes on through continue-as-new, works the same. -Each create and publish is a Local Activity, and each close is a regular Activity, so each has -a place in the run's commands. To add, remove or move one in a Workflow that has open runs, gate -the change with ``workflow.patched``, else the replay of those runs fails as nondeterministic. -Publishes that wait behind a write to the same Stream go as one Local Activity. A close is -committed with the Workflow Task that schedules it, so a task that fails, or a crash, before -then leaves the Stream open for the task's retry. +Each create and publish is a Local Activity, and each close is a command that the backend +carries out once the Workflow Task commits, a regular Activity on Redis. So each has a place in +the run's commands. To add, remove or move one in a Workflow that has open runs, gate the change +with ``workflow.patched``, else the replay of those runs fails as nondeterministic. Publishes +that wait behind a write to the same Stream go as one Local Activity. A task that fails, or a +crash, before the close commits leaves the Stream open for the task's retry. """ from __future__ import annotations @@ -24,7 +24,6 @@ import google.protobuf.message import temporalio.api.streamservice.v1 as stream_pb -import temporalio.common import temporalio.converter import temporalio.exceptions import temporalio.workflow @@ -44,15 +43,11 @@ # The backend's Activities that write for Workflow code. The create and the append are Local # Activities, whose marker keeps the answer only, since a Local Activity's input is not in -# History. The close is a regular Activity on the Workflow's task queue. +# History. The Redis close is a regular Activity on the Workflow's task queue. CREATE_ACTIVITY = "__temporal_streams_create" APPEND_ACTIVITY = "__temporal_streams_append" CLOSE_ACTIVITY = "__temporal_streams_close" _WRITE_TIMEOUT = timedelta(seconds=30) -# A Store error is retryable and every other Stream error is not, as each one says itself. -_CLOSE_RETRY = temporalio.common.RetryPolicy( - initial_interval=timedelta(seconds=1), maximum_interval=timedelta(seconds=30) -) # The most that one merged append takes before the codec. The Local Activity checks the size # after the codec, and sends each call as its own batch when the codec grew the merge past the # Store's 4 MiB batch limit. Half of it leaves room for most codecs. @@ -136,20 +131,36 @@ class WorkerBackend(NamedTuple): kind: str -def worker_backend() -> WorkerBackend | None: - """The Worker's backend, which marks the run's outbound interceptor, or None.""" +def _transport() -> Any: + # The run's outbound interceptor that the Worker's backend marks, or None. outbound = getattr(temporalio.workflow._Runtime.current(), "_outbound", None) while outbound is not None: if getattr(outbound, "_temporal_stream_transport", False): - return WorkerBackend( - getattr(outbound, "_temporal_stream_backend_name", ""), - getattr(outbound, "_temporal_stream_default", False), - getattr(outbound, "_temporal_stream_kind", ""), - ) + return outbound outbound = getattr(outbound, "next", None) return None +def worker_backend() -> WorkerBackend | None: + """The Worker's backend, which marks the run's outbound interceptor, or None.""" + outbound = _transport() + if outbound is None: + return None + return WorkerBackend( + getattr(outbound, "_temporal_stream_backend_name", ""), + getattr(outbound, "_temporal_stream_default", False), + getattr(outbound, "_temporal_stream_kind", ""), + ) + + +async def _close_in_workflow(request: stream_pb.CloseStreamRequest) -> int: + # The Worker's backend carries out the close once the Workflow Task commits. + outbound = _transport() + if outbound is None: + raise no_worker_backend() + return await outbound._temporal_stream_backend._close_in_workflow(request) + + def no_worker_backend() -> _errors.StreamConfigError: """The error of Workflow code on a Worker with no Stream backend.""" return _errors.StreamConfigError( @@ -238,18 +249,8 @@ async def run() -> None: start_to_close_timeout=_WRITE_TIMEOUT, ) elif activity == CLOSE_ACTIVITY: - # A close that a failed Workflow Task scheduled never runs, so the task's retry - # still writes to the Stream. A Local Activity would close it before the task - # completes. - answers = [ - await temporalio.workflow.execute_activity( - activity, - write.request, - result_type=int, - start_to_close_timeout=_WRITE_TIMEOUT, - retry_policy=_CLOSE_RETRY, - ) - ] + assert isinstance(write.request, stream_pb.CloseStreamRequest) + answers = [await _close_in_workflow(write.request)] else: answers = [ await temporalio.workflow.execute_local_activity( @@ -431,13 +432,14 @@ def _append(self, topic: str, values: tuple[Any, ...]) -> asyncio.Future[int]: async def close( self, *, result: Any | None = None, failure: BaseException | None = None ) -> None: - """Close the Stream with one regular Activity, after the writes before it. + """Close the Stream after the writes before it, once the Workflow Task commits. - The Activity runs once the Workflow Task that schedules it completes, so a task that - fails before then, or a crash, leaves the Stream open, and the task's retry writes to it - again. It closes as the run chain, the Stream's owner. Readers get the Records before - it, then a ``Closed`` item with ``result`` or ``failure``. A ``CancelledError`` failure - closes it as canceled. Later writes in this run raise :py:class:`StreamClosedError`. + A task that fails before then, or a crash, leaves the Stream open, and the task's retry + writes to it again. On Redis the close is a regular Activity on the Workflow's task + queue, which closes as the run chain, the Stream's owner. So every Worker on that queue + registers the Redis backend. Readers get the Records before it, then a ``Closed`` item + with ``result`` or ``failure``. A ``CancelledError`` failure closes it as canceled. + Later writes in this run raise :py:class:`StreamClosedError`. Raises: StreamNotOwnerError: This run chain didn't create the Stream. @@ -449,9 +451,8 @@ def close_nowait( ) -> None: """Close the Stream with no await, for sync Workflow code. - It is to :py:meth:`close` what ``asyncio.Queue.put_nowait`` is to ``put``. It schedules - the close Activity, and the end of the run waits for it. A backend error fails the - Workflow Task. + It is to :py:meth:`close` what ``asyncio.Queue.put_nowait`` is to ``put``. It starts the + close, and the end of the run waits for it. A backend error fails the Workflow Task. """ _write_nowait(self._close(result, failure)) @@ -462,7 +463,7 @@ def _close(self, result: Any, failure: BaseException | None) -> asyncio.Future[i info.first_execution_run_id, ): # Only the run chain that created a Stream ends it for every reader. The backend - # refuses it too, and this error comes before an Activity is spent. + # refuses it too, and this error comes before a command is spent. raise _errors.StreamNotOwnerError( f"Can't close the Stream {self.id}, since this run chain didn't create it", reason="stream_not_owner", diff --git a/temporalio/contrib/stream/__init__.py b/temporalio/contrib/stream/__init__.py index 4dd740c693..7c05ae410b 100644 --- a/temporalio/contrib/stream/__init__.py +++ b/temporalio/contrib/stream/__init__.py @@ -40,6 +40,10 @@ class RedisStreamBackend(ExtensionStreamBackend): name ``redis``, which the Stream handles it creates carry. For now a Worker serves one backend. A client may have more than one, and ``backend=`` picks one. + A Workflow's close is a regular Activity on the Workflow's task queue, which runs once the + Workflow Task commits. So a Worker with this backend polls Activities on its task queue, and + Workers that share a task queue all register the backend. + .. warning:: This API is experimental. """ diff --git a/temporalio/contrib/stream/testing.py b/temporalio/contrib/stream/testing.py index b0ea5eec44..b8901500ae 100644 --- a/temporalio/contrib/stream/testing.py +++ b/temporalio/contrib/stream/testing.py @@ -20,7 +20,8 @@ class MemoryStreamBackend(ExtensionStreamBackend): shares one backend shares its Streams, and they go when the process ends. It registers the backend name ``memory``. It keeps Redis's rules, so the calls of :py:mod:`temporalio.contrib.stream` take it as a Redis backend, and tests swap the Store - with no change to Workflow code. + with no change to Workflow code. As on Redis, a Workflow's close is a regular Activity, so + Workers that share a task queue all register the backend. .. warning:: This API is experimental. diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 69bec68721..dacdedae59 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -25,10 +25,10 @@ def create_stream( Its id is ``::``, so a client finds it from the Workflow handle with no call. The create and each publish are Local Activities, and the create's - error raises here. Only the run chain closes the Stream, with a regular Activity that runs - once its Workflow Task completed, so a task that fails before then leaves the Stream open - for its retry. On Redis, the end of the run chain closes nothing, so a Stream that nobody - closes stays open until its ``safety_ttl``. + error raises here. Only the run chain closes the Stream. A close runs once its Workflow + Task commits, as a regular Activity on Redis, so a task that fails before then leaves the + Stream open for its retry. On Redis, the end of the run chain closes nothing, so a Stream + that nobody closes stays open until its ``safety_ttl``. Await the call for the handle, or use it in ``async with``, which closes the Stream at the end of the block:: diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index ad6c05dc6a..7ecefeb01a 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -709,3 +709,24 @@ async def test_default_is_native_with_no_default_backend(client: Client): items = [item async for item in stream.subscribe()] assert data(items) == ["on redis"] assert isinstance(items[-1], streams.Closed) and items[-1].result == "closed" + + +class _RecordsCloses(MemoryStreamBackend): + # A backend that closes another way overrides the close of a Workflow's Stream. + closes: list[str] = [] + + async def _close_in_workflow(self, request: Any) -> int: + self.closes.append(request.stream.id) + return await super()._close_in_workflow(request) + + +async def test_backend_carries_the_workflow_close(client: Client): + config = client.config() + config["stream_backends"] = [_RecordsCloses(default=True)] + recording = Client(**config) + async with new_worker(recording, Agent) as worker: + handle = await recording.start_workflow( + Agent.run, "hi", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue + ) + await handle.result() + assert _RecordsCloses.closes == [handle.stream(Events).id] From 8f139d52863cec3130960160f5d26446791c4516 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:41:15 -0700 Subject: [PATCH 22/79] Closed a native Stream with a System Nexus command from Workflow code. The server carries out the close once the Workflow Task commits, so a task that fails after the close leaves the Stream open for its retry. drain() awaits it as it awaits the writes. --- temporalio/common/streams/_native.py | 31 ++++++++++++++++++++++++ temporalio/common/streams/_workflow.py | 33 +++++++++++++++++++------- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/temporalio/common/streams/_native.py b/temporalio/common/streams/_native.py index c17360d860..ed438c99df 100644 --- a/temporalio/common/streams/_native.py +++ b/temporalio/common/streams/_native.py @@ -40,6 +40,8 @@ # Nothing waits for the answer of a wake that the reader dropped, so its cancel ends it at once. _TRY_CANCEL = temporalio.workflow.NexusOperationCancellationType.TRY_CANCEL +# A close goes on when the run is cancelled, since the cancel may be what the close records. +_ABANDON = temporalio.workflow.NexusOperationCancellationType.ABANDON # How long a wake parks on the server. A parked wake adds nothing to History, so an idle reader # costs nothing. The server caps it at `stream.maxWakeWait`, and an older server at 60 s. _WAKE_WAIT = timedelta(days=7) @@ -269,6 +271,35 @@ async def _temporal_stream_start_wake( ) ) + async def _temporal_stream_close( + self, request: stream_pb.CloseStreamRequest + ) -> None: + # A System Nexus command, which the server carries out once the Workflow Task commits. + # A task that fails after the close leaves the Stream open, and its retry closes it. The + # server takes the closing run from the operation's links. + close = stream_pb.CloseStreamRequest() + close.CopyFrom(request) + close.stream.ClearField("backend") + handle = await self.next.start_system_nexus_operation( + temporalio.worker.StartNexusOperationInput( + endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, + service=_SERVICE, + operation="CloseStream", + input=close, + output_type=stream_pb.CloseStreamResponse, + schedule_to_close_timeout=None, + schedule_to_start_timeout=None, + start_to_close_timeout=None, + cancellation_type=_ABANDON, + headers=None, + summary=None, + ) + ) + try: + await handle + except temporalio.exceptions.NexusOperationError as err: + raise _workflow_reader.error_of(err) from None + async def _temporal_stream_describe( self, request: stream_pb.DescribeStreamRequest ) -> stream_pb.DescribeStreamResponse: diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 7540a7d958..5afda1fdef 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -5,16 +5,18 @@ an argument, such as a :py:class:`WorkflowStream` that a run passes on through continue-as-new, works the same. -Each create, publish and close is a Local Activity, so it has a place in the run's commands, -as an Activity call has. To add, remove or move one in a Workflow that has open runs, gate the -change with ``workflow.patched``, else the replay of those runs fails as nondeterministic. -Publishes that wait behind a write to the same Stream go as one Local Activity. +Each create and publish is a Local Activity, so it has a place in the run's commands, as an +Activity call has. A close on the native backend is a System Nexus command, which the server +carries out once the Workflow Task commits. To add, remove or move one in a Workflow that has +open runs, gate the change with ``workflow.patched``, else the replay of those runs fails as +nondeterministic. Publishes that wait behind a write to the same Stream go as one Local +Activity. """ from __future__ import annotations import asyncio -from collections.abc import Sequence +from collections.abc import Awaitable, Callable, Sequence from dataclasses import dataclass, field from datetime import timedelta from types import TracebackType @@ -230,7 +232,12 @@ async def run() -> None: if state.waiting.get(stream_id) is write: del state.waiting[stream_id] try: - if isinstance(write.request, stream_pb.AppendRecordsRequest): + if isinstance(write.request, stream_pb.CloseStreamRequest) and ( + closer := _closer(backend) + ): + await closer(write.request) + answers: list[Any] = [0] + elif isinstance(write.request, stream_pb.AppendRecordsRequest): # The Local Activity answers each merged call with its last offset or its error. counts = [ count - before @@ -281,6 +288,14 @@ async def run() -> None: return future +def _closer( + backend: str, +) -> Callable[[stream_pb.CloseStreamRequest], Awaitable[None]] | None: + # A backend whose server takes a close as a command of the Workflow Task closes the Stream + # only once the task commits. The others close with a Local Activity. + return getattr(_workflow_reader.reader_of(backend), "_temporal_stream_close", None) + + def _write_nowait(future: asyncio.Future[int]) -> None: runtime = temporalio.workflow._Runtime.current() @@ -500,11 +515,13 @@ def _append(self, topic: str, values: tuple[Any, ...]) -> asyncio.Future[int]: async def close( self, *, result: Any | None = None, failure: BaseException | None = None ) -> None: - """Close the Stream with one Local Activity, after the writes before it. + """Close the Stream, after the writes before it. Readers get the Records before it, then a ``Closed`` item with ``result`` or ``failure``. A ``CancelledError`` failure closes it as canceled. Later writes in this run - raise :py:class:`StreamClosedError`. + raise :py:class:`StreamClosedError`. On the native backend the close is a System Nexus + command, which the server carries out once this Workflow Task commits, so a task that + fails after the close leaves the Stream open for its retry to close. Raises: StreamNotOwnerError: This run chain didn't create the Stream. From 398ae2c39b261c2715514375773ddbba502fa087 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:41:30 -0700 Subject: [PATCH 23/79] Left a block's Stream open at the end of a cron run or a retried failure. A cron chain writes on to the Stream in its next run, and so does a retry run after a failure under a retry policy. An explicit close() still closes. --- temporalio/common/streams/_workflow.py | 16 +++- tests/contrib/streams/test_workflow_closes.py | 90 +++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 tests/contrib/streams/test_workflow_closes.py diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 5afda1fdef..1261cd2a42 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -577,13 +577,23 @@ async def __aexit__( """Close the Stream, with the exception as its failure if there is one. A cancel closes it with a ``CancelledError`` failure. Another ``BaseException``, such as - an eviction, leaves it open, since the block did not end in Workflow code. + an eviction, leaves it open, since the block did not end in Workflow code. The Stream + also stays open in a cron run, since it spans the cron chain, and on a failure of a run + with a retry policy, since a retry run writes on to it. An explicit :py:meth:`close` + always closes. """ if _state().stream(self._model_ref()).closed: return close, failure = exit_failure(exc) - if close: - await self.close(failure=failure) + if not close: + return + info = temporalio.workflow.info() + if info.cron_schedule: + return + retried = exc is not None and not isinstance(exc, asyncio.CancelledError) + if retried and info.retry_policy is not None: + return + await self.close(failure=failure) def _own_handle(definition: type[E], backend: str | None) -> WorkflowStream[E]: diff --git a/tests/contrib/streams/test_workflow_closes.py b/tests/contrib/streams/test_workflow_closes.py new file mode 100644 index 0000000000..172af6e121 --- /dev/null +++ b/tests/contrib/streams/test_workflow_closes.py @@ -0,0 +1,90 @@ +import uuid +from datetime import timedelta +from typing import Any + +from temporalio import workflow +from temporalio.client import Client +from temporalio.common import RetryPolicy, streams +from temporalio.exceptions import ApplicationError +from tests.helpers import new_worker + + +@streams.defn(name="events") +class Events: + tokens = streams.topic(str) + + +def data(items: list[Any]) -> list[Any]: + return [item.data for item in items if isinstance(item, streams.Record)] + + +def closes(items: list[Any]) -> list[streams.Closed]: + return [item for item in items if isinstance(item, streams.Closed)] + + +@workflow.defn +class CronBlock: + @workflow.run + async def run(self) -> str: + async with workflow.create_stream(Events) as stream: + await stream.topic(Events.tokens).publish("turn") + return "done" + + +async def test_a_cron_run_leaves_its_block_open(stream_client: Client): + # The Stream spans the cron chain, so the end of each run's block leaves it open. + async with new_worker(stream_client, CronBlock) as worker: + handle = await stream_client.start_workflow( + CronBlock.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + cron_schedule="@every 1s", + ) + stream = handle.stream(Events) + reader = stream.subscribe(create_wait=timedelta(seconds=20)) + first = await reader.__anext__() + second = await reader.__anext__() + await reader.aclose() + # Read before the terminate, since a native server closes the Stream with it. + items = [item async for item in stream.subscribe(wait_new_records=False)] + await handle.terminate() + assert [first.data, second.data] == ["turn", "turn"] # type: ignore[union-attr] + assert not closes(items) + + +@workflow.defn +class RetriesInBlock: + @workflow.run + async def run(self) -> str: + attempt = workflow.info().attempt + async with workflow.create_stream(Events) as stream: + await stream.topic(Events.tokens).publish(f"run-{attempt}") + if attempt == 1: + raise ApplicationError("the first run fails") + return "done" + + +async def test_a_retried_run_writes_on_and_its_block_closes(stream_client: Client): + # The first run fails under a retry policy, so its block leaves the Stream open, and the + # retry run writes on to it and closes it. + async with new_worker(stream_client, RetriesInBlock) as worker: + handle = await stream_client.start_workflow( + RetriesInBlock.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + retry_policy=RetryPolicy( + initial_interval=timedelta(milliseconds=10), maximum_attempts=2 + ), + ) + assert await handle.result() == "done" + items = [item async for item in handle.stream(Events).subscribe()] + withdrawn = [item for item in items if isinstance(item, streams.Superseded)] + visible = [ + item.data + for item in items + if isinstance(item, streams.Record) + and not any(w.withdraws(item) for w in withdrawn) + ] + assert visible == ["run-2"] + [closed] = closes(items) + assert items[-1] is closed and closed.status == streams.CloseStatus.COMPLETED From 0a8927ff8c0799da392b8df3cd9a5123c399101f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:43:45 -0700 Subject: [PATCH 24/79] Noted in the Redis package docs that Workers poll the close Activity. --- temporalio/contrib/stream/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/temporalio/contrib/stream/__init__.py b/temporalio/contrib/stream/__init__.py index 7c05ae410b..bfd1cffc0d 100644 --- a/temporalio/contrib/stream/__init__.py +++ b/temporalio/contrib/stream/__init__.py @@ -9,6 +9,10 @@ "localhost:7233", stream_backends=[redis.RedisStreamBackend("redis://localhost")] ) +A Workflow's close is a regular Activity on the Workflow's task queue. So a Worker with a Redis +backend polls Activities on its task queue for the close Activity, and Workers that share a task +queue must all register the Redis backend. + .. warning:: This package is experimental. """ From 386f0f56d2113b26b3409152a05d4c901f4b7159 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:49:24 -0700 Subject: [PATCH 25/79] Tested that a close waits for its Workflow Task to commit. A Local Activity in the task of the close runs, the task fails, and the Stream stays open until the retry closes it once, through the Activity on Redis and the command on native. --- temporalio/common/streams/_writes.py | 3 +- tests/contrib/streams/test_workflow_closes.py | 71 ++++++++++++++++++- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/temporalio/common/streams/_writes.py b/temporalio/common/streams/_writes.py index df86c78b79..5e3687e720 100644 --- a/temporalio/common/streams/_writes.py +++ b/temporalio/common/streams/_writes.py @@ -2,7 +2,8 @@ Each Workflow ``create_stream`` is a Local Activity, and the publishes that wait behind a write to the same Stream share one. History has the answer and not the values. Each Workflow -``close`` is a regular Activity, so it runs only once its Workflow Task completed. +``close`` on Redis is a regular Activity, so it runs only once its Workflow Task completed. The +native backend closes with a System Nexus command instead. """ from __future__ import annotations diff --git a/tests/contrib/streams/test_workflow_closes.py b/tests/contrib/streams/test_workflow_closes.py index 172af6e121..2cb34d4a69 100644 --- a/tests/contrib/streams/test_workflow_closes.py +++ b/tests/contrib/streams/test_workflow_closes.py @@ -2,11 +2,12 @@ from datetime import timedelta from typing import Any -from temporalio import workflow +from temporalio import activity, workflow from temporalio.client import Client from temporalio.common import RetryPolicy, streams from temporalio.exceptions import ApplicationError -from tests.helpers import new_worker +from temporalio.worker import UnsandboxedWorkflowRunner +from tests.helpers import assert_task_fail_eventually, new_worker @streams.defn(name="events") @@ -22,6 +23,72 @@ def closes(items: list[Any]) -> list[streams.Closed]: return [item for item in items if isinstance(item, streams.Closed)] +failed_once: set[str] = set() + + +@activity.defn +async def noop() -> None: + pass + + +@workflow.defn +class ClosesThenFailsItsTask: + def __init__(self) -> None: + self._go = False + + @workflow.run + async def run(self, key: str) -> str: + stream = await workflow.create_stream(Events) + await stream.topic(Events.tokens).publish("a") + # The create and the publish commit with this task. + await workflow.sleep(timedelta(milliseconds=10)) + if key not in failed_once: + failed_once.add(key) + stream.close_nowait(result="closed") + # A Local Activity in the same task runs before the task fails, and the close + # must not. + await workflow.execute_local_activity( + noop, start_to_close_timeout=timedelta(seconds=10) + ) + raise RuntimeError("fail the Workflow Task of the close") + await workflow.wait_condition(lambda: self._go) + await stream.close(result="closed") + return "done" + + @workflow.signal + def go(self) -> None: + self._go = True + + +async def test_a_close_waits_for_its_workflow_task_to_commit(stream_client: Client): + # A close runs only once its Workflow Task commits, so a task that fails after it leaves + # the Stream open, and the retry closes it once. + key = str(uuid.uuid4()) + # The test's set must survive the retry of the task, so no sandbox. + async with new_worker( + stream_client, + ClosesThenFailsItsTask, + activities=[noop], + workflow_runner=UnsandboxedWorkflowRunner(), + ) as worker: + handle = await stream_client.start_workflow( + ClosesThenFailsItsTask.run, + key, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await assert_task_fail_eventually(handle, message_contains="of the close") + stream = handle.stream(Events) + before = [item async for item in stream.subscribe(wait_new_records=False)] + assert data(before) == ["a"] and not closes(before) + await handle.signal(ClosesThenFailsItsTask.go) + assert await handle.result() == "done" + items = [item async for item in stream.subscribe()] + assert data(items) == ["a"] + [closed] = closes(items) + assert closed.result == "closed" + + @workflow.defn class CronBlock: @workflow.run From e73731bc3dd458ae613f55e2514ac508b4604864 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:49:58 -0700 Subject: [PATCH 26/79] Tested that only a Worker with a Redis backend polls Activities. --- tests/contrib/streams/test_stream_client.py | 23 +++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 6c3bfd0fd1..572cdeb428 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -10,14 +10,14 @@ from temporalio import workflow from temporalio.client import Client from temporalio.common import streams -from temporalio.common.streams import _plugin, _stream_client +from temporalio.common.streams import _plugin, _stream_client, _writes from temporalio.contrib import stream as redis from temporalio.contrib.stream import RedisStreamBackend from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.converter import DataConverter from temporalio.plugin import SimplePlugin from temporalio.testing import WorkflowEnvironment -from temporalio.worker import Replayer +from temporalio.worker import Replayer, Worker from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner from tests.contrib.streams._support import create_owned from tests.contrib.streams.fake_extension import fake_client @@ -432,6 +432,25 @@ async def test_registration_rules(client: Client): ) +async def test_only_a_redis_backend_polls_activities(client: Client): + # The native close is a command, so a Worker with no Redis backend polls no Activities and + # shares its task queue with any other Worker. The Redis close is a regular Activity. + def polled(worker_client: Client) -> list[Any]: + worker = Worker( + worker_client, task_queue=f"q-{uuid.uuid4()}", workflows=[Agent] + ) + return list(worker.config(active_config=True).get("activities") or []) + + assert polled(client) == [] + config = client.config() + config["stream_backends"] = [MemoryStreamBackend(default=True)] + assert polled(Client(**config)) == [_writes.close_stream] + config["stream_backends"] = [ + streams.NativeStreamBackend(retention=timedelta(days=1)) + ] + assert polled(Client(**config)) == [] + + async def test_backend_is_not_a_plugin(client: Client): # The connect refuses it before it connects, so the unreachable target doesn't matter. with pytest.raises(streams.StreamConfigError, match="not a plugin"): From 42440f7615b0de9fe006a0006a38bf4af210b13b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 13:53:27 -0700 Subject: [PATCH 27/79] Ran p35's close tests on the native lane, and dropped j9's copies. On native the close is a CloseStream operation in History, not an Activity, and the server closes the Stream at the terminate. p35's block tests cover the cron and retry cases. --- tests/contrib/streams/test_redis.py | 25 +++++-- tests/contrib/streams/test_workflow_closes.py | 71 +------------------ tests/contrib/streams/test_workflow_module.py | 23 +++--- 3 files changed, 35 insertions(+), 84 deletions(-) diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index a66af1229f..891f1c85ed 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -881,13 +881,15 @@ async def run(self) -> str: async def close_activity_events( handle: WorkflowHandle[Any, Any], ) -> list[EventType.ValueType]: - # The events of the close Activity, from its schedule on. + # The events of the close command, from its schedule on: the Activity on Redis, and the + # System Nexus operation on native. history = await handle.fetch_history() scheduled = { event.event_id for event in history.events if event.activity_task_scheduled_event_attributes.activity_type.name == "__temporal_streams_close" + or event.nexus_operation_scheduled_event_attributes.operation == "CloseStream" } return [ event.event_type @@ -896,6 +898,10 @@ async def close_activity_events( or event.activity_task_started_event_attributes.scheduled_event_id in scheduled or event.activity_task_completed_event_attributes.scheduled_event_id in scheduled + or event.nexus_operation_started_event_attributes.scheduled_event_id + in scheduled + or event.nexus_operation_completed_event_attributes.scheduled_event_id + in scheduled ] @@ -925,11 +931,18 @@ async def test_close_survives_a_failed_last_task(stream_client: Client, cache: i assert len(closes) == 1 and items[-1] is closes[0] assert closes[0].result == "done" assert closes[0].status == streams.CloseStatus.COMPLETED - assert await close_activity_events(handle) == [ - EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED, - EventType.EVENT_TYPE_ACTIVITY_TASK_STARTED, - EventType.EVENT_TYPE_ACTIVITY_TASK_COMPLETED, - ] + assert await close_activity_events(handle) == ( + [ + EventType.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED, + EventType.EVENT_TYPE_NEXUS_OPERATION_COMPLETED, + ] + if on_native(stream_client) + else [ + EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED, + EventType.EVENT_TYPE_ACTIVITY_TASK_STARTED, + EventType.EVENT_TYPE_ACTIVITY_TASK_COMPLETED, + ] + ) @workflow.defn diff --git a/tests/contrib/streams/test_workflow_closes.py b/tests/contrib/streams/test_workflow_closes.py index 2cb34d4a69..e0a128044d 100644 --- a/tests/contrib/streams/test_workflow_closes.py +++ b/tests/contrib/streams/test_workflow_closes.py @@ -4,8 +4,7 @@ from temporalio import activity, workflow from temporalio.client import Client -from temporalio.common import RetryPolicy, streams -from temporalio.exceptions import ApplicationError +from temporalio.common import streams from temporalio.worker import UnsandboxedWorkflowRunner from tests.helpers import assert_task_fail_eventually, new_worker @@ -87,71 +86,3 @@ async def test_a_close_waits_for_its_workflow_task_to_commit(stream_client: Clie assert data(items) == ["a"] [closed] = closes(items) assert closed.result == "closed" - - -@workflow.defn -class CronBlock: - @workflow.run - async def run(self) -> str: - async with workflow.create_stream(Events) as stream: - await stream.topic(Events.tokens).publish("turn") - return "done" - - -async def test_a_cron_run_leaves_its_block_open(stream_client: Client): - # The Stream spans the cron chain, so the end of each run's block leaves it open. - async with new_worker(stream_client, CronBlock) as worker: - handle = await stream_client.start_workflow( - CronBlock.run, - id=f"wf-{uuid.uuid4()}", - task_queue=worker.task_queue, - cron_schedule="@every 1s", - ) - stream = handle.stream(Events) - reader = stream.subscribe(create_wait=timedelta(seconds=20)) - first = await reader.__anext__() - second = await reader.__anext__() - await reader.aclose() - # Read before the terminate, since a native server closes the Stream with it. - items = [item async for item in stream.subscribe(wait_new_records=False)] - await handle.terminate() - assert [first.data, second.data] == ["turn", "turn"] # type: ignore[union-attr] - assert not closes(items) - - -@workflow.defn -class RetriesInBlock: - @workflow.run - async def run(self) -> str: - attempt = workflow.info().attempt - async with workflow.create_stream(Events) as stream: - await stream.topic(Events.tokens).publish(f"run-{attempt}") - if attempt == 1: - raise ApplicationError("the first run fails") - return "done" - - -async def test_a_retried_run_writes_on_and_its_block_closes(stream_client: Client): - # The first run fails under a retry policy, so its block leaves the Stream open, and the - # retry run writes on to it and closes it. - async with new_worker(stream_client, RetriesInBlock) as worker: - handle = await stream_client.start_workflow( - RetriesInBlock.run, - id=f"wf-{uuid.uuid4()}", - task_queue=worker.task_queue, - retry_policy=RetryPolicy( - initial_interval=timedelta(milliseconds=10), maximum_attempts=2 - ), - ) - assert await handle.result() == "done" - items = [item async for item in handle.stream(Events).subscribe()] - withdrawn = [item for item in items if isinstance(item, streams.Superseded)] - visible = [ - item.data - for item in items - if isinstance(item, streams.Record) - and not any(w.withdraws(item) for w in withdrawn) - ] - assert visible == ["run-2"] - [closed] = closes(items) - assert items[-1] is closed and closed.status == streams.CloseStatus.COMPLETED diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index b97c1d8280..a2e71d5b91 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -335,9 +335,10 @@ async def test_block_end_leaves_a_cron_stream_open(stream_client: Client): first = await asyncio.wait_for(anext(reader), 20) second = await asyncio.wait_for(anext(reader), 20) await reader.aclose() + # Read before the terminate, since a native server closes the Stream with it. + items = await read(handle) await handle.terminate() assert isinstance(first, streams.Record) and isinstance(second, streams.Record) - items = await read(handle) assert not any(isinstance(item, streams.Closed) for item in items) @@ -374,16 +375,22 @@ async def test_failed_task_writes_no_close(stream_client: Client): closes = [item for item in items if isinstance(item, streams.Closed)] assert len(closes) == 1 and closes[0].result == "closed" history = await handle.fetch_history() - # The failed task's close never went out, so History has one close Activity. - scheduled = [ - event + # The failed task's close never went out, so History has one close: an Activity on + # Redis, and a System Nexus operation on native. + activities = [ + event.activity_task_scheduled_event_attributes.activity_type.name for event in history.events if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED ] - assert [ - event.activity_task_scheduled_event_attributes.activity_type.name - for event in scheduled - ] == ["__temporal_streams_close"] + operations = [ + event.nexus_operation_scheduled_event_attributes.operation + for event in history.events + if event.event_type == EventType.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED + ] + if on_native(stream_client): + assert (activities, operations) == ([], ["CloseStream"]) + else: + assert (activities, operations) == (["__temporal_streams_close"], []) @workflow.defn From 0b4e4509e2b1ef454abd689788313eb7f832dc8b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 17:26:16 -0700 Subject: [PATCH 28/79] Added Client.create_stream for a client-owned Stream on the default backend. It mirrors workflow.create_stream and activity.create_stream, so a client creates on native with no registration. redis.create stays as the Redis shorthand, and the native lane uses the new call in place of the internal create. --- temporalio/client/_client.py | 49 ++++++++++++++++-- temporalio/common/streams/__init__.py | 8 +-- temporalio/common/streams/_stream_client.py | 9 ++-- tests/contrib/streams/_support.py | 20 -------- tests/contrib/streams/test_native_server.py | 10 ++-- tests/contrib/streams/test_redis.py | 12 ++--- tests/contrib/streams/test_stream_client.py | 50 ++++++++++++++----- tests/contrib/streams/test_workflow_module.py | 4 +- tests/contrib/streams/test_workflow_reads.py | 4 +- 9 files changed, 108 insertions(+), 58 deletions(-) diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index 8c6ac29560..ec3a491646 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -111,7 +111,12 @@ ) if TYPE_CHECKING: - from temporalio.common.streams import ClientStream, StreamBackend, StreamHandle + from temporalio.common.streams import ( + ClientStream, + Creating, + StreamBackend, + StreamHandle, + ) from ._interceptor import Interceptor from ._plugin import Plugin @@ -926,8 +931,8 @@ def stream( ) -> ClientStream[_StreamDefinitionT]: """A Stream by its id, or a handle that came as data, such as in a Record. - It makes no call. To create a Stream that the client owns, use the create of a - backend, such as :py:func:`temporalio.contrib.stream.create`. + It makes no call. To create a Stream that the client owns, use + :py:meth:`create_stream`. .. warning:: This API is experimental. @@ -949,6 +954,44 @@ def stream( self, definition, id=id, stream=stream, backend=backend ) + def create_stream( + self, + definition: type[_StreamDefinitionT], + *, + id: str, + retention: timedelta | None = None, + backend: str | None = None, + ) -> Creating[ClientStream[_StreamDefinitionT]]: + """Create a Stream that this client owns, so only a client closes it. + + It is idempotent for one id and one Retention. Await the call for the handle, or use it + in ``async with``, which closes the Stream at the end of the block:: + + async with client.create_stream(Events, id="chat-1") as chat: + await chat.topic(Events.turns).publish(Turn("user", "hi")) + + .. warning:: + This API is experimental. + + Args: + definition: The Stream definition class. + id: The Stream id, as a Workflow takes a Workflow id. + retention: How long the Records stay after the close. Defaults to the backend's. + backend: The name of the backend. Defaults to the registered backend with + ``default=True``, else to ``native``. + + Raises: + StreamConfigError: The client has no backend of that name. No call is made. + StreamClosedError: A Stream of that id was created and closed before. + StreamError: The Stream exists with another Retention, with the reason + ``stream_already_exists``. + """ + from temporalio.common.streams import _stream_client + + return _stream_client.create( + self, definition, id=id, retention=retention, backend=backend + ) + def get_workflow_handle_for( self, workflow: ( diff --git a/temporalio/common/streams/__init__.py b/temporalio/common/streams/__init__.py index bbba04cc42..82789c4ea7 100644 --- a/temporalio/common/streams/__init__.py +++ b/temporalio/common/streams/__init__.py @@ -1,9 +1,11 @@ """Streams of Records that Workflows, Activities and clients write and read. A Stream definition, a class with ``@streams.defn``, gives the Stream name and the type of each -Topic. Workflow code creates a Stream with :py:func:`temporalio.workflow.create_stream`. A -client gets a Workflow's Stream with :py:meth:`temporalio.client.WorkflowHandle.stream`, and -any Stream with :py:meth:`temporalio.client.Client.stream`. A Stream is data: a Workflow +Topic. Workflow code creates a Stream with :py:func:`temporalio.workflow.create_stream`, a +Standalone Activity with :py:func:`temporalio.activity.create_stream`, and a client with +:py:meth:`temporalio.client.Client.create_stream`. A client gets a Workflow's Stream with +:py:meth:`temporalio.client.WorkflowHandle.stream`, and any Stream with +:py:meth:`temporalio.client.Client.stream`. A Stream is data: a Workflow passes it to an Activity, a child Workflow or the next run as an argument, and the parameter's type gives the handle class of that context, such as :py:class:`ActivityStream`. diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index 18fda7f26b..2bd94d1239 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -91,7 +91,8 @@ class ClientStream(StreamHandle[E]): """A Stream that a client writes, reads and closes. Get one from :py:meth:`temporalio.client.Client.stream`, - :py:meth:`temporalio.client.WorkflowHandle.stream` or a backend's create, such as + :py:meth:`temporalio.client.WorkflowHandle.stream`, + :py:meth:`temporalio.client.Client.create_stream`, or a backend's create, such as :py:func:`temporalio.contrib.stream.create`. Each publish writes before it returns. Use :py:class:`BufferedPublisher` to batch. The @@ -406,10 +407,10 @@ def create( *, id: str, retention: timedelta | None, - backend: str, + backend: str | None, ) -> Creating[ClientStream[E]]: - """Create a Stream that this client owns on a registered backend. See - :py:func:`temporalio.contrib.stream.create`. + """Create a Stream that this client owns. See + :py:meth:`temporalio.client.Client.create_stream`. """ async def create() -> ClientStream[E]: diff --git a/tests/contrib/streams/_support.py b/tests/contrib/streams/_support.py index 2481990f6a..cf2f9ab80c 100644 --- a/tests/contrib/streams/_support.py +++ b/tests/contrib/streams/_support.py @@ -50,23 +50,3 @@ def on_native(client: Any) -> bool: name, _ = _plugin.client_backend(client, None) return name == "native" - - -def create_owned( - client: Any, definition: Any, *, id: str, retention: Any = None -) -> Any: - """Create a Stream that the client owns, on the client's default backend. - - ``temporalio.contrib.stream.create`` makes one on Redis. The native server keeps one too, - with no public create, so a native lane makes it through the internal call. Await the - result, or use it in ``async with``. - """ - from temporalio.common.streams import _plugin, _stream_client - from temporalio.contrib import stream as redis - - name, _ = _plugin.client_backend(client, None) - if name != "native": - return redis.create(client, definition, id=id, retention=retention) - return _stream_client.create( - client, definition, id=id, retention=retention, backend=name - ) diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index de846e355f..d1ef30fff7 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -19,7 +19,6 @@ from temporalio.contrib.stream import RedisStreamBackend from temporalio.worker import Replayer, UnsandboxedWorkflowRunner from tests.contrib.streams import test_redis -from tests.contrib.streams._support import create_owned from tests.contrib.streams.test_workflow_reads import ( Events, Reader, @@ -65,7 +64,7 @@ async def run(self) -> list[str | None]: async def test_native_client_round_trip(native_client: Client): - stream = await create_owned(native_client, Events, id=f"native-{uuid.uuid4()}") + stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") assert await stream.topic(Events.tokens).publish("a", "b") await stream.close(result="done") items = [item async for item in stream.subscribe()] @@ -264,7 +263,7 @@ async def test_a_native_read_from_the_end_keeps_what_comes_after_its_first_page( ): # The server resolves END on each call, so the wait and the read after it start where the # first page ended. Else the Record that wakes the reader is behind its start. - stream = await create_owned(native_client, Events, id=f"native-{uuid.uuid4()}") + stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("old") async with new_worker(native_client, ReadsNewTokens) as worker: reader = await native_client.start_workflow( @@ -303,7 +302,7 @@ def stop(self) -> None: async def test_closing_a_native_read_cancels_its_wake(native_client: Client): # A wake may park for days, so a read that its code closed must not keep it. - stream = await create_owned(native_client, Events, id=f"native-{uuid.uuid4()}") + stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") async with new_worker(native_client, ClosesItsRead) as worker: reader = await native_client.start_workflow( ClosesItsRead.run, @@ -335,8 +334,7 @@ async def test_a_native_reader_of_a_client_stream_ends_at_half_its_retention( ): # Retention trims a Stream that no Workflow owns under its readers, and a replay reads the # Stream again. So the read ends in time for the run to continue as new. - stream = await create_owned( - native_client, + stream = await native_client.create_stream( Events, id=f"native-{uuid.uuid4()}", retention=timedelta(seconds=6), diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 891f1c85ed..c0179938e8 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -37,7 +37,7 @@ SandboxedWorkflowRunner, SandboxRestrictions, ) -from tests.contrib.streams._support import create_owned, on_native +from tests.contrib.streams._support import on_native from tests.helpers import new_worker @@ -599,9 +599,9 @@ async def test_client_close_goes_through_the_payload_codec(stream_client: Client DataConverter.default, payload_codec=_Rot13Codec() ) codec_client = Client(**config) - done = await create_owned(codec_client, Events, id=f"done-{uuid.uuid4()}") + done = await codec_client.create_stream(Events, id=f"done-{uuid.uuid4()}") await done.close(result=Turn("done")) - failed = await create_owned(codec_client, Events, id=f"failed-{uuid.uuid4()}") + failed = await codec_client.create_stream(Events, id=f"failed-{uuid.uuid4()}") await failed.close(failure=ApplicationError("bad turn")) [closed] = [item async for item in done.subscribe()] assert isinstance(closed, streams.Closed) @@ -678,7 +678,7 @@ async def test_backend_replays_a_history_made_without_it( async def test_backend_shutdown_after_client_writes(stream_client: Client): backend = _plugin.backend_of(stream_client, None) assert isinstance(backend, ExtensionStreamBackend) - stream = await create_owned(stream_client, Events, id=f"shutdown-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"shutdown-{uuid.uuid4()}") async with streams.BufferedPublisher(stream.topic(Events.tokens)) as tokens: tokens.publish("buffered") await backend.shutdown() @@ -795,7 +795,7 @@ async def test_final_failure_under_a_retry_policy( async def test_reader_stops_on_its_own_timer_and_resumes(stream_client: Client): - stream = await create_owned(stream_client, Events, id=f"timer-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"timer-{uuid.uuid4()}") first = await stream.topic(Events.tokens).publish("one") seen: list[streams.Record[Any]] = [] @@ -970,7 +970,7 @@ async def test_write_to_a_missing_stream_raises_at_the_await(stream_client: Clie async def test_write_to_a_closed_stream_raises_at_the_await(stream_client: Client): - stream = await create_owned(stream_client, Events, id=f"closed-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"closed-{uuid.uuid4()}") await stream.close() async with new_worker(stream_client, WritesToStream) as worker: assert ( diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 572cdeb428..e4127990ae 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -19,7 +19,6 @@ from temporalio.testing import WorkflowEnvironment from temporalio.worker import Replayer, Worker from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner -from tests.contrib.streams._support import create_owned from tests.contrib.streams.fake_extension import fake_client from tests.helpers import new_worker @@ -166,7 +165,7 @@ async def run(self, stream: streams.WorkflowStream[Events]) -> None: async def test_client_stream_takes_workflow_writes(stream_client: Client): - stream = await create_owned(stream_client, Events, id=f"chat-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"chat-{uuid.uuid4()}") assert await stream.get_offset() == 0 first = await stream.topic(Events.turns).publish(Turn("client")) last = await stream.topic(Events.tokens).publish("a", "b") @@ -237,20 +236,47 @@ async def test_client_reads_a_workflow_stream(stream_client: Client): await stream.close() +async def test_client_creates_a_stream_on_its_default_backend(stream_client: Client): + # A client owns the Stream it creates, as a Workflow and a Standalone Activity own theirs. + # It goes on the default backend: native, unless a registered backend has default=True. + lane, _ = _plugin.client_backend(stream_client, None) + stream_id = f"own-{uuid.uuid4()}" + stream = await stream_client.create_stream(Events, id=stream_id) + assert (stream.backend, stream.workflow_id) == (lane, None) + await stream.topic(Events.tokens).publish("a", "b") + # A create of the same id and Retention finds the Stream. + again = await stream_client.create_stream(Events, id=stream_id, backend=lane) + await again.topic(Events.tokens).publish("c") + await again.close(result="done") + items = [ + item async for item in stream_client.stream(Events, id=stream_id).subscribe() + ] + assert [item.data for item in items if isinstance(item, streams.Record)] == [ + "a", + "b", + "c", + ] + assert isinstance(items[-1], streams.Closed) and items[-1].result == "done" + with pytest.raises(streams.StreamClosedError): + await stream_client.create_stream(Events, id=stream_id) + with pytest.raises(streams.StreamConfigError, match="elsewhere"): + await stream_client.create_stream(Events, id=stream_id, backend="elsewhere") + + async def test_async_with_closes_a_client_stream(stream_client: Client): - async with create_owned(stream_client, Events, id=f"block-{uuid.uuid4()}") as done: + async with stream_client.create_stream(Events, id=f"block-{uuid.uuid4()}") as done: await done.topic(Events.tokens).publish("a") failed = stream_client.stream(Events, id=f"block-{uuid.uuid4()}") with pytest.raises(ValueError): - async with create_owned(stream_client, Events, id=failed.id): + async with stream_client.create_stream(Events, id=failed.id): raise ValueError("turn failed") - async with await create_owned( - stream_client, Events, id=f"block-{uuid.uuid4()}" + async with await stream_client.create_stream( + Events, id=f"block-{uuid.uuid4()}" ) as early: await early.close(result="early") cancelled = stream_client.stream(Events, id=f"block-{uuid.uuid4()}") with pytest.raises(asyncio.CancelledError): - async with create_owned(stream_client, Events, id=cancelled.id): + async with stream_client.create_stream(Events, id=cancelled.id): raise asyncio.CancelledError() for stream, result, failure, status in ( (done, None, None, streams.CloseStatus.COMPLETED), @@ -267,7 +293,7 @@ async def test_async_with_closes_a_client_stream(stream_client: Client): async def test_next_offset_resumes_after_the_last_item(stream_client: Client): - stream = await create_owned(stream_client, Events, id=f"resume-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"resume-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("a") turn = await stream.topic(Events.turns).publish(Turn("model")) tokens = stream.subscribe(Events.tokens, wait_new_records=False) @@ -289,7 +315,7 @@ async def test_next_offset_resumes_after_the_last_item(stream_client: Client): async def test_last_reads_the_newest_records_then_follows(stream_client: Client): - stream = await create_owned(stream_client, Events, id=f"last-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"last-{uuid.uuid4()}") tokens = stream.topic(Events.tokens) await tokens.publish("a", "b", "c") await stream.topic(Events.turns).publish(Turn("model")) @@ -314,7 +340,7 @@ async def test_last_reads_the_newest_records_then_follows(stream_client: Client) async def test_read_type_decodes_topics_outside_the_definition(stream_client: Client): - stream = await create_owned(stream_client, Events, id=f"typed-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"typed-{uuid.uuid4()}") await stream.topic("extra", type=Turn).publish(Turn("extra")) await stream.topic(Events.tokens).publish("a") @@ -327,7 +353,7 @@ async def read(**options: Any) -> list[object]: async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: Client): - stream = await create_owned(stream_client, Events, id=f"buffered-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"buffered-{uuid.uuid4()}") async with streams.BufferedPublisher(stream) as publisher: publisher.publish(Events.turns, Turn("user")) publisher.publish(Events.tokens, "a", "b") @@ -348,7 +374,7 @@ async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: C async def test_named_producer_fences_its_earlier_epoch(stream_client: Client): - stream = await create_owned(stream_client, Events, id=f"named-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"named-{uuid.uuid4()}") # A server that restarts writes as one name with a higher epoch, so the zombie it # replaces can't write after it. before = stream.producer("ingest", epoch=1) diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index a2e71d5b91..acb27e4826 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -25,7 +25,7 @@ from temporalio.converter import DataConverter, PayloadCodec from temporalio.exceptions import ApplicationError from temporalio.worker import Replayer, UnsandboxedWorkflowRunner -from tests.contrib.streams._support import create_owned, on_native +from tests.contrib.streams._support import on_native from tests.helpers import assert_task_fail_eventually, new_worker @@ -235,7 +235,7 @@ async def run(self, stream: streams.WorkflowStream[Events]) -> str: async def test_workflow_closes_only_its_own_streams(stream_client: Client): - stream = await create_owned(stream_client, Events, id=f"owned-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"owned-{uuid.uuid4()}") async with new_worker(stream_client, ClosesAReceivedStream) as worker: assert ( await stream_client.execute_workflow( diff --git a/tests/contrib/streams/test_workflow_reads.py b/tests/contrib/streams/test_workflow_reads.py index fc4d74f479..6eeb6e0a84 100644 --- a/tests/contrib/streams/test_workflow_reads.py +++ b/tests/contrib/streams/test_workflow_reads.py @@ -18,7 +18,7 @@ from temporalio.exceptions import ApplicationError from temporalio.worker import Replayer from temporalio.workflow import NondeterminismError -from tests.contrib.streams._support import create_owned, on_native +from tests.contrib.streams._support import on_native from tests.contrib.streams.test_redis import _Rot13Codec from tests.helpers import new_worker @@ -288,7 +288,7 @@ async def run(self, stream: streams.WorkflowStream[Events]) -> int: async def test_a_workflow_page_stays_below_the_marker_limit(stream_client: Client): # A Redis page is a History marker, and the server terminates a run whose marker is over # 2 MiB. So a page of large Records splits. - stream = await create_owned(stream_client, Events, id=f"big-{uuid.uuid4()}") + stream = await stream_client.create_stream(Events, id=f"big-{uuid.uuid4()}") for _ in range(25): await stream.topic(Events.tokens).publish("x" * 100_000) async with new_worker(stream_client, CountsTokens) as worker: From 90b0286a6d9fdc628dc3f2119206ae813feed4a6 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 17:31:23 -0700 Subject: [PATCH 29/79] Kept a native shutdown off the runtimes that no call built. The process-wide native backend has a runtime for each client connection it served. One that never made a call has no writer, so a shutdown no longer connects to close it. --- temporalio/common/streams/_native.py | 3 +++ tests/contrib/streams/test_native_server.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/temporalio/common/streams/_native.py b/temporalio/common/streams/_native.py index 49e6a7327e..9e2389856d 100644 --- a/temporalio/common/streams/_native.py +++ b/temporalio/common/streams/_native.py @@ -200,6 +200,9 @@ def __init__( self._streams: temporalio.bridge.streams.StreamRuntime | None = None async def call(self, method: str, input: bytes) -> bytes: + if self._streams is None and method == "CloseWriters": + # A runtime that no call built has no writer, so a shutdown makes no connection. + return ext_pb.CloseWritersResponse().SerializeToString() if self._streams is None: bridge_client = await _bridge_client(self._service) # Two first calls may both wait for the connection. One runtime serves both, so diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index d1ef30fff7..c039f10818 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -419,3 +419,5 @@ async def test_each_client_of_a_native_backend_calls_on_its_own_connection(): _, calls = _plugin.client_backend(plain, None) assert cast(Any, calls)._runtime._service is plain.service_client assert _plugin.client_backend(plain, None)[1] is calls + # No call built these runtimes, so a shutdown connects to none of them. + await native.shutdown() From 452ffcd07c04ba6d4b4432a9bedc1ea05050eb32 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 17:31:23 -0700 Subject: [PATCH 30/79] Keyed a Workflow's Activities only when its run uses Streams. The key is in History, so a run that never creates, binds or receives a Stream adds nothing there. A Local Activity keeps its key always, since its headers stay out of History. --- temporalio/common/streams/_extension.py | 29 ++++- temporalio/common/streams/_workflow.py | 11 +- .../contrib/streams/test_activity_handles.py | 105 ++++++++++++++++++ tests/contrib/streams/test_redis.py | 2 +- 4 files changed, 144 insertions(+), 3 deletions(-) diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index 8001633d11..8e09a00366 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -425,6 +425,7 @@ async def execute_workflow( self, input: temporalio.worker.ExecuteWorkflowInput ) -> Any: _take_owned_streams(input.headers) + _receive(input.args) try: result = await super().execute_workflow(input) except BaseException as err: @@ -434,6 +435,23 @@ async def execute_workflow( await _end_run() return result + async def handle_signal(self, input: temporalio.worker.HandleSignalInput) -> None: + _receive(input.args) + await super().handle_signal(input) + + async def handle_update_handler( + self, input: temporalio.worker.HandleUpdateInput + ) -> Any: + _receive(input.args) + return await super().handle_update_handler(input) + + +def _receive(args: Sequence[Any]) -> None: + # A run that receives a Stream uses Streams. The inputs, Signals and Updates are in + # History, so a replay finds the same. + if _stream_arguments(args): + streams_workflow._state().uses_streams = True + async def _end_run() -> None: # The run's reads end with it, so the server drops their parked reads. Its writes that @@ -471,6 +489,7 @@ def _take_owned_streams( if payload is None: return state = streams_workflow._state() + state.uses_streams = True converter = temporalio.converter.PayloadConverter.default for stream_id, backend, *_ in converter.from_payload(payload, list[Any]): state.stream(_owned_ref(stream_id)).backend = backend @@ -554,12 +573,20 @@ def continue_as_new(self, input: temporalio.worker.ContinueAsNewInput) -> NoRetu def start_activity( self, input: temporalio.worker.StartActivityInput ) -> temporalio.workflow.ActivityHandle[Any]: - input.headers = {**input.headers, **self._activity_headers()} + # The key goes into History, so only a run that uses Streams adds it, or a run whose + # Activity gets a Stream. Each call still counts, so a key is the same on replay. + headers = self._activity_headers() + state = streams_workflow._state() + if not state.uses_streams and _stream_arguments(input.args): + state.uses_streams = True + if state.uses_streams: + input.headers = {**input.headers, **headers} return super().start_activity(input) def start_local_activity( self, input: temporalio.worker.StartLocalActivityInput ) -> temporalio.workflow.ActivityHandle[Any]: + # A Local Activity's headers stay out of History, so each one gets its key. input.headers = {**input.headers, **self._activity_headers()} return super().start_local_activity(input) diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 8f9d6a2dd0..a7b8818f3b 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -107,6 +107,9 @@ class _RunState: waiting: dict[str, _Write] = field(default_factory=dict) # The Streams that a retry run's first write has withdrawn the previous run on. withdrawn: set[str] = field(default_factory=set) + # Whether the run created, bound or received a Stream. Only then do its Activities carry + # a Producer key, so a run with no Stream adds nothing to History. + uses_streams: bool = False def stream(self, ref: models.StreamRef) -> _RunStream: known = self.streams.get(ref.id) @@ -458,6 +461,8 @@ def subscribe( StreamConfigError: The Stream is of a backend that the Worker does not have. It fails the Workflow Task, not the run. """ + if not temporalio.workflow.unsafe.is_read_only(): + _state().uses_streams = True read = read_spec( self, topics, @@ -492,7 +497,9 @@ def _check( raise _errors.ReadOnlyContextError( f"Can't {action} a Stream in read-only code" ) - run_stream = _state().stream(self._model_ref()) + state = _state() + state.uses_streams = True + run_stream = state.stream(self._model_ref()) if run_stream.closed: raise _errors.StreamClosedError( f"Can't {action} the Stream {self.id}, since this run closed it", @@ -688,6 +695,8 @@ def stream_handle( :py:func:`temporalio.workflow.stream_handle`. """ stream = _own_handle(definition, backend) + if not temporalio.workflow.unsafe.is_read_only(): + _state().uses_streams = True if backend is None: known = _state().streams.get(stream.id) stream._backend = (known.backend if known else None) or _check_backend( diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index ee3c516667..a1b2d8ed5e 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -640,3 +640,108 @@ async def test_only_a_standalone_activity_creates_a_stream(stream_client: Client task_queue=worker.task_queue, ) assert "Only a Standalone Activity" in message + + +@activity.defn +async def plain(name: str) -> str: + return f"hello {name}" + + +@workflow.defn +class NeverTouchesStreams: + @workflow.run + async def run(self) -> str: + return await workflow.execute_activity( + plain, "ada", start_to_close_timeout=timedelta(seconds=10) + ) + + +@workflow.defn +class ReceivesAStream: + @workflow.run + async def run(self, stream: streams.WorkflowStream[Events]) -> str: + # The Activity gets no Stream, but the run received one. + return await workflow.execute_activity( + plain, stream.id, start_to_close_timeout=timedelta(seconds=10) + ) + + +@workflow.defn +class RetriesAfterContinueAsNew: + @workflow.run + async def run( + self, key: str, stream: streams.WorkflowStream[Events] | None + ) -> None: + if stream is None: + created = await workflow.create_stream(Events) + workflow.continue_as_new(args=[key, created]) + await workflow.execute_activity( + flaky, + CallModelInput(key, streams.ActivityStream.from_handle(stream)), + start_to_close_timeout=timedelta(seconds=30), + retry_policy=RetryPolicy(initial_interval=timedelta(milliseconds=10)), + ) + + +async def keyed(handle: WorkflowHandle[Any, Any], activity_type: str) -> list[bool]: + # Whether each schedule of the Activity carries the Producer key. + history = await handle.fetch_history() + return [ + "__temporal_stream_key" in attributes.header.fields + for event in history.events + if ( + attributes := event.activity_task_scheduled_event_attributes + ).activity_type.name + == activity_type + ] + + +async def test_only_a_run_that_uses_streams_keys_its_activities(stream_client: Client): + # The key is in History, so a run with no Stream adds none. A run that created, bound or + # received one keys each Activity, so an Activity that writes keeps its Producer id. + key = str(uuid.uuid4()) + async with new_worker( + stream_client, + NeverTouchesStreams, + ReceivesAStream, + Retrying, + RetriesAfterContinueAsNew, + activities=[plain, flaky], + ) as worker: + none = await stream_client.start_workflow( + NeverTouchesStreams.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await none.result() == "hello ada" + assert await keyed(none, "plain") == [False] + creates = await stream_client.start_workflow( + Retrying.run, key, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue + ) + await creates.result() + assert await keyed(creates, "flaky") == [True] + receives = await stream_client.start_workflow( + ReceivesAStream.run, + streams.WorkflowStream.from_handle(creates.stream(Events)), + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await receives.result() + assert await keyed(receives, "plain") == [True] + # The next run gets the Stream as its argument, and its retried Activity withdraws its + # first attempt as in the run that created the Stream. + continued_key = str(uuid.uuid4()) + continued = await stream_client.start_workflow( + RetriesAfterContinueAsNew.run, + args=[continued_key, None], + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await continued.result() + assert await keyed(continued, "flaky") == [True] + items = await read(continued) + superseded = [item for item in items if isinstance(item, streams.Superseded)] + assert len(superseded) == 1 + assert [r.data for r in records(items) if superseded[0].withdraws(r)] == [ + "attempt-1" + ] diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index c0179938e8..0d5fff2fa0 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -718,7 +718,7 @@ def test_closing_an_evicted_run_touches_no_runtime(): # Python prints "Exception ignored". inbound = _WorkflowInbound(_NeverEnds()) run = inbound.execute_workflow( - cast(ExecuteWorkflowInput, cast(object, SimpleNamespace(headers={}))) + cast(ExecuteWorkflowInput, cast(object, SimpleNamespace(headers={}, args=[]))) ) run.send(None) run.close() From ed0deb89a5af13d0339c968a5ab33d2a73bd6c87 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 17:37:21 -0700 Subject: [PATCH 31/79] Waited for the server's close in native reads after a run chain ends. The native server closes the Stream a moment after the run closes, so a read that ended at the newest Record could miss the close. --- tests/contrib/streams/_support.py | 16 ++++++++++++++++ tests/contrib/streams/test_activity_handles.py | 5 ++--- tests/contrib/streams/test_redis.py | 5 ++--- tests/contrib/streams/test_workflow_module.py | 4 ++-- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/contrib/streams/_support.py b/tests/contrib/streams/_support.py index cf2f9ab80c..968ee4f896 100644 --- a/tests/contrib/streams/_support.py +++ b/tests/contrib/streams/_support.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import functools import json import subprocess @@ -50,3 +51,18 @@ def on_native(client: Any) -> bool: name, _ = _plugin.client_backend(client, None) return name == "native" + + +async def read_after_run_end(client: Any, stream: Any) -> list[Any]: + """The items of a Stream whose owner's run chain ended. + + The native server closes the Stream at the chain's end, a moment after the run closes, so + a native read waits for the close. A Redis read ends at the newest Record. + """ + if not on_native(client): + return [item async for item in stream.subscribe(wait_new_records=False)] + + async def to_close() -> list[Any]: + return [item async for item in stream.subscribe()] + + return await asyncio.wait_for(to_close(), 30) diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index a1b2d8ed5e..17a7f24d51 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -16,7 +16,7 @@ from temporalio.common import RetryPolicy, streams from temporalio.common.streams import _activity from temporalio.exceptions import CancelledError -from tests.contrib.streams._support import on_native +from tests.contrib.streams._support import on_native, read_after_run_end from tests.helpers import new_worker @@ -361,8 +361,7 @@ async def test_store_refuses_another_chains_close(stream_client: Client): id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) == ("stream_not_found" if native else "stream_not_owner") - stream = owner.stream(Events) - items = [item async for item in stream.subscribe(wait_new_records=False)] + items = await read_after_run_end(stream_client, owner.stream(Events)) # The server closed the native Stream when its run chain ended. expected: list[type] = [streams.Closed] if native else [streams.Record] assert [type(item) for item in items] == expected diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 0d5fff2fa0..80987a6824 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -37,7 +37,7 @@ SandboxedWorkflowRunner, SandboxRestrictions, ) -from tests.contrib.streams._support import on_native +from tests.contrib.streams._support import on_native, read_after_run_end from tests.helpers import new_worker @@ -774,8 +774,7 @@ async def test_final_failure_under_a_retry_policy( ) with pytest.raises(WorkflowFailureError): await handle.result() - stream = workflow_stream(handle) - items = [item async for item in stream.subscribe(wait_new_records=False)] + items = await read_after_run_end(stream_client, workflow_stream(handle)) written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] # The end of the chain closes only a native Stream, with the last run's failure. diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index acb27e4826..3756c40dfb 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -25,7 +25,7 @@ from temporalio.converter import DataConverter, PayloadCodec from temporalio.exceptions import ApplicationError from temporalio.worker import Replayer, UnsandboxedWorkflowRunner -from tests.contrib.streams._support import on_native +from tests.contrib.streams._support import on_native, read_after_run_end from tests.helpers import assert_task_fail_eventually, new_worker @@ -267,7 +267,7 @@ async def test_run_end_closes_only_a_native_stream(stream_client: Client): ) with pytest.raises(WorkflowFailureError): await handle.result() - items = await read(handle) + items = await read_after_run_end(stream_client, handle.stream(Events)) assert data(items) == ["before"] if not on_native(stream_client): # The Stream stays open, so a later reader still waits for a close. From e77169329bf3a0efa732b90559d9cd5d88e8b25f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 18:30:57 -0700 Subject: [PATCH 32/79] Decoded the Stream test vectors as UTF-8 on every OS. --- tests/contrib/streams/_support.py | 2 +- tests/contrib/streams/test_vectors.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/contrib/streams/_support.py b/tests/contrib/streams/_support.py index e6cd47603d..90b2e05c7e 100644 --- a/tests/contrib/streams/_support.py +++ b/tests/contrib/streams/_support.py @@ -38,4 +38,4 @@ def crate_file(relative: str) -> Path: def vectors(name: str) -> object: - return json.loads(crate_file(f"testdata/{name}").read_text()) + return json.loads(crate_file(f"testdata/{name}").read_text(encoding="utf-8")) diff --git a/tests/contrib/streams/test_vectors.py b/tests/contrib/streams/test_vectors.py index 6580f10d56..792985576e 100644 --- a/tests/contrib/streams/test_vectors.py +++ b/tests/contrib/streams/test_vectors.py @@ -27,8 +27,8 @@ ], ) def test_vendored_protos_match_the_crate(relative: str): - crate = crate_file(f"protos/{relative}").read_text() - assert (_SCRIPTS / relative).read_text() == crate + crate = crate_file(f"protos/{relative}").read_text(encoding="utf-8") + assert (_SCRIPTS / relative).read_text(encoding="utf-8") == crate def test_stream_ids_match_the_crate(): From b880a6d2a7425077fa1d497b7da8f4d1655153f2 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 18:30:57 -0700 Subject: [PATCH 33/79] Took a client config with no interceptors in the Stream backends. --- temporalio/common/streams/_extension.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index 32683193a2..8815c4f61a 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -111,7 +111,7 @@ def _configure_client( def _backend_for(self, client: temporalio.client.Client) -> BackendCalls: # The Stream runtime that this backend gave the client. interceptor = self._interceptor_of( - client.config(active_config=True)["interceptors"] + client.config(active_config=True).get("interceptors", []) ) if interceptor is None or interceptor.backend is None: raise StreamConfigError( @@ -136,7 +136,7 @@ def _configure_worker( config = super()._configure_worker(config) client = config["client"] # type: ignore[reportTypedDictNotRequiredAccess] interceptors = [ - *client.config(active_config=True)["interceptors"], + *client.config(active_config=True).get("interceptors", []), *config.get("interceptors", []), ] if any( From 3ef573b1a2487bfe43b38817cc42217b82202638 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 18:30:57 -0700 Subject: [PATCH 34/79] Hid the generated Stream Nexus clients from the API docs. --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index af19d3f8f6..f2e97120ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -239,6 +239,8 @@ privacy = [ "HIDDEN:temporalio.worker.workflow_instance", "HIDDEN:temporalio.worker.workflow_sandbox.importer", "HIDDEN:temporalio.worker.workflow_sandbox.in_sandbox", + "HIDDEN:temporalio.common.streams.system", + "HIDDEN:temporalio.common.streams.writer_system", "HIDDEN:**.*_pb2*", ] project-name = "Temporal Python" From c3b37981aaec5c0627c225e73b77ca2dd43d4c8c Mon Sep 17 00:00:00 2001 From: Justin Anderson <44687433+jmaeagle99@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:19:44 -0700 Subject: [PATCH 35/79] fix: annotate pydantic TypeAdapter cache for pydantic 2.14 (#2011) (cherry picked from commit 437eddeec6835f6425b4b8f95fc70d524b322124) --- temporalio/contrib/pydantic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/temporalio/contrib/pydantic.py b/temporalio/contrib/pydantic.py index c6ea2a2da9..94e4363db0 100644 --- a/temporalio/contrib/pydantic.py +++ b/temporalio/contrib/pydantic.py @@ -75,8 +75,8 @@ def __init__( raise ValueError("max_cached_type_adapters cannot be negative") self._schema_serializer = SchemaSerializer(any_schema()) self._to_json_options = to_json_options - self._type_adapter = functools.lru_cache(maxsize=max_cached_type_adapters)( - TypeAdapter + self._type_adapter: functools._lru_cache_wrapper[TypeAdapter[Any]] = ( + functools.lru_cache(maxsize=max_cached_type_adapters)(TypeAdapter) ) def __getstate__(self) -> dict[str, Any]: From e033b82266c30c4e8623959703edf163a7ef686b Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 18:30:57 -0700 Subject: [PATCH 36/79] Added a changelog fragment for Streams. --- changelog/added/babbling-brook.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/added/babbling-brook.md diff --git a/changelog/added/babbling-brook.md b/changelog/added/babbling-brook.md new file mode 100644 index 0000000000..eedb94fa74 --- /dev/null +++ b/changelog/added/babbling-brook.md @@ -0,0 +1,2 @@ +Add experimental Streams in `temporalio.common.streams`: ordered logs of Records that Workflows, Activities and clients write, and that clients and Activities read. +Add a Redis Stream backend in `temporalio.contrib.stream`, and an in-memory one for tests in `temporalio.contrib.stream.testing`. From 3d88b43b3f1338347ebfc0cc72fe52052b2f2622 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 18:32:03 -0700 Subject: [PATCH 37/79] Added a changelog fragment for the native Stream backend. --- changelog/added/jaunty-jetty.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changelog/added/jaunty-jetty.md diff --git a/changelog/added/jaunty-jetty.md b/changelog/added/jaunty-jetty.md new file mode 100644 index 0000000000..43b4f8616d --- /dev/null +++ b/changelog/added/jaunty-jetty.md @@ -0,0 +1,2 @@ +Add a native Stream backend that keeps Streams in the Temporal server. It is the default and needs no registration, and a Worker or client can register more than one backend. +Add Workflow reads of a Stream with `WorkflowStream.subscribe`, Streams owned by Standalone Activities with `activity.create_stream` and `ActivityHandle.stream`, and client-owned Streams with `Client.create_stream`. From cb68aaa8f411bdcdcd6fd75cbee20dee62dc3499 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 18:33:51 -0700 Subject: [PATCH 38/79] Named the Subscription in a WorkflowItems docs link. --- temporalio/common/streams/_workflow_reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/temporalio/common/streams/_workflow_reader.py b/temporalio/common/streams/_workflow_reader.py index 933aa002af..548bfdce3c 100644 --- a/temporalio/common/streams/_workflow_reader.py +++ b/temporalio/common/streams/_workflow_reader.py @@ -88,8 +88,8 @@ class WorkflowItems(Subscription[Any]): It ends at a close, or at the newest Record when ``wait_new_records`` is False. It also ends when this run should continue as new, after at least one page, and then - :py:attr:`continue_as_new_suggested` is True. Pass :py:attr:`next_offset` to the next run, - which reads on from there. + :py:attr:`continue_as_new_suggested` is True. Pass + :py:attr:`Subscription.next_offset` to the next run, which reads on from there. .. warning:: This API is experimental. From 71ae5398fd2d2b5a8e201e2b2082b779200651d4 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 19:28:47 -0700 Subject: [PATCH 39/79] Hid only the generated Stream clients from the API docs. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f2e97120ac..8b05971325 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -239,8 +239,8 @@ privacy = [ "HIDDEN:temporalio.worker.workflow_instance", "HIDDEN:temporalio.worker.workflow_sandbox.importer", "HIDDEN:temporalio.worker.workflow_sandbox.in_sandbox", - "HIDDEN:temporalio.common.streams.system", - "HIDDEN:temporalio.common.streams.writer_system", + "HIDDEN:temporalio.common.streams.system.client", + "HIDDEN:temporalio.common.streams.writer_system.client", "HIDDEN:**.*_pb2*", ] project-name = "Temporal Python" From da1da37b981e73f829eb502b7d65eebe0d12a880 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 19:28:47 -0700 Subject: [PATCH 40/79] Named the Stream entry points and other API changes in the changelog. --- changelog/added/babbling-brook.md | 5 +++-- changelog/changed/babbling-brook.md | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelog/changed/babbling-brook.md diff --git a/changelog/added/babbling-brook.md b/changelog/added/babbling-brook.md index eedb94fa74..9d6abdeb19 100644 --- a/changelog/added/babbling-brook.md +++ b/changelog/added/babbling-brook.md @@ -1,2 +1,3 @@ -Add experimental Streams in `temporalio.common.streams`: ordered logs of Records that Workflows, Activities and clients write, and that clients and Activities read. -Add a Redis Stream backend in `temporalio.contrib.stream`, and an in-memory one for tests in `temporalio.contrib.stream.testing`. +Add experimental Streams in `temporalio.common.streams`: ordered logs of Records that Workflows, Activities and clients write, and that clients and Activities read. Register a backend with `stream_backends=` on `Client.connect`, `Replayer` or `WorkflowEnvironment`, then use `workflow.create_stream`, `Client.stream` or `WorkflowHandle.stream`. +Add a Redis Stream backend in `temporalio.contrib.stream`, and an in-memory one for tests in `temporalio.contrib.stream.testing`. The Redis backend needs a build with the bridge's `streams` feature, which the official builds have. +Add `activity.Info.workflow_task`, the Workflow Task attempt in which a Worker dispatched a Local Activity. diff --git a/changelog/changed/babbling-brook.md b/changelog/changed/babbling-brook.md new file mode 100644 index 0000000000..7d4504d693 --- /dev/null +++ b/changelog/changed/babbling-brook.md @@ -0,0 +1 @@ +Convert values of a `transfer_type_convertible` class inside JSON values too, such as list items and dataclass fields, not only at the top level of a payload. A class that is itself a dataclass is still written as its fields there. From b4a68f81cec2961ef6ff6f06d36557f932f6cffc Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sat, 24 Oct 2026 19:29:21 -0700 Subject: [PATCH 41/79] Corrected the native backend changelog on Workers and the server. --- changelog/added/jaunty-jetty.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog/added/jaunty-jetty.md b/changelog/added/jaunty-jetty.md index 43b4f8616d..a23c22acfc 100644 --- a/changelog/added/jaunty-jetty.md +++ b/changelog/added/jaunty-jetty.md @@ -1,2 +1,2 @@ -Add a native Stream backend that keeps Streams in the Temporal server. It is the default and needs no registration, and a Worker or client can register more than one backend. -Add Workflow reads of a Stream with `WorkflowStream.subscribe`, Streams owned by Standalone Activities with `activity.create_stream` and `ActivityHandle.stream`, and client-owned Streams with `Client.create_stream`. +Add a native Stream backend that keeps Streams in the Temporal server, which needs a server with the Stream service. It is the default and needs no registration. A client can register more than one backend, and its Workers serve them all. +Add Workflow reads of a Stream with `WorkflowStream.subscribe`, Streams owned by Standalone Activities with `activity.create_stream` and `client.ActivityHandle.stream`, and client-owned Streams with `Client.create_stream`. From 466aff54a0a5561b49ebe622a8dc9260406c2054 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 00:56:13 -0700 Subject: [PATCH 42/79] Left a Standalone Activity's Stream open at the end of its block. The server closes it when the Activity ends, with the result, and a retry creates it again. A close at the block end failed every retry. --- temporalio/activity.py | 8 +-- temporalio/common/streams/_activity_stream.py | 9 +++- tests/contrib/streams/test_native_server.py | 51 ++++++++++++++++--- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/temporalio/activity.py b/temporalio/activity.py index 09a0db712f..ac85070511 100644 --- a/temporalio/activity.py +++ b/temporalio/activity.py @@ -530,11 +530,11 @@ def create_stream( Its id is ``::``, so a client finds it from the Activity handle with :py:meth:`temporalio.client.ActivityHandle.stream` and no call. The Stream goes on across the Activity's retries, and the server closes it when the Activity - ends, with its status and result. Await the call for the handle, or use it in - ``async with``, which closes the Stream at the end of the block:: + ends, with its status and result. So the end of an ``async with`` block doesn't close it. + Await the call for the handle:: - async with activity.create_stream(Events) as stream: - await stream.topic(Events.tokens).publish("hi") + stream = await activity.create_stream(Events) + await stream.topic(Events.tokens).publish("hi") Only the native backend keeps the Stream of a Standalone Activity, since nothing else would close it when the Activity ends. diff --git a/temporalio/common/streams/_activity_stream.py b/temporalio/common/streams/_activity_stream.py index 09db973687..85846be3a9 100644 --- a/temporalio/common/streams/_activity_stream.py +++ b/temporalio/common/streams/_activity_stream.py @@ -184,7 +184,9 @@ async def close( await self._access().close(result, failure) async def __aenter__(self) -> ActivityStream[E]: - """Use the Stream in ``async with``, which closes it at the end.""" + """Use the Stream in ``async with``, which closes it at the end, except a Standalone + Activity's Stream. + """ return self async def __aexit__( @@ -196,8 +198,11 @@ async def __aexit__( """Close the Stream, with the exception as its failure if there is one. A cancel closes it with a ``CancelledError`` failure. Another ``BaseException`` leaves - it open. + it open. A Standalone Activity's Stream stays open, since the server closes it when the + Activity ends, with its status and result, and a retry of the Activity writes on to it. """ + if self.activity_id: + return close, failure = exit_failure(exc) if close: await self.close(failure=failure) diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index c039f10818..7b30a603cd 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -13,7 +13,7 @@ import temporalio.api.streamservice.v1 as stream_pb from temporalio import activity, workflow from temporalio.client import Client, WorkflowHandle -from temporalio.common import streams +from temporalio.common import RetryPolicy, streams from temporalio.common.streams import _native, _plugin from temporalio.common.streams._native import POSITIONS_TYPE from temporalio.contrib.stream import RedisStreamBackend @@ -351,9 +351,10 @@ async def test_a_native_reader_of_a_client_stream_ends_at_half_its_retention( @activity.defn -async def streams_its_tokens() -> str: - async with activity.create_stream(Events) as stream: - await stream.topic(Events.tokens).publish("a", "b") +async def closes_its_stream() -> str: + stream = await activity.create_stream(Events) + await stream.topic(Events.tokens).publish("a", "b") + await stream.close() return "done" @@ -370,11 +371,11 @@ async def test_a_standalone_activity_owns_its_stream(native_client: Client): task_queue = f"tq-{uuid.uuid4()}" async with new_worker( native_client, - activities=[streams_its_tokens, leaves_its_stream_open], + activities=[closes_its_stream, leaves_its_stream_open], task_queue=task_queue, ): closed = await native_client.start_activity( - streams_its_tokens, + closes_its_stream, id=f"act-{uuid.uuid4()}", task_queue=task_queue, start_to_close_timeout=timedelta(seconds=30), @@ -400,6 +401,44 @@ async def test_a_standalone_activity_owns_its_stream(native_client: Client): assert left[-1].status == streams.CloseStatus.COMPLETED +@activity.defn +async def streams_in_a_block() -> str: + attempt = activity.info().attempt + async with activity.create_stream(Events) as stream: + await stream.topic(Events.tokens).publish(f"attempt {attempt}") + if attempt == 1: + raise RuntimeError("transient") + return f"done on attempt {attempt}" + + +async def test_a_block_end_leaves_a_standalone_activitys_stream_open( + native_client: Client, +): + # A retry of the Activity creates its Stream again, so a close at the end of the failed + # attempt's block would fail every retry. + task_queue = f"tq-{uuid.uuid4()}" + async with new_worker( + native_client, activities=[streams_in_a_block], task_queue=task_queue + ): + handle = await native_client.start_activity( + streams_in_a_block, + id=f"act-{uuid.uuid4()}", + task_queue=task_queue, + start_to_close_timeout=timedelta(seconds=30), + retry_policy=RetryPolicy( + initial_interval=timedelta(milliseconds=100), maximum_attempts=2 + ), + ) + assert await handle.result() == "done on attempt 2" + items = [item async for item in handle.stream(Events).subscribe()] + assert "attempt 2" in [ + item.data for item in items if isinstance(item, streams.Record) + ] + # The server's close carries the result of the attempt that ended the Activity. + assert isinstance(items[-1], streams.Closed) + assert items[-1].result == "done on attempt 2" + + async def test_each_client_of_a_native_backend_calls_on_its_own_connection(): # One backend on two clients gives each its own runtime, so a call doesn't go on the # connection of the client that came last. From 3fe0dfa62282f79c3398db15b3bfd7fb2b6eddec Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 00:57:00 -0700 Subject: [PATCH 43/79] Refused a Local Activity's close of a Workflow's Stream. A Local Activity writes before its Workflow Task commits, so a close there stayed when the task failed, and the task's retry could not create the Stream again. --- temporalio/common/streams/_activity_stream.py | 19 ++++++++++--------- .../contrib/streams/test_activity_handles.py | 11 ++++++++--- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/temporalio/common/streams/_activity_stream.py b/temporalio/common/streams/_activity_stream.py index cf25b08e1f..524e24b5f7 100644 --- a/temporalio/common/streams/_activity_stream.py +++ b/temporalio/common/streams/_activity_stream.py @@ -50,15 +50,14 @@ def _bound(handle: StreamHandle[Any]) -> _activity.ActivityStreams: def _check_close(handle: StreamHandle[Any]) -> None: - # A Workflow's Stream closes only from its run chain, and a client's only from a client. So - # only a Local Activity of the chain may close here. The Store refuses the others too, and - # this error comes with no call. + # A Workflow's Stream closes only from Workflow code, once its task commits, and a client's + # only from a client. The Store refuses the others too, and this error comes with no call. streams = _bound(handle) local = streams.producer.kind == stream_pb.ProducerKind.PRODUCER_KIND_LOCAL_ACTIVITY - if not local or (handle.workflow_id and handle.workflow_id != streams.workflow_id): + if not local or handle.workflow_id: raise _errors.StreamNotOwnerError( f"Can't close the Stream {handle.id} from this Activity. A Workflow's Stream " - "closes only from its run chain, and a client's only from a client.", + "closes only from Workflow code, and a client's only from a client.", reason="stream_not_owner", ) @@ -77,7 +76,7 @@ async def known_id() -> str: stream_id=known_id, writer=lambda: streams.writer(stream_id), # A handle with only an id names no chain, so the close names this Activity's own, and - # the Store refuses it when another chain owns the Stream. + # the Store refuses it for a Workflow's Stream. workflow_id=handle.workflow_id or streams.workflow_id, first_execution_run_id=handle.first_execution_run_id, ) @@ -160,9 +159,11 @@ async def close( Readers get the Records before it, then a ``Closed`` item with ``result`` or ``failure``. A ``CancelledError`` failure closes it as canceled. - Only a Local Activity of the run chain that created the Stream may close it. Any other - Activity gets :py:class:`StreamNotOwnerError`, so it can't end the Stream for every - reader. + No Activity closes a Workflow's Stream, also a Local Activity of the run chain that + created it, and no Activity closes a client's Stream. They get + :py:class:`StreamNotOwnerError`. A Local Activity writes before its Workflow Task + commits, so its close would stay when the task fails, and the task's retry could not + create the Stream again. Workflow code closes a Workflow's Stream. """ _check_close(self) await self._access().close(result, failure) diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index ee7191a475..a288f4ddaa 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -284,12 +284,13 @@ async def run(self) -> list[str]: streams.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) - # A Local Activity is part of the run chain, so it may close the chain's Stream. + # A Local Activity writes before its Workflow Task commits, so it can't close either. local = await workflow.execute_local_activity( close_it, streams.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) + await stream.close(result="closed by the run") return [backend, owner, local] @@ -301,11 +302,15 @@ async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): Closing.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) # An Activity writes to the Workflow's Stream but can't end it for every reader. - assert await handle.result() == ["config", "stream_not_owner", "closed"] + assert await handle.result() == [ + "config", + "stream_not_owner", + "stream_not_owner", + ] items = await read(handle) assert [r.data for r in records(items)] == ["last", "last"] assert isinstance(items[-1], streams.Closed) - assert items[-1].result == "closed by activity" + assert items[-1].result == "closed by the run" @workflow.defn From 47aa31e8a4409bc9102e7fe6a0880d8448eef315 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 00:57:24 -0700 Subject: [PATCH 44/79] Named a Standalone Activity's Stream by its Activity in Workflow calls. The server nests the Stream in its Activity, so a ref with only the id found no Stream. The server keeps the reader's floor there, so the read asks no Retention. --- temporalio/common/streams/_workflow.py | 9 ++-- temporalio/common/streams/_workflow_reader.py | 14 ++++-- tests/contrib/streams/test_native_server.py | 47 +++++++++++++++++++ 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index a7b8818f3b..79463f015c 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -380,10 +380,13 @@ def _model_ref(self) -> models.StreamRef: ) def _ref(self) -> stream_pb.StreamRef: + # A server that nests a Stream in its owner finds it only by the owner's fields. return stream_pb.StreamRef( id=self.id, workflow_id=self.workflow_id or "", first_execution_run_id=self.first_execution_run_id or "", + activity_id=self.activity_id or "", + activity_run_id=self.activity_run_id or "", ) @overload @@ -476,11 +479,7 @@ def subscribe( reader = _workflow_reader.reader_of(_check_backend(self.backend, "read")) return WorkflowItems( reader, - stream_pb.StreamRef( - id=self.id, - workflow_id=self.workflow_id or "", - first_execution_run_id=self.first_execution_run_id or "", - ), + self._ref(), topics=read.topics, topic_prefixes=topic_prefixes, types=read.types, diff --git a/temporalio/common/streams/_workflow_reader.py b/temporalio/common/streams/_workflow_reader.py index 548bfdce3c..ccd7614387 100644 --- a/temporalio/common/streams/_workflow_reader.py +++ b/temporalio/common/streams/_workflow_reader.py @@ -143,8 +143,8 @@ def __init__( self.continue_as_new_suggested = False """Whether the read ended because this run should continue as new. The server suggests it for the run's History or for the Records that the reader keeps on the - server. A reader of a native Stream that no Workflow owns gets it at half the Stream's - Retention after its first page, so its replay finds the Records.""" + server. A reader of a client's native Stream gets it at half the Stream's Retention + after its first page, so its replay finds the Records.""" self.owner_ended = False """Whether the read ended because the run chain that owns the Stream ended with no close, such as a terminated run on Redis.""" @@ -278,10 +278,14 @@ async def _take( await self._learn_retention() async def _learn_retention(self) -> None: - # Retention trims a Stream that no Workflow owns under its readers. A backend whose - # replay reads the Stream again needs the reader to move on before that. + # Retention trims a client's Stream under its readers. A backend whose replay reads the + # Stream again needs the reader to move on before that. stream = self._request.stream - if stream.workflow_id or not self._reader._temporal_stream_parks_reads: + if ( + stream.workflow_id + or stream.activity_id + or not self._reader._temporal_stream_parks_reads + ): return description = await self._reader._temporal_stream_describe( stream_pb.DescribeStreamRequest(stream=stream) diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index 7b30a603cd..aeb7abfec0 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -6,6 +6,7 @@ import asyncio import base64 import concurrent.futures +import json import uuid from datetime import timedelta from typing import Any, cast @@ -439,6 +440,52 @@ async def test_a_block_end_leaves_a_standalone_activitys_stream_open( assert items[-1].result == "done on attempt 2" +@workflow.defn +class ReadsAll: + @workflow.run + async def run(self, stream: streams.WorkflowStream[Events]) -> list[Any]: + return [ + item.data if isinstance(item, streams.Record) else f"closed {item.result}" + async for item in stream.subscribe() + if not isinstance(item, streams.Superseded) + ] + + +async def test_a_workflow_reads_a_standalone_activitys_stream(native_client: Client): + # The server nests the Stream in its Activity, so the read names the Activity. + task_queue = f"tq-{uuid.uuid4()}" + async with new_worker( + native_client, + ReadsAll, + activities=[leaves_its_stream_open], + task_queue=task_queue, + ): + activity_handle = await native_client.start_activity( + leaves_its_stream_open, + id=f"act-{uuid.uuid4()}", + task_queue=task_queue, + start_to_close_timeout=timedelta(seconds=30), + ) + await activity_handle.result() + reader = await native_client.start_workflow( + ReadsAll.run, + streams.WorkflowStream.from_handle(activity_handle.stream(Events)), + id=f"wf-{uuid.uuid4()}", + task_queue=task_queue, + ) + assert await reader.result() == ["a", "closed done"] + # The server keeps the reader's floor on a nested Stream, so the read asks no Retention. + local_activities = [ + json.loads( + event.marker_recorded_event_attributes.details["data"].payloads[0].data + )["activity_type"] + for event in (await reader.fetch_history()).events + if event.HasField("marker_recorded_event_attributes") + ] + assert local_activities + assert _native._describe_activity_name("native") not in local_activities + + async def test_each_client_of_a_native_backend_calls_on_its_own_connection(): # One backend on two clients gives each its own runtime, so a call doesn't go on the # connection of the client that came last. From 006e9b0a0bd884cf015d87a35f8561018db56c5f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 00:58:22 -0700 Subject: [PATCH 45/79] Held each client weakly in the native backend's runtimes. The backend keeps a client's runtime in a weak map by the client, and the implicit backend lives as long as the process. A strong ref in the runtime kept every client and its connection alive. --- temporalio/common/streams/_native.py | 11 ++++++--- tests/contrib/streams/test_native_server.py | 26 +++++++++++++++++++-- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/temporalio/common/streams/_native.py b/temporalio/common/streams/_native.py index 9e2389856d..2165905f00 100644 --- a/temporalio/common/streams/_native.py +++ b/temporalio/common/streams/_native.py @@ -195,7 +195,9 @@ class _ConnectionRuntime: def __init__( self, service: temporalio.service.ServiceClient, config: bytes ) -> None: - self._service = service + # The backend keeps the runtime by the client in a weak map, so a strong ref here + # would keep every client alive. + self._service = weakref.ref(service) self._config = config self._streams: temporalio.bridge.streams.StreamRuntime | None = None @@ -204,12 +206,15 @@ async def call(self, method: str, input: bytes) -> bytes: # A runtime that no call built has no writer, so a shutdown makes no connection. return ext_pb.CloseWritersResponse().SerializeToString() if self._streams is None: - bridge_client = await _bridge_client(self._service) + service = self._service() + if service is None: + raise RuntimeError("The client of the native Stream backend is gone") + bridge_client = await _bridge_client(service) # Two first calls may both wait for the connection. One runtime serves both, so # a writer that one opens is known to the other. if self._streams is None: self._streams = temporalio.bridge.streams.StreamRuntime.on_client( - _runtime_of(self._service)._core_runtime, + _runtime_of(service)._core_runtime, bridge_client, self._config, ) diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index aeb7abfec0..5141e1b59f 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -6,8 +6,10 @@ import asyncio import base64 import concurrent.futures +import gc import json import uuid +import weakref from datetime import timedelta from typing import Any, cast @@ -496,14 +498,34 @@ async def test_each_client_of_a_native_backend_calls_on_its_own_connection(): cast(Any, native._backend_for(client))._runtime for client in (first, second) ] assert runtimes[0] is not runtimes[1] - assert [runtime._service for runtime in runtimes] == [ + assert [runtime._service() for runtime in runtimes] == [ first.service_client, second.service_client, ] # The native backend that no client registers works the same, and makes no call. plain = await Client.connect("127.0.0.1:3", lazy=True) _, calls = _plugin.client_backend(plain, None) - assert cast(Any, calls)._runtime._service is plain.service_client + assert cast(Any, calls)._runtime._service() is plain.service_client assert _plugin.client_backend(plain, None)[1] is calls # No call built these runtimes, so a shutdown connects to none of them. await native.shutdown() + + +async def test_a_native_backend_keeps_no_client_alive(): + # The implicit backend lives as long as the process, and every Worker's client goes + # through it. + native = streams.NativeStreamBackend() + registered = await Client.connect( + "127.0.0.1:1", lazy=True, stream_backends=[native] + ) + plain = await Client.connect("127.0.0.1:2", lazy=True) + native._backend_for(registered) + _plugin.client_backend(plain, None) + services = [ + weakref.ref(registered.service_client), + weakref.ref(plain.service_client), + ] + del registered, plain + gc.collect() + assert [service() for service in services] == [None, None] + assert not native._connections From e78e39e0d9a1a923e010ee106c7b573e5cf4f461 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 00:58:39 -0700 Subject: [PATCH 46/79] Described Retention as a Record's age in the Jetty create docstrings. The Stores drop Records older than the Retention on every append, open or not. --- temporalio/activity.py | 3 ++- temporalio/client/_client.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/temporalio/activity.py b/temporalio/activity.py index ac85070511..fdc26b3c56 100644 --- a/temporalio/activity.py +++ b/temporalio/activity.py @@ -544,7 +544,8 @@ def create_stream( Args: definition: The Stream definition class. Its name is the Stream name. - retention: How long the Records stay after the close. Defaults to the backend's. + retention: The age after which a Record is dropped. A closed Stream stays one + Retention after its close. Defaults to the backend's. backend: The name of a native backend. Defaults to ``native``. Raises: diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index ec3a491646..a94f7f852f 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -976,7 +976,8 @@ def create_stream( Args: definition: The Stream definition class. id: The Stream id, as a Workflow takes a Workflow id. - retention: How long the Records stay after the close. Defaults to the backend's. + retention: The age after which a Record is dropped. A closed Stream stays one + Retention after its close. Defaults to the backend's. backend: The name of the backend. Defaults to the registered backend with ``default=True``, else to ``native``. From 85894c540af27327967d1c88d2b779248a14aaf8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 00:59:05 -0700 Subject: [PATCH 47/79] Closed a Workflow's Stream as canceled when a cancel ends its block. A run's cancel reaches the block as the error of an Activity, a Local Activity or a child, and no retry run follows a cancelled run. --- temporalio/common/streams/_activity_stream.py | 2 +- temporalio/common/streams/_items.py | 7 +-- temporalio/common/streams/_stream_client.py | 4 +- temporalio/common/streams/_workflow.py | 16 ++++- tests/contrib/streams/test_workflow_module.py | 58 ++++++++++++++++++- 5 files changed, 76 insertions(+), 11 deletions(-) diff --git a/temporalio/common/streams/_activity_stream.py b/temporalio/common/streams/_activity_stream.py index 524e24b5f7..b0a9302638 100644 --- a/temporalio/common/streams/_activity_stream.py +++ b/temporalio/common/streams/_activity_stream.py @@ -157,7 +157,7 @@ async def close( """Write this attempt's Records, then close the Stream. Readers get the Records before it, then a ``Closed`` item with ``result`` or - ``failure``. A ``CancelledError`` failure closes it as canceled. + ``failure``. A cancel failure, such as a ``CancelledError``, closes it as canceled. No Activity closes a Workflow's Stream, also a Local Activity of the run chain that created it, and no Activity closes a client's Stream. They get diff --git a/temporalio/common/streams/_items.py b/temporalio/common/streams/_items.py index ade0ea70c4..312610a983 100644 --- a/temporalio/common/streams/_items.py +++ b/temporalio/common/streams/_items.py @@ -2,7 +2,6 @@ from __future__ import annotations -import asyncio import enum from dataclasses import dataclass from typing import Any, Generic, TypeVar @@ -10,7 +9,7 @@ from temporalio.api.streamservice.v1 import CloseStatus as _CloseStatus from temporalio.api.streamservice.v1 import ProducerKind from temporalio.common.streams.system import models -from temporalio.exceptions import CancelledError +from temporalio.exceptions import is_cancelled_exception T = TypeVar("T") @@ -92,7 +91,7 @@ class CloseStatus(enum.IntEnum): """Closed with a failure.""" CANCELED = int(_CloseStatus.CLOSE_STATUS_CANCELED) - """Closed with a ``CancelledError`` failure, such as by a cancelled owner run.""" + """Closed with a cancel failure, such as by a cancelled owner run.""" TERMINATED = int(_CloseStatus.CLOSE_STATUS_TERMINATED) """The server closed it, since the owner run was terminated.""" @@ -103,7 +102,7 @@ class CloseStatus(enum.IntEnum): def close_status(failure: BaseException | None) -> _CloseStatus.ValueType | None: """The status a caller's close asks for. None lets the backend pick completed or failed.""" - if isinstance(failure, (asyncio.CancelledError, CancelledError)): + if failure is not None and is_cancelled_exception(failure): return _CloseStatus.CLOSE_STATUS_CANCELED return None diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index 8cdb0f74d3..c0f85b3814 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -349,8 +349,8 @@ async def close( """Write this handle's Records, then close the Stream. Readers get the Records before it, then a ``Closed`` item with ``result`` or - ``failure``. A ``CancelledError`` failure closes it as canceled. ``rpc_metadata`` and - ``rpc_timeout`` are as for :py:meth:`get_offset`. + ``failure``. A cancel failure, such as a ``CancelledError``, closes it as canceled. + ``rpc_metadata`` and ``rpc_timeout`` are as for :py:meth:`get_offset`. Raises: StreamNotOwnerError: A Workflow owns the Stream, so only its run chain closes it. diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index d6213fb682..01433aceff 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -438,8 +438,9 @@ async def close( writes to it again. On Redis the close is a regular Activity on the Workflow's task queue, which closes as the run chain, the Stream's owner. So every Worker on that queue registers the Redis backend. Readers get the Records before it, then a ``Closed`` item - with ``result`` or ``failure``. A ``CancelledError`` failure closes it as canceled. - Later writes in this run raise :py:class:`StreamClosedError`. + with ``result`` or ``failure``. A cancel failure, such as a ``CancelledError`` or an + ``ActivityError`` that a cancel caused, closes it as canceled. Later writes in this run + raise :py:class:`StreamClosedError`. Raises: StreamNotOwnerError: This run chain didn't create the Stream. @@ -512,13 +513,22 @@ def _closes_at_block_end(exc: BaseException | None) -> bool: info = temporalio.workflow.info() if info.cron_schedule: return False - if exc is None or isinstance(exc, asyncio.CancelledError): + if exc is None or isinstance(exc, asyncio.CancelledError) or _cancelled(exc): return True if not temporalio.workflow._Runtime.current().workflow_is_failure_exception(exc): return False return info.retry_policy is None +def _cancelled(exc: BaseException) -> bool: + # A cancelled run ends as canceled with no retry, also when the cancel reached the block as + # the error of an Activity, a Local Activity or a child. + return ( + temporalio.workflow.cancellation_reason() is not None + and temporalio.exceptions.is_cancelled_exception(exc) + ) + + def _own_handle(definition: type[E], backend: str | None) -> WorkflowStream[E]: info = temporalio.workflow.info() return WorkflowStream( diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 7ecefeb01a..8cb50aa42d 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -9,7 +9,7 @@ import pytest -from temporalio import workflow +from temporalio import activity, workflow from temporalio.api.common.v1 import Payload from temporalio.api.enums.v1 import EventType from temporalio.client import ( @@ -306,6 +306,62 @@ async def test_block_end_leaves_the_stream_open_for_a_retry_run(stream_client: C assert closes[0].failure is None +# Set by the Activity of a test's run once it runs, so the test cancels the run there. +waiting: dict[str, asyncio.Event] = {} + + +@activity.defn +async def waits_for_cancel() -> None: + waiting[activity.info().workflow_id].set() + while True: + activity.heartbeat() + await asyncio.sleep(0.1) + + +@workflow.defn +class CancelledInItsBlock: + @workflow.run + async def run(self, local: bool) -> None: + async with workflow.create_stream(Events) as stream: + await stream.topic(Events.tokens).publish("started") + if local: + await workflow.execute_local_activity( + waits_for_cancel, start_to_close_timeout=timedelta(minutes=1) + ) + else: + await workflow.execute_activity( + waits_for_cancel, + start_to_close_timeout=timedelta(minutes=1), + heartbeat_timeout=timedelta(seconds=1), + ) + + +async def test_cancel_during_an_activity_closes_as_canceled(stream_client: Client): + # The cancel reaches the block as an ActivityError, and no retry follows a cancelled run. + async with new_worker( + stream_client, CancelledInItsBlock, activities=[waits_for_cancel] + ) as worker: + for local, policy in ((False, RetryPolicy(maximum_attempts=3)), (True, None)): + workflow_id = f"wf-{uuid.uuid4()}" + waiting[workflow_id] = asyncio.Event() + handle = await stream_client.start_workflow( + CancelledInItsBlock.run, + local, + id=workflow_id, + task_queue=worker.task_queue, + retry_policy=policy, + ) + await asyncio.wait_for(waiting[workflow_id].wait(), 20) + await handle.cancel() + with pytest.raises(WorkflowFailureError): + await handle.result() + items = await read(handle) + assert data(items) == ["started"] + closed = items[-1] + assert isinstance(closed, streams.Closed), f"local={local}" + assert closed.status == streams.CloseStatus.CANCELED + + @workflow.defn class CronBlock: @workflow.run From 49595538b6630f92299a4ab423cb57543a312423 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:03:52 -0700 Subject: [PATCH 48/79] Settled a deferred block-end close from the run's real outcome. The block end decided from the exception and the retry policy alone, so it left Streams open when no retry run followed. --- temporalio/common/streams/_extension.py | 2 + temporalio/common/streams/_workflow.py | 96 +++++++++++++++---- temporalio/workflow/_streams.py | 6 +- tests/contrib/streams/test_workflow_module.py | 72 ++++++++++++++ 4 files changed, 156 insertions(+), 20 deletions(-) diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index 8815c4f61a..5e87ebea47 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -279,8 +279,10 @@ async def execute_workflow( result = await super().execute_workflow(input) except BaseException as err: if _ends_run(err): + streams_workflow.close_at_run_end(err) await streams_workflow.drain() raise + streams_workflow.close_at_run_end(None) await streams_workflow.drain() return result diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 01433aceff..241bbd4c39 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -103,6 +103,10 @@ class _RunState: waiting: dict[str, _Write] = field(default_factory=dict) # The Streams that a retry run's first write has withdrawn the previous run on. withdrawn: set[str] = field(default_factory=set) + # The block ends whose close waits for the run's outcome, with the block's failure. + deferred_closes: list[tuple[WorkflowStream[Any], BaseException | None]] = field( + default_factory=list + ) def stream(self, ref: models.StreamRef) -> _RunStream: known = self.streams.get(ref.id) @@ -354,7 +358,8 @@ class WorkflowStream(StreamHandle[E]): Pass it to an Activity, a child Workflow or the next run of continue-as-new as an argument. Use it in ``async with`` to close it at the end of the block. A run end closes - nothing, so a Stream that the run chain never closes stays open until its ``safety_ttl``. + only a Stream whose block end waited for it, so a Stream that the run chain never closes + stays open until its ``safety_ttl``. .. warning:: This API is experimental. @@ -457,7 +462,7 @@ def close_nowait( """ _write_nowait(self._close(result, failure)) - def _close(self, result: Any, failure: BaseException | None) -> asyncio.Future[int]: + def _closing(self) -> _RunStream: info = temporalio.workflow.info() if (self.workflow_id, self.first_execution_run_id) != ( info.workflow_id, @@ -469,7 +474,10 @@ def _close(self, result: Any, failure: BaseException | None) -> asyncio.Future[i f"Can't close the Stream {self.id}, since this run chain didn't create it", reason="stream_not_owner", ) - self._check("close").closed = True + return self._check("close") + + def _close(self, result: Any, failure: BaseException | None) -> asyncio.Future[int]: + self._closing().closed = True request = stream_pb.CloseStreamRequest(stream=self._ref()) if result is not None: request.result.CopyFrom(_payload(result)) @@ -494,30 +502,84 @@ async def __aexit__( ) -> None: """Close the Stream, with the exception as its failure if there is one. - A cancel closes it with a ``CancelledError`` failure. The block end leaves the Stream - open in a cron run, since the Stream spans the cron chain and the next run creates it - again. It also leaves it open on an exception that fails the run when the run has a - retry policy, since a retry run writes on to the same Stream, and on an exception that - fails only the Workflow Task, since the task's retry runs the block again. Another - ``BaseException``, such as an eviction, leaves it open, since the block did not end in - Workflow code. An explicit :py:meth:`close` always closes. + A cancel of the block or of the run closes it as canceled. An exception that fails the + run with no retry run to follow closes it too, such as one in a run with no retry + policy, a non-retryable ``ApplicationError``, or one on the last attempt. The close of + any other exception waits for the run's end. The Stream then closes with that exception + when the run returns or continues as new, or fails with no retry run to follow. It + stays open when a retry run follows, since that run writes on to the same Stream, and + when the exception fails only the Workflow Task, since the task's retry runs the block + again. In a cron run the block end leaves the Stream open, since the Stream spans the + cron chain and the next run creates it again. An eviction cancels the block, and the + Worker sends no command of an evicted run. An explicit :py:meth:`close` always closes. """ - if _state().stream(self._model_ref()).closed: + state = _state() + if state.stream(self._model_ref()).closed: return close, failure = exit_failure(exc) - if close and _closes_at_block_end(exc): + if not close or temporalio.workflow.info().cron_schedule: + return + if _closes_at_block_end(exc): await self.close(failure=failure) + else: + # A refused close raises here, as it does for a close at the block end. + self._closing() + state.deferred_closes.append((self, failure)) def _closes_at_block_end(exc: BaseException | None) -> bool: - info = temporalio.workflow.info() - if info.cron_schedule: - return False + # Else only the run's end tells whether a retry run writes on to the Stream. if exc is None or isinstance(exc, asyncio.CancelledError) or _cancelled(exc): return True - if not temporalio.workflow._Runtime.current().workflow_is_failure_exception(exc): + runtime = temporalio.workflow._Runtime.current() + return runtime.workflow_is_failure_exception(exc) and not _retried(exc) + + +def close_at_run_end(err: BaseException | None) -> None: + """Close the Streams whose block end waited for the run's outcome, unless a retry run + follows. ``err`` is the exception that ends the run, or None when the run returns. + """ + state = _state() + runtime = temporalio.workflow._Runtime.current() + if ( + isinstance(err, Exception) + and not _cancelled(err) + and runtime.workflow_is_failure_exception(err) + and _retried(err) + ): + return + for stream, failure in state.deferred_closes: + if not state.stream(stream._model_ref()).closed: + _write_nowait(stream._close(None, failure)) + + +def _retried(exc: BaseException) -> bool: + # Whether the server starts a retry run once the run fails with exc, by the server's rule. + info = temporalio.workflow.info() + policy = info.retry_policy + if policy is None or 0 < policy.maximum_attempts <= info.attempt: return False - return info.retry_policy is None + failure_type: str | None + if isinstance(exc, temporalio.exceptions.ApplicationError): + if exc.non_retryable: + return False + failure_type = exc.type + elif isinstance( + exc, + (temporalio.exceptions.CancelledError, temporalio.exceptions.TerminatedError), + ): + return False + elif isinstance(exc, temporalio.exceptions.TimeoutError): + return exc.type in ( + temporalio.exceptions.TimeoutType.START_TO_CLOSE, + temporalio.exceptions.TimeoutType.HEARTBEAT, + ) + elif isinstance(exc, temporalio.exceptions.FailureError): + return True + else: + # The run's failure carries any other exception as the type of its class name. + failure_type = type(exc).__name__ + return failure_type not in (policy.non_retryable_error_types or ()) def _cancelled(exc: BaseException) -> bool: diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index dacdedae59..c0f2980d09 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -36,9 +36,9 @@ def create_stream( async with workflow.create_stream(streams.with_name(Events, "turn-3")) as turn: await turn.topic(Events.tokens).publish("hi") - The block end leaves the Stream open in a cron run, on an exception that fails the run when - the run has a retry policy, and on an exception that fails only the Workflow Task, since a - later run or the task's retry writes on to the same Stream. See + An exception closes the Stream with that exception as the failure. When a retry run may + follow, the close waits for the run's end, and the Stream stays open only for a retry run + or for the retry of a Workflow Task that failed. A cron run leaves the Stream open. See :py:meth:`temporalio.common.streams.WorkflowStream.__aexit__`. .. warning:: diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 8cb50aa42d..f7b46af991 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -306,6 +306,78 @@ async def test_block_end_leaves_the_stream_open_for_a_retry_run(stream_client: C assert closes[0].failure is None +@workflow.defn +class EndsItsBlock: + @workflow.run + async def run(self, mode: str) -> str: + try: + async with workflow.create_stream(Events) as stream: + await stream.topic(Events.tokens).publish(mode) + if mode == "non_retryable": + raise ApplicationError("block failed", non_retryable=True) + if mode == "listed_type": + raise ApplicationError("block failed", type="Listed") + if mode == "caught_task_failure": + raise ValueError("block failed") + raise ApplicationError("block failed") + except (ApplicationError, ValueError) as err: + if mode.startswith("caught"): + return "caught" + if mode == "then_non_retryable": + raise ApplicationError("run failed", non_retryable=True) from err + raise + + +async def test_block_end_closes_when_no_retry_run_follows(stream_client: Client): + # Each run ends with no retry run, so each block end closes its Stream, at once or at the + # run's end, with the block's exception. + retries = RetryPolicy( + initial_interval=timedelta(milliseconds=100), maximum_attempts=3 + ) + cases = { + "non_retryable": retries, + "last_attempt": RetryPolicy(maximum_attempts=1), + "listed_type": dataclasses.replace( + retries, non_retryable_error_types=["Listed"] + ), + "caught": retries, + "caught_task_failure": None, + "then_non_retryable": retries, + } + async with new_worker(stream_client, EndsItsBlock) as worker: + handles = { + mode: await stream_client.start_workflow( + EndsItsBlock.run, + mode, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + retry_policy=policy, + ) + for mode, policy in cases.items() + } + for mode, handle in handles.items(): + if mode.startswith("caught"): + assert await handle.result() == "caught" + else: + with pytest.raises(WorkflowFailureError): + await handle.result() + items = await read(handle) + assert data(items) == [mode] + closed = items[-1] + assert isinstance(closed, streams.Closed), mode + assert closed.status == streams.CloseStatus.FAILED + assert ( + closed.failure is not None and closed.failure.message == "block failed" + ) + # A close at the run's end is a command after the run's code, also on replay. + replayer = Replayer( + workflows=[EndsItsBlock], + stream_backends=stream_client.config()["stream_backends"], + ) + for mode in ("caught", "then_non_retryable"): + await replayer.replay_workflow(await handles[mode].fetch_history()) + + # Set by the Activity of a test's run once it runs, so the test cancels the run there. waiting: dict[str, asyncio.Event] = {} From 76e918c089552d06e74ee39013376173cf64544d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:04:32 -0700 Subject: [PATCH 49/79] Kept a no-wait read going while an empty page moves on. A Store scans a limited number of entries per call, so a filtered read ended early after 10,000 entries of other Topics. --- temporalio/common/streams/_reader.py | 9 +++++++-- tests/contrib/streams/test_stream_client.py | 11 +++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/temporalio/common/streams/_reader.py b/temporalio/common/streams/_reader.py index af2b526339..87919a2d9d 100644 --- a/temporalio/common/streams/_reader.py +++ b/temporalio/common/streams/_reader.py @@ -187,7 +187,7 @@ async def _pages( wait_new_records: bool, ) -> AsyncGenerator[models.ReadRecordsResponse, None]: # A read that waits long-polls each page. A read with no wait reads pages until one has no - # Record. + # Record and moves no further. if request.last: # The newest Records come in one page that never waits. A read that follows goes on # after them. @@ -206,7 +206,12 @@ async def _pages( yield page if page.done: return - if not wait_new_records and (not page.records or page.next_offset is None): + if not wait_new_records and ( + page.next_offset is None + # A Store scans a limited number of entries per call, so an empty page that moved + # on stopped in Records that a Topic filter left out, not at the newest Record. + or (not page.records and page.next_offset <= (request.from_offset or 0)) + ): return if page.next_offset is not None: request = dataclasses.replace(request, from_offset=page.next_offset) diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 611e436e5e..34330a53ab 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -288,6 +288,17 @@ async def test_next_offset_resumes_after_the_last_item(stream_client: Client): ] == ["b"] +async def test_no_wait_read_goes_past_records_of_other_topics(stream_client: Client): + # A Store scans at most 10,000 entries per call, so the first pages of this read are empty. + stream = await redis.create(stream_client, Events, id=f"scan-{uuid.uuid4()}") + for _ in range(11): + await stream.topic(Events.tokens).publish(*["x"] * 1000) + turn = await stream.topic(Events.turns).publish(Turn("model")) + turns = stream.subscribe(Events.turns, wait_new_records=False) + assert [item.data async for item in turns] == [Turn("model")] + assert turns.next_offset == turn + 1 + + async def test_last_reads_the_newest_records_then_follows(stream_client: Client): stream = await redis.create(stream_client, Events, id=f"last-{uuid.uuid4()}") tokens = stream.topic(Events.tokens) From 3485b975e15ccca3b9f5b1a0586314c2efb22d09 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:05:15 -0700 Subject: [PATCH 50/79] Reported a NondeterminismError of the codec step as nondeterminism. A native read's replay checks its page in the codec step, outside the Workflow. The task carries the nondeterminism cause, and the Replayer reports a NondeterminismError. --- temporalio/worker/_workflow.py | 40 +++++++++++ tests/contrib/streams/test_native_server.py | 75 +++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index f50955c4cf..9d1722ffb4 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -25,6 +25,7 @@ import temporalio.converter._extstore import temporalio.exceptions import temporalio.workflow +from temporalio.api.enums.v1 import WorkflowTaskFailedCause from temporalio.bridge.worker import PollShutdownError from temporalio.converter import StorageDriverStoreContext, StorageDriverWorkflowInfo from temporalio.worker.workflow_sandbox._runner import SandboxedWorkflowRunner @@ -54,6 +55,15 @@ # Set to true to log all activations and completions LOG_PROTOS = False +_NONDETERMINISM = ( + WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_NON_DETERMINISTIC_ERROR +) +_EvictionReason = ( + temporalio.bridge.proto.workflow_activation.RemoveFromCache.EvictionReason +) +_LANG_FAIL = _EvictionReason.LANG_FAIL +_NONDETERMINISM_EVICTION = _EvictionReason.NONDETERMINISM + # Value was chosen abitrarily as a small number that allows some concurrency and prevents # large numbers of concurrent external storage operations causing resource contention. @@ -174,6 +184,8 @@ def __init__( max_workflow_task_external_storage_concurrency ) self._throw_after_activation: Exception | None = None + # The runs whose last Workflow Task a NondeterminismError of the codec step failed. + self._nondeterministic_runs: set[str] = set() # If debug mode is enabled, disable deadlock detection # otherwise set to 2 seconds @@ -465,6 +477,25 @@ async def _handle_activation( ) else: completion.failed.failure.SetInParent() + if isinstance(err, temporalio.workflow.NondeterminismError): + # A codec step may compare the data with what the run saw, such as a + # native Stream read. + self._nondeterministic_runs.add(act.run_id) + workflow_type = ( + workflow.get_info().workflow_type + if workflow + else init_job.workflow_type + if init_job + else "" + ) + # Core fails a run for this cause in the middle of the replay, and again at + # the next marker, so only a run that fails the task takes it. + if not ( + self.nondeterminism_as_workflow_fail() + or workflow_type + in self.nondeterminism_as_workflow_fail_for_types() + ): + completion.failed.force_cause = _NONDETERMINISM try: data_converter.failure_converter.to_failure( err, @@ -647,6 +678,15 @@ async def _handle_cache_eviction( act.run_id, ) + if act.run_id in self._nondeterministic_runs: + self._nondeterministic_runs.discard(act.run_id) + if job.reason == _LANG_FAIL: + # Core evicts a run that lang failed as a lang failure, so a hook such as the + # Replayer's would not see the nondeterminism. + job = temporalio.bridge.proto.workflow_activation.RemoveFromCache( + message=job.message, reason=_NONDETERMINISM_EVICTION + ) + # Run eviction hook if present if self._on_eviction_hook is not None: try: diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index 5141e1b59f..41f80e72c6 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -13,8 +13,11 @@ from datetime import timedelta from typing import Any, cast +import pytest + import temporalio.api.streamservice.v1 as stream_pb from temporalio import activity, workflow +from temporalio.api.enums.v1 import WorkflowTaskFailedCause from temporalio.client import Client, WorkflowHandle from temporalio.common import RetryPolicy, streams from temporalio.common.streams import _native, _plugin @@ -488,6 +491,78 @@ async def test_a_workflow_reads_a_standalone_activitys_stream(native_client: Cli assert _native._describe_activity_name("native") not in local_activities +@workflow.defn +class CountsThenWaits: + def __init__(self) -> None: + self._go = False + + @workflow.signal + def finish(self) -> None: + self._go = True + + @workflow.run + async def run(self, stream: streams.WorkflowStream[Events]) -> int: + count = len([item async for item in stream.subscribe(wait_new_records=False)]) + await workflow.wait_condition(lambda: self._go) + return count + + +async def until_marker(handle: WorkflowHandle[Any, Any]) -> None: + for _ in range(100): + history = await handle.fetch_history() + if any( + event.HasField("marker_recorded_event_attributes") + for event in history.events + ): + return + await asyncio.sleep(0.1) + raise AssertionError("The reader never read") + + +async def until_task_failure(handle: WorkflowHandle[Any, Any]) -> Any: + for _ in range(100): + history = await handle.fetch_history() + for event in history.events: + if event.HasField("workflow_task_failed_event_attributes"): + return event.workflow_task_failed_event_attributes + await asyncio.sleep(0.1) + raise AssertionError("No Workflow Task failed") + + +async def test_a_replay_that_reads_other_records_is_nondeterministic( + native_client: Client, monkeypatch: pytest.MonkeyPatch +): + # The codec step checks a replayed page against its digest, outside the Workflow. The + # server and the Replayer still learn that the replay was nondeterministic. + stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") + await stream.topic(Events.tokens).publish("a", "b") + async with new_worker( + native_client, CountsThenWaits, max_cached_workflows=0 + ) as worker: + handle = await native_client.start_workflow( + CountsThenWaits.run, + streams.WorkflowStream.from_handle(stream), + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await until_marker(handle) + monkeypatch.setattr(_native, "_digest", lambda records: b"other") + _native._pages.clear() + await handle.signal(CountsThenWaits.finish) + failed = await until_task_failure(handle) + await handle.terminate() + assert ( + failed.cause + == WorkflowTaskFailedCause.WORKFLOW_TASK_FAILED_CAUSE_NON_DETERMINISTIC_ERROR + ) + assert "found other Records" in failed.failure.message + replayer = Replayer(workflows=[CountsThenWaits], stream_client=native_client) + result = await replayer.replay_workflow( + await handle.fetch_history(), raise_on_replay_failure=False + ) + assert isinstance(result.replay_failure, workflow.NondeterminismError) + + async def test_each_client_of_a_native_backend_calls_on_its_own_connection(): # One backend on two clients gives each its own runtime, so a call doesn't go on the # connection of the client that came last. From 8382d328ec509f6269f2bea3be9658cfe679496f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:05:32 -0700 Subject: [PATCH 51/79] Let an Activity retry go on with a closed or expired Stream argument. The open of a new attempt withdraws earlier Records, and a Stream that takes no write failed the attempt for good before user code ran. --- temporalio/common/streams/_writer.py | 9 ++- .../contrib/streams/test_activity_handles.py | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/temporalio/common/streams/_writer.py b/temporalio/common/streams/_writer.py index 670817997b..1263722342 100644 --- a/temporalio/common/streams/_writer.py +++ b/temporalio/common/streams/_writer.py @@ -51,11 +51,16 @@ def producer(self) -> stream_pb.Producer: async def withdraw_earlier_attempts(self) -> None: """Open the writer of a new attempt, which withdraws the Records of the earlier ones. - A Stream that does not exist has nothing to withdraw. + A Stream that does not exist has nothing to withdraw. A closed or expired Stream takes + no write, so the attempt goes on, and a later write raises the real error. """ try: await self.open() - except _errors.StreamNotFoundError: + except ( + _errors.StreamNotFoundError, + _errors.StreamClosedError, + _errors.StreamExpiredError, + ): pass async def open(self) -> writer_client.Writer: diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index a288f4ddaa..ce029e960b 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -15,6 +15,7 @@ from temporalio.client import Client, WorkflowHandle from temporalio.common import RetryPolicy, streams from temporalio.common.streams import _activity +from temporalio.contrib import stream as redis from temporalio.exceptions import CancelledError from tests.helpers import new_worker @@ -180,6 +181,65 @@ async def test_retry_withdraws_a_received_stream(stream_client: Client): assert withdrawn == ["attempt-1"] +@dataclass +class FinishedStreams: + key: str + closed: streams.ActivityStream[Events] + expired: streams.ActivityStream[Events] + + +@activity.defn +async def reads_after_a_failed_attempt(input: FinishedStreams) -> list[Any]: + attempts[input.key] = attempts.get(input.key, 0) + 1 + if attempts[input.key] == 1: + raise RuntimeError("first attempt fails") + items = input.closed.subscribe(Events.tokens, wait_new_records=False) + return [item.data async for item in items if isinstance(item, streams.Record)] + + +@workflow.defn +class ReadsFinishedStreams: + @workflow.run + async def run(self, key: str, expired: streams.WorkflowStream[Events]) -> list[Any]: + stream = await workflow.create_stream(Events) + await stream.topic(Events.tokens).publish("turn") + await stream.close() + return await workflow.execute_activity( + reads_after_a_failed_attempt, + FinishedStreams( + key, + streams.ActivityStream.from_handle(stream), + streams.ActivityStream.from_handle(expired), + ), + start_to_close_timeout=timedelta(seconds=30), + retry_policy=RetryPolicy(initial_interval=timedelta(milliseconds=10)), + ) + + +async def test_retry_goes_on_with_a_closed_or_expired_stream(stream_client: Client): + # The second attempt's open withdraws nothing on a Stream that takes no write, and the + # attempt only reads. + expired = await redis.create( + stream_client, + Events, + id=f"expired-{uuid.uuid4()}", + retention=timedelta(milliseconds=200), + ) + await expired.topic(Events.tokens).publish("old") + await asyncio.sleep(0.5) + key = str(uuid.uuid4()) + async with new_worker( + stream_client, ReadsFinishedStreams, activities=[reads_after_a_failed_attempt] + ) as worker: + result = await stream_client.execute_workflow( + ReadsFinishedStreams.run, + args=[key, streams.WorkflowStream.from_handle(expired)], + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert result == ["turn"] and attempts[key] == 2 + + # Set by a test once its readers showed the first attempt's Record. shown: dict[str, asyncio.Event] = {} From 802b7a47b4e96fd9a9348dad61585fca1366a249 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:06:30 -0700 Subject: [PATCH 52/79] Decoded a Closed item's failure with the reader's converter. The read path converts Record bodies later, so the failure kept RawValue details and an encoded message. --- temporalio/common/streams/_convert.py | 17 +++++++++++++ temporalio/common/streams/_reader.py | 7 +++++- tests/contrib/streams/test_stream_client.py | 27 ++++++++++++++++++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/temporalio/common/streams/_convert.py b/temporalio/common/streams/_convert.py index ba24d64902..47261f9b0d 100644 --- a/temporalio/common/streams/_convert.py +++ b/temporalio/common/streams/_convert.py @@ -9,6 +9,7 @@ import temporalio.api.common.v1 import temporalio.common import temporalio.converter +import temporalio.exceptions class _RawPayloadConverter(temporalio.converter.PayloadConverter): @@ -39,6 +40,22 @@ def raw( return dataclasses.replace(converter, payload_converter_class=_RawPayloadConverter) +def failure( + converter: temporalio.converter.DataConverter, failure: BaseException +) -> BaseException: + """Convert a close failure that a read gave with RawValue payloads, after the codec, with + the user's payload converter. + """ + if ( + not isinstance(failure, temporalio.exceptions.FailureError) + or not failure.failure + ): + return failure + return converter.failure_converter.from_failure( + failure.failure, converter.payload_converter + ) + + def decode( converter: temporalio.converter.DataConverter, body: Any, type_hint: Any ) -> Any: diff --git a/temporalio/common/streams/_reader.py b/temporalio/common/streams/_reader.py index 87919a2d9d..283fe0b65c 100644 --- a/temporalio/common/streams/_reader.py +++ b/temporalio/common/streams/_reader.py @@ -241,6 +241,11 @@ async def _item( CloseStatus.FAILED if failure is not None else CloseStatus.COMPLETED ) return Closed( - offset=offset, result=result, failure=failure, status=CloseStatus(status) + offset=offset, + result=result, + failure=_convert.failure(converter, failure) + if failure is not None + else None, + status=CloseStatus(status), ) return None diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 34330a53ab..296cc4f6ff 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -1,4 +1,5 @@ import asyncio +import dataclasses import uuid from dataclasses import dataclass from datetime import timedelta @@ -14,7 +15,8 @@ from temporalio.contrib import stream as redis from temporalio.contrib.stream import RedisStreamBackend from temporalio.contrib.stream.testing import MemoryStreamBackend -from temporalio.converter import DataConverter +from temporalio.converter import DataConverter, DefaultFailureConverter +from temporalio.exceptions import ApplicationError from temporalio.plugin import SimplePlugin from temporalio.testing import WorkflowEnvironment from temporalio.worker import Replayer @@ -288,6 +290,29 @@ async def test_next_offset_resumes_after_the_last_item(stream_client: Client): ] == ["b"] +class _EncodesCommonAttributes(DefaultFailureConverter): + def __init__(self) -> None: + super().__init__(encode_common_attributes=True) + + +async def test_closed_failure_decodes_with_the_clients_converter(stream_client: Client): + # The message goes into the encoded attributes, which only the payload converter reads. + config = stream_client.config() + config["data_converter"] = dataclasses.replace( + DataConverter.default, failure_converter_class=_EncodesCommonAttributes + ) + client = Client(**config) + stream = await redis.create(client, Events, id=f"failed-{uuid.uuid4()}") + await stream.close( + failure=ApplicationError("tool failed", "detail", {"k": 1}, type="Tool") + ) + [closed] = [item async for item in stream.subscribe()] + assert isinstance(closed, streams.Closed) + assert isinstance(closed.failure, ApplicationError) + assert closed.failure.message == "tool failed" and closed.failure.type == "Tool" + assert closed.failure.details == ("detail", {"k": 1}) + + async def test_no_wait_read_goes_past_records_of_other_topics(stream_client: Client): # A Store scans at most 10,000 entries per call, so the first pages of this read are empty. stream = await redis.create(stream_client, Events, id=f"scan-{uuid.uuid4()}") From 38852775384fe588cbc07accbfdc6f24d09b05db Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:06:57 -0700 Subject: [PATCH 53/79] Wrote a transfer type dataclass in a top-level list as its fields. The decode of a dataclass inside a JSON value reads its fields, so the encoder's transfer type broke the round trip. --- temporalio/converter/_payload_converter.py | 6 +++--- tests/test_converter.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/temporalio/converter/_payload_converter.py b/temporalio/converter/_payload_converter.py index 3540d743cb..35ce411207 100644 --- a/temporalio/converter/_payload_converter.py +++ b/temporalio/converter/_payload_converter.py @@ -689,9 +689,9 @@ def default(self, o: Any) -> Any: # Datetime support if isinstance(o, datetime): return o.isoformat() - # A transfer type value inside another value. A dataclass never gets here, since - # dataclasses.asdict has already turned it into a dict. - converter = _get_transfer_type_converter(type(o)) + # A transfer type value inside another value. A dataclass there is written as its + # fields, such as in a top-level list, since value_to_type decodes it from them. + converter = _nested_transfer_type_converter(type(o)) if converter is not None: value = converter.to_transfer_type(o) if isinstance(value, google.protobuf.message.Message): diff --git a/tests/test_converter.py b/tests/test_converter.py index d21d9193ab..97618e2904 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -512,6 +512,18 @@ def test_transfer_type_values_nested_in_json(): assert decoded.proto == TemporalTransferTypeValueWithoutHint("c") +def test_transfer_type_dataclass_in_a_top_level_list_keeps_its_fields(): + # The decode of a dataclass inside a JSON value reads its fields, so the encode writes them. + converter = DataConverter.default.payload_converter + value = [TemporalTransferTypeValueWithoutHint("c")] + payload = converter.to_payload(value) + assert json.loads(payload.data) == [{"value": "c"}] + decoded = converter.from_payload( + payload, list[TemporalTransferTypeValueWithoutHint] + ) + assert decoded == value + + def test_transfer_type_convertible_rejects_existing_converter(): with pytest.raises(TypeError, match="already has a transfer type converter"): transfer_type_convertible(TemporalTransferTypeValueConverter)( From 78359143cd4dba506b250b701c78019eece770a8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:09:00 -0700 Subject: [PATCH 54/79] Failed the Workflow Task when the Stream backend lacks a capability. Such as a native server with direct Nexus calls off. It is setup, not History, so a Worker or server with the capability can retry the task. --- temporalio/common/streams/_errors.py | 20 +++++++++- temporalio/common/streams/_workflow.py | 10 ++++- temporalio/common/streams/_workflow_reader.py | 2 +- tests/contrib/streams/test_workflow_reads.py | 40 ++++++++++++++++++- 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/temporalio/common/streams/_errors.py b/temporalio/common/streams/_errors.py index 1ca99b7f4d..79111ab864 100644 --- a/temporalio/common/streams/_errors.py +++ b/temporalio/common/streams/_errors.py @@ -77,7 +77,10 @@ class StreamOffsetError(StreamError): class StreamUnsupportedError(StreamError): - """The Store lacks a capability that the call needs. The message ends with its name.""" + """The Store lacks a capability that the call needs. The message ends with its name. + + Workflow code gets :py:class:`StreamConfigError` for it, since it is setup. + """ class StreamNotOwnerError(StreamError): @@ -100,7 +103,9 @@ class StreamSeqDivergentError(StreamError): class StreamConfigError(RuntimeError): """No Stream backend, or no backend of a given name. It is config, not a Store error. - ``except StreamError`` does not catch it. It fails a Workflow Task, and an Activity retries. + Workflow code also gets it for a backend that lacks a capability, such as a native server + with ``frontend.enableDirectNexusCalls`` off. ``except StreamError`` does not catch it. It + fails a Workflow Task, and an Activity retries. """ @@ -190,6 +195,17 @@ def from_activity_error(err: Exception) -> Exception: return err +def in_workflow(err: BaseException) -> BaseException: + """The error that Workflow code raises for the Stream error of a failed call. + + A missing capability is not in History, so it fails the Workflow Task, and a Worker or a + server with that capability retries it. + """ + if isinstance(err, StreamUnsupportedError): + return StreamConfigError(err.message) + return err + + def from_rpc_error(err: temporalio.service.RPCError) -> Exception: """The Stream error for a server failure with Stream error details, else ``err``.""" for detail in err.grpc_status.details: diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 79463f015c..65ce5d9704 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -277,7 +277,9 @@ async def run() -> None: ) as err: for _, call in write.calls: if not call.done(): - call.set_exception(_errors.from_activity_error(err)) + call.set_exception( + _errors.in_workflow(_errors.from_activity_error(err)) + ) return except BaseException: for _, call in write.calls: @@ -288,7 +290,11 @@ async def run() -> None: continue if isinstance(answer, dict): call.set_exception( - _errors.from_reason(str(answer["reason"]), str(answer["message"])) + _errors.in_workflow( + _errors.from_reason( + str(answer["reason"]), str(answer["message"]) + ) + ) ) else: call.set_result(int(answer)) diff --git a/temporalio/common/streams/_workflow_reader.py b/temporalio/common/streams/_workflow_reader.py index ccd7614387..e87bcebbd6 100644 --- a/temporalio/common/streams/_workflow_reader.py +++ b/temporalio/common/streams/_workflow_reader.py @@ -454,6 +454,6 @@ def error_of(err: BaseException) -> BaseException: if isinstance(cause, temporalio.exceptions.ApplicationError): stream_error = _errors.from_application_error(cause) if stream_error is not None: - return stream_error + return _errors.in_workflow(stream_error) cause = cause.__cause__ return err diff --git a/tests/contrib/streams/test_workflow_reads.py b/tests/contrib/streams/test_workflow_reads.py index 6eeb6e0a84..2cd569617c 100644 --- a/tests/contrib/streams/test_workflow_reads.py +++ b/tests/contrib/streams/test_workflow_reads.py @@ -12,7 +12,7 @@ from temporalio import workflow from temporalio.client import Client, WorkflowFailureError from temporalio.common import streams -from temporalio.common.streams import _errors, _workflow_reader +from temporalio.common.streams import _backend, _errors, _workflow_reader from temporalio.common.streams._native import _DirectReadCodec from temporalio.converter import DataConverter from temporalio.exceptions import ApplicationError @@ -303,6 +303,44 @@ async def test_a_workflow_page_stays_below_the_marker_limit(stream_client: Clien ) +async def test_a_backend_that_lacks_a_capability_fails_the_task( + stream_client: Client, monkeypatch: pytest.MonkeyPatch +): + # Such as a native server with direct Nexus calls off. A Worker or a server with the + # capability can retry the task, while a failed run stays failed. + stream = await stream_client.create_stream(Events, id=f"cap-{uuid.uuid4()}") + await stream.topic(Events.tokens).publish("a") + call_async = _backend._ExtensionCalls.call_async + refused: list[str] = [] + + async def refuse_a_read(self: Any, method: str, request: Any, response: Any) -> Any: + if method == "WorkflowReadRecords" and not refused: + refused.append(method) + raise streams.StreamUnsupportedError( + "Direct Nexus calls are disabled", reason="stream_unsupported" + ) + return await call_async(self, method, request, response) + + monkeypatch.setattr(_backend._ExtensionCalls, "call_async", refuse_a_read) + async with new_worker(stream_client, CountsTokens) as worker: + handle = await stream_client.start_workflow( + CountsTokens.run, + streams.WorkflowStream.from_handle(stream), + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await handle.result() == 1 + failures = [ + event.workflow_task_failed_event_attributes.failure + async for event in handle.fetch_history_events() + if event.HasField("workflow_task_failed_event_attributes") + ] + assert [failure.application_failure_info.type for failure in failures] == [ + "StreamConfigError" + ] + assert failures[0].message == "Direct Nexus calls are disabled" + + def test_a_server_failure_maps_by_its_reason(): by_type = _errors.from_application_error( ApplicationError("closed", type="stream_closed") From 4aec6df69874785c2b9614e0ad79be008c14b6a0 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:14:27 -0700 Subject: [PATCH 55/79] Waited at a run's end until the server has each wake's cancel. The server drops the cancel of a closed run, so a wake that the run's last task cancelled kept its parked read and a slot of stream.maxParkedReads. --- temporalio/common/streams/_extension.py | 2 +- temporalio/common/streams/_native.py | 7 +- temporalio/common/streams/_workflow_reader.py | 32 +++++--- tests/contrib/streams/test_native_server.py | 76 ++++++++++++++++++- 4 files changed, 101 insertions(+), 16 deletions(-) diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index f5b5be316e..3774d8b25e 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -456,7 +456,7 @@ def _receive(args: Sequence[Any]) -> None: async def _end_run() -> None: # The run's reads end with it, so the server drops their parked reads. Its writes that # nobody awaits go out before its end. - _workflow_reader.cancel_wakes() + await _workflow_reader.cancel_wakes() await streams_workflow.drain() diff --git a/temporalio/common/streams/_native.py b/temporalio/common/streams/_native.py index 2165905f00..20bdb6f1e5 100644 --- a/temporalio/common/streams/_native.py +++ b/temporalio/common/streams/_native.py @@ -38,8 +38,9 @@ _SERVICE = "temporal.api.streamservice.v1.StreamService" -# Nothing waits for the answer of a wake that the reader dropped, so its cancel ends it at once. -_TRY_CANCEL = temporalio.workflow.NexusOperationCancellationType.TRY_CANCEL +# A wake's cancel ends it once the server has the cancel. The server drops the cancel of a run +# that closed, so the run's end waits for it. +_WAIT_REQUESTED = temporalio.workflow.NexusOperationCancellationType.WAIT_REQUESTED # A close goes on when the run is cancelled, since the cancel may be what the close records. _ABANDON = temporalio.workflow.NexusOperationCancellationType.ABANDON # How long a wake parks on the server. A parked wake adds nothing to History, so an idle reader @@ -273,7 +274,7 @@ async def _temporal_stream_start_wake( schedule_to_close_timeout=None, schedule_to_start_timeout=None, start_to_close_timeout=None, - cancellation_type=_TRY_CANCEL, + cancellation_type=_WAIT_REQUESTED, headers=None, summary=None, ) diff --git a/temporalio/common/streams/_workflow_reader.py b/temporalio/common/streams/_workflow_reader.py index e87bcebbd6..dba398b280 100644 --- a/temporalio/common/streams/_workflow_reader.py +++ b/temporalio/common/streams/_workflow_reader.py @@ -7,6 +7,7 @@ from __future__ import annotations +import asyncio from collections import deque from collections.abc import AsyncGenerator, Mapping, Sequence from dataclasses import dataclass, field @@ -60,8 +61,8 @@ def reader_of(backend: str) -> Any: @dataclass class _ReadState: - # The run's wakes that wait on the server, so the run's end cancels them. - wakes: set[Any] = field(default_factory=set) + # The run's wakes that wait on the server, with their reads, so the run's end cancels them. + wakes: dict[Any, WorkflowItems] = field(default_factory=dict) # The lowest offset that the run's replayed pages read, per Stream, and whether a live # page sent it. replay_floors: dict[tuple[str, str], int] = field(default_factory=dict) @@ -77,10 +78,22 @@ def _read_state() -> _ReadState: return state -def cancel_wakes() -> None: - """Cancel the run's wakes at the end of the run, so the server drops their parked reads.""" - for wake in list(_read_state().wakes): +async def cancel_wakes() -> None: + """Cancel the run's wakes at the end of the run, and wait until the server has each cancel. + + The server drops the cancel of a run that closed, and a parked read holds its slot until + the next append or its wait ends. + """ + wakes = list(_read_state().wakes.items()) + for wake, items in wakes: + # The read ends with no error, as when its code closes it. + items._closed = True wake.cancel() + for wake, _ in wakes: + try: + await wake + except (Exception, asyncio.CancelledError): + pass class WorkflowItems(Subscription[Any]): @@ -347,13 +360,12 @@ async def _wake_once(self, request: stream_pb.ReadRecordsRequest) -> None: # The server parks the wake until a Record passes the filter, the Stream closes, or # the wait ends. Its answer holds no Records, and the next page reads them. wake = await self._reader._temporal_stream_start_wake(request) + wakes = _read_state().wakes + wakes[wake] = self + self._wake = wake if self._closed: # The iterator closed while the server took the wake. wake.cancel() - return - wakes = _read_state().wakes - self._wake = wake - wakes.add(wake) try: answer = await wake except temporalio.exceptions.NexusOperationError as err: @@ -366,7 +378,7 @@ async def _wake_once(self, request: stream_pb.ReadRecordsRequest) -> None: raise error_of(err) from None finally: self._wake = None - wakes.discard(wake) + wakes.pop(wake, None) if ( self._request.start == stream_pb.ReadStart.READ_START_END and answer.next_offset diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index 41f80e72c6..1fe9c0f834 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -18,7 +18,7 @@ import temporalio.api.streamservice.v1 as stream_pb from temporalio import activity, workflow from temporalio.api.enums.v1 import WorkflowTaskFailedCause -from temporalio.client import Client, WorkflowHandle +from temporalio.client import Client, WorkflowExecutionStatus, WorkflowHandle from temporalio.common import RetryPolicy, streams from temporalio.common.streams import _native, _plugin from temporalio.common.streams._native import POSITIONS_TYPE @@ -31,7 +31,7 @@ ReadInput, Writer, ) -from tests.helpers import new_worker +from tests.helpers import assert_eventually, new_worker @activity.defn @@ -326,6 +326,78 @@ async def test_closing_a_native_read_cancels_its_wake(native_client: Client): ) +@workflow.defn +class EndsWhileItReads: + def __init__(self) -> None: + self._end = False + + @workflow.signal + def end(self) -> None: + self._end = True + + @workflow.run + async def run(self, stream: streams.WorkflowStream[Events], first: bool) -> str: + async def read() -> None: + async for _ in stream.subscribe(offset=streams.END): + pass + + asyncio.create_task(read()) + await workflow.wait_condition(lambda: self._end) + if first: + workflow.continue_as_new(args=[stream, False]) + return "done" + + +async def until_parked(handle: WorkflowHandle[Any, Any]) -> None: + for _ in range(100): + history = await handle.fetch_history() + if any( + event.HasField("nexus_operation_started_event_attributes") + for event in history.events + ): + return + await asyncio.sleep(0.1) + raise AssertionError("The reader never parked") + + +async def test_a_run_end_waits_until_the_server_has_its_wakes_cancels( + native_client: Client, +): + # The server drops the cancel of a run that closed, and its parked read would keep a slot + # of the Stream's stream.maxParkedReads until the next append, for up to 7 days. + stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") + async with new_worker(native_client, EndsWhileItReads) as worker: + first = await native_client.start_workflow( + EndsWhileItReads.run, + args=[streams.WorkflowStream.from_handle(stream), True], + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await until_parked(first) + await first.signal(EndsWhileItReads.end) + + async def continued() -> None: + description = await native_client.get_workflow_handle( + first.id, run_id=first.first_execution_run_id + ).describe() + assert description.status == WorkflowExecutionStatus.CONTINUED_AS_NEW + + await assert_eventually(continued) + latest = native_client.get_workflow_handle(first.id) + await until_parked(latest) + await latest.signal(EndsWhileItReads.end) + assert await latest.result() == "done" + # Nothing comes after a run's close, so the server took each cancel while the run was open. + for run_id in (first.first_execution_run_id, latest.result_run_id): + history = await native_client.get_workflow_handle( + first.id, run_id=run_id + ).fetch_history() + assert any( + event.HasField("nexus_operation_cancel_request_completed_event_attributes") + for event in history.events + ) + + @workflow.defn class ReadsUntilHalfRetention: @workflow.run From 0d39c781f741e944673a7a7780f5d74d744874c6 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:17:03 -0700 Subject: [PATCH 56/79] Bounded the pending Workflow appends by bytes and dropped refused ones. An entry left only on success, a rerun under a new epoch added another, and the count bound let up to 2 GiB stay. --- temporalio/common/streams/_writes.py | 72 ++++++++++++++----- tests/contrib/streams/test_workflow_module.py | 46 ++++++++++++ 2 files changed, 100 insertions(+), 18 deletions(-) diff --git a/temporalio/common/streams/_writes.py b/temporalio/common/streams/_writes.py index a9b91f91a7..aea926f539 100644 --- a/temporalio/common/streams/_writes.py +++ b/temporalio/common/streams/_writes.py @@ -113,14 +113,49 @@ class _Encoded: # The batches of one append and the calls of each, with the bodies after the codec, and # the answer of each batch that the Store took. batches: list[tuple[list[int], list[stream_pb.AppendRecord]]] + size: int answers: dict[int, Any] = dataclasses.field(default_factory=dict) -# The appends that have not succeeded yet, by Producer and epoch. A retry of the Local Activity -# in this process sends the same bytes, so the Store's dedupe of a resend matches them, also -# when the codec adds random bytes. A rerun after a crash is a new epoch, so it needs none. -_PENDING: OrderedDict[bytes, _Encoded] = OrderedDict() -_PENDING_LIMIT = 1024 +_Key = tuple[str, str, str] +_Epoch = tuple[int, int, int, int] + + +class _Pending: + # The appends that have not succeeded, one for each Producer. A retry in this process sends + # the same bytes, so the Store's dedupe of a resend matches them, also with a random codec. + + def __init__(self, limit: int) -> None: + self._limit = limit + self._size = 0 + self._entries: OrderedDict[_Key, tuple[_Epoch, _Encoded]] = OrderedDict() + + def get(self, key: _Key, epoch: _Epoch) -> _Encoded | None: + found = self._entries.get(key) + return found[1] if found is not None and found[0] == epoch else None + + def put(self, key: _Key, epoch: _Epoch, encoded: _Encoded) -> None: + found = self._entries.get(key) + if found is not None: + if found[0] > epoch: + # The Store fences this write, since the Producer has a later epoch. + return + self.pop(key, found[0]) + self._entries[key] = (epoch, encoded) + self._size += encoded.size + while self._size > self._limit and len(self._entries) > 1: + _, (_, oldest) = self._entries.popitem(last=False) + self._size -= oldest.size + + def pop(self, key: _Key, epoch: _Epoch) -> None: + found = self._entries.get(key) + if found is not None and found[0] == epoch: + del self._entries[key] + self._size -= found[1].size + + +# A run that ends while its append retries leaves its entry, so the oldest go past this size. +_PENDING = _Pending(64 * 1024 * 1024) async def _encode( @@ -148,7 +183,7 @@ async def _encode( for record in records ] if sum(sizes) <= _MAX_BATCH_BYTES and max(sizes, default=0) <= _MAX_RECORD_BYTES: - return _Encoded([(list(range(len(counts))), records)]) + return _Encoded([(list(range(len(counts))), records)], sum(sizes)) # The codec grew the merge past the Store's limit, so each call goes as its own batch, as # it would with no merge. A call that is too large alone fails alone. batches: list[tuple[list[int], list[stream_pb.AppendRecord]]] = [] @@ -156,7 +191,7 @@ async def _encode( for call, count in enumerate(counts): batches.append(([call], records[start : start + count])) start += count - return _Encoded(batches) + return _Encoded(batches, sum(sizes)) @temporalio.activity.defn(name=APPEND_ACTIVITY) @@ -171,19 +206,18 @@ async def append_records( streams, service = _context() producer = _producer(streams) producer_pb = models._ProducerTransferTypeConverter().to_transfer_type(producer) - key = b"\n".join( - [ - streams.namespace.encode(), - request.stream.id.encode(), - producer_pb.SerializeToString(deterministic=True), - ] + key = (streams.namespace, request.stream.id, producer_pb.id) + task = producer_pb.epoch + epoch = ( + task.task_started_event_id, + task.task_attempt, + task.task_started_time.seconds, + task.task_started_time.nanos, ) - encoded = _PENDING.get(key) + encoded = _PENDING.get(key, epoch) if encoded is None: encoded = await _encode(streams, request, counts, producer_pb) - _PENDING[key] = encoded - while len(_PENDING) > _PENDING_LIMIT: - _PENDING.popitem(last=False) + _PENDING.put(key, epoch, encoded) # The bodies went through the codec once already. service = streams.backend.service( streams.namespace, dataclasses.replace(streams.converter, payload_codec=None) @@ -214,6 +248,8 @@ async def append_records( answer = response.last_offset or 0 except _errors.StreamError as err: if err.reason != "stream_too_large": + if err.non_retryable: + _PENDING.pop(key, epoch) raise answer = {"reason": err.reason, "message": err.message} encoded.answers[index] = answer @@ -223,7 +259,7 @@ async def append_records( for call in calls: after -= counts[call] answers[call] = answer - after if isinstance(answer, int) else answer - _PENDING.pop(key, None) + _PENDING.pop(key, epoch) return answers diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index f7b46af991..5b5e6bbf31 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -18,6 +18,7 @@ WorkflowHandle, ) from temporalio.common import RetryPolicy, streams +from temporalio.common.streams import _writes from temporalio.common.streams._backend import _ExtensionCalls from temporalio.common.streams._defn import definition_of from temporalio.contrib import stream as redis @@ -766,6 +767,51 @@ async def loses_the_first_answer( assert sent[0] == sent[1] +@workflow.defn +class PublishesToAMissingStream: + @workflow.run + async def run(self) -> None: + # No create, so the Store refuses the append, and no retry sends it again. + stream = workflow.stream_handle(Events) + await stream.topic(Events.tokens).publish("x" * 100_000) + + +async def test_refused_write_keeps_no_pending_bytes(stream_client: Client): + async with new_worker(stream_client, PublishesToAMissingStream) as worker: + handle = await stream_client.start_workflow( + PublishesToAMissingStream.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + with pytest.raises(WorkflowFailureError): + await handle.result() + stream_id = handle.stream(Events).id + assert not [key for key in _writes._PENDING._entries if key[1] == stream_id] + + +def _entry(size: int) -> Any: + return _writes._Encoded([], size) + + +def test_pending_appends_keep_the_newest_epoch_and_a_bounded_size(): + pending = _writes._Pending(100) + one, two = ("ns", "s", "la:1"), ("ns", "s", "la:2") + first, retried = _entry(10), _entry(10) + pending.put(one, (5, 1, 0, 0), first) + # A rerun under a later epoch takes the place of the earlier one. + pending.put(one, (9, 1, 0, 0), retried) + assert pending.get(one, (5, 1, 0, 0)) is None + assert pending.get(one, (9, 1, 0, 0)) is retried + # A write of an earlier epoch is fenced, so it leaves the later one, and so does its pop. + pending.put(one, (5, 1, 0, 0), first) + pending.pop(one, (5, 1, 0, 0)) + assert pending.get(one, (9, 1, 0, 0)) is retried + # Past the size, the oldest entry goes. + pending.put(two, (9, 1, 0, 0), _entry(95)) + assert pending.get(one, (9, 1, 0, 0)) is None + assert pending.get(two, (9, 1, 0, 0)) is not None + + @workflow.defn class PublishesLargeValues: @workflow.run From a488efeb617f4006972c6ae4f044c645a7d73f28 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:17:38 -0700 Subject: [PATCH 57/79] Warned when a Worker with Streams polls no Activities. The close of a Workflow's Stream is a regular Activity, so such a Worker alone left every close waiting. Another Worker on the queue may run it, so it is a warning. --- temporalio/common/streams/_extension.py | 7 +++++++ temporalio/common/streams/_workflow.py | 3 ++- tests/contrib/streams/test_stream_client.py | 13 +++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index 5e87ebea47..5f8d03e497 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -153,6 +153,13 @@ def _configure_worker( f"The Worker's client has no {type(self).__name__}. Register it with " "stream_backends=." ) + if config.get("no_remote_activities"): + # Another Worker on the queue may run the close Activity, so it is no error. + logger.warning( + "The Worker on the task queue %s polls no Activities, so the close of a " + "Workflow's Stream waits for another Worker on that queue with the backend", + config.get("task_queue"), + ) # Records get no serialization context, so Workflows, Activities and clients encode # them alike. mine.converter = client.data_converter diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 241bbd4c39..33a6d486f7 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -442,7 +442,8 @@ async def close( A task that fails before then, or a crash, leaves the Stream open, and the task's retry writes to it again. On Redis the close is a regular Activity on the Workflow's task queue, which closes as the run chain, the Stream's owner. So every Worker on that queue - registers the Redis backend. Readers get the Records before it, then a ``Closed`` item + registers the Redis backend, and one that polls Activities runs the close, since a + Worker with ``no_remote_activities=True`` runs none. Readers get the Records before it, then a ``Closed`` item with ``result`` or ``failure``. A cancel failure, such as a ``CancelledError`` or an ``ActivityError`` that a cancel caused, closes it as canceled. Later writes in this run raise :py:class:`StreamClosedError`. diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 296cc4f6ff..2a82b55726 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -1,5 +1,6 @@ import asyncio import dataclasses +import logging import uuid from dataclasses import dataclass from datetime import timedelta @@ -470,6 +471,18 @@ async def test_registration_rules(client: Client): ) +async def test_worker_that_polls_no_activities_warns_of_the_close( + client: Client, caplog: pytest.LogCaptureFixture +): + # The close of a Workflow's Stream is a regular Activity, which another Worker may run. + config = client.config() + config["stream_backends"] = [MemoryStreamBackend(default=True)] + with caplog.at_level(logging.WARNING, logger="temporalio.common.streams"): + async with new_worker(Client(**config), Agent, no_remote_activities=True): + pass + assert "polls no Activities" in caplog.text + + async def test_backend_is_not_a_plugin(client: Client): # The connect refuses it before it connects, so the unreachable target doesn't matter. with pytest.raises(streams.StreamConfigError, match="not a plugin"): From a68192842130e3f42feaf02511df843f5d20b45e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:16:29 -0700 Subject: [PATCH 58/79] Paused native reads on a timer past 20 wakes in one run. Wakes and the native close share the run's 30 pending Nexus operations. Past the limit the server failed every task of the run, so a close never went out. --- temporalio/common/streams/_workflow.py | 6 ++- temporalio/common/streams/_workflow_reader.py | 29 +++++++--- tests/contrib/streams/test_native_server.py | 53 ++++++++++++++++++- 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 65ce5d9704..0a5f15012c 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -437,8 +437,10 @@ def subscribe( History, so a replay reads the same items. On a native Stream the marker holds only the page's offsets, and a replay reads the page again from the server. A read at the end of a native Stream waits on the server until a Record comes or the Stream closes. - On Redis the marker holds the page, the Local Activity waits a short time for new - Records, and a page with none is followed by a durable timer from 1 s up to 60 s. + Such waits take at most 20 of the run's 30 pending Nexus operations, and a read past + them waits as on Redis. On Redis the marker holds the page, the Local Activity waits a + short time for new Records, and a page with none is followed by a durable timer from + 1 s up to 60 s. A read sees this run's writes that it awaited, since each write is a Local Activity that ends before the next read starts. diff --git a/temporalio/common/streams/_workflow_reader.py b/temporalio/common/streams/_workflow_reader.py index dba398b280..96c2aadef1 100644 --- a/temporalio/common/streams/_workflow_reader.py +++ b/temporalio/common/streams/_workflow_reader.py @@ -31,6 +31,9 @@ # from the first pause up to the last. _FIRST_PAUSE = timedelta(seconds=1) _LAST_PAUSE = timedelta(seconds=60) +# A run may have 30 pending Nexus operations on the server. Wakes take at most 20, so the run's +# closes and its own operations still fit, and a read past them pauses on a timer. +_MAX_WAKES = 20 # How long a read waits for a close after the Stream's owner ended. The close of a run comes # from its Worker, so it may come after the run closed, or never. _OWNER_END_GRACE = timedelta(minutes=1) @@ -63,6 +66,8 @@ def reader_of(backend: str) -> Any: class _ReadState: # The run's wakes that wait on the server, with their reads, so the run's end cancels them. wakes: dict[Any, WorkflowItems] = field(default_factory=dict) + # The wakes that the server has not taken yet, which count against the cap too. + starting: int = 0 # The lowest offset that the run's replayed pages read, per Stream, and whether a live # page sent it. replay_floors: dict[tuple[str, str], int] = field(default_factory=dict) @@ -323,9 +328,9 @@ def _owner_gone(self, page: Page, now: datetime) -> bool: return now - self._owner_ended_at >= _OWNER_END_GRACE async def _wait_for_records(self, now: datetime) -> None: - if not self._reader._temporal_stream_parks_reads: - # The Store can't wake a Workflow, so a quiet Stream costs one timer and one read - # per pause. + if not self._can_wake(): + # The Store can't wake a Workflow, or the run has the most wakes. A quiet Stream + # then costs one timer and one read per pause. await self._sleep() return request = self._waiting_request() @@ -339,7 +344,7 @@ async def _wait_for_records(self, now: datetime) -> None: await self._wake_once(request) async def _wait_for_create(self) -> None: - if not self._reader._temporal_stream_parks_reads: + if not self._can_wake(): await self._sleep() return request = self._waiting_request() @@ -352,6 +357,13 @@ async def _wait_for_create(self) -> None: # The next page tells whether time is left. pass + def _can_wake(self) -> bool: + state = _read_state() + return ( + self._reader._temporal_stream_parks_reads + and len(state.wakes) + state.starting < _MAX_WAKES + ) + async def _sleep(self) -> None: await temporalio.workflow.sleep(self._pause) self._pause = min(self._pause * 2, _LAST_PAUSE) @@ -359,8 +371,13 @@ async def _sleep(self) -> None: async def _wake_once(self, request: stream_pb.ReadRecordsRequest) -> None: # The server parks the wake until a Record passes the filter, the Stream closes, or # the wait ends. Its answer holds no Records, and the next page reads them. - wake = await self._reader._temporal_stream_start_wake(request) - wakes = _read_state().wakes + state = _read_state() + state.starting += 1 + try: + wake = await self._reader._temporal_stream_start_wake(request) + finally: + state.starting -= 1 + wakes = state.wakes wakes[wake] = self self._wake = wake if self._closed: diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index 1fe9c0f834..cb875e7ea9 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -20,7 +20,7 @@ from temporalio.api.enums.v1 import WorkflowTaskFailedCause from temporalio.client import Client, WorkflowExecutionStatus, WorkflowHandle from temporalio.common import RetryPolicy, streams -from temporalio.common.streams import _native, _plugin +from temporalio.common.streams import _native, _plugin, _workflow_reader from temporalio.common.streams._native import POSITIONS_TYPE from temporalio.contrib.stream import RedisStreamBackend from temporalio.worker import Replayer, UnsandboxedWorkflowRunner @@ -398,6 +398,57 @@ async def continued() -> None: ) +@workflow.defn +class ReadsTwoAndCloses: + @workflow.run + async def run( + self, a: streams.WorkflowStream[Events], b: streams.WorkflowStream[Events] + ) -> str: + async def read(stream: streams.WorkflowStream[Events]) -> None: + async for _ in stream.subscribe(offset=streams.END): + pass + + asyncio.create_task(read(a)) + asyncio.create_task(read(b)) + await workflow.sleep(timedelta(seconds=2)) + own = await workflow.create_stream(Events) + await own.close() + return "closed" + + +async def test_reads_past_the_wake_cap_pause_on_a_timer( + native_client: Client, monkeypatch: pytest.MonkeyPatch +): + # Wakes and the close share the run's 30 pending Nexus operations, and the server fails a + # task with one more. So reads past the cap leave room for the close. + monkeypatch.setattr(_workflow_reader, "_MAX_WAKES", 1) + refs: list[streams.WorkflowStream[Events]] = [] + for _ in range(2): + stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") + refs.append(streams.WorkflowStream.from_handle(stream)) + async with new_worker(native_client, ReadsTwoAndCloses) as worker: + handle = await native_client.start_workflow( + ReadsTwoAndCloses.run, + args=refs, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await handle.result() == "closed" + history = await handle.fetch_history() + operations = [ + event.nexus_operation_scheduled_event_attributes.operation + for event in history.events + if event.HasField("nexus_operation_scheduled_event_attributes") + ] + assert sorted(operations) == ["CloseStream", "ReadRecords"] + timers = [ + event + for event in history.events + if event.HasField("timer_started_event_attributes") + ] + assert len(timers) > 1 + + @workflow.defn class ReadsUntilHalfRetention: @workflow.run From 0c173b5511cd1051fed520aa45589628aad0ecc4 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:18:26 -0700 Subject: [PATCH 59/79] Raised StreamExpiredError at the create of an expired Stream. The create took the expired status as a success, and the error came only at the first publish. --- temporalio/common/streams/_stream_client.py | 4 +++ temporalio/common/streams/_writes.py | 4 +++ temporalio/contrib/stream/__init__.py | 1 + temporalio/workflow/_streams.py | 4 ++- tests/contrib/streams/test_stream_client.py | 13 ++++++++ tests/contrib/streams/test_workflow_module.py | 31 +++++++++++++++++++ 6 files changed, 56 insertions(+), 1 deletion(-) diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index c0f85b3814..c890aeed82 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -412,6 +412,10 @@ async def create() -> ClientStream[E]: f"The Stream {id} was created and closed before", reason="stream_closed", ) + if response.status == stream_pb.StreamStatus.STREAM_STATUS_EXPIRED: + raise _errors.StreamExpiredError( + f"The Stream {id} expired", reason="stream_expired" + ) stream = ClientStream(id=id, backend=backend_name, definition=definition) return stream._bind(client, calls) diff --git a/temporalio/common/streams/_writes.py b/temporalio/common/streams/_writes.py index aea926f539..bf07508bc1 100644 --- a/temporalio/common/streams/_writes.py +++ b/temporalio/common/streams/_writes.py @@ -100,6 +100,10 @@ async def create_stream(request: stream_pb.CreateStreamRequest) -> int: f"The Stream {request.stream_id} was created and closed before", reason="stream_closed", ) + if response.status == stream_pb.StreamStatus.STREAM_STATUS_EXPIRED: + raise _errors.StreamExpiredError( + f"The Stream {request.stream_id} expired", reason="stream_expired" + ) return 0 diff --git a/temporalio/contrib/stream/__init__.py b/temporalio/contrib/stream/__init__.py index bfd1cffc0d..c77f870aa5 100644 --- a/temporalio/contrib/stream/__init__.py +++ b/temporalio/contrib/stream/__init__.py @@ -131,6 +131,7 @@ def create( Raises: StreamConfigError: The client has no Redis backend of that name. No call is made. StreamClosedError: A Stream of that id was created and closed before. + StreamExpiredError: A Stream of that id expired. StreamError: The Stream exists with another Retention, with the reason ``stream_already_exists``. """ diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index c0f2980d09..4c9ed08001 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -54,7 +54,9 @@ def create_stream( that of the Worker's backend. Raises: - StreamError: The backend refused the create, such as for another Retention. + StreamError: The backend refused the create, such as for another Retention, or + ``StreamClosedError`` and ``StreamExpiredError`` for a Stream of that id that + closed or expired. ReadOnlyContextError: The code must not change state, such as a query handler. StreamConfigError: No Stream backend is on the Worker, none of that name, or none has ``default=True`` for a create with no backend. It fails the Workflow Task, diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 2a82b55726..c52da8ddc4 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -384,6 +384,19 @@ async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: C ] +async def test_create_of_an_expired_stream_raises(stream_client: Client): + # An open client Stream expires one Retention after its last write. + stream_id = f"expired-{uuid.uuid4()}" + retention = timedelta(milliseconds=200) + stream = await redis.create( + stream_client, Events, id=stream_id, retention=retention + ) + await stream.topic(Events.tokens).publish("old") + await asyncio.sleep(0.5) + with pytest.raises(streams.StreamExpiredError): + await redis.create(stream_client, Events, id=stream_id, retention=retention) + + async def test_named_producer_fences_its_earlier_epoch(stream_client: Client): stream = await redis.create(stream_client, Events, id=f"named-{uuid.uuid4()}") # A server that restarts writes as one name with a higher epoch, so the zombie it diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 5b5e6bbf31..7fda37b25e 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -859,6 +859,37 @@ async def run(self) -> str: return stream.backend or "" +@workflow.defn +class CreatesAfterAnIdleTime: + @workflow.run + async def run(self) -> str: + await workflow.create_stream(Events) + await workflow.sleep(timedelta(seconds=1.5)) + try: + await workflow.create_stream(Events) + except streams.StreamExpiredError as err: + return err.reason + return "created" + + +async def test_create_of_an_expired_workflow_stream_raises(client: Client): + # An open Workflow Stream expires one safety TTL after its last write. + config = client.config() + config["stream_backends"] = [ + MemoryStreamBackend(default=True, safety_ttl=timedelta(milliseconds=500)) + ] + expiring = Client(**config) + async with new_worker(expiring, CreatesAfterAnIdleTime) as worker: + assert ( + await expiring.execute_workflow( + CreatesAfterAnIdleTime.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + == "stream_expired" + ) + + async def test_default_is_native_with_no_default_backend(client: Client): # With no backend marked default=True, the default is native, which this SDK lacks. The # Redis calls still pick the registered Redis backend, and the memory one stands in for it. From 2e47ec14afa7133c4181417a30faeabd8ca86b24 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:19:06 -0700 Subject: [PATCH 60/79] Raised a StreamError past a Record that fails to decode. A plain TypeError ended the read with next_offset before the Record, so every resume failed on it again. --- temporalio/common/streams/_reader.py | 26 +++++++++++++++++---- tests/contrib/streams/test_stream_client.py | 22 +++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/temporalio/common/streams/_reader.py b/temporalio/common/streams/_reader.py index 283fe0b65c..22975f55b6 100644 --- a/temporalio/common/streams/_reader.py +++ b/temporalio/common/streams/_reader.py @@ -69,8 +69,9 @@ def next_offset(self) -> int | None: After a ``Superseded`` item it is that item's own offset, since the first Record of its batch has the same offset and comes next. A read from there gives the ``Superseded`` - item again, which withdraws nothing new. Pass it as ``offset`` to a later read. It - is None only before the first page of a read from ``END``. + item again, which withdraws nothing new. After a Record that failed to decode, it is + past that Record. Pass it as ``offset`` to a later read. It is None before the first + page of a read from ``END`` or of a read with ``last``. """ return self._position.next_offset @@ -79,7 +80,12 @@ def __aiter__(self) -> Subscription[T]: return self async def __anext__(self) -> Record[T] | Superseded | Closed: - """The next item.""" + """The next item. + + Raises: + StreamError: A Record's value failed to decode with its Topic's type. The read + ends, and :py:attr:`next_offset` is past that Record. + """ with _backend.using(self._options): item = await self._items.__anext__() if isinstance(item, Superseded): @@ -162,7 +168,19 @@ def request() -> models.ReadRecordsRequest: try: async for page in pages: for record in page.records or []: - item = await _item(converter, record, types, default_type) + try: + item = await _item(converter, record, types, default_type) + except Exception as err: + # A resume from next_offset goes on past the Record, so it can't stop + # every later read. The error names its offset for a read of it. + offset = record.offset or 0 + position.next_offset = max( + position.next_offset or 0, offset + 1 + ) + raise _errors.StreamError( + f"Can't decode the Record at the offset {offset} of the Topic " + f"{record.topic or ''}: {err}" + ) from err if item is None: continue yield item diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index c52da8ddc4..dec42988c3 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -384,6 +384,28 @@ async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: C ] +@streams.defn(name="events") +class LooseEvents: + turns = streams.topic(str) + tokens = streams.topic(str) + + +async def test_record_that_fails_to_decode_raises_and_resumes_past_it( + stream_client: Client, +): + stream_id = f"poison-{uuid.uuid4()}" + writer = await redis.create(stream_client, LooseEvents, id=stream_id) + bad = await writer.topic(LooseEvents.turns).publish("not a turn") + await writer.topic(LooseEvents.tokens).publish("after") + reader = stream_client.stream(Events, id=stream_id) + read = reader.subscribe(wait_new_records=False) + with pytest.raises(streams.StreamError, match=f"offset {bad} of the Topic turns"): + await anext(read) + assert read.next_offset == bad + 1 + resumed = reader.subscribe(offset=read.next_offset, wait_new_records=False) + assert [item.data async for item in resumed] == ["after"] + + async def test_create_of_an_expired_stream_raises(stream_client: Client): # An open client Stream expires one Retention after its last write. stream_id = f"expired-{uuid.uuid4()}" From 8358aef6873b67d2b3a48d7765287ca72474c9cb Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:19:53 -0700 Subject: [PATCH 61/79] Refused async with on a client handle of a Workflow's Stream. The close at the block end always raised StreamNotOwnerError and hid the block's own error. --- temporalio/common/streams/_stream_client.py | 27 ++++++++++++++------- tests/contrib/streams/test_stream_client.py | 14 +++++++++++ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index c890aeed82..dcb3589b92 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -338,6 +338,16 @@ async def get_offset( with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): return await self._access().get_offset() + def _check_close(self) -> None: + if self._workflow_id: + # A Workflow's Stream closes only from its run chain. The Store refuses a client's + # close too, and this error comes with no call. + raise _errors.StreamNotOwnerError( + f"Can't close the Stream {self._id or self._stream_name} from a client, since " + "a Workflow owns it", + reason="stream_not_owner", + ) + async def close( self, *, @@ -355,20 +365,19 @@ async def close( Raises: StreamNotOwnerError: A Workflow owns the Stream, so only its run chain closes it. """ - if self._workflow_id: - # A Workflow's Stream closes only from its run chain. The Store refuses a client's - # close too, and this error comes with no call. - raise _errors.StreamNotOwnerError( - f"Can't close the Stream {self._id or self._stream_name} from a client, since " - "a Workflow owns it", - reason="stream_not_owner", - ) + self._check_close() self._closed = True with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): await self._access().close(result, failure) async def __aenter__(self) -> ClientStream[E]: - """Use the Stream in ``async with``, which closes it at the end.""" + """Use the Stream in ``async with``, which closes it at the end. + + Raises: + StreamNotOwnerError: A Workflow owns the Stream, so only its run chain closes it. + """ + # The close at the end would raise and hide the block's own error. + self._check_close() return self async def __aexit__( diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index dec42988c3..138f455856 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -406,6 +406,20 @@ async def test_record_that_fails_to_decode_raises_and_resumes_past_it( assert [item.data async for item in resumed] == ["after"] +async def test_async_with_refuses_a_workflow_stream_before_the_block( + stream_client: Client, +): + # The close at the block end would raise and hide the block's own error. + handle = stream_client.get_workflow_handle( + "agent-1", first_execution_run_id="run-1" + ) + ran = False + with pytest.raises(streams.StreamNotOwnerError): + async with handle.stream(Events): + ran = True + assert not ran + + async def test_create_of_an_expired_stream_raises(stream_client: Client): # An open client Stream expires one Retention after its last write. stream_id = f"expired-{uuid.uuid4()}" From 0cd056c19040940296dc9036106a1d1d8914371d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:20:35 -0700 Subject: [PATCH 62/79] Refused an empty Stream id at Client.stream. An empty id failed only at the first call, with a bare AssertionError. --- temporalio/client/_client.py | 2 ++ temporalio/common/streams/_stream_client.py | 6 +++++- tests/contrib/streams/test_stream_client.py | 9 +++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index e3f823078c..ac51fbb48e 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -942,6 +942,8 @@ def stream( Raises: StreamConfigError: The client has no backend of that name, or none has ``default=True`` for a call with no backend. + TypeError: Neither or both of ``id`` and ``stream`` are given. + ValueError: The id is empty, or ``backend`` is not the handle's backend. """ from temporalio.common.streams import _stream_client diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index dcb3589b92..98ceb35525 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -451,9 +451,13 @@ def client_stream( f"The handle is of the backend {stream.backend}, not {backend}" ) backend = backend or stream.backend + stream_id = stream.id if stream else id + if not stream_id: + # An empty id would fail only at the first call, with no cause in it. + raise ValueError("The Stream id is empty") backend_name, calls = client_backend(client, backend) handle = ClientStream( - id=stream.id if stream else id or "", + id=stream_id, workflow_id=stream.workflow_id if stream else None, first_execution_run_id=stream.first_execution_run_id if stream else None, backend=backend_name, diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 138f455856..a24f6e1801 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -420,6 +420,15 @@ async def test_async_with_refuses_a_workflow_stream_before_the_block( assert not ran +async def test_empty_stream_id_raises_with_no_call(client: Client): + client, extension = fake_client(client) + with pytest.raises(ValueError, match="id is empty"): + client.stream(Events, id="") + with pytest.raises(ValueError, match="id is empty"): + client.stream(Events, stream=streams.WorkflowStream(id="")) + assert not extension.namespaces + + async def test_create_of_an_expired_stream_raises(stream_client: Client): # An open client Stream expires one Retention after its last write. stream_id = f"expired-{uuid.uuid4()}" From 1de5999621dc4ddc7a24aed8fa921b8c54a22a2e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:21:01 -0700 Subject: [PATCH 63/79] Kept no offsets of client Records in a read. Nothing withdraws a client's Records, so a long UI read grew its withdrawal map with no end. --- temporalio/common/streams/_reader.py | 9 +++++++-- tests/contrib/streams/test_stream_client.py | 9 +++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/temporalio/common/streams/_reader.py b/temporalio/common/streams/_reader.py index 22975f55b6..843908fe8d 100644 --- a/temporalio/common/streams/_reader.py +++ b/temporalio/common/streams/_reader.py @@ -9,7 +9,7 @@ from typing import Any, Generic, TypeVar import temporalio.converter -from temporalio.api.streamservice.v1 import ReadStart, StreamRecordKind +from temporalio.api.streamservice.v1 import ProducerKind, ReadStart, StreamRecordKind from temporalio.common.streams import _backend, _convert, _errors from temporalio.common.streams._items import ( Closed, @@ -95,7 +95,12 @@ async def __anext__(self) -> Record[T] | Superseded | Closed: self._position.next_offset = max( self._position.next_offset or 0, item.offset + 1 ) - if isinstance(item, Record) and item.producer is not None: + if ( + isinstance(item, Record) + and item.producer is not None + # Nothing withdraws a client's Records, so a long read of them keeps no offsets. + and item.producer.kind != ProducerKind.PRODUCER_KIND_CLIENT + ): key = producer_key(item.producer) self._shown.setdefault(key, (item.producer, []))[1].append(item.offset) elif isinstance(item, Superseded): diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index a24f6e1801..d9af7c5c5e 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -420,6 +420,15 @@ async def test_async_with_refuses_a_workflow_stream_before_the_block( assert not ran +async def test_read_keeps_no_offsets_of_client_records(stream_client: Client): + # Nothing withdraws a client's Records, so a long UI read of them keeps no state. + stream = await redis.create(stream_client, Events, id=f"chat-{uuid.uuid4()}") + await stream.topic(Events.tokens).publish("a", "b", "c") + read = stream.subscribe(wait_new_records=False) + assert [item.data async for item in read] == ["a", "b", "c"] + assert read._shown == {} + + async def test_empty_stream_id_raises_with_no_call(client: Client): client, extension = fake_client(client) with pytest.raises(ValueError, match="id is empty"): From 6c60f9744e5b2328ebfff9f91673b666d0e8b9ac Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:21:33 -0700 Subject: [PATCH 64/79] Exported End and dropped the unused RPC error mapping. subscribe takes offset: int | End | None, and nothing in Brook or Jetty calls from_rpc_error. --- temporalio/common/streams/__init__.py | 2 ++ temporalio/common/streams/_errors.py | 18 ------------------ tests/contrib/streams/test_facade.py | 18 ------------------ tests/contrib/streams/test_handles.py | 5 +++++ 4 files changed, 7 insertions(+), 36 deletions(-) diff --git a/temporalio/common/streams/__init__.py b/temporalio/common/streams/__init__.py index 214b376368..19b6e56e49 100644 --- a/temporalio/common/streams/__init__.py +++ b/temporalio/common/streams/__init__.py @@ -52,6 +52,7 @@ END, Closed, CloseStatus, + End, Record, Superseded, ) @@ -70,6 +71,7 @@ "CloseStatus", "Closed", "Creating", + "End", "ReadOnlyContextError", "Record", "StreamClosedError", diff --git a/temporalio/common/streams/_errors.py b/temporalio/common/streams/_errors.py index 48ac3fde49..d33bc01821 100644 --- a/temporalio/common/streams/_errors.py +++ b/temporalio/common/streams/_errors.py @@ -134,13 +134,6 @@ class ReadOnlyContextError(StreamError): } -# The details move to temporal.api.streamservice.v1. Both names carry the same message. -_DETAILS_TYPES = ( - "temporal.api.streamservice.v1.StreamErrorDetails", - StreamErrorDetails.DESCRIPTOR.full_name, -) - - def from_call_error(err: Exception) -> StreamError: """Turn a failed call of the crate's runtime into the Stream error for its reason. @@ -174,14 +167,3 @@ def from_activity_error(err: Exception) -> Exception: reason = str(cause.details[0]) if cause.details else "" return _BY_TYPE[cause.type](cause.message, reason=reason) return err - - -def from_rpc_error(err: temporalio.service.RPCError) -> Exception: - """The Stream error for a server failure with Stream error details, else ``err``.""" - for detail in err.grpc_status.details: - if detail.type_url.rsplit("/", 1)[-1] in _DETAILS_TYPES: - reason = StreamErrorDetails.FromString(detail.value).reason - return _BY_REASON.get(reason, StreamError)( - err.message, reason=reason, code=err.status - ) - return err diff --git a/tests/contrib/streams/test_facade.py b/tests/contrib/streams/test_facade.py index d0894825f7..a0e66004ee 100644 --- a/tests/contrib/streams/test_facade.py +++ b/tests/contrib/streams/test_facade.py @@ -3,10 +3,8 @@ from dataclasses import dataclass from datetime import timedelta -import google.protobuf.any_pb2 import pytest -import temporalio.api.common.v1 import temporalio.api.streamservice.v1 as stream_pb from temporalio.bridge.streams import StreamCallError from temporalio.client import Client @@ -186,19 +184,3 @@ def test_unsupported_capability_maps_to_its_error(): assert isinstance(err, streams.StreamUnsupportedError) assert err.non_retryable and err.code == RPCStatusCode.UNIMPLEMENTED assert err.message.endswith("replica_wait") - - -def test_server_failure_maps_to_its_stream_error(): - detail = google.protobuf.any_pb2.Any() - detail.Pack(ext_pb.StreamErrorDetails(reason="stream_closed")) - status = temporalio.api.common.v1.GrpcStatus( - code=9, message="closed", details=[detail] - ) - err = _errors.from_rpc_error( - RPCError( - "closed", RPCStatusCode.FAILED_PRECONDITION, status.SerializeToString() - ) - ) - assert isinstance(err, streams.StreamClosedError) - plain = RPCError("down", RPCStatusCode.UNAVAILABLE, b"") - assert _errors.from_rpc_error(plain) is plain diff --git a/tests/contrib/streams/test_handles.py b/tests/contrib/streams/test_handles.py index b00d3d66f4..a1df381fb2 100644 --- a/tests/contrib/streams/test_handles.py +++ b/tests/contrib/streams/test_handles.py @@ -149,3 +149,8 @@ def test_any_type_argument_has_no_definition(): decoded = converter.from_payload(converter.to_payload(OWNED), PlainStream[Any]) assert type(decoded) is PlainStream and decoded.definition is None assert decoded.id == OWNED.id + + +def test_end_type_is_public(): + # subscribe takes offset: int | End | None, so the type of END is a public name. + assert isinstance(streams.END, streams.End) and "End" in streams.__all__ From c0da52753bc67025995d4837889c6fef0d53851c Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:22:46 -0700 Subject: [PATCH 65/79] Keyed a Workflow's Activities only when its run uses Streams. The key is in History, so a run that never creates, binds or receives a Stream adds nothing there. A Local Activity keeps its key always, since its headers stay out of History. --- temporalio/common/streams/_extension.py | 33 +++++- temporalio/common/streams/_workflow.py | 9 +- .../contrib/streams/test_activity_handles.py | 105 ++++++++++++++++++ tests/contrib/streams/test_redis.py | 2 +- 4 files changed, 144 insertions(+), 5 deletions(-) diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/streams/_extension.py index 5f8d03e497..03afdfbc24 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/streams/_extension.py @@ -33,8 +33,8 @@ logger = logging.getLogger(__name__) -# The backend puts this key into the headers of each Activity and Local Activity that a Workflow -# schedules. It keeps the Producer id the same across retries. +# The backend puts this key into the headers of each Local Activity that a Workflow schedules, +# and of each Activity of a run that uses Streams. It keeps the Producer id across retries. _KEY_HEADER = "__temporal_stream_key" # Continue-as-new passes the run chain's open Streams to the next run in this header. _OWNED_STREAMS_HEADER = "__temporal_owned_streams" @@ -282,6 +282,7 @@ async def execute_workflow( self, input: temporalio.worker.ExecuteWorkflowInput ) -> Any: _take_owned_streams(input.headers) + _receive(input.args) try: result = await super().execute_workflow(input) except BaseException as err: @@ -293,6 +294,23 @@ async def execute_workflow( await streams_workflow.drain() return result + async def handle_signal(self, input: temporalio.worker.HandleSignalInput) -> None: + _receive(input.args) + await super().handle_signal(input) + + async def handle_update_handler( + self, input: temporalio.worker.HandleUpdateInput + ) -> Any: + _receive(input.args) + return await super().handle_update_handler(input) + + +def _receive(args: Sequence[Any]) -> None: + # A run that receives a Stream uses Streams. The inputs, Signals and Updates are in + # History, so a replay finds the same. + if _stream_arguments(args): + streams_workflow._state().uses_streams = True + def _ends_run(err: BaseException) -> bool: # The run's writes that nobody awaits go out before its end. Any other exception fails the @@ -323,6 +341,7 @@ def _take_owned_streams( if payload is None: return state = streams_workflow._state() + state.uses_streams = True converter = temporalio.converter.PayloadConverter.default for stream_id, backend, *_ in converter.from_payload(payload, list[Any]): state.stream(_owned_ref(stream_id)).backend = backend @@ -374,12 +393,20 @@ def continue_as_new(self, input: temporalio.worker.ContinueAsNewInput) -> NoRetu def start_activity( self, input: temporalio.worker.StartActivityInput ) -> temporalio.workflow.ActivityHandle[Any]: - input.headers = {**input.headers, **self._activity_headers()} + # The key goes into History, so only a run that uses Streams adds it, or a run whose + # Activity gets a Stream. Each call still counts, so a key is the same on replay. + headers = self._activity_headers() + state = streams_workflow._state() + if not state.uses_streams and _stream_arguments(input.args): + state.uses_streams = True + if state.uses_streams: + input.headers = {**input.headers, **headers} return super().start_activity(input) def start_local_activity( self, input: temporalio.worker.StartLocalActivityInput ) -> temporalio.workflow.ActivityHandle[Any]: + # A Local Activity's headers stay out of History, so each one gets its key. input.headers = {**input.headers, **self._activity_headers()} return super().start_local_activity(input) diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/streams/_workflow.py index 33a6d486f7..79bae3c8d5 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/streams/_workflow.py @@ -103,6 +103,9 @@ class _RunState: waiting: dict[str, _Write] = field(default_factory=dict) # The Streams that a retry run's first write has withdrawn the previous run on. withdrawn: set[str] = field(default_factory=set) + # Whether the run created, bound or received a Stream. Only then do its Activities carry + # a Producer key, so a run with no Stream adds nothing to History. + uses_streams: bool = False # The block ends whose close waits for the run's outcome, with the block's failure. deferred_closes: list[tuple[WorkflowStream[Any], BaseException | None]] = field( default_factory=list @@ -411,7 +414,9 @@ def _check(self, action: str, *, backend_checked: bool = False) -> _RunStream: raise _errors.ReadOnlyContextError( f"Can't {action} a Stream in read-only code" ) - run_stream = _state().stream(self._model_ref()) + state = _state() + state.uses_streams = True + run_stream = state.stream(self._model_ref()) if run_stream.closed: raise _errors.StreamClosedError( f"Can't {action} the Stream {self.id}, since this run closed it", @@ -671,6 +676,8 @@ def stream_handle( :py:func:`temporalio.workflow.stream_handle`. """ stream = _own_handle(definition, backend) + if not temporalio.workflow.unsafe.is_read_only(): + _state().uses_streams = True if backend is None: known = _state().streams.get(stream.id) stream._backend = (known.backend if known else None) or _check_backend( diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index ce029e960b..96d7803642 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -659,3 +659,108 @@ def call() -> None: assert stopped.wait(5) assert errors and isinstance(errors[0], CancelledError) loop.call_soon_threadsafe(loop.stop) + + +@activity.defn +async def plain(name: str) -> str: + return f"hello {name}" + + +@workflow.defn +class NeverTouchesStreams: + @workflow.run + async def run(self) -> str: + return await workflow.execute_activity( + plain, "ada", start_to_close_timeout=timedelta(seconds=10) + ) + + +@workflow.defn +class ReceivesAStream: + @workflow.run + async def run(self, stream: streams.WorkflowStream[Events]) -> str: + # The Activity gets no Stream, but the run received one. + return await workflow.execute_activity( + plain, stream.id, start_to_close_timeout=timedelta(seconds=10) + ) + + +@workflow.defn +class RetriesAfterContinueAsNew: + @workflow.run + async def run( + self, key: str, stream: streams.WorkflowStream[Events] | None + ) -> None: + if stream is None: + created = await workflow.create_stream(Events) + workflow.continue_as_new(args=[key, created]) + await workflow.execute_activity( + flaky, + CallModelInput(key, streams.ActivityStream.from_handle(stream)), + start_to_close_timeout=timedelta(seconds=30), + retry_policy=RetryPolicy(initial_interval=timedelta(milliseconds=10)), + ) + + +async def keyed(handle: WorkflowHandle[Any, Any], activity_type: str) -> list[bool]: + # Whether each schedule of the Activity carries the Producer key. + history = await handle.fetch_history() + return [ + "__temporal_stream_key" in attributes.header.fields + for event in history.events + if ( + attributes := event.activity_task_scheduled_event_attributes + ).activity_type.name + == activity_type + ] + + +async def test_only_a_run_that_uses_streams_keys_its_activities(stream_client: Client): + # The key is in History, so a run with no Stream adds none. A run that created, bound or + # received one keys each Activity, so an Activity that writes keeps its Producer id. + key = str(uuid.uuid4()) + async with new_worker( + stream_client, + NeverTouchesStreams, + ReceivesAStream, + Retrying, + RetriesAfterContinueAsNew, + activities=[plain, flaky], + ) as worker: + none = await stream_client.start_workflow( + NeverTouchesStreams.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await none.result() == "hello ada" + assert await keyed(none, "plain") == [False] + creates = await stream_client.start_workflow( + Retrying.run, key, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue + ) + await creates.result() + assert await keyed(creates, "flaky") == [True] + receives = await stream_client.start_workflow( + ReceivesAStream.run, + streams.WorkflowStream.from_handle(creates.stream(Events)), + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await receives.result() + assert await keyed(receives, "plain") == [True] + # The next run gets the Stream as its argument, and its retried Activity withdraws its + # first attempt as in the run that created the Stream. + continued_key = str(uuid.uuid4()) + continued = await stream_client.start_workflow( + RetriesAfterContinueAsNew.run, + args=[continued_key, None], + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + await continued.result() + assert await keyed(continued, "flaky") == [True] + items = await read(continued) + superseded = [item for item in items if isinstance(item, streams.Superseded)] + assert len(superseded) == 1 + assert [r.data for r in records(items) if superseded[0].withdraws(r)] == [ + "attempt-1" + ] diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 39c6a29a07..48bbd18dda 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -661,7 +661,7 @@ def test_closing_an_evicted_run_touches_no_runtime(): # Python prints "Exception ignored". inbound = _WorkflowInbound(_NeverEnds()) run = inbound.execute_workflow( - cast(ExecuteWorkflowInput, cast(object, SimpleNamespace(headers={}))) + cast(ExecuteWorkflowInput, cast(object, SimpleNamespace(headers={}, args=[]))) ) run.send(None) run.close() From d06cba494e4b1a37b21b580742c9855e084b3485 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:23:33 -0700 Subject: [PATCH 66/79] Corrected the Retention, aclose and Stream error docstrings. Retention is a Record's age and trims open Streams, a trimmed offset raises StreamExpiredError, and aclose from another task raises. --- temporalio/common/streams/_errors.py | 4 ++-- temporalio/common/streams/_handle.py | 2 +- temporalio/common/streams/_reader.py | 6 +++++- temporalio/contrib/stream/__init__.py | 3 ++- temporalio/contrib/stream/workflow.py | 3 ++- temporalio/workflow/_streams.py | 3 ++- 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/temporalio/common/streams/_errors.py b/temporalio/common/streams/_errors.py index d33bc01821..17dfe40323 100644 --- a/temporalio/common/streams/_errors.py +++ b/temporalio/common/streams/_errors.py @@ -61,7 +61,7 @@ class StreamClosedError(StreamError): class StreamExpiredError(StreamError): - """Retention removed the Stream.""" + """Retention removed the Stream, or the Records at the offset that a read asked for.""" class StreamFencedError(StreamError): @@ -75,7 +75,7 @@ class StreamStorageError(StreamError): class StreamOffsetError(StreamError): - """The offset is past the newest one plus one, or Retention removed it.""" + """The offset is below zero or past the newest one plus one.""" class StreamUnsupportedError(StreamError): diff --git a/temporalio/common/streams/_handle.py b/temporalio/common/streams/_handle.py index 7d5c321ec6..51e91fe56d 100644 --- a/temporalio/common/streams/_handle.py +++ b/temporalio/common/streams/_handle.py @@ -23,7 +23,7 @@ def exit_failure(exc: BaseException | None) -> tuple[bool, BaseException | None] """Whether the end of an ``async with`` block closes the Stream, and with which failure. A cancel closes it with a ``CancelledError`` failure. Another ``BaseException``, such as - an eviction, leaves it open, since the block did not end in the caller's code. + ``KeyboardInterrupt``, leaves it open, since the block did not end in the caller's code. """ if isinstance(exc, asyncio.CancelledError): return True, temporalio.exceptions.CancelledError(str(exc) or "Cancelled") diff --git a/temporalio/common/streams/_reader.py b/temporalio/common/streams/_reader.py index 843908fe8d..0184722e94 100644 --- a/temporalio/common/streams/_reader.py +++ b/temporalio/common/streams/_reader.py @@ -119,7 +119,11 @@ def _withdraw(self, supersession: models.Supersession) -> tuple[int, ...]: return tuple(sorted(offsets)) async def aclose(self) -> None: - """End the read early. It makes no more calls.""" + """End the read early, from the task that reads it. It makes no more calls. + + A read that waits for new Records in another task is still running, so this raises + ``RuntimeError`` there. Cancel that task instead. + """ with _backend.using(self._options): await self._items.aclose() diff --git a/temporalio/contrib/stream/__init__.py b/temporalio/contrib/stream/__init__.py index c77f870aa5..d73a80bb1c 100644 --- a/temporalio/contrib/stream/__init__.py +++ b/temporalio/contrib/stream/__init__.py @@ -124,7 +124,8 @@ def create( client: The client, with a registered Redis backend. definition: The Stream definition class. id: The Stream id, as a Workflow takes a Workflow id. - retention: How long the Records stay after the close. Defaults to the backend's. + retention: The age after which a Record is dropped, also while the Stream is open. + A closed Stream stays one Retention after its close. Defaults to the backend's. backend: The name of the Redis backend. Defaults to the client's only Redis backend, or the one with ``default=True``. diff --git a/temporalio/contrib/stream/workflow.py b/temporalio/contrib/stream/workflow.py index b9316efdd5..a68ad49d9e 100644 --- a/temporalio/contrib/stream/workflow.py +++ b/temporalio/contrib/stream/workflow.py @@ -51,7 +51,8 @@ def create_stream( Args: definition: The Stream definition class. Its name is the Stream name. - retention: How long the Records stay after the close. Defaults to the backend's. + retention: The age after which a Record is dropped, also while the Stream is open. + A closed Stream stays one Retention after its close. Defaults to the backend's. backend: The name of the Redis backend. Defaults to the Worker's. Raises: diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 4c9ed08001..70cda08bad 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -47,7 +47,8 @@ def create_stream( Args: definition: The Stream definition class. Its name is the Stream name. For one Stream per turn, for example, pass ``streams.with_name(Events, f"turn-{n}")``. - retention: How long the Records stay after the close. Defaults to the backend's. + retention: The age after which a Record is dropped, also while the Stream is open. + A closed Stream stays one Retention after its close. Defaults to the backend's. backend: The name of the backend. Defaults to the registered backend with ``default=True``. The default is part of the Workflow's code, since a replay must create on the same backend. For now a Worker serves one backend, so the name is From 68bb7b7a9eecb6e18f62cbddb7887ec1dd1e0474 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:25:42 -0700 Subject: [PATCH 67/79] Narrowed the item and failure types in the new Stream tests. --- tests/contrib/streams/test_stream_client.py | 14 +++++++++++--- tests/contrib/streams/test_workflow_module.py | 7 +++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index d9af7c5c5e..7aa422421e 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -321,7 +321,9 @@ async def test_no_wait_read_goes_past_records_of_other_topics(stream_client: Cli await stream.topic(Events.tokens).publish(*["x"] * 1000) turn = await stream.topic(Events.turns).publish(Turn("model")) turns = stream.subscribe(Events.turns, wait_new_records=False) - assert [item.data async for item in turns] == [Turn("model")] + assert [item.data async for item in turns if isinstance(item, streams.Record)] == [ + Turn("model") + ] assert turns.next_offset == turn + 1 @@ -403,7 +405,9 @@ async def test_record_that_fails_to_decode_raises_and_resumes_past_it( await anext(read) assert read.next_offset == bad + 1 resumed = reader.subscribe(offset=read.next_offset, wait_new_records=False) - assert [item.data async for item in resumed] == ["after"] + assert [ + item.data async for item in resumed if isinstance(item, streams.Record) + ] == ["after"] async def test_async_with_refuses_a_workflow_stream_before_the_block( @@ -425,7 +429,11 @@ async def test_read_keeps_no_offsets_of_client_records(stream_client: Client): stream = await redis.create(stream_client, Events, id=f"chat-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("a", "b", "c") read = stream.subscribe(wait_new_records=False) - assert [item.data async for item in read] == ["a", "b", "c"] + assert [item.data async for item in read if isinstance(item, streams.Record)] == [ + "a", + "b", + "c", + ] assert read._shown == {} diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 7fda37b25e..6580df0314 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -367,9 +367,8 @@ async def test_block_end_closes_when_no_retry_run_follows(stream_client: Client) closed = items[-1] assert isinstance(closed, streams.Closed), mode assert closed.status == streams.CloseStatus.FAILED - assert ( - closed.failure is not None and closed.failure.message == "block failed" - ) + assert isinstance(closed.failure, ApplicationError) + assert closed.failure.message == "block failed" # A close at the run's end is a command after the run's code, also on replay. replayer = Replayer( workflows=[EndsItsBlock], @@ -385,7 +384,7 @@ async def test_block_end_closes_when_no_retry_run_follows(stream_client: Client) @activity.defn async def waits_for_cancel() -> None: - waiting[activity.info().workflow_id].set() + waiting[activity.info().workflow_id or ""].set() while True: activity.heartbeat() await asyncio.sleep(0.1) From a45425e4828e2bc19571524d61706e7cac32a4f1 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:26:11 -0700 Subject: [PATCH 68/79] Relied on the server's reader floors for client Streams too. The server keeps a Workflow reader's range of a client's Stream and refuses an append that the size cap would make drop it, so a reader no longer stops at half the Retention. StreamTooLargeError names that refusal. --- temporalio/common/streams/__init__.py | 2 + temporalio/common/streams/_errors.py | 11 ++++ temporalio/common/streams/_native.py | 49 +-------------- temporalio/common/streams/_workflow_reader.py | 50 +++------------- tests/contrib/streams/test_native_server.py | 60 ++++++++++--------- 5 files changed, 56 insertions(+), 116 deletions(-) diff --git a/temporalio/common/streams/__init__.py b/temporalio/common/streams/__init__.py index 82789c4ea7..5ae504dad5 100644 --- a/temporalio/common/streams/__init__.py +++ b/temporalio/common/streams/__init__.py @@ -50,6 +50,7 @@ StreamOffsetError, StreamSeqDivergentError, StreamStorageError, + StreamTooLargeError, StreamUnsupportedError, ) from temporalio.common.streams._handle import Creating, StreamHandle @@ -93,6 +94,7 @@ "StreamOffsetError", "StreamSeqDivergentError", "StreamStorageError", + "StreamTooLargeError", "StreamUnsupportedError", "Subscription", "Superseded", diff --git a/temporalio/common/streams/_errors.py b/temporalio/common/streams/_errors.py index 79111ab864..f08474bee7 100644 --- a/temporalio/common/streams/_errors.py +++ b/temporalio/common/streams/_errors.py @@ -76,6 +76,16 @@ class StreamOffsetError(StreamError): """The offset is past the newest one plus one, or Retention removed it.""" +class StreamTooLargeError(StreamError): + """A Record or a batch is over the Store's limit, or the Stream is full. + + A native Stream keeps the Records that an open Workflow reader may read again on replay, + so an append that the Stream's size cap would make drop them fails. It passes once the + reader moves on, such as when it continues as new, which the server suggests to a reader + that keeps half of the cap. + """ + + class StreamUnsupportedError(StreamError): """The Store lacks a capability that the call needs. The message ends with its name. @@ -120,6 +130,7 @@ class ReadOnlyContextError(StreamError): "stream_fenced": StreamFencedError, "stream_storage": StreamStorageError, "stream_offset": StreamOffsetError, + "stream_too_large": StreamTooLargeError, "stream_unsupported": StreamUnsupportedError, "stream_not_owner": StreamNotOwnerError, "seq_divergent": StreamSeqDivergentError, diff --git a/temporalio/common/streams/_native.py b/temporalio/common/streams/_native.py index 20bdb6f1e5..fc08075357 100644 --- a/temporalio/common/streams/_native.py +++ b/temporalio/common/streams/_native.py @@ -8,9 +8,8 @@ from collections import OrderedDict from collections.abc import Callable, Sequence from datetime import timedelta -from typing import Any, cast +from typing import Any -import temporalio.activity import temporalio.api.streamservice.v1 as stream_pb import temporalio.bridge.client import temporalio.bridge.streams @@ -27,8 +26,6 @@ from temporalio.common.streams._backend import _ExtensionCalls from temporalio.common.streams._extension import ( ExtensionStreamBackend, - _activity_error, - _attempt_streams, _decode_page, _runtime_of, _WorkflowOutbound, @@ -62,11 +59,6 @@ _CACHE_SIZE = 256 _pages: OrderedDict[tuple[str, bytes], list[stream_pb.StreamRecord]] = OrderedDict() RANGE_GONE_TYPE = "STREAM_RANGE_GONE" -_DESCRIBE_TIMEOUT = timedelta(seconds=10) - - -def _describe_activity_name(backend: str) -> str: - return f"__temporal_stream_describe:{backend}" def _digest(records: Sequence[stream_pb.StreamRecord]) -> bytes: @@ -309,22 +301,6 @@ async def _temporal_stream_close( except temporalio.exceptions.NexusOperationError as err: raise _workflow_reader.error_of(err) from None - async def _temporal_stream_describe( - self, request: stream_pb.DescribeStreamRequest - ) -> stream_pb.DescribeStreamResponse: - try: - return await temporalio.workflow.execute_local_activity( - _describe_activity_name(self._temporal_stream_backend_name), - request, - result_type=stream_pb.DescribeStreamResponse, - start_to_close_timeout=_DESCRIBE_TIMEOUT, - ) - except ( - temporalio.exceptions.ActivityError, - temporalio.exceptions.ApplicationError, - ) as err: - raise _workflow_reader.error_of(err) from None - class NativeStreamBackend(ExtensionStreamBackend): """Carries Stream calls to the Temporal server, which keeps the Streams. @@ -446,9 +422,6 @@ async def _close_in_workflow(self, request: stream_pb.CloseStreamRequest) -> int ) return 0 - def _activities(self) -> list[Any]: - return [*super()._activities(), _describe_activity(self)] - def _configure_client( self, config: temporalio.client.ClientConfig ) -> temporalio.client.ClientConfig: @@ -457,26 +430,6 @@ def _configure_client( return config -def _describe_activity(backend: NativeStreamBackend) -> Any: - # A Workflow reader of a standalone Stream learns its Retention, since a replay reads the - # Stream again and Retention trims it. - @temporalio.activity.defn(name=_describe_activity_name(backend.backend_name)) - async def describe( - request: stream_pb.DescribeStreamRequest, - ) -> stream_pb.DescribeStreamResponse: - request.namespace = temporalio.activity.info().namespace - # The Worker's client gave the attempt its Stream runtime. - streams = _attempt_streams(backend.backend_name) - try: - return await cast(_ExtensionCalls, streams.backend).call_async( - "DescribeStream", request, stream_pb.DescribeStreamResponse - ) - except _errors.StreamError as err: - raise _activity_error(err) from None - - return describe - - def wrap( backend: Callable[[], _ExtensionCalls], converter: temporalio.converter.DataConverter, diff --git a/temporalio/common/streams/_workflow_reader.py b/temporalio/common/streams/_workflow_reader.py index 96c2aadef1..f68ec6f5c3 100644 --- a/temporalio/common/streams/_workflow_reader.py +++ b/temporalio/common/streams/_workflow_reader.py @@ -150,8 +150,6 @@ def __init__( self._create_until: datetime | None = None self._pause = _FIRST_PAUSE self._server_suggests = False - self._retention_known = False - self._range_until: datetime | None = None self._owner_ended_at: datetime | None = None self._wake: Any = None super().__init__( @@ -161,8 +159,7 @@ def __init__( self.continue_as_new_suggested = False """Whether the read ended because this run should continue as new. The server suggests it for the run's History or for the Records that the reader keeps on the - server. A reader of a client's native Stream gets it at half the Stream's Retention - after its first page, so its replay finds the Records.""" + server.""" self.owner_ended = False """Whether the read ended because the run chain that owns the Stream ended with no close, such as a terminated run on Redis.""" @@ -204,7 +201,7 @@ async def _read(self) -> None: if self._create_wait: self._create_until = now + self._create_wait # A read ends only after a page, so next_offset says where the next run goes on. - if self._paged and self._suggests_continue_as_new(now): + if self._paged and self._suggests_continue_as_new(): self.continue_as_new_suggested = True self._ended = True return @@ -219,7 +216,7 @@ async def _read(self) -> None: await self._wait_for_create() return response = page.response - await self._take(request, response) + self._take(request, response) if response.records or response.done: self._pause = _FIRST_PAUSE return @@ -230,11 +227,11 @@ async def _read(self) -> None: self.owner_ended = True self._ended = True return - if self._suggests_continue_as_new(now): + if self._suggests_continue_as_new(): self.continue_as_new_suggested = True self._ended = True return - await self._wait_for_records(now) + await self._wait_for_records() def _waiting_request(self) -> stream_pb.ReadRecordsRequest: # The next read with the backend's waits. A backend that parks reads on the server @@ -260,7 +257,7 @@ def _send_floor(self, request: stream_pb.ReadRecordsRequest) -> None: request.floor_from = floor state.sent_floors.add(key) - async def _take( + def _take( self, request: stream_pb.ReadRecordsRequest, page: stream_pb.ReadRecordsResponse ) -> None: self._paged = True @@ -291,32 +288,11 @@ async def _take( floors = _read_state().replay_floors low = request.from_offset or page.records[0].offset floors[key] = min(floors.get(key, low), low) - if not self._retention_known: - self._retention_known = True - await self._learn_retention() - - async def _learn_retention(self) -> None: - # Retention trims a client's Stream under its readers. A backend whose replay reads the - # Stream again needs the reader to move on before that. - stream = self._request.stream - if ( - stream.workflow_id - or stream.activity_id - or not self._reader._temporal_stream_parks_reads - ): - return - description = await self._reader._temporal_stream_describe( - stream_pb.DescribeStreamRequest(stream=stream) - ) - retention = description.retention.ToTimedelta() - if retention > timedelta(): - self._range_until = temporalio.workflow.now() + retention / 2 - def _suggests_continue_as_new(self, now: datetime) -> bool: + def _suggests_continue_as_new(self) -> bool: return ( self._server_suggests or temporalio.workflow.info().is_continue_as_new_suggested() - or (self._range_until is not None and now >= self._range_until) ) def _owner_gone(self, page: Page, now: datetime) -> bool: @@ -327,21 +303,13 @@ def _owner_gone(self, page: Page, now: datetime) -> bool: self._owner_ended_at = now return now - self._owner_ended_at >= _OWNER_END_GRACE - async def _wait_for_records(self, now: datetime) -> None: + async def _wait_for_records(self) -> None: if not self._can_wake(): # The Store can't wake a Workflow, or the run has the most wakes. A quiet Stream # then costs one timer and one read per pause. await self._sleep() return - request = self._waiting_request() - if self._range_until is not None: - # The wake ends in time for the read to end at half the Retention. - wait = min( - request.wait.ToTimedelta(), - max(self._range_until - now, _FIRST_PAUSE), - ) - request.wait.FromTimedelta(wait) - await self._wake_once(request) + await self._wake_once(self._waiting_request()) async def _wait_for_create(self) -> None: if not self._can_wake(): diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index cb875e7ea9..d72f27f5ca 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -7,7 +7,6 @@ import base64 import concurrent.futures import gc -import json import uuid import weakref from datetime import timedelta @@ -450,33 +449,50 @@ async def test_reads_past_the_wake_cap_pause_on_a_timer( @workflow.defn -class ReadsUntilHalfRetention: +class ReadsUntilToldToContinue: + def __init__(self) -> None: + self._end = False + + @workflow.signal + def end(self) -> None: + self._end = True + @workflow.run async def run(self, stream: streams.WorkflowStream[Events]) -> list[Any]: items = stream.subscribe(Events.tokens) - seen = [item.data async for item in items if isinstance(item, streams.Record)] - return [seen, items.continue_as_new_suggested, items.next_offset] + sizes = [ + len(item.data) async for item in items if isinstance(item, streams.Record) + ] + await workflow.wait_condition(lambda: self._end) + return [sizes, items.continue_as_new_suggested] -async def test_a_native_reader_of_a_client_stream_ends_at_half_its_retention( +async def test_a_workflow_reader_keeps_its_range_of_a_client_stream( native_client: Client, ): - # Retention trims a Stream that no Workflow owns under its readers, and a replay reads the - # Stream again. So the read ends in time for the run to continue as new. - stream = await native_client.create_stream( - Events, - id=f"native-{uuid.uuid4()}", - retention=timedelta(seconds=6), - ) - offset = await stream.topic(Events.tokens).publish("a") - async with new_worker(native_client, ReadsUntilHalfRetention) as worker: - seen, suggested, next_offset = await native_client.execute_workflow( - ReadsUntilHalfRetention.run, + # The size cap would drop what the reader's replays read again, and every task of the run + # would fail. So the server refuses the append and suggests that the reader move on. + stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") + await stream.topic(Events.tokens).publish("a") + # Two of them keep more than half of the 4 MiB stream.maxStoredBytes, three pass it. + big = "x" * 1_450_000 + async with new_worker( + native_client, ReadsUntilToldToContinue, max_cached_workflows=0 + ) as worker: + reader = await native_client.start_workflow( + ReadsUntilToldToContinue.run, streams.WorkflowStream.from_handle(stream), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) - assert seen == ["a"] and suggested and next_offset > offset + await until_parked(reader) + for _ in range(2): + await stream.topic(Events.tokens).publish(big) + with pytest.raises(streams.StreamTooLargeError) as refused: + await stream.topic(Events.tokens).publish(big) + assert refused.value.reason == "stream_too_large" + await reader.signal(ReadsUntilToldToContinue.end) + assert await reader.result() == [[1, len(big), len(big)], True] @activity.defn @@ -602,16 +618,6 @@ async def test_a_workflow_reads_a_standalone_activitys_stream(native_client: Cli task_queue=task_queue, ) assert await reader.result() == ["a", "closed done"] - # The server keeps the reader's floor on a nested Stream, so the read asks no Retention. - local_activities = [ - json.loads( - event.marker_recorded_event_attributes.details["data"].payloads[0].data - )["activity_type"] - for event in (await reader.fetch_history()).events - if event.HasField("marker_recorded_event_attributes") - ] - assert local_activities - assert _native._describe_activity_name("native") not in local_activities @workflow.defn From 2ef3bcf03422166005d0a752ec1ca857e80d6e1e Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:29:16 -0700 Subject: [PATCH 69/79] Added StreamTooLargeError for the reason stream_too_large. The Brook crate and the Jetty server both answer a Record or a batch over the Store's limit with it, so both stacks raise one class. --- temporalio/common/streams/__init__.py | 2 ++ temporalio/common/streams/_errors.py | 5 +++++ tests/contrib/streams/test_stream_client.py | 9 ++++++++ tests/contrib/streams/test_workflow_module.py | 22 +++++++++++++++++++ 4 files changed, 38 insertions(+) diff --git a/temporalio/common/streams/__init__.py b/temporalio/common/streams/__init__.py index 19b6e56e49..aca56388ca 100644 --- a/temporalio/common/streams/__init__.py +++ b/temporalio/common/streams/__init__.py @@ -45,6 +45,7 @@ StreamOffsetError, StreamSeqDivergentError, StreamStorageError, + StreamTooLargeError, StreamUnsupportedError, ) from temporalio.common.streams._handle import Creating, StreamHandle @@ -87,6 +88,7 @@ "StreamOffsetError", "StreamSeqDivergentError", "StreamStorageError", + "StreamTooLargeError", "StreamUnsupportedError", "Subscription", "Superseded", diff --git a/temporalio/common/streams/_errors.py b/temporalio/common/streams/_errors.py index 17dfe40323..16b67c9c98 100644 --- a/temporalio/common/streams/_errors.py +++ b/temporalio/common/streams/_errors.py @@ -78,6 +78,10 @@ class StreamOffsetError(StreamError): """The offset is below zero or past the newest one plus one.""" +class StreamTooLargeError(StreamError): + """A Record or a batch is over the Store's limit.""" + + class StreamUnsupportedError(StreamError): """The Store lacks a capability that the call needs. The message ends with its name.""" @@ -128,6 +132,7 @@ class ReadOnlyContextError(StreamError): "stream_fenced": StreamFencedError, "stream_storage": StreamStorageError, "stream_offset": StreamOffsetError, + "stream_too_large": StreamTooLargeError, "stream_unsupported": StreamUnsupportedError, "stream_not_owner": StreamNotOwnerError, "seq_divergent": StreamSeqDivergentError, diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 7aa422421e..5a3371f8d6 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -424,6 +424,15 @@ async def test_async_with_refuses_a_workflow_stream_before_the_block( assert not ran +async def test_record_over_the_store_limit_raises_too_large(stream_client: Client): + # The client's writer splits a batch at the Store's limit, so only a Record can be over. + stream = await redis.create(stream_client, Events, id=f"large-{uuid.uuid4()}") + with pytest.raises(streams.StreamTooLargeError) as refused: + await stream.topic(Events.tokens).publish("x" * 2_000_000) + assert refused.value.reason == "stream_too_large" and refused.value.non_retryable + assert await stream.get_offset() == 0 + + async def test_read_keeps_no_offsets_of_client_records(stream_client: Client): # Nothing withdraws a client's Records, so a long UI read of them keeps no state. stream = await redis.create(stream_client, Events, id=f"chat-{uuid.uuid4()}") diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 6580df0314..081e9af974 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -841,6 +841,28 @@ async def test_merge_that_the_codec_grows_past_the_batch_limit(stream_client: Cl assert offsets == [written[2].offset, written[5].offset] +@workflow.defn +class PublishesTooMuch: + @workflow.run + async def run(self) -> str: + tokens = (await workflow.create_stream(Events)).topic(Events.tokens) + try: + await tokens.publish(*["x" * 1_400_000] * 4) + except streams.StreamTooLargeError as err: + return err.reason + return "written" + + +async def test_batch_over_the_store_limit_raises_too_large(stream_client: Client): + # Each Record is under the Store's Record limit, and the publish's batch is over 4 MiB. + async with new_worker(stream_client, PublishesTooMuch) as worker: + handle = await stream_client.start_workflow( + PublishesTooMuch.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue + ) + assert await handle.result() == "stream_too_large" + assert data(await read(handle)) == [] + + @workflow.defn class CreatesOnTheDefault: @workflow.run From 870e8414ab48fc49d154f3ac1ffd3e7c3d834fb8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:41:09 -0700 Subject: [PATCH 70/79] Moved the streams crate to k3 abd8955b and sdk-core to c10 2de24aef. The vendored extension.proto and stream-service.wit changed in comments only, so only their docstrings in the generated code moved. --- scripts/_proto/temporal/sdk/streams/v1/extension.proto | 4 ++-- scripts/_wit/stream-service.wit | 2 +- temporalio/bridge/Cargo.lock | 2 +- temporalio/bridge/Cargo.toml | 2 +- temporalio/bridge/sdk-core | 2 +- temporalio/common/streams/proto/v1/extension_pb2.pyi | 4 ++-- temporalio/common/streams/system/operations/create_stream.py | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/_proto/temporal/sdk/streams/v1/extension.proto b/scripts/_proto/temporal/sdk/streams/v1/extension.proto index 1c4081738a..d405d8a73f 100644 --- a/scripts/_proto/temporal/sdk/streams/v1/extension.proto +++ b/scripts/_proto/temporal/sdk/streams/v1/extension.proto @@ -18,7 +18,7 @@ message NativeStreamConfig { google.protobuf.Duration retention = 2; } -// The settings of the Redis Stream extension. Lang builds it from the plugin options. +// The settings of the Redis Stream extension. Lang builds it from the backend options. message RedisStreamConfig { // One URL, or the seed URLs of a cluster. Credentials and TLS come from the URL. repeated string urls = 1; @@ -120,7 +120,7 @@ message FreeWriterRequest { message FreeWriterResponse { } -// Method `CloseWriters`, asynchronous. Drops every open writer, for example when the plugin's +// Method `CloseWriters`, asynchronous. Drops every open writer, for example when the backend's // client or Worker shuts down. message CloseWritersRequest { } diff --git a/scripts/_wit/stream-service.wit b/scripts/_wit/stream-service.wit index 4a74bcd0ba..fa655f0099 100644 --- a/scripts/_wit/stream-service.wit +++ b/scripts/_wit/stream-service.wit @@ -171,7 +171,7 @@ interface stream-service { /// @nexus.doc "Create a Stream idempotently. ALREADY_EXISTS for another Retention." /// @nexus.proto "temporal.api.streamservice.v1.CreateStreamRequest" record create-stream-request { - /// @nexus.doc "Required from Workflow code. Empty elsewhere asks the service for an id." + /// @nexus.doc "Required on Redis and memory. On native, empty asks the server for an id." stream-id: string, /// @nexus.doc "The age after which a Record is dropped. Empty is the Store default." retention: option, diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 396244f8ab..e47f16e4cd 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -2938,7 +2938,7 @@ dependencies = [ [[package]] name = "temporalio-streams" version = "0.1.0" -source = "git+https://github.com/moetemp/sdk-streams?rev=a0a287ea97c3478d632b34239f341b9f202a0bc7#a0a287ea97c3478d632b34239f341b9f202a0bc7" +source = "git+https://github.com/moetemp/sdk-streams?rev=abd8955bc5d1314e6126daa0c0b91f0d694b2426#abd8955bc5d1314e6126daa0c0b91f0d694b2426" dependencies = [ "async-trait", "aws-lc-rs", diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index 5582639bdc..2abf8c8c2b 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -94,7 +94,7 @@ url = "2.2" # crates, so the bridge sees one type of each. [dependencies.temporalio-streams] git = "https://github.com/moetemp/sdk-streams" -rev = "a0a287ea97c3478d632b34239f341b9f202a0bc7" +rev = "abd8955bc5d1314e6126daa0c0b91f0d694b2426" default-features = false optional = true diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 191c8b8b64..2de24aeff9 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 191c8b8b64e08b4f81d17d3204b893186295d023 +Subproject commit 2de24aeff98dfef14247cf8f96b02dd1a5092a5c diff --git a/temporalio/common/streams/proto/v1/extension_pb2.pyi b/temporalio/common/streams/proto/v1/extension_pb2.pyi index 596afc199d..07af95096e 100644 --- a/temporalio/common/streams/proto/v1/extension_pb2.pyi +++ b/temporalio/common/streams/proto/v1/extension_pb2.pyi @@ -59,7 +59,7 @@ class NativeStreamConfig(google.protobuf.message.Message): global___NativeStreamConfig = NativeStreamConfig class RedisStreamConfig(google.protobuf.message.Message): - """The settings of the Redis Stream extension. Lang builds it from the plugin options.""" + """The settings of the Redis Stream extension. Lang builds it from the backend options.""" DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -427,7 +427,7 @@ class FreeWriterResponse(google.protobuf.message.Message): global___FreeWriterResponse = FreeWriterResponse class CloseWritersRequest(google.protobuf.message.Message): - """Method `CloseWriters`, asynchronous. Drops every open writer, for example when the plugin's + """Method `CloseWriters`, asynchronous. Drops every open writer, for example when the backend's client or Worker shuts down. """ diff --git a/temporalio/common/streams/system/operations/create_stream.py b/temporalio/common/streams/system/operations/create_stream.py index 947b5d3608..10da759aad 100644 --- a/temporalio/common/streams/system/operations/create_stream.py +++ b/temporalio/common/streams/system/operations/create_stream.py @@ -53,7 +53,7 @@ async def create_stream( ) -> CreateStreamResult: """ Args: - stream_id: Required from Workflow code. Empty elsewhere asks the service for an + stream_id: Required on Redis and memory. On native, empty asks the server for an id. retention: The age after which a Record is dropped. Empty is the Store default. withdraw_run: Retry run only. The previous run, whose Records this withdraws. From 2e2ffe5df96f20bb7b7eb7dcd3adaaf6a18e5690 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 01:57:50 -0700 Subject: [PATCH 71/79] Pinned crate s23 ee6d1e51 and Core c8 f7826366. The crate refuses a Local Activity's close of a Workflow's Stream, so a close by id from one is refused too. The vendored proto and WIT differ in comments only. --- .../_proto/temporal/sdk/streams/v1/extension.proto | 4 ++-- scripts/_wit/stream-service.wit | 2 +- temporalio/bridge/Cargo.lock | 2 +- temporalio/bridge/Cargo.toml | 2 +- temporalio/bridge/sdk-core | 2 +- .../common/streams/system/operations/create_stream.py | 2 +- tests/contrib/streams/test_activity_handles.py | 11 +++++++++-- 7 files changed, 16 insertions(+), 9 deletions(-) diff --git a/scripts/_proto/temporal/sdk/streams/v1/extension.proto b/scripts/_proto/temporal/sdk/streams/v1/extension.proto index c61590419e..88d0a7cf90 100644 --- a/scripts/_proto/temporal/sdk/streams/v1/extension.proto +++ b/scripts/_proto/temporal/sdk/streams/v1/extension.proto @@ -8,7 +8,7 @@ package temporal.sdk.streams.v1; import "google/protobuf/duration.proto"; import "temporal/api/streamservice/v1/stream_service.proto"; -// The settings of the Redis Stream extension. Lang builds it from the plugin options. +// The settings of the Redis Stream extension. Lang builds it from the backend options. message RedisStreamConfig { // One URL, or the seed URLs of a cluster. Credentials and TLS come from the URL. repeated string urls = 1; @@ -96,7 +96,7 @@ message FreeWriterRequest { message FreeWriterResponse { } -// Method `CloseWriters`, asynchronous. Drops every open writer, for example when the plugin's +// Method `CloseWriters`, asynchronous. Drops every open writer, for example when the backend's // client or Worker shuts down. message CloseWritersRequest { } diff --git a/scripts/_wit/stream-service.wit b/scripts/_wit/stream-service.wit index 6c307a43da..0222790f7c 100644 --- a/scripts/_wit/stream-service.wit +++ b/scripts/_wit/stream-service.wit @@ -172,7 +172,7 @@ interface stream-service { /// @nexus.doc "Create a Stream idempotently. ALREADY_EXISTS for another Retention." /// @nexus.proto "temporal.api.streamservice.v1.CreateStreamRequest" record create-stream-request { - /// @nexus.doc "Required from Workflow code. Empty elsewhere asks the service for an id." + /// @nexus.doc "Required on Redis and memory. On native, empty asks the server for an id." stream-id: string, /// @nexus.doc "The age after which a Record is dropped. Empty is the Store default." retention: option, diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index d09168f0e1..6591afb5a3 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -2938,7 +2938,7 @@ dependencies = [ [[package]] name = "temporalio-streams" version = "0.1.0" -source = "git+https://github.com/moetemp/sdk-streams?rev=2866bc971b0419e8861108d4b392319dccd0472c#2866bc971b0419e8861108d4b392319dccd0472c" +source = "git+https://github.com/moetemp/sdk-streams?rev=ee6d1e51d91c96f0c661fb59ad0957a97a579cbe#ee6d1e51d91c96f0c661fb59ad0957a97a579cbe" dependencies = [ "async-trait", "aws-lc-rs", diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index 683d6af213..f90fbef389 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -94,7 +94,7 @@ url = "2.2" # crates, so the bridge sees one type of each. [dependencies.temporalio-streams] git = "https://github.com/moetemp/sdk-streams" -rev = "2866bc971b0419e8861108d4b392319dccd0472c" +rev = "ee6d1e51d91c96f0c661fb59ad0957a97a579cbe" default-features = false optional = true diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 320e725ff5..f782636659 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 320e725ff5e0bfd838089e293dba48a8f46954b3 +Subproject commit f782636659f25f2765260c9d061c9165afa34b3a diff --git a/temporalio/common/streams/system/operations/create_stream.py b/temporalio/common/streams/system/operations/create_stream.py index 947b5d3608..10da759aad 100644 --- a/temporalio/common/streams/system/operations/create_stream.py +++ b/temporalio/common/streams/system/operations/create_stream.py @@ -53,7 +53,7 @@ async def create_stream( ) -> CreateStreamResult: """ Args: - stream_id: Required from Workflow code. Empty elsewhere asks the service for an + stream_id: Required on Redis and memory. On native, empty asks the server for an id. retention: The age after which a Record is dropped. Empty is the Store default. withdraw_run: Retry run only. The previous run, whose Records this withdraws. diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index 96d7803642..c7c7820c14 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -350,8 +350,14 @@ async def run(self) -> list[str]: streams.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) + # A handle with only the id names no Workflow, so the Store refuses it. + by_id = await workflow.execute_local_activity( + close_it, + streams.ActivityStream(id=stream.id), + start_to_close_timeout=timedelta(seconds=30), + ) await stream.close(result="closed by the run") - return [backend, owner, local] + return [backend, owner, local, by_id] async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): @@ -366,9 +372,10 @@ async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): "config", "stream_not_owner", "stream_not_owner", + "stream_not_owner", ] items = await read(handle) - assert [r.data for r in records(items)] == ["last", "last"] + assert [r.data for r in records(items)] == ["last", "last", "last"] assert isinstance(items[-1], streams.Closed) assert items[-1].result == "closed by the run" From 2ff42e1f9eb25cf26321476d5923bb88d2f71e81 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 02:02:16 -0700 Subject: [PATCH 72/79] Moved the Streams API into `temporalio.common`. The code is in the private temporalio.common._streams, and temporalio.common loads its names on first use, since the Stream modules import temporalio.client and temporalio.workflow. --- changelog/added/babbling-brook.md | 2 +- pyproject.toml | 18 +- scripts/gen_protos.py | 8 +- scripts/gen_stream_service.py | 10 +- temporalio/client/_client.py | 8 +- temporalio/client/_workflow.py | 4 +- temporalio/common/__init__.py | 201 +++++++++++++++++- temporalio/common/_streams/__init__.py | 86 ++++++++ .../common/{streams => _streams}/_access.py | 16 +- .../common/{streams => _streams}/_activity.py | 4 +- .../{streams => _streams}/_activity_stream.py | 12 +- .../common/{streams => _streams}/_backend.py | 10 +- .../common/{streams => _streams}/_buffered.py | 10 +- .../common/{streams => _streams}/_convert.py | 0 .../common/{streams => _streams}/_defn.py | 6 +- .../common/{streams => _streams}/_errors.py | 4 +- .../{streams => _streams}/_extension.py | 18 +- .../common/{streams => _streams}/_handle.py | 4 +- .../common/{streams => _streams}/_ids.py | 0 .../common/{streams => _streams}/_items.py | 2 +- .../common/{streams => _streams}/_plugin.py | 4 +- .../common/{streams => _streams}/_reader.py | 8 +- .../{streams => _streams}/_stream_client.py | 22 +- .../common/{streams => _streams}/_workflow.py | 10 +- .../common/{streams => _streams}/_writer.py | 8 +- .../common/{streams => _streams}/_writes.py | 8 +- .../{streams => _streams}/proto/__init__.py | 0 .../proto/v1/__init__.py | 0 .../proto/v1/extension_pb2.py | 24 +-- .../proto/v1/extension_pb2.pyi | 0 .../{streams => _streams}/system/__init__.py | 0 .../system/_resources/__init__.py | 0 .../system/_resources/stream.py | 0 .../system/_system_nexus_interceptor.py | 0 .../{streams => _streams}/system/client.py | 0 .../{streams => _streams}/system/models.py | 0 .../system/operations/__init__.py | 0 .../system/operations/create_stream.py | 0 .../{streams => _streams}/system/services.py | 0 .../writer_system/__init__.py | 0 .../writer_system/client.py | 0 .../writer_system/models.py | 131 ++++++------ temporalio/common/streams/__init__.py | 104 --------- temporalio/contrib/stream/__init__.py | 12 +- temporalio/contrib/stream/testing.py | 4 +- temporalio/contrib/stream/workflow.py | 8 +- temporalio/testing/_workflow.py | 2 +- temporalio/worker/_replayer.py | 4 +- temporalio/worker/_worker.py | 2 +- temporalio/workflow/_streams.py | 18 +- tests/contrib/streams/conftest.py | 2 +- .../examples/activity_streams_tokens.py | 21 +- .../streams/examples/agent_publishes_turns.py | 19 +- .../streams/examples/client_creates_stream.py | 13 +- .../streams/examples/resume_from_offset.py | 21 +- .../contrib/streams/examples/test_examples.py | 6 +- .../streams/examples/turn_scoped_stream.py | 29 ++- .../streams/examples/ui_backend_reads.py | 21 +- .../contrib/streams/examples/worker_setup.py | 9 +- tests/contrib/streams/fake_extension.py | 6 +- .../contrib/streams/test_activity_handles.py | 154 +++++++------- tests/contrib/streams/test_bindings.py | 2 +- tests/contrib/streams/test_facade.py | 53 ++--- tests/contrib/streams/test_handles.py | 65 ++++-- tests/contrib/streams/test_pydantic.py | 19 +- tests/contrib/streams/test_redis.py | 134 ++++++------ tests/contrib/streams/test_stream_client.py | 147 +++++++------ tests/contrib/streams/test_vectors.py | 4 +- tests/contrib/streams/test_workflow_module.py | 100 ++++----- 69 files changed, 911 insertions(+), 706 deletions(-) create mode 100644 temporalio/common/_streams/__init__.py rename temporalio/common/{streams => _streams}/_access.py (92%) rename temporalio/common/{streams => _streams}/_activity.py (95%) rename temporalio/common/{streams => _streams}/_activity_stream.py (96%) rename temporalio/common/{streams => _streams}/_backend.py (96%) rename temporalio/common/{streams => _streams}/_buffered.py (96%) rename temporalio/common/{streams => _streams}/_convert.py (100%) rename temporalio/common/{streams => _streams}/_defn.py (96%) rename temporalio/common/{streams => _streams}/_errors.py (97%) rename temporalio/common/{streams => _streams}/_extension.py (97%) rename temporalio/common/{streams => _streams}/_handle.py (99%) rename temporalio/common/{streams => _streams}/_ids.py (100%) rename temporalio/common/{streams => _streams}/_items.py (99%) rename temporalio/common/{streams => _streams}/_plugin.py (98%) rename temporalio/common/{streams => _streams}/_reader.py (97%) rename temporalio/common/{streams => _streams}/_stream_client.py (96%) rename temporalio/common/{streams => _streams}/_workflow.py (98%) rename temporalio/common/{streams => _streams}/_writer.py (94%) rename temporalio/common/{streams => _streams}/_writes.py (97%) rename temporalio/common/{streams => _streams}/proto/__init__.py (100%) rename temporalio/common/{streams => _streams}/proto/v1/__init__.py (100%) rename temporalio/common/{streams => _streams}/proto/v1/extension_pb2.py (90%) rename temporalio/common/{streams => _streams}/proto/v1/extension_pb2.pyi (100%) rename temporalio/common/{streams => _streams}/system/__init__.py (100%) rename temporalio/common/{streams => _streams}/system/_resources/__init__.py (100%) rename temporalio/common/{streams => _streams}/system/_resources/stream.py (100%) rename temporalio/common/{streams => _streams}/system/_system_nexus_interceptor.py (100%) rename temporalio/common/{streams => _streams}/system/client.py (100%) rename temporalio/common/{streams => _streams}/system/models.py (100%) rename temporalio/common/{streams => _streams}/system/operations/__init__.py (100%) rename temporalio/common/{streams => _streams}/system/operations/create_stream.py (100%) rename temporalio/common/{streams => _streams}/system/services.py (100%) rename temporalio/common/{streams => _streams}/writer_system/__init__.py (100%) rename temporalio/common/{streams => _streams}/writer_system/client.py (100%) rename temporalio/common/{streams => _streams}/writer_system/models.py (82%) delete mode 100644 temporalio/common/streams/__init__.py diff --git a/changelog/added/babbling-brook.md b/changelog/added/babbling-brook.md index 9d6abdeb19..66af4b2540 100644 --- a/changelog/added/babbling-brook.md +++ b/changelog/added/babbling-brook.md @@ -1,3 +1,3 @@ -Add experimental Streams in `temporalio.common.streams`: ordered logs of Records that Workflows, Activities and clients write, and that clients and Activities read. Register a backend with `stream_backends=` on `Client.connect`, `Replayer` or `WorkflowEnvironment`, then use `workflow.create_stream`, `Client.stream` or `WorkflowHandle.stream`. +Add experimental Streams to `temporalio.common`: ordered logs of Records that Workflows, Activities and clients write, and that clients and Activities read. Register a backend with `stream_backends=` on `Client.connect`, `Replayer` or `WorkflowEnvironment`, then use `workflow.create_stream`, `Client.stream` or `WorkflowHandle.stream`. Add a Redis Stream backend in `temporalio.contrib.stream`, and an in-memory one for tests in `temporalio.contrib.stream.testing`. The Redis backend needs a build with the bridge's `streams` feature, which the official builds have. Add `activity.Info.workflow_task`, the Workflow Task attempt in which a Worker dispatched a Local Activity. diff --git a/pyproject.toml b/pyproject.toml index 8b05971325..9f5333e595 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -186,9 +186,9 @@ exclude = [ 'temporalio/api', 'temporalio/bridge/proto', 'temporalio/nexus/system/workflow_service', - 'temporalio/common/streams/proto', - 'temporalio/common/streams/system', - 'temporalio/common/streams/writer_system', + 'temporalio/common/_streams/proto', + 'temporalio/common/_streams/system', + 'temporalio/common/_streams/writer_system', ] [[tool.mypy.overrides]] @@ -197,9 +197,9 @@ ignore_errors = true [[tool.mypy.overrides]] module = [ - "temporalio.common.streams.proto.*", - "temporalio.common.streams.system.*", - "temporalio.common.streams.writer_system.*", + "temporalio.common._streams.proto.*", + "temporalio.common._streams.system.*", + "temporalio.common._streams.writer_system.*", ] ignore_errors = true @@ -239,8 +239,8 @@ privacy = [ "HIDDEN:temporalio.worker.workflow_instance", "HIDDEN:temporalio.worker.workflow_sandbox.importer", "HIDDEN:temporalio.worker.workflow_sandbox.in_sandbox", - "HIDDEN:temporalio.common.streams.system.client", - "HIDDEN:temporalio.common.streams.writer_system.client", + "HIDDEN:temporalio.common._streams.system.client", + "HIDDEN:temporalio.common._streams.writer_system.client", "HIDDEN:**.*_pb2*", ] project-name = "Temporal Python" @@ -274,7 +274,7 @@ exclude = [ "temporalio/api", "temporalio/bridge/proto", "temporalio/bridge/_visitor.py", - "temporalio/common/streams/proto", + "temporalio/common/_streams/proto", "tests/worker/workflow_sandbox/testmodules/proto", ] diff --git a/scripts/gen_protos.py b/scripts/gen_protos.py index a3330d8e87..adb42a3a5a 100644 --- a/scripts/gen_protos.py +++ b/scripts/gen_protos.py @@ -33,7 +33,7 @@ api_out_dir = base_dir / "temporalio" / "api" sdk_out_dir = base_dir / "temporalio" / "bridge" / "proto" -streams_out_dir = base_dir / "temporalio" / "common" / "streams" / "proto" +streams_out_dir = base_dir / "temporalio" / "common" / "_streams" / "proto" py_fixes = [ partial(re.compile(r"from temporal\.api\.").sub, r"from temporalio.api."), @@ -53,11 +53,11 @@ ), partial( re.compile(r"from temporal\.sdk\.streams\.").sub, - r"from temporalio.common.streams.proto.", + r"from temporalio.common._streams.proto.", ), partial( re.compile(r"'__module__' : 'temporal\.sdk\.streams\.").sub, - r"'__module__' : 'temporalio.common.streams.proto.", + r"'__module__' : 'temporalio.common._streams.proto.", ), partial( re.compile(r"'__module__' : 'temporal\.api\.").sub, @@ -82,7 +82,7 @@ partial(re.compile(r"temporal\.sdk\.core\.").sub, r"temporalio.bridge.proto."), partial( re.compile(r"temporal\.sdk\.streams\.").sub, - r"temporalio.common.streams.proto.", + r"temporalio.common._streams.proto.", ), ] diff --git a/scripts/gen_stream_service.py b/scripts/gen_stream_service.py index afe4f09347..f990c63948 100644 --- a/scripts/gen_stream_service.py +++ b/scripts/gen_stream_service.py @@ -1,4 +1,4 @@ -"""Generates the Stream service bindings of temporalio.common.streams. +"""Generates the Stream service bindings of the Streams API in temporalio.common. The WIT and the protos are copies of the temporalio-streams crate's, in scripts/_wit and scripts/_proto. Run scripts/gen_protos.py first for the proto modules. No nexgen release has the @@ -30,9 +30,9 @@ / "v1" / "stream_service.proto" ) -output_dir = base_dir / "temporalio" / "common" / "streams" / "system" +output_dir = base_dir / "temporalio" / "common" / "_streams" / "system" # The writer is client-only, so it has a caller and its resource class, and no Workflow code. -writer_output_dir = base_dir / "temporalio" / "common" / "streams" / "writer_system" +writer_output_dir = base_dir / "temporalio" / "common" / "_streams" / "writer_system" NEX_GEN_REPOSITORY = "https://github.com/moetemp/nexgen" # The head of moe/brook/v3/n8-stream-resource. A branch moves, so the pin is the commit. @@ -81,14 +81,14 @@ def generate_stream_service() -> None: raise RuntimeError(f"missing generator input: {path}") with tempfile.TemporaryDirectory(dir=base_dir) as temp_dir: descriptor_path = Path(temp_dir) / "stream_service.bin" - # The extension messages live in temporalio.common.streams.proto, so the descriptor + # The extension messages live in temporalio.common._streams.proto, so the descriptor # names their file by that module path. extension_root = Path(temp_dir) / "extension" extension_path = ( extension_root / "temporalio" / "common" - / "streams" + / "_streams" / "proto" / "v1" / "extension.proto" diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index ac51fbb48e..66937dfe8b 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -111,7 +111,7 @@ ) if TYPE_CHECKING: - from temporalio.common.streams import ClientStream, StreamBackend, StreamHandle + from temporalio.common import ClientStream, StreamBackend, StreamHandle from ._interceptor import Interceptor from ._plugin import Plugin @@ -234,7 +234,7 @@ async def connect( stream_backends: The Stream backends of this client and its Workers, such as :py:class:`temporalio.contrib.stream.RedisStreamBackend`. Names are unique. With more than one, exactly one has ``default=True``. See - :py:class:`temporalio.common.streams.StreamBackend`. + :py:class:`temporalio.common.StreamBackend`. .. warning:: This API is experimental. @@ -312,7 +312,7 @@ def __init__( config = plugin.configure_client(config) if config.get("stream_backends"): # Only a client with Stream backends loads the Streams package. - from temporalio.common.streams import _plugin as _stream_plugin + from temporalio.common._streams import _plugin as _stream_plugin config = _stream_plugin.configure_client(config) @@ -945,7 +945,7 @@ def stream( TypeError: Neither or both of ``id`` and ``stream`` are given. ValueError: The id is empty, or ``backend`` is not the handle's backend. """ - from temporalio.common.streams import _stream_client + from temporalio.common._streams import _stream_client return _stream_client.client_stream( self, definition, id=id, stream=stream, backend=backend diff --git a/temporalio/client/_workflow.py b/temporalio/client/_workflow.py index 82735f0761..e1a1d2a3aa 100644 --- a/temporalio/client/_workflow.py +++ b/temporalio/client/_workflow.py @@ -79,7 +79,7 @@ ) if TYPE_CHECKING: - from temporalio.common.streams import ClientStream + from temporalio.common import ClientStream from ._client import Client from ._interceptor import ListWorkflowsInput @@ -213,7 +213,7 @@ def stream( StreamConfigError: The client has no backend of that name, or none has ``default=True`` for a call with no backend. """ - from temporalio.common.streams import _stream_client + from temporalio.common._streams import _stream_client return _stream_client.workflow_stream(self, definition, backend=backend) diff --git a/temporalio/common/__init__.py b/temporalio/common/__init__.py index 578d714c6a..f0cd0d72ed 100644 --- a/temporalio/common/__init__.py +++ b/temporalio/common/__init__.py @@ -1,4 +1,24 @@ -"""Common code used in the Temporal SDK.""" +"""Common code used in the Temporal SDK. + +It also has the Streams API: Streams of Records that Workflows, Activities and clients write +and read. A Stream definition, a class with ``@common.defn``, gives the Stream name and the +type of each Topic. Workflow code creates a Stream with +:py:func:`temporalio.workflow.create_stream`. A client gets a Workflow's Stream with +:py:meth:`temporalio.client.WorkflowHandle.stream`, and any Stream with +:py:meth:`temporalio.client.Client.stream`. A Stream is data: a Workflow passes it to an +Activity, a child Workflow or the next run as an argument, and the parameter's type gives the +handle class of that context, such as :py:class:`ActivityStream`. + +A :py:class:`StreamBackend` carries the calls and registers a backend name, such as +:py:class:`temporalio.contrib.stream.RedisStreamBackend` (``redis``) and +:py:class:`temporalio.contrib.stream.testing.MemoryStreamBackend` (``memory``). Register +backends with ``Client.connect(..., stream_backends=[...])``. A handle carries the name of its +backend. With more than one registered backend, exactly one has ``default=True``, the one +that a create with no ``backend=`` uses. For now a Worker serves one backend. + +.. warning:: + The Streams API is experimental. +""" from __future__ import annotations @@ -13,6 +33,7 @@ from datetime import datetime, timedelta from enum import IntEnum from typing import ( + TYPE_CHECKING, Any, ClassVar, Generic, @@ -32,6 +53,130 @@ import temporalio.api.workflow.v1 import temporalio.types +if TYPE_CHECKING: + from temporalio.common._streams._activity_stream import ( + ActivityStream, + ActivityTopic, + SyncActivityStream, + SyncActivityTopic, + SyncSubscription, + ) + from temporalio.common._streams._buffered import BufferedPublisher + from temporalio.common._streams._defn import ( + StreamDefinition, + TopicDefinition, + defn, + topic, + with_name, + ) + from temporalio.common._streams._errors import ( + ReadOnlyContextError, + StreamClosedError, + StreamConfigError, + StreamError, + StreamExpiredError, + StreamFencedError, + StreamNotFoundError, + StreamNotOwnerError, + StreamOffsetError, + StreamSeqDivergentError, + StreamStorageError, + StreamTooLargeError, + StreamUnsupportedError, + ) + from temporalio.common._streams._handle import Creating, StreamHandle + from temporalio.common._streams._items import ( + END, + Closed, + CloseStatus, + End, + Record, + Superseded, + ) + from temporalio.common._streams._plugin import StreamBackend + from temporalio.common._streams._reader import Subscription + from temporalio.common._streams._stream_client import ClientStream, ClientTopic + from temporalio.common._streams._workflow import WorkflowStream, WorkflowTopic + +__all__ = [ + "ActivityIDConflictPolicy", + "ActivityIDReusePolicy", + "ActivityStream", + "ActivityTopic", + "AutoUpgradeVersioningOverride", + "BufferedPublisher", + "ClientStream", + "ClientTopic", + "CloseStatus", + "Closed", + "Creating", + "END", + "End", + "HeaderCodecBehavior", + "MetricAttributes", + "MetricCommon", + "MetricCounter", + "MetricGauge", + "MetricGaugeFloat", + "MetricHistogram", + "MetricHistogramFloat", + "MetricHistogramTimedelta", + "MetricMeter", + "NexusOperationCancellationState", + "NexusOperationExecutionStatus", + "NexusOperationIDConflictPolicy", + "NexusOperationIDReusePolicy", + "OneTimeVersioningOverride", + "PendingNexusOperationExecutionState", + "PinnedVersioningOverride", + "Priority", + "QueryRejectCondition", + "RawValue", + "ReadOnlyContextError", + "Record", + "RetryPolicy", + "SearchAttributeIndexedValueType", + "SearchAttributeKey", + "SearchAttributePair", + "SearchAttributeUpdate", + "SearchAttributeValue", + "SearchAttributeValueType", + "SearchAttributeValues", + "SearchAttributes", + "StreamBackend", + "StreamClosedError", + "StreamConfigError", + "StreamDefinition", + "StreamError", + "StreamExpiredError", + "StreamFencedError", + "StreamHandle", + "StreamNotFoundError", + "StreamNotOwnerError", + "StreamOffsetError", + "StreamSeqDivergentError", + "StreamStorageError", + "StreamTooLargeError", + "StreamUnsupportedError", + "Subscription", + "Superseded", + "SyncActivityStream", + "SyncActivityTopic", + "SyncSubscription", + "TopicDefinition", + "TypedSearchAttributes", + "VersioningBehavior", + "VersioningOverride", + "WorkerDeploymentVersion", + "WorkflowIDConflictPolicy", + "WorkflowIDReusePolicy", + "WorkflowStream", + "WorkflowTopic", + "defn", + "topic", + "with_name", +] + @dataclass class RetryPolicy: @@ -1482,3 +1627,57 @@ def wait_sync(self, timeout: float | None = None) -> None: if not self.thread_event: raise RuntimeError("Missing event") self.thread_event.wait(timeout) + + +# The Stream modules import temporalio.client and temporalio.workflow, which import this module, +# so the Stream names load on first use. +_STREAM_NAMES = frozenset( + [ + "ActivityStream", + "ActivityTopic", + "BufferedPublisher", + "ClientStream", + "ClientTopic", + "CloseStatus", + "Closed", + "Creating", + "END", + "End", + "ReadOnlyContextError", + "Record", + "StreamBackend", + "StreamClosedError", + "StreamConfigError", + "StreamDefinition", + "StreamError", + "StreamExpiredError", + "StreamFencedError", + "StreamHandle", + "StreamNotFoundError", + "StreamNotOwnerError", + "StreamOffsetError", + "StreamSeqDivergentError", + "StreamStorageError", + "StreamTooLargeError", + "StreamUnsupportedError", + "Subscription", + "Superseded", + "SyncActivityStream", + "SyncActivityTopic", + "SyncSubscription", + "TopicDefinition", + "WorkflowStream", + "WorkflowTopic", + "defn", + "topic", + "with_name", + ] +) + + +def __getattr__(name: str) -> Any: + if name in _STREAM_NAMES: + import temporalio.common._streams + + return getattr(temporalio.common._streams, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/temporalio/common/_streams/__init__.py b/temporalio/common/_streams/__init__.py new file mode 100644 index 0000000000..a4001dbd31 --- /dev/null +++ b/temporalio/common/_streams/__init__.py @@ -0,0 +1,86 @@ +"""The Streams API, which :py:mod:`temporalio.common` exports.""" + +from temporalio.common._streams._activity_stream import ( + ActivityStream, + ActivityTopic, + SyncActivityStream, + SyncActivityTopic, + SyncSubscription, +) +from temporalio.common._streams._buffered import BufferedPublisher +from temporalio.common._streams._defn import ( + StreamDefinition, + TopicDefinition, + defn, + topic, + with_name, +) +from temporalio.common._streams._errors import ( + ReadOnlyContextError, + StreamClosedError, + StreamConfigError, + StreamError, + StreamExpiredError, + StreamFencedError, + StreamNotFoundError, + StreamNotOwnerError, + StreamOffsetError, + StreamSeqDivergentError, + StreamStorageError, + StreamTooLargeError, + StreamUnsupportedError, +) +from temporalio.common._streams._handle import Creating, StreamHandle +from temporalio.common._streams._items import ( + END, + Closed, + CloseStatus, + End, + Record, + Superseded, +) +from temporalio.common._streams._plugin import StreamBackend +from temporalio.common._streams._reader import Subscription +from temporalio.common._streams._stream_client import ClientStream, ClientTopic +from temporalio.common._streams._workflow import WorkflowStream, WorkflowTopic + +__all__ = [ + "END", + "ActivityStream", + "ActivityTopic", + "BufferedPublisher", + "ClientStream", + "ClientTopic", + "CloseStatus", + "Closed", + "Creating", + "End", + "ReadOnlyContextError", + "Record", + "StreamClosedError", + "StreamConfigError", + "StreamDefinition", + "StreamBackend", + "StreamError", + "StreamExpiredError", + "StreamFencedError", + "StreamHandle", + "StreamNotFoundError", + "StreamNotOwnerError", + "StreamOffsetError", + "StreamSeqDivergentError", + "StreamStorageError", + "StreamTooLargeError", + "StreamUnsupportedError", + "Subscription", + "Superseded", + "SyncActivityStream", + "SyncActivityTopic", + "SyncSubscription", + "TopicDefinition", + "WorkflowStream", + "WorkflowTopic", + "defn", + "topic", + "with_name", +] diff --git a/temporalio/common/streams/_access.py b/temporalio/common/_streams/_access.py similarity index 92% rename from temporalio/common/streams/_access.py rename to temporalio/common/_streams/_access.py index cd007be620..19fc94de0a 100644 --- a/temporalio/common/streams/_access.py +++ b/temporalio/common/_streams/_access.py @@ -8,14 +8,14 @@ from typing import Any, TypeVar import temporalio.converter -from temporalio.common.streams import _convert, _reader -from temporalio.common.streams._backend import BackendCalls, CallOptions -from temporalio.common.streams._defn import TopicDefinition -from temporalio.common.streams._handle import StreamHandle, TopicBase -from temporalio.common.streams._items import End, close_status -from temporalio.common.streams._reader import Item, Position, Subscription -from temporalio.common.streams._writer import StreamWriter -from temporalio.common.streams.system import models +from temporalio.common._streams import _convert, _reader +from temporalio.common._streams._backend import BackendCalls, CallOptions +from temporalio.common._streams._defn import TopicDefinition +from temporalio.common._streams._handle import StreamHandle, TopicBase +from temporalio.common._streams._items import End, close_status +from temporalio.common._streams._reader import Item, Position, Subscription +from temporalio.common._streams._writer import StreamWriter +from temporalio.common._streams.system import models T = TypeVar("T") diff --git a/temporalio/common/streams/_activity.py b/temporalio/common/_streams/_activity.py similarity index 95% rename from temporalio/common/streams/_activity.py rename to temporalio/common/_streams/_activity.py index 9b6f15491b..593d9dfdc9 100644 --- a/temporalio/common/streams/_activity.py +++ b/temporalio/common/_streams/_activity.py @@ -9,8 +9,8 @@ import temporalio.api.streamservice.v1 as stream_pb import temporalio.converter -from temporalio.common.streams._backend import BackendCalls -from temporalio.common.streams._writer import StreamWriter +from temporalio.common._streams._backend import BackendCalls +from temporalio.common._streams._writer import StreamWriter @dataclass diff --git a/temporalio/common/streams/_activity_stream.py b/temporalio/common/_streams/_activity_stream.py similarity index 96% rename from temporalio/common/streams/_activity_stream.py rename to temporalio/common/_streams/_activity_stream.py index b0a9302638..81cfc09e23 100644 --- a/temporalio/common/streams/_activity_stream.py +++ b/temporalio/common/_streams/_activity_stream.py @@ -9,18 +9,18 @@ from typing import Any, Generic, TypeVar, overload import temporalio.api.streamservice.v1 as stream_pb -from temporalio.common.streams import _activity, _errors -from temporalio.common.streams._access import ( +from temporalio.common._streams import _activity, _errors +from temporalio.common._streams._access import ( Access, SyncTopicBase, _AsyncTopic, read_spec, subscription, ) -from temporalio.common.streams._defn import TopicDefinition -from temporalio.common.streams._handle import StreamHandle -from temporalio.common.streams._items import Closed, End, Record, Superseded -from temporalio.common.streams._reader import Subscription +from temporalio.common._streams._defn import TopicDefinition +from temporalio.common._streams._handle import StreamHandle +from temporalio.common._streams._items import Closed, End, Record, Superseded +from temporalio.common._streams._reader import Subscription E = TypeVar("E") T = TypeVar("T") diff --git a/temporalio/common/streams/_backend.py b/temporalio/common/_streams/_backend.py similarity index 96% rename from temporalio/common/streams/_backend.py rename to temporalio/common/_streams/_backend.py index 24f02b2a48..d0fbf167e3 100644 --- a/temporalio/common/streams/_backend.py +++ b/temporalio/common/_streams/_backend.py @@ -17,13 +17,13 @@ import temporalio.bridge.streams import temporalio.converter import temporalio.service -from temporalio.common.streams import _errors -from temporalio.common.streams.proto import v1 as ext_pb -from temporalio.common.streams.system import client as service_client +from temporalio.common._streams import _errors +from temporalio.common._streams.proto import v1 as ext_pb +from temporalio.common._streams.system import client as service_client if TYPE_CHECKING: # The writer bindings import this package's protos, so they load after it. - from temporalio.common.streams.writer_system import client as writer_client + from temporalio.common._streams.writer_system import client as writer_client _M = TypeVar("_M", bound=google.protobuf.message.Message) @@ -164,7 +164,7 @@ def writers( self, namespace: str, data_converter: temporalio.converter.DataConverter ) -> writer_client.StreamWriterClient: """The generated writer client over the runtime.""" - from temporalio.common.streams.writer_system import client + from temporalio.common._streams.writer_system import client return client.StreamWriterClient( _ExtensionTransport(self, namespace), data_converter=data_converter diff --git a/temporalio/common/streams/_buffered.py b/temporalio/common/_streams/_buffered.py similarity index 96% rename from temporalio/common/streams/_buffered.py rename to temporalio/common/_streams/_buffered.py index 26f2f55071..4af1a9d488 100644 --- a/temporalio/common/streams/_buffered.py +++ b/temporalio/common/_streams/_buffered.py @@ -7,11 +7,11 @@ from types import TracebackType from typing import Any, Generic, TypeVar, overload -from temporalio.common.streams import _activity -from temporalio.common.streams._activity_stream import ActivityStream, ActivityTopic -from temporalio.common.streams._defn import TopicDefinition -from temporalio.common.streams._handle import TopicBase -from temporalio.common.streams._stream_client import ClientStream, ClientTopic +from temporalio.common._streams import _activity +from temporalio.common._streams._activity_stream import ActivityStream, ActivityTopic +from temporalio.common._streams._defn import TopicDefinition +from temporalio.common._streams._handle import TopicBase +from temporalio.common._streams._stream_client import ClientStream, ClientTopic T = TypeVar("T") U = TypeVar("U") diff --git a/temporalio/common/streams/_convert.py b/temporalio/common/_streams/_convert.py similarity index 100% rename from temporalio/common/streams/_convert.py rename to temporalio/common/_streams/_convert.py diff --git a/temporalio/common/streams/_defn.py b/temporalio/common/_streams/_defn.py similarity index 96% rename from temporalio/common/streams/_defn.py rename to temporalio/common/_streams/_defn.py index 2708a5fe1f..4485b379e4 100644 --- a/temporalio/common/streams/_defn.py +++ b/temporalio/common/_streams/_defn.py @@ -171,9 +171,11 @@ def definition_of(cls: type) -> StreamDefinition: """The definition that :py:func:`defn` gave a class. Raises: - TypeError: The class has no ``@streams.defn``. + TypeError: The class has no ``@common.defn``. """ definition = cls.__dict__.get(_DEFINITION_ATTR) if isinstance(cls, type) else None if not isinstance(definition, StreamDefinition): - raise TypeError(f"{cls!r} is not a Stream definition. Add @streams.defn.") + raise TypeError( + f"{cls!r} is not a Stream definition. Add @temporalio.common.defn." + ) return definition diff --git a/temporalio/common/streams/_errors.py b/temporalio/common/_streams/_errors.py similarity index 97% rename from temporalio/common/streams/_errors.py rename to temporalio/common/_streams/_errors.py index 16b67c9c98..6ff59ba6fd 100644 --- a/temporalio/common/streams/_errors.py +++ b/temporalio/common/_streams/_errors.py @@ -1,4 +1,4 @@ -"""Errors of temporalio.common.streams.""" +"""Errors of the Streams API in temporalio.common.""" from __future__ import annotations @@ -6,7 +6,7 @@ import temporalio.exceptions import temporalio.service -from temporalio.common.streams.proto.v1 import StreamErrorDetails +from temporalio.common._streams.proto.v1 import StreamErrorDetails class StreamError(temporalio.exceptions.ApplicationError): diff --git a/temporalio/common/streams/_extension.py b/temporalio/common/_streams/_extension.py similarity index 97% rename from temporalio/common/streams/_extension.py rename to temporalio/common/_streams/_extension.py index 03afdfbc24..9719670b88 100644 --- a/temporalio/common/streams/_extension.py +++ b/temporalio/common/_streams/_extension.py @@ -21,14 +21,14 @@ import temporalio.service import temporalio.worker import temporalio.workflow -from temporalio.common.streams import _activity, _ids, _writes -from temporalio.common.streams import _workflow as streams_workflow -from temporalio.common.streams._backend import BackendCalls, _ExtensionCalls -from temporalio.common.streams._errors import StreamConfigError -from temporalio.common.streams._handle import StreamHandle -from temporalio.common.streams._plugin import StreamBackend -from temporalio.common.streams.proto import v1 as ext_pb -from temporalio.common.streams.system import models +from temporalio.common._streams import _activity, _ids, _writes +from temporalio.common._streams import _workflow as streams_workflow +from temporalio.common._streams._backend import BackendCalls, _ExtensionCalls +from temporalio.common._streams._errors import StreamConfigError +from temporalio.common._streams._handle import StreamHandle +from temporalio.common._streams._plugin import StreamBackend +from temporalio.common._streams.proto import v1 as ext_pb +from temporalio.common._streams.system import models from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner logger = logging.getLogger(__name__) @@ -170,7 +170,7 @@ def _configure_worker( config["workflow_runner"] = dataclasses.replace( runner, restrictions=runner.restrictions.with_passthrough_modules( - "temporalio.common.streams" + "temporalio.common._streams" ), ) return config diff --git a/temporalio/common/streams/_handle.py b/temporalio/common/_streams/_handle.py similarity index 99% rename from temporalio/common/streams/_handle.py rename to temporalio/common/_streams/_handle.py index 51e91fe56d..2e8f4ef2de 100644 --- a/temporalio/common/streams/_handle.py +++ b/temporalio/common/_streams/_handle.py @@ -12,8 +12,8 @@ import temporalio.converter import temporalio.exceptions -from temporalio.common.streams import _errors -from temporalio.common.streams._defn import TopicDefinition, definition_of +from temporalio.common._streams import _errors +from temporalio.common._streams._defn import TopicDefinition, definition_of E = TypeVar("E") T = TypeVar("T") diff --git a/temporalio/common/streams/_ids.py b/temporalio/common/_streams/_ids.py similarity index 100% rename from temporalio/common/streams/_ids.py rename to temporalio/common/_streams/_ids.py diff --git a/temporalio/common/streams/_items.py b/temporalio/common/_streams/_items.py similarity index 99% rename from temporalio/common/streams/_items.py rename to temporalio/common/_streams/_items.py index 312610a983..3b60e39a34 100644 --- a/temporalio/common/streams/_items.py +++ b/temporalio/common/_streams/_items.py @@ -8,7 +8,7 @@ from temporalio.api.streamservice.v1 import CloseStatus as _CloseStatus from temporalio.api.streamservice.v1 import ProducerKind -from temporalio.common.streams.system import models +from temporalio.common._streams.system import models from temporalio.exceptions import is_cancelled_exception T = TypeVar("T") diff --git a/temporalio/common/streams/_plugin.py b/temporalio/common/_streams/_plugin.py similarity index 98% rename from temporalio/common/streams/_plugin.py rename to temporalio/common/_streams/_plugin.py index cf71ce70b1..cf7b378da9 100644 --- a/temporalio/common/streams/_plugin.py +++ b/temporalio/common/_streams/_plugin.py @@ -8,8 +8,8 @@ import temporalio.client import temporalio.service import temporalio.worker -from temporalio.common.streams import _errors -from temporalio.common.streams._backend import BackendCalls +from temporalio.common._streams import _errors +from temporalio.common._streams._backend import BackendCalls from temporalio.plugin import SimplePlugin # The default backend when no registered backend has default=True. This SDK has no native diff --git a/temporalio/common/streams/_reader.py b/temporalio/common/_streams/_reader.py similarity index 97% rename from temporalio/common/streams/_reader.py rename to temporalio/common/_streams/_reader.py index 0184722e94..cf2a6606bd 100644 --- a/temporalio/common/streams/_reader.py +++ b/temporalio/common/_streams/_reader.py @@ -10,8 +10,8 @@ import temporalio.converter from temporalio.api.streamservice.v1 import ProducerKind, ReadStart, StreamRecordKind -from temporalio.common.streams import _backend, _convert, _errors -from temporalio.common.streams._items import ( +from temporalio.common._streams import _backend, _convert, _errors +from temporalio.common._streams._items import ( Closed, CloseStatus, End, @@ -20,8 +20,8 @@ producer_key, withdraws, ) -from temporalio.common.streams.system import client as service_client -from temporalio.common.streams.system import models +from temporalio.common._streams.system import client as service_client +from temporalio.common._streams.system import models T = TypeVar("T") diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/_streams/_stream_client.py similarity index 96% rename from temporalio/common/streams/_stream_client.py rename to temporalio/common/_streams/_stream_client.py index 98ceb35525..53817ac3f6 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/_streams/_stream_client.py @@ -14,22 +14,22 @@ import temporalio.api.streamservice.v1 as stream_pb import temporalio.client -from temporalio.common.streams import _backend, _errors, _ids -from temporalio.common.streams._access import ( +from temporalio.common._streams import _backend, _errors, _ids +from temporalio.common._streams._access import ( Access, ReadSpec, _AsyncTopic, read_spec, subscription, ) -from temporalio.common.streams._backend import BackendCalls -from temporalio.common.streams._defn import TopicDefinition, definition_of -from temporalio.common.streams._handle import Creating, StreamHandle, exit_failure -from temporalio.common.streams._items import End -from temporalio.common.streams._plugin import client_backend -from temporalio.common.streams._reader import Item, Position, Subscription -from temporalio.common.streams._writer import StreamWriter -from temporalio.common.streams.system import models +from temporalio.common._streams._backend import BackendCalls +from temporalio.common._streams._defn import TopicDefinition, definition_of +from temporalio.common._streams._handle import Creating, StreamHandle, exit_failure +from temporalio.common._streams._items import End +from temporalio.common._streams._plugin import client_backend +from temporalio.common._streams._reader import Item, Position, Subscription +from temporalio.common._streams._writer import StreamWriter +from temporalio.common._streams.system import models E = TypeVar("E") T = TypeVar("T") @@ -279,7 +279,7 @@ def subscribe( Args: topics: Read these Topics, by Topic of the definition, name or Topic object. With no Topics and no ``topic_prefixes``, read every Topic. - offset: Start at this offset, inclusive, or at ``streams.END`` for new + offset: Start at this offset, inclusive, or at ``common.END`` for new Records only. Defaults to the first Record. last: Start with the newest this many Records that pass the filter, oldest first, such as the first view of a chat. Not with ``offset``. A Record diff --git a/temporalio/common/streams/_workflow.py b/temporalio/common/_streams/_workflow.py similarity index 98% rename from temporalio/common/streams/_workflow.py rename to temporalio/common/_streams/_workflow.py index 79bae3c8d5..fae2f06400 100644 --- a/temporalio/common/streams/_workflow.py +++ b/temporalio/common/_streams/_workflow.py @@ -27,16 +27,16 @@ import temporalio.converter import temporalio.exceptions import temporalio.workflow -from temporalio.common.streams import _errors, _ids -from temporalio.common.streams._defn import TopicDefinition, definition_of -from temporalio.common.streams._handle import ( +from temporalio.common._streams import _errors, _ids +from temporalio.common._streams._defn import TopicDefinition, definition_of +from temporalio.common._streams._handle import ( Creating, StreamHandle, TopicBase, exit_failure, ) -from temporalio.common.streams._items import close_status -from temporalio.common.streams.system import models +from temporalio.common._streams._items import close_status +from temporalio.common._streams.system import models E = TypeVar("E") T = TypeVar("T") diff --git a/temporalio/common/streams/_writer.py b/temporalio/common/_streams/_writer.py similarity index 94% rename from temporalio/common/streams/_writer.py rename to temporalio/common/_streams/_writer.py index 1263722342..c189d6a23f 100644 --- a/temporalio/common/streams/_writer.py +++ b/temporalio/common/_streams/_writer.py @@ -8,10 +8,10 @@ from typing import TYPE_CHECKING, Any import temporalio.api.streamservice.v1 as stream_pb -from temporalio.common.streams import _errors +from temporalio.common._streams import _errors if TYPE_CHECKING: - from temporalio.common.streams.writer_system import client as writer_client + from temporalio.common._streams.writer_system import client as writer_client class StreamWriter: @@ -67,7 +67,7 @@ async def open(self) -> writer_client.Writer: """Open the crate's writer once. The open of a new attempt withdraws the earlier one's Records, even when it writes nothing. """ - from temporalio.common.streams.writer_system import models + from temporalio.common._streams.writer_system import models async with self._open_lock: if self._writer is None: @@ -99,7 +99,7 @@ async def write(self, records: Sequence[tuple[str, Any]]) -> int: """Write ``(topic, value)`` pairs as Records now, in order. Returns the last Record's offset. """ - from temporalio.common.streams.writer_system import models + from temporalio.common._streams.writer_system import models if self._closed: raise _errors.StreamClosedError( diff --git a/temporalio/common/streams/_writes.py b/temporalio/common/_streams/_writes.py similarity index 97% rename from temporalio/common/streams/_writes.py rename to temporalio/common/_streams/_writes.py index bf07508bc1..057fb106f6 100644 --- a/temporalio/common/streams/_writes.py +++ b/temporalio/common/_streams/_writes.py @@ -14,14 +14,14 @@ import temporalio.activity import temporalio.api.streamservice.v1 as stream_pb import temporalio.common -from temporalio.common.streams import _activity, _errors, _ids -from temporalio.common.streams._workflow import ( +from temporalio.common._streams import _activity, _errors, _ids +from temporalio.common._streams._workflow import ( APPEND_ACTIVITY, CLOSE_ACTIVITY, CREATE_ACTIVITY, ) -from temporalio.common.streams.system import client as service_client -from temporalio.common.streams.system import models +from temporalio.common._streams.system import client as service_client +from temporalio.common._streams.system import models def _context() -> tuple[_activity.ActivityStreams, service_client.StreamServiceClient]: diff --git a/temporalio/common/streams/proto/__init__.py b/temporalio/common/_streams/proto/__init__.py similarity index 100% rename from temporalio/common/streams/proto/__init__.py rename to temporalio/common/_streams/proto/__init__.py diff --git a/temporalio/common/streams/proto/v1/__init__.py b/temporalio/common/_streams/proto/v1/__init__.py similarity index 100% rename from temporalio/common/streams/proto/v1/__init__.py rename to temporalio/common/_streams/proto/v1/__init__.py diff --git a/temporalio/common/streams/proto/v1/extension_pb2.py b/temporalio/common/_streams/proto/v1/extension_pb2.py similarity index 90% rename from temporalio/common/streams/proto/v1/extension_pb2.py rename to temporalio/common/_streams/proto/v1/extension_pb2.py index 3596df46f8..ea592706d7 100644 --- a/temporalio/common/streams/proto/v1/extension_pb2.py +++ b/temporalio/common/_streams/proto/v1/extension_pb2.py @@ -42,7 +42,7 @@ (_message.Message,), { "DESCRIPTOR": _REDISSTREAMCONFIG, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.RedisStreamConfig) }, ) @@ -53,7 +53,7 @@ (_message.Message,), { "DESCRIPTOR": _OPENWRITERREQUEST, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.OpenWriterRequest) }, ) @@ -64,7 +64,7 @@ (_message.Message,), { "DESCRIPTOR": _OPENWRITERRESPONSE, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.OpenWriterResponse) }, ) @@ -75,7 +75,7 @@ (_message.Message,), { "DESCRIPTOR": _WRITERECORDSREQUEST, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.WriteRecordsRequest) }, ) @@ -86,7 +86,7 @@ (_message.Message,), { "DESCRIPTOR": _WRITERECORDSRESPONSE, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.WriteRecordsResponse) }, ) @@ -97,7 +97,7 @@ (_message.Message,), { "DESCRIPTOR": _CLOSEWRITERREQUEST, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.CloseWriterRequest) }, ) @@ -108,7 +108,7 @@ (_message.Message,), { "DESCRIPTOR": _CLOSEWRITERRESPONSE, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.CloseWriterResponse) }, ) @@ -119,7 +119,7 @@ (_message.Message,), { "DESCRIPTOR": _FREEWRITERREQUEST, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.FreeWriterRequest) }, ) @@ -130,7 +130,7 @@ (_message.Message,), { "DESCRIPTOR": _FREEWRITERRESPONSE, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.FreeWriterResponse) }, ) @@ -141,7 +141,7 @@ (_message.Message,), { "DESCRIPTOR": _CLOSEWRITERSREQUEST, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.CloseWritersRequest) }, ) @@ -152,7 +152,7 @@ (_message.Message,), { "DESCRIPTOR": _CLOSEWRITERSRESPONSE, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.CloseWritersResponse) }, ) @@ -163,7 +163,7 @@ (_message.Message,), { "DESCRIPTOR": _STREAMERRORDETAILS, - "__module__": "temporalio.common.streams.proto.v1.extension_pb2", + "__module__": "temporalio.common._streams.proto.v1.extension_pb2", # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.StreamErrorDetails) }, ) diff --git a/temporalio/common/streams/proto/v1/extension_pb2.pyi b/temporalio/common/_streams/proto/v1/extension_pb2.pyi similarity index 100% rename from temporalio/common/streams/proto/v1/extension_pb2.pyi rename to temporalio/common/_streams/proto/v1/extension_pb2.pyi diff --git a/temporalio/common/streams/system/__init__.py b/temporalio/common/_streams/system/__init__.py similarity index 100% rename from temporalio/common/streams/system/__init__.py rename to temporalio/common/_streams/system/__init__.py diff --git a/temporalio/common/streams/system/_resources/__init__.py b/temporalio/common/_streams/system/_resources/__init__.py similarity index 100% rename from temporalio/common/streams/system/_resources/__init__.py rename to temporalio/common/_streams/system/_resources/__init__.py diff --git a/temporalio/common/streams/system/_resources/stream.py b/temporalio/common/_streams/system/_resources/stream.py similarity index 100% rename from temporalio/common/streams/system/_resources/stream.py rename to temporalio/common/_streams/system/_resources/stream.py diff --git a/temporalio/common/streams/system/_system_nexus_interceptor.py b/temporalio/common/_streams/system/_system_nexus_interceptor.py similarity index 100% rename from temporalio/common/streams/system/_system_nexus_interceptor.py rename to temporalio/common/_streams/system/_system_nexus_interceptor.py diff --git a/temporalio/common/streams/system/client.py b/temporalio/common/_streams/system/client.py similarity index 100% rename from temporalio/common/streams/system/client.py rename to temporalio/common/_streams/system/client.py diff --git a/temporalio/common/streams/system/models.py b/temporalio/common/_streams/system/models.py similarity index 100% rename from temporalio/common/streams/system/models.py rename to temporalio/common/_streams/system/models.py diff --git a/temporalio/common/streams/system/operations/__init__.py b/temporalio/common/_streams/system/operations/__init__.py similarity index 100% rename from temporalio/common/streams/system/operations/__init__.py rename to temporalio/common/_streams/system/operations/__init__.py diff --git a/temporalio/common/streams/system/operations/create_stream.py b/temporalio/common/_streams/system/operations/create_stream.py similarity index 100% rename from temporalio/common/streams/system/operations/create_stream.py rename to temporalio/common/_streams/system/operations/create_stream.py diff --git a/temporalio/common/streams/system/services.py b/temporalio/common/_streams/system/services.py similarity index 100% rename from temporalio/common/streams/system/services.py rename to temporalio/common/_streams/system/services.py diff --git a/temporalio/common/streams/writer_system/__init__.py b/temporalio/common/_streams/writer_system/__init__.py similarity index 100% rename from temporalio/common/streams/writer_system/__init__.py rename to temporalio/common/_streams/writer_system/__init__.py diff --git a/temporalio/common/streams/writer_system/client.py b/temporalio/common/_streams/writer_system/client.py similarity index 100% rename from temporalio/common/streams/writer_system/client.py rename to temporalio/common/_streams/writer_system/client.py diff --git a/temporalio/common/streams/writer_system/models.py b/temporalio/common/_streams/writer_system/models.py similarity index 82% rename from temporalio/common/streams/writer_system/models.py rename to temporalio/common/_streams/writer_system/models.py index 0542ae5ded..93767c40c5 100644 --- a/temporalio/common/streams/writer_system/models.py +++ b/temporalio/common/_streams/writer_system/models.py @@ -10,7 +10,7 @@ import typing_extensions import temporalio.api.streamservice.v1.stream_service_pb2 -import temporalio.common.streams.proto.v1.extension_pb2 +import temporalio.common._streams.proto.v1.extension_pb2 import temporalio.converter import temporalio.nexus.system._support as _support @@ -30,17 +30,17 @@ class OpenWriterRequest: class _OpenWriterRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ OpenWriterRequest, - temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest, + temporalio.common._streams.proto.v1.extension_pb2.OpenWriterRequest, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest + type[temporalio.common._streams.proto.v1.extension_pb2.OpenWriterRequest] | None + ) = temporalio.common._streams.proto.v1.extension_pb2.OpenWriterRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest, + value: temporalio.common._streams.proto.v1.extension_pb2.OpenWriterRequest, type_hint: type[OpenWriterRequest], ) -> OpenWriterRequest: return OpenWriterRequest( @@ -70,8 +70,8 @@ def from_transfer_type( def to_transfer_type( self, value: OpenWriterRequest, - ) -> temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest: - message = temporalio.common.streams.proto.v1.extension_pb2.OpenWriterRequest() + ) -> temporalio.common._streams.proto.v1.extension_pb2.OpenWriterRequest: + message = temporalio.common._streams.proto.v1.extension_pb2.OpenWriterRequest() if value.stream is not None: message.stream.CopyFrom( _StreamRefTransferTypeConverter().to_transfer_type(value.stream) @@ -416,17 +416,18 @@ class WriterResult: class _WriterResultTransferTypeConverter( temporalio.converter.TransferTypeConverter[ WriterResult, - temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse, + temporalio.common._streams.proto.v1.extension_pb2.OpenWriterResponse, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse + type[temporalio.common._streams.proto.v1.extension_pb2.OpenWriterResponse] + | None + ) = temporalio.common._streams.proto.v1.extension_pb2.OpenWriterResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse, + value: temporalio.common._streams.proto.v1.extension_pb2.OpenWriterResponse, type_hint: type[WriterResult], ) -> WriterResult: return WriterResult( @@ -437,8 +438,8 @@ def from_transfer_type( def to_transfer_type( self, value: WriterResult, - ) -> temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse: - message = temporalio.common.streams.proto.v1.extension_pb2.OpenWriterResponse() + ) -> temporalio.common._streams.proto.v1.extension_pb2.OpenWriterResponse: + message = temporalio.common._streams.proto.v1.extension_pb2.OpenWriterResponse() if value.writer_id is not None: message.writer_id = value.writer_id return message @@ -458,18 +459,18 @@ class WriteRecordsRequest: class _WriteRecordsRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ WriteRecordsRequest, - temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest, + temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsRequest, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest] + type[temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsRequest] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest + ) = temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest, + value: temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsRequest, type_hint: type[WriteRecordsRequest], ) -> WriteRecordsRequest: return WriteRecordsRequest( @@ -486,8 +487,10 @@ def from_transfer_type( def to_transfer_type( self, value: WriteRecordsRequest, - ) -> temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest: - message = temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsRequest() + ) -> temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsRequest: + message = ( + temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsRequest() + ) if value.writer_id is not None: message.writer_id = value.writer_id if value.records: @@ -563,18 +566,18 @@ class WriteRecordsResponse: class _WriteRecordsResponseTransferTypeConverter( temporalio.converter.TransferTypeConverter[ WriteRecordsResponse, - temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse, + temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsResponse, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse] + type[temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsResponse] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse + ) = temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse, + value: temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsResponse, type_hint: type[WriteRecordsResponse], ) -> WriteRecordsResponse: return WriteRecordsResponse( @@ -586,9 +589,9 @@ def from_transfer_type( def to_transfer_type( self, value: WriteRecordsResponse, - ) -> temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse: + ) -> temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsResponse: message = ( - temporalio.common.streams.proto.v1.extension_pb2.WriteRecordsResponse() + temporalio.common._streams.proto.v1.extension_pb2.WriteRecordsResponse() ) if value.first_offset is not None: message.first_offset = value.first_offset @@ -610,17 +613,18 @@ class CloseWriterRequest: class _CloseWriterRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ CloseWriterRequest, - temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest, + temporalio.common._streams.proto.v1.extension_pb2.CloseWriterRequest, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest + type[temporalio.common._streams.proto.v1.extension_pb2.CloseWriterRequest] + | None + ) = temporalio.common._streams.proto.v1.extension_pb2.CloseWriterRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest, + value: temporalio.common._streams.proto.v1.extension_pb2.CloseWriterRequest, type_hint: type[CloseWriterRequest], ) -> CloseWriterRequest: return CloseWriterRequest( @@ -631,8 +635,8 @@ def from_transfer_type( def to_transfer_type( self, value: CloseWriterRequest, - ) -> temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest: - message = temporalio.common.streams.proto.v1.extension_pb2.CloseWriterRequest() + ) -> temporalio.common._streams.proto.v1.extension_pb2.CloseWriterRequest: + message = temporalio.common._streams.proto.v1.extension_pb2.CloseWriterRequest() if value.writer_id is not None: message.writer_id = value.writer_id return message @@ -651,18 +655,18 @@ class CloseWriterResponse: class _CloseWriterResponseTransferTypeConverter( temporalio.converter.TransferTypeConverter[ CloseWriterResponse, - temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse, + temporalio.common._streams.proto.v1.extension_pb2.CloseWriterResponse, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse] + type[temporalio.common._streams.proto.v1.extension_pb2.CloseWriterResponse] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse + ) = temporalio.common._streams.proto.v1.extension_pb2.CloseWriterResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse, + value: temporalio.common._streams.proto.v1.extension_pb2.CloseWriterResponse, type_hint: type[CloseWriterResponse], ) -> CloseWriterResponse: return CloseWriterResponse( @@ -673,8 +677,10 @@ def from_transfer_type( def to_transfer_type( self, value: CloseWriterResponse, - ) -> temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse: - message = temporalio.common.streams.proto.v1.extension_pb2.CloseWriterResponse() + ) -> temporalio.common._streams.proto.v1.extension_pb2.CloseWriterResponse: + message = ( + temporalio.common._streams.proto.v1.extension_pb2.CloseWriterResponse() + ) if value.last_offset is not None: message.last_offset = value.last_offset return message @@ -693,17 +699,17 @@ class FreeWriterRequest: class _FreeWriterRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ FreeWriterRequest, - temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest, + temporalio.common._streams.proto.v1.extension_pb2.FreeWriterRequest, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest + type[temporalio.common._streams.proto.v1.extension_pb2.FreeWriterRequest] | None + ) = temporalio.common._streams.proto.v1.extension_pb2.FreeWriterRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest, + value: temporalio.common._streams.proto.v1.extension_pb2.FreeWriterRequest, type_hint: type[FreeWriterRequest], ) -> FreeWriterRequest: return FreeWriterRequest( @@ -714,8 +720,8 @@ def from_transfer_type( def to_transfer_type( self, value: FreeWriterRequest, - ) -> temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest: - message = temporalio.common.streams.proto.v1.extension_pb2.FreeWriterRequest() + ) -> temporalio.common._streams.proto.v1.extension_pb2.FreeWriterRequest: + message = temporalio.common._streams.proto.v1.extension_pb2.FreeWriterRequest() if value.writer_id is not None: message.writer_id = value.writer_id return message @@ -734,17 +740,18 @@ class FreeWriterResponse: class _FreeWriterResponseTransferTypeConverter( temporalio.converter.TransferTypeConverter[ FreeWriterResponse, - temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse, + temporalio.common._streams.proto.v1.extension_pb2.FreeWriterResponse, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse + type[temporalio.common._streams.proto.v1.extension_pb2.FreeWriterResponse] + | None + ) = temporalio.common._streams.proto.v1.extension_pb2.FreeWriterResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse, + value: temporalio.common._streams.proto.v1.extension_pb2.FreeWriterResponse, type_hint: type[FreeWriterResponse], ) -> FreeWriterResponse: return FreeWriterResponse() @@ -753,8 +760,8 @@ def from_transfer_type( def to_transfer_type( self, value: FreeWriterResponse, - ) -> temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse: - message = temporalio.common.streams.proto.v1.extension_pb2.FreeWriterResponse() + ) -> temporalio.common._streams.proto.v1.extension_pb2.FreeWriterResponse: + message = temporalio.common._streams.proto.v1.extension_pb2.FreeWriterResponse() return message @@ -771,18 +778,18 @@ class CloseWritersRequest: class _CloseWritersRequestTransferTypeConverter( temporalio.converter.TransferTypeConverter[ CloseWritersRequest, - temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest, + temporalio.common._streams.proto.v1.extension_pb2.CloseWritersRequest, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest] + type[temporalio.common._streams.proto.v1.extension_pb2.CloseWritersRequest] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest + ) = temporalio.common._streams.proto.v1.extension_pb2.CloseWritersRequest @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest, + value: temporalio.common._streams.proto.v1.extension_pb2.CloseWritersRequest, type_hint: type[CloseWritersRequest], ) -> CloseWritersRequest: return CloseWritersRequest() @@ -791,8 +798,10 @@ def from_transfer_type( def to_transfer_type( self, value: CloseWritersRequest, - ) -> temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest: - message = temporalio.common.streams.proto.v1.extension_pb2.CloseWritersRequest() + ) -> temporalio.common._streams.proto.v1.extension_pb2.CloseWritersRequest: + message = ( + temporalio.common._streams.proto.v1.extension_pb2.CloseWritersRequest() + ) return message @@ -809,18 +818,18 @@ class CloseWritersResponse: class _CloseWritersResponseTransferTypeConverter( temporalio.converter.TransferTypeConverter[ CloseWritersResponse, - temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse, + temporalio.common._streams.proto.v1.extension_pb2.CloseWritersResponse, ] ): transfer_type: ( - type[temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse] + type[temporalio.common._streams.proto.v1.extension_pb2.CloseWritersResponse] | None - ) = temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse + ) = temporalio.common._streams.proto.v1.extension_pb2.CloseWritersResponse @typing_extensions.override def from_transfer_type( self, - value: temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse, + value: temporalio.common._streams.proto.v1.extension_pb2.CloseWritersResponse, type_hint: type[CloseWritersResponse], ) -> CloseWritersResponse: return CloseWritersResponse( @@ -831,9 +840,9 @@ def from_transfer_type( def to_transfer_type( self, value: CloseWritersResponse, - ) -> temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse: + ) -> temporalio.common._streams.proto.v1.extension_pb2.CloseWritersResponse: message = ( - temporalio.common.streams.proto.v1.extension_pb2.CloseWritersResponse() + temporalio.common._streams.proto.v1.extension_pb2.CloseWritersResponse() ) if value.failed is not None: message.failed = value.failed diff --git a/temporalio/common/streams/__init__.py b/temporalio/common/streams/__init__.py deleted file mode 100644 index aca56388ca..0000000000 --- a/temporalio/common/streams/__init__.py +++ /dev/null @@ -1,104 +0,0 @@ -"""Streams of Records that Workflows, Activities and clients write and read. - -A Stream definition, a class with ``@streams.defn``, gives the Stream name and the type of each -Topic. Workflow code creates a Stream with :py:func:`temporalio.workflow.create_stream`. A -client gets a Workflow's Stream with :py:meth:`temporalio.client.WorkflowHandle.stream`, and -any Stream with :py:meth:`temporalio.client.Client.stream`. A Stream is data: a Workflow -passes it to an Activity, a child Workflow or the next run as an argument, and the -parameter's type gives the handle class of that context, such as :py:class:`ActivityStream`. - -A :py:class:`StreamBackend` carries the calls and registers a backend name, such as -:py:class:`temporalio.contrib.stream.RedisStreamBackend` (``redis``) and -:py:class:`temporalio.contrib.stream.testing.MemoryStreamBackend` (``memory``). Register -backends with ``Client.connect(..., stream_backends=[...])``. A handle carries the name of its -backend. With more than one registered backend, exactly one has ``default=True``, the one -that a create with no ``backend=`` uses. For now a Worker serves one backend. - -.. warning:: - This package is experimental. -""" - -from temporalio.common.streams._activity_stream import ( - ActivityStream, - ActivityTopic, - SyncActivityStream, - SyncActivityTopic, - SyncSubscription, -) -from temporalio.common.streams._buffered import BufferedPublisher -from temporalio.common.streams._defn import ( - StreamDefinition, - TopicDefinition, - defn, - topic, - with_name, -) -from temporalio.common.streams._errors import ( - ReadOnlyContextError, - StreamClosedError, - StreamConfigError, - StreamError, - StreamExpiredError, - StreamFencedError, - StreamNotFoundError, - StreamNotOwnerError, - StreamOffsetError, - StreamSeqDivergentError, - StreamStorageError, - StreamTooLargeError, - StreamUnsupportedError, -) -from temporalio.common.streams._handle import Creating, StreamHandle -from temporalio.common.streams._items import ( - END, - Closed, - CloseStatus, - End, - Record, - Superseded, -) -from temporalio.common.streams._plugin import StreamBackend -from temporalio.common.streams._reader import Subscription -from temporalio.common.streams._stream_client import ClientStream, ClientTopic -from temporalio.common.streams._workflow import WorkflowStream, WorkflowTopic - -__all__ = [ - "END", - "ActivityStream", - "ActivityTopic", - "BufferedPublisher", - "ClientStream", - "ClientTopic", - "CloseStatus", - "Closed", - "Creating", - "End", - "ReadOnlyContextError", - "Record", - "StreamClosedError", - "StreamConfigError", - "StreamDefinition", - "StreamBackend", - "StreamError", - "StreamExpiredError", - "StreamFencedError", - "StreamHandle", - "StreamNotFoundError", - "StreamNotOwnerError", - "StreamOffsetError", - "StreamSeqDivergentError", - "StreamStorageError", - "StreamTooLargeError", - "StreamUnsupportedError", - "Subscription", - "Superseded", - "SyncActivityStream", - "SyncActivityTopic", - "SyncSubscription", - "TopicDefinition", - "WorkflowStream", - "WorkflowTopic", - "defn", - "topic", - "with_name", -] diff --git a/temporalio/contrib/stream/__init__.py b/temporalio/contrib/stream/__init__.py index d73a80bb1c..aa8d5f1cc4 100644 --- a/temporalio/contrib/stream/__init__.py +++ b/temporalio/contrib/stream/__init__.py @@ -24,12 +24,12 @@ from typing import TypeVar import temporalio.client -from temporalio.common.streams import _plugin, _stream_client -from temporalio.common.streams._errors import StreamConfigError -from temporalio.common.streams._extension import ExtensionStreamBackend -from temporalio.common.streams._handle import Creating -from temporalio.common.streams._stream_client import ClientStream -from temporalio.common.streams.proto import v1 as ext_pb +from temporalio.common._streams import _plugin, _stream_client +from temporalio.common._streams._errors import StreamConfigError +from temporalio.common._streams._extension import ExtensionStreamBackend +from temporalio.common._streams._handle import Creating +from temporalio.common._streams._stream_client import ClientStream +from temporalio.common._streams.proto import v1 as ext_pb from temporalio.contrib.stream import workflow D = TypeVar("D") diff --git a/temporalio/contrib/stream/testing.py b/temporalio/contrib/stream/testing.py index b8901500ae..b9ad6de74b 100644 --- a/temporalio/contrib/stream/testing.py +++ b/temporalio/contrib/stream/testing.py @@ -9,8 +9,8 @@ from datetime import timedelta import temporalio.runtime -from temporalio.common.streams._extension import ExtensionStreamBackend -from temporalio.common.streams.proto import v1 as ext_pb +from temporalio.common._streams._extension import ExtensionStreamBackend +from temporalio.common._streams.proto import v1 as ext_pb class MemoryStreamBackend(ExtensionStreamBackend): diff --git a/temporalio/contrib/stream/workflow.py b/temporalio/contrib/stream/workflow.py index a68ad49d9e..acb147a711 100644 --- a/temporalio/contrib/stream/workflow.py +++ b/temporalio/contrib/stream/workflow.py @@ -15,10 +15,10 @@ from datetime import timedelta from typing import TypeVar -from temporalio.common.streams import _workflow -from temporalio.common.streams._errors import StreamConfigError -from temporalio.common.streams._handle import Creating -from temporalio.common.streams._workflow import WorkflowStream +from temporalio.common._streams import _workflow +from temporalio.common._streams._errors import StreamConfigError +from temporalio.common._streams._handle import Creating +from temporalio.common._streams._workflow import WorkflowStream D = TypeVar("D") diff --git a/temporalio/testing/_workflow.py b/temporalio/testing/_workflow.py index 3d363a9215..b4af035279 100644 --- a/temporalio/testing/_workflow.py +++ b/temporalio/testing/_workflow.py @@ -33,7 +33,7 @@ ) if TYPE_CHECKING: - from temporalio.common.streams import StreamBackend + from temporalio.common import StreamBackend logger = logging.getLogger(__name__) diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 528db5c77a..1ed28e186a 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -34,7 +34,7 @@ from .workflow_sandbox import SandboxedWorkflowRunner if TYPE_CHECKING: - from temporalio.common.streams import StreamBackend + from temporalio.common import StreamBackend logger = logging.getLogger(__name__) @@ -103,7 +103,7 @@ def __init__( for plugin in plugins: self._config = plugin.configure_replayer(self._config) if self._config.get("stream_backends"): - from temporalio.common.streams import _plugin as _stream_plugin + from temporalio.common._streams import _plugin as _stream_plugin self._config = _stream_plugin.configure_replayer(self._config) diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index fba239d25d..5a99ffbd0b 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -414,7 +414,7 @@ def __init__( # so a plugin that replaces the Workflow runner keeps the Streams passthrough. stream_backends = client.config(active_config=True).get("stream_backends") if stream_backends: - from temporalio.common.streams import _plugin as _stream_plugin + from temporalio.common._streams import _plugin as _stream_plugin config = _stream_plugin.configure_worker(stream_backends, config) diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 70cda08bad..3d55d4f342 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -1,7 +1,7 @@ """Streams in Workflow code. -The calls load :py:mod:`temporalio.common.streams` on first use, so a Workflow with no Stream -never loads it. +The calls load the Streams API of :py:mod:`temporalio.common` on first use, so a Workflow with +no Stream never loads it. """ from __future__ import annotations @@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, TypeVar if TYPE_CHECKING: - from temporalio.common.streams import Creating, WorkflowStream + from temporalio.common import Creating, WorkflowStream D = TypeVar("D") @@ -33,20 +33,20 @@ def create_stream( Await the call for the handle, or use it in ``async with``, which closes the Stream at the end of the block:: - async with workflow.create_stream(streams.with_name(Events, "turn-3")) as turn: + async with workflow.create_stream(common.with_name(Events, "turn-3")) as turn: await turn.topic(Events.tokens).publish("hi") An exception closes the Stream with that exception as the failure. When a retry run may follow, the close waits for the run's end, and the Stream stays open only for a retry run or for the retry of a Workflow Task that failed. A cron run leaves the Stream open. See - :py:meth:`temporalio.common.streams.WorkflowStream.__aexit__`. + :py:meth:`temporalio.common.WorkflowStream.__aexit__`. .. warning:: This API is experimental. Args: definition: The Stream definition class. Its name is the Stream name. For one Stream - per turn, for example, pass ``streams.with_name(Events, f"turn-{n}")``. + per turn, for example, pass ``common.with_name(Events, f"turn-{n}")``. retention: The age after which a Record is dropped, also while the Stream is open. A closed Stream stays one Retention after its close. Defaults to the backend's. backend: The name of the backend. Defaults to the registered backend with @@ -63,7 +63,7 @@ def create_stream( has ``default=True`` for a create with no backend. It fails the Workflow Task, not the run. """ - from temporalio.common.streams import _workflow + from temporalio.common._streams import _workflow return _workflow.create_stream(definition, retention=retention, backend=backend) @@ -83,7 +83,7 @@ def create_stream_nowait( .. warning:: This API is experimental. """ - from temporalio.common.streams import _workflow + from temporalio.common._streams import _workflow return _workflow.create_stream_nowait( definition, retention=retention, backend=backend @@ -111,6 +111,6 @@ def stream_handle( Raises: StreamConfigError: No backend is given, known or marked as the default. """ - from temporalio.common.streams import _workflow + from temporalio.common._streams import _workflow return _workflow.stream_handle(definition, backend=backend) diff --git a/tests/contrib/streams/conftest.py b/tests/contrib/streams/conftest.py index bf93a3526d..0de80034b5 100644 --- a/tests/contrib/streams/conftest.py +++ b/tests/contrib/streams/conftest.py @@ -9,7 +9,7 @@ import pytest from temporalio.client import Client -from temporalio.common.streams import StreamBackend +from temporalio.common import StreamBackend from temporalio.contrib.stream import RedisStreamBackend from temporalio.contrib.stream.testing import MemoryStreamBackend diff --git a/tests/contrib/streams/examples/activity_streams_tokens.py b/tests/contrib/streams/examples/activity_streams_tokens.py index cc4bcfc5f8..862e793cd1 100644 --- a/tests/contrib/streams/examples/activity_streams_tokens.py +++ b/tests/contrib/streams/examples/activity_streams_tokens.py @@ -12,27 +12,26 @@ from dataclasses import dataclass from datetime import timedelta -from temporalio import activity, workflow +from temporalio import activity, common, workflow from temporalio.client import Client -from temporalio.common import streams from temporalio.worker import Worker -@streams.defn(name="events") +@common.defn(name="events") class Events: - tokens = streams.topic(str) + tokens = common.topic(str) @dataclass class CallModelInput: prompt: str - stream: streams.ActivityStream[Events] + stream: common.ActivityStream[Events] @dataclass class SummarizeInput: text: str - stream: streams.SyncActivityStream[Events] + stream: common.SyncActivityStream[Events] @activity.defn @@ -42,7 +41,7 @@ async def call_model(input: CallModelInput) -> int: # Each awaited publish has written when it returns. last = await tokens.publish(words[0]) # A fast model sends many small tokens, so the helper batches them. - async with streams.BufferedPublisher( + async with common.BufferedPublisher( tokens, max_delay=timedelta(milliseconds=100) ) as fast: for word in words[1:]: @@ -62,12 +61,12 @@ async def run(self, prompt: str) -> list[int]: stream = await workflow.create_stream(Events) first = await workflow.execute_activity( call_model, - CallModelInput(prompt, streams.ActivityStream.from_handle(stream)), + CallModelInput(prompt, common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(minutes=5), ) summary = await workflow.execute_activity( summarize, - SummarizeInput(prompt, streams.SyncActivityStream.from_handle(stream)), + SummarizeInput(prompt, common.SyncActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(minutes=5), ) await stream.close() @@ -76,7 +75,7 @@ async def run(self, prompt: str) -> list[int]: async def main( client: Client, task_queue: str -) -> tuple[list[int], list[streams.Record[str]]]: +) -> tuple[list[int], list[common.Record[str]]]: with concurrent.futures.ThreadPoolExecutor() as executor: async with Worker( client, @@ -94,6 +93,6 @@ async def main( offsets = await handle.result() stream = handle.stream(Events) records = [ - item async for item in stream.subscribe() if isinstance(item, streams.Record) + item async for item in stream.subscribe() if isinstance(item, common.Record) ] return offsets, records diff --git a/tests/contrib/streams/examples/agent_publishes_turns.py b/tests/contrib/streams/examples/agent_publishes_turns.py index b869d61276..74f1ba9655 100644 --- a/tests/contrib/streams/examples/agent_publishes_turns.py +++ b/tests/contrib/streams/examples/agent_publishes_turns.py @@ -11,9 +11,8 @@ from dataclasses import dataclass, field from datetime import timedelta -from temporalio import activity, workflow +from temporalio import activity, common, workflow from temporalio.client import Client -from temporalio.common import streams from temporalio.worker import Worker @@ -23,10 +22,10 @@ class Turn: text: str = "" -@streams.defn(name="events") +@common.defn(name="events") class Events: - turns = streams.topic(Turn) - tokens = streams.topic(str) + turns = common.topic(Turn) + tokens = common.topic(str) @dataclass @@ -38,7 +37,7 @@ class Reply: @dataclass class CallModelInput: messages: list[str] - stream: streams.ActivityStream[Events] + stream: common.ActivityStream[Events] @activity.defn @@ -65,7 +64,7 @@ async def run(self, messages: list[str]) -> str: await turns.publish(Turn("model")) reply = await workflow.execute_activity( call_model, - CallModelInput(messages, streams.ActivityStream.from_handle(stream)), + CallModelInput(messages, common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(minutes=5), ) if not reply.tool_calls: @@ -81,9 +80,7 @@ async def run(self, messages: list[str]) -> str: ) -async def main( - client: Client, task_queue: str -) -> list[streams.Record | streams.Closed]: +async def main(client: Client, task_queue: str) -> list[common.Record | common.Closed]: async with Worker( client, task_queue=task_queue, @@ -101,5 +98,5 @@ async def main( return [ item async for item in stream.subscribe() - if isinstance(item, (streams.Record, streams.Closed)) + if isinstance(item, (common.Record, common.Closed)) ] diff --git a/tests/contrib/streams/examples/client_creates_stream.py b/tests/contrib/streams/examples/client_creates_stream.py index e42a0775cf..5fddc3810e 100644 --- a/tests/contrib/streams/examples/client_creates_stream.py +++ b/tests/contrib/streams/examples/client_creates_stream.py @@ -10,9 +10,8 @@ from dataclasses import dataclass from datetime import timedelta -from temporalio import workflow +from temporalio import common, workflow from temporalio.client import Client -from temporalio.common import streams from temporalio.contrib import stream as redis from temporalio.worker import Worker @@ -23,15 +22,15 @@ class Turn: text: str = "" -@streams.defn(name="events") +@common.defn(name="events") class Events: - turns = streams.topic(Turn) + turns = common.topic(Turn) @workflow.defn class Chat: @workflow.run - async def run(self, stream: streams.WorkflowStream[Events]) -> None: + async def run(self, stream: common.WorkflowStream[Events]) -> None: # The Workflow only writes. The client closes the Stream. await stream.topic(Events.turns).publish( Turn("model", "hi"), Turn("last", "bye") @@ -51,12 +50,12 @@ async def main(client: Client, task_queue: str) -> list[Turn]: await client.start_workflow( Chat.run, # A Stream is data, so the Workflow gets it as an argument. - streams.WorkflowStream.from_handle(chat_stream), + common.WorkflowStream.from_handle(chat_stream), id=f"chat-{uuid.uuid4()}", task_queue=task_queue, ) async for item in chat_stream.subscribe(): - if not isinstance(item, streams.Record): + if not isinstance(item, common.Record): continue turns.append(item.data) if item.data.kind == "last": diff --git a/tests/contrib/streams/examples/resume_from_offset.py b/tests/contrib/streams/examples/resume_from_offset.py index 95d3091293..607b7d9565 100644 --- a/tests/contrib/streams/examples/resume_from_offset.py +++ b/tests/contrib/streams/examples/resume_from_offset.py @@ -1,8 +1,8 @@ """Brook API proposal, "A client resumes a scoped Stream from an offset". -The Workflow writes one Stream per turn, a renamed definition from ``streams.with_name``. The +The Workflow writes one Stream per turn, a renamed definition from ``common.with_name``. The Reader saves the read's ``next_offset`` after each Record, stops, and later resumes from it with -``offset``. A view that wants only new Records starts at ``streams.END``. +``offset``. A view that wants only new Records starts at ``common.END``. """ from __future__ import annotations @@ -10,18 +10,17 @@ import uuid from datetime import timedelta -from temporalio import workflow +from temporalio import common, workflow from temporalio.client import Client, WorkflowHandle -from temporalio.common import streams from temporalio.worker import Worker -@streams.defn(name="events") +@common.defn(name="events") class Events: - tokens = streams.topic(str) + tokens = common.topic(str) -Turn3 = streams.with_name(Events, "turn-3") +Turn3 = common.with_name(Events, "turn-3") @workflow.defn @@ -65,9 +64,9 @@ async def read_some( Events.tokens, offset=store.load("turn-3"), create_wait=timedelta(seconds=10) ) async for item in read: - if isinstance(item, streams.Closed): + if isinstance(item, common.Closed): break - if isinstance(item, streams.Superseded): + if isinstance(item, common.Superseded): continue seen.append(item.data) # The read's next offset is after this Record, so the next read starts there. @@ -84,9 +83,9 @@ async def read_new(handle: WorkflowHandle[Turns, None]) -> list[str]: return [ item.data async for item in turn3.subscribe( - Events.tokens, offset=streams.END, wait_new_records=False + Events.tokens, offset=common.END, wait_new_records=False ) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] diff --git a/tests/contrib/streams/examples/test_examples.py b/tests/contrib/streams/examples/test_examples.py index f7c72eb190..b9afcce37c 100644 --- a/tests/contrib/streams/examples/test_examples.py +++ b/tests/contrib/streams/examples/test_examples.py @@ -2,8 +2,8 @@ import uuid +from temporalio import common from temporalio.client import Client -from temporalio.common import streams from tests.contrib.streams.examples import ( activity_streams_tokens, agent_publishes_turns, @@ -45,12 +45,12 @@ async def test_client_creates_stream(stream_client: Client): async def test_agent_publishes_turns(stream_client: Client): items = await agent_publishes_turns.main(stream_client, task_queue()) - data = [item.data for item in items if isinstance(item, streams.Record)] + data = [item.data for item in items if isinstance(item, common.Record)] Turn = agent_publishes_turns.Turn assert data == [Turn("model"), "find", "it", Turn("model"), "search", "result"] + [ Turn("done", "the answer") ] - assert isinstance(items[-1], streams.Closed) and items[-1].result == "the answer" + assert isinstance(items[-1], common.Closed) and items[-1].result == "the answer" async def test_activity_streams_tokens(stream_client: Client): diff --git a/tests/contrib/streams/examples/turn_scoped_stream.py b/tests/contrib/streams/examples/turn_scoped_stream.py index ab79a9a4d0..c29ea6de6a 100644 --- a/tests/contrib/streams/examples/turn_scoped_stream.py +++ b/tests/contrib/streams/examples/turn_scoped_stream.py @@ -1,6 +1,6 @@ """Brook API proposal, "A Workflow scopes a Stream to one turn and closes it". -Each turn gets its own Stream, a renamed definition from ``streams.with_name``, which +Each turn gets its own Stream, a renamed definition from ``common.with_name``, which ``async with`` closes at the end of the turn. An index Stream lists the turns, and continue-as-new passes it to the next run. """ @@ -11,32 +11,31 @@ from dataclasses import dataclass from datetime import timedelta -from temporalio import activity, workflow +from temporalio import activity, common, workflow from temporalio.client import Client -from temporalio.common import streams from temporalio.worker import Worker -@streams.defn(name="events") +@common.defn(name="events") class Events: - tokens = streams.topic(str) + tokens = common.topic(str) @dataclass class TurnStarted: n: int - stream: streams.ClientStream[Events] + stream: common.ClientStream[Events] -@streams.defn(name="index") +@common.defn(name="index") class TurnIndex: - started = streams.topic(TurnStarted) + started = common.topic(TurnStarted) @dataclass class CallModelInput: prompt: str - stream: streams.ActivityStream[Events] + stream: common.ActivityStream[Events] @activity.defn @@ -50,7 +49,7 @@ class ChatTurnsInput: # The first turn number of this run. A Stream name must be unique in the run chain. next: int = 0 # From the run before continue-as-new. - index: streams.WorkflowStream[TurnIndex] | None = None + index: common.WorkflowStream[TurnIndex] | None = None @workflow.defn @@ -64,16 +63,16 @@ async def run(self, input: ChatTurnsInput) -> None: batch = input.prompts[:2] for i, prompt in enumerate(batch): async with workflow.create_stream( - streams.with_name(Events, f"turn-{input.next + i}"), + common.with_name(Events, f"turn-{input.next + i}"), retention=timedelta(days=1), ) as turn: # A Stream is data, so the index Record carries the turn's Stream. await started.publish( - TurnStarted(input.next + i, streams.ClientStream.from_handle(turn)) + TurnStarted(input.next + i, common.ClientStream.from_handle(turn)) ) await workflow.execute_activity( call_model_into, - CallModelInput(prompt, streams.ActivityStream.from_handle(turn)), + CallModelInput(prompt, common.ActivityStream.from_handle(turn)), start_to_close_timeout=timedelta(minutes=5), ) if len(input.prompts) > len(batch): @@ -101,13 +100,13 @@ async def main(client: Client, task_queue: str) -> dict[int, list[str]]: index = handle.stream(TurnIndex) turns = {} async for item in index.subscribe(): - if not isinstance(item, streams.Record): + if not isinstance(item, common.Record): continue # A Stream from a payload has no client, so the reader binds it to this one. turn = client.stream(Events, stream=item.data.stream) turns[item.data.n] = [ token.data async for token in turn.subscribe() - if isinstance(token, streams.Record) + if isinstance(token, common.Record) ] return turns diff --git a/tests/contrib/streams/examples/ui_backend_reads.py b/tests/contrib/streams/examples/ui_backend_reads.py index ce54b92621..1960c05523 100644 --- a/tests/contrib/streams/examples/ui_backend_reads.py +++ b/tests/contrib/streams/examples/ui_backend_reads.py @@ -11,9 +11,8 @@ from datetime import timedelta from typing import Any -from temporalio import activity, workflow +from temporalio import activity, common, workflow from temporalio.client import Client -from temporalio.common import streams from temporalio.worker import Worker @@ -23,16 +22,16 @@ class Turn: text: str = "" -@streams.defn(name="events") +@common.defn(name="events") class Events: - turns = streams.topic(Turn) - tokens = streams.topic(str) + turns = common.topic(Turn) + tokens = common.topic(str) @dataclass class CallModelInput: prompt: str - stream: streams.ActivityStream[Events] + stream: common.ActivityStream[Events] @activity.defn @@ -52,7 +51,7 @@ async def run(self, prompt: str) -> str: await turns.publish(Turn("model")) reply = await workflow.execute_activity( call_model, - CallModelInput(prompt, streams.ActivityStream.from_handle(stream)), + CallModelInput(prompt, common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(minutes=5), ) await turns.publish(Turn("done", reply)) @@ -67,10 +66,10 @@ class Screen: """ def __init__(self) -> None: - self.shown: dict[int, streams.Record[Any]] = {} + self.shown: dict[int, common.Record[Any]] = {} self.result: Any = None - def render(self, record: streams.Record[Any]) -> None: + def render(self, record: common.Record[Any]) -> None: self.shown[record.offset] = record def unrender(self, offset: int) -> None: @@ -90,9 +89,9 @@ async def main(client: Client, task_queue: str) -> Screen: async for item in stream.subscribe( Events.turns, Events.tokens, create_wait=timedelta(seconds=10) ): - if isinstance(item, streams.Closed): + if isinstance(item, common.Closed): screen.result = item.result if item.failure is None else item.failure - elif isinstance(item, streams.Superseded): + elif isinstance(item, common.Superseded): # The offsets of the Records that this read showed and the item withdraws. for offset in item.withdrawn_offsets: screen.unrender(offset) diff --git a/tests/contrib/streams/examples/worker_setup.py b/tests/contrib/streams/examples/worker_setup.py index 3c12967b34..04b3a12b15 100644 --- a/tests/contrib/streams/examples/worker_setup.py +++ b/tests/contrib/streams/examples/worker_setup.py @@ -10,17 +10,16 @@ import uuid -from temporalio import workflow +from temporalio import common, workflow from temporalio.client import Client -from temporalio.common import streams from temporalio.contrib import stream as redis from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.worker import Worker -@streams.defn(name="events") +@common.defn(name="events") class Events: - tokens = streams.topic(str) + tokens = common.topic(str) @workflow.defn @@ -79,7 +78,7 @@ async def main(client: Client, task_queue: str) -> list[str]: return [ item.data async for item in stream.subscribe() - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] diff --git a/tests/contrib/streams/fake_extension.py b/tests/contrib/streams/fake_extension.py index b2cdd59a05..8b9619b848 100644 --- a/tests/contrib/streams/fake_extension.py +++ b/tests/contrib/streams/fake_extension.py @@ -7,9 +7,9 @@ import temporalio.api.streamservice.v1 as stream_pb from temporalio.bridge.streams import StreamCallError from temporalio.client import Client -from temporalio.common.streams import StreamBackend -from temporalio.common.streams._backend import _ExtensionCalls -from temporalio.common.streams.proto import v1 as ext_pb +from temporalio.common import StreamBackend +from temporalio.common._streams._backend import _ExtensionCalls +from temporalio.common._streams.proto import v1 as ext_pb from temporalio.service import RPCStatusCode diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index c7c7820c14..ca1c305c75 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -11,10 +11,10 @@ import pytest -from temporalio import activity, workflow +from temporalio import activity, common, workflow from temporalio.client import Client, WorkflowHandle -from temporalio.common import RetryPolicy, streams -from temporalio.common.streams import _activity +from temporalio.common import RetryPolicy +from temporalio.common._streams import _activity from temporalio.contrib import stream as redis from temporalio.exceptions import CancelledError from tests.helpers import new_worker @@ -26,31 +26,31 @@ class Turn: text: str = "" -@streams.defn(name="events") +@common.defn(name="events") class Events: - turns = streams.topic(Turn) - tokens = streams.topic(str) + turns = common.topic(Turn) + tokens = common.topic(str) @dataclass class CallModelInput: prompt: str - stream: streams.ActivityStream[Events] + stream: common.ActivityStream[Events] @dataclass class SyncInput: prompt: str - stream: streams.SyncActivityStream[Events] + stream: common.SyncActivityStream[Events] async def read(handle: WorkflowHandle[Any, Any], name: str = "events") -> list[object]: - stream = handle.stream(streams.with_name(Events, name)) + stream = handle.stream(common.with_name(Events, name)) return [item async for item in stream.subscribe(wait_new_records=False)] -def records(items: Sequence[object]) -> list[streams.Record[Any]]: - return [item for item in items if isinstance(item, streams.Record)] +def records(items: Sequence[object]) -> list[common.Record[Any]]: + return [item for item in items if isinstance(item, common.Record)] @activity.defn @@ -58,7 +58,7 @@ async def call_model(input: CallModelInput) -> list[int]: assert input.stream.definition is Events tokens = input.stream.topic(Events.tokens) offsets = [await tokens.publish(word) for word in input.prompt.split()] - async with streams.BufferedPublisher( + async with common.BufferedPublisher( input.stream.topic(Events.turns), max_delay=timedelta(seconds=10) ) as turns: turns.publish(Turn("model", "a")) @@ -75,11 +75,11 @@ def call_model_sync(input: SyncInput) -> int: @activity.defn -async def read_back(stream: streams.ActivityStream[Events]) -> list[Any]: +async def read_back(stream: common.ActivityStream[Events]) -> list[Any]: return [ item.data async for item in stream.subscribe(Events.tokens, wait_new_records=False) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] @@ -90,17 +90,17 @@ async def run(self, prompt: str) -> list[Any]: stream = await workflow.create_stream(Events) offsets = await workflow.execute_activity( call_model, - CallModelInput(prompt, streams.ActivityStream.from_handle(stream)), + CallModelInput(prompt, common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), ) last = await workflow.execute_activity( call_model_sync, - SyncInput("sync words", streams.SyncActivityStream.from_handle(stream)), + SyncInput("sync words", common.SyncActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), ) read = await workflow.execute_activity( read_back, - streams.ActivityStream.from_handle(stream), + common.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) await stream.close() @@ -132,7 +132,7 @@ async def test_activity_publishes_and_returns_offsets(stream_client: Client): assert offsets[2] == data[3].offset + 1 assert last == data[5].offset assert read_tokens == ["hi", "there", "sync", "words"] - assert isinstance(items[-1], streams.Closed) + assert isinstance(items[-1], common.Closed) def _executor() -> Any: @@ -160,7 +160,7 @@ async def run(self, key: str) -> None: await stream.topic(Events.tokens).publish("start") await workflow.execute_activity( flaky, - CallModelInput(key, streams.ActivityStream.from_handle(stream)), + CallModelInput(key, common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), retry_policy=RetryPolicy(initial_interval=timedelta(milliseconds=10)), ) @@ -174,7 +174,7 @@ async def test_retry_withdraws_a_received_stream(stream_client: Client): ) await handle.result() items = await read(handle) - superseded = [item for item in items if isinstance(item, streams.Superseded)] + superseded = [item for item in items if isinstance(item, common.Superseded)] # The second attempt writes nothing, and its start still withdraws the first one's Record. assert len(superseded) == 1 withdrawn = [r.data for r in records(items) if superseded[0].withdraws(r)] @@ -184,8 +184,8 @@ async def test_retry_withdraws_a_received_stream(stream_client: Client): @dataclass class FinishedStreams: key: str - closed: streams.ActivityStream[Events] - expired: streams.ActivityStream[Events] + closed: common.ActivityStream[Events] + expired: common.ActivityStream[Events] @activity.defn @@ -194,13 +194,13 @@ async def reads_after_a_failed_attempt(input: FinishedStreams) -> list[Any]: if attempts[input.key] == 1: raise RuntimeError("first attempt fails") items = input.closed.subscribe(Events.tokens, wait_new_records=False) - return [item.data async for item in items if isinstance(item, streams.Record)] + return [item.data async for item in items if isinstance(item, common.Record)] @workflow.defn class ReadsFinishedStreams: @workflow.run - async def run(self, key: str, expired: streams.WorkflowStream[Events]) -> list[Any]: + async def run(self, key: str, expired: common.WorkflowStream[Events]) -> list[Any]: stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("turn") await stream.close() @@ -208,8 +208,8 @@ async def run(self, key: str, expired: streams.WorkflowStream[Events]) -> list[A reads_after_a_failed_attempt, FinishedStreams( key, - streams.ActivityStream.from_handle(stream), - streams.ActivityStream.from_handle(expired), + common.ActivityStream.from_handle(stream), + common.ActivityStream.from_handle(expired), ), start_to_close_timeout=timedelta(seconds=30), retry_policy=RetryPolicy(initial_interval=timedelta(milliseconds=10)), @@ -233,7 +233,7 @@ async def test_retry_goes_on_with_a_closed_or_expired_stream(stream_client: Clie ) as worker: result = await stream_client.execute_workflow( ReadsFinishedStreams.run, - args=[key, streams.WorkflowStream.from_handle(expired)], + args=[key, common.WorkflowStream.from_handle(expired)], id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -262,7 +262,7 @@ async def run(self, key: str) -> None: stream = await workflow.create_stream(Events) await workflow.execute_activity( fails_after_it_is_shown, - CallModelInput(key, streams.ActivityStream.from_handle(stream)), + CallModelInput(key, common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), retry_policy=RetryPolicy(initial_interval=timedelta(milliseconds=10)), ) @@ -297,34 +297,34 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( resumed = [ item async for item in stream.subscribe(Events.tokens, offset=resume_at) ] - assert isinstance(first, streams.Record) and first.data == "attempt-1" - assert isinstance(before, streams.Record) and before.offset == first.offset - [live_superseded] = [i for i in live_items if isinstance(i, streams.Superseded)] + assert isinstance(first, common.Record) and first.data == "attempt-1" + assert isinstance(before, common.Record) and before.offset == first.offset + [live_superseded] = [i for i in live_items if isinstance(i, common.Superseded)] assert live_superseded.withdrawn_offsets == (first.offset,) # A new read gave no Record of the failed attempt, so it names no offset. The kept # Record is still withdrawn. - [superseded] = [i for i in resumed if isinstance(i, streams.Superseded)] + [superseded] = [i for i in resumed if isinstance(i, common.Superseded)] assert superseded.withdrawn_offsets == () assert superseded.withdraws(before) assert [r.data for r in records(resumed)] == ["attempt-2"] - assert isinstance(resumed[-1], streams.Closed) + assert isinstance(resumed[-1], common.Closed) @activity.defn -async def wrong_backend(stream: streams.ActivityStream[Events]) -> str: +async def wrong_backend(stream: common.ActivityStream[Events]) -> str: try: await stream.topic(Events.tokens).publish("x") - except streams.StreamConfigError: + except common.StreamConfigError: return "config" return "wrote" @activity.defn -async def close_it(stream: streams.ActivityStream[Events]) -> str: +async def close_it(stream: common.ActivityStream[Events]) -> str: await stream.topic(Events.tokens).publish("last") try: await stream.close(result="closed by activity") - except streams.StreamNotOwnerError as err: + except common.StreamNotOwnerError as err: return err.reason return "closed" @@ -336,24 +336,24 @@ async def run(self) -> list[str]: stream = await workflow.create_stream(Events) backend = await workflow.execute_activity( wrong_backend, - streams.ActivityStream(id=stream.id, backend="elsewhere"), + common.ActivityStream(id=stream.id, backend="elsewhere"), start_to_close_timeout=timedelta(seconds=30), ) owner = await workflow.execute_activity( close_it, - streams.ActivityStream.from_handle(stream), + common.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) # A Local Activity writes before its Workflow Task commits, so it can't close either. local = await workflow.execute_local_activity( close_it, - streams.ActivityStream.from_handle(stream), + common.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) # A handle with only the id names no Workflow, so the Store refuses it. by_id = await workflow.execute_local_activity( close_it, - streams.ActivityStream(id=stream.id), + common.ActivityStream(id=stream.id), start_to_close_timeout=timedelta(seconds=30), ) await stream.close(result="closed by the run") @@ -376,7 +376,7 @@ async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): ] items = await read(handle) assert [r.data for r in records(items)] == ["last", "last", "last"] - assert isinstance(items[-1], streams.Closed) + assert isinstance(items[-1], common.Closed) assert items[-1].result == "closed by the run" @@ -395,7 +395,7 @@ async def run(self, stream_id: str) -> str: # The handle has only the id, so the SDK can't tell that another chain owns it. return await workflow.execute_local_activity( close_it, - streams.ActivityStream(id=stream_id), + common.ActivityStream(id=stream_id), start_to_close_timeout=timedelta(seconds=30), ) @@ -424,11 +424,11 @@ async def test_store_refuses_another_chains_close(stream_client: Client): ) stream = stream_client.stream(Events, id=stream_id) items = [item async for item in stream.subscribe(wait_new_records=False)] - assert [type(item) for item in items] == [streams.Record] + assert [type(item) for item in items] == [common.Record] @dataclass -class _Batches(streams.ActivityTopic[int]): +class _Batches(common.ActivityTopic[int]): batches: list[tuple[int, ...]] = field(default_factory=list) fail: bool = False in_flight: int = 0 @@ -439,7 +439,7 @@ def __post_init__(self) -> None: async def publish(self, *values: int) -> int: if self.fail: - raise streams.StreamStorageError("store down") + raise common.StreamStorageError("store down") self.in_flight += 1 self.most_in_flight = max(self.most_in_flight, self.in_flight) await asyncio.sleep(0.01) @@ -450,7 +450,7 @@ async def publish(self, *values: int) -> int: async def test_buffered_publisher_batches_by_delay(): topic = _Batches() - publisher = streams.BufferedPublisher(topic, max_delay=timedelta(milliseconds=20)) + publisher = common.BufferedPublisher(topic, max_delay=timedelta(milliseconds=20)) publisher.publish(1) publisher.publish(2, 3) await asyncio.sleep(0.1) @@ -463,7 +463,7 @@ async def test_buffered_publisher_batches_by_delay(): async def test_buffered_publisher_writes_a_full_batch_with_one_write_in_flight(): topic = _Batches() - publisher = streams.BufferedPublisher(topic, max_delay=timedelta(minutes=10)) + publisher = common.BufferedPublisher(topic, max_delay=timedelta(minutes=10)) publisher.publish(*range(1000)) publisher.publish(*range(1000, 2500)) await asyncio.sleep(0.2) @@ -476,21 +476,21 @@ async def test_buffered_publisher_writes_a_full_batch_with_one_write_in_flight() async def test_buffered_publisher_raises_a_failed_batch(): topic = _Batches(fail=True) - with pytest.raises(streams.StreamStorageError): - async with streams.BufferedPublisher(topic) as publisher: + with pytest.raises(common.StreamStorageError): + async with common.BufferedPublisher(topic) as publisher: publisher.publish(1) with pytest.raises(TypeError): - streams.BufferedPublisher(topic).publish() + common.BufferedPublisher(topic).publish() @activity.defn -async def local_step(stream: streams.ActivityStream[Events]) -> list[Any]: +async def local_step(stream: common.ActivityStream[Events]) -> list[Any]: # Each Workflow write is an awaited Local Activity, so the run's create and Records are in # the Store before a later Local Activity starts. seen = [ item.data async for item in stream.subscribe(wait_new_records=False) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] await stream.topic(Events.tokens).publish("local") return seen @@ -505,7 +505,7 @@ async def run(self) -> list[Any]: await tokens.publish("before") seen = await workflow.execute_local_activity( local_step, - streams.ActivityStream.from_handle(stream), + common.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) await tokens.publish("after") @@ -524,27 +524,27 @@ async def test_workflow_writes_go_out_before_a_local_activity(stream_client: Cli assert [record.data for record in written] == ["before", "local", "after"] producer = written[1].producer assert producer is not None and producer.id.startswith("la:") - assert isinstance(items[-1], streams.Closed) + assert isinstance(items[-1], common.Closed) async def test_buffered_publisher_stays_failed(): topic = _Batches(fail=True) - publisher = streams.BufferedPublisher(topic) + publisher = common.BufferedPublisher(topic) publisher.publish(1, 2) - with pytest.raises(streams.StreamStorageError): + with pytest.raises(common.StreamStorageError): await publisher.flush() topic.fail = False # A later batch would leave a gap, so the publisher refuses it and keeps the error. - with pytest.raises(streams.StreamStorageError): + with pytest.raises(common.StreamStorageError): publisher.publish(3) - with pytest.raises(streams.StreamStorageError): + with pytest.raises(common.StreamStorageError): await publisher.flush() assert topic.batches == [] and publisher.unwritten == 2 @activity.defn async def never_flushes(input: CallModelInput) -> None: - publisher = streams.BufferedPublisher( + publisher = common.BufferedPublisher( input.stream.topic(Events.tokens), max_delay=timedelta(minutes=10) ) publisher.publish(*input.prompt.split()) @@ -553,8 +553,8 @@ async def never_flushes(input: CallModelInput) -> None: @dataclass class ManyStreams: key: str - targets: list[streams.ActivityStream[Events]] - maybe: streams.ActivityStream[Events] | None = None + targets: list[common.ActivityStream[Events]] + maybe: common.ActivityStream[Events] | None = None @activity.defn @@ -568,11 +568,11 @@ async def flaky_many(input: ManyStreams) -> None: @activity.defn -def sync_read(stream: streams.SyncActivityStream[Events]) -> list[Any]: +def sync_read(stream: common.SyncActivityStream[Events]) -> list[Any]: return [ item.data for item in stream.subscribe(Events.tokens, wait_new_records=False) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] @@ -581,25 +581,25 @@ class Leftovers: @workflow.run async def run(self, key: str) -> list[Any]: stream = await workflow.create_stream(Events) - other = await workflow.create_stream(streams.with_name(Events, "other")) + other = await workflow.create_stream(common.with_name(Events, "other")) await workflow.execute_activity( never_flushes, - CallModelInput("a b", streams.ActivityStream.from_handle(stream)), + CallModelInput("a b", common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), ) await workflow.execute_activity( flaky_many, ManyStreams( key, - [streams.ActivityStream.from_handle(stream)], - streams.ActivityStream.from_handle(other), + [common.ActivityStream.from_handle(stream)], + common.ActivityStream.from_handle(other), ), start_to_close_timeout=timedelta(seconds=30), retry_policy=RetryPolicy(initial_interval=timedelta(milliseconds=10)), ) return await workflow.execute_activity( sync_read, - streams.SyncActivityStream.from_handle(stream), + common.SyncActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) @@ -622,7 +622,7 @@ async def test_attempt_end_flushes_and_retry_finds_nested_streams( items = await read(handle) other = await read(handle, "other") for read_items in (items, other): - superseded = [i for i in read_items if isinstance(i, streams.Superseded)] + superseded = [i for i in read_items if isinstance(i, common.Superseded)] withdrawn = [r.data for r in records(read_items) if superseded[0].withdraws(r)] assert withdrawn == ["attempt-1"] @@ -641,7 +641,7 @@ async def long_poll() -> None: finally: stopped.set() - stream: streams.SyncActivityStream[Any] = streams.SyncActivityStream(id="s") + stream: common.SyncActivityStream[Any] = common.SyncActivityStream(id="s") token = _activity.current.set( cast(Any, SimpleNamespace(loop=loop, backend_name="", default=True)) ) @@ -685,7 +685,7 @@ async def run(self) -> str: @workflow.defn class ReceivesAStream: @workflow.run - async def run(self, stream: streams.WorkflowStream[Events]) -> str: + async def run(self, stream: common.WorkflowStream[Events]) -> str: # The Activity gets no Stream, but the run received one. return await workflow.execute_activity( plain, stream.id, start_to_close_timeout=timedelta(seconds=10) @@ -695,15 +695,13 @@ async def run(self, stream: streams.WorkflowStream[Events]) -> str: @workflow.defn class RetriesAfterContinueAsNew: @workflow.run - async def run( - self, key: str, stream: streams.WorkflowStream[Events] | None - ) -> None: + async def run(self, key: str, stream: common.WorkflowStream[Events] | None) -> None: if stream is None: created = await workflow.create_stream(Events) workflow.continue_as_new(args=[key, created]) await workflow.execute_activity( flaky, - CallModelInput(key, streams.ActivityStream.from_handle(stream)), + CallModelInput(key, common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), retry_policy=RetryPolicy(initial_interval=timedelta(milliseconds=10)), ) @@ -748,7 +746,7 @@ async def test_only_a_run_that_uses_streams_keys_its_activities(stream_client: C assert await keyed(creates, "flaky") == [True] receives = await stream_client.start_workflow( ReceivesAStream.run, - streams.WorkflowStream.from_handle(creates.stream(Events)), + common.WorkflowStream.from_handle(creates.stream(Events)), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -766,7 +764,7 @@ async def test_only_a_run_that_uses_streams_keys_its_activities(stream_client: C await continued.result() assert await keyed(continued, "flaky") == [True] items = await read(continued) - superseded = [item for item in items if isinstance(item, streams.Superseded)] + superseded = [item for item in items if isinstance(item, common.Superseded)] assert len(superseded) == 1 assert [r.data for r in records(items) if superseded[0].withdraws(r)] == [ "attempt-1" diff --git a/tests/contrib/streams/test_bindings.py b/tests/contrib/streams/test_bindings.py index 95e5d0dcb4..61ae5e45d5 100644 --- a/tests/contrib/streams/test_bindings.py +++ b/tests/contrib/streams/test_bindings.py @@ -4,7 +4,7 @@ import temporalio.api.streamservice.v1 as stream_pb import temporalio.converter import temporalio.nexus.system as nexus_system -from temporalio.common.streams.system import client, models +from temporalio.common._streams.system import client, models class FakeTransport: diff --git a/tests/contrib/streams/test_facade.py b/tests/contrib/streams/test_facade.py index a0e66004ee..d3b6fb5310 100644 --- a/tests/contrib/streams/test_facade.py +++ b/tests/contrib/streams/test_facade.py @@ -6,12 +6,13 @@ import pytest import temporalio.api.streamservice.v1 as stream_pb +from temporalio import common from temporalio.bridge.streams import StreamCallError from temporalio.client import Client -from temporalio.common import RawValue, streams -from temporalio.common.streams import _errors -from temporalio.common.streams.proto import v1 as ext_pb -from temporalio.common.streams.system import models +from temporalio.common import RawValue +from temporalio.common._streams import _errors +from temporalio.common._streams.proto import v1 as ext_pb +from temporalio.common._streams.system import models from temporalio.contrib import stream as redis from temporalio.exceptions import ApplicationError from temporalio.service import RPCError, RPCStatusCode @@ -29,12 +30,12 @@ def test_runtime_errors_map_to_stream_errors(): err = _errors.from_call_error( StreamCallError(int(RPCStatusCode.FAILED_PRECONDITION), "fenced", details) ) - assert isinstance(err, streams.StreamFencedError) + assert isinstance(err, common.StreamFencedError) assert err.reason == "stream_fenced" other = _errors.from_call_error( StreamCallError(int(RPCStatusCode.INTERNAL), "odd", b"") ) - assert type(other) is streams.StreamError + assert type(other) is common.StreamError def test_superseded_withdraws_earlier_attempts_of_one_producer(): @@ -46,7 +47,7 @@ def producer(attempt: int) -> models.Producer: epoch=models.ProducerEpoch(attempt=attempt), ) - item = streams.Superseded( + item = common.Superseded( offset=9, supersession=models.Supersession( producer_kind=stream_pb.ProducerKind.PRODUCER_KIND_ACTIVITY, @@ -54,16 +55,16 @@ def producer(attempt: int) -> models.Producer: epoch=models.ProducerEpoch(attempt=2), ), ) - old = streams.Record(topic="t", data=1, offset=1, producer=producer(1)) - new = streams.Record(topic="t", data=2, offset=10, producer=producer(2)) + old = common.Record(topic="t", data=1, offset=1, producer=producer(1)) + new = common.Record(topic="t", data=2, offset=10, producer=producer(2)) assert item.withdraws(old) assert not item.withdraws(new) -@streams.defn(name="events") +@common.defn(name="events") class Events: - turns = streams.topic(Turn) - tokens = streams.topic(str) + turns = common.topic(Turn) + tokens = common.topic(str) async def test_client_writes_reads_and_closes(client: Client): @@ -76,16 +77,16 @@ async def test_client_writes_reads_and_closes(client: Client): await stream.close(result="bye") items = [item async for item in stream.subscribe()] assert [type(item) for item in items] == [ - streams.Record, - streams.Record, - streams.Closed, + common.Record, + common.Record, + common.Closed, ] turn, token, closed = items - assert isinstance(turn, streams.Record) and turn.data == Turn("model") - assert isinstance(token, streams.Record) and token.data == "hello" + assert isinstance(turn, common.Record) and turn.data == Turn("model") + assert isinstance(token, common.Record) and token.data == "hello" assert turn.producer is not None assert turn.producer.kind == stream_pb.ProducerKind.PRODUCER_KIND_CLIENT - assert isinstance(closed, streams.Closed) and closed.result == "bye" + assert isinstance(closed, common.Closed) and closed.result == "bye" assert extension.namespaces == {client.namespace} raw = stream.topic("raw", type=RawValue) @@ -98,7 +99,7 @@ async def test_read_waits_for_create(client: Client): client, _ = fake_client(client) handle = client.get_workflow_handle("wf-1", first_execution_run_id="run-1") stream = handle.stream(Events) - with pytest.raises(streams.StreamNotFoundError): + with pytest.raises(common.StreamNotFoundError): async for _ in stream.subscribe(): pass @@ -110,15 +111,15 @@ async def create_later() -> None: task = asyncio.create_task(create_later()) items = [item async for item in stream.subscribe(create_wait=timedelta(seconds=5))] await task - assert isinstance(items[0], streams.Record) and items[0].data == "five" - assert isinstance(items[-1], streams.Closed) + assert isinstance(items[0], common.Record) and items[0].data == "five" + assert isinstance(items[-1], common.Closed) def test_stream_errors_fail_the_workflow_not_the_task(): - err = streams.StreamClosedError("closed", reason="stream_closed") + err = common.StreamClosedError("closed", reason="stream_closed") assert isinstance(err, ApplicationError) assert err.non_retryable and err.type == "StreamClosedError" - assert not streams.StreamStorageError("down").non_retryable + assert not common.StreamStorageError("down").non_retryable details = ext_pb.StreamErrorDetails(reason="stream_closed").SerializeToString() mapped = _errors.from_call_error( StreamCallError(int(RPCStatusCode.FAILED_PRECONDITION), "closed", details) @@ -136,7 +137,7 @@ async def test_handles_of_one_client_share_a_producer_and_free_it(client: Client producers = { item.producer.id for item in items - if isinstance(item, streams.Record) and item.producer + if isinstance(item, common.Record) and item.producer } assert len(producers) == 1 # A handle that goes with no close frees its writer, so a server that makes a handle per @@ -171,7 +172,7 @@ async def publish_later() -> None: item = await anext(reader) await reader.aclose() await publisher - assert isinstance(item, streams.Record) and item.data == "late" + assert isinstance(item, common.Record) and item.data == "late" def test_unsupported_capability_maps_to_its_error(): @@ -181,6 +182,6 @@ def test_unsupported_capability_maps_to_its_error(): int(RPCStatusCode.UNIMPLEMENTED), "The Store lacks replica_wait", details ) ) - assert isinstance(err, streams.StreamUnsupportedError) + assert isinstance(err, common.StreamUnsupportedError) assert err.non_retryable and err.code == RPCStatusCode.UNIMPLEMENTED assert err.message.endswith("replica_wait") diff --git a/tests/contrib/streams/test_handles.py b/tests/contrib/streams/test_handles.py index a1df381fb2..e7de9095cf 100644 --- a/tests/contrib/streams/test_handles.py +++ b/tests/contrib/streams/test_handles.py @@ -1,12 +1,16 @@ import json +import subprocess +import sys +import types from dataclasses import dataclass from typing import Any, TypeVar import pydantic import pytest -from temporalio.common import RawValue, streams -from temporalio.common.streams._defn import definition_of +from temporalio import common +from temporalio.common import RawValue +from temporalio.common._streams._defn import definition_of from temporalio.contrib.pydantic import pydantic_data_converter from temporalio.converter import DataConverter @@ -18,19 +22,19 @@ class Turn: kind: str -@streams.defn(name="events") +@common.defn(name="events") class Events: - turns = streams.topic(Turn) - tokens = streams.topic(str) - raw = streams.topic(RawValue, name="raw-values") + turns = common.topic(Turn) + tokens = common.topic(str) + raw = common.topic(RawValue, name="raw-values") -@streams.defn +@common.defn class Index: - turns = streams.topic(int) + turns = common.topic(int) -class PlainStream(streams.StreamHandle[E]): +class PlainStream(common.StreamHandle[E]): pass @@ -46,19 +50,19 @@ class PlainStream(streams.StreamHandle[E]): def test_definition_rejects_two_topics_of_one_name(): with pytest.raises(ValueError, match="two Topics named t"): - @streams.defn + @common.defn class Twice: # pyright: ignore[reportUnusedClass] - a = streams.topic(int, name="t") - b = streams.topic(str, name="t") + a = common.topic(int, name="t") + b = common.topic(str, name="t") def test_subclass_topic_replaces_the_base_topic(): class Base: - turns = streams.topic(int) + turns = common.topic(int) - @streams.defn + @common.defn class Sub(Base): - turns = streams.topic(str) + turns = common.topic(str) assert definition_of(Sub).topics["turns"].type is str @@ -72,10 +76,10 @@ def test_topic_of_the_definition_keeps_its_type(): assert OWNED._topic_spec(Events.turns, None) == ("turns", Turn) assert OWNED._topic_spec("tokens", None) == ("tokens", str) assert OWNED._topic_spec("custom", Turn) == ("custom", Turn) - with pytest.raises(streams.StreamError, match="has no Topic turns") as err: + with pytest.raises(common.StreamError, match="has no Topic turns") as err: OWNED._topic_spec(Index.turns, None) assert err.value.reason == "unknown_topic" - with pytest.raises(streams.StreamError, match="Pass type="): + with pytest.raises(common.StreamError, match="Pass type="): OWNED._topic_spec("custom", None) with pytest.raises(TypeError, match="has the type"): OWNED._topic_spec("tokens", int) @@ -105,7 +109,7 @@ def test_type_hint_picks_the_class_and_the_definition(): bare = converter.from_payload(payload, PlainStream) assert bare.definition is None and bare.backend == "redis" with pytest.raises(Exception, match="handle type of its context"): - converter.from_payload(payload, streams.StreamHandle) + converter.from_payload(payload, common.StreamHandle) @dataclass @@ -153,4 +157,27 @@ def test_any_type_argument_has_no_definition(): def test_end_type_is_public(): # subscribe takes offset: int | End | None, so the type of END is a public name. - assert isinstance(streams.END, streams.End) and "End" in streams.__all__ + assert isinstance(common.END, common.End) and "End" in common.__all__ + + +def test_temporalio_common_exports_the_streams_api(): + # The Stream names load on first use, and each is the object of the private package. + import temporalio.common._streams as streams_api + + names = { + name + for name, value in vars(streams_api).items() + if not name.startswith("_") and not isinstance(value, types.ModuleType) + } + assert names == common._STREAM_NAMES and names <= set(common.__all__) + for name in common._STREAM_NAMES: + assert getattr(common, name) is getattr(streams_api, name) + + +def test_import_of_temporalio_common_loads_no_stream_code(): + # The Stream modules import temporalio.client and temporalio.workflow. + code = "import sys, temporalio.common; print('temporalio.common._streams' in sys.modules)" + loaded = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, check=True + ).stdout + assert loaded.strip() == "False" diff --git a/tests/contrib/streams/test_pydantic.py b/tests/contrib/streams/test_pydantic.py index 89ead4949d..f9745414c1 100644 --- a/tests/contrib/streams/test_pydantic.py +++ b/tests/contrib/streams/test_pydantic.py @@ -3,27 +3,26 @@ import pydantic -from temporalio import activity, workflow +from temporalio import activity, common, workflow from temporalio.client import Client -from temporalio.common import streams from temporalio.contrib.pydantic import pydantic_data_converter from temporalio.contrib.stream.testing import MemoryStreamBackend from tests.helpers import new_worker -@streams.defn(name="events") +@common.defn(name="events") class Events: - tokens = streams.topic(str) + tokens = common.topic(str) class CallModelInput(pydantic.BaseModel): prompt: str - stream: streams.ActivityStream[Events] + stream: common.ActivityStream[Events] class TurnResult(pydantic.BaseModel): text: str - stream: streams.ClientStream[Events] + stream: common.ClientStream[Events] @activity.defn @@ -40,13 +39,13 @@ async def run(self, prompt: str) -> TurnResult: await workflow.execute_activity( call_model, CallModelInput( - prompt=prompt, stream=streams.ActivityStream.from_handle(stream) + prompt=prompt, stream=common.ActivityStream.from_handle(stream) ), start_to_close_timeout=timedelta(seconds=30), ) await stream.close(result=prompt.upper()) return TurnResult( - text=prompt.upper(), stream=streams.ClientStream.from_handle(stream) + text=prompt.upper(), stream=common.ClientStream.from_handle(stream) ) @@ -69,5 +68,5 @@ async def test_handles_in_models_round_trip_on_the_memory_store(client: Client): assert result.stream.backend == "memory" stream = client.stream(Events, id=result.stream.id) items = [item async for item in stream.subscribe()] - assert [i.data for i in items if isinstance(i, streams.Record)] == ["hi", "there"] - assert isinstance(items[-1], streams.Closed) and items[-1].result is not None + assert [i.data for i in items if isinstance(i, common.Record)] == ["hi", "there"] + assert isinstance(items[-1], common.Closed) and items[-1].result is not None diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 48bbd18dda..0388c787d8 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -10,12 +10,12 @@ import pytest -from temporalio import activity, workflow +from temporalio import activity, common, workflow from temporalio.api.common.v1 import Payload from temporalio.api.enums.v1 import EventType from temporalio.client import Client, Interceptor, WorkflowFailureError, WorkflowHandle -from temporalio.common import RawValue, RetryPolicy, streams -from temporalio.common.streams._extension import ( +from temporalio.common import RawValue, RetryPolicy +from temporalio.common._streams._extension import ( ExtensionStreamBackend, _WorkflowInbound, ) @@ -45,30 +45,30 @@ class Turn: text: str = "" -@streams.defn(name="events") +@common.defn(name="events") class Events: - turns = streams.topic(Turn) - tokens = streams.topic(str) + turns = common.topic(Turn) + tokens = common.topic(str) @dataclass class CallModelInput: prompt: str - stream: streams.ActivityStream[Events] + stream: common.ActivityStream[Events] def workflow_stream( handle: WorkflowHandle[Any, Any], name: str | None = None -) -> streams.ClientStream[Events]: - return handle.stream(streams.with_name(Events, name) if name else Events) +) -> common.ClientStream[Events]: + return handle.stream(common.with_name(Events, name) if name else Events) -async def read_all(stream: streams.ClientStream[Any]) -> list[Any]: +async def read_all(stream: common.ClientStream[Any]) -> list[Any]: return [item async for item in stream.subscribe(create_wait=timedelta(seconds=20))] -def records(items: list[Any]) -> list[streams.Record[Any]]: - return [item for item in items if isinstance(item, streams.Record)] +def records(items: list[Any]) -> list[common.Record[Any]]: + return [item for item in items if isinstance(item, common.Record)] @activity.defn @@ -88,7 +88,7 @@ async def run(self, prompt: str) -> str: await turns.publish(Turn("model")) reply = await workflow.execute_activity( call_model, - CallModelInput(prompt, streams.ActivityStream.from_handle(stream)), + CallModelInput(prompt, common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), ) await turns.publish(Turn("done", reply)) @@ -112,12 +112,12 @@ async def test_workflow_and_activity_write_one_stream(stream_client: Client): ("turns", Turn("done", "HI THERE")), ] closed = items[-1] - assert isinstance(closed, streams.Closed) and closed.result == "HI THERE" + assert isinstance(closed, common.Closed) and closed.result == "HI THERE" only_turns = [item async for item in stream.subscribe(Events.turns)] assert [type(item) for item in only_turns] == [ - streams.Record, - streams.Record, - streams.Closed, + common.Record, + common.Record, + common.Closed, ] @@ -157,11 +157,11 @@ async def test_failed_workflow_task_is_withdrawn(stream_client: Client): items = await read_all(workflow_stream(handle)) # The retried task writes "after" again, and withdraws the copy of the failed task. written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] + superseded = [item for item in items if isinstance(item, common.Superseded)] assert [record.data for record in written] == ["before", "after", "after"] withdrawn = [r for r in written if any(i.withdraws(r) for i in superseded)] assert withdrawn == [written[1]] - assert isinstance(items[-1], streams.Closed) + assert isinstance(items[-1], common.Closed) async def test_resume_after_a_superseded_item_keeps_its_record(stream_client: Client): @@ -185,7 +185,7 @@ async def test_resume_after_a_superseded_item_keeps_its_record(stream_client: Cl [shared] = [ i for i in items - if isinstance(i, streams.Superseded) and i.offset in record_offsets + if isinstance(i, common.Superseded) and i.offset in record_offsets ] reader = stream.subscribe(wait_new_records=False) async for item in reader: @@ -254,7 +254,7 @@ async def test_later_run_closes_a_stream_it_did_not_write(stream_client: Client) items = await read_all(workflow_stream(handle)) assert [i.data for i in records(items)] == ["first run"] closed = items[-1] - assert isinstance(closed, streams.Closed) and closed.result == "second run" + assert isinstance(closed, common.Closed) and closed.result == "second run" @workflow.defn @@ -262,7 +262,7 @@ class CancelledBlock: @workflow.run async def run(self) -> None: async with await workflow.create_stream( - streams.with_name(Events, "turn") + common.with_name(Events, "turn") ) as turn: await turn.topic(Events.tokens).publish("started") await workflow.wait_condition(lambda: False) @@ -275,19 +275,19 @@ async def test_cancelled_block_closes_with_a_failure(stream_client: Client): ) turn = workflow_stream(handle, "turn") reader = turn.subscribe(create_wait=timedelta(seconds=20)) - assert isinstance(await anext(reader), streams.Record) + assert isinstance(await anext(reader), common.Record) await reader.aclose() await handle.cancel() items = await read_all(turn) closed = items[-1] - assert isinstance(closed, streams.Closed) + assert isinstance(closed, common.Closed) assert closed.failure is not None - assert closed.status == streams.CloseStatus.CANCELED + assert closed.status == common.CloseStatus.CANCELED -@streams.defn(name="plain") +@common.defn(name="plain") class Plain: - t = streams.topic(str) + t = common.topic(str) @workflow.defn @@ -297,7 +297,7 @@ async def run(self) -> str: stream = await workflow.create_stream(Plain) await stream.topic(Plain.t).publish("plain") async with await workflow.create_stream( - streams.with_name(Plain, "scoped") + common.with_name(Plain, "scoped") ) as scoped: await scoped.topic(Plain.t).publish("scoped") await stream.close(result="done") @@ -319,11 +319,11 @@ async def test_temporal_only_passthrough_keeps_records(stream_client: Client): ) await handle.result() items = await read_all(handle.stream(Plain)) - scoped = await read_all(handle.stream(streams.with_name(Plain, "scoped"))) + scoped = await read_all(handle.stream(common.with_name(Plain, "scoped"))) assert [i.data for i in records(items)] == ["plain"] - assert isinstance(items[-1], streams.Closed) and items[-1].result == "done" + assert isinstance(items[-1], common.Closed) and items[-1].result == "done" assert [i.data for i in records(scoped)] == ["scoped"] - assert isinstance(scoped[-1], streams.Closed) + assert isinstance(scoped[-1], common.Closed) @activity.defn @@ -338,7 +338,7 @@ async def run(self) -> None: stream = await workflow.create_stream(Events) await workflow.execute_local_activity( local_tokens, - CallModelInput("0 1", streams.ActivityStream.from_handle(stream)), + CallModelInput("0 1", common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), ) await stream.close() @@ -369,7 +369,7 @@ async def run(self, key: str) -> None: await workflow.execute_local_activity( local_tokens, CallModelInput( - "0 1" if first else "2 3", streams.ActivityStream.from_handle(stream) + "0 1" if first else "2 3", common.ActivityStream.from_handle(stream) ), start_to_close_timeout=timedelta(seconds=30), ) @@ -395,7 +395,7 @@ async def test_rerun_local_activity_withdraws_its_failed_task(stream_client: Cli await handle.result() items = await read_all(workflow_stream(handle)) written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] + superseded = [item for item in items if isinstance(item, common.Superseded)] visible = [r for r in written if not any(i.withdraws(r) for i in superseded)] assert [r.data for r in visible] == ["2", "3"] # The rerun keeps the Producer id and moves to the retry task's epoch. @@ -417,7 +417,7 @@ async def run(self, key: str) -> None: attempts[key] = 1 await workflow.execute_local_activity( local_tokens, - CallModelInput("0 1", streams.ActivityStream.from_handle(stream)), + CallModelInput("0 1", common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), ) raise RuntimeError("fail this Workflow Task once") @@ -443,10 +443,10 @@ async def test_later_write_withdraws_a_skipped_local_activity(stream_client: Cli await handle.result() items = await read_all(workflow_stream(handle)) written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] + superseded = [item for item in items if isinstance(item, common.Superseded)] visible = [r for r in written if not any(s.withdraws(r) for s in superseded)] assert [r.data for r in visible] == ["other"] - assert isinstance(items[-1], streams.Closed) + assert isinstance(items[-1], common.Closed) @workflow.defn @@ -475,12 +475,12 @@ async def test_retry_run_withdraws_the_failed_run(stream_client: Client): assert await handle.result() == "done" items = await read_all(workflow_stream(handle)) written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] + superseded = [item for item in items if isinstance(item, common.Superseded)] assert [record.data for record in written] == ["run-1", "run-2"] assert len(superseded) == 1 assert [r.data for r in written if superseded[0].withdraws(r)] == ["run-1"] closed = items[-1] - assert isinstance(closed, streams.Closed) and closed.result == "done" + assert isinstance(closed, common.Closed) and closed.result == "done" class _Rot13Codec(PayloadCodec): @@ -532,7 +532,7 @@ async def test_records_go_through_the_payload_codec(stream_client: Client): Turn("done", "A B"), ] first = raw[0] - assert isinstance(first, streams.Record) + assert isinstance(first, common.Record) assert first.data.payload.metadata["encoding"] == b"json/plain" @@ -547,10 +547,10 @@ async def test_client_close_goes_through_the_payload_codec(stream_client: Client failed = await redis.create(codec_client, Events, id=f"failed-{uuid.uuid4()}") await failed.close(failure=ApplicationError("bad turn")) [closed] = [item async for item in done.subscribe()] - assert isinstance(closed, streams.Closed) + assert isinstance(closed, common.Closed) assert closed.result == {"kind": "done", "text": ""} [closed] = [item async for item in failed.subscribe()] - assert isinstance(closed, streams.Closed) and str(closed.failure) == "bad turn" + assert isinstance(closed, common.Closed) and str(closed.failure) == "bad turn" @workflow.defn @@ -564,7 +564,7 @@ async def run(self) -> str: turns = stream.topic(Events.tokens) await turns.publish("before restart") async with await workflow.create_stream( - streams.with_name(Events, "scoped") + common.with_name(Events, "scoped") ) as scoped: await scoped.topic(Events.tokens).publish("scoped") await workflow.wait_condition(lambda: self.go) @@ -585,7 +585,7 @@ async def test_new_worker_replays_and_writes_each_record_once(stream_client: Cli ) first = workflow_stream(handle) reader = first.subscribe(create_wait=timedelta(seconds=20)) - assert isinstance(await anext(reader), streams.Record) + assert isinstance(await anext(reader), common.Record) await reader.aclose() # The new Worker has no cache, so it replays the run and sends the creates and the close # of the scoped Stream again. @@ -595,9 +595,9 @@ async def test_new_worker_replays_and_writes_each_record_once(stream_client: Cli items = await read_all(first) scoped = await read_all(workflow_stream(handle, "scoped")) assert [i.data for i in records(items)] == ["before restart", "after restart"] - assert isinstance(items[-1], streams.Closed) + assert isinstance(items[-1], common.Closed) assert [i.data for i in records(scoped)] == ["scoped"] - assert [type(i) for i in scoped].count(streams.Closed) == 1 + assert [type(i) for i in scoped].count(common.Closed) == 1 async def test_backend_replays_a_history_made_without_it( @@ -622,13 +622,13 @@ async def test_backend_shutdown_after_client_writes(stream_client: Client): [backend] = stream_client.config()["stream_backends"] assert isinstance(backend, ExtensionStreamBackend) stream = await redis.create(stream_client, Events, id=f"shutdown-{uuid.uuid4()}") - async with streams.BufferedPublisher(stream.topic(Events.tokens)) as tokens: + async with common.BufferedPublisher(stream.topic(Events.tokens)) as tokens: tokens.publish("buffered") await backend.shutdown() written = [ item.data async for item in stream.subscribe(wait_new_records=False) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] assert written == ["buffered"] @@ -683,7 +683,7 @@ async def test_eviction_prints_nothing( WaitsForever.run, id=f"wf-{uuid.uuid4()}", task_queue=w.task_queue ) reader = workflow_stream(handle).subscribe(create_wait=timedelta(seconds=20)) - assert isinstance(await anext(reader), streams.Record) + assert isinstance(await anext(reader), common.Record) await reader.aclose() await handle.terminate() gc.collect() @@ -720,9 +720,9 @@ async def test_final_failure_under_a_retry_policy( stream = workflow_stream(handle) items = [item async for item in stream.subscribe(wait_new_records=False)] written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] + superseded = [item for item in items if isinstance(item, common.Superseded)] # The end of the chain closes nothing. - assert not any(isinstance(item, streams.Closed) for item in items) + assert not any(isinstance(item, common.Closed) for item in items) if non_retryable: # A non-retryable failure ends the chain on attempt 1. assert [r.data for r in written] == ["run-1"] @@ -739,11 +739,11 @@ async def test_final_failure_under_a_retry_policy( async def test_reader_stops_on_its_own_timer_and_resumes(stream_client: Client): stream = await redis.create(stream_client, Events, id=f"timer-{uuid.uuid4()}") first = await stream.topic(Events.tokens).publish("one") - seen: list[streams.Record[Any]] = [] + seen: list[common.Record[Any]] = [] async def read_until_quiet() -> None: async for item in stream.subscribe(): - assert isinstance(item, streams.Record) + assert isinstance(item, common.Record) seen.append(item) # A read has no idle end, so a reader that wants one sets its own timer. @@ -756,7 +756,7 @@ async def read_until_quiet() -> None: async for item in stream.subscribe( offset=seen[-1].offset + 1, wait_new_records=False ) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] assert resumed == ["two"] @@ -783,7 +783,7 @@ async def test_two_backends_on_one_client(client: Client, redis_url: str): data = [ item.data async for item in again.subscribe(wait_new_records=False) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] assert data == [expected] @@ -863,10 +863,10 @@ async def test_close_survives_a_failed_last_task(stream_client: Client, cache: i stream = workflow_stream(handle) items = [item async for item in stream.subscribe(wait_new_records=False)] assert [r.data for r in records(items)] == ["token"] - closes = [item for item in items if isinstance(item, streams.Closed)] + closes = [item for item in items if isinstance(item, common.Closed)] assert len(closes) == 1 and items[-1] is closes[0] assert closes[0].result == "done" - assert closes[0].status == streams.CloseStatus.COMPLETED + assert closes[0].status == common.CloseStatus.COMPLETED assert await close_activity_events(handle) == [ EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED, EventType.EVENT_TYPE_ACTIVITY_TASK_STARTED, @@ -877,10 +877,10 @@ async def test_close_survives_a_failed_last_task(stream_client: Client, cache: i @workflow.defn class WritesToStream: @workflow.run - async def run(self, stream: streams.WorkflowStream[Events]) -> str: + async def run(self, stream: common.WorkflowStream[Events]) -> str: try: await stream.topic(Events.tokens).publish("from workflow") - except streams.StreamError as err: + except common.StreamError as err: return err.reason return "wrote" @@ -890,7 +890,7 @@ async def test_write_to_a_missing_stream_raises_at_the_await(stream_client: Clie assert ( await stream_client.execute_workflow( WritesToStream.run, - streams.WorkflowStream(id=f"missing-{uuid.uuid4()}"), + common.WorkflowStream(id=f"missing-{uuid.uuid4()}"), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -905,21 +905,21 @@ async def test_write_to_a_closed_stream_raises_at_the_await(stream_client: Clien assert ( await stream_client.execute_workflow( WritesToStream.run, - streams.WorkflowStream.from_handle(stream), + common.WorkflowStream.from_handle(stream), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) == "stream_closed" ) items = [item async for item in stream.subscribe()] - assert len(items) == 1 and isinstance(items[0], streams.Closed) + assert len(items) == 1 and isinstance(items[0], common.Closed) @activity.defn async def unknown_topic(input: CallModelInput) -> str: try: input.stream.topic("missing") - except streams.StreamError as err: + except common.StreamError as err: return err.reason return "found" @@ -931,7 +931,7 @@ async def run(self) -> str: stream = await workflow.create_stream(Events) return await workflow.execute_activity( unknown_topic, - CallModelInput("", streams.ActivityStream.from_handle(stream)), + CallModelInput("", common.ActivityStream.from_handle(stream)), start_to_close_timeout=timedelta(seconds=30), ) @@ -949,7 +949,7 @@ async def test_activity_refuses_a_topic_outside_the_definition(stream_client: Cl @workflow.defn class ContinuesThenRetries: @workflow.run - async def run(self, stream: streams.WorkflowStream[Events] | None) -> str: + async def run(self, stream: common.WorkflowStream[Events] | None) -> str: if stream is None: stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("first") @@ -977,7 +977,7 @@ async def test_retry_of_a_continued_run_writes_the_same_stream(stream_client: Cl items = await read_all(workflow_stream(handle)) written = records(items) assert [r.data for r in written] == ["first", "attempt-1", "attempt-2"] - superseded = [item for item in items if isinstance(item, streams.Superseded)] + superseded = [item for item in items if isinstance(item, common.Superseded)] assert [r.data for r in written if superseded[0].withdraws(r)] == ["attempt-1"] - closes = [item for item in items if isinstance(item, streams.Closed)] + closes = [item for item in items if isinstance(item, common.Closed)] assert len(closes) == 1 and closes[0].result == "done" diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 5a3371f8d6..7b0f252d78 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -9,10 +9,9 @@ import pytest -from temporalio import workflow +from temporalio import common, workflow from temporalio.client import Client -from temporalio.common import streams -from temporalio.common.streams import _stream_client +from temporalio.common._streams import _stream_client from temporalio.contrib import stream as redis from temporalio.contrib.stream import RedisStreamBackend from temporalio.contrib.stream.testing import MemoryStreamBackend @@ -32,19 +31,17 @@ class Turn: text: str = "" -@streams.defn(name="events") +@common.defn(name="events") class Events: - turns = streams.topic(Turn) - tokens = streams.topic(str) + turns = common.topic(Turn) + tokens = common.topic(str) async def test_unknown_backend_raises_before_any_call(client: Client): client, extension = fake_client(client) - with pytest.raises( - streams.StreamConfigError, match="no Redis Stream backend redis" - ): + with pytest.raises(common.StreamConfigError, match="no Redis Stream backend redis"): await redis.create(client, Events, id="chat-1", backend="redis") - with pytest.raises(streams.StreamConfigError, match="It has fake"): + with pytest.raises(common.StreamConfigError, match="It has fake"): client.stream(Events, id="chat-1", backend="redis") assert not extension.streams and not extension.namespaces stream = await redis.create(client, Events, id="chat-1") @@ -52,9 +49,9 @@ async def test_unknown_backend_raises_before_any_call(client: Client): async def test_client_without_backend_has_no_transport(client: Client): - with pytest.raises(streams.StreamConfigError, match="no Stream backend"): + with pytest.raises(common.StreamConfigError, match="no Stream backend"): client.stream(Events, id="chat-1") - with pytest.raises(streams.StreamConfigError, match="no Redis Stream backend"): + with pytest.raises(common.StreamConfigError, match="no Redis Stream backend"): await redis.create(client, Events, id="chat-1") @@ -63,10 +60,10 @@ async def test_decoded_client_stream_has_no_client(client: Client): stream = await redis.create(client, Events, id="chat-2") converter = DataConverter.default.payload_converter decoded = converter.from_payload( - converter.to_payload(stream), streams.ClientStream[Events] + converter.to_payload(stream), common.ClientStream[Events] ) assert decoded == stream - with pytest.raises(streams.StreamConfigError, match="Client.stream"): + with pytest.raises(common.StreamConfigError, match="Client.stream"): await decoded.get_offset() @@ -91,7 +88,7 @@ class Description: setattr(handle, "describe", describe) await redis.create(client, Events, id="agent-1:first:n") - stream = handle.stream(streams.with_name(Events, "n")) + stream = handle.stream(common.with_name(Events, "n")) with pytest.raises(RuntimeError, match="not known yet"): stream.id assert await stream.get_offset() == 0 @@ -99,7 +96,7 @@ class Description: assert stream.first_execution_run_id == "first" assert await stream.get_offset() == 0 # The handle keeps the first run id, so its other Streams make no call. - other = handle.stream(streams.with_name(Events, "n")) + other = handle.stream(common.with_name(Events, "n")) assert await other.get_offset() == 0 assert calls == [1] @@ -164,7 +161,7 @@ async def describe(**_: Any) -> Any: @workflow.defn class WriteToStream: @workflow.run - async def run(self, stream: streams.WorkflowStream[Events]) -> None: + async def run(self, stream: common.WorkflowStream[Events]) -> None: await stream.topic(Events.turns).publish(Turn("workflow")) @@ -177,35 +174,35 @@ async def test_client_stream_takes_workflow_writes(stream_client: Client): async with new_worker(stream_client, WriteToStream) as worker: await stream_client.execute_workflow( WriteToStream.run, - streams.WorkflowStream.from_handle(stream), + common.WorkflowStream.from_handle(stream), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) await stream.close(result="bye") items = [item async for item in stream.subscribe()] - data = [item.data for item in items if isinstance(item, streams.Record)] + data = [item.data for item in items if isinstance(item, common.Record)] assert data == [Turn("client"), "a", "b", Turn("workflow")] - assert isinstance(items[-1], streams.Closed) and items[-1].result == "bye" + assert isinstance(items[-1], common.Closed) and items[-1].result == "bye" # Another handle of the same Stream reads it, from an offset and by Topic. again = stream_client.stream(Events, id=stream.id) resumed = [ item.data async for item in again.subscribe(Events.tokens, offset=first + 1) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] assert resumed == ["a", "b"] by_prefix = [ item.data async for item in again.subscribe(topic_prefixes=["tok"]) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] assert by_prefix == ["a", "b"] only_turns = [item async for item in again.subscribe(again.topic(Events.turns))] assert [type(item) for item in only_turns] == [ - streams.Record, - streams.Record, - streams.Closed, + common.Record, + common.Record, + common.Closed, ] @@ -230,13 +227,13 @@ async def test_client_reads_a_workflow_stream(stream_client: Client): items = [ item async for item in stream.subscribe(create_wait=timedelta(seconds=20)) ] - assert [item.data for item in items if isinstance(item, streams.Record)] == [ + assert [item.data for item in items if isinstance(item, common.Record)] == [ "hi", "there", ] - assert isinstance(items[-1], streams.Closed) and items[-1].result == "HI THERE" + assert isinstance(items[-1], common.Closed) and items[-1].result == "HI THERE" assert stream.workflow_id == handle.id - with pytest.raises(streams.StreamNotOwnerError): + with pytest.raises(common.StreamNotOwnerError): await stream.close() @@ -256,13 +253,13 @@ async def test_async_with_closes_a_client_stream(stream_client: Client): async with redis.create(stream_client, Events, id=cancelled.id): raise asyncio.CancelledError() for stream, result, failure, status in ( - (done, None, None, streams.CloseStatus.COMPLETED), - (failed, None, "turn failed", streams.CloseStatus.FAILED), - (early, "early", None, streams.CloseStatus.COMPLETED), - (cancelled, None, "Cancelled", streams.CloseStatus.CANCELED), + (done, None, None, common.CloseStatus.COMPLETED), + (failed, None, "turn failed", common.CloseStatus.FAILED), + (early, "early", None, common.CloseStatus.COMPLETED), + (cancelled, None, "Cancelled", common.CloseStatus.CANCELED), ): items = [item async for item in stream.subscribe()] - closes = [item for item in items if isinstance(item, streams.Closed)] + closes = [item for item in items if isinstance(item, common.Closed)] assert len(closes) == 1 and closes[0].result == result assert closes[0].status == status assert (failure is None) == (closes[0].failure is None) @@ -275,20 +272,20 @@ async def test_next_offset_resumes_after_the_last_item(stream_client: Client): turn = await stream.topic(Events.turns).publish(Turn("model")) tokens = stream.subscribe(Events.tokens, wait_new_records=False) assert tokens.next_offset == 0 - assert [item.data async for item in tokens if isinstance(item, streams.Record)] == [ + assert [item.data async for item in tokens if isinstance(item, common.Record)] == [ "a" ] # The page ends past the Record that the Topic filter left out. assert tokens.next_offset == turn + 1 - new = stream.subscribe(offset=streams.END, wait_new_records=False) + new = stream.subscribe(offset=common.END, wait_new_records=False) assert new.next_offset is None assert [item async for item in new] == [] assert new.next_offset == turn + 1 await stream.topic(Events.tokens).publish("b") resumed = stream.subscribe(offset=tokens.next_offset, wait_new_records=False) - assert [ - item.data async for item in resumed if isinstance(item, streams.Record) - ] == ["b"] + assert [item.data async for item in resumed if isinstance(item, common.Record)] == [ + "b" + ] class _EncodesCommonAttributes(DefaultFailureConverter): @@ -308,7 +305,7 @@ async def test_closed_failure_decodes_with_the_clients_converter(stream_client: failure=ApplicationError("tool failed", "detail", {"k": 1}, type="Tool") ) [closed] = [item async for item in stream.subscribe()] - assert isinstance(closed, streams.Closed) + assert isinstance(closed, common.Closed) assert isinstance(closed.failure, ApplicationError) assert closed.failure.message == "tool failed" and closed.failure.type == "Tool" assert closed.failure.details == ("detail", {"k": 1}) @@ -321,7 +318,7 @@ async def test_no_wait_read_goes_past_records_of_other_topics(stream_client: Cli await stream.topic(Events.tokens).publish(*["x"] * 1000) turn = await stream.topic(Events.turns).publish(Turn("model")) turns = stream.subscribe(Events.turns, wait_new_records=False) - assert [item.data async for item in turns if isinstance(item, streams.Record)] == [ + assert [item.data async for item in turns if isinstance(item, common.Record)] == [ Turn("model") ] assert turns.next_offset == turn + 1 @@ -339,13 +336,13 @@ async def test_last_reads_the_newest_records_then_follows(stream_client: Client) await tokens.publish("d") shown.append(await anext(tail)) await tail.aclose() - assert [item.data for item in shown if isinstance(item, streams.Record)] == [ + assert [item.data for item in shown if isinstance(item, common.Record)] == [ "b", "c", "d", ] newest = stream.subscribe(last=1, wait_new_records=False) - assert [item.data async for item in newest if isinstance(item, streams.Record)] == [ + assert [item.data async for item in newest if isinstance(item, common.Record)] == [ "d" ] with pytest.raises(ValueError, match="no offset"): @@ -359,7 +356,7 @@ async def test_read_type_decodes_topics_outside_the_definition(stream_client: Cl async def read(**options: Any) -> list[object]: items = stream.subscribe(wait_new_records=False, **options) - return [item.data async for item in items if isinstance(item, streams.Record)] + return [item.data async for item in items if isinstance(item, common.Record)] assert await read() == [{"kind": "extra", "text": ""}, "a"] assert await read(type=Turn) == [Turn("extra"), "a"] @@ -367,16 +364,16 @@ async def read(**options: Any) -> list[object]: async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: Client): stream = await redis.create(stream_client, Events, id=f"buffered-{uuid.uuid4()}") - async with streams.BufferedPublisher(stream) as publisher: + async with common.BufferedPublisher(stream) as publisher: publisher.publish(Events.turns, Turn("user")) publisher.publish(Events.tokens, "a", "b") publisher.publish("turns", Turn("model")) - with pytest.raises(streams.StreamError, match="no Topic"): + with pytest.raises(common.StreamError, match="no Topic"): publisher.publish("turn", Turn("typo")) written = [ (item.topic, item.data) async for item in stream.subscribe(wait_new_records=False) - if isinstance(item, streams.Record) + if isinstance(item, common.Record) ] assert written == [ ("turns", Turn("user")), @@ -386,10 +383,10 @@ async def test_buffered_publisher_keeps_the_order_across_topics(stream_client: C ] -@streams.defn(name="events") +@common.defn(name="events") class LooseEvents: - turns = streams.topic(str) - tokens = streams.topic(str) + turns = common.topic(str) + tokens = common.topic(str) async def test_record_that_fails_to_decode_raises_and_resumes_past_it( @@ -401,13 +398,13 @@ async def test_record_that_fails_to_decode_raises_and_resumes_past_it( await writer.topic(LooseEvents.tokens).publish("after") reader = stream_client.stream(Events, id=stream_id) read = reader.subscribe(wait_new_records=False) - with pytest.raises(streams.StreamError, match=f"offset {bad} of the Topic turns"): + with pytest.raises(common.StreamError, match=f"offset {bad} of the Topic turns"): await anext(read) assert read.next_offset == bad + 1 resumed = reader.subscribe(offset=read.next_offset, wait_new_records=False) - assert [ - item.data async for item in resumed if isinstance(item, streams.Record) - ] == ["after"] + assert [item.data async for item in resumed if isinstance(item, common.Record)] == [ + "after" + ] async def test_async_with_refuses_a_workflow_stream_before_the_block( @@ -418,7 +415,7 @@ async def test_async_with_refuses_a_workflow_stream_before_the_block( "agent-1", first_execution_run_id="run-1" ) ran = False - with pytest.raises(streams.StreamNotOwnerError): + with pytest.raises(common.StreamNotOwnerError): async with handle.stream(Events): ran = True assert not ran @@ -427,7 +424,7 @@ async def test_async_with_refuses_a_workflow_stream_before_the_block( async def test_record_over_the_store_limit_raises_too_large(stream_client: Client): # The client's writer splits a batch at the Store's limit, so only a Record can be over. stream = await redis.create(stream_client, Events, id=f"large-{uuid.uuid4()}") - with pytest.raises(streams.StreamTooLargeError) as refused: + with pytest.raises(common.StreamTooLargeError) as refused: await stream.topic(Events.tokens).publish("x" * 2_000_000) assert refused.value.reason == "stream_too_large" and refused.value.non_retryable assert await stream.get_offset() == 0 @@ -438,7 +435,7 @@ async def test_read_keeps_no_offsets_of_client_records(stream_client: Client): stream = await redis.create(stream_client, Events, id=f"chat-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("a", "b", "c") read = stream.subscribe(wait_new_records=False) - assert [item.data async for item in read if isinstance(item, streams.Record)] == [ + assert [item.data async for item in read if isinstance(item, common.Record)] == [ "a", "b", "c", @@ -451,7 +448,7 @@ async def test_empty_stream_id_raises_with_no_call(client: Client): with pytest.raises(ValueError, match="id is empty"): client.stream(Events, id="") with pytest.raises(ValueError, match="id is empty"): - client.stream(Events, stream=streams.WorkflowStream(id="")) + client.stream(Events, stream=common.WorkflowStream(id="")) assert not extension.namespaces @@ -464,7 +461,7 @@ async def test_create_of_an_expired_stream_raises(stream_client: Client): ) await stream.topic(Events.tokens).publish("old") await asyncio.sleep(0.5) - with pytest.raises(streams.StreamExpiredError): + with pytest.raises(common.StreamExpiredError): await redis.create(stream_client, Events, id=stream_id, retention=retention) @@ -476,11 +473,11 @@ async def test_named_producer_fences_its_earlier_epoch(stream_client: Client): await before.topic(Events.tokens).publish("before restart") after = stream.producer("ingest", epoch=2) await after.topic(Events.tokens).publish("after restart") - with pytest.raises(streams.StreamFencedError): + with pytest.raises(common.StreamFencedError): await before.topic(Events.tokens).publish("zombie") # The earlier epoch's Records are real messages, so nothing withdraws them. items = [item async for item in stream.subscribe(wait_new_records=False)] - assert [type(item) for item in items] == [streams.Record, streams.Record] + assert [type(item) for item in items] == [common.Record, common.Record] with pytest.raises(ValueError): stream.producer("ingest", epoch=0) @@ -490,7 +487,7 @@ async def test_unresolved_workflow_stream_compares_and_prints(client: Client): handle = client.get_workflow_handle("agent-2") first = handle.stream(Events) same = handle.stream(Events) - other = handle.stream(streams.with_name(Events, "other")) + other = handle.stream(common.with_name(Events, "other")) key = hash(first) assert first == same and first != other describes = [] @@ -530,19 +527,19 @@ async def test_registration_rules(client: Client): ], ): config["stream_backends"] = backends - with pytest.raises(streams.StreamConfigError, match="exactly one"): + with pytest.raises(common.StreamConfigError, match="exactly one"): Client(**config) config["stream_backends"] = [MemoryStreamBackend(), MemoryStreamBackend()] - with pytest.raises(streams.StreamConfigError, match="two Stream backends"): + with pytest.raises(common.StreamConfigError, match="two Stream backends"): Client(**config) # For now a Worker serves one backend, and so does a Replayer. config["stream_backends"] = [ MemoryStreamBackend(), MemoryStreamBackend(name="b", default=True), ] - with pytest.raises(streams.StreamConfigError, match="one backend"): + with pytest.raises(common.StreamConfigError, match="one backend"): new_worker(Client(**config), Agent) - with pytest.raises(streams.StreamConfigError, match="one backend"): + with pytest.raises(common.StreamConfigError, match="one backend"): Replayer(workflows=[Agent], stream_backends=config["stream_backends"]) # A generator of backends works too, and a copy of the client keeps them. config["stream_backends"] = cast( @@ -561,7 +558,7 @@ async def test_worker_that_polls_no_activities_warns_of_the_close( # The close of a Workflow's Stream is a regular Activity, which another Worker may run. config = client.config() config["stream_backends"] = [MemoryStreamBackend(default=True)] - with caplog.at_level(logging.WARNING, logger="temporalio.common.streams"): + with caplog.at_level(logging.WARNING, logger="temporalio.common._streams"): async with new_worker(Client(**config), Agent, no_remote_activities=True): pass assert "polls no Activities" in caplog.text @@ -569,15 +566,15 @@ async def test_worker_that_polls_no_activities_warns_of_the_close( async def test_backend_is_not_a_plugin(client: Client): # The connect refuses it before it connects, so the unreachable target doesn't matter. - with pytest.raises(streams.StreamConfigError, match="not a plugin"): + with pytest.raises(common.StreamConfigError, match="not a plugin"): await Client.connect("localhost:1", plugins=[MemoryStreamBackend()]) config = client.config() config["plugins"] = [MemoryStreamBackend()] - with pytest.raises(streams.StreamConfigError, match="not a plugin"): + with pytest.raises(common.StreamConfigError, match="not a plugin"): Client(**config) - with pytest.raises(streams.StreamConfigError, match="not a plugin"): + with pytest.raises(common.StreamConfigError, match="not a plugin"): new_worker(client, Agent, plugins=[RedisStreamBackend("redis://-")]) - with pytest.raises(streams.StreamConfigError, match="not a plugin"): + with pytest.raises(common.StreamConfigError, match="not a plugin"): Replayer(workflows=[Agent], plugins=[MemoryStreamBackend()]) @@ -590,16 +587,16 @@ async def test_worker_plugin_keeps_the_streams_passthrough(client: Client): worker = new_worker(Client(**config), Agent, plugins=[plugin]) runner = worker.config(active_config=True).get("workflow_runner") assert isinstance(runner, SandboxedWorkflowRunner) - assert "temporalio.common.streams" in runner.restrictions.passthrough_modules + assert "temporalio.common._streams" in runner.restrictions.passthrough_modules async def test_default_is_native_with_no_default_backend(client: Client): config = client.config() config["stream_backends"] = [MemoryStreamBackend()] one = Client(**config) - with pytest.raises(streams.StreamConfigError, match="default is native"): + with pytest.raises(common.StreamConfigError, match="default is native"): one.stream(Events, id="d-2") - with pytest.raises(streams.StreamConfigError, match="default is native"): + with pytest.raises(common.StreamConfigError, match="default is native"): one.get_workflow_handle("agent-3").stream(Events) assert one.stream(Events, id="d-2", backend="memory").backend == "memory" # The Redis create picks the client's only Redis backend, and memory stands in for it. @@ -617,7 +614,7 @@ async def test_stream_takes_the_handle_backend_or_the_default(client: Client): await created.topic(Events.tokens).publish("on b") assert await two.stream(Events, stream=created).get_offset() > 0 # A handle with no backend name is of the default backend. - assert two.stream(Events, stream=streams.ClientStream(id="moving")).backend == "a" + assert two.stream(Events, stream=common.ClientStream(id="moving")).backend == "a" with pytest.raises(ValueError): two.stream(Events, stream=created, backend="a") with pytest.raises(TypeError): @@ -645,5 +642,5 @@ async def test_workflow_environment_takes_stream_backends(): ) await handle.result() items = [item async for item in handle.stream(Events).subscribe()] - data = [item.data for item in items if isinstance(item, streams.Record)] + data = [item.data for item in items if isinstance(item, common.Record)] assert data == ["from the environment"] diff --git a/tests/contrib/streams/test_vectors.py b/tests/contrib/streams/test_vectors.py index 792985576e..72bf764fe4 100644 --- a/tests/contrib/streams/test_vectors.py +++ b/tests/contrib/streams/test_vectors.py @@ -4,8 +4,8 @@ import pytest import temporalio.api.streamservice.v1 as stream_pb -from temporalio.common.streams import _ids, _items -from temporalio.common.streams.system import models +from temporalio.common._streams import _ids, _items +from temporalio.common._streams.system import models from tests.contrib.streams._support import crate_file, vectors _KINDS = { diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 081e9af974..7d5b267010 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -9,7 +9,7 @@ import pytest -from temporalio import activity, workflow +from temporalio import activity, common, workflow from temporalio.api.common.v1 import Payload from temporalio.api.enums.v1 import EventType from temporalio.client import ( @@ -17,10 +17,10 @@ WorkflowFailureError, WorkflowHandle, ) -from temporalio.common import RetryPolicy, streams -from temporalio.common.streams import _writes -from temporalio.common.streams._backend import _ExtensionCalls -from temporalio.common.streams._defn import definition_of +from temporalio.common import RetryPolicy +from temporalio.common._streams import _writes +from temporalio.common._streams._backend import _ExtensionCalls +from temporalio.common._streams._defn import definition_of from temporalio.contrib import stream as redis from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.converter import DataConverter, PayloadCodec @@ -35,19 +35,19 @@ class Turn: text: str = "" -@streams.defn(name="events") +@common.defn(name="events") class Events: - turns = streams.topic(Turn) - tokens = streams.topic(str) + turns = common.topic(Turn) + tokens = common.topic(str) async def read(handle: WorkflowHandle[Any, Any], name: str = "events") -> list[object]: - stream = handle.stream(streams.with_name(Events, name)) + stream = handle.stream(common.with_name(Events, name)) return [item async for item in stream.subscribe(wait_new_records=False)] def data(items: Sequence[object]) -> list[object]: - return [item.data for item in items if isinstance(item, streams.Record)] + return [item.data for item in items if isinstance(item, common.Record)] @workflow.defn @@ -72,7 +72,7 @@ async def test_create_publish_and_close(stream_client: Client): items = await read(handle) assert data(items) == [Turn("user", "hi there"), Turn("model"), "hi", "there"] closed = items[-1] - assert isinstance(closed, streams.Closed) and closed.result == "HI THERE" + assert isinstance(closed, common.Closed) and closed.result == "HI THERE" @workflow.defn @@ -82,7 +82,7 @@ async def run(self, fail_at: int) -> None: for n in range(2): try: async with workflow.create_stream( - streams.with_name(Events, f"turn-{n}") + common.with_name(Events, f"turn-{n}") ) as turn: await turn.topic(Events.tokens).publish(f"token-{n}") if n == fail_at: @@ -101,16 +101,16 @@ async def test_async_with_closes_the_stream(stream_client: Client): first = await read(handle, "turn-0") second = await read(handle, "turn-1") assert data(first) == ["token-0"] - assert isinstance(first[-1], streams.Closed) and first[-1].failure is None + assert isinstance(first[-1], common.Closed) and first[-1].failure is None assert data(second) == ["token-1"] - assert isinstance(second[-1], streams.Closed) + assert isinstance(second[-1], common.Closed) assert str(second[-1].failure) == "turn failed" @workflow.defn class Continuing: @workflow.run - async def run(self, stream: streams.WorkflowStream[Events] | None) -> str: + async def run(self, stream: common.WorkflowStream[Events] | None) -> str: if stream is None: stream = await workflow.create_stream(Events) await stream.topic(Events.tokens).publish("run 1") @@ -126,7 +126,7 @@ async def run(self, stream: streams.WorkflowStream[Events] | None) -> str: class RenamedAcrossRuns: @workflow.run async def run(self, first: bool) -> list[str]: - renamed = streams.with_name(Events, "turn-1") + renamed = common.with_name(Events, "turn-1") if first: turn = await workflow.create_stream(renamed) await turn.topic(Events.tokens).publish("run 1") @@ -152,7 +152,7 @@ async def test_renamed_definition_is_another_stream(stream_client: Client): def test_with_name_leaves_the_definition_unchanged(): - renamed = streams.with_name(Events, "turn-1") + renamed = common.with_name(Events, "turn-1") assert definition_of(Events).name == "events" assert definition_of(renamed).name == "turn-1" assert definition_of(renamed).topics == definition_of(Events).topics @@ -166,14 +166,14 @@ async def test_continue_as_new_passes_the_stream(stream_client: Client): assert await handle.result() == "done" items = await read(handle) assert data(items) == ["run 1", "run 2"] - closed = [item for item in items if isinstance(item, streams.Closed)] + closed = [item for item in items if isinstance(item, common.Closed)] assert len(closed) == 1 and closed[0].result == "done" @workflow.defn class Mistakes: def __init__(self) -> None: - self._stream: streams.WorkflowStream[Events] | None = None + self._stream: common.WorkflowStream[Events] | None = None @workflow.run async def run(self) -> list[str]: @@ -182,16 +182,16 @@ async def run(self) -> list[str]: self._stream = stream try: stream.topic("unknown") - except streams.StreamError as err: + except common.StreamError as err: errors.append(err.reason) await stream.close(result="early") try: await stream.topic(Events.tokens).publish("late") - except streams.StreamClosedError as err: + except common.StreamClosedError as err: errors.append(err.reason) try: await workflow.create_stream(Events) - except streams.StreamClosedError as err: + except common.StreamClosedError as err: errors.append(err.reason) return errors @@ -200,7 +200,7 @@ def publish_in_query(self) -> str: assert self._stream is not None try: self._stream.topic(Events.tokens)._stream._append("tokens", ("x",)) - except streams.ReadOnlyContextError as err: + except common.ReadOnlyContextError as err: return type(err).__name__ return "wrote" @@ -219,17 +219,17 @@ async def test_replay_reproducible_errors_raise_in_workflow_code( ] assert await handle.query(Mistakes.publish_in_query) == "ReadOnlyContextError" items = await read(handle) - assert isinstance(items[-1], streams.Closed) and items[-1].result == "early" + assert isinstance(items[-1], common.Closed) and items[-1].result == "early" @workflow.defn class ClosesAReceivedStream: @workflow.run - async def run(self, stream: streams.WorkflowStream[Events]) -> str: + async def run(self, stream: common.WorkflowStream[Events]) -> str: await stream.topic(Events.tokens).publish("from workflow") try: await stream.close(result="ended by a writer") - except streams.StreamNotOwnerError as err: + except common.StreamNotOwnerError as err: return err.reason return "closed" @@ -240,7 +240,7 @@ async def test_workflow_closes_only_its_own_streams(stream_client: Client): assert ( await stream_client.execute_workflow( ClosesAReceivedStream.run, - streams.WorkflowStream.from_handle(stream), + common.WorkflowStream.from_handle(stream), id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -270,7 +270,7 @@ async def test_run_end_closes_nothing(stream_client: Client): items = await read(handle) # The Stream stays open, so a later reader still waits for a close. assert data(items) == ["before"] - assert not any(isinstance(item, streams.Closed) for item in items) + assert not any(isinstance(item, common.Closed) for item in items) @workflow.defn @@ -297,12 +297,12 @@ async def test_block_end_leaves_the_stream_open_for_a_retry_run(stream_client: C ) assert await handle.result() == "done on attempt 2" items = await read(handle) - written = [item for item in items if isinstance(item, streams.Record)] - superseded = [item for item in items if isinstance(item, streams.Superseded)] + written = [item for item in items if isinstance(item, common.Record)] + superseded = [item for item in items if isinstance(item, common.Superseded)] assert [r.data for r in written] == ["run-1", "run-2"] assert [r.data for r in written if superseded[0].withdraws(r)] == ["run-1"] # Only the block end of the run that succeeds closes the Stream. - closes = [item for item in items if isinstance(item, streams.Closed)] + closes = [item for item in items if isinstance(item, common.Closed)] assert len(closes) == 1 and items[-1] is closes[0] assert closes[0].failure is None @@ -365,8 +365,8 @@ async def test_block_end_closes_when_no_retry_run_follows(stream_client: Client) items = await read(handle) assert data(items) == [mode] closed = items[-1] - assert isinstance(closed, streams.Closed), mode - assert closed.status == streams.CloseStatus.FAILED + assert isinstance(closed, common.Closed), mode + assert closed.status == common.CloseStatus.FAILED assert isinstance(closed.failure, ApplicationError) assert closed.failure.message == "block failed" # A close at the run's end is a command after the run's code, also on replay. @@ -430,8 +430,8 @@ async def test_cancel_during_an_activity_closes_as_canceled(stream_client: Clien items = await read(handle) assert data(items) == ["started"] closed = items[-1] - assert isinstance(closed, streams.Closed), f"local={local}" - assert closed.status == streams.CloseStatus.CANCELED + assert isinstance(closed, common.Closed), f"local={local}" + assert closed.status == common.CloseStatus.CANCELED @workflow.defn @@ -457,9 +457,9 @@ async def test_block_end_leaves_a_cron_stream_open(stream_client: Client): second = await asyncio.wait_for(anext(reader), 20) await reader.aclose() await handle.terminate() - assert isinstance(first, streams.Record) and isinstance(second, streams.Record) + assert isinstance(first, common.Record) and isinstance(second, common.Record) items = await read(handle) - assert not any(isinstance(item, streams.Closed) for item in items) + assert not any(isinstance(item, common.Closed) for item in items) failed_close_tasks: set[str] = set() @@ -492,7 +492,7 @@ async def test_failed_task_writes_no_close(stream_client: Client): ) await handle.result() items = await read(handle) - closes = [item for item in items if isinstance(item, streams.Closed)] + closes = [item for item in items if isinstance(item, common.Closed)] assert len(closes) == 1 and closes[0].result == "closed" history = await handle.fetch_history() # The failed task's close never went out, so History has one close Activity. @@ -558,10 +558,10 @@ async def test_cron_runs_share_an_open_stream(stream_client: Client): second = await reader.__anext__() await reader.aclose() await handle.terminate() - assert isinstance(first, streams.Record) and first.data == "turn" - assert isinstance(second, streams.Record) and second.data == "turn" + assert isinstance(first, common.Record) and first.data == "turn" + assert isinstance(second, common.Record) and second.data == "turn" items = [item async for item in stream.subscribe(wait_new_records=False)] - assert not any(isinstance(item, streams.Closed) for item in items) + assert not any(isinstance(item, common.Closed) for item in items) @workflow.defn @@ -587,7 +587,7 @@ def finish(self) -> str: self._stream.close_nowait(result="finished") try: self._stream.topic(Events.tokens).publish_nowait("late") - except streams.StreamClosedError as err: + except common.StreamClosedError as err: return err.reason finally: self._done = True @@ -604,7 +604,7 @@ async def test_sync_handlers_publish_and_close_with_no_await(stream_client: Clie await handle.result() items = await read(handle) assert data(items) == ["a", "p"] - assert isinstance(items[-1], streams.Closed) and items[-1].result == "finished" + assert isinstance(items[-1], common.Closed) and items[-1].result == "finished" @workflow.defn @@ -632,7 +632,7 @@ async def test_init_creates_and_publishes_with_no_await(stream_client: Client): assert await handle.result() == "done" items = await read(handle) assert data(items) == ["hi", "HI"] - assert isinstance(items[-1], streams.Closed) and items[-1].result == "done" + assert isinstance(items[-1], common.Closed) and items[-1].result == "done" @workflow.defn @@ -660,7 +660,7 @@ async def test_concurrent_publishes_keep_their_order(stream_client: Client): # its own Record. assert ( offsets - == [item.offset for item in items if isinstance(item, streams.Record)][1:] + == [item.offset for item in items if isinstance(item, common.Record)][1:] ) history = await handle.fetch_history() markers = [ @@ -836,7 +836,7 @@ async def test_merge_that_the_codec_grows_past_the_batch_limit(stream_client: Cl ) offsets = await handle.result() items = await read(handle) - written = [item for item in items if isinstance(item, streams.Record)] + written = [item for item in items if isinstance(item, common.Record)] assert [len(r.data) for r in written] == [300_000] * 6 assert offsets == [written[2].offset, written[5].offset] @@ -848,7 +848,7 @@ async def run(self) -> str: tokens = (await workflow.create_stream(Events)).topic(Events.tokens) try: await tokens.publish(*["x" * 1_400_000] * 4) - except streams.StreamTooLargeError as err: + except common.StreamTooLargeError as err: return err.reason return "written" @@ -888,7 +888,7 @@ async def run(self) -> str: await workflow.sleep(timedelta(seconds=1.5)) try: await workflow.create_stream(Events) - except streams.StreamExpiredError as err: + except common.StreamExpiredError as err: return err.reason return "created" @@ -929,12 +929,12 @@ async def test_default_is_native_with_no_default_backend(client: Client): CreatesOnRedis.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await on_redis.result() == "memory" - with pytest.raises(streams.StreamConfigError, match="default=True"): + with pytest.raises(common.StreamConfigError, match="default=True"): on_redis.stream(Events) stream = on_redis.stream(Events, backend="memory") items = [item async for item in stream.subscribe()] assert data(items) == ["on redis"] - assert isinstance(items[-1], streams.Closed) and items[-1].result == "closed" + assert isinstance(items[-1], common.Closed) and items[-1].result == "closed" class _RecordsCloses(MemoryStreamBackend): From 1e5866aaf698108c2cd4a6e3db6095445366c70d Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 02:20:58 -0700 Subject: [PATCH 73/79] Gave a Workflow read's Closed item its status and decoded failure. The model's converter keeps payloads raw for the Topic types, so the failure kept RawValue details, as p35 fixed for client reads. --- temporalio/common/streams/_workflow_reader.py | 19 +++++++++-- tests/contrib/streams/test_workflow_reads.py | 32 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/temporalio/common/streams/_workflow_reader.py b/temporalio/common/streams/_workflow_reader.py index f68ec6f5c3..e35be09dd2 100644 --- a/temporalio/common/streams/_workflow_reader.py +++ b/temporalio/common/streams/_workflow_reader.py @@ -21,7 +21,13 @@ import temporalio.nexus.system import temporalio.workflow from temporalio.common.streams import _convert, _errors -from temporalio.common.streams._items import Closed, End, Record, Superseded +from temporalio.common.streams._items import ( + Closed, + CloseStatus, + End, + Record, + Superseded, +) from temporalio.common.streams._reader import Position, Subscription from temporalio.common.streams.system import models @@ -416,10 +422,19 @@ def workflow_item( return None return Superseded(offset=offset, supersession=model.supersession) if model.kind == stream_pb.StreamRecordKind.STREAM_RECORD_KIND_CLOSE: + failure = model.close_failure + if isinstance(failure, temporalio.exceptions.FailureError) and failure.failure: + # The model's converter keeps payloads raw for the Topic types. + failure = _failure_converter().from_failure(failure.failure, converter) + # A close Record from before close statuses has none, and its failure tells. + status = model.close_status or ( + CloseStatus.FAILED if failure is not None else CloseStatus.COMPLETED + ) return Closed( offset=offset, result=_decode(converter, model.close_result, None), - failure=model.close_failure, + failure=failure, + status=CloseStatus(status), ) return None diff --git a/tests/contrib/streams/test_workflow_reads.py b/tests/contrib/streams/test_workflow_reads.py index 2cd569617c..c4cb17e58a 100644 --- a/tests/contrib/streams/test_workflow_reads.py +++ b/tests/contrib/streams/test_workflow_reads.py @@ -303,6 +303,38 @@ async def test_a_workflow_page_stays_below_the_marker_limit(stream_client: Clien ) +@workflow.defn +class ReadsTheClose: + @workflow.run + async def run(self, stream: streams.WorkflowStream[Events]) -> list[Any]: + [closed] = [item async for item in stream.subscribe()] + assert isinstance(closed, streams.Closed) + assert isinstance(closed.failure, ApplicationError) + return [ + closed.status.name, + closed.failure.message, + closed.failure.type, + [type(detail).__name__ for detail in closed.failure.details], + ] + + +async def test_a_workflow_read_gives_the_close_status_and_failure( + stream_client: Client, +): + # The model's converter keeps the payloads raw, as Topic types decode the bodies later. + stream = await stream_client.create_stream(Events, id=f"failed-{uuid.uuid4()}") + await stream.close( + failure=ApplicationError("tool failed", "detail", {"k": 1}, type="Tool") + ) + async with new_worker(stream_client, ReadsTheClose) as worker: + assert await stream_client.execute_workflow( + ReadsTheClose.run, + streams.WorkflowStream.from_handle(stream), + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) == ["FAILED", "tool failed", "Tool", ["str", "dict"]] + + async def test_a_backend_that_lacks_a_capability_fails_the_task( stream_client: Client, monkeypatch: pytest.MonkeyPatch ): From 8a25f59aa554619b0bdb3168655a41c703819d96 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 02:20:58 -0700 Subject: [PATCH 74/79] Refused a client close of a Standalone Activity's Stream with no call. Only that Activity closes it. As with a Workflow's Stream, the block end of async with would raise and hide the block's own error. --- temporalio/common/streams/_stream_client.py | 14 ++++++++------ tests/contrib/streams/test_stream_client.py | 12 ++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/temporalio/common/streams/_stream_client.py b/temporalio/common/streams/_stream_client.py index d6a8ee46e6..39ebc9a383 100644 --- a/temporalio/common/streams/_stream_client.py +++ b/temporalio/common/streams/_stream_client.py @@ -351,12 +351,12 @@ async def get_offset( return await self._access().get_offset() def _check_close(self) -> None: - if self._workflow_id: - # A Workflow's Stream closes only from its run chain. The Store refuses a client's - # close too, and this error comes with no call. + if self._workflow_id or self._activity_id: + # A Workflow's Stream closes only from its run chain, and a Standalone Activity's + # only from that Activity. The Store refuses a client's close too. raise _errors.StreamNotOwnerError( f"Can't close the Stream {self._id or self._stream_name} from a client, since " - "a Workflow owns it", + f"{'a Workflow' if self._workflow_id else 'an Activity'} owns it", reason="stream_not_owner", ) @@ -375,7 +375,8 @@ async def close( ``rpc_metadata`` and ``rpc_timeout`` are as for :py:meth:`get_offset`. Raises: - StreamNotOwnerError: A Workflow owns the Stream, so only its run chain closes it. + StreamNotOwnerError: A Workflow or a Standalone Activity owns the Stream, so only + its run chain or that Activity closes it. """ self._check_close() self._closed = True @@ -386,7 +387,8 @@ async def __aenter__(self) -> ClientStream[E]: """Use the Stream in ``async with``, which closes it at the end. Raises: - StreamNotOwnerError: A Workflow owns the Stream, so only its run chain closes it. + StreamNotOwnerError: A Workflow or a Standalone Activity owns the Stream, so only + its run chain or that Activity closes it. """ # The close at the end would raise and hide the block's own error. self._check_close() diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index f764eabae9..8222958f97 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -450,6 +450,18 @@ async def test_async_with_refuses_a_workflow_stream_before_the_block( assert not ran +async def test_async_with_refuses_an_activity_stream_before_the_block( + stream_client: Client, +): + # Only the Standalone Activity closes its Stream, so the block end would raise too. + handle = stream_client.get_activity_handle("tool-1", run_id="run-1") + ran = False + with pytest.raises(streams.StreamNotOwnerError, match="an Activity owns it"): + async with handle.stream(Events): + ran = True + assert not ran + + async def test_record_over_the_store_limit_raises_too_large(stream_client: Client): # The client's writer splits a batch at the Store's limit, so only a Record can be over. stream = await stream_client.create_stream(Events, id=f"large-{uuid.uuid4()}") From b4c5ceef15f81f1f948a705d4c61a0a6706b9eb7 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 02:55:35 -0700 Subject: [PATCH 75/79] Tightened the comments and docs on this branch. --- changelog/added/babbling-brook.md | 2 +- changelog/changed/babbling-brook.md | 2 +- scripts/gen_protos.py | 2 +- scripts/gen_stream_service.py | 13 +- temporalio/activity.py | 4 +- temporalio/bridge/Cargo.toml | 9 +- temporalio/bridge/src/streams.rs | 14 +- temporalio/bridge/streams.py | 4 +- temporalio/client/_client.py | 20 +- temporalio/client/_workflow.py | 15 +- temporalio/common/__init__.py | 22 +-- temporalio/common/_streams/_access.py | 17 +- temporalio/common/_streams/_activity.py | 6 +- .../common/_streams/_activity_stream.py | 51 +++--- temporalio/common/_streams/_backend.py | 38 ++-- temporalio/common/_streams/_buffered.py | 49 ++--- temporalio/common/_streams/_convert.py | 6 +- temporalio/common/_streams/_defn.py | 12 +- temporalio/common/_streams/_errors.py | 33 ++-- temporalio/common/_streams/_extension.py | 69 ++++--- temporalio/common/_streams/_handle.py | 45 +++-- temporalio/common/_streams/_items.py | 38 ++-- temporalio/common/_streams/_plugin.py | 43 ++--- temporalio/common/_streams/_reader.py | 52 +++--- temporalio/common/_streams/_stream_client.py | 104 ++++++----- temporalio/common/_streams/_workflow.py | 171 +++++++++--------- temporalio/common/_streams/_writer.py | 30 +-- temporalio/common/_streams/_writes.py | 31 ++-- temporalio/contrib/stream/__init__.py | 55 +++--- temporalio/contrib/stream/testing.py | 18 +- temporalio/contrib/stream/workflow.py | 25 +-- temporalio/converter/_payload_converter.py | 16 +- temporalio/worker/_replayer.py | 6 +- temporalio/worker/_worker.py | 4 +- temporalio/workflow/_streams.py | 72 ++++---- tests/contrib/streams/conftest.py | 6 +- tests/contrib/streams/examples/README.md | 5 +- .../examples/activity_streams_tokens.py | 2 +- .../streams/examples/agent_publishes_turns.py | 4 +- .../contrib/streams/examples/test_examples.py | 2 +- .../streams/examples/turn_scoped_stream.py | 4 +- .../streams/examples/ui_backend_reads.py | 10 +- .../contrib/streams/examples/worker_setup.py | 12 +- tests/contrib/streams/fake_extension.py | 4 +- .../contrib/streams/test_activity_handles.py | 28 +-- tests/contrib/streams/test_facade.py | 6 +- tests/contrib/streams/test_handles.py | 5 +- tests/contrib/streams/test_redis.py | 28 +-- tests/contrib/streams/test_stream_client.py | 23 +-- tests/contrib/streams/test_workflow_module.py | 43 ++--- 50 files changed, 647 insertions(+), 633 deletions(-) diff --git a/changelog/added/babbling-brook.md b/changelog/added/babbling-brook.md index 66af4b2540..d038ff2b7a 100644 --- a/changelog/added/babbling-brook.md +++ b/changelog/added/babbling-brook.md @@ -1,3 +1,3 @@ -Add experimental Streams to `temporalio.common`: ordered logs of Records that Workflows, Activities and clients write, and that clients and Activities read. Register a backend with `stream_backends=` on `Client.connect`, `Replayer` or `WorkflowEnvironment`, then use `workflow.create_stream`, `Client.stream` or `WorkflowHandle.stream`. +Add experimental Streams to `temporalio.common`. A Stream is an ordered log of Records that Workflows, Activities and clients write, and that clients and Activities read. Register a backend with `stream_backends=` on `Client.connect`, `Replayer` or `WorkflowEnvironment`, then use `workflow.create_stream`, `Client.stream` or `WorkflowHandle.stream`. Add a Redis Stream backend in `temporalio.contrib.stream`, and an in-memory one for tests in `temporalio.contrib.stream.testing`. The Redis backend needs a build with the bridge's `streams` feature, which the official builds have. Add `activity.Info.workflow_task`, the Workflow Task attempt in which a Worker dispatched a Local Activity. diff --git a/changelog/changed/babbling-brook.md b/changelog/changed/babbling-brook.md index 7d4504d693..f22c0d8a31 100644 --- a/changelog/changed/babbling-brook.md +++ b/changelog/changed/babbling-brook.md @@ -1 +1 @@ -Convert values of a `transfer_type_convertible` class inside JSON values too, such as list items and dataclass fields, not only at the top level of a payload. A class that is itself a dataclass is still written as its fields there. +Convert values of a `transfer_type_convertible` class inside JSON values too, such as list items and dataclass fields, not only at the top level of a payload. A dataclass with the decorator is still written as its fields inside a JSON value. diff --git a/scripts/gen_protos.py b/scripts/gen_protos.py index adb42a3a5a..4f6c2ad862 100644 --- a/scripts/gen_protos.py +++ b/scripts/gen_protos.py @@ -231,7 +231,7 @@ def generate_protos(output_dir: Path): for p in (output_dir / "temporal" / "sdk" / "core").iterdir(): shutil.rmtree(sdk_out_dir / p.name, ignore_errors=True) p.replace(sdk_out_dir / p.name) - # The stream extension's own protos go with the Streams package that uses them + # Only the Streams package imports the stream extension protos streams_out_dir.mkdir(parents=True, exist_ok=True) for p in (output_dir / "temporal" / "sdk" / "streams").iterdir(): shutil.rmtree(streams_out_dir / p.name, ignore_errors=True) diff --git a/scripts/gen_stream_service.py b/scripts/gen_stream_service.py index f990c63948..c5d5bbc306 100644 --- a/scripts/gen_stream_service.py +++ b/scripts/gen_stream_service.py @@ -2,9 +2,8 @@ The WIT and the protos are copies of the temporalio-streams crate's, in scripts/_wit and scripts/_proto. Run scripts/gen_protos.py first for the proto modules. No nexgen release has the -offset, long-poll and client features yet, so the script installs nexgen at -NEX_GEN_REVISION from moetemp/nexgen. NEX_GEN_STREAMS_BIN may name a build of that revision -instead. +offset, long-poll and client features, so the script installs nexgen at NEX_GEN_REVISION from +moetemp/nexgen. Set NEX_GEN_STREAMS_BIN to use a local build of that revision instead. """ import os @@ -31,11 +30,11 @@ / "stream_service.proto" ) output_dir = base_dir / "temporalio" / "common" / "_streams" / "system" -# The writer is client-only, so it has a caller and its resource class, and no Workflow code. +# Only clients call the writer service, so its bindings have no Workflow code. writer_output_dir = base_dir / "temporalio" / "common" / "_streams" / "writer_system" NEX_GEN_REPOSITORY = "https://github.com/moetemp/nexgen" -# The head of moe/brook/v3/n8-stream-resource. A branch moves, so the pin is the commit. +# A commit on the moe/brook/v3/n8-stream-resource branch. NEX_GEN_REVISION = "204b566ee996939b165ccd1d18c8c432bfdcc0dd" nex_gen_root = base_dir / ".nexgen-streams" / NEX_GEN_REVISION @@ -81,8 +80,8 @@ def generate_stream_service() -> None: raise RuntimeError(f"missing generator input: {path}") with tempfile.TemporaryDirectory(dir=base_dir) as temp_dir: descriptor_path = Path(temp_dir) / "stream_service.bin" - # The extension messages live in temporalio.common._streams.proto, so the descriptor - # names their file by that module path. + # The generated extension messages are in temporalio.common._streams.proto, so the + # descriptor must name their file by that module path. extension_root = Path(temp_dir) / "extension" extension_path = ( extension_root diff --git a/temporalio/activity.py b/temporalio/activity.py index 434f707bdf..712d0a30ee 100644 --- a/temporalio/activity.py +++ b/temporalio/activity.py @@ -112,7 +112,7 @@ class WorkflowTaskAttempt: """ID of the ``WorkflowTaskStarted`` event of the run's last completed task. 0 if none.""" failed_tasks: Sequence[int] = () - """``WorkflowTaskStarted`` event IDs of the run's failed and timed out tasks, the last 100.""" + """``WorkflowTaskStarted`` event IDs of the run's last 100 failed and timed out tasks.""" @dataclass(frozen=True) @@ -167,7 +167,7 @@ class Info: workflow_task: WorkflowTaskAttempt | None = None """For a local activity, the Workflow Task in which the Worker dispatched this execution. - Retries that the Worker runs without a new task keep it. None for other activities. + Retries that the Worker runs in the same task have the same value. None for other activities. .. warning:: This API is experimental. diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index f90fbef389..e103624418 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -39,9 +39,8 @@ tls-aws-lc = [ "tokio-rustls/aws_lc_rs", ] fips = ["tls-aws-lc", "dep:aws-lc-rs", "streams", "temporalio-streams?/fips"] -# Links the Stream runtime, which a Stream plugin builds for each client. No stream code runs -# unless a plugin builds it. The official builds turn it on. It is off by default, since its git -# dependencies would break offline builds of the sdist. +# Off by default, since the git dependencies of temporalio-streams break offline sdist builds. +# The official builds turn it on. No stream code runs unless a Stream plugin builds a runtime. streams = ["dep:temporalio-streams"] [dependencies] @@ -90,8 +89,8 @@ tonic = "0.14" tracing = "0.1" url = "2.2" -# The rev moves with the sdk-core pin, and the patch below makes the crate use the submodule's -# crates, so the bridge sees one type of each. +# Update this rev with the sdk-core pin. The patch below makes this crate use the submodule's +# crates, so the bridge sees one copy of each type. [dependencies.temporalio-streams] git = "https://github.com/moetemp/sdk-streams" rev = "ee6d1e51d91c96f0c661fb59ad0957a97a579cbe" diff --git a/temporalio/bridge/src/streams.rs b/temporalio/bridge/src/streams.rs index 789b093f45..c2dde7255c 100644 --- a/temporalio/bridge/src/streams.rs +++ b/temporalio/bridge/src/streams.rs @@ -1,5 +1,5 @@ -//! The Stream runtime of the temporalio-streams crate, when this build links it. A Stream plugin -//! builds one for each client from its config, and Python calls it by method name. +//! Bindings to the Stream runtime of the temporalio-streams crate. A Stream plugin builds one +//! runtime for each client from its config. Python calls it by method name. use pyo3::exceptions::PyException; #[cfg(feature = "streams")] @@ -8,10 +8,10 @@ use pyo3::prelude::*; #[cfg(feature = "streams")] use crate::runtime; -// Its args are the gRPC code, the message and the encoded details, as with `RPCError`. +// The exception args are the gRPC code, the message and the encoded details, as in `RPCError`. pyo3::create_exception!(temporal_sdk_bridge, StreamCallError, PyException); -/// The crate's runtime object. Python keeps it on the plugin for the client that it serves. +/// The crate's runtime object. Python keeps it with the client that it serves. #[cfg(feature = "streams")] #[pyclass(frozen)] pub struct StreamRuntimeRef { @@ -41,8 +41,8 @@ impl StreamRuntimeRef { }) } - /// Releases one handle of a writer with no wait. It needs no runtime, so a finalizer of a - /// handle that Python drops can call it. + /// Releases one handle of a writer without waiting. It needs no Tokio runtime, so a Python + /// finalizer can call it. fn free_writer(&self, writer_id: u64) -> PyResult<()> { self.streams.free_writer(writer_id).map_err(call_error) } @@ -64,7 +64,7 @@ pub fn new_stream_runtime( let runtime = runtime_ref.runtime.clone(); runtime.assert_same_process("create a Stream runtime")?; - // The runtime may start tasks on the Tokio runtime that it lives on. + // Building the Stream runtime can spawn Tokio tasks, which need an entered Tokio runtime. let _guard = runtime.core.tokio_handle().enter(); let meter = runtime.core.telemetry().get_temporal_metric_meter(); let streams = if memory { diff --git a/temporalio/bridge/streams.py b/temporalio/bridge/streams.py index 23097bc503..4486ea32ff 100644 --- a/temporalio/bridge/streams.py +++ b/temporalio/bridge/streams.py @@ -31,7 +31,7 @@ def __init__( @property def name(self) -> str: - """Name the runtime gives itself.""" + """Name that the runtime reports.""" return self._ref.name async def call(self, method: str, input: bytes) -> bytes: @@ -43,5 +43,5 @@ async def call(self, method: str, input: bytes) -> bytes: return await self._ref.call(method, input) def free_writer(self, writer_id: int) -> None: - """Release one handle of a writer with no wait, such as from a finalizer.""" + """Release one handle of a writer without waiting, such as from a finalizer.""" self._ref.free_writer(writer_id) diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index 66937dfe8b..73dd8c600c 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -232,8 +232,8 @@ async def connect( fields are logged but still sent. Set a threshold to 0 to disable it. header_codec_behavior: Encoding behavior for headers sent by the client. stream_backends: The Stream backends of this client and its Workers, such as - :py:class:`temporalio.contrib.stream.RedisStreamBackend`. Names are unique. - With more than one, exactly one has ``default=True``. See + :py:class:`temporalio.contrib.stream.RedisStreamBackend`. Names must be unique. + With more than one backend, exactly one must have ``default=True``. See :py:class:`temporalio.common.StreamBackend`. .. warning:: @@ -923,10 +923,10 @@ def stream( stream: StreamHandle[Any] | None = None, backend: str | None = None, ) -> ClientStream[_StreamDefinitionT]: - """A Stream by its id, or a handle that came as data, such as in a Record. + """Get a Stream by its id, or from a handle that came in data, such as in a Record. - It makes no call. To create a Stream that the client owns, use the create of a - backend, such as :py:func:`temporalio.contrib.stream.create`. + This method makes no call. To create a Stream that the client owns, use a backend's + ``create`` function, such as :py:func:`temporalio.contrib.stream.create`. .. warning:: This API is experimental. @@ -934,14 +934,14 @@ def stream( Args: definition: The Stream definition class. id: The Stream id. - stream: A handle of the Stream, such as one in a Record. Its id, owner and - backend stay. - backend: The name of the backend. Defaults to the handle's, and else to the + stream: A handle of the Stream, such as one in a Record. The result keeps its id, + owner and backend. + backend: The name of the backend. Defaults to the handle's backend, or else to the registered backend with ``default=True``. Raises: - StreamConfigError: The client has no backend of that name, or none has - ``default=True`` for a call with no backend. + StreamConfigError: The client has no backend with that name, or no backend has + ``default=True`` and the call gives no backend. TypeError: Neither or both of ``id`` and ``stream`` are given. ValueError: The id is empty, or ``backend`` is not the handle's backend. """ diff --git a/temporalio/client/_workflow.py b/temporalio/client/_workflow.py index e1a1d2a3aa..400204ea96 100644 --- a/temporalio/client/_workflow.py +++ b/temporalio/client/_workflow.py @@ -195,23 +195,24 @@ def first_execution_run_id(self) -> str | None: def stream( self, definition: type[_StreamDefinitionT], *, backend: str | None = None ) -> ClientStream[_StreamDefinitionT]: - """A Stream that this Workflow created, by its definition. + """Get a Stream that this Workflow created, by its definition. - It builds the Stream id from the Workflow id and the first run id, so it makes no - call. A handle with no first run id makes one Describe call on the first use of its - Streams, and keeps the answer. + This method builds the Stream id from the Workflow id and the first run id, so it makes + no call. A handle with no first run id makes one Describe call when one of its Streams + is first used, and caches the result. .. warning:: This API is experimental. Args: - definition: The Stream definition class, renamed as the Workflow's create. + definition: The Stream definition class, with the same name as in the Workflow's + create. backend: The name of the backend. Defaults to the registered backend with ``default=True``. Raises: - StreamConfigError: The client has no backend of that name, or none has - ``default=True`` for a call with no backend. + StreamConfigError: The client has no backend with that name, or no backend has + ``default=True`` and the call gives no backend. """ from temporalio.common._streams import _stream_client diff --git a/temporalio/common/__init__.py b/temporalio/common/__init__.py index f0cd0d72ed..2e6ce3b1af 100644 --- a/temporalio/common/__init__.py +++ b/temporalio/common/__init__.py @@ -1,20 +1,20 @@ """Common code used in the Temporal SDK. -It also has the Streams API: Streams of Records that Workflows, Activities and clients write -and read. A Stream definition, a class with ``@common.defn``, gives the Stream name and the -type of each Topic. Workflow code creates a Stream with +This module also has the Streams API. A Stream is an ordered log of Records that Workflows, +Activities and clients write and read. A Stream definition is a class with ``@common.defn``. +It gives the Stream name and the type of each Topic. Workflow code creates a Stream with :py:func:`temporalio.workflow.create_stream`. A client gets a Workflow's Stream with :py:meth:`temporalio.client.WorkflowHandle.stream`, and any Stream with -:py:meth:`temporalio.client.Client.stream`. A Stream is data: a Workflow passes it to an -Activity, a child Workflow or the next run as an argument, and the parameter's type gives the -handle class of that context, such as :py:class:`ActivityStream`. +:py:meth:`temporalio.client.Client.stream`. A Stream is data. A Workflow can pass it as an +argument to an Activity, a child Workflow or the next run. The parameter's type sets the handle +class for that context, such as :py:class:`ActivityStream`. -A :py:class:`StreamBackend` carries the calls and registers a backend name, such as +A :py:class:`StreamBackend` makes the Stream calls and registers a backend name. Examples are :py:class:`temporalio.contrib.stream.RedisStreamBackend` (``redis``) and :py:class:`temporalio.contrib.stream.testing.MemoryStreamBackend` (``memory``). Register -backends with ``Client.connect(..., stream_backends=[...])``. A handle carries the name of its -backend. With more than one registered backend, exactly one has ``default=True``, the one -that a create with no ``backend=`` uses. For now a Worker serves one backend. +backends with ``Client.connect(..., stream_backends=[...])``. A handle stores the name of its +backend. With more than one registered backend, exactly one must have ``default=True``. A +create with no ``backend=`` uses that backend. A Worker serves only one backend. .. warning:: The Streams API is experimental. @@ -1630,7 +1630,7 @@ def wait_sync(self, timeout: float | None = None) -> None: # The Stream modules import temporalio.client and temporalio.workflow, which import this module, -# so the Stream names load on first use. +# so this module imports the Stream names on first use. _STREAM_NAMES = frozenset( [ "ActivityStream", diff --git a/temporalio/common/_streams/_access.py b/temporalio/common/_streams/_access.py index 19fc94de0a..3dba3ff860 100644 --- a/temporalio/common/_streams/_access.py +++ b/temporalio/common/_streams/_access.py @@ -29,7 +29,7 @@ class Access: namespace: str stream_id: Callable[[], Awaitable[str]] writer: Callable[[], StreamWriter] - # The run chain that a close names, which the Store checks against the Stream's owner. + # A close sends these, and the Store checks them against the Stream's owner. workflow_id: str | None = None first_execution_run_id: str | None = None @@ -70,17 +70,15 @@ async def items( await items.aclose() async def get_offset(self) -> int: - """The next offset of the Stream, the newest offset plus one. + """The next offset of the Stream, the newest offset plus one, or 0 if it has no Record. - It is 0 for a Stream with no Record, and ``offset=0`` reads from the start. - Offsets are sparse and above zero, so 0 is never a Record's offset. + Offsets are sparse and above zero, so ``offset=0`` reads from the start. """ response = await self.backend.service(self.namespace).describe_stream( models.DescribeStreamRequest( stream=models.StreamRef(id=await self.stream_id()) ) ) - # Offsets start above zero, so zero means the Stream has no Record. return response.last_offset + 1 if response.last_offset else 0 async def close(self, result: Any | None, failure: BaseException | None) -> None: @@ -135,7 +133,7 @@ def read_spec( """The Topic names of a read, the decode type of each known Topic, and its options. Raises: - ValueError: ``last`` is not above zero, or it comes with ``offset``. + ValueError: ``last`` is not above zero, or the call also gives ``offset``. """ if last is not None and (last < 1 or offset is not None): raise ValueError("last takes a count above zero and no offset") @@ -147,7 +145,7 @@ def read_spec( elif isinstance(topic, TopicDefinition): name, topic_type = handle._topic_spec(topic, None) else: - # A read names a Topic outside the definition with no type, and its values + # A Topic name that the definition doesn't have gets no type here, so its values # decode with the read's default type. name, topic_type = topic, types.get(topic) names.append(name) @@ -171,8 +169,7 @@ def subscription( options: CallOptions | None = None, ) -> Subscription[Any]: """A read over the items that ``items`` gives.""" - # Before its first page, a read from the start resumes at 0, and a read from END or of the - # last Records has no offset yet. + # A read from END or of the last Records has no resume offset until its first page. if read.last is not None or read.offset is End.END: position = Position(None) else: @@ -192,4 +189,4 @@ async def publish(self, *values: T) -> int: class SyncTopicBase(TopicBase[T]): - """Base of a Topic whose calls block, for sync Activities.""" + """Base class of a Topic whose calls block, for sync Activities.""" diff --git a/temporalio/common/_streams/_activity.py b/temporalio/common/_streams/_activity.py index 593d9dfdc9..09ff5360b4 100644 --- a/temporalio/common/_streams/_activity.py +++ b/temporalio/common/_streams/_activity.py @@ -25,9 +25,9 @@ class ActivityStreams: backend_name: str = "" # Whether the Worker's backend is the default, which a handle with no backend name uses. default: bool = False - # The Workflow of the attempt, so a close checks that the Stream is its run chain's. + # The attempt's Workflow id, which a close sends so the Store can check the Stream's owner. workflow_id: str | None = None - # A Local Activity's run history, which lets its writes withdraw the run's orphaned Records. + # A Local Activity's writes use these to withdraw the run's orphaned Records. previous_completed_task: stream_pb.WorkflowTaskEpoch | None = None failed_tasks: tuple[int, ...] = () writers: dict[str, StreamWriter] = field(default_factory=dict) @@ -54,7 +54,7 @@ async def known_id() -> str: return writer async def close(self) -> None: - """Flush the attempt's publishers, then drop every writer of the attempt.""" + """Flush the attempt's publishers, then close every writer of the attempt.""" errors = [] for publisher in self.publishers: try: diff --git a/temporalio/common/_streams/_activity_stream.py b/temporalio/common/_streams/_activity_stream.py index 81cfc09e23..d3af608ae8 100644 --- a/temporalio/common/_streams/_activity_stream.py +++ b/temporalio/common/_streams/_activity_stream.py @@ -50,8 +50,8 @@ def _bound(handle: StreamHandle[Any]) -> _activity.ActivityStreams: def _check_close(handle: StreamHandle[Any]) -> None: - # A Workflow's Stream closes only from Workflow code, once its task commits, and a client's - # only from a client. The Store refuses the others too, and this error comes with no call. + # Only Workflow code can close a Workflow's Stream, and only a client can close a client's + # Stream. This check refuses some closes without a call, and the Store refuses the rest. streams = _bound(handle) local = streams.producer.kind == stream_pb.ProducerKind.PRODUCER_KIND_LOCAL_ACTIVITY if not local or handle.workflow_id: @@ -75,8 +75,8 @@ async def known_id() -> str: namespace=streams.namespace, stream_id=known_id, writer=lambda: streams.writer(stream_id), - # A handle with only an id names no chain, so the close names this Activity's own, and - # the Store refuses it for a Workflow's Stream. + # A handle with only an id has no run chain, so the close sends the Activity's Workflow + # id. The Store refuses that close for a Workflow's Stream. workflow_id=handle.workflow_id or streams.workflow_id, first_execution_run_id=handle.first_execution_run_id, ) @@ -91,7 +91,7 @@ class ActivityTopic(_AsyncTopic[T]): class ActivityStream(StreamHandle[E]): - """A Stream in an async Activity. The Activity writes, reads and closes it. + """A Stream in an async Activity. The Activity can write and read it. An Activity gets a Stream only as an argument, such as a field of its input. Each publish writes before it returns. Use :py:class:`BufferedPublisher` to batch. @@ -144,10 +144,9 @@ def subscribe( return subscription(lambda position: access.items(read, position), read) async def get_offset(self) -> int: - """The next offset of the Stream, the newest offset plus one. + """The next offset of the Stream, the newest offset plus one, or 0 if it has no Record. - It is 0 for a Stream with no Record, and ``offset=0`` reads from the start. - Offsets are sparse and above zero, so 0 is never a Record's offset. + Offsets are sparse and above zero, so ``offset=0`` reads from the start. """ return await self._access().get_offset() @@ -156,14 +155,15 @@ async def close( ) -> None: """Write this attempt's Records, then close the Stream. - Readers get the Records before it, then a ``Closed`` item with ``result`` or - ``failure``. A cancel failure, such as a ``CancelledError``, closes it as canceled. + Readers get the Records before the close, then a ``Closed`` item with ``result`` or + ``failure``. A cancel failure, such as a ``CancelledError``, closes the Stream as + canceled. - No Activity closes a Workflow's Stream, also a Local Activity of the run chain that - created it, and no Activity closes a client's Stream. They get - :py:class:`StreamNotOwnerError`. A Local Activity writes before its Workflow Task - commits, so its close would stay when the task fails, and the task's retry could not - create the Stream again. Workflow code closes a Workflow's Stream. + An Activity can't close a Workflow's Stream or a client's Stream, and this call raises + :py:class:`StreamNotOwnerError`. The rule includes a Local Activity of the run chain + that created the Stream. A Local Activity runs before its Workflow Task commits. If the + task then failed, the close would stay, and the task's retry couldn't create the Stream + again. Close a Workflow's Stream from Workflow code. """ _check_close(self) await self._access().close(result, failure) @@ -187,9 +187,9 @@ def publish(self, *values: T) -> int: class SyncSubscription(Generic[T]): - """The items of one read of a Stream in a sync Activity, in order. Use it in ``for``. + """The items of one read of a Stream in a sync Activity, in order. Iterate it with ``for``. - Each step blocks the Activity's thread. See :py:class:`Subscription`. + Each iteration blocks the Activity's thread. See :py:class:`Subscription`. .. warning:: This API is experimental. @@ -246,15 +246,14 @@ def _run(self, call: Coroutine[Any, Any, _R]) -> _R: future = asyncio.run_coroutine_threadsafe(call, loop) try: while True: - # Short waits let an exception that the Worker raises in this thread, such as - # a cancel, come in. + # Short waits let the Worker raise an exception in this thread, such as a cancel. try: return future.result(timeout=_WAIT_STEP) except concurrent.futures.TimeoutError: continue except BaseException: - # A cancel of the Activity thread, such as from a heartbeat, stops the call too, - # so a long poll does not run on in the Worker's loop. + # Cancel the call too, so a long poll doesn't keep running in the Worker's loop after + # the Activity thread is canceled, such as from a heartbeat. future.cancel() raise @@ -314,18 +313,18 @@ def _items( self._run(items.aclose()) def get_offset(self) -> int: - """The next offset of the Stream, the newest offset plus one. + """The next offset of the Stream, the newest offset plus one, or 0 if it has no Record. - It is 0 for a Stream with no Record, and ``offset=0`` reads from the start. - Offsets are sparse and above zero, so 0 is never a Record's offset. + Offsets are sparse and above zero, so ``offset=0`` reads from the start. """ return self._run(self._access().get_offset()) def close( self, *, result: Any | None = None, failure: BaseException | None = None ) -> None: - """Write this attempt's Records, then close the Stream. See - :py:meth:`ActivityStream.close`. + """Write this attempt's Records, then close the Stream. + + See :py:meth:`ActivityStream.close`. """ _check_close(self) self._run(self._access().close(result, failure)) diff --git a/temporalio/common/_streams/_backend.py b/temporalio/common/_streams/_backend.py index d0fbf167e3..9d77298f5c 100644 --- a/temporalio/common/_streams/_backend.py +++ b/temporalio/common/_streams/_backend.py @@ -22,7 +22,7 @@ from temporalio.common._streams.system import client as service_client if TYPE_CHECKING: - # The writer bindings import this package's protos, so they load after it. + # The writer bindings import this package's protos, so writers() imports them on first use. from temporalio.common._streams.writer_system import client as writer_client _M = TypeVar("_M", bound=google.protobuf.message.Message) @@ -60,8 +60,8 @@ class CallOptions: timeout: timedelta | None -# The options of the client call in progress. Each transport reads them, so they reach the -# generated clients' calls and the writer's with no new argument there. +# The options of the client call in progress. Each transport reads them, so the generated +# clients and the writer don't need a new argument for them. call_options: contextvars.ContextVar[CallOptions | None] = contextvars.ContextVar( "__temporal_stream_call_options", default=None ) @@ -78,7 +78,7 @@ def options_of( @contextlib.contextmanager def using(options: CallOptions | None) -> Iterator[None]: - """Give the calls in the block these options.""" + """Apply these options to the calls in the block.""" if options is None: yield return @@ -90,14 +90,12 @@ def using(options: CallOptions | None) -> Iterator[None]: def longest_wait(longest: timedelta) -> timedelta: - """The longest wait of one call. It is less than ``longest`` to answer before the call's - rpc_timeout. - """ + """The longest wait of one call, at most ``longest`` and below the call's rpc_timeout.""" options = call_options.get() if options is None or options.timeout is None: return longest - # A long poll answers before the call's deadline, as the server does for its own long - # polls, so a read that waits goes on with its next poll and does not fail. + # A long poll returns before the call's deadline, as the server's long polls do, so a + # waiting read starts its next poll and doesn't fail. timeout = options.timeout return min(longest, max(timeout - timedelta(seconds=1), timeout / 2)) @@ -120,7 +118,7 @@ def writers( ... def free_writer(self, writer_id: int) -> None: - """Release one handle of a writer with no wait, for a writer that Python drops.""" + """Release one handle of a writer without waiting, such as when Python frees it.""" ... @@ -128,19 +126,19 @@ class StreamRuntime(Protocol): """The calls of the crate's runtime object, which the bridge builds.""" async def call(self, method: str, input: bytes) -> bytes: - """Call a method with encoded input, and give the encoded output.""" + """Call a method with encoded input, and return the encoded output.""" ... def free_writer(self, writer_id: int) -> None: - """Release one handle of a writer with no wait.""" + """Release one handle of a writer without waiting.""" ... class _ExtensionCalls: - """Carries Stream calls to the crate's runtime object, which serves one client. + """Makes Stream calls to the crate's runtime object, which serves one client. - The five Stream service operations go through the generated client, and the writer - methods are the crate's own. + The five Stream service operations use the generated client. The writer methods are the + crate's own. """ def __init__(self, runtime: StreamRuntime) -> None: @@ -154,7 +152,8 @@ def service( ) -> service_client.StreamServiceClient: """A generated Stream service client for ``namespace``. - With a ``data_converter``, user values in the requests go through it and its codec. + With a ``data_converter``, the client encodes user values in the requests with it and + its codec. """ return service_client.StreamServiceClient( _ExtensionTransport(self, namespace), data_converter=data_converter @@ -171,7 +170,7 @@ def writers( ) def free_writer(self, writer_id: int) -> None: - """Release one handle of a writer with no wait.""" + """Release one handle of a writer without waiting.""" try: self._runtime.free_writer(writer_id) except temporalio.bridge.streams.StreamCallError: @@ -190,7 +189,7 @@ async def call_async( return response.FromString(data) async def close_writers(self) -> int: - """Drop every open writer of the runtime, and return how many failed.""" + """Close every open writer of the runtime, and return how many failed.""" response = await self.call_async( "CloseWriters", ext_pb.CloseWritersRequest(), ext_pb.CloseWritersResponse ) @@ -198,8 +197,7 @@ async def close_writers(self) -> int: class _ExtensionTransport: - # The generated client takes a transport, so the five operations reach the runtime as they - # would reach a server later. + # The generated client calls a transport. This one calls the runtime in place of a server. def __init__(self, backend: _ExtensionCalls, namespace: str) -> None: self._backend = backend self._namespace = namespace diff --git a/temporalio/common/_streams/_buffered.py b/temporalio/common/_streams/_buffered.py index 4af1a9d488..678681d2ef 100644 --- a/temporalio/common/_streams/_buffered.py +++ b/temporalio/common/_streams/_buffered.py @@ -16,7 +16,7 @@ T = TypeVar("T") U = TypeVar("U") -# The Records of one batch of the crate's writer. A buffer that holds a batch writes at once. +# Records in one batch of the crate's writer. A full batch is written at once. _BATCH = 1000 @@ -26,23 +26,24 @@ class _AnyTopic: class BufferedPublisher(Generic[T]): - """Batches the publishes of one Topic, or of every Topic of one Stream, for an Activity or - a client that writes often. + """Batches the publishes of one Topic, or of every Topic of one Stream. - :py:meth:`publish` returns at once. The publisher writes the buffered values as one batch - at most ``max_delay`` after the first of them, or at once when they fill a batch. One write - is in flight at a time, and values published during it wait for the next one. Use it in - ``async with``, which writes the rest at the end of the block. In an Activity, the end of - the attempt also writes the rest. + Use it in an Activity or a client that writes often. :py:meth:`publish` returns at once. + The publisher writes the buffered values as one batch at most ``max_delay`` after the first + of them, or at once when they fill a batch. One write is in flight at a time, and values + published during it wait for the next one. Use it in ``async with``, which writes the + remaining values at the end of the block. In an Activity, the end of the attempt also + writes the remaining values. ``await flush()`` is the backpressure point. A caller that publishes faster than the Store - takes Records awaits it now and then, so the buffer does not grow without bound. + accepts Records must await it from time to time, so the buffer doesn't grow without bound. - Over a Stream, ``publish(topic, *values)`` takes the Topic of each call first, and all the - Topics share one buffer, so the Records keep the publish order across Topics. + A publisher over a Stream takes the Topic as the first argument of + ``publish(topic, *values)``. All the Topics share one buffer, so the Records keep the + publish order across Topics. - After a batch fails, the publisher writes nothing more, so a reader sees no gap, and each - later call raises the error. + After a batch fails, the publisher writes nothing more, so a reader sees no gap. Each later + call raises the error. .. warning:: This API is experimental. @@ -75,7 +76,7 @@ def __init__( Args: target: A Topic of an ``ActivityStream`` or a ``ClientStream``, or the Stream itself for a publisher of all its Topics. - max_delay: How long a value waits in the buffer at most. + max_delay: The longest time that a value waits in the buffer. """ self._topic: ActivityTopic[Any] | ClientTopic[Any] | None = None self._stream: ActivityStream[Any] | ClientStream[Any] | None = None @@ -85,12 +86,12 @@ def __init__( self._stream = target self._max_delay = max_delay self._buffer: list[tuple[str, Any]] = [] - # When the oldest buffered value came, so the next batch goes at most max_delay later. + # When the oldest buffered value arrived, so the next batch is at most max_delay later. self._first_at = 0.0 self._timer: asyncio.TimerHandle | None = None self._writing: asyncio.Task[None] | None = None self._published = 0 - # Values that a write took, written or not. + # Count of the values that a write took, written or not. self._settled = 0 self._error: BaseException | None = None self._last_offset = 0 @@ -119,8 +120,8 @@ def publish(self, *values: T) -> None: ... def publish(self, *args: Any) -> None: """Add one Record for each value to the buffer, and return at once. - A publisher over a Stream takes the Topic first. It is a Topic of the definition, such - as ``Events.tokens``, a name of the definition, or a Topic of the Stream. + A publisher over a Stream takes the Topic first. The Topic can be a Topic of the + definition, such as ``Events.tokens``, a Topic name, or a Topic of the Stream. Raises: StreamError: A batch failed to write, so the publisher takes no more values. @@ -151,10 +152,9 @@ def _topic_name(self, topic: Any) -> str: return name async def flush(self) -> int: - """Write every value published before the call, and return the offset of the last - Record written. + """Write every value published before the call, and return the last written offset. - Returns 0 when the publisher wrote nothing yet. + Returns 0 if the publisher hasn't written anything. """ target = self._published while self._settled < target: @@ -171,8 +171,8 @@ def _raise_error(self) -> None: raise self._error async def _flush_at_attempt_end(self) -> None: - # The attempt fails for values that never went out, unless the Activity already saw - # the error and went on. + # Fail the attempt for unwritten values, unless the Activity already got the error and + # continued. reported = self._reported try: await self.flush() @@ -212,7 +212,8 @@ async def _write(self, batch: list[tuple[str, Any]]) -> None: finally: self._settled += len(batch) self._writing = None - # Values published during the write go next, when their delay or a batch is up. + # Schedule the values published during the write, for when their delay ends or they + # fill a batch. self._schedule() async def _send(self, batch: list[tuple[str, Any]]) -> int: diff --git a/temporalio/common/_streams/_convert.py b/temporalio/common/_streams/_convert.py index 47261f9b0d..5b769b82e9 100644 --- a/temporalio/common/_streams/_convert.py +++ b/temporalio/common/_streams/_convert.py @@ -14,7 +14,7 @@ class _RawPayloadConverter(temporalio.converter.PayloadConverter): # The generated client runs the codec on Record bodies and keeps them as RawValue, so the - # facade converts each body with its Topic's type. + # handle converts each body with its Topic's type. def to_payloads( self, values: Sequence[Any] ) -> list[temporalio.api.common.v1.Payload]: @@ -43,9 +43,7 @@ def raw( def failure( converter: temporalio.converter.DataConverter, failure: BaseException ) -> BaseException: - """Convert a close failure that a read gave with RawValue payloads, after the codec, with - the user's payload converter. - """ + """Convert the RawValue payloads of a read's close failure with the user's converter.""" if ( not isinstance(failure, temporalio.exceptions.FailureError) or not failure.failure diff --git a/temporalio/common/_streams/_defn.py b/temporalio/common/_streams/_defn.py index 4485b379e4..76a37089fa 100644 --- a/temporalio/common/_streams/_defn.py +++ b/temporalio/common/_streams/_defn.py @@ -1,4 +1,4 @@ -"""Stream definitions: the Stream name and the type of each Topic.""" +"""Stream definitions, which give the Stream name and the type of each Topic.""" from __future__ import annotations @@ -26,7 +26,7 @@ def __init__(self, type: Any, name: str | None = None) -> None: self._owner: type | None = None def __set_name__(self, owner: type, name: str) -> None: - """Take the attribute name as the Topic name, unless the Topic has a name.""" + """Use the attribute name as the Topic name, unless the Topic has one.""" if self._name is None: self._name = name self._owner = owner @@ -61,7 +61,7 @@ def topic(type: Any, *, name: str | None = None) -> TopicDefinition[Any]: ... def topic(type: Any, *, name: str | None = None) -> TopicDefinition[Any]: """Declare a Topic in a Stream definition class. - ``RawValue`` keeps each Record's Payload. + Use ``RawValue`` as the type to read each Record's Payload with no conversion. Args: type: The type of the Topic's values. @@ -101,7 +101,7 @@ def defn( ) -> ClassT | Callable[[ClassT], ClassT]: """Mark a class as a Stream definition. Its :py:func:`topic` attributes are the Topics. - A definition has no backend in it, so Workflows, Activities and clients share it. + A definition doesn't name a backend, so Workflows, Activities and clients can share it. Args: cls: The class. @@ -140,9 +140,9 @@ def decorate(cls: ClassT) -> ClassT: def with_name(definition: ClassT, name: str) -> ClassT: - """The definition with the same Topics and another Stream name, such as one per turn. + """Return a definition with the same Topics and another Stream name, such as one per turn. - It is a subclass of ``definition``, which stays unchanged. Its Topics are the same + The result is a subclass of ``definition``, which doesn't change. Its Topics are the same attributes, so ``Events.tokens`` is a Topic of ``with_name(Events, "turn-3")`` too. Args: diff --git a/temporalio/common/_streams/_errors.py b/temporalio/common/_streams/_errors.py index 6ff59ba6fd..c8f480b7e1 100644 --- a/temporalio/common/_streams/_errors.py +++ b/temporalio/common/_streams/_errors.py @@ -10,12 +10,12 @@ class StreamError(temporalio.exceptions.ApplicationError): - """Base of every Stream error. + """Base class of every Stream error. - It is an application error that is not retryable, except - :py:class:`StreamStorageError`. So in Workflow code, where only - :py:class:`StreamClosedError` and the backend errors are raised, an uncaught one fails the - Workflow and does not fail the Workflow Task again and again. + Each Stream error is a non-retryable application error, except + :py:class:`StreamStorageError`. So an uncaught Stream error in Workflow code fails the + Workflow, and doesn't fail the Workflow Task again and again. In Workflow code, only + :py:class:`StreamClosedError` and the backend errors are raised. .. warning:: This API is experimental. @@ -83,30 +83,33 @@ class StreamTooLargeError(StreamError): class StreamUnsupportedError(StreamError): - """The Store lacks a capability that the call needs. The message ends with its name.""" + """The Store lacks a capability that the call needs. + + The message ends with the name of the capability. + """ class StreamNotOwnerError(StreamError): - """The caller may not close the Stream or write as that Producer, since the Stream's owner - doesn't allow it. + """The caller isn't allowed to close the Stream or write as that Producer. - A Workflow's Stream closes only from its own run chain, and a client's Stream only from a - client. + Only the Workflow's own run chain can close a Workflow's Stream. Only a client can close a + client's Stream. """ class StreamSeqDivergentError(StreamError): - """A resend has the sequence of the Producer's last batch and other content. + """A resent batch has the sequence number of the Producer's last batch, but other content. - A writer that starts again with the same Producer id and epoch sends its first sequence + A writer that restarts with the same Producer id and epoch sends its first sequence number again. Give it a higher epoch instead. """ class StreamConfigError(RuntimeError): - """No Stream backend, or no backend of a given name. It is config, not a Store error. + """The Stream backend config is wrong, such as when no backend has a given name. - ``except StreamError`` does not catch it. It fails a Workflow Task, and an Activity retries. + It isn't a :py:class:`StreamError`, so ``except StreamError`` doesn't catch it. In Workflow + code it fails the Workflow Task. An Activity that raises it retries. """ @@ -142,7 +145,7 @@ class ReadOnlyContextError(StreamError): def from_call_error(err: Exception) -> StreamError: """Turn a failed call of the crate's runtime into the Stream error for its reason. - Its args are the gRPC code, the message and the encoded ``StreamErrorDetails``. + ``err.args`` are the gRPC code, the message and the encoded ``StreamErrorDetails``. """ code, message, details = err.args reason = StreamErrorDetails.FromString(details).reason if details else "" diff --git a/temporalio/common/_streams/_extension.py b/temporalio/common/_streams/_extension.py index 9719670b88..97d6c502f8 100644 --- a/temporalio/common/_streams/_extension.py +++ b/temporalio/common/_streams/_extension.py @@ -1,4 +1,4 @@ -"""The Stream backends whose calls go to the temporalio-streams runtime in the bridge.""" +"""The Stream backends that call the temporalio-streams runtime in the bridge.""" from __future__ import annotations @@ -34,19 +34,19 @@ logger = logging.getLogger(__name__) # The backend puts this key into the headers of each Local Activity that a Workflow schedules, -# and of each Activity of a run that uses Streams. It keeps the Producer id across retries. +# and of each Activity of a run that uses Streams. It keeps the Producer id the same on retries. _KEY_HEADER = "__temporal_stream_key" # Continue-as-new passes the run chain's open Streams to the next run in this header. _OWNED_STREAMS_HEADER = "__temporal_owned_streams" _CLOSE_TIMEOUT = timedelta(seconds=30) -# A Store error is retryable and every other Stream error is not, as each one says itself. +# Only a Store error is retryable. Each Stream error sets its own non_retryable flag. _CLOSE_RETRY = temporalio.common.RetryPolicy( initial_interval=timedelta(seconds=1), maximum_interval=timedelta(seconds=30) ) class ExtensionStreamBackend(StreamBackend): - """Base of the Redis and memory backends, whose calls go to the temporalio-streams runtime. + """Base class of the Redis and memory backends, which call the temporalio-streams runtime. .. warning:: This API is experimental. @@ -63,7 +63,7 @@ def __init__( ) -> None: """Create the backend over the bridge's Stream runtime, on Redis or in memory. - ``backend_name`` is the name that Stream handles carry for this backend. ``default`` + ``backend_name`` is the name that Stream handles store for this backend. ``default`` makes it the backend of a create with no backend name. """ config.backend = backend_name @@ -99,7 +99,6 @@ def _new_backend( def _configure_client( self, config: temporalio.client.ClientConfig ) -> temporalio.client.ClientConfig: - # Each client gets its own Stream runtime. config = super()._configure_client(config) backend = self._new_backend(_runtime_of(config["service_client"])) config["interceptors"] = [ @@ -132,7 +131,7 @@ def _interceptor_of(self, interceptors: Sequence[Any]) -> _StreamInterceptor | N def _configure_worker( self, config: temporalio.worker.WorkerConfig ) -> temporalio.worker.WorkerConfig: - # Add the backend's Activities. The Worker takes the interceptor from its client. + # The Worker gets the Stream interceptor from its client. config = super()._configure_worker(config) client = config["client"] # type: ignore[reportTypedDictNotRequiredAccess] interceptors = [ @@ -154,7 +153,7 @@ def _configure_worker( "stream_backends=." ) if config.get("no_remote_activities"): - # Another Worker on the queue may run the close Activity, so it is no error. + # Another Worker on the task queue can run the close Activity, so it isn't an error. logger.warning( "The Worker on the task queue %s polls no Activities, so the close of a " "Workflow's Stream waits for another Worker on that queue with the backend", @@ -165,8 +164,8 @@ def _configure_worker( mine.converter = client.data_converter runner = config.get("workflow_runner") if isinstance(runner, SandboxedWorkflowRunner): - # The interceptor and the facade must share the run's Stream state, also when the - # restrictions don't pass temporalio through. + # The interceptor and temporalio.workflow must share the run's Stream state, even + # when the restrictions don't pass temporalio through. config["workflow_runner"] = dataclasses.replace( runner, restrictions=runner.restrictions.with_passthrough_modules( @@ -195,10 +194,8 @@ def _configure_replayer( return config async def _close_in_workflow(self, request: stream_pb.CloseStreamRequest) -> int: - # Workflow code closes a Workflow's Stream here. The close must run only once the - # Workflow Task commits, so a task that fails before then leaves the Stream open for its - # retry. A regular Activity on the Workflow's task queue does that, and it closes as the - # run chain, the Stream's owner. A backend that closes another way overrides this. + # An Activity runs only after the Workflow Task commits, so a failed task leaves the + # Stream open for its retry. The Activity closes the Stream as the run chain that owns it. return await temporalio.workflow.execute_activity( streams_workflow.CLOSE_ACTIVITY, request, @@ -208,8 +205,9 @@ async def _close_in_workflow(self, request: stream_pb.CloseStreamRequest) -> int ) async def shutdown(self) -> None: - """Drop every open writer, including the client's. Call it once the process no longer - writes Streams. + """Close every open writer, including the client's. + + Call it when the process stops writing to Streams. """ failed = 0 for backend in list(self._backends): @@ -221,7 +219,7 @@ async def shutdown(self) -> None: def _runtime_of( service: temporalio.service.ServiceClient, ) -> temporalio.runtime.Runtime: - # The runtime of a client's connection, as the Worker finds it. + # Find the runtime of a client's connection the same way the Worker does. service = getattr(service, "worker_service_client", service) runtime = getattr(service, "_runtime", None) or getattr( getattr(service, "config", None), "runtime", None @@ -230,8 +228,8 @@ def _runtime_of( class _StreamInterceptor(temporalio.client.Interceptor, temporalio.worker.Interceptor): - # One for each client of the backend, with that client's Stream runtime. The client's - # Workers take it from the client. + # There is one for each client of the backend, with that client's Stream runtime. The + # client's Workers get it from the client. def __init__( self, plugin: ExtensionStreamBackend, @@ -246,7 +244,7 @@ def __init__( def workflow_interceptor_class( self, input: temporalio.worker.WorkflowInterceptorClassInput ) -> type[temporalio.worker.WorkflowInboundInterceptor] | None: - # Workflow code learns the Worker's backend from the interceptor class. + # Workflow code reads the Worker's backend from the interceptor class. return type( "_WorkflowInbound", (_WorkflowInbound,), @@ -307,14 +305,14 @@ async def handle_update_handler( def _receive(args: Sequence[Any]) -> None: # A run that receives a Stream uses Streams. The inputs, Signals and Updates are in - # History, so a replay finds the same. + # History, so a replay sets the same flag. if _stream_arguments(args): streams_workflow._state().uses_streams = True def _ends_run(err: BaseException) -> bool: - # The run's writes that nobody awaits go out before its end. Any other exception fails the - # Workflow Task, and an eviction closes the coroutine outside the Workflow's loop. + # True if err ends the run, which must send its unawaited writes first. Any other exception + # fails the Workflow Task, and an eviction closes the coroutine outside the Workflow's loop. if isinstance(err, temporalio.workflow.ContinueAsNewError): return True if isinstance(err, asyncio.CancelledError): @@ -325,7 +323,7 @@ def _ends_run(err: BaseException) -> bool: def _owned_ref(stream_id: str) -> models.StreamRef: - # A Stream that the run chain creates is the chain's own. + # The run chain owns each Stream that it creates. info = temporalio.workflow.info() return models.StreamRef( id=stream_id, @@ -363,9 +361,8 @@ def __init__( self._temporal_stream_backend_name = backend_name self._temporal_stream_default = default self._temporal_stream_kind = kind - # Numbers each Activity call of the run in order, so the key is the same on replay and - # a retry keeps it. A counter, not workflow.uuid4(), leaves the user's random values - # alone. + # Counts the run's Activity calls, so a key is the same on replay and on a retry. It + # doesn't call workflow.uuid4(), so the user's random values don't change. self._activity_calls = 0 def _activity_headers(self) -> dict[str, temporalio.api.common.v1.Payload]: @@ -393,8 +390,8 @@ def continue_as_new(self, input: temporalio.worker.ContinueAsNewInput) -> NoRetu def start_activity( self, input: temporalio.worker.StartActivityInput ) -> temporalio.workflow.ActivityHandle[Any]: - # The key goes into History, so only a run that uses Streams adds it, or a run whose - # Activity gets a Stream. Each call still counts, so a key is the same on replay. + # The key is in History, so add it only for a run that uses Streams or passes one to + # this Activity. Count every call, so the keys are the same on replay. headers = self._activity_headers() state = streams_workflow._state() if not state.uses_streams and _stream_arguments(input.args): @@ -406,7 +403,7 @@ def start_activity( def start_local_activity( self, input: temporalio.worker.StartLocalActivityInput ) -> temporalio.workflow.ActivityHandle[Any]: - # A Local Activity's headers stay out of History, so each one gets its key. + # A Local Activity's headers aren't in History, so every Local Activity gets the key. input.headers = {**input.headers, **self._activity_headers()} return super().start_local_activity(input) @@ -427,8 +424,8 @@ async def execute_activity( token = _activity.current.set(streams) try: if temporalio.activity.info().attempt > 1: - # The open of a new attempt withdraws the Records of the earlier ones, also - # when this attempt publishes nothing. It creates no Stream. + # Withdraw the Records of earlier attempts, even if this attempt publishes + # nothing. This creates no Stream. stream_ids = [handle.id for handle in _stream_arguments(input.args)] for stream_id in dict.fromkeys(stream_ids): await streams.writer(stream_id).withdraw_earlier_attempts() @@ -444,8 +441,8 @@ async def execute_activity( def _stream_arguments(args: Sequence[Any]) -> list[StreamHandle[Any]]: - # The Streams that an Activity received, which an earlier attempt may have written. They - # may sit in fields of its input object, in slots, and in lists and dicts. + # Find the Stream handles in call arguments. A handle can be in a field, a slot, a list or + # a dict. handles: list[StreamHandle[Any]] = [] seen: set[int] = set() @@ -506,8 +503,8 @@ def _activity_streams( if info.is_local: task = info.workflow_task if task is not None: - # Always set, with 0 for none completed. An unset one turns off the withdrawal of - # the run's orphaned Local Activity Records. + # Always set it, with 0 if no task completed. If it's unset, the Store doesn't + # withdraw the run's orphaned Local Activity Records. previous = stream_pb.WorkflowTaskEpoch( started_event_id=task.previous_started_event_id ) diff --git a/temporalio/common/_streams/_handle.py b/temporalio/common/_streams/_handle.py index 2e8f4ef2de..f657dcb6d1 100644 --- a/temporalio/common/_streams/_handle.py +++ b/temporalio/common/_streams/_handle.py @@ -23,7 +23,7 @@ def exit_failure(exc: BaseException | None) -> tuple[bool, BaseException | None] """Whether the end of an ``async with`` block closes the Stream, and with which failure. A cancel closes it with a ``CancelledError`` failure. Another ``BaseException``, such as - ``KeyboardInterrupt``, leaves it open, since the block did not end in the caller's code. + ``KeyboardInterrupt``, leaves it open, since the caller's code didn't end the block. """ if isinstance(exc, asyncio.CancelledError): return True, temporalio.exceptions.CancelledError(str(exc) or "Cancelled") @@ -45,26 +45,26 @@ async def __aexit__( class Creating(Generic[S]): - """A Stream create. Await it for the handle, or use it in ``async with``, which closes the - Stream at the end of the block. + """A pending Stream create. - The create starts when it is awaited or entered, not before. + Await it to get the handle, or use it in ``async with``, which closes the Stream at the end + of the block. The create starts when it's awaited or entered, not before. .. warning:: This API is experimental. """ def __init__(self, create: Callable[[], Coroutine[Any, Any, S]]) -> None: - """Create it. Use a context's create call instead.""" + """Create the object. Use a context's create function instead.""" self._create = create self._stream: S | None = None def __await__(self) -> Generator[Any, None, S]: - """Create the Stream and give its handle.""" + """Create the Stream and return its handle.""" return self._create().__await__() async def __aenter__(self) -> S: - """Create the Stream and give its handle for the block.""" + """Create the Stream and return its handle for the block.""" self._stream = await self._create() return self._stream @@ -80,7 +80,7 @@ async def __aexit__( class TopicBase(Generic[T]): - """Base of the Topic of each handle class. All Topics of a Stream share one order.""" + """Base class of the Topic of each handle class. All Topics of a Stream share one order.""" def __init__(self, stream: Any, name: str, type: Any) -> None: """Create a Topic. Use the Stream's ``topic`` instead.""" @@ -147,8 +147,8 @@ def _from_json(value: Any, type_hint: Any) -> StreamHandle[Any]: class _StreamHandleJson( temporalio.converter.TransferTypeConverter["StreamHandle[Any]", dict[str, str]] ): - # One form for every handle type, and the parameter's type hint picks the class, so a - # Workflow passes a Stream to an Activity as data. + # Every handle type has the same JSON form, and the parameter's type hint picks the class. + # So a Workflow can pass a Stream to an Activity as data. transfer_type: type[dict[str, str]] | None = dict def to_transfer_type(self, value: StreamHandle[Any]) -> dict[str, str]: @@ -161,11 +161,11 @@ def from_transfer_type( class StreamHandle(Generic[E]): - """Base of the Stream handle of each context. + """Base class of the Stream handle of each context. - A handle is data: it passes as a Workflow or Activity argument, as the proto JSON of the - ``StreamRef`` message. The parameter's type, such as ``ActivityStream[Events]``, gives the - handle class and the definition. + A handle is data. It's passed as a Workflow or Activity argument in the proto JSON form of + the ``StreamRef`` message. The parameter's type, such as ``ActivityStream[Events]``, sets + the handle class and the definition. .. warning:: This API is experimental. @@ -180,7 +180,7 @@ def __init__( backend: str | None = None, definition: type[E] | None = None, ) -> None: - """Create a handle. Use the create and get calls of a context instead.""" + """Create a handle. Use a context's create and get functions instead.""" self._id = id self._workflow_id = workflow_id self._first_execution_run_id = first_execution_run_id @@ -191,11 +191,10 @@ def __init__( @classmethod def from_handle(cls: type[H], handle: StreamHandle[Any]) -> H: - """The same Stream as a handle of this class, such as an ``ActivityStream`` from a - ``WorkflowStream``. + """Return the same Stream as a handle of this class. - A handle passes as data, so a Workflow uses it to give an Activity's input the type - that the Activity declares. + For example, a Workflow converts a ``WorkflowStream`` to an ``ActivityStream`` for an + Activity's input, so the input has the type that the Activity declares. """ return cls( id=handle.id, @@ -222,7 +221,7 @@ def first_execution_run_id(self) -> str | None: @property def backend(self) -> str | None: - """Name of the backend that keeps the Stream. None means the default one.""" + """Name of the backend that stores the Stream. None means the default one.""" return self._backend @property @@ -234,7 +233,7 @@ def _topic_spec( self, topic: str | TopicDefinition[Any], type: Any ) -> tuple[str, Any]: # The name and the decode type of a Topic. A Topic of a definition keeps its type, - # and a name outside the definition needs a type, so a typo does not pass. + # and a name outside the definition needs a type, so a typo raises an error. definition = definition_of(self._definition) if self._definition else None if isinstance(topic, TopicDefinition): if type is not None: @@ -292,7 +291,7 @@ def __eq__(self, other: object) -> bool: return self._fields() == other._fields() def __hash__(self) -> int: - """Hash of the fields that never change, so it stays when a client learns the id.""" + """Hash of the fields that never change, so it's the same after a client learns the id.""" return hash((type(self), self._backend, self._workflow_id or self._id)) def __repr__(self) -> str: @@ -305,7 +304,7 @@ def __copy__(self) -> Self: return self def __deepcopy__(self, memo: dict[int, Any]) -> Self: - """A deep copy is the same handle, since dataclasses.asdict copies its fields.""" + """Return the same handle, since dataclasses.asdict deep-copies a handle field.""" return self @classmethod diff --git a/temporalio/common/_streams/_items.py b/temporalio/common/_streams/_items.py index 3b60e39a34..94b9388e34 100644 --- a/temporalio/common/_streams/_items.py +++ b/temporalio/common/_streams/_items.py @@ -15,7 +15,7 @@ class End(enum.Enum): - """An ``offset`` after the newest Record, for new Records only. Use ``END``.""" + """An ``offset`` past the newest Record, to read only new Records. Use ``END``.""" END = "END" @@ -38,7 +38,7 @@ class Record(Generic[T]): """Value of the Record, decoded with the Topic's type.""" offset: int - """Place of the Record in the Stream. Ordered and unique, with gaps.""" + """Position of the Record in the Stream. Offsets are ordered and unique, with gaps.""" producer: models.Producer | None """Producer that wrote the Record.""" @@ -46,11 +46,11 @@ class Record(Generic[T]): @dataclass(frozen=True) class Superseded: - """A failed attempt's Records are withdrawn. + """An item that withdraws the Records of a failed attempt. - It comes at the same point for every Reader. A live view takes back the Records in - :py:attr:`withdrawn_offsets`. A view that keeps Records from an earlier read, such as one - before a reconnect, uses :py:meth:`withdraws` on them. + Every reader gets it at the same point in the Stream. A live view removes the Records in + :py:attr:`withdrawn_offsets`. A view that keeps Records from an earlier read, such as from + before a reconnect, checks them with :py:meth:`withdraws`. The next write of a run to a Stream withdraws the Records that the run's Local Activities wrote there in Workflow Tasks that never completed. A Stream that the run never writes @@ -61,7 +61,7 @@ class Superseded: """ offset: int - """Place of the item in the Stream.""" + """Position of the item in the Stream.""" supersession: models.Supersession """Which earlier Records the item withdraws.""" @@ -69,7 +69,7 @@ class Superseded: withdrawn_offsets: tuple[int, ...] = () """The offsets of the Records that this read gave and this item withdraws, in order. - A read gives each offset once. Records from before the read's start are not in it. + A read gives each offset once. It doesn't include Records from before the read's start. """ def withdraws(self, record: Record[Any]) -> bool: @@ -116,17 +116,19 @@ class Closed: """ offset: int - """Place of the close in the Stream.""" + """Position of the close in the Stream.""" result: Any | None - """Result that the close carries, decoded with no type. None if it has none.""" + """Result of the close, decoded with no type hint. None if it has none.""" failure: BaseException | None - """Failure that the close carries. None if it has none.""" + """Failure of the close. None if it has none.""" status: CloseStatus = CloseStatus.COMPLETED - """How the Stream closed, so a reader tells a cancelled or terminated owner from a failed - one with no check of the failure's type. + """How the Stream closed. + + A reader can tell a cancelled or terminated owner from a failed one without checking the + failure's type. """ @@ -159,9 +161,7 @@ def _completed(task: models.WorkflowTaskEpoch) -> tuple[int, int, tuple[int, int def producer_key(producer: models.Producer) -> tuple[Any, ...]: - """The fields of a Producer that :py:func:`withdraws` looks at, so equal keys are withdrawn - together. - """ + """The Producer fields that :py:func:`withdraws` reads, so equal keys are withdrawn together.""" epoch = _epoch(producer.epoch) if producer.epoch is not None else None return (producer.kind or 0, producer.id, producer.run_id or "", epoch) @@ -179,7 +179,7 @@ def withdraws( ) -> bool: """Whether a Superseded item withdraws a Record of ``producer``. - It is the same rule as the crate's, so every SDK withdraws the same Records. + The rule matches the crate's, so every SDK withdraws the same Records. """ if producer is None: return False @@ -214,6 +214,6 @@ def withdraws( return False if (completed.attempt or 0) > 0: return old < _completed(completed) - # Core gives the completed task with attempt 0 when it no longer has the attempt. The - # attempts of one task start in time order, so the started time still tells the lower ones. + # Core sends the completed task with attempt 0 when it no longer knows the attempt. A task's + # attempts start in time order, so the started time still orders them. return old[2] < _time(completed.started_time) diff --git a/temporalio/common/_streams/_plugin.py b/temporalio/common/_streams/_plugin.py index cf7b378da9..d3c9fcc17f 100644 --- a/temporalio/common/_streams/_plugin.py +++ b/temporalio/common/_streams/_plugin.py @@ -18,14 +18,15 @@ class StreamBackend(SimplePlugin): - """Base of the Stream backends. Each registers one backend under its name. + """Base class of the Stream backends. Each instance registers one backend under its name. Register backends on the client with ``Client.connect(..., stream_backends=[...])``. A - Worker takes its client's backends, and a ``Replayer`` takes ``stream_backends=`` too. - Names are unique. With more than one registered backend, exactly one has ``default=True``. - With none marked, the default is ``native``, which this SDK does not have, so a create with - no backend raises ``StreamConfigError``. Mark a backend with ``default=True``, or use the - calls of its package, such as ``temporalio.contrib.stream.workflow.create_stream``. + Worker uses its client's backends. A ``Replayer`` takes ``stream_backends=`` too. Names + must be unique. With more than one registered backend, exactly one must have + ``default=True``. With no backend marked, the default is ``native``. This SDK has no + ``native`` backend, so a create with no backend raises ``StreamConfigError``. Mark a + backend with ``default=True``, or use the functions of its package, such as + ``temporalio.contrib.stream.workflow.create_stream``. A backend is not a plugin, so passing it in ``plugins=`` raises ``StreamConfigError``. @@ -33,8 +34,8 @@ class StreamBackend(SimplePlugin): This API is experimental. """ - # The kind of Store, such as "redis", so the calls of a backend's package find the - # registered backend of their kind. + # The kind of Store, such as "redis". The functions of a backend's package use it to find + # the registered backend of their kind. _kind = "" def __init__( @@ -52,7 +53,7 @@ def __init__( @property def backend_name(self) -> str: - """The name that Stream handles carry for this backend.""" + """The backend name that Stream handles store.""" return self._backend_name @property @@ -64,7 +65,7 @@ def _backend_for( self, client: temporalio.client.Client, # pyright: ignore[reportUnusedParameter] ) -> BackendCalls: - # The calls of this backend for the client and its Workers. Each backend gives its own. + # Each backend subclass returns its calls for the client and its Workers. raise NotImplementedError(f"{type(self).__name__} gives no calls") async def connect_service_client( @@ -96,8 +97,8 @@ def configure_replayer( """Refuse a backend in ``plugins=``.""" raise _not_a_plugin(self) - # The registration calls these, so the plugin hooks above only see a backend that went - # into plugins= by mistake. + # Stream registration calls these, so the public hooks above only run for a backend that + # was passed in plugins= by mistake. def _configure_client( self, config: temporalio.client.ClientConfig @@ -125,13 +126,13 @@ def _not_a_plugin(backend: StreamBackend) -> _errors.StreamConfigError: def check_backends(backends: Sequence[object], owner: str) -> list[StreamBackend]: """Check the registered backends, and return them. - The default backend is part of Workflow code, since a replayed create uses it. So the - default must not follow the order of the list. + A replayed create uses the default backend, so the default must not depend on the order + of the list. Raises: TypeError: An entry is not a ``StreamBackend``. - StreamConfigError: Two backends have one name, or more than one backend and not - exactly one with ``default=True``. + StreamConfigError: Two backends have the same name, or there is more than one backend + and not exactly one has ``default=True``. """ checked: list[StreamBackend] = [] for backend in backends: @@ -156,14 +157,14 @@ def check_backends(backends: Sequence[object], owner: str) -> list[StreamBackend def default_name(backends: Sequence[StreamBackend]) -> str: - """The name of the default backend: the one with ``default=True``, else ``native``.""" + """The name of the backend with ``default=True``, or ``native`` if none has it.""" return next((b.backend_name for b in backends if b.default), NATIVE) def configure_client( config: temporalio.client.ClientConfig, ) -> temporalio.client.ClientConfig: - """Check the client's backends, and add the parts of each to the client.""" + """Check the client's backends, and apply the plugin configuration of each one.""" backends = check_backends(config.get("stream_backends", ()), "client") config["stream_backends"] = backends for backend in backends: @@ -174,7 +175,7 @@ def configure_client( def configure_worker( backends: Sequence[StreamBackend], config: temporalio.worker.WorkerConfig ) -> temporalio.worker.WorkerConfig: - """Add the parts of the client's backends to a Worker.""" + """Apply the plugin configuration of the client's backends to a Worker.""" for backend in backends: config = backend._configure_worker(config) return config @@ -183,7 +184,7 @@ def configure_worker( def configure_replayer( config: temporalio.worker.ReplayerConfig, ) -> temporalio.worker.ReplayerConfig: - """Check the Replayer's backends, and add the parts of each to it.""" + """Check the Replayer's backends, and apply the plugin configuration of each one.""" backends = check_backends(config.get("stream_backends", ()), "Replayer") config["stream_backends"] = backends for backend in backends: @@ -199,7 +200,7 @@ def client_backends(client: temporalio.client.Client) -> list[StreamBackend]: def client_backend( client: temporalio.client.Client, name: str | None, *, hint: str = "backend=" ) -> tuple[str, BackendCalls]: - """The name and the calls of a client's backend. None picks the default backend. + """The name and the calls of a client's backend. A ``name`` of None picks the default. Raises: StreamConfigError: The client has no backend of that name, or none has diff --git a/temporalio/common/_streams/_reader.py b/temporalio/common/_streams/_reader.py index cf2a6606bd..7713031108 100644 --- a/temporalio/common/_streams/_reader.py +++ b/temporalio/common/_streams/_reader.py @@ -33,15 +33,15 @@ @dataclasses.dataclass class Position: - """Where a read goes on from. The read moves it past each page it finished.""" + """Where a read resumes from. The read moves it past each page that it finished.""" next_offset: int | None class Subscription(Generic[T]): - """The items of one read of a Stream, in order. Use it in ``async for``. + """The items of one read of a Stream, in order. Iterate it with ``async for``. - It gives :py:class:`Record` items, :py:class:`Superseded` items that withdraw earlier + It yields :py:class:`Record` items, :py:class:`Superseded` items that withdraw earlier Records, and a :py:class:`Closed` item last when the Stream is closed. .. warning:: @@ -59,19 +59,19 @@ def __init__( self._position = position # A client read's rpc_metadata and rpc_timeout, for each call of each step. self._options = options - # The Producer and the offsets of each Record this read gave, with no values, so a - # Superseded item can give the offsets it withdraws. + # The Producer and the offsets of each Record that this read gave, without values, so a + # Superseded item can list the offsets that it withdraws. self._shown: dict[tuple[Any, ...], tuple[models.Producer, list[int]]] = {} @property def next_offset(self) -> int | None: - """The offset to resume from, after the last item given or the last page read. + """The offset to resume from, after the last item or the last page that the read got. - After a ``Superseded`` item it is that item's own offset, since the first Record of its - batch has the same offset and comes next. A read from there gives the ``Superseded`` - item again, which withdraws nothing new. After a Record that failed to decode, it is - past that Record. Pass it as ``offset`` to a later read. It is None before the first - page of a read from ``END`` or of a read with ``last``. + Pass it as ``offset`` to a later read. After a ``Superseded`` item, it's that item's own + offset, since the first Record of its batch has the same offset and comes next. A read + from there gives the ``Superseded`` item again, which withdraws nothing new. After a + Record that failed to decode, it's past that Record. It's None before the first page of + a read from ``END`` or of a read with ``last``. """ return self._position.next_offset @@ -89,7 +89,7 @@ async def __anext__(self) -> Record[T] | Superseded | Closed: with _backend.using(self._options): item = await self._items.__anext__() if isinstance(item, Superseded): - # The Record of the same entry shares the offset and is not given yet. + # The Record of the same entry has the same offset and comes next. self._position.next_offset = item.offset else: self._position.next_offset = max( @@ -121,8 +121,8 @@ def _withdraw(self, supersession: models.Supersession) -> tuple[int, ...]: async def aclose(self) -> None: """End the read early, from the task that reads it. It makes no more calls. - A read that waits for new Records in another task is still running, so this raises - ``RuntimeError`` there. Cancel that task instead. + If another task is waiting in the read for new Records, this raises ``RuntimeError``. + Cancel that task instead. """ with _backend.using(self._options): await self._items.aclose() @@ -143,8 +143,10 @@ async def read( wait_new_records: bool, create_wait: timedelta | None, ) -> AsyncGenerator[Item, None]: - """Give the items of a Stream in order, until a close, or the newest Record when - ``wait_new_records`` is False. With ``last``, start with the newest ``last`` Records. + """Yield the items of a Stream in order, until a close. + + With ``wait_new_records`` False, stop at the newest Record. With ``last``, start with the + newest ``last`` Records. """ loop = asyncio.get_running_loop() create_until = loop.time() + create_wait.total_seconds() if create_wait else None @@ -157,7 +159,7 @@ def request() -> models.ReadRecordsRequest: ) return models.ReadRecordsRequest( stream=models.StreamRef(id=stream_id), - # END is only the first page's start. Each answer's next offset pins later pages. + # END applies only to the first page. Later pages start at the last page's next offset. from_offset=from_offset if isinstance(from_offset, int) else None, start=ReadStart.READ_START_END if from_offset is End.END @@ -180,8 +182,8 @@ def request() -> models.ReadRecordsRequest: try: item = await _item(converter, record, types, default_type) except Exception as err: - # A resume from next_offset goes on past the Record, so it can't stop - # every later read. The error names its offset for a read of it. + # Move next_offset past the Record, so a resume doesn't fail on it again. + # The error gives the offset, for a read of that Record. offset = record.offset or 0 position.next_offset = max( position.next_offset or 0, offset + 1 @@ -213,10 +215,10 @@ async def _pages( request: models.ReadRecordsRequest, wait_new_records: bool, ) -> AsyncGenerator[models.ReadRecordsResponse, None]: - # A read that waits long-polls each page. A read with no wait reads pages until one has no - # Record and moves no further. + # A read that waits long-polls each page. A read with no wait reads pages until a page has + # no Record and doesn't advance. if request.last: - # The newest Records come in one page that never waits. A read that follows goes on + # The newest Records come in one page that never waits. A waiting read then continues # after them. page = await service.read_records(request) yield page @@ -235,8 +237,8 @@ async def _pages( return if not wait_new_records and ( page.next_offset is None - # A Store scans a limited number of entries per call, so an empty page that moved - # on stopped in Records that a Topic filter left out, not at the newest Record. + # A Store scans a limited number of entries per call. So an empty page that advanced + # stopped in Records that a Topic filter skipped, not at the newest Record. or (not page.records and page.next_offset <= (request.from_offset or 0)) ): return @@ -263,7 +265,7 @@ async def _item( if record.kind == StreamRecordKind.STREAM_RECORD_KIND_CLOSE: result = _convert.decode(converter, record.close_result, None) failure = record.close_failure - # A close Record from before close statuses has none, and its failure tells. + # An older close Record has no status, so its failure decides the status. status = record.close_status or ( CloseStatus.FAILED if failure is not None else CloseStatus.COMPLETED ) diff --git a/temporalio/common/_streams/_stream_client.py b/temporalio/common/_streams/_stream_client.py index 53817ac3f6..49a808053c 100644 --- a/temporalio/common/_streams/_stream_client.py +++ b/temporalio/common/_streams/_stream_client.py @@ -34,9 +34,8 @@ E = TypeVar("E") T = TypeVar("T") -# The handles of one client and Stream write as one Producer, so the crate shares one writer -# between them. Each client gets its own key, since two clients may have two runtime objects, -# and two writers of one Producer would send clashing sequences. +# The handles of one client and Stream write as one Producer. Each client gets its own key, +# since two clients can have two runtime objects, whose writers would send clashing sequences. _PROCESS_KEY = str(uuid.uuid4()) _CLIENTS = itertools.count(1) _CLIENT_KEYS: weakref.WeakKeyDictionary[temporalio.client.Client, str] = ( @@ -88,16 +87,16 @@ async def publish( class ClientStream(StreamHandle[E]): - """A Stream that a client writes, reads and closes. + """A Stream in client code. The client can write and read it, and close a client's Stream. Get one from :py:meth:`temporalio.client.Client.stream`, - :py:meth:`temporalio.client.WorkflowHandle.stream` or a backend's create, such as - :py:func:`temporalio.contrib.stream.create`. + :py:meth:`temporalio.client.WorkflowHandle.stream` or a backend's ``create`` function, such + as :py:func:`temporalio.contrib.stream.create`. - Each publish writes before it returns. Use :py:class:`BufferedPublisher` to batch. The - handles of one client and Stream write as one Producer, and :py:meth:`producer` gives a - handle that writes as a named one. Use it in ``async with`` to close it at the end of the - block. + Each publish writes before it returns. Use :py:class:`BufferedPublisher` to batch. All + handles of one client and Stream write as one Producer. :py:meth:`producer` returns a + handle that writes as a named Producer. Use a handle in ``async with`` to close the Stream + at the end of the block. .. warning:: This API is experimental. @@ -158,8 +157,8 @@ def id(self) -> str: async def _stream_id(self) -> str: if not self._id: - # Calls that start together share one Describe call. A failed call does not - # stay, so a later call tries again. + # Concurrent calls share one Describe call. A failed Describe is discarded, so a + # later call tries again. if self._finding is None or _failed(self._finding): assert self._find_first_run is not None self._finding = asyncio.ensure_future(self._find_first_run()) @@ -206,13 +205,13 @@ def writer() -> StreamWriter: ) def producer(self, name: str, epoch: int) -> ClientStream[E]: - """This Stream as a handle that writes as the Producer ``name`` at ``epoch``. + """Return this Stream as a handle that writes as the Producer ``name`` at ``epoch``. - It is for a writer that must fence its earlier self, such as a server that restarts. - Keep ``epoch`` in durable storage, and raise it at each start. After a write at a - higher epoch, a write at a lower one raises :py:class:`StreamFencedError`. The Records - of the earlier epochs stay, since they are real messages. Run one live writer for a - name. + Use it for a writer that must fence its earlier instances, such as a server that + restarts. Keep ``epoch`` in durable storage, and raise it at each start. After a write + at a higher epoch, a write at a lower one raises :py:class:`StreamFencedError`. The + Records of the earlier epochs aren't withdrawn, since they're real messages. Run only + one live writer for each name. Args: name: The Producer name, the same across restarts. @@ -249,7 +248,7 @@ def topic( Args: topic: A Topic of the Stream's definition, such as ``Events.turns``, or a name. - A name of the definition takes its type. + A name in the definition gets the definition's type. type: The type of a Topic outside the definition, or of a Stream with no definition. """ @@ -271,29 +270,28 @@ def subscribe( """Read the Stream's items in order. The read ends after a ``Closed`` item, or at the newest Record when - ``wait_new_records`` is False. A reader that wants to stop after a quiet time sets its - own timer, and resumes later with ``offset=`` the read's ``next_offset``. A - ``Superseded`` item withdraws earlier Records of a failed attempt, and it passes every - Topic filter. + ``wait_new_records`` is False. To stop after a quiet period, use your own timer, and + resume later with ``offset=`` set to the read's ``next_offset``. A ``Superseded`` item + withdraws earlier Records of a failed attempt. Topic filters never skip it. Args: topics: Read these Topics, by Topic of the definition, name or Topic object. With no Topics and no ``topic_prefixes``, read every Topic. offset: Start at this offset, inclusive, or at ``common.END`` for new Records only. Defaults to the first Record. - last: Start with the newest this many Records that pass the filter, oldest - first, such as the first view of a chat. Not with ``offset``. A Record + last: Start with the newest ``last`` Records that pass the filter, oldest first, + such as for the first view of a chat. Can't be used with ``offset``. A Record that a later ``Superseded`` item withdraws doesn't count. At most 1000. wait_new_records: Wait for new Records at the end of the Stream. topic_prefixes: Also read the Topics whose names start with one of these. - create_wait: How long to wait for a Stream that nobody created yet, such as the + create_wait: How long to wait for a Stream that doesn't exist yet, such as the Stream of a Workflow that just started. Defaults to no wait. type: The type of the values of Topics outside the definition, such as Topics that ``topic_prefixes`` match. Topics of the definition keep their own types. - Defaults to no type, so the values decode as the converter gives them. + Defaults to no type, so the converter decodes the values with no type hint. rpc_metadata: Headers of each call of the read. See :py:meth:`get_offset`. rpc_timeout: The deadline of each call of the read, not of the whole read. A call - that waits for new Records answers before it. + that waits for new Records returns before the deadline. """ read = read_spec( self, @@ -327,7 +325,9 @@ async def get_offset( rpc_metadata: Mapping[str, str | bytes] = {}, rpc_timeout: timedelta | None = None, ) -> int: - """The next offset of the Stream, the newest offset plus one. + """The next offset of the Stream, the newest offset plus one, or 0 if it has no Record. + + Offsets are sparse and above zero, so ``offset=0`` reads from the start. Args: rpc_metadata: Headers of each call. The Redis and memory backends make no RPC, so @@ -340,8 +340,8 @@ async def get_offset( def _check_close(self) -> None: if self._workflow_id: - # A Workflow's Stream closes only from its run chain. The Store refuses a client's - # close too, and this error comes with no call. + # Only the run chain can close a Workflow's Stream. The Store refuses a client's + # close too, but this check makes no call. raise _errors.StreamNotOwnerError( f"Can't close the Stream {self._id or self._stream_name} from a client, since " "a Workflow owns it", @@ -358,12 +358,12 @@ async def close( ) -> None: """Write this handle's Records, then close the Stream. - Readers get the Records before it, then a ``Closed`` item with ``result`` or - ``failure``. A cancel failure, such as a ``CancelledError``, closes it as canceled. - ``rpc_metadata`` and ``rpc_timeout`` are as for :py:meth:`get_offset`. + Readers get the Records before the close, then a ``Closed`` item with ``result`` or + ``failure``. A cancel failure, such as a ``CancelledError``, closes the Stream as + canceled. ``rpc_metadata`` and ``rpc_timeout`` are as for :py:meth:`get_offset`. Raises: - StreamNotOwnerError: A Workflow owns the Stream, so only its run chain closes it. + StreamNotOwnerError: A Workflow owns the Stream, so only its run chain can close it. """ self._check_close() self._closed = True @@ -374,9 +374,9 @@ async def __aenter__(self) -> ClientStream[E]: """Use the Stream in ``async with``, which closes it at the end. Raises: - StreamNotOwnerError: A Workflow owns the Stream, so only its run chain closes it. + StreamNotOwnerError: A Workflow owns the Stream, so only its run chain can close it. """ - # The close at the end would raise and hide the block's own error. + # Fail early, since the close at the end would raise and hide the block's own error. self._check_close() return self @@ -389,7 +389,8 @@ async def __aexit__( """Close the Stream, with the exception as its failure if there is one. A cancel closes it with a ``CancelledError`` failure. Another ``BaseException``, such as - ``KeyboardInterrupt``, leaves it open. A Stream that this handle closed stays as it is. + ``KeyboardInterrupt``, leaves it open. If this handle already closed the Stream, this + does nothing. """ if self._closed: return @@ -406,8 +407,9 @@ def create( retention: timedelta | None, backend: str, ) -> Creating[ClientStream[E]]: - """Create a Stream that this client owns on a registered backend. See - :py:func:`temporalio.contrib.stream.create`. + """Create a Stream that the client owns, on a registered backend. + + See :py:func:`temporalio.contrib.stream.create`. """ async def create() -> ClientStream[E]: @@ -439,8 +441,9 @@ def client_stream( stream: StreamHandle[Any] | None, backend: str | None, ) -> ClientStream[E]: - """A Stream by its id, or a handle that came as data. See - :py:meth:`temporalio.client.Client.stream`. + """Get a Stream by its id, or from a handle that came in data. + + See :py:meth:`temporalio.client.Client.stream`. """ definition_of(definition) if (id is None) == (stream is None): @@ -453,7 +456,7 @@ def client_stream( backend = backend or stream.backend stream_id = stream.id if stream else id if not stream_id: - # An empty id would fail only at the first call, with no cause in it. + # An empty id would fail only at the first call, with an unclear error. raise ValueError("The Stream id is empty") backend_name, calls = client_backend(client, backend) handle = ClientStream( @@ -466,8 +469,8 @@ def client_stream( return handle._bind(client, calls) -# The first run id of a Workflow handle, or the Describe call that looks for it, so the -# Streams of one handle share the call. +# The first run id of a Workflow handle, or the Describe call that fetches it, so the Streams +# of one handle share the call. _FIRST_RUNS: weakref.WeakKeyDictionary[ temporalio.client.WorkflowHandle[Any, Any], asyncio.Future[str] | str ] = weakref.WeakKeyDictionary() @@ -480,7 +483,7 @@ def _failed(future: asyncio.Future[Any]) -> bool: async def _describe_first_run( handle: temporalio.client.WorkflowHandle[Any, Any], ) -> str: - # The first call that needs the id gives its own call options. + # The Describe uses the call options of the first call that needs the id. options = _backend.call_options.get() description = await handle.describe( rpc_metadata=options.metadata if options else {}, @@ -507,8 +510,8 @@ def _settle( handle_ref: weakref.ref[temporalio.client.WorkflowHandle[Any, Any]], finding: asyncio.Future[str], ) -> None: - # A found id stays as text. A failed call goes, so a later call tries again, and the - # failure's traceback does not keep the handle in the map. + # Store a found id as text. Remove a failed call, so a later call tries again and the + # failure's traceback doesn't keep the handle alive. handle = handle_ref() if handle is None or _FIRST_RUNS.get(handle) is not finding: return @@ -524,8 +527,9 @@ def workflow_stream( *, backend: str | None, ) -> ClientStream[E]: - """A Stream that a Workflow created. See - :py:meth:`temporalio.client.WorkflowHandle.stream`. + """Get a Stream that a Workflow created. + + See :py:meth:`temporalio.client.WorkflowHandle.stream`. """ stream_name = definition_of(definition).name client = handle._client diff --git a/temporalio/common/_streams/_workflow.py b/temporalio/common/_streams/_workflow.py index fae2f06400..1cd655ab02 100644 --- a/temporalio/common/_streams/_workflow.py +++ b/temporalio/common/_streams/_workflow.py @@ -1,16 +1,16 @@ """Streams in Workflow code. -Workflow code creates a Stream with :py:func:`temporalio.workflow.create_stream` and gets one +Workflow code creates a Stream with :py:func:`temporalio.workflow.create_stream` and gets it again with :py:func:`temporalio.workflow.stream_handle`. A Stream that the Workflow receives as -an argument, such as a :py:class:`WorkflowStream` that a run passes on through continue-as-new, -works the same. - -Each create and publish is a Local Activity, and each close is a command that the backend -carries out once the Workflow Task commits, a regular Activity on Redis. So each has a place in -the run's commands. To add, remove or move one in a Workflow that has open runs, gate the change -with ``workflow.patched``, else the replay of those runs fails as nondeterministic. Publishes -that wait behind a write to the same Stream go as one Local Activity. A task that fails, or a -crash, before the close commits leaves the Stream open for the task's retry. +an argument works the same, such as a :py:class:`WorkflowStream` that a run passes on through +continue-as-new. + +Each create and publish is a Local Activity. Each close is a command that the backend runs after +the Workflow Task commits, and on Redis it's a regular Activity. So each one has a place in the +run's commands. To add, remove or move one in a Workflow that has open runs, gate the change +with ``workflow.patched``, or the replay of those runs fails as nondeterministic. Publishes that +wait behind a write to the same Stream are sent as one Local Activity. If a task fails or the +Worker crashes before the close commits, the Stream stays open for the task's retry. """ from __future__ import annotations @@ -41,23 +41,21 @@ E = TypeVar("E") T = TypeVar("T") -# The backend's Activities that write for Workflow code. The create and the append are Local -# Activities, whose marker keeps the answer only, since a Local Activity's input is not in -# History. The Redis close is a regular Activity on the Workflow's task queue. +# The create and the append are Local Activities, so History has only their results. The Redis +# close is a regular Activity on the Workflow's task queue. CREATE_ACTIVITY = "__temporal_streams_create" APPEND_ACTIVITY = "__temporal_streams_append" CLOSE_ACTIVITY = "__temporal_streams_close" _WRITE_TIMEOUT = timedelta(seconds=30) -# The most that one merged append takes before the codec. The Local Activity checks the size -# after the codec, and sends each call as its own batch when the codec grew the merge past the -# Store's 4 MiB batch limit. Half of it leaves room for most codecs. +# Limits of one merged append before the codec. Half of the Store's 4 MiB batch limit leaves room +# for most codecs. The Local Activity splits a merge by call if the codec grows it past the limit. _MERGE_RECORDS = 1000 _MERGE_BYTES = 2 * 1024 * 1024 @dataclass class _RunStream: - # What this run knows about one Stream. Many handles may name it, so it is kept by id. + # What this run knows about one Stream. Many handles can name it, so the state is keyed by id. ref: models.StreamRef closed: bool = False # Set for a Stream that the run chain created, which continue-as-new passes on. @@ -66,8 +64,8 @@ class _RunStream: @dataclass class _Write: - # One Local Activity that writes to a Stream, and the calls it answers. Each call gets the - # Records up to its own last one, and its future. + # One Local Activity that writes to a Stream, and the calls that it answers. Each call has the + # Record count up to its own last Record, and its future. activity: str request: google.protobuf.message.Message calls: list[tuple[int, asyncio.Future[int]]] @@ -75,8 +73,8 @@ class _Write: started: bool = False def take(self, request: google.protobuf.message.Message) -> bool: - # An append that waits for the write before it takes the appends that come after it, - # so a burst of awaits costs two markers and not one for each await. + # An append that waits behind the previous write merges the appends that come after it. + # So a burst of awaits costs two markers, not one for each await. if self.started or not isinstance(request, stream_pb.AppendRecordsRequest): return False mine = self.request @@ -99,14 +97,14 @@ class _RunState: streams: dict[str, _RunStream] = field(default_factory=dict) # The last write of each Stream. Each write waits for it, so writes keep their order. tails: dict[str, asyncio.Task[None]] = field(default_factory=dict) - # The last write of each Stream that has not started, which later appends may join. + # The last write of each Stream that hasn't started, which later appends can join. waiting: dict[str, _Write] = field(default_factory=dict) - # The Streams that a retry run's first write has withdrawn the previous run on. + # The Streams on which a retry run's first write withdrew the previous run's Records. withdrawn: set[str] = field(default_factory=set) - # Whether the run created, bound or received a Stream. Only then do its Activities carry - # a Producer key, so a run with no Stream adds nothing to History. + # Whether the run created, bound or received a Stream. Only then do its Activities get a + # Producer key, so a run with no Stream adds nothing to History. uses_streams: bool = False - # The block ends whose close waits for the run's outcome, with the block's failure. + # The async with blocks whose close waits for the run's outcome, with each block's failure. deferred_closes: list[tuple[WorkflowStream[Any], BaseException | None]] = field( default_factory=list ) @@ -120,8 +118,8 @@ def stream(self, ref: models.StreamRef) -> _RunStream: def _state() -> _RunState: - # The state is an attribute of the run, not a weak map entry, since its tasks refer to the - # run's loop and would keep a map entry alive. + # Keep the state on the run, since its tasks refer to the run's loop and would keep a weak + # map entry alive. runtime = temporalio.workflow._Runtime.current() state = getattr(runtime, "_temporal_streams_state", None) if state is None: @@ -161,7 +159,7 @@ def worker_backend() -> WorkerBackend | None: async def _close_in_workflow(request: stream_pb.CloseStreamRequest) -> int: - # The Worker's backend carries out the close once the Workflow Task commits. + # The Worker's backend runs the close after the Workflow Task commits. outbound = _transport() if outbound is None: raise no_worker_backend() @@ -177,8 +175,8 @@ def no_worker_backend() -> _errors.StreamConfigError: def _check_backend(backend: str | None, action: str) -> str: - # The Worker's config gives these errors, not the history, so they fail the Workflow Task - # and a Worker with the right config retries it. An ApplicationError would fail the run. + # These errors come from the Worker's config, not from History. So they fail the Workflow + # Task, and a Worker with the right config retries it. worker = worker_backend() if worker is None: raise no_worker_backend() @@ -242,7 +240,7 @@ async def run() -> None: del state.waiting[stream_id] try: if isinstance(write.request, stream_pb.AppendRecordsRequest): - # The Local Activity answers each merged call with its last offset or its error. + # The Local Activity returns each merged call's last offset or error. counts = [ count - before for (before, _), (count, _) in zip( @@ -331,26 +329,26 @@ class WorkflowTopic(TopicBase[T]): async def publish(self, *values: T) -> int: """Write one Record for each value, in order, and return the last Record's offset. - The write is a Local Activity, so History has its answer and not the values. Publishes - that wait behind a write to the same Stream go as one Local Activity, and each call - gets the offset of its own last Record. A list is one value. Writes to one Stream keep - the order of the calls. + The write is a Local Activity, so History has its result and not the values. Publishes + that wait behind a write to the same Stream are sent as one Local Activity, and each + call gets the offset of its own last Record. A list is one value. Writes to one Stream + keep the order of the calls. Raises: StreamError: The backend refused the write, such as ``StreamClosedError`` for a - Stream that something else closed. + Stream that another caller closed. StreamClosedError: This run closed the Stream. ReadOnlyContextError: The code must not change state, such as a query handler. - StreamConfigError: No Stream backend is on the Worker, or the Stream is of a - backend that the Worker does not have. It fails the Workflow Task, not the run. + StreamConfigError: The Worker has no Stream backend, or not the Stream's backend. + It fails the Workflow Task, not the run. """ return await self._stream._append(self._name, values) def publish_nowait(self, *values: T) -> None: - """Write one Record for each value, in order, with no await. + """Write one Record for each value, in order, without an await. - It is for sync Workflow code, such as ``@workflow.init`` and sync signal and update - handlers, as ``asyncio.Queue.put_nowait`` is for code that can't await ``put``. A + Use it in sync Workflow code, such as ``@workflow.init`` and sync signal and update + handlers. It's like ``asyncio.Queue.put_nowait`` for code that can't await ``put``. A backend error fails the Workflow Task. """ _write_nowait(self._stream._append(self._name, values)) @@ -360,9 +358,9 @@ class WorkflowStream(StreamHandle[E]): """A Stream in Workflow code. Workflow code writes and closes it, and can't read it. Pass it to an Activity, a child Workflow or the next run of continue-as-new as an - argument. Use it in ``async with`` to close it at the end of the block. A run end closes - only a Stream whose block end waited for it, so a Stream that the run chain never closes - stays open until its ``safety_ttl``. + argument. Use it in ``async with`` to close it at the end of the block. The end of a run + closes only the Streams whose ``async with`` block deferred the close to the run end. A + Stream that the run chain never closes stays open until its ``safety_ttl``. .. warning:: This API is experimental. @@ -398,7 +396,7 @@ def topic( Args: topic: A Topic of the Stream's definition, such as ``Events.turns``, or a name. - A name of the definition takes its type. + A name in the definition gets the definition's type. type: The type of a Topic outside the definition, or of a Stream with no definition. @@ -444,14 +442,16 @@ async def close( ) -> None: """Close the Stream after the writes before it, once the Workflow Task commits. - A task that fails before then, or a crash, leaves the Stream open, and the task's retry - writes to it again. On Redis the close is a regular Activity on the Workflow's task - queue, which closes as the run chain, the Stream's owner. So every Worker on that queue - registers the Redis backend, and one that polls Activities runs the close, since a - Worker with ``no_remote_activities=True`` runs none. Readers get the Records before it, then a ``Closed`` item - with ``result`` or ``failure``. A cancel failure, such as a ``CancelledError`` or an - ``ActivityError`` that a cancel caused, closes it as canceled. Later writes in this run - raise :py:class:`StreamClosedError`. + If the task fails first, or the Worker crashes, the Stream stays open, and the task's + retry writes to it again. On Redis the close is a regular Activity on the Workflow's + task queue, which closes the Stream as the run chain that owns it. So every Worker on + that queue must register the Redis backend. A Worker that polls Activities runs the + close, and a Worker with ``no_remote_activities=True`` doesn't. + + Readers get the Records before the close, then a ``Closed`` item with ``result`` or + ``failure``. A cancel failure, such as a ``CancelledError`` or an ``ActivityError`` + that a cancel caused, closes the Stream as canceled. Later writes in this run raise + :py:class:`StreamClosedError`. Raises: StreamNotOwnerError: This run chain didn't create the Stream. @@ -461,7 +461,7 @@ async def close( def close_nowait( self, *, result: Any | None = None, failure: BaseException | None = None ) -> None: - """Close the Stream with no await, for sync Workflow code. + """Close the Stream without an await, for sync Workflow code. It is to :py:meth:`close` what ``asyncio.Queue.put_nowait`` is to ``put``. It starts the close, and the end of the run waits for it. A backend error fails the Workflow Task. @@ -474,8 +474,8 @@ def _closing(self) -> _RunStream: info.workflow_id, info.first_execution_run_id, ): - # Only the run chain that created a Stream ends it for every reader. The backend - # refuses it too, and this error comes before a command is spent. + # Only the run chain that created a Stream can close it. The backend refuses the + # close too, but this check fails before the Workflow schedules a command. raise _errors.StreamNotOwnerError( f"Can't close the Stream {self.id}, since this run chain didn't create it", reason="stream_not_owner", @@ -508,16 +508,21 @@ async def __aexit__( ) -> None: """Close the Stream, with the exception as its failure if there is one. - A cancel of the block or of the run closes it as canceled. An exception that fails the - run with no retry run to follow closes it too, such as one in a run with no retry - policy, a non-retryable ``ApplicationError``, or one on the last attempt. The close of - any other exception waits for the run's end. The Stream then closes with that exception - when the run returns or continues as new, or fails with no retry run to follow. It - stays open when a retry run follows, since that run writes on to the same Stream, and - when the exception fails only the Workflow Task, since the task's retry runs the block - again. In a cron run the block end leaves the Stream open, since the Stream spans the - cron chain and the next run creates it again. An eviction cancels the block, and the - Worker sends no command of an evicted run. An explicit :py:meth:`close` always closes. + A cancel of the block or of the run closes the Stream as canceled. An exception that + fails the run with no retry run closes it too. Examples are an exception in a run with + no retry policy, a non-retryable ``ApplicationError``, and an exception on the last + attempt. + + For any other exception, the close waits for the run's end. The Stream then closes with + that exception if the run returns, continues as new, or fails with no retry run. It + stays open if a retry run comes next, since that run writes to the same Stream. It also + stays open if the exception fails only the Workflow Task, since the task's retry runs + the block again. + + In a cron run, the block end leaves the Stream open, since the Stream spans the cron + chain and the next run creates it again. An eviction cancels the block, and the Worker + sends no commands for an evicted run. An explicit :py:meth:`close` always closes the + Stream. """ state = _state() if state.stream(self._model_ref()).closed: @@ -534,7 +539,7 @@ async def __aexit__( def _closes_at_block_end(exc: BaseException | None) -> bool: - # Else only the run's end tells whether a retry run writes on to the Stream. + # For other exceptions, only the run's end shows whether a retry run will write to the Stream. if exc is None or isinstance(exc, asyncio.CancelledError) or _cancelled(exc): return True runtime = temporalio.workflow._Runtime.current() @@ -542,8 +547,9 @@ def _closes_at_block_end(exc: BaseException | None) -> bool: def close_at_run_end(err: BaseException | None) -> None: - """Close the Streams whose block end waited for the run's outcome, unless a retry run - follows. ``err`` is the exception that ends the run, or None when the run returns. + """Close the Streams whose block end deferred the close, unless a retry run comes next. + + ``err`` is the exception that ends the run, or None if the run returns. """ state = _state() runtime = temporalio.workflow._Runtime.current() @@ -560,7 +566,7 @@ def close_at_run_end(err: BaseException | None) -> None: def _retried(exc: BaseException) -> bool: - # Whether the server starts a retry run once the run fails with exc, by the server's rule. + # Whether the server starts a retry run after the run fails with exc, by the server's rule. info = temporalio.workflow.info() policy = info.retry_policy if policy is None or 0 < policy.maximum_attempts <= info.attempt: @@ -583,13 +589,13 @@ def _retried(exc: BaseException) -> bool: elif isinstance(exc, temporalio.exceptions.FailureError): return True else: - # The run's failure carries any other exception as the type of its class name. + # For any other exception, the run's failure has the class name as its type. failure_type = type(exc).__name__ return failure_type not in (policy.non_retryable_error_types or ()) def _cancelled(exc: BaseException) -> bool: - # A cancelled run ends as canceled with no retry, also when the cancel reached the block as + # A cancelled run ends as canceled with no retry, even when the cancel reached the block as # the error of an Activity, a Local Activity or a child. return ( temporalio.workflow.cancellation_reason() is not None @@ -618,8 +624,9 @@ def create_stream( retention: timedelta | None = None, backend: str | None = None, ) -> Creating[WorkflowStream[E]]: - """Create a Stream that this run chain owns. See - :py:func:`temporalio.workflow.create_stream`. + """Create a Stream that this run chain owns. + + See :py:func:`temporalio.workflow.create_stream`. """ async def create() -> WorkflowStream[E]: @@ -636,8 +643,9 @@ def create_stream_nowait( retention: timedelta | None = None, backend: str | None = None, ) -> WorkflowStream[E]: - """Create a Stream that this run chain owns, with no await. See - :py:func:`temporalio.workflow.create_stream_nowait`. + """Create a Stream that this run chain owns, without an await. + + See :py:func:`temporalio.workflow.create_stream_nowait`. """ stream, task = _create(definition, retention, backend) _write_nowait(task) @@ -652,8 +660,8 @@ def _create( stream = _own_handle(definition, _check_backend(backend, "create")) run_stream = stream._check("create", backend_checked=True) run_id = temporalio.workflow.info().run_id - # The owner makes the Stream the run chain's, and names a retry run's withdrawal. - # The ref names the run chain, which the Store keeps for its owner check of a close. + # A retry run's withdrawal names the owner's run id. The Store keeps the ref's run chain, and + # checks each close against it. request = stream_pb.CreateStreamRequest( stream_id=stream.id, owner=stream_pb.Producer( @@ -672,8 +680,9 @@ def _create( def stream_handle( definition: type[E], *, backend: str | None = None ) -> WorkflowStream[E]: - """A Stream that this run chain created, by its definition. See - :py:func:`temporalio.workflow.stream_handle`. + """Get a Stream that this run chain created, by its definition. + + See :py:func:`temporalio.workflow.stream_handle`. """ stream = _own_handle(definition, backend) if not temporalio.workflow.unsafe.is_read_only(): diff --git a/temporalio/common/_streams/_writer.py b/temporalio/common/_streams/_writer.py index c189d6a23f..1ee5f2726d 100644 --- a/temporalio/common/_streams/_writer.py +++ b/temporalio/common/_streams/_writer.py @@ -15,7 +15,7 @@ class StreamWriter: - """Writes the Records of one Producer on one Stream, through the generated Writer.""" + """Writes the Records of one Producer to one Stream, through the generated Writer.""" def __init__( self, @@ -27,10 +27,10 @@ def __init__( previous_completed_task: stream_pb.WorkflowTaskEpoch | None = None, failed_tasks: Sequence[int] = (), ) -> None: - """Create a writer. It opens on its first write. + """Create a writer. It opens on the first write. - ``writers`` runs the values through the user's converter and codec. ``free`` releases - the crate's writer when Python drops this one with no close. + ``writers`` encodes the values with the user's converter and codec. ``free`` releases + the crate's writer if Python frees this one without a close. """ self._writers = writers self._resolve_id = stream_id @@ -49,10 +49,10 @@ def producer(self) -> stream_pb.Producer: return self._producer async def withdraw_earlier_attempts(self) -> None: - """Open the writer of a new attempt, which withdraws the Records of the earlier ones. + """Open the writer of a new attempt, which withdraws the Records of earlier attempts. - A Stream that does not exist has nothing to withdraw. A closed or expired Stream takes - no write, so the attempt goes on, and a later write raises the real error. + A Stream that doesn't exist has nothing to withdraw. A closed or expired Stream accepts + no write, so the attempt continues, and a later write raises the real error. """ try: await self.open() @@ -64,8 +64,10 @@ async def withdraw_earlier_attempts(self) -> None: pass async def open(self) -> writer_client.Writer: - """Open the crate's writer once. The open of a new attempt withdraws the earlier one's - Records, even when it writes nothing. + """Open the crate's writer once. + + The open of a new attempt withdraws the Records of earlier attempts, even if it writes + nothing. """ from temporalio.common._streams.writer_system import models @@ -88,17 +90,15 @@ async def open(self) -> writer_client.Writer: ) ) if self._free is not None: - # The crate keeps a writer for each open that no one closed or freed, so a - # writer that Python drops frees its open. + # The crate keeps a writer for each open that nobody closed or freed. So + # free it when Python frees this writer. self._release = weakref.finalize( self, self._free, self._writer.writer_id ) return self._writer async def write(self, records: Sequence[tuple[str, Any]]) -> int: - """Write ``(topic, value)`` pairs as Records now, in order. Returns the last Record's - offset. - """ + """Write ``(topic, value)`` pairs as Records now, in order, and return the last offset.""" from temporalio.common._streams.writer_system import models if self._closed: @@ -112,7 +112,7 @@ async def write(self, records: Sequence[tuple[str, Any]]) -> int: return answer.last_offset or 0 async def close(self) -> None: - """Drop the writer. The Stream stays open.""" + """Close the writer. The Stream stays open.""" if self._closed: return self._closed = True diff --git a/temporalio/common/_streams/_writes.py b/temporalio/common/_streams/_writes.py index 057fb106f6..04e85d428b 100644 --- a/temporalio/common/_streams/_writes.py +++ b/temporalio/common/_streams/_writes.py @@ -1,8 +1,8 @@ """The Activities that write for Workflow code. -Each Workflow ``create_stream`` is a Local Activity, and the publishes that wait behind a write -to the same Stream share one. History has the answer and not the values. Each Workflow -``close`` is a regular Activity, so it runs only once its Workflow Task completed. +Each Workflow ``create_stream`` is a Local Activity. The publishes that wait behind a write to +the same Stream share one Local Activity. History has the result and not the values. Each +Workflow ``close`` is a regular Activity, so it runs only after its Workflow Task completes. """ from __future__ import annotations @@ -32,9 +32,8 @@ def _context() -> tuple[_activity.ActivityStreams, service_client.StreamServiceC def _producer(streams: _activity.ActivityStreams) -> models.Producer: - # Every attempt of one Workflow Task writes as one epoch with one sequence, so the backend - # drops the copy that a retry sends. A later task has a higher epoch, so its write - # withdraws what a failed task wrote under the same call key. + # All attempts of one Workflow Task write with one epoch and sequence, so the Store dedupes a + # retry. A later task's higher epoch withdraws a failed task's Records for the same call key. producer = streams.producer epoch = producer.epoch started = epoch.task_started_time if epoch.HasField("task_started_time") else None @@ -114,8 +113,8 @@ async def create_stream(request: stream_pb.CreateStreamRequest) -> int: @dataclasses.dataclass class _Encoded: - # The batches of one append and the calls of each, with the bodies after the codec, and - # the answer of each batch that the Store took. + # The batches of one append, with each batch's calls and encoded bodies, and the result of + # each batch that the Store accepted. batches: list[tuple[list[int], list[stream_pb.AppendRecord]]] size: int answers: dict[int, Any] = dataclasses.field(default_factory=dict) @@ -126,8 +125,8 @@ class _Encoded: class _Pending: - # The appends that have not succeeded, one for each Producer. A retry in this process sends - # the same bytes, so the Store's dedupe of a resend matches them, also with a random codec. + # The appends that haven't succeeded, one for each Producer. A retry in this process resends + # the same bytes, so the Store's dedupe matches them, even with a random codec. def __init__(self, limit: int) -> None: self._limit = limit @@ -158,7 +157,7 @@ def pop(self, key: _Key, epoch: _Epoch) -> None: self._size -= found[1].size -# A run that ends while its append retries leaves its entry, so the oldest go past this size. +# A run that ends while its append retries leaves its entry, so evict the oldest past this size. _PENDING = _Pending(64 * 1024 * 1024) @@ -188,8 +187,8 @@ async def _encode( ] if sum(sizes) <= _MAX_BATCH_BYTES and max(sizes, default=0) <= _MAX_RECORD_BYTES: return _Encoded([(list(range(len(counts))), records)], sum(sizes)) - # The codec grew the merge past the Store's limit, so each call goes as its own batch, as - # it would with no merge. A call that is too large alone fails alone. + # The codec grew the merge past the Store's limit, so send each call as its own batch, as + # with no merge. A call that's too large by itself fails by itself. batches: list[tuple[list[int], list[stream_pb.AppendRecord]]] = [] start = 0 for call, count in enumerate(counts): @@ -204,8 +203,8 @@ async def append_records( ) -> list[Any]: """Append the Records of a Workflow's merged publishes. - ``counts`` has the Records of each publish. Gives each publish the offset of its last - Record, or the error of a publish that the Store refused as too large. + ``counts`` has the Record count of each publish. Returns each publish's last offset, or the + error of a publish that the Store refused as too large. """ streams, service = _context() producer = _producer(streams) @@ -222,7 +221,7 @@ async def append_records( if encoded is None: encoded = await _encode(streams, request, counts, producer_pb) _PENDING.put(key, epoch, encoded) - # The bodies went through the codec once already. + # The codec already ran on the bodies. service = streams.backend.service( streams.namespace, dataclasses.replace(streams.converter, payload_codec=None) ) diff --git a/temporalio/contrib/stream/__init__.py b/temporalio/contrib/stream/__init__.py index aa8d5f1cc4..a1f132dee9 100644 --- a/temporalio/contrib/stream/__init__.py +++ b/temporalio/contrib/stream/__init__.py @@ -1,7 +1,7 @@ """The Redis Stream backend. -Register it on the client. :py:func:`create` makes a Stream that the client owns, and -:py:mod:`temporalio.contrib.stream.workflow` has the Workflow calls that pick this backend:: +Register it on the client. :py:func:`create` creates a Stream that the client owns. +:py:mod:`temporalio.contrib.stream.workflow` has the Workflow functions that use this backend:: from temporalio.contrib import stream as redis @@ -10,8 +10,8 @@ ) A Workflow's close is a regular Activity on the Workflow's task queue. So a Worker with a Redis -backend polls Activities on its task queue for the close Activity, and Workers that share a task -queue must all register the Redis backend. +backend must poll Activities on its task queue for the close Activity. All Workers that share a +task queue must register the Redis backend. .. warning:: This package is experimental. @@ -36,17 +36,17 @@ class RedisStreamBackend(ExtensionStreamBackend): - """Carries Stream calls to Redis, through the temporalio-streams runtime in the bridge. + """Sends Stream calls to Redis, through the temporalio-streams runtime in the bridge. - Register it on the client with ``stream_backends=``. A Worker takes its client's backends, + Register it on the client with ``stream_backends=``. A Worker uses its client's backends, so one backend serves Workflow code, Activities and the client. Each client of the backend gets its own runtime object, and the client's Workers share it. It registers the backend - name ``redis``, which the Stream handles it creates carry. For now a Worker serves one - backend. A client may have more than one, and ``backend=`` picks one. + name ``redis``, and each Stream handle that it creates stores that name. A Worker serves + only one backend. A client can have more than one, and ``backend=`` picks one. - A Workflow's close is a regular Activity on the Workflow's task queue, which runs once the - Workflow Task commits. So a Worker with this backend polls Activities on its task queue, and - Workers that share a task queue all register the backend. + A Workflow's close is a regular Activity on the Workflow's task queue, which runs after the + Workflow Task commits. So a Worker with this backend must poll Activities on its task queue, + and all Workers that share a task queue must register the backend. .. warning:: This API is experimental. @@ -66,8 +66,10 @@ def __init__( name: str = "redis", default: bool = False, ) -> None: - """Create the backend. Each client of the backend connects to Redis on its first - Stream call, on the client's runtime. + """Create the backend. + + Each client of the backend connects to Redis on its first Stream call, on the client's + runtime. Args: url: A Redis URL, or the seed URLs of a cluster. Credentials and TLS come from the @@ -75,13 +77,13 @@ def __init__( cluster: Whether the URLs are of a Redis cluster. key_prefix: The prefix of every key. Defaults to ``temporal-streams``. retention: The default Retention of a new Stream. Defaults to 7 days. - safety_ttl: How long an open Workflow-owned Stream stays after its last write. + safety_ttl: How long an open Workflow-owned Stream is kept after its last write. Defaults to 90 days. - response_timeout: How long one Redis call takes at most. Defaults to 5 s. - name: The backend name that Stream handles carry, so every process routes a - handle it receives. + response_timeout: The longest time that one Redis call can take. Defaults to 5 s. + name: The backend name that Stream handles store, so every process can route a + handle that it receives. default: Make this the backend of a create with no backend name. With more than - one registered backend, exactly one has it, else the client raises + one registered backend, exactly one must have it, or the client raises ``StreamConfigError``. """ config = ext_pb.RedisStreamConfig( @@ -109,10 +111,10 @@ def create( retention: timedelta | None = None, backend: str | None = None, ) -> Creating[ClientStream[D]]: - """Create a Stream that this client owns, so only a client closes it. + """Create a Stream that the client owns, so only a client can close it. - It is idempotent for one id and one Retention. Await the call for the handle, or use it in - ``async with``, which closes the Stream at the end of the block:: + The call is idempotent for the same id and Retention. Await the call to get the handle, or + use it in ``async with``, which closes the Stream at the end of the block:: async with redis.create(client, Events, id="chat-1") as chat: await chat.topic(Events.turns).publish(Turn("user", "hi")) @@ -123,9 +125,10 @@ def create( Args: client: The client, with a registered Redis backend. definition: The Stream definition class. - id: The Stream id, as a Workflow takes a Workflow id. - retention: The age after which a Record is dropped, also while the Stream is open. - A closed Stream stays one Retention after its close. Defaults to the backend's. + id: The Stream id, which the caller picks, as for a Workflow id. + retention: The age after which a Record is deleted, even while the Stream is open. + A closed Stream is kept for one Retention after its close. Defaults to the + backend's Retention. backend: The name of the Redis backend. Defaults to the client's only Redis backend, or the one with ``default=True``. @@ -133,7 +136,7 @@ def create( StreamConfigError: The client has no Redis backend of that name. No call is made. StreamClosedError: A Stream of that id was created and closed before. StreamExpiredError: A Stream of that id expired. - StreamError: The Stream exists with another Retention, with the reason + StreamError: The Stream exists with a different Retention, with the reason ``stream_already_exists``. """ @@ -147,7 +150,7 @@ async def create() -> ClientStream[D]: def _redis_backend(backends: Sequence[_plugin.StreamBackend], name: str | None) -> str: - # The memory backend keeps Redis's rules, so it stands in for Redis in tests. + # The memory backend has Redis's rules, so tests can use it in place of Redis. redis = [backend for backend in backends if backend._kind == "redis"] if name: if not any(backend.backend_name == name for backend in redis): diff --git a/temporalio/contrib/stream/testing.py b/temporalio/contrib/stream/testing.py index b9ad6de74b..0002ec03e8 100644 --- a/temporalio/contrib/stream/testing.py +++ b/temporalio/contrib/stream/testing.py @@ -14,14 +14,14 @@ class MemoryStreamBackend(ExtensionStreamBackend): - """Keeps Streams in this process, with the same rules as on Redis, for tests. + """Keeps Streams in this process, with the same rules as Redis, for tests. Register it on the client, as with ``RedisStreamBackend``. Every Worker and client that - shares one backend shares its Streams, and they go when the process ends. It registers the - backend name ``memory``. It keeps Redis's rules, so the calls of - :py:mod:`temporalio.contrib.stream` take it as a Redis backend, and tests swap the Store - with no change to Workflow code. As on Redis, a Workflow's close is a regular Activity, so - Workers that share a task queue all register the backend. + shares one backend shares its Streams. The Streams are lost when the process ends. It + registers the backend name ``memory``. It has Redis's rules, so the functions of + :py:mod:`temporalio.contrib.stream` accept it as a Redis backend, and tests can swap the + Store without changing Workflow code. As on Redis, a Workflow's close is a regular + Activity, so all Workers that share a task queue must register the backend. .. warning:: This API is experimental. @@ -42,13 +42,13 @@ def __init__( Args: retention: The default Retention of a new Stream. Defaults to 7 days. - safety_ttl: How long an open Workflow-owned Stream stays after its last write. + safety_ttl: How long an open Workflow-owned Stream is kept after its last write. Defaults to 90 days. runtime: The runtime of the backend's memory Store. Defaults to the default runtime. - name: The backend name that Stream handles carry. + name: The backend name that Stream handles store. default: Make this the backend of a create with no backend name. With more than - one registered backend, exactly one has it. + one registered backend, exactly one must have it. """ config = ext_pb.RedisStreamConfig() if retention is not None: diff --git a/temporalio/contrib/stream/workflow.py b/temporalio/contrib/stream/workflow.py index acb147a711..a25d3aeea9 100644 --- a/temporalio/contrib/stream/workflow.py +++ b/temporalio/contrib/stream/workflow.py @@ -1,10 +1,10 @@ """Streams in Workflow code on the registered Redis backend. -These calls work as :py:func:`temporalio.workflow.create_stream`, +These functions work like :py:func:`temporalio.workflow.create_stream`, :py:func:`temporalio.workflow.create_stream_nowait` and -:py:func:`temporalio.workflow.stream_handle`, with the Worker's Redis backend in place of the -default one. A ``MemoryStreamBackend`` keeps Redis's rules, so tests use it here with no change -to Workflow code. +:py:func:`temporalio.workflow.stream_handle`, but use the Worker's Redis backend in place of the +default one. A ``MemoryStreamBackend`` has Redis's rules, so tests can use it here without +changing Workflow code. .. warning:: This module is experimental. @@ -24,7 +24,7 @@ def _backend(backend: str | None) -> str: - # The Worker's config gives this error, so it fails the Workflow Task and not the run. + # This error comes from the Worker's config, so it fails the Workflow Task and not the run. worker = _workflow.worker_backend() if worker is None: raise _workflow.no_worker_backend() @@ -43,20 +43,21 @@ def create_stream( ) -> Creating[WorkflowStream[D]]: """Create a Stream that this run chain owns, on the Worker's Redis backend. - See :py:func:`temporalio.workflow.create_stream`. On Redis, the end of the run chain closes - nothing, so close the Stream before the run returns. + See :py:func:`temporalio.workflow.create_stream`. On Redis, the end of the run chain doesn't + close a Stream outside an ``async with`` block, so close the Stream before the run returns. .. warning:: This API is experimental. Args: definition: The Stream definition class. Its name is the Stream name. - retention: The age after which a Record is dropped, also while the Stream is open. - A closed Stream stays one Retention after its close. Defaults to the backend's. + retention: The age after which a Record is deleted, even while the Stream is open. + A closed Stream is kept for one Retention after its close. Defaults to the + backend's Retention. backend: The name of the Redis backend. Defaults to the Worker's. Raises: - StreamError: The backend refused the create, such as for another Retention. + StreamError: The backend refused the create, such as for a different Retention. StreamConfigError: The Worker has no Redis backend, or none of that name. It fails the Workflow Task, not the run. """ @@ -75,7 +76,7 @@ def create_stream_nowait( retention: timedelta | None = None, backend: str | None = None, ) -> WorkflowStream[D]: - """Create a Stream on the Worker's Redis backend, with no await. + """Create a Stream on the Worker's Redis backend, without an await. See :py:func:`temporalio.workflow.create_stream_nowait`. @@ -90,7 +91,7 @@ def create_stream_nowait( def stream_handle( definition: type[D], *, backend: str | None = None ) -> WorkflowStream[D]: - """A Stream that this run chain created on the Worker's Redis backend. It makes no call. + """Get a Stream that this run chain created on the Worker's Redis backend. It makes no call. See :py:func:`temporalio.workflow.stream_handle`. diff --git a/temporalio/converter/_payload_converter.py b/temporalio/converter/_payload_converter.py index 35ce411207..210aa33c36 100644 --- a/temporalio/converter/_payload_converter.py +++ b/temporalio/converter/_payload_converter.py @@ -110,9 +110,10 @@ def transfer_type_convertible( ) -> _TransferTypeConvertibleDecorator[ValueT, TransferTypeT]: """Decorate a class with a transfer type converter class. - A value of the class takes its transfer type at the top level of a payload, and inside a - JSON value such as a list or a dataclass field. A dataclass is the exception inside a - JSON value: ``dataclasses.asdict`` writes it as its fields, and it decodes from them. + The converter converts a value of the class to its transfer type at the top level of a + payload, and inside a JSON value such as a list item or a dataclass field. Inside a JSON + value, a dataclass is the exception. ``dataclasses.asdict`` writes it as its fields, and + the converter decodes it from them. .. warning:: This API is experimental and subject to change. @@ -132,8 +133,8 @@ def _get_transfer_type_converter( def _nested_transfer_type_converter( hint: object, ) -> TransferTypeConverter[Any, Any] | None: - # A dataclass inside another value is written as its fields, so only other classes take - # their transfer type there. + # dataclasses.asdict writes a nested dataclass as its fields, so only a class that isn't a + # dataclass is converted there. origin = typing.get_origin(hint) or hint if not inspect.isclass(origin) or dataclasses.is_dataclass(origin): return None @@ -689,8 +690,7 @@ def default(self, o: Any) -> Any: # Datetime support if isinstance(o, datetime): return o.isoformat() - # A transfer type value inside another value. A dataclass there is written as its - # fields, such as in a top-level list, since value_to_type decodes it from them. + # Nested transfer type support converter = _nested_transfer_type_converter(type(o)) if converter is not None: value = converter.to_transfer_type(o) @@ -866,7 +866,7 @@ def value_to_type( if ret is not JSONTypeConverter.Unhandled: return ret - # A transfer type value inside another value, as AdvancedJSONEncoder writes it + # Nested transfer type values, as AdvancedJSONEncoder writes them converter = _nested_transfer_type_converter(hint) if converter is not None: transfer_type = converter.transfer_type diff --git a/temporalio/worker/_replayer.py b/temporalio/worker/_replayer.py index 1ed28e186a..e4ae5e9d94 100644 --- a/temporalio/worker/_replayer.py +++ b/temporalio/worker/_replayer.py @@ -67,9 +67,9 @@ def __init__( See :py:meth:`temporalio.worker.Worker.__init__` for a description of most of the arguments. Most of the same arguments need to be passed to the replayer that were passed to the worker when the workflow originally - ran. ``stream_backends`` are the Stream backends of the Worker's client, - such as :py:class:`temporalio.contrib.stream.RedisStreamBackend`. A - replay makes no Stream call, so they need not reach their Store. + ran. Pass ``stream_backends`` with the same backends as the Worker's + client, such as :py:class:`temporalio.contrib.stream.RedisStreamBackend`. + A replay makes no Stream call, so the Store doesn't need to be reachable. Note, unlike the worker, for the replayer the workflow_task_executor will default to a new thread pool executor with no max_workers set that diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index 5a99ffbd0b..af10ffc05a 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -410,8 +410,8 @@ def __init__( self._plugins = plugins for plugin in plugins: config = plugin.configure_worker(config) - # The Worker takes its client's Stream backends after the plugins, as the client does, - # so a plugin that replaces the Workflow runner keeps the Streams passthrough. + # Configure Streams after the plugins, so a plugin that replaces the Workflow runner + # doesn't lose the Streams sandbox passthrough. stream_backends = client.config(active_config=True).get("stream_backends") if stream_backends: from temporalio.common._streams import _plugin as _stream_plugin diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 3d55d4f342..8eb77b71dc 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -1,7 +1,7 @@ """Streams in Workflow code. -The calls load the Streams API of :py:mod:`temporalio.common` on first use, so a Workflow with -no Stream never loads it. +These functions import the Streams API of :py:mod:`temporalio.common` on first use, so a +Workflow with no Stream never imports it. """ from __future__ import annotations @@ -23,23 +23,24 @@ def create_stream( ) -> Creating[WorkflowStream[D]]: """Create a Stream that this run chain owns. - Its id is ``::``, so a client finds it from the Workflow - handle with no call. The create and each publish are Local Activities, and the create's - error raises here. Only the run chain closes the Stream. A close runs once its Workflow - Task commits, as a regular Activity on Redis, so a task that fails before then leaves the - Stream open for its retry. On Redis, the end of the run chain closes nothing, so a Stream - that nobody closes stays open until its ``safety_ttl``. + The Stream id is ``::``, so a client can get it from the + Workflow handle without a call. The create and each publish are Local Activities, and the + create's error is raised here. Only the run chain can close the Stream. A close runs after + its Workflow Task commits, as a regular Activity on Redis, so a task that fails first leaves + the Stream open for its retry. On Redis, the end of the run chain doesn't close a Stream + outside an ``async with`` block. So a Stream that nobody closes stays open until its + ``safety_ttl``. - Await the call for the handle, or use it in ``async with``, which closes the Stream at the - end of the block:: + Await the call to get the handle, or use it in ``async with``, which closes the Stream at + the end of the block:: async with workflow.create_stream(common.with_name(Events, "turn-3")) as turn: await turn.topic(Events.tokens).publish("hi") - An exception closes the Stream with that exception as the failure. When a retry run may - follow, the close waits for the run's end, and the Stream stays open only for a retry run - or for the retry of a Workflow Task that failed. A cron run leaves the Stream open. See - :py:meth:`temporalio.common.WorkflowStream.__aexit__`. + An exception in the block closes the Stream with that exception as the failure. If a retry + run can come next, the close waits for the run's end. The Stream then stays open only for a + retry run, or for the retry of a failed Workflow Task. A cron run leaves the Stream open. + See :py:meth:`temporalio.common.WorkflowStream.__aexit__`. .. warning:: This API is experimental. @@ -47,21 +48,22 @@ def create_stream( Args: definition: The Stream definition class. Its name is the Stream name. For one Stream per turn, for example, pass ``common.with_name(Events, f"turn-{n}")``. - retention: The age after which a Record is dropped, also while the Stream is open. - A closed Stream stays one Retention after its close. Defaults to the backend's. + retention: The age after which a Record is deleted, even while the Stream is open. + A closed Stream is kept for one Retention after its close. Defaults to the + backend's Retention. backend: The name of the backend. Defaults to the registered backend with ``default=True``. The default is part of the Workflow's code, since a replay must - create on the same backend. For now a Worker serves one backend, so the name is - that of the Worker's backend. + create the Stream on the same backend. A Worker serves only one backend, so the + name must be the Worker's backend name. Raises: - StreamError: The backend refused the create, such as for another Retention, or - ``StreamClosedError`` and ``StreamExpiredError`` for a Stream of that id that - closed or expired. + StreamError: The backend refused the create, such as for a different Retention. It's + ``StreamClosedError`` or ``StreamExpiredError`` if a Stream with that id closed or + expired. ReadOnlyContextError: The code must not change state, such as a query handler. - StreamConfigError: No Stream backend is on the Worker, none of that name, or none - has ``default=True`` for a create with no backend. It fails the Workflow Task, - not the run. + StreamConfigError: The Worker has no Stream backend or none with that name, or no + backend has ``default=True`` and the call gives no backend. It fails the Workflow + Task, not the run. """ from temporalio.common._streams import _workflow @@ -74,11 +76,11 @@ def create_stream_nowait( retention: timedelta | None = None, backend: str | None = None, ) -> WorkflowStream[D]: - """Create a Stream that this run chain owns, with no await. + """Create a Stream that this run chain owns, without an await. - It is for sync Workflow code, such as ``@workflow.init``, as ``asyncio.Queue.put_nowait`` - is for code that can't await ``put``. It takes the options of :py:func:`create_stream`. A - backend error fails the Workflow Task. + Use it in sync Workflow code, such as ``@workflow.init``. It's like + ``asyncio.Queue.put_nowait`` for code that can't await ``put``. It takes the options of + :py:func:`create_stream`. A backend error fails the Workflow Task. .. warning:: This API is experimental. @@ -93,20 +95,20 @@ def create_stream_nowait( def stream_handle( definition: type[D], *, backend: str | None = None ) -> WorkflowStream[D]: - """A Stream that this run chain created, by its definition. It makes no call. + """Get a Stream that this run chain created, by its definition. It makes no call. - A later run of the chain, such as one after continue-as-new, uses it to write or close the - Stream again. In ``@workflow.init`` of a run that continued as new, the run has not read - the backends of the chain's Streams yet, so pass ``backend=`` there, or call it in the run + A later run of the chain, such as one after continue-as-new, uses it to write to or close + the Stream. In ``@workflow.init`` of a run that continued as new, the run hasn't read the + backends of the chain's Streams yet. So pass ``backend=`` there, or call it in the run method. .. warning:: This API is experimental. Args: - definition: The Stream definition class, renamed as for the create. - backend: The backend of the Stream. Defaults to the one that the run chain created it - on, and else to the registered backend with ``default=True``. + definition: The Stream definition class, with the same name as in the create. + backend: The backend of the Stream. Defaults to the backend that the run chain created + it on, or else to the registered backend with ``default=True``. Raises: StreamConfigError: No backend is given, known or marked as the default. diff --git a/tests/contrib/streams/conftest.py b/tests/contrib/streams/conftest.py index 0de80034b5..ba32594c13 100644 --- a/tests/contrib/streams/conftest.py +++ b/tests/contrib/streams/conftest.py @@ -26,8 +26,8 @@ def _free_port() -> int: return port -# One scratch server serves every test module, and each test takes its own key prefix. A run -# that must use a known port gives its own server in TEMPORAL_STREAMS_REDIS_URL. +# One scratch server serves every test module, and each test uses its own key prefix. To use a +# known port, set TEMPORAL_STREAMS_REDIS_URL to your own server. @pytest.fixture(scope="session") def redis_url() -> Iterator[str]: given = os.environ.get("TEMPORAL_STREAMS_REDIS_URL") @@ -59,7 +59,7 @@ def redis_url() -> Iterator[str]: async def stream_client( client: Client, request: pytest.FixtureRequest ) -> AsyncIterator[Client]: - # Every live test runs on the memory Store and on Redis, since both must keep one set of + # Every live test runs on the memory Store and on Redis, since both must have the same # rules. The backend is the default, so a create with no backend name uses it. backend: StreamBackend if request.param == "memory": diff --git a/tests/contrib/streams/examples/README.md b/tests/contrib/streams/examples/README.md index 9564002ee9..d559350efa 100644 --- a/tests/contrib/streams/examples/README.md +++ b/tests/contrib/streams/examples/README.md @@ -1,6 +1,7 @@ # Examples for the Brook streams API proposal -Each file is a complete program for one section of the proposal. `test_examples.py` runs each one end to end on the memory Store and on Redis. +Each file is a complete program for one section of the proposal. `test_examples.py` runs each one +end to end on the memory Store and on Redis. | Proposal section | File | |---|---| @@ -12,4 +13,4 @@ Each file is a complete program for one section of the proposal. `test_examples. | A Workflow scopes a Stream to one turn and closes it | [`turn_scoped_stream.py`](turn_scoped_stream.py) | | Worker setup | [`worker_setup.py`](worker_setup.py) | -The proposal's Nexus examples are a later stage, so they have no file here. +The proposal's Nexus examples aren't in this directory. diff --git a/tests/contrib/streams/examples/activity_streams_tokens.py b/tests/contrib/streams/examples/activity_streams_tokens.py index 862e793cd1..b68f0f3558 100644 --- a/tests/contrib/streams/examples/activity_streams_tokens.py +++ b/tests/contrib/streams/examples/activity_streams_tokens.py @@ -38,7 +38,7 @@ class SummarizeInput: async def call_model(input: CallModelInput) -> int: tokens = input.stream.topic(Events.tokens) words = input.prompt.split() - # Each awaited publish has written when it returns. + # Each awaited publish has written its Record when it returns. last = await tokens.publish(words[0]) # A fast model sends many small tokens, so the helper batches them. async with common.BufferedPublisher( diff --git a/tests/contrib/streams/examples/agent_publishes_turns.py b/tests/contrib/streams/examples/agent_publishes_turns.py index 74f1ba9655..660ac4b9ab 100644 --- a/tests/contrib/streams/examples/agent_publishes_turns.py +++ b/tests/contrib/streams/examples/agent_publishes_turns.py @@ -2,7 +2,7 @@ The Workflow creates its Stream, publishes each turn with an awaited ``publish``, and passes the Stream to the model Activity in its one input object. It closes the Stream with its result -before it returns, since a run end closes nothing. +before it returns, since a run's end doesn't close the Stream. """ from __future__ import annotations @@ -69,7 +69,7 @@ async def run(self, messages: list[str]) -> str: ) if not reply.tool_calls: await turns.publish(Turn("done", reply.text)) - # A run end closes nothing, so the Workflow closes its Stream. + # A run's end doesn't close the Stream, so the Workflow closes it. await stream.close(result=reply.text) return reply.text for call in reply.tool_calls: diff --git a/tests/contrib/streams/examples/test_examples.py b/tests/contrib/streams/examples/test_examples.py index b9afcce37c..0e41708670 100644 --- a/tests/contrib/streams/examples/test_examples.py +++ b/tests/contrib/streams/examples/test_examples.py @@ -1,4 +1,4 @@ -"""Runs each example end to end on the memory Store and on Redis, so none goes stale.""" +"""Runs each example end to end on the memory Store and on Redis, so a broken one fails.""" import uuid diff --git a/tests/contrib/streams/examples/turn_scoped_stream.py b/tests/contrib/streams/examples/turn_scoped_stream.py index c29ea6de6a..8c1223147e 100644 --- a/tests/contrib/streams/examples/turn_scoped_stream.py +++ b/tests/contrib/streams/examples/turn_scoped_stream.py @@ -66,7 +66,7 @@ async def run(self, input: ChatTurnsInput) -> None: common.with_name(Events, f"turn-{input.next + i}"), retention=timedelta(days=1), ) as turn: - # A Stream is data, so the index Record carries the turn's Stream. + # A Stream is data, so the index Record contains the turn's Stream. await started.publish( TurnStarted(input.next + i, common.ClientStream.from_handle(turn)) ) @@ -79,7 +79,7 @@ async def run(self, input: ChatTurnsInput) -> None: workflow.continue_as_new( ChatTurnsInput(input.prompts[2:], input.next + len(batch), index) ) - # The last run closes the index, since a run end closes nothing. + # The last run closes the index, since a run's end doesn't close it. await index.close() diff --git a/tests/contrib/streams/examples/ui_backend_reads.py b/tests/contrib/streams/examples/ui_backend_reads.py index 1960c05523..f0d71198ab 100644 --- a/tests/contrib/streams/examples/ui_backend_reads.py +++ b/tests/contrib/streams/examples/ui_backend_reads.py @@ -1,7 +1,7 @@ """Brook API proposal, "A UI backend reads a Workflow's Stream". The agent Workflow creates the ``events`` Stream. The UI backend starts it, gets the Stream -from the Workflow handle with no call to Temporal, and renders each item as it comes. +from the Workflow handle with no call to Temporal, and renders each item as it arrives. """ from __future__ import annotations @@ -55,15 +55,13 @@ async def run(self, prompt: str) -> str: start_to_close_timeout=timedelta(minutes=5), ) await turns.publish(Turn("done", reply)) - # A run end closes nothing, so the Workflow closes its Stream before it returns. + # A run's end doesn't close the Stream, so the Workflow closes it before it returns. await stream.close(result=reply) return reply class Screen: - """Stands in for the UI. It keeps what it shows by offset, so a withdrawal can take items - back. - """ + """A fake UI. It keeps what it shows by offset, so a withdrawal can remove items.""" def __init__(self) -> None: self.shown: dict[int, common.Record[Any]] = {} @@ -96,6 +94,6 @@ async def main(client: Client, task_queue: str) -> Screen: for offset in item.withdrawn_offsets: screen.unrender(offset) else: - # item.data is a Turn or a str, by item.topic. + # item.data is a Turn or a str, depending on item.topic. screen.render(item) return screen diff --git a/tests/contrib/streams/examples/worker_setup.py b/tests/contrib/streams/examples/worker_setup.py index 04b3a12b15..d48ec404ad 100644 --- a/tests/contrib/streams/examples/worker_setup.py +++ b/tests/contrib/streams/examples/worker_setup.py @@ -1,9 +1,9 @@ """Brook API proposal, "Worker setup". -The Stream backends go on the client with ``stream_backends=``, and the Worker takes the -client's backends. Each backend registers a name. A create with no backend name goes to the -one with ``default=True``. Tests use ``MemoryStreamBackend`` with no change to Workflow code. -Sync Workflow code, such as ``@workflow.init``, uses the ``_nowait`` calls. +Register the Stream backends on the client with ``stream_backends=``. The Worker uses the +client's backends. Each backend registers a name. A create with no backend name uses the +backend with ``default=True``. Tests use ``MemoryStreamBackend`` without changing Workflow code. +Sync Workflow code, such as ``@workflow.init``, uses the ``_nowait`` functions. """ from __future__ import annotations @@ -53,7 +53,7 @@ async def connect_for_tests(target_host: str) -> Client: async def connect_while_migrating(target_host: str, url_a: str, url_b: str) -> Client: - # Two backends during a move. Creates with no backend= go to the default one. + # Two backends during a migration. A create with no backend= uses the default one. return await Client.connect( target_host, stream_backends=[ @@ -64,7 +64,7 @@ async def connect_while_migrating(target_host: str, url_a: str, url_b: str) -> C def run_worker(client: Client, task_queue: str) -> Worker: - # The Worker takes the client's backends, so it needs no Stream setup of its own. + # The Worker uses the client's backends, so it needs no Stream setup of its own. return Worker(client, task_queue=task_queue, workflows=[Greeter]) diff --git a/tests/contrib/streams/fake_extension.py b/tests/contrib/streams/fake_extension.py index 8b9619b848..152b5583bc 100644 --- a/tests/contrib/streams/fake_extension.py +++ b/tests/contrib/streams/fake_extension.py @@ -1,4 +1,4 @@ -"""An in-memory stand-in for the temporalio-streams runtime, for facade tests.""" +"""An in-memory fake of the temporalio-streams runtime, for the facade tests.""" from __future__ import annotations @@ -152,7 +152,7 @@ async def _read( class _FakeBackend(StreamBackend): - # The fake keeps Redis's rules, so the Redis calls take it. + # The fake has Redis's rules, so the Redis functions accept it. _kind = "redis" def __init__(self, calls: _ExtensionCalls) -> None: diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index ca1c305c75..9b8b395470 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -217,7 +217,7 @@ async def run(self, key: str, expired: common.WorkflowStream[Events]) -> list[An async def test_retry_goes_on_with_a_closed_or_expired_stream(stream_client: Client): - # The second attempt's open withdraws nothing on a Stream that takes no write, and the + # On a Stream that accepts no write, the second attempt's open withdraws nothing, and the # attempt only reads. expired = await redis.create( stream_client, @@ -240,7 +240,7 @@ async def test_retry_goes_on_with_a_closed_or_expired_stream(stream_client: Clie assert result == ["turn"] and attempts[key] == 2 -# Set by a test once its readers showed the first attempt's Record. +# Set by a test after its readers showed the first attempt's Record. shown: dict[str, asyncio.Event] = {} @@ -288,8 +288,8 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( reconnects = stream.subscribe(Events.tokens, create_wait=timedelta(seconds=20)) first = await anext(live) before = await anext(reconnects) - # The reader drops its connection and keeps only the Records it showed and where to - # go on from, as a UI does across a page reload. + # The reader closes its connection and keeps only the Records that it showed and its + # resume offset, as a UI does across a page reload. resume_at = reconnects.next_offset await reconnects.aclose() shown[key].set() @@ -301,7 +301,7 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( assert isinstance(before, common.Record) and before.offset == first.offset [live_superseded] = [i for i in live_items if isinstance(i, common.Superseded)] assert live_superseded.withdrawn_offsets == (first.offset,) - # A new read gave no Record of the failed attempt, so it names no offset. The kept + # A new read got no Record of the failed attempt, so the item lists no offset. The kept # Record is still withdrawn. [superseded] = [i for i in resumed if isinstance(i, common.Superseded)] assert superseded.withdrawn_offsets == () @@ -344,13 +344,13 @@ async def run(self) -> list[str]: common.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) - # A Local Activity writes before its Workflow Task commits, so it can't close either. + # A Local Activity runs before its Workflow Task commits, so it can't close it either. local = await workflow.execute_local_activity( close_it, common.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) - # A handle with only the id names no Workflow, so the Store refuses it. + # A handle with only the id names no Workflow, so the Store refuses the close. by_id = await workflow.execute_local_activity( close_it, common.ActivityStream(id=stream.id), @@ -367,7 +367,7 @@ async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): handle = await stream_client.start_workflow( Closing.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) - # An Activity writes to the Workflow's Stream but can't end it for every reader. + # An Activity writes to the Workflow's Stream but can't close it. assert await handle.result() == [ "config", "stream_not_owner", @@ -467,8 +467,8 @@ async def test_buffered_publisher_writes_a_full_batch_with_one_write_in_flight() publisher.publish(*range(1000)) publisher.publish(*range(1000, 2500)) await asyncio.sleep(0.2) - # A full batch goes with no wait for the delay, and the values published during its - # write go in the next one. + # A full batch is written without waiting for the delay. The values published during its + # write are in the next batch. assert [len(batch) for batch in topic.batches] == [1000, 1500] assert topic.most_in_flight == 1 assert await publisher.flush() == 2500 @@ -708,7 +708,7 @@ async def run(self, key: str, stream: common.WorkflowStream[Events] | None) -> N async def keyed(handle: WorkflowHandle[Any, Any], activity_type: str) -> list[bool]: - # Whether each schedule of the Activity carries the Producer key. + # Whether each schedule of the Activity has the Producer key. history = await handle.fetch_history() return [ "__temporal_stream_key" in attributes.header.fields @@ -722,7 +722,7 @@ async def keyed(handle: WorkflowHandle[Any, Any], activity_type: str) -> list[bo async def test_only_a_run_that_uses_streams_keys_its_activities(stream_client: Client): # The key is in History, so a run with no Stream adds none. A run that created, bound or - # received one keys each Activity, so an Activity that writes keeps its Producer id. + # received a Stream adds it to each Activity, so an Activity that writes keeps its Producer id. key = str(uuid.uuid4()) async with new_worker( stream_client, @@ -752,8 +752,8 @@ async def test_only_a_run_that_uses_streams_keys_its_activities(stream_client: C ) await receives.result() assert await keyed(receives, "plain") == [True] - # The next run gets the Stream as its argument, and its retried Activity withdraws its - # first attempt as in the run that created the Stream. + # The next run gets the Stream as its argument. Its retried Activity withdraws the first + # attempt's Records, as in the run that created the Stream. continued_key = str(uuid.uuid4()) continued = await stream_client.start_workflow( RetriesAfterContinueAsNew.run, diff --git a/tests/contrib/streams/test_facade.py b/tests/contrib/streams/test_facade.py index d3b6fb5310..c30f8ae603 100644 --- a/tests/contrib/streams/test_facade.py +++ b/tests/contrib/streams/test_facade.py @@ -140,8 +140,8 @@ async def test_handles_of_one_client_share_a_producer_and_free_it(client: Client if isinstance(item, common.Record) and item.producer } assert len(producers) == 1 - # A handle that goes with no close frees its writer, so a server that makes a handle per - # request keeps no writer for it. + # A handle that Python frees without a close frees its writer. So a server that makes a + # handle per request doesn't keep a writer for each. del other gc.collect() assert len(extension.freed) == 1 @@ -166,7 +166,7 @@ async def publish_later() -> None: await asyncio.sleep(1) await stream.topic(Events.tokens).publish("late") - # Each call of a read that waits answers before the deadline, so the read goes on. + # Each call of a waiting read returns before the deadline, so the read continues. reader = stream.subscribe(Events.tokens, rpc_timeout=timedelta(milliseconds=300)) publisher = asyncio.create_task(publish_later()) item = await anext(reader) diff --git a/tests/contrib/streams/test_handles.py b/tests/contrib/streams/test_handles.py index e7de9095cf..6054d4bfca 100644 --- a/tests/contrib/streams/test_handles.py +++ b/tests/contrib/streams/test_handles.py @@ -161,7 +161,7 @@ def test_end_type_is_public(): def test_temporalio_common_exports_the_streams_api(): - # The Stream names load on first use, and each is the object of the private package. + # temporalio.common imports the Stream names on first use. Each is the private package's. import temporalio.common._streams as streams_api names = { @@ -175,7 +175,8 @@ def test_temporalio_common_exports_the_streams_api(): def test_import_of_temporalio_common_loads_no_stream_code(): - # The Stream modules import temporalio.client and temporalio.workflow. + # The Stream modules import temporalio.client and temporalio.workflow, which import + # temporalio.common. code = "import sys, temporalio.common; print('temporalio.common._streams' in sys.modules)" loaded = subprocess.run( [sys.executable, "-c", code], capture_output=True, text=True, check=True diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 0388c787d8..9b3367e964 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -141,7 +141,8 @@ async def run(self, key: str) -> None: async def test_failed_workflow_task_is_withdrawn(stream_client: Client): key = str(uuid.uuid4()) - # The test's counter must survive the retry of the task, so no sandbox. + # The test's counter must keep its value across the task's retry, so the Workflow has no + # sandbox. async with new_worker( stream_client, PublishThenFailTask, @@ -398,7 +399,7 @@ async def test_rerun_local_activity_withdraws_its_failed_task(stream_client: Cli superseded = [item for item in items if isinstance(item, common.Superseded)] visible = [r for r in written if not any(i.withdraws(r) for i in superseded)] assert [r.data for r in visible] == ["2", "3"] - # The rerun keeps the Producer id and moves to the retry task's epoch. + # The rerun keeps the Producer id and uses the retry task's epoch. first, second = written[0].producer, visible[0].producer assert first is not None and second is not None and first.id == second.id assert first.epoch is not None and second.epoch is not None @@ -427,7 +428,7 @@ async def run(self, key: str) -> None: async def test_later_write_withdraws_a_skipped_local_activity(stream_client: Client): # The retry task skips the Local Activity, so only its own write can withdraw "0 1". The - # write also takes the skipped call's key. + # write also uses the skipped call's key. async with new_worker( stream_client, LocalOnlyInFailedTask, @@ -517,7 +518,7 @@ async def test_records_go_through_the_payload_codec(stream_client: Client): await handle.result() stream = workflow_stream(handle) decoded = await read_all(stream) - # A Topic outside the definition reads the raw Payloads. + # A read of a Topic outside the definition gets the raw Payloads. plain = codec_client.stream(Plain, id=stream.id) raw = [ item @@ -656,9 +657,8 @@ async def execute_workflow(self, input: ExecuteWorkflowInput) -> Any: def test_closing_an_evicted_run_touches_no_runtime(): - # Python closes the coroutine of an evicted run with GeneratorExit, outside the - # Workflow's loop. The interceptor must let it pass with no call to the runtime, or - # Python prints "Exception ignored". + # Python closes an evicted run's coroutine with GeneratorExit, outside the Workflow's loop. + # The interceptor must let it pass without a call to the runtime. inbound = _WorkflowInbound(_NeverEnds()) run = inbound.execute_workflow( cast(ExecuteWorkflowInput, cast(object, SimpleNamespace(headers={}, args=[]))) @@ -721,14 +721,14 @@ async def test_final_failure_under_a_retry_policy( items = [item async for item in stream.subscribe(wait_new_records=False)] written = records(items) superseded = [item for item in items if isinstance(item, common.Superseded)] - # The end of the chain closes nothing. + # The end of the chain doesn't close the Stream. assert not any(isinstance(item, common.Closed) for item in items) if non_retryable: # A non-retryable failure ends the chain on attempt 1. assert [r.data for r in written] == ["run-1"] assert superseded == [] return - # Each retry run withdraws the run before it. + # Each retry run withdraws the Records of the run before it. assert [r.data for r in written] == ["run-1", "run-2", "run-3"] assert len(superseded) == 2 assert [r.data for r in written if superseded[0].withdraws(r)] == ["run-1"] @@ -746,7 +746,7 @@ async def read_until_quiet() -> None: assert isinstance(item, common.Record) seen.append(item) - # A read has no idle end, so a reader that wants one sets its own timer. + # A read has no idle timeout, so a reader that wants one uses its own timer. with pytest.raises(asyncio.TimeoutError): await asyncio.wait_for(read_until_quiet(), timeout=0.5) assert [item.offset for item in seen] == [first] @@ -762,8 +762,8 @@ async def read_until_quiet() -> None: async def test_two_backends_on_one_client(client: Client, redis_url: str): - # A client may carry two Stream backends, such as during a move from one backend to - # another. Each handle carries its backend, so a call goes to the backend that has it. + # A client can have two Stream backends, such as during a migration to another backend. Each + # handle stores its backend name, so a call goes to the backend that has the Stream. config = client.config() config["stream_backends"] = [ RedisStreamBackend(redis_url, key_prefix=f"t-{uuid.uuid4()}", default=True), @@ -792,8 +792,8 @@ async def test_two_backends_on_one_client(client: Client, redis_url: str): class _FailsTheLastTaskOnce(Interceptor, WorkerInterceptor): - # On the client before the Stream backend's interceptor, so it is the outer Workflow - # interceptor and fails the task after the backend drained the run's writes. + # It's on the client before the Stream backend's interceptor, so it's the outer Workflow + # interceptor. It fails the task after the backend drained the run's writes. def workflow_interceptor_class( self, input: WorkflowInterceptorClassInput ) -> type[WorkflowInboundInterceptor]: diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 7b0f252d78..97db96eaee 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -294,7 +294,8 @@ def __init__(self) -> None: async def test_closed_failure_decodes_with_the_clients_converter(stream_client: Client): - # The message goes into the encoded attributes, which only the payload converter reads. + # The failure converter puts the message in the encoded attributes, which only the payload + # converter reads. config = stream_client.config() config["data_converter"] = dataclasses.replace( DataConverter.default, failure_converter_class=_EncodesCommonAttributes @@ -329,7 +330,7 @@ async def test_last_reads_the_newest_records_then_follows(stream_client: Client) tokens = stream.topic(Events.tokens) await tokens.publish("a", "b", "c") await stream.topic(Events.turns).publish(Turn("model")) - # A chat view shows the last few Records first, then follows. + # A chat view shows the last few Records first, then waits for new ones. tail = stream.subscribe(tokens, last=2) assert tail.next_offset is None shown = [await anext(tail), await anext(tail)] @@ -422,7 +423,7 @@ async def test_async_with_refuses_a_workflow_stream_before_the_block( async def test_record_over_the_store_limit_raises_too_large(stream_client: Client): - # The client's writer splits a batch at the Store's limit, so only a Record can be over. + # The client's writer splits a batch at the Store's limit, so only one Record can be over it. stream = await redis.create(stream_client, Events, id=f"large-{uuid.uuid4()}") with pytest.raises(common.StreamTooLargeError) as refused: await stream.topic(Events.tokens).publish("x" * 2_000_000) @@ -467,7 +468,7 @@ async def test_create_of_an_expired_stream_raises(stream_client: Client): async def test_named_producer_fences_its_earlier_epoch(stream_client: Client): stream = await redis.create(stream_client, Events, id=f"named-{uuid.uuid4()}") - # A server that restarts writes as one name with a higher epoch, so the zombie it + # A server that restarts writes as the same name with a higher epoch, so the zombie that it # replaces can't write after it. before = stream.producer("ingest", epoch=1) await before.topic(Events.tokens).publish("before restart") @@ -502,7 +503,7 @@ async def describe(**_: Any) -> Any: setattr(handle, "describe", describe) await redis.create(client, Events, id="agent-2:run-2:events") - # Calls that start together share one Describe call. + # Concurrent calls share one Describe call. assert await asyncio.gather(first.get_offset(), first.get_offset()) == [0, 0] assert describes == [1] assert first.id == "agent-2:run-2:events" @@ -518,7 +519,7 @@ async def test_registration_rules(client: Client): two = Client(**config) assert (await redis.create(two, Events, id="d-1")).backend == "b" assert two.stream(Events, id="d-1").backend == "b" - # The default is part of Workflow code, so it must not follow the order of the list. + # The default is part of Workflow code, so it must not depend on the order of the list. for backends in ( [MemoryStreamBackend(), MemoryStreamBackend(name="b")], [ @@ -532,7 +533,7 @@ async def test_registration_rules(client: Client): config["stream_backends"] = [MemoryStreamBackend(), MemoryStreamBackend()] with pytest.raises(common.StreamConfigError, match="two Stream backends"): Client(**config) - # For now a Worker serves one backend, and so does a Replayer. + # A Worker serves only one backend, and so does a Replayer. config["stream_backends"] = [ MemoryStreamBackend(), MemoryStreamBackend(name="b", default=True), @@ -555,7 +556,7 @@ async def test_registration_rules(client: Client): async def test_worker_that_polls_no_activities_warns_of_the_close( client: Client, caplog: pytest.LogCaptureFixture ): - # The close of a Workflow's Stream is a regular Activity, which another Worker may run. + # The close of a Workflow's Stream is a regular Activity, which another Worker can run. config = client.config() config["stream_backends"] = [MemoryStreamBackend(default=True)] with caplog.at_level(logging.WARNING, logger="temporalio.common._streams"): @@ -580,7 +581,7 @@ async def test_backend_is_not_a_plugin(client: Client): async def test_worker_plugin_keeps_the_streams_passthrough(client: Client): # The backends configure the Worker after its plugins, so a plugin that replaces the - # Workflow runner doesn't drop the passthrough. + # Workflow runner doesn't lose the Streams sandbox passthrough. config = client.config() config["stream_backends"] = [MemoryStreamBackend(default=True)] plugin = SimplePlugin("replaces-runner", workflow_runner=SandboxedWorkflowRunner()) @@ -599,7 +600,7 @@ async def test_default_is_native_with_no_default_backend(client: Client): with pytest.raises(common.StreamConfigError, match="default is native"): one.get_workflow_handle("agent-3").stream(Events) assert one.stream(Events, id="d-2", backend="memory").backend == "memory" - # The Redis create picks the client's only Redis backend, and memory stands in for it. + # The Redis create picks the client's only Redis backend. The memory backend counts as one. assert (await redis.create(one, Events, id="d-2")).backend == "memory" @@ -613,7 +614,7 @@ async def test_stream_takes_the_handle_backend_or_the_default(client: Client): created = await redis.create(two, Events, id="moving", backend="b") await created.topic(Events.tokens).publish("on b") assert await two.stream(Events, stream=created).get_offset() > 0 - # A handle with no backend name is of the default backend. + # A handle with no backend name uses the default backend. assert two.stream(Events, stream=common.ClientStream(id="moving")).backend == "a" with pytest.raises(ValueError): two.stream(Events, stream=created, backend="a") diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 7d5b267010..d30a0948b7 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -58,7 +58,7 @@ async def run(self, prompt: str) -> str: turns = stream.topic(Events.turns) await turns.publish(Turn("user", prompt), Turn("model")) await stream.topic("tokens").publish(*prompt.split()) - # A run end closes nothing, so the run closes its Stream before it returns. + # A run's end doesn't close this Stream, so the run closes it before it returns. await stream.close(result=prompt.upper()) return prompt.upper() @@ -246,7 +246,7 @@ async def test_workflow_closes_only_its_own_streams(stream_client: Client): ) == "stream_not_owner" ) - # The writer can't end the Stream for every reader, so it stays open. + # The Workflow that received the Stream can't close it, so it stays open. items = [item async for item in stream.subscribe(wait_new_records=False)] assert data(items) == ["from workflow"] @@ -369,7 +369,7 @@ async def test_block_end_closes_when_no_retry_run_follows(stream_client: Client) assert closed.status == common.CloseStatus.FAILED assert isinstance(closed.failure, ApplicationError) assert closed.failure.message == "block failed" - # A close at the run's end is a command after the run's code, also on replay. + # A close at the run's end is a command after the run's code, even on replay. replayer = Replayer( workflows=[EndsItsBlock], stream_backends=stream_client.config()["stream_backends"], @@ -378,7 +378,7 @@ async def test_block_end_closes_when_no_retry_run_follows(stream_client: Client) await replayer.replay_workflow(await handles[mode].fetch_history()) -# Set by the Activity of a test's run once it runs, so the test cancels the run there. +# The Activity of a test's run sets this when it starts, so the test cancels the run then. waiting: dict[str, asyncio.Event] = {} @@ -409,7 +409,7 @@ async def run(self, local: bool) -> None: async def test_cancel_during_an_activity_closes_as_canceled(stream_client: Client): - # The cancel reaches the block as an ActivityError, and no retry follows a cancelled run. + # The cancel reaches the block as an ActivityError, and a cancelled run gets no retry run. async with new_worker( stream_client, CancelledInItsBlock, activities=[waits_for_cancel] ) as worker: @@ -479,7 +479,8 @@ async def run(self) -> None: async def test_failed_task_writes_no_close(stream_client: Client): - # The test's set must survive the retry of the task, so no sandbox. + # The test's set must keep its value across the task's retry, so the Workflow has no + # sandbox. async with new_worker( stream_client, SchedulesACloseThenFailsItsTask, @@ -495,7 +496,7 @@ async def test_failed_task_writes_no_close(stream_client: Client): closes = [item for item in items if isinstance(item, common.Closed)] assert len(closes) == 1 and closes[0].result == "closed" history = await handle.fetch_history() - # The failed task's close never went out, so History has one close Activity. + # The failed task's close was never sent, so History has one close Activity. scheduled = [ event for event in history.events @@ -518,7 +519,7 @@ async def test_worker_config_errors_fail_the_task( client: Client, stream_client: Client ): # A Worker with no plugin, or with another backend, fails the Workflow Task, so a Worker - # with the right config can take the run on. + # with the right config can continue the run. for task_client, backend, message in ( (client, None, "No Stream backend"), (stream_client, "elsewhere", "backend elsewhere"), @@ -568,7 +569,7 @@ async def test_cron_runs_share_an_open_stream(stream_client: Client): class SyncHandlers: @workflow.init def __init__(self, prompt: str) -> None: - # The create goes first, so the signal's writes find the Stream. + # The create runs first, so the Stream exists for the signal's writes. self._stream = workflow.create_stream_nowait(Events) self._prompt = prompt self._done = False @@ -656,8 +657,8 @@ async def test_concurrent_publishes_keep_their_order(stream_client: Client): offsets = await handle.result() items = await read(handle) assert data(items) == ["0", "1", "2", "3", "4"] - # The four writes that waited went as one Local Activity, and each call got the offset of - # its own Record. + # The four writes that waited were sent as one Local Activity, and each call got the offset + # of its own Record. assert ( offsets == [item.offset for item in items if isinstance(item, common.Record)][1:] @@ -670,7 +671,7 @@ async def test_concurrent_publishes_keep_their_order(stream_client: Client): ] # The create, the first write and the four that waited. assert len(markers) == 3 - # Replay answers from the markers and writes nothing. + # Replay uses the markers and writes nothing. await Replayer( workflows=[Gathers], stream_backends=stream_client.config()["stream_backends"] ).replay_workflow(history) @@ -695,7 +696,7 @@ async def decode(self, payloads: Sequence[Payload]) -> list[Payload]: class _GrowingCodec(PayloadCodec): - # Triples each payload, so a merge under the Workflow's cap passes the Store's batch limit + # Triples each payload, so a merge under the Workflow's cap exceeds the Store's batch limit # after the codec. async def encode(self, payloads: Sequence[Payload]) -> list[Payload]: return [ @@ -761,8 +762,8 @@ async def loses_the_first_answer( await handle.result() items = await read(handle) assert data(items) == ["0", "1", "2", "3", "4"] - # The retry sends the bytes of the first attempt, so the Store's dedupe takes it as the - # same batch, though the codec adds random bytes. + # The retry sends the bytes of the first attempt, so the Store's dedupe matches it as the + # same batch, even though the codec adds random bytes. assert sent[0] == sent[1] @@ -797,15 +798,15 @@ def test_pending_appends_keep_the_newest_epoch_and_a_bounded_size(): one, two = ("ns", "s", "la:1"), ("ns", "s", "la:2") first, retried = _entry(10), _entry(10) pending.put(one, (5, 1, 0, 0), first) - # A rerun under a later epoch takes the place of the earlier one. + # A rerun under a later epoch replaces the earlier one. pending.put(one, (9, 1, 0, 0), retried) assert pending.get(one, (5, 1, 0, 0)) is None assert pending.get(one, (9, 1, 0, 0)) is retried - # A write of an earlier epoch is fenced, so it leaves the later one, and so does its pop. + # A write of an earlier epoch is fenced, so its put and its pop don't change the later entry. pending.put(one, (5, 1, 0, 0), first) pending.pop(one, (5, 1, 0, 0)) assert pending.get(one, (9, 1, 0, 0)) is retried - # Past the size, the oldest entry goes. + # Past the size limit, the oldest entry is evicted. pending.put(two, (9, 1, 0, 0), _entry(95)) assert pending.get(one, (9, 1, 0, 0)) is None assert pending.get(two, (9, 1, 0, 0)) is not None @@ -824,8 +825,8 @@ async def run(self, size: int) -> list[int]: async def test_merge_that_the_codec_grows_past_the_batch_limit(stream_client: Client): - # The two publishes merge under the Workflow's cap, and the codec makes the merge larger - # than one batch may be, so each publish goes as its own batch. + # The two publishes merge under the Workflow's cap. The codec makes the merge larger than the + # batch limit, so each publish is sent as its own batch. client = with_codec(stream_client, _GrowingCodec()) async with new_worker(client, PublishesLargeValues) as worker: handle = await client.start_workflow( @@ -913,7 +914,7 @@ async def test_create_of_an_expired_workflow_stream_raises(client: Client): async def test_default_is_native_with_no_default_backend(client: Client): # With no backend marked default=True, the default is native, which this SDK lacks. The - # Redis calls still pick the registered Redis backend, and the memory one stands in for it. + # Redis functions still pick the registered Redis backend, and the memory backend counts as one. config = client.config() config["stream_backends"] = [MemoryStreamBackend()] memory_client = Client(**config) From 41f52c0dffdc1dba8b7f76775bd5739ddc5d4829 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 02:56:14 -0700 Subject: [PATCH 76/79] Expected a native close by id to find no Stream in the owner test. The native server finds a Workflow's Stream only by its owner fields, so the by-id case p35 added answers stream_not_found there. --- tests/contrib/streams/test_activity_handles.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/contrib/streams/test_activity_handles.py b/tests/contrib/streams/test_activity_handles.py index fcb6dd6f52..e39983aad9 100644 --- a/tests/contrib/streams/test_activity_handles.py +++ b/tests/contrib/streams/test_activity_handles.py @@ -331,7 +331,7 @@ async def close_it(stream: common.ActivityStream[Events]) -> str: @workflow.defn class Closing: @workflow.run - async def run(self) -> list[str]: + async def run(self, native: bool) -> list[str]: stream = await workflow.create_stream(Events) backend = await workflow.execute_activity( wrong_backend, @@ -349,9 +349,10 @@ async def run(self) -> list[str]: common.ActivityStream.from_handle(stream), start_to_close_timeout=timedelta(seconds=30), ) - # A handle with only the id names no Workflow, so the Store refuses it. + # A handle with only the id names no Workflow, so the Store refuses it. The native + # server finds a Workflow's Stream only by its owner fields. by_id = await workflow.execute_local_activity( - close_it, + only_closes if native else close_it, common.ActivityStream(id=stream.id), start_to_close_timeout=timedelta(seconds=30), ) @@ -360,21 +361,22 @@ async def run(self) -> list[str]: async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): + native = on_native(stream_client) async with new_worker( - stream_client, Closing, activities=[wrong_backend, close_it] + stream_client, Closing, activities=[wrong_backend, close_it, only_closes] ) as worker: handle = await stream_client.start_workflow( - Closing.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue + Closing.run, native, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) # An Activity writes to the Workflow's Stream but can't end it for every reader. assert await handle.result() == [ "config", "stream_not_owner", "stream_not_owner", - "stream_not_owner", + "stream_not_found" if native else "stream_not_owner", ] items = await read(handle) - assert [r.data for r in records(items)] == ["last", "last", "last"] + assert [r.data for r in records(items)] == ["last"] * (2 if native else 3) assert isinstance(items[-1], common.Closed) assert items[-1].result == "closed by the run" From 8f71f3eebcca3f8f39ffd9fc99bacc2039b90ce8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 02:59:01 -0700 Subject: [PATCH 77/79] Dropped the figurative wording from three Stream error messages. --- temporalio/common/_streams/_activity_stream.py | 2 +- temporalio/common/_streams/_extension.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/temporalio/common/_streams/_activity_stream.py b/temporalio/common/_streams/_activity_stream.py index d3af608ae8..9212f3d72d 100644 --- a/temporalio/common/_streams/_activity_stream.py +++ b/temporalio/common/_streams/_activity_stream.py @@ -44,7 +44,7 @@ def _bound(handle: StreamHandle[Any]) -> _activity.ActivityStreams: ) if not handle.backend and not streams.default: raise _errors.no_default_backend( - "Worker", [streams.backend_name], "a handle that carries its backend name" + "Worker", [streams.backend_name], "a handle that has its backend name" ) return streams diff --git a/temporalio/common/_streams/_extension.py b/temporalio/common/_streams/_extension.py index 97d6c502f8..b27dd1e940 100644 --- a/temporalio/common/_streams/_extension.py +++ b/temporalio/common/_streams/_extension.py @@ -143,8 +143,8 @@ def _configure_worker( for other in interceptors ): raise StreamConfigError( - "The Worker's client has another Stream backend. For now a Worker serves " - "one backend." + "The Worker's client has another Stream backend. A Worker serves only one " + "backend." ) mine = self._interceptor_of(interceptors) if mine is None: @@ -184,7 +184,7 @@ def _configure_replayer( for other in config.get("interceptors", []) ): raise StreamConfigError( - "The Replayer has another Stream backend. For now a Replayer serves one " + "The Replayer has another Stream backend. A Replayer serves only one " "backend, as a Worker does." ) config["interceptors"] = [ From 300d460a7e3e2c3287ac69c0d3c6268a3c01d1e8 Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 03:09:48 -0700 Subject: [PATCH 78/79] Tightened the comments and docs on this branch. --- changelog/added/jaunty-jetty.md | 2 +- temporalio/activity.py | 18 ++-- temporalio/bridge/src/streams.rs | 2 +- temporalio/bridge/streams.py | 3 +- temporalio/client/_activity.py | 12 +-- temporalio/client/_client.py | 20 ++--- temporalio/client/_workflow.py | 2 +- temporalio/common/_streams/_access.py | 6 +- temporalio/common/_streams/_activity.py | 4 +- .../common/_streams/_activity_stream.py | 29 ++++--- temporalio/common/_streams/_backend.py | 5 +- temporalio/common/_streams/_buffered.py | 2 +- temporalio/common/_streams/_errors.py | 14 ++-- temporalio/common/_streams/_extension.py | 42 +++++----- temporalio/common/_streams/_handle.py | 4 +- temporalio/common/_streams/_native.py | 83 ++++++++++--------- temporalio/common/_streams/_plugin.py | 24 +++--- temporalio/common/_streams/_stream_client.py | 11 +-- temporalio/common/_streams/_workflow.py | 28 +++---- .../common/_streams/_workflow_reader.py | 54 ++++++------ temporalio/worker/_activity.py | 4 +- temporalio/worker/_worker.py | 4 +- temporalio/worker/_workflow.py | 12 +-- temporalio/workflow/_streams.py | 2 +- tests/contrib/streams/_support.py | 4 +- tests/contrib/streams/conftest.py | 9 +- .../examples/migrate_redis_to_native.py | 18 ++-- .../examples/standalone_activity_stream.py | 5 +- .../contrib/streams/examples/test_examples.py | 5 +- .../contrib/streams/examples/two_backends.py | 12 +-- .../streams/examples/workflow_reader.py | 4 +- tests/contrib/streams/test_backends.py | 6 +- tests/contrib/streams/test_native_server.py | 40 ++++----- tests/contrib/streams/test_redis.py | 2 +- tests/contrib/streams/test_stream_client.py | 8 +- tests/contrib/streams/test_workflow_closes.py | 5 +- tests/contrib/streams/test_workflow_module.py | 2 +- tests/contrib/streams/test_workflow_reads.py | 14 ++-- 38 files changed, 263 insertions(+), 258 deletions(-) diff --git a/changelog/added/jaunty-jetty.md b/changelog/added/jaunty-jetty.md index 249840b269..ea10281179 100644 --- a/changelog/added/jaunty-jetty.md +++ b/changelog/added/jaunty-jetty.md @@ -1,2 +1,2 @@ -Add `temporalio.common.NativeStreamBackend`, a Stream backend that keeps Streams in the Temporal server, which needs a server with the Stream service. It is the default and needs no registration. A client can register more than one backend, and its Workers serve them all. +Add `temporalio.common.NativeStreamBackend`, a Stream backend that stores Streams in the Temporal server. It needs a server with the Stream service. It's the default and needs no registration. A client can register more than one backend, and its Workers serve all of them. Add Workflow reads of a Stream with `WorkflowStream.subscribe`, which returns a `temporalio.common.WorkflowItems`. Add Streams owned by Standalone Activities with `activity.create_stream` and `client.ActivityHandle.stream`, and client-owned Streams with `Client.create_stream`. diff --git a/temporalio/activity.py b/temporalio/activity.py index d05c2c6938..51b2804aa0 100644 --- a/temporalio/activity.py +++ b/temporalio/activity.py @@ -527,30 +527,30 @@ def create_stream( ) -> Creating[ActivityStream[_StreamDefinitionT]]: """Create a Stream that this Standalone Activity owns, on the native backend. - Its id is ``::``, so a client finds it from the - Activity handle with :py:meth:`temporalio.client.ActivityHandle.stream` and no call. The - Stream goes on across the Activity's retries, and the server closes it when the Activity + The Stream id is ``::``, so a client can get it from the + Activity handle with :py:meth:`temporalio.client.ActivityHandle.stream` without a call. The + Stream stays open across the Activity's retries, and the server closes it when the Activity ends, with its status and result. So the end of an ``async with`` block doesn't close it. - Await the call for the handle:: + Await the call to get the handle:: stream = await activity.create_stream(Events) await stream.topic(Events.tokens).publish("hi") - Only the native backend keeps the Stream of a Standalone Activity, since nothing else - would close it when the Activity ends. + Only the native backend can store the Stream of a Standalone Activity, since no other + backend would close it when the Activity ends. .. warning:: This API is experimental. Args: definition: The Stream definition class. Its name is the Stream name. - retention: The age after which a Record is dropped. A closed Stream stays one - Retention after its close. Defaults to the backend's. + retention: The age after which a Record is deleted. A closed Stream is kept for one + Retention after its close. Defaults to the backend's Retention. backend: The name of a native backend. Defaults to ``native``. Raises: StreamConfigError: The Activity runs for a Workflow, which owns its Streams, or the - backend is not native. No call is made. + backend isn't native. No call is made. StreamClosedError: The Stream was created and closed before. """ from temporalio.common._streams import _activity_stream diff --git a/temporalio/bridge/src/streams.rs b/temporalio/bridge/src/streams.rs index 12e3b6a029..f7d16501a6 100644 --- a/temporalio/bridge/src/streams.rs +++ b/temporalio/bridge/src/streams.rs @@ -77,7 +77,7 @@ pub fn new_stream_runtime( } /// Builds the native runtime on a client's connection, from its encoded `NativeStreamConfig`. -/// Each client gets its own, so a call goes on the connection of the client that makes it. +/// Each client gets its own, so a call uses the connection of the client that makes it. #[cfg(feature = "streams")] pub fn new_native_stream_runtime( runtime_ref: &runtime::RuntimeRef, diff --git a/temporalio/bridge/streams.py b/temporalio/bridge/streams.py index 024d7fd667..063e68960c 100644 --- a/temporalio/bridge/streams.py +++ b/temporalio/bridge/streams.py @@ -37,8 +37,7 @@ def on_client( client: temporalio.bridge.client.Client, config: bytes, ) -> StreamRuntime: - """Build the native runtime on a connected client's connection, from its encoded - ``NativeStreamConfig``. + """Build the native runtime on a connected client, from its encoded ``NativeStreamConfig``. Raises: ValueError: The bridge build doesn't link the crate, or the config is not valid. diff --git a/temporalio/client/_activity.py b/temporalio/client/_activity.py index f35a1e3f45..40511bbf2f 100644 --- a/temporalio/client/_activity.py +++ b/temporalio/client/_activity.py @@ -1010,21 +1010,23 @@ def run_id(self) -> str | None: def stream( self, definition: type[_StreamDefinitionT], *, backend: str | None = None ) -> ClientStream[_StreamDefinitionT]: - """A Stream that this Standalone Activity created, by its definition. + """Get a Stream that this Standalone Activity created, by its definition. - It builds the Stream id from the Activity id and run id, so it makes no call. Only the - native backend keeps the Stream of a Standalone Activity, which + This method builds the Stream id from the Activity id and run id, so it makes no call. + Only the native backend stores the Stream of a Standalone Activity, which :py:func:`temporalio.activity.create_stream` creates. .. warning:: This API is experimental. Args: - definition: The Stream definition class, renamed as the Activity's create. + definition: The Stream definition class, with the same name as in the Activity's + create. backend: The name of a native backend. Defaults to ``native``. Raises: - StreamConfigError: The client has no backend of that name, or it is not native. + StreamConfigError: The client has no backend with that name, or the backend isn't + native. ValueError: The handle has no run id. """ from temporalio.common._streams import _stream_client diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index d04881fe23..10cd660b2a 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -965,10 +965,10 @@ def create_stream( retention: timedelta | None = None, backend: str | None = None, ) -> Creating[ClientStream[_StreamDefinitionT]]: - """Create a Stream that this client owns, so only a client closes it. + """Create a Stream that the client owns, so only a client can close it. - It is idempotent for one id and one Retention. Await the call for the handle, or use it - in ``async with``, which closes the Stream at the end of the block:: + The call is idempotent for the same id and Retention. Await the call to get the handle, + or use it in ``async with``, which closes the Stream at the end of the block:: async with client.create_stream(Events, id="chat-1") as chat: await chat.topic(Events.turns).publish(Turn("user", "hi")) @@ -978,16 +978,16 @@ def create_stream( Args: definition: The Stream definition class. - id: The Stream id, as a Workflow takes a Workflow id. - retention: The age after which a Record is dropped. A closed Stream stays one - Retention after its close. Defaults to the backend's. + id: The Stream id, which the caller picks, as for a Workflow id. + retention: The age after which a Record is deleted. A closed Stream is kept for one + Retention after its close. Defaults to the backend's Retention. backend: The name of the backend. Defaults to the registered backend with - ``default=True``, else to ``native``. + ``default=True``, or else to ``native``. Raises: - StreamConfigError: The client has no backend of that name. No call is made. - StreamClosedError: A Stream of that id was created and closed before. - StreamError: The Stream exists with another Retention, with the reason + StreamConfigError: The client has no backend with that name. No call is made. + StreamClosedError: A Stream with that id was created and closed before. + StreamError: The Stream exists with a different Retention, with the reason ``stream_already_exists``. """ from temporalio.common._streams import _stream_client diff --git a/temporalio/client/_workflow.py b/temporalio/client/_workflow.py index 45b03e92a1..bab75e58a9 100644 --- a/temporalio/client/_workflow.py +++ b/temporalio/client/_workflow.py @@ -208,7 +208,7 @@ def stream( definition: The Stream definition class, with the same name as in the Workflow's create. backend: The name of the backend. Defaults to the registered backend with - ``default=True``, else to ``native``. + ``default=True``, or else to ``native``. Raises: StreamConfigError: The client has no backend with that name. diff --git a/temporalio/common/_streams/_access.py b/temporalio/common/_streams/_access.py index 516320d3f3..3f2b9c78ea 100644 --- a/temporalio/common/_streams/_access.py +++ b/temporalio/common/_streams/_access.py @@ -81,8 +81,8 @@ async def get_offset(self) -> int: return response.last_offset + 1 if response.last_offset else 0 async def _close_ref(self) -> models.StreamRef: - # A handle with only an id names no chain, so a close names the chain that the Access - # gives, and the Store refuses it when another chain owns the Stream. + # A handle with only an id has no run chain, so the close sends the Access's run chain. + # The Store refuses it if another chain owns the Stream. ref = await self.ref() if ref.workflow_id or not self.workflow_id: return ref @@ -187,7 +187,7 @@ def subscription( class AsyncTopicBase(TopicBase[T]): - """Base of a Topic of an Activity or client handle. A publish writes before it returns.""" + """Base class of a Topic of an Activity or client handle. A publish writes before it returns.""" async def publish(self, *values: T) -> int: """Write one Record for each value, in order, and return the last Record's offset. diff --git a/temporalio/common/_streams/_activity.py b/temporalio/common/_streams/_activity.py index 029af4a0b6..7c69b251ce 100644 --- a/temporalio/common/_streams/_activity.py +++ b/temporalio/common/_streams/_activity.py @@ -33,9 +33,9 @@ class ActivityStreams: # An Activity's task token, which its writes send, so the native server checks the # Producer against the Activity's current attempt. Empty for a Local Activity. task_token: bytes = b"" - # Whether the backend keeps the Streams of a Standalone Activity. + # Whether the backend stores the Streams of a Standalone Activity. activity_streams: bool = False - # The most bytes of one append that the backend's Store takes. + # The most bytes of one append that the backend's Store accepts. max_batch_bytes: int = 4 * 1024 * 1024 # Whether the backend's Store finds a Workflow's Stream only by its owner fields. nests_workflow_streams: bool = False diff --git a/temporalio/common/_streams/_activity_stream.py b/temporalio/common/_streams/_activity_stream.py index 4f3f9f0d00..716efe5447 100644 --- a/temporalio/common/_streams/_activity_stream.py +++ b/temporalio/common/_streams/_activity_stream.py @@ -84,8 +84,8 @@ async def ref() -> models.StreamRef: namespace=streams.namespace, ref=ref, writer=lambda: streams.writer(handle), - # A handle with only an id has no run chain, so the close sends the Activity's Workflow id - # and the Store refuses it. A nesting Store gets none, since a Workflow id alone isn't a ref. + # A handle with only an id has no run chain, so the close sends the Activity's Workflow + # id and the Store refuses it. A nesting Store gets no id, since one alone isn't a ref. workflow_id=handle.workflow_id or (None if streams.nests_workflow_streams else streams.workflow_id), first_execution_run_id=handle.first_execution_run_id, @@ -181,8 +181,9 @@ async def close( await self._access().close(result, failure) async def __aenter__(self) -> ActivityStream[E]: - """Use the Stream in ``async with``, which closes it at the end, except a Standalone - Activity's Stream. + """Use the Stream in ``async with``, which closes it at the end of the block. + + The block's end doesn't close a Standalone Activity's Stream. """ return self @@ -195,8 +196,8 @@ async def __aexit__( """Close the Stream, with the exception as its failure if there is one. A cancel closes it with a ``CancelledError`` failure. Another ``BaseException`` leaves - it open. A Standalone Activity's Stream stays open, since the server closes it when the - Activity ends, with its status and result, and a retry of the Activity writes on to it. + it open. A Standalone Activity's Stream stays open, since a retry of the Activity writes + to it, and the server closes it when the Activity ends, with its status and result. """ if self.activity_id: return @@ -376,8 +377,9 @@ def create_stream( retention: timedelta | None = None, backend: str | None = None, ) -> Creating[ActivityStream[E]]: - """Create a Stream that this Standalone Activity owns. See - :py:func:`temporalio.activity.create_stream`. + """Create a Stream that this Standalone Activity owns. + + See :py:func:`temporalio.activity.create_stream`. """ info = temporalio.activity.info() if info.in_workflow: @@ -386,15 +388,16 @@ def create_stream( "the Streams that it receives." ) attempt = _activity.current.get() - # Only a native backend keeps the Stream, so the default rule doesn't apply. + # Only a native backend stores the Stream, so the default rule doesn't apply. name = backend or _ids.NATIVE_BACKEND streams = attempt.backends.get(name) if attempt is not None else None if streams is None: raise _errors.StreamConfigError(f"No Stream backend {name} runs this Activity") if not streams.activity_streams: raise _errors.StreamConfigError( - f"The backend {streams.backend_name} keeps no Stream of a Standalone Activity, " - "since nothing would close it when the Activity ends. Use the native backend." + f"The backend {streams.backend_name} can't store the Stream of a Standalone " + "Activity, since it wouldn't close the Stream when the Activity ends. Use the " + "native backend." ) run_id = info.activity_run_id or "" stream_id = _ids.scoped_stream_id( @@ -413,8 +416,8 @@ async def create() -> ActivityStream[E]: id=stream_id, activity_id=info.activity_id, activity_run_id=run_id ), ) - # The server checks the token against the Activity's current attempt, so a caller - # can't put a Stream into an Activity it isn't. + # The server checks the token against the Activity's current attempt, so only that + # attempt can create the Activity's Stream. with _backend.with_task_token(streams.task_token): response = await streams.backend.service( streams.namespace, streams.converter diff --git a/temporalio/common/_streams/_backend.py b/temporalio/common/_streams/_backend.py index bdb8b1042a..0d468507b8 100644 --- a/temporalio/common/_streams/_backend.py +++ b/temporalio/common/_streams/_backend.py @@ -89,9 +89,8 @@ def using(options: CallOptions | None) -> Iterator[None]: call_options.reset(token) -# The task token of the Activity attempt whose write is in progress. The generated models leave -# it out, so the transport puts it into the request, and the native server checks the Producer -# against the Activity's current attempt. +# The task token of the Activity attempt whose write is in progress. The generated models lack +# it, so the transport adds it to the request for the native server's Producer check. _task_token: contextvars.ContextVar[bytes] = contextvars.ContextVar( "__temporal_stream_task_token", default=b"" ) diff --git a/temporalio/common/_streams/_buffered.py b/temporalio/common/_streams/_buffered.py index 71577450d3..1c40ff212b 100644 --- a/temporalio/common/_streams/_buffered.py +++ b/temporalio/common/_streams/_buffered.py @@ -98,7 +98,7 @@ def __init__( self._unwritten = 0 self._reported = False attempt = _activity.current.get() - # The attempt's end flushes the publisher before its backend drops the writers. + # The attempt's end flushes the publisher before its backend closes the writers. stream = self._topic._stream if self._topic is not None else self._stream streams = ( attempt.get(stream.backend) diff --git a/temporalio/common/_streams/_errors.py b/temporalio/common/_streams/_errors.py index fce736d4ce..824ca7049f 100644 --- a/temporalio/common/_streams/_errors.py +++ b/temporalio/common/_streams/_errors.py @@ -79,10 +79,10 @@ class StreamOffsetError(StreamError): class StreamTooLargeError(StreamError): """A Record or a batch is over the Store's limit. - A native Stream also keeps the Records that an open Workflow reader may read again on - replay, so an append that the Stream's size cap would make drop them fails. It passes once - the reader moves on, such as when it continues as new, which the server suggests to a - reader that keeps half of the cap. + A native Stream also keeps the Records that an open Workflow reader can read again on + replay. An append fails if the Stream's size cap would make the Store drop them. The append + succeeds after the reader advances, such as when it continues as new. The server suggests + continue-as-new to a reader that holds half of the cap. """ @@ -135,7 +135,7 @@ class ReadOnlyContextError(StreamError): "stream_unsupported": StreamUnsupportedError, "stream_not_owner": StreamNotOwnerError, "seq_divergent": StreamSeqDivergentError, - # The native server's Workflow reader cap. Readers leave, so a retry may get in. + # The native server's cap on Workflow readers. Readers finish, so a retry can succeed. "stream_exhausted": StreamStorageError, } @@ -159,9 +159,9 @@ def from_call_error(err: Exception) -> StreamError: def from_application_error( err: temporalio.exceptions.ApplicationError, ) -> StreamError | None: - """The Stream error that an application error of a failed call carries, if any. + """The Stream error in the application error of a failed call, if any. - A Stream error's type is its class name, and its first detail is its reason. A server may + A Stream error's type is its class name, and its first detail is its reason. A server can give the reason as the type, or at the start of the message, such as ``stream_closed: ...``. """ token = err.message.split(":", 1)[0].strip() diff --git a/temporalio/common/_streams/_extension.py b/temporalio/common/_streams/_extension.py index e30fda2776..f127e80073 100644 --- a/temporalio/common/_streams/_extension.py +++ b/temporalio/common/_streams/_extension.py @@ -40,9 +40,8 @@ logger = logging.getLogger(__name__) -# How long a Workflow read's Local Activity waits for new Records or for the create. It is below -# Core's 8 s Workflow Task heartbeat, so a read that waits spans one task. The Store waits for -# both from one start, so the timeout's margin covers the calls. +# How long a Workflow read's Local Activity waits for new Records or the create, below Core's 8 s +# task heartbeat. The Store waits for both as one wait, so the timeout's margin covers the calls. _READ_WAIT = timedelta(seconds=5) _READ_MARGIN = timedelta(seconds=10) _PAGE_TYPE = stream_pb.ReadRecordsResponse.DESCRIPTOR.full_name.encode() @@ -98,7 +97,7 @@ def __init__( runtime or temporalio.runtime.Runtime.default(), True ) super().__init__(name, backend_name=backend_name, default=default) - # The read activity's name has the backend name, which the base keeps. + # The read Activity's name includes the backend name, so this runs after the base __init__. self._read_activity = _read_activity(self) async def _read_result( @@ -109,9 +108,8 @@ async def _read_result( owner_ended: bool, converter: temporalio.converter.DataConverter, ) -> Any: - # The result of a Workflow read's Local Activity, which its marker holds. Workflow code - # can't run the codec, so the page's bodies go to it decoded. The page as a whole goes - # through the codec as the Local Activity's result. + # The result of a Workflow read's Local Activity. Workflow code can't run the codec, so the + # bodies are decoded here, and the codec runs on the whole page as the Activity's result. await _decode_page(converter, page) payload = page_payload(page) if owner_ended: @@ -119,12 +117,12 @@ async def _read_result( return temporalio.common.RawValue(payload) async def _owner_ended(self, ref: stream_pb.StreamRef) -> bool: - # A reader at the end of a Stream whose owner ended with no close would wait until - # the Stream expires, so an empty page tells it. + # A reader at the end of a Stream whose owner ended without a close would wait until + # the Stream expires. So an empty page tells the reader that the owner ended. return await _owner_chain_ended(ref) def _activities(self) -> list[Callable[..., Any]]: - # The Local Activities that Workflow reads run, so the Worker must have them. + # Workflow reads run these Local Activities, so the Worker must register them. return [self._read_activity] def _runtime_backend( @@ -184,8 +182,8 @@ def _configure_worker( # Add the backend's interceptor if the client has none, and on Redis add the close Activity. config = super()._configure_worker(config) if config.get("workflows") and self._kind == "redis": - # The close runs once the Workflow Task commits, as a regular Activity, so this - # Worker polls Activities on its task queue. + # The close is a regular Activity that runs after the Workflow Task commits, so this + # Worker must poll Activities on its task queue. activities = list(config.get("activities", [])) if _writes.close_stream not in activities: activities.append(_writes.close_stream) @@ -275,8 +273,8 @@ async def read_page(request: stream_pb.ReadRecordsRequest) -> Any: # The Worker's client gave the attempt its Stream runtime. streams = _attempt_streams(backend.backend_name) try: - # The reading run goes with the read, so a native server keeps the Records that - # the open run may read again on replay. + # The read sends the reading run, so a native server keeps the Records that the open + # run can read again on replay. page = await cast(_ExtensionCalls, streams.backend).call_async( "WorkflowReadRecords", ext_pb.WorkflowReadRecordsRequest( @@ -320,7 +318,7 @@ async def _owner_chain_ended(ref: stream_pb.StreamRef) -> bool: except temporalio.service.RPCError as err: if err.status == temporalio.service.RPCStatusCode.NOT_FOUND: return True - # Only a hint, so the reader waits on as before. + # The result is only a hint, so the reader keeps waiting. logger.debug("Describing the owner of Stream %s failed", ref.id, exc_info=True) return False first_run_id = description.raw_description.workflow_execution_info.first_run_id @@ -461,8 +459,8 @@ def _receive(args: Sequence[Any]) -> None: async def _end_run(err: BaseException | None) -> None: - # The run's reads end with it, so the server drops their parked reads. Its block-end closes - # settle from its outcome, and its writes that nobody awaits go out before its end. + # Cancel the run's reads, so the server drops their parked reads. Then run the deferred closes + # by the run's outcome, and send the unawaited writes before the run ends. await _workflow_reader.cancel_wakes() streams_workflow.close_at_run_end(err) await streams_workflow.drain() @@ -505,7 +503,7 @@ def _take_owned_streams( class _WorkflowOutbound(temporalio.worker.WorkflowOutboundInterceptor): _temporal_stream_transport = True - # The runtime can't park a read yet, so a reader pauses on a timer after an empty page. + # The runtime can't park a read, so a reader waits on a timer after an empty page. _temporal_stream_parks_reads = False _temporal_stream_read_wait = _READ_WAIT # The Local Activity's timeout bounds its wait for the create too. @@ -553,7 +551,7 @@ async def _temporal_stream_read_page( temporalio.exceptions.ActivityError, temporalio.exceptions.ApplicationError, ) as err: - # A Local Activity's failure may come as its application error, with no + # A Local Activity's failure can come as its application error, with no # ActivityError around it. raise _workflow_reader.error_of(err) from None return _workflow_reader.Page( @@ -643,8 +641,8 @@ def enter_attempt( ) -> tuple[_activity.AttemptStreams, Any]: """Add a backend's state to the attempt, and return the attempt and the token to reset. - The outermost backend's interceptor makes the attempt, so it reads the backends of the - Worker from the chain, and each inner one adds its state. + The outermost backend's interceptor creates the attempt and reads the Worker's backends + from the chain. Each inner interceptor adds its state. """ attempt = _activity.current.get() if attempt is not None: @@ -658,7 +656,7 @@ def enter_attempt( (link._temporal_stream_backend_name, link._temporal_stream_default) ) link = getattr(link, "next", None) - # The default is the backend with default=True, else native. + # The default is the backend with default=True, or else native. default = next((name for name, marked in backends if marked), _ids.NATIVE_BACKEND) attempt = _activity.AttemptStreams(default=default) attempt.backends[streams.backend_name] = streams diff --git a/temporalio/common/_streams/_handle.py b/temporalio/common/_streams/_handle.py index 748a6f0a90..1e6f163cda 100644 --- a/temporalio/common/_streams/_handle.py +++ b/temporalio/common/_streams/_handle.py @@ -288,8 +288,8 @@ def _topic_spec( return topic, type def _owner_ref(self) -> models.StreamRef: - # A Workflow's or a Standalone Activity's Stream carries its owner on every call, so a - # server that nests it in its owner finds it. + # A Workflow's or a Standalone Activity's Stream sends its owner on every call, so a + # server that nests the Stream under its owner can find it. return models.StreamRef( id=self._id, workflow_id=self._workflow_id or None, diff --git a/temporalio/common/_streams/_native.py b/temporalio/common/_streams/_native.py index 97895b9683..36e0e4499a 100644 --- a/temporalio/common/_streams/_native.py +++ b/temporalio/common/_streams/_native.py @@ -1,4 +1,4 @@ -"""The native Stream backend, whose Streams the Temporal server keeps.""" +"""The native Stream backend, whose Streams the Temporal server stores.""" from __future__ import annotations @@ -35,27 +35,27 @@ _SERVICE = "temporal.api.streamservice.v1.StreamService" -# A wake's cancel ends it once the server has the cancel. The server drops the cancel of a run -# that closed, so the run's end waits for it. +# A wake's cancel completes after the server gets the cancel. The server drops the cancel of a +# closed run, so the run's end waits for it. _WAIT_REQUESTED = temporalio.workflow.NexusOperationCancellationType.WAIT_REQUESTED -# A close goes on when the run is cancelled, since the cancel may be what the close records. +# A close continues when the run is cancelled, since the close can record that cancel. _ABANDON = temporalio.workflow.NexusOperationCancellationType.ABANDON # How long a wake parks on the server. A parked wake adds nothing to History, so an idle reader # costs nothing. The server caps it at `stream.maxWakeWait`, and an older server at 60 s. _WAKE_WAIT = timedelta(days=7) -# A Workflow read's Local Activity answers with its positions only: the request, and the answer -# with no Records. The codec step turns them back into the page before Workflow code sees it. +# A Workflow read's Local Activity returns only positions, the request and the answer without +# Records. The codec step turns them back into the page before Workflow code gets it. POSITIONS_TYPE = b"temporal.io/stream-read-positions" _COUNT_KEY = "temporal.io/stream-read-count" # The SHA-256 of the page's Records as the Store gave them, so a replay that reads other # Records fails. _DIGEST_KEY = "temporal.io/stream-read-digest" -# The run that read the page. A reset run replays the old run's pages, which this process may +# The run that read the page. A reset run replays the old run's pages, which this process might # never have read. _RUN_KEY = "temporal.io/stream-read-run" -# The pages that this process read and no Workflow Task took yet. A Workflow Task takes its -# pages, and the bound covers the runs that left the Worker before theirs. +# The pages that this process read and no Workflow Task has used yet. A Workflow Task removes its +# pages, and the size bound covers runs that left the Worker before their task. _CACHE_SIZE = 256 _pages: OrderedDict[tuple[str, bytes], list[stream_pb.StreamRecord]] = OrderedDict() RANGE_GONE_TYPE = "STREAM_RANGE_GONE" @@ -67,7 +67,8 @@ def _digest(records: Sequence[stream_pb.StreamRecord]) -> bytes: def _range_gone(stream_id: str, detail: str) -> temporalio.exceptions.ApplicationError: - # Retryable, so it fails the Workflow Task and not the run. The operator picks the way out. + # Retryable, so it fails the Workflow Task and not the run. The operator then resets or + # terminates the run. return temporalio.exceptions.ApplicationError( f"A recorded read of the Stream {stream_id} is no longer in the Store, so the run can't " f"replay: {detail}. Reset the run to a point before its reads of the Stream, or " @@ -86,8 +87,8 @@ def __init__( inner: temporalio.converter.PayloadCodec | None, records: temporalio.converter.PayloadCodec | None, ) -> None: - # The runtime that a replay reads again on: the Worker's client's, or the Replayer's - # stream_client's. + # The runtime for a replay's second read. It's the Worker's client's runtime, or the + # Replayer's stream_client's. self._backend = backend self._inner = inner # Writers encode Records with no serialization context, so a page decodes with the @@ -153,8 +154,8 @@ async def _read_again( count: int, digest: bytes, ) -> Sequence[stream_pb.StreamRecord]: - # The range is fixed by History, and a Stream only appends, so a read of the same range - # gives the same Records. It runs on replay, or when the page left the cache. + # History fixes the range, and a Stream only appends, so a read of the same range gives + # the same Records. This runs on replay, or when the page is no longer in the cache. request.to_offset = answer.next_offset stream_id = request.stream.id try: @@ -184,7 +185,7 @@ async def _read_again( class _ConnectionRuntime: # The native runtime of one client, on that client's connection. A lazy client connects at - # its first call, so the runtime comes then too. + # its first call, so the runtime is built then too. def __init__( self, service: temporalio.service.ServiceClient, config: bytes ) -> None: @@ -203,8 +204,8 @@ async def call(self, method: str, input: bytes) -> bytes: if service is None: raise RuntimeError("The client of the native Stream backend is gone") bridge_client = await _bridge_client(service) - # Two first calls may both wait for the connection. One runtime serves both, so - # a writer that one opens is known to the other. + # Two first calls can both wait for the connection. One runtime serves both, so each + # sees the writers that the other opens. if self._streams is None: self._streams = temporalio.bridge.streams.StreamRuntime.on_client( _runtime_of(service)._core_runtime, @@ -252,7 +253,7 @@ async def _temporal_stream_read_page( async def _temporal_stream_start_wake( self, request: stream_pb.ReadRecordsRequest ) -> temporalio.workflow.NexusOperationHandle[stream_pb.ReadRecordsResponse]: - # Its answer holds no Records, so History holds only offsets. + # The wake's answer has no Records, so History has only offsets. wake = stream_pb.ReadRecordsRequest() wake.CopyFrom(request) wake.wake_only = True @@ -275,9 +276,8 @@ async def _temporal_stream_start_wake( async def _temporal_stream_close( self, request: stream_pb.CloseStreamRequest ) -> None: - # A System Nexus command, which the server carries out once the Workflow Task commits. - # A task that fails after the close leaves the Stream open, and its retry closes it. The - # server takes the closing run from the operation's links. + # A System Nexus command, which the server runs after the Workflow Task commits. The + # server gets the closing run from the operation's links. close = stream_pb.CloseStreamRequest() close.CopyFrom(request) close.stream.ClearField("backend") @@ -303,19 +303,20 @@ async def _temporal_stream_close( class NativeStreamBackend(ExtensionStreamBackend): - """Carries Stream calls to the Temporal server, which keeps the Streams. + """Sends Stream calls to the Temporal server, which stores the Streams. Every client, Worker and Replayer has it with no registration, under the name - ``native``, and it is the default backend unless a registered backend has + ``native``, and it's the default backend unless a registered backend has ``default=True``. Register one only to configure it, such as for its Retention, or to make it the default next to another backend. Workflow writes run as Local Activities, as on Redis, and the client's Stream runtime writes them to the server's ``temporal.api.streamservice.v1.StreamService``. Activity and - client calls go to the same service. Each client gets its own runtime on its own + client calls use the same service. Each client gets its own runtime on its own connection, built at the first call, and the client's Workers share it. A page of a - Workflow read is a direct read in a Local Activity whose marker holds only its positions, - and a replay reads the page again, on the connection of the Replayer's ``stream_client``. + Workflow read is a direct read in a Local Activity whose marker has only the page's + positions. A replay reads the page again, with the connection of the Replayer's + ``stream_client``. .. warning:: This API is experimental. @@ -335,14 +336,15 @@ def __init__( name: str = "native", default: bool = False, ) -> None: - """Create the backend. It makes no call. Each client of the backend calls the server - on its own connection. + """Create the backend. It makes no call. + + Each client of the backend calls the server on its own connection. Args: retention: The default Retention of a new Stream. Defaults to the server's. - name: The backend name that Stream handles carry. + name: The backend name that Stream handles store. default: Make this the backend of a create with no backend name. With more than - one registered backend, exactly one has it. + one registered backend, exactly one must have it. """ config = ext_pb.NativeStreamConfig() if retention is not None: @@ -358,8 +360,8 @@ def __init__( def _new_backend( self, service: temporalio.service.ServiceClient ) -> _ExtensionCalls: - # Each connection gets its own runtime, so a call goes on the connection of the - # client that makes it. The runtime comes at the first call. + # Each connection gets its own runtime, so a call uses the connection of the client + # that makes it. The runtime is built at the first call. backend = self._connections.get(service) if backend is None: backend = _ExtensionCalls(_ConnectionRuntime(service, self._config)) @@ -379,8 +381,8 @@ async def _read_result( owner_ended: bool, # pyright: ignore[reportUnusedParameter] converter: temporalio.converter.DataConverter, # pyright: ignore[reportUnusedParameter] ) -> Any: - # The marker holds the positions, and the Records wait as the Store gave them for the - # codec step, which decodes them as on replay. + # The marker holds the positions. The Records stay in the cache as the Store gave them, + # and the codec step decodes them as on replay. answer = stream_pb.ReadRecordsResponse() answer.CopyFrom(page) answer.ClearField("records") @@ -415,8 +417,8 @@ async def _owner_ended( return False async def _close_in_workflow(self, request: stream_pb.CloseStreamRequest) -> int: - # The close is a System Nexus command of the Workflow Task, which the server carries - # out once the task commits, so the Worker polls no Activities for it. + # The close is a System Nexus command of the Workflow Task, which the server runs after + # the task commits, so the Worker doesn't poll Activities for it. await _workflow_reader.reader_of(self.backend_name)._temporal_stream_close( request ) @@ -425,8 +427,8 @@ async def _close_in_workflow(self, request: stream_pb.CloseStreamRequest) -> int def _configure_client( self, config: temporalio.client.ClientConfig ) -> temporalio.client.ClientConfig: - # The calls come from the client's connection at the first call, so the client needs - # no interceptor. + # The runtime is built on the client's connection at the first call, so the client + # needs no interceptor. return config @@ -434,8 +436,9 @@ def wrap( backend: Callable[[], _ExtensionCalls], converter: temporalio.converter.DataConverter, ) -> temporalio.converter.DataConverter: - """The converter with a codec step that turns a native read's positions back into its - page, reading it again with ``backend`` when the page is not in the cache. + """The converter with a codec step that turns a native read's positions back into its page. + + The step reads the page again with ``backend`` if the page isn't in the cache. """ if isinstance(converter.payload_codec, _DirectReadCodec): return converter diff --git a/temporalio/common/_streams/_plugin.py b/temporalio/common/_streams/_plugin.py index 2f7c88bb7e..c180d337fe 100644 --- a/temporalio/common/_streams/_plugin.py +++ b/temporalio/common/_streams/_plugin.py @@ -38,10 +38,10 @@ class StreamBackend(SimplePlugin): # The kind of Store, such as "redis". The functions of a backend's package use it to find # the registered backend of their kind. _kind = "" - # Whether the backend keeps the Streams of a Standalone Activity, which it closes with the - # Activity. Only the server does, so a Stream that nobody closes can't stay. + # Whether the backend stores the Streams of a Standalone Activity and closes each with its + # Activity. Only the server does, so no Stream of an ended Activity stays open. _activity_streams = False - # The most bytes of one append that the Store takes, as it counts them. + # The most bytes of one append that the Store accepts, as it counts them. _max_batch_bytes = 4 * 1024 * 1024 # Whether the Store finds a Workflow's Stream only by its owner fields, so a ref with a # Workflow id and no first run id names nothing. @@ -225,7 +225,7 @@ def local_activities( def replayer_backends(config: temporalio.worker.ReplayerConfig) -> list[StreamBackend]: - """The Replayer's backends, else those of its ``stream_client``, and the native one.""" + """The Replayer's backends, or else those of its ``stream_client``, and the native one.""" backends = config.get("stream_backends") or () client = config.get("stream_client") if not backends and client is not None: @@ -245,8 +245,9 @@ def configure_replayer( def worker_converter( client: temporalio.client.Client, converter: temporalio.converter.DataConverter ) -> temporalio.converter.DataConverter: - """The converter of a Worker's Workflows, whose codec step turns a native read's - positions back into its page. + """The converter of a Worker's Workflows. + + Its codec step turns a native read's positions back into its page. """ from temporalio.common._streams import _native @@ -261,8 +262,9 @@ def replayer_converter( config: temporalio.worker.ReplayerConfig, converter: temporalio.converter.DataConverter, ) -> temporalio.converter.DataConverter: - """The converter of a Replayer's Workflows, as :py:func:`worker_converter`. A replay - reads a native page again on the connection of the Replayer's ``stream_client``. + """The converter of a Replayer's Workflows, as :py:func:`worker_converter`. + + A replay reads a native page again with the connection of the Replayer's ``stream_client``. """ from temporalio.common._streams import _native @@ -290,7 +292,7 @@ def backend_of(client: temporalio.client.Client, name: str | None) -> StreamBack """A client's backend. A ``name`` of None picks the default. Raises: - StreamConfigError: The client has no backend of that name. + StreamConfigError: The client has no backend with that name. """ backends = with_native(client_backends(client)) wanted = name or default_name(backends) @@ -306,10 +308,10 @@ def backend_of(client: temporalio.client.Client, name: str | None) -> StreamBack def client_backend( client: temporalio.client.Client, name: str | None ) -> tuple[str, BackendCalls]: - """The name and the calls of a client's backend. None picks the default backend. + """The name and the calls of a client's backend. A ``name`` of None picks the default. Raises: - StreamConfigError: The client has no backend of that name. + StreamConfigError: The client has no backend with that name. """ backend = backend_of(client, name) return backend.backend_name, backend._backend_for(client) diff --git a/temporalio/common/_streams/_stream_client.py b/temporalio/common/_streams/_stream_client.py index 39d0e11d74..15b0be923a 100644 --- a/temporalio/common/_streams/_stream_client.py +++ b/temporalio/common/_streams/_stream_client.py @@ -184,7 +184,7 @@ def _fields(self) -> dict[str, str]: return fields async def _resolved_ref(self) -> models.StreamRef: - # The first call finds the first run id of a Workflow's Stream, which the ref carries. + # The first call fetches the first run id of a Workflow's Stream, which the ref includes. await self._stream_id() return self._owner_ref() @@ -572,16 +572,17 @@ def activity_stream( *, backend: str | None, ) -> ClientStream[E]: - """A Stream that a Standalone Activity created. See - :py:meth:`temporalio.client.ActivityHandle.stream`. + """Get a Stream that a Standalone Activity created. + + See :py:meth:`temporalio.client.ActivityHandle.stream`. """ stream_name = definition_of(definition).name client = handle._client - # Only a native backend keeps the Stream, so the default rule doesn't apply. + # Only a native backend stores the Stream, so the default rule doesn't apply. stream_backend = backend_of(client, backend or _ids.NATIVE_BACKEND) if not stream_backend._activity_streams: raise _errors.StreamConfigError( - f"The backend {stream_backend.backend_name} keeps no Stream of a Standalone " + f"The backend {stream_backend.backend_name} can't store the Stream of a Standalone " "Activity. Use the native backend." ) if not handle.run_id: diff --git a/temporalio/common/_streams/_workflow.py b/temporalio/common/_streams/_workflow.py index 0a6a968539..aaa8bdc88b 100644 --- a/temporalio/common/_streams/_workflow.py +++ b/temporalio/common/_streams/_workflow.py @@ -184,7 +184,7 @@ def _check_backend(backend: str | None, action: str) -> str: raise no_worker_backend() names = [b.name for b in backends] if not backend: - # The default is the backend with default=True, else native, which every Worker has. + # The default is the backend with default=True, or else native, which every Worker has. return next((b.name for b in backends if b.default), _ids.NATIVE_BACKEND) if backend not in names: raise _errors.StreamConfigError( @@ -389,7 +389,7 @@ def _model_ref(self) -> models.StreamRef: ) def _ref(self) -> stream_pb.StreamRef: - # A server that nests a Stream in its owner finds it only by the owner's fields. + # A server that nests a Stream under its owner finds it only by the owner's fields. return stream_pb.StreamRef( id=self.id, workflow_id=self.workflow_id or "", @@ -435,15 +435,15 @@ def subscribe( ) -> WorkflowItems: """Read the Stream's items in order, in Workflow code. - It is an async iterator, as on the client, and it yields the same ``Record``, - ``Superseded`` and ``Closed`` items. Each page is a Local Activity whose answer is in + It returns an async iterator, as on the client, which yields the same ``Record``, + ``Superseded`` and ``Closed`` items. Each page is a Local Activity whose result is in History, so a replay reads the same items. On a native Stream the marker holds only the page's offsets, and a replay reads the page again from the server. A read at the - end of a native Stream waits on the server until a Record comes or the Stream closes. - Such waits take at most 20 of the run's 30 pending Nexus operations, and a read past - them waits as on Redis. On Redis the marker holds the page, the Local Activity waits a - short time for new Records, and a page with none is followed by a durable timer from - 1 s up to 60 s. + end of a native Stream waits on the server until a Record arrives or the Stream closes. + These waits use at most 20 of the run's 30 pending Nexus operations. A read past that + limit waits as on Redis. On Redis the marker holds the page, and the Local Activity + waits a short time for new Records. After a page with no Records, the read waits on a + durable timer, from 1 s up to 60 s. A read sees this run's writes that it awaited, since each write is a Local Activity that ends before the next read starts. @@ -466,14 +466,14 @@ def subscribe( wait_new_records: Wait for new Records at the end of the Stream. When False, the read ends at the newest Record. topic_prefixes: Also read the Topics whose names start with one of these. - create_wait: How long to wait for a Stream that nobody created yet. Defaults to + create_wait: How long to wait for a Stream that doesn't exist yet. Defaults to no wait. Raises: StreamNotFoundError: The Stream does not exist, or it was not created in time. ReadOnlyContextError: The code must not change state, such as a query handler. - StreamConfigError: The Stream is of a backend that the Worker does not have. It - fails the Workflow Task, not the run. + StreamConfigError: The Worker doesn't have the Stream's backend. It fails the + Workflow Task, not the run. """ if not temporalio.workflow.unsafe.is_read_only(): _state().uses_streams = True @@ -502,7 +502,7 @@ def subscribe( def _check( self, action: str, *, backend: str | None = None ) -> tuple[_RunStream, str]: - # The run's state of the Stream, and the backend that its calls go to. + # The run's state of the Stream, and the backend for its calls. if temporalio.workflow.unsafe.is_read_only(): raise _errors.ReadOnlyContextError( f"Can't {action} a Stream in read-only code" @@ -770,7 +770,7 @@ def _create( ) if retention is not None: request.retention.FromTimedelta(retention) - # The native server closes the Stream when the run chain ends. Redis ignores it. + # The native server closes the Stream when the run chain ends. Redis ignores this flag. request.close_with_run = True run_stream.backend = stream.backend return stream, _write(stream.id, CREATE_ACTIVITY, request, backend_name) diff --git a/temporalio/common/_streams/_workflow_reader.py b/temporalio/common/_streams/_workflow_reader.py index e3e2411727..5dc5beffe4 100644 --- a/temporalio/common/_streams/_workflow_reader.py +++ b/temporalio/common/_streams/_workflow_reader.py @@ -1,6 +1,6 @@ """Reads of a Stream in Workflow code. -Each page is a call whose answer is in History, so a replay reads the same items. A Stream +Each page is a call whose result is in History, so a replay reads the same items. A Stream backend's outbound interceptor reads a page with ``_temporal_stream_read_page``. A backend whose server parks reads also starts a wake with ``_temporal_stream_start_wake``. """ @@ -33,15 +33,15 @@ Item = Record[Any] | Superseded | Closed -# A backend that can't park a read pauses on a durable timer after a page with no Record, -# from the first pause up to the last. +# A backend that can't park a read waits on a durable timer after a page with no Record. The +# pause doubles from the first to the last. _FIRST_PAUSE = timedelta(seconds=1) _LAST_PAUSE = timedelta(seconds=60) -# A run may have 30 pending Nexus operations on the server. Wakes take at most 20, so the run's -# closes and its own operations still fit, and a read past them pauses on a timer. +# A run can have 30 pending Nexus operations. Wakes use at most 20, so the run's closes and its +# own operations still fit. A read past the cap waits on a timer. _MAX_WAKES = 20 # How long a read waits for a close after the Stream's owner ended. The close of a run comes -# from its Worker, so it may come after the run closed, or never. +# from its Worker, so it can arrive after the run closed, or never. _OWNER_END_GRACE = timedelta(minutes=1) @@ -72,7 +72,7 @@ def reader_of(backend: str) -> Any: class _ReadState: # The run's wakes that wait on the server, with their reads, so the run's end cancels them. wakes: dict[Any, WorkflowItems] = field(default_factory=dict) - # The wakes that the server has not taken yet, which count against the cap too. + # The wakes that the server hasn't accepted yet, which count against the cap too. starting: int = 0 # The lowest offset that the run's replayed pages read, per Stream, and whether a live # page sent it. @@ -113,7 +113,7 @@ class WorkflowItems(Subscription[Any]): It ends at a close, or at the newest Record when ``wait_new_records`` is False. It also ends when this run should continue as new, after at least one page, and then :py:attr:`continue_as_new_suggested` is True. Pass - :py:attr:`Subscription.next_offset` to the next run, which reads on from there. + :py:attr:`Subscription.next_offset` to the next run, which continues reading from there. .. warning:: This API is experimental. @@ -140,8 +140,8 @@ def __init__( stream=ref, topics=list(topics or []), topic_prefixes=list(topic_prefixes) ) if offset is End.END: - # END is only the first page's start. Each answer's next offset pins later - # pages and the waits. + # END applies only to the first page. Later pages and the waits start at the last + # page's next offset. self._request.start = stream_pb.ReadStart.READ_START_END elif offset: self._request.from_offset = offset @@ -164,14 +164,14 @@ def __init__( ) self.continue_as_new_suggested = False """Whether the read ended because this run should continue as new. The server - suggests it for the run's History or for the Records that the reader keeps on the - server.""" + suggests it because of the run's History size, or the Records that the reader keeps + on the server.""" self.owner_ended = False - """Whether the read ended because the run chain that owns the Stream ended with no + """Whether the read ended because the run chain that owns the Stream ended without a close, such as a terminated run on Redis.""" async def _pages(self) -> AsyncGenerator[Item, None]: - # Gives the items of each page, and reads or waits when none is left. + # Yields the items of each page, and reads or waits when none are left. while True: while not self._queue: if self._ended or self._closed: @@ -188,7 +188,7 @@ async def _pages(self) -> AsyncGenerator[Item, None]: yield item async def aclose(self) -> None: - """End the read. A wait on the server ends with it.""" + """End the read. This also cancels a wait on the server.""" self._closed = True self._queue.clear() if self._wake is not None: @@ -206,7 +206,7 @@ async def _read(self) -> None: self._started = True if self._create_wait: self._create_until = now + self._create_wait - # A read ends only after a page, so next_offset says where the next run goes on. + # A read ends only after a page, so next_offset is where the next run continues. if self._paged and self._suggests_continue_as_new(): self.continue_as_new_suggested = True self._ended = True @@ -240,8 +240,8 @@ async def _read(self) -> None: await self._wait_for_records() def _waiting_request(self) -> stream_pb.ReadRecordsRequest: - # The next read with the backend's waits. A backend that parks reads on the server - # takes the waits off its pages and puts them on its wakes. + # The next read request, with the backend's waits. A backend that parks reads on the + # server moves the waits from its pages to its wakes. request = stream_pb.ReadRecordsRequest() request.CopyFrom(self._request) if self._wait_new_records: @@ -311,7 +311,7 @@ def _owner_gone(self, page: Page, now: datetime) -> bool: async def _wait_for_records(self) -> None: if not self._can_wake(): - # The Store can't wake a Workflow, or the run has the most wakes. A quiet Stream + # The Store can't wake a Workflow, or the run is at the wake cap. A quiet Stream # then costs one timer and one read per pause. await self._sleep() return @@ -323,7 +323,7 @@ async def _wait_for_create(self) -> None: return request = self._waiting_request() if not self._wait_new_records: - # After the create, the read takes what is there and ends. + # After the create, the read reads what's there and ends. request.wait.FromTimedelta(_FIRST_PAUSE) try: await self._wake_once(request) @@ -344,7 +344,7 @@ async def _sleep(self) -> None: async def _wake_once(self, request: stream_pb.ReadRecordsRequest) -> None: # The server parks the wake until a Record passes the filter, the Stream closes, or - # the wait ends. Its answer holds no Records, and the next page reads them. + # the wait ends. The wake's answer has no Records, and the next page reads them. state = _read_state() state.starting += 1 try: @@ -355,7 +355,7 @@ async def _wake_once(self, request: stream_pb.ReadRecordsRequest) -> None: wakes[wake] = self self._wake = wake if self._closed: - # The iterator closed while the server took the wake. + # The iterator closed while the server was accepting the wake. wake.cancel() try: answer = await wake @@ -363,8 +363,8 @@ async def _wake_once(self, request: stream_pb.ReadRecordsRequest) -> None: if self._closed or isinstance( err.cause, temporalio.exceptions.TimeoutError ): - # A namespace may cap a Nexus operation below the wake's wait, which ends the - # wake as an empty one would. + # A namespace can cap a Nexus operation below the wake's wait. That timeout + # ends the wake as an empty answer would. return raise error_of(err) from None finally: @@ -374,7 +374,7 @@ async def _wake_once(self, request: stream_pb.ReadRecordsRequest) -> None: self._request.start == stream_pb.ReadStart.READ_START_END and answer.next_offset ): - # A wait for the create answers where END was when the Stream came. + # A wait for the create returns the END position at the time of the create. self._request.from_offset = answer.next_offset self._request.ClearField("start") @@ -426,7 +426,7 @@ def workflow_item( if isinstance(failure, temporalio.exceptions.FailureError) and failure.failure: # The model's converter keeps payloads raw for the Topic types. failure = _failure_converter().from_failure(failure.failure, converter) - # A close Record from before close statuses has none, and its failure tells. + # An older close Record has no status, so its failure decides the status. status = model.close_status or ( CloseStatus.FAILED if failure is not None else CloseStatus.COMPLETED ) @@ -457,7 +457,7 @@ def _failure_converter() -> temporalio.converter.FailureConverter: def error_of(err: BaseException) -> BaseException: - """The Stream error that a failed page carries as its cause, or the error itself. + """The Stream error in a failed page's cause, or the error itself. A page's failure is in History, so a replay raises the same error. """ diff --git a/temporalio/worker/_activity.py b/temporalio/worker/_activity.py index f3967e19fd..d7a439e625 100644 --- a/temporalio/worker/_activity.py +++ b/temporalio/worker/_activity.py @@ -83,8 +83,8 @@ def __init__( # Validate and build activity dict self._activities: dict[str, temporalio.activity._Definition] = {} self._dynamic_activity: temporalio.activity._Definition | None = None - # The SDK's own Local Activities, such as those of Streams, which the errors about - # the user's activities leave out. + # The SDK's own Local Activities, such as the Streams ones. The errors about the user's + # activities don't list them. self._internal_activities = { temporalio.activity._Definition.must_from_callable(activity).name for activity in internal_activities diff --git a/temporalio/worker/_worker.py b/temporalio/worker/_worker.py index ccb3bde4e8..07b618e7b6 100644 --- a/temporalio/worker/_worker.py +++ b/temporalio/worker/_worker.py @@ -503,8 +503,8 @@ def _init_from_config(self, client: temporalio.client.Client, config: WorkerConf or temporalio.runtime.Runtime.default() ) activities = config.get("activities") - # The Local Activities that the Worker's Workflows run for Streams. They are the SDK's - # own, so they enable no polling of the task queue's Activities. + # The Local Activities that the Worker's Workflows run for Streams. They're the SDK's + # own, so they don't turn on polling for the task queue's Activities. internal_activities: Sequence[Callable[..., Any]] = [] if config.get("workflows"): from temporalio.common._streams import _plugin as _stream_plugin diff --git a/temporalio/worker/_workflow.py b/temporalio/worker/_workflow.py index 9d1722ffb4..16ed38a4a7 100644 --- a/temporalio/worker/_workflow.py +++ b/temporalio/worker/_workflow.py @@ -184,7 +184,7 @@ def __init__( max_workflow_task_external_storage_concurrency ) self._throw_after_activation: Exception | None = None - # The runs whose last Workflow Task a NondeterminismError of the codec step failed. + # The runs whose last Workflow Task failed with a NondeterminismError from the codec step. self._nondeterministic_runs: set[str] = set() # If debug mode is enabled, disable deadlock detection @@ -478,8 +478,8 @@ async def _handle_activation( else: completion.failed.failure.SetInParent() if isinstance(err, temporalio.workflow.NondeterminismError): - # A codec step may compare the data with what the run saw, such as a - # native Stream read. + # A codec step can compare the data with what the run saw before, such as + # for a native Stream read. self._nondeterministic_runs.add(act.run_id) workflow_type = ( workflow.get_info().workflow_type @@ -489,7 +489,7 @@ async def _handle_activation( else "" ) # Core fails a run for this cause in the middle of the replay, and again at - # the next marker, so only a run that fails the task takes it. + # the next marker. So set it only for a run that fails the task. if not ( self.nondeterminism_as_workflow_fail() or workflow_type @@ -681,8 +681,8 @@ async def _handle_cache_eviction( if act.run_id in self._nondeterministic_runs: self._nondeterministic_runs.discard(act.run_id) if job.reason == _LANG_FAIL: - # Core evicts a run that lang failed as a lang failure, so a hook such as the - # Replayer's would not see the nondeterminism. + # Core evicts a run that lang failed with the lang failure reason, so a hook such + # as the Replayer's wouldn't see the nondeterminism. job = temporalio.bridge.proto.workflow_activation.RemoveFromCache( message=job.message, reason=_NONDETERMINISM_EVICTION ) diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 4f71f3428e..26fec155c5 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -113,7 +113,7 @@ def stream_handle( ``native``. Raises: - StreamConfigError: The Worker has no backend of that name. + StreamConfigError: The Worker has no backend with that name. """ from temporalio.common._streams import _workflow diff --git a/tests/contrib/streams/_support.py b/tests/contrib/streams/_support.py index a66bb1ab2e..5aba69268a 100644 --- a/tests/contrib/streams/_support.py +++ b/tests/contrib/streams/_support.py @@ -44,9 +44,7 @@ def vectors(name: str) -> object: def on_native(client: Any) -> bool: - """Whether a client's default backend is the native server, whose rules differ in - places. - """ + """Whether a client's default backend is the native server, whose rules differ in places.""" from temporalio.common._streams import _plugin name, _ = _plugin.client_backend(client, None) diff --git a/tests/contrib/streams/conftest.py b/tests/contrib/streams/conftest.py index 59c4568d62..495e139a03 100644 --- a/tests/contrib/streams/conftest.py +++ b/tests/contrib/streams/conftest.py @@ -80,9 +80,8 @@ async def _brook_client( @pytest.fixture(params=["memory", "redis", *(["native"] if NATIVE_TARGET else [])]) async def stream_client(client: Client, request: pytest.FixtureRequest) -> Client: - # Every live test runs on the memory Store and on Redis, since both must keep one set of - # rules, and on the native server when one is given. The native backend is every - # client's with no registration, and the default when no backend has default=True. + # Every live test runs on the memory Store, on Redis and, when one is given, on the native + # server. Every client has the native backend, the default when no backend has default=True. if request.param == "native": return await Client.connect(NATIVE_TARGET, namespace=NATIVE_NAMESPACE) return await _brook_client(client, request, request.param) @@ -90,8 +89,8 @@ async def stream_client(client: Client, request: pytest.FixtureRequest) -> Clien @pytest.fixture(params=["memory", "redis"]) async def redis_client(client: Client, request: pytest.FixtureRequest) -> Client: - # A test of the Redis calls, such as temporalio.contrib.stream.create, which the memory - # Store keeps the rules of. + # A test of the Redis functions, such as temporalio.contrib.stream.create. The memory Store + # has the same rules. return await _brook_client(client, request, request.param) diff --git a/tests/contrib/streams/examples/migrate_redis_to_native.py b/tests/contrib/streams/examples/migrate_redis_to_native.py index 43b76e2eeb..07970ac0b2 100644 --- a/tests/contrib/streams/examples/migrate_redis_to_native.py +++ b/tests/contrib/streams/examples/migrate_redis_to_native.py @@ -1,11 +1,11 @@ """Jetty v2, "Move from Redis to native Streams". -Old Streams stay on Redis, and new ones go to the server. The client registers -``RedisStreamBackend`` with ``default=True``, so Workflow code that names no backend keeps -creating on Redis, and the native backend is there with no registration. Step 1 runs the old -code. Step 2 deploys new code that picks ``native`` behind ``workflow.patched``, so a run that -started on the old code replays the same and goes on with its Redis Stream. Redis stays the -default, since a change of the default changes the commands of a replayed create. +Old Streams stay on Redis, and new ones are created on the server. The client registers +``RedisStreamBackend`` with ``default=True``, so Workflow code that names no backend still +creates on Redis. The native backend needs no registration. Step 1 runs the old code. Step 2 +deploys new code that picks ``native`` behind ``workflow.patched``, so a run that started on the +old code replays the same and continues with its Redis Stream. Redis stays the default, since a +change of the default changes the commands of a replayed create. """ from __future__ import annotations @@ -63,7 +63,7 @@ async def connect(target_host: str, redis_url: str) -> Client: async def read(handle: WorkflowHandle[Any, None]) -> list[str]: await handle.result() - # A UI that reads chats from both sides of the move looks on native first, then Redis. + # A UI that reads chats from both sides of the move checks native first, then Redis. for backend in ("native", "redis"): stream = handle.stream(Events, backend=backend) try: @@ -81,7 +81,7 @@ async def read(handle: WorkflowHandle[Any, None]) -> list[str]: async def main(client: Client, task_queue: str) -> list[str]: - # Step 1: the old code starts a chat on Redis. + # Step 1. The old code starts a chat on Redis. async with Worker(client, task_queue=task_queue, workflows=[ChatBeforeTheMove]): old = await client.start_workflow( ChatBeforeTheMove.run, id=f"chat-{uuid.uuid4()}", task_queue=task_queue @@ -89,7 +89,7 @@ async def main(client: Client, task_queue: str) -> list[str]: async for _ in old.stream(Events).subscribe(create_wait=timedelta(seconds=30)): # The old chat wrote its first Record, so it runs on the old code. break - # Step 2: the new code goes on with the old chat, and starts a new one on the server. + # Step 2. The new code continues the old chat, and starts a new one on the server. async with Worker(client, task_queue=task_queue, workflows=[Chat]): await old.signal(ChatBeforeTheMove.more, "old chat goes on") new = await client.start_workflow( diff --git a/tests/contrib/streams/examples/standalone_activity_stream.py b/tests/contrib/streams/examples/standalone_activity_stream.py index 7765a0ae61..c2367dc3c9 100644 --- a/tests/contrib/streams/examples/standalone_activity_stream.py +++ b/tests/contrib/streams/examples/standalone_activity_stream.py @@ -1,8 +1,9 @@ """Jetty v2, "A Standalone Activity streams its progress". A Standalone Activity, one that no Workflow runs, owns a Stream on the native backend with -``activity.create_stream``. A client reads it from the Activity handle with no call. The server -closes the Stream when the Activity ends, with its result, so the Activity need not close it. +``activity.create_stream``. A client gets it from the Activity handle without a call. The server +closes the Stream when the Activity ends, with its result, so the Activity doesn't need to close +it. """ from __future__ import annotations diff --git a/tests/contrib/streams/examples/test_examples.py b/tests/contrib/streams/examples/test_examples.py index 28daeb1ac6..7acd33055c 100644 --- a/tests/contrib/streams/examples/test_examples.py +++ b/tests/contrib/streams/examples/test_examples.py @@ -105,9 +105,8 @@ async def test_worker_setup_on_redis(client: Client, redis_url: str): assert await worker_setup.create_on_second_backend(migrating) == "redis-b" -# The Jetty v2 examples. The Workflow reader runs on one backend: the memory Store, Redis or -# the native server. The others need the native server, with Redis, or a memory Store named -# `redis` in its place, so they run only when TEMPORAL_STREAMS_NATIVE_TARGET is set. +# The Workflow reader runs on the memory Store, Redis or native. The other Jetty v2 examples need +# native and a Store named `redis`, so they run only when TEMPORAL_STREAMS_NATIVE_TARGET is set. async def native_server(*backends: common.StreamBackend) -> Client: if not NATIVE_TARGET: pytest.skip( diff --git a/tests/contrib/streams/examples/two_backends.py b/tests/contrib/streams/examples/two_backends.py index 6f158d037a..39ca732354 100644 --- a/tests/contrib/streams/examples/two_backends.py +++ b/tests/contrib/streams/examples/two_backends.py @@ -1,10 +1,10 @@ """Jetty v2, "Two backends in one Worker". The client registers ``RedisStreamBackend``, and has the native backend with no registration. -The Worker takes both from it. Each Stream handle carries its backend name, so every call goes -to the backend that keeps the Stream. A create with no ``backend=`` goes to the default one, -here native, since the Redis backend has no ``default=True``. Tests swap Redis for a -``MemoryStreamBackend(name="redis")`` with no change to Workflow code. +The Worker uses both. Each Stream handle stores its backend name, so every call goes to the +backend that stores the Stream. A create with no ``backend=`` uses the default backend, which +here is native, since the Redis backend has no ``default=True``. Tests swap Redis for a +``MemoryStreamBackend(name="redis")`` without changing Workflow code. """ from __future__ import annotations @@ -60,8 +60,8 @@ async def run(self) -> str: ), start_to_close_timeout=timedelta(seconds=30), ) - # The server closes the audit log when the run chain ends. The end of a run closes - # nothing on Redis, so the run closes the live view. + # The server closes the audit log when the run chain ends. On Redis, a run's end doesn't + # close the Stream, so the run closes the live view. await live.close(result=result) return result diff --git a/tests/contrib/streams/examples/workflow_reader.py b/tests/contrib/streams/examples/workflow_reader.py index fc2a3e7621..f6750f6277 100644 --- a/tests/contrib/streams/examples/workflow_reader.py +++ b/tests/contrib/streams/examples/workflow_reader.py @@ -1,6 +1,6 @@ """Jetty v2, "A Workflow reads another Workflow's Stream". -An orchestrator Workflow follows the turns of an agent Workflow. It gets the agent's Stream as +An orchestrator Workflow reads the turns of an agent Workflow. It gets the agent's Stream as an argument and reads it with ``subscribe``, an async iterator in Workflow code. Each page is in History, so a replay reads the same items. On a native Stream a read at the end waits on the server. On Redis it waits on a Local Activity, then on a durable timer. The Workflow code is @@ -28,7 +28,7 @@ class Agent: async def run(self, prompt: str) -> str: stream = await workflow.create_stream(Events) for word in prompt.split(): - # Each word goes out in its own Workflow Task, so the follower waits for some. + # Each word is written in its own Workflow Task, so the reader waits for some. await workflow.sleep(timedelta(milliseconds=200)) await stream.topic(Events.tokens).publish(word) # The server would close a native Stream when the run chain ends. A Redis Stream diff --git a/tests/contrib/streams/test_backends.py b/tests/contrib/streams/test_backends.py index ab7a739a82..fe558eeab6 100644 --- a/tests/contrib/streams/test_backends.py +++ b/tests/contrib/streams/test_backends.py @@ -94,8 +94,8 @@ async def read_all( async def test_one_worker_serves_two_backends(client: Client, redis_url: str): - # Each call goes to the backend that the handle names: Workflow buffers, Activity - # writes, a Local Activity's flush, retry withdrawals and the closes. + # Each call goes to the backend that the handle names. That includes Workflow buffers, + # Activity writes, a Local Activity's flush, retry withdrawals and the closes. two = two_backend_client(client, redis_url) async with new_worker( two, WritesBoth, activities=[write_both, write_memory_locally] @@ -203,7 +203,7 @@ async def run(self, again: bool) -> str: async def test_continued_run_closes_each_backend(client: Client, redis_url: str): - # The next run learns the backend of each Stream that the chain created. + # The next run gets the backend of each Stream that the chain created. two = two_backend_client(client, redis_url) async with new_worker(two, ContinuesOnBoth) as worker: handle = await two.start_workflow( diff --git a/tests/contrib/streams/test_native_server.py b/tests/contrib/streams/test_native_server.py index 5187e82663..85f6e0935d 100644 --- a/tests/contrib/streams/test_native_server.py +++ b/tests/contrib/streams/test_native_server.py @@ -1,4 +1,4 @@ -"""The native lane: the same calls against a server that serves the Stream service. +"""The native lane, which makes the same calls against a server with the Stream service. Set ``TEMPORAL_STREAMS_NATIVE_TARGET`` to its address to run it. """ @@ -111,8 +111,8 @@ async def test_native_workflow_reads_and_replays(native_client: Client): if event.HasField("nexus_operation_completed_event_attributes") ] assert wakes and not any(wake.records for wake in wakes) - # A replay reads each page again on the connection of its stream_client, once the cache - # is gone. + # A replay reads each page again on the connection of its stream_client, after the cache + # is cleared. replayer = Replayer(workflows=[Writer, Reader], stream_client=native_client) await replayer.replay_workflow(await writer.fetch_history()) await replayer.replay_workflow(history) @@ -195,8 +195,8 @@ async def run(self, stream_id: str) -> str: async def test_a_write_without_the_owner_fields_raises(native_client: Client): - # A Workflow's Stream nests in its Workflow, so a handle without the owner fields finds - # nothing, and the write's Local Activity raises at the await. + # The server nests a Workflow's Stream under its Workflow, so a handle without the owner + # fields finds nothing, and the write's Local Activity raises at the await. async with new_worker(native_client, Writer, WritesByBareId) as worker: writer = await native_client.start_workflow( Writer.run, None, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue @@ -267,7 +267,7 @@ async def test_a_native_read_from_the_end_keeps_what_comes_after_its_first_page( native_client: Client, ): # The server resolves END on each call, so the wait and the read after it start where the - # first page ended. Else the Record that wakes the reader is behind its start. + # first page ended. Otherwise the Record that wakes the reader would be before its start. stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("old") async with new_worker(native_client, ReadsNewTokens) as worker: @@ -306,7 +306,7 @@ def stop(self) -> None: async def test_closing_a_native_read_cancels_its_wake(native_client: Client): - # A wake may park for days, so a read that its code closed must not keep it. + # A wake can park for days, so closing the read must cancel it. stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") async with new_worker(native_client, ClosesItsRead) as worker: reader = await native_client.start_workflow( @@ -386,7 +386,7 @@ async def continued() -> None: await until_parked(latest) await latest.signal(EndsWhileItReads.end) assert await latest.result() == "done" - # Nothing comes after a run's close, so the server took each cancel while the run was open. + # Nothing comes after a run's close, so the server accepted each cancel while the run was open. for run_id in (first.first_execution_run_id, latest.result_run_id): history = await native_client.get_workflow_handle( first.id, run_id=run_id @@ -471,10 +471,10 @@ async def test_a_workflow_reader_keeps_its_range_of_a_client_stream( native_client: Client, ): # The size cap would drop what the reader's replays read again, and every task of the run - # would fail. So the server refuses the append and suggests that the reader move on. + # would fail. So the server refuses the append and suggests that the reader continue as new. stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("a") - # Two of them keep more than half of the 4 MiB stream.maxStoredBytes, three pass it. + # Two of them keep more than half of the 4 MiB stream.maxStoredBytes, three exceed it. big = "x" * 1_450_000 async with new_worker( native_client, ReadsUntilToldToContinue, max_cached_workflows=0 @@ -511,8 +511,8 @@ async def leaves_its_stream_open() -> str: async def test_a_standalone_activity_owns_its_stream(native_client: Client): - # The Activity may close its own Stream. The server closes it when the Activity ends, with - # its status and result, so a Stream that the Activity leaves open doesn't stay. + # The Activity can close its own Stream. If it doesn't, the server closes the Stream when + # the Activity ends, with its status and result. task_queue = f"tq-{uuid.uuid4()}" async with new_worker( native_client, @@ -538,7 +538,7 @@ async def test_a_standalone_activity_owns_its_stream(native_client: Client): "a", "b", ] - # The Activity's own close got past the early owner check, so its close has no result. + # The Activity's own close passed the SDK's owner check, so the Closed item has no result. assert isinstance(items[-1], common.Closed) and items[-1].result is None left = [item async for item in left_open.stream(Events).subscribe()] assert [item.data for item in left if isinstance(item, common.Record)] == ["a"] @@ -579,7 +579,7 @@ async def test_a_block_end_leaves_a_standalone_activitys_stream_open( assert "attempt 2" in [ item.data for item in items if isinstance(item, common.Record) ] - # The server's close carries the result of the attempt that ended the Activity. + # The server's close has the result of the attempt that ended the Activity. assert isinstance(items[-1], common.Closed) assert items[-1].result == "done on attempt 2" @@ -596,7 +596,7 @@ async def run(self, stream: common.WorkflowStream[Events]) -> list[Any]: async def test_a_workflow_reads_a_standalone_activitys_stream(native_client: Client): - # The server nests the Stream in its Activity, so the read names the Activity. + # The server nests the Stream under its Activity, so the read names the Activity. task_queue = f"tq-{uuid.uuid4()}" async with new_worker( native_client, @@ -662,7 +662,7 @@ async def test_a_replay_that_reads_other_records_is_nondeterministic( native_client: Client, monkeypatch: pytest.MonkeyPatch ): # The codec step checks a replayed page against its digest, outside the Workflow. The - # server and the Replayer still learn that the replay was nondeterministic. + # server and the Replayer still see that the replay was nondeterministic. stream = await native_client.create_stream(Events, id=f"native-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("a", "b") async with new_worker( @@ -693,8 +693,8 @@ async def test_a_replay_that_reads_other_records_is_nondeterministic( async def test_each_client_of_a_native_backend_calls_on_its_own_connection(): - # One backend on two clients gives each its own runtime, so a call doesn't go on the - # connection of the client that came last. + # One backend on two clients gives each its own runtime, so a call doesn't use the + # connection of the last client. native = common.NativeStreamBackend() first = await Client.connect("127.0.0.1:1", lazy=True, stream_backends=[native]) second = await Client.connect("127.0.0.1:2", lazy=True, stream_backends=[native]) @@ -716,8 +716,8 @@ async def test_each_client_of_a_native_backend_calls_on_its_own_connection(): async def test_a_native_backend_keeps_no_client_alive(): - # The implicit backend lives as long as the process, and every Worker's client goes - # through it. + # The implicit backend exists for the life of the process, and every Worker's client uses + # it. native = common.NativeStreamBackend() registered = await Client.connect( "127.0.0.1:1", lazy=True, stream_backends=[native] diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 6afbef32af..1b713b5dda 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -880,7 +880,7 @@ async def run(self) -> str: async def close_activity_events( handle: WorkflowHandle[Any, Any], ) -> list[EventType.ValueType]: - # The events of the close command, from its schedule on: the Activity on Redis, and the + # The events of the close command, from its schedule on. It's the Activity on Redis, and the # System Nexus operation on native. history = await handle.fetch_history() scheduled = { diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 3cf152c341..6debe33664 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -237,8 +237,8 @@ async def test_client_reads_a_workflow_stream(stream_client: Client): async def test_client_creates_a_stream_on_its_default_backend(stream_client: Client): - # A client owns the Stream it creates, as a Workflow and a Standalone Activity own theirs. - # It goes on the default backend: native, unless a registered backend has default=True. + # A client owns the Stream that it creates, as a Workflow and a Standalone Activity own theirs. + # It's on the default backend, which is native unless a registered backend has default=True. lane, _ = _plugin.client_backend(stream_client, None) stream_id = f"own-{uuid.uuid4()}" stream = await stream_client.create_stream(Events, id=stream_id) @@ -451,7 +451,7 @@ async def test_async_with_refuses_a_workflow_stream_before_the_block( async def test_async_with_refuses_an_activity_stream_before_the_block( stream_client: Client, ): - # Only the Standalone Activity closes its Stream, so the block end would raise too. + # Only the Standalone Activity can close its Stream, so the block end would raise too. handle = stream_client.get_activity_handle("tool-1", run_id="run-1") ran = False with pytest.raises(common.StreamNotOwnerError, match="an Activity owns it"): @@ -657,7 +657,7 @@ async def test_default_is_native_with_no_default_backend(client: Client): assert one.stream(Events, id="d-2", backend="memory").backend == "memory" # The Redis create picks the client's only Redis backend. The memory backend counts as one. assert (await redis.create(one, Events, id="d-2")).backend == "memory" - # A registered NativeStreamBackend configures native, and only it takes that name. + # A registered NativeStreamBackend configures native, and only it can use that name. config["stream_backends"] = [ common.NativeStreamBackend(default=True), MemoryStreamBackend(), diff --git a/tests/contrib/streams/test_workflow_closes.py b/tests/contrib/streams/test_workflow_closes.py index 901cc3b4e8..bd7d92907c 100644 --- a/tests/contrib/streams/test_workflow_closes.py +++ b/tests/contrib/streams/test_workflow_closes.py @@ -59,10 +59,11 @@ def go(self) -> None: async def test_a_close_waits_for_its_workflow_task_to_commit(stream_client: Client): - # A close runs only once its Workflow Task commits, so a task that fails after it leaves + # A close runs only after its Workflow Task commits, so a task that fails after it leaves # the Stream open, and the retry closes it once. key = str(uuid.uuid4()) - # The test's set must survive the retry of the task, so no sandbox. + # The test's set must keep its value across the task's retry, so the Workflow has no + # sandbox. async with new_worker( stream_client, ClosesThenFailsItsTask, diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index b6f7b1a3e7..8c1f07c3c3 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -837,7 +837,7 @@ async def test_merge_that_the_codec_grows_past_the_batch_limit(stream_client: Cl # The two publishes merge under the Workflow's cap. The codec makes the merge larger than the # batch limit, so each publish is sent as its own batch. client = with_codec(stream_client, _GrowingCodec()) - # The native server takes half the bytes of one append, so each publish there is smaller. + # The native server accepts half the bytes per append, so each publish there is smaller. size = 150_000 if on_native(stream_client) else 300_000 async with new_worker(client, PublishesLargeValues) as worker: handle = await client.start_workflow( diff --git a/tests/contrib/streams/test_workflow_reads.py b/tests/contrib/streams/test_workflow_reads.py index e001f1353e..127b1085db 100644 --- a/tests/contrib/streams/test_workflow_reads.py +++ b/tests/contrib/streams/test_workflow_reads.py @@ -41,7 +41,7 @@ async def run(self, backend: str | None) -> str: stream = await workflow.create_stream(Events, backend=backend) await stream.topic(Events.turns).publish(Turn("user", "hi")) for token in ("a", "b", "c"): - # Each token goes out in its own Workflow Task, so the reader waits for some. + # Each token is written in its own Workflow Task, so the reader waits for some. await workflow.sleep(0.3) await stream.topic(Events.tokens).publish(token) await stream.close(result="written") @@ -172,7 +172,7 @@ def __init__(self) -> None: @workflow.run async def run(self) -> None: - # Set in the first task, since a query may come before a create's Local Activity ends. + # Set in the first task, since a query can arrive before a create's Local Activity ends. self._stream = workflow.create_stream_nowait(Events) await workflow.wait_condition(lambda: False) @@ -181,7 +181,7 @@ def read(self) -> str: assert self._stream is not None items = self._stream.subscribe() try: - # A query can't await, so it takes the read's first step by hand. + # A query can't await, so it runs the read's first step by hand. cast(Coroutine[Any, Any, Any], items.__anext__()).send(None) except common.ReadOnlyContextError as err: return str(err) @@ -286,7 +286,7 @@ async def run(self, stream: common.WorkflowStream[Events]) -> int: async def test_a_workflow_page_stays_below_the_marker_limit(stream_client: Client): # A Redis page is a History marker, and the server terminates a run whose marker is over - # 2 MiB. So a page of large Records splits. + # 2 MiB. So a page of large Records is split. stream = await stream_client.create_stream(Events, id=f"big-{uuid.uuid4()}") for _ in range(25): await stream.topic(Events.tokens).publish("x" * 100_000) @@ -337,8 +337,8 @@ async def test_a_workflow_read_gives_the_close_status_and_failure( async def test_a_backend_that_lacks_a_capability_fails_the_task( stream_client: Client, monkeypatch: pytest.MonkeyPatch ): - # Such as a native server with direct Nexus calls off. A Worker or a server with the - # capability can retry the task, while a failed run stays failed. + # A native server with direct Nexus calls off is such a backend. A Worker or a server with + # the capability can retry the task, but a failed run stays failed. stream = await stream_client.create_stream(Events, id=f"cap-{uuid.uuid4()}") await stream.topic(Events.tokens).publish("a") call_async = _backend._ExtensionCalls.call_async @@ -422,7 +422,7 @@ async def call_async(_method: str, request: Any, _response: Any) -> Any: return answer backend.call_async = call_async - # The live task takes the page from the cache, and a replay reads it again. + # The live task gets the page from the cache, and a replay reads it again. for _ in range(2): [page] = await codec.decode([positions]) assert stream_pb.ReadRecordsResponse.FromString(page.data) == live From 415fb36a6d7b3ed175c357a3482e32369089818f Mon Sep 17 00:00:00 2001 From: Mohammad Dashti Date: Sun, 25 Oct 2026 11:40:05 -0700 Subject: [PATCH 79/79] Made WorkflowTopic.publish start the write and return its future. The write takes its place in the Stream's order at the call, also with no await. An error that no caller gets raises at the run's end, as with publish_nowait. --- temporalio/common/_streams/_workflow.py | 64 +++++++++--- temporalio/workflow/_streams.py | 3 +- tests/contrib/streams/test_workflow_module.py | 99 ++++++++++++++++++- 3 files changed, 150 insertions(+), 16 deletions(-) diff --git a/temporalio/common/_streams/_workflow.py b/temporalio/common/_streams/_workflow.py index 1cd655ab02..bd3955e0a4 100644 --- a/temporalio/common/_streams/_workflow.py +++ b/temporalio/common/_streams/_workflow.py @@ -108,6 +108,9 @@ class _RunState: deferred_closes: list[tuple[WorkflowStream[Any], BaseException | None]] = field( default_factory=list ) + # The publishes that failed before a caller got the error. A caller can still get it, so + # the run's end checks them again. + failed_publishes: list[asyncio.Future[int]] = field(default_factory=list) def stream(self, ref: models.StreamRef) -> _RunStream: known = self.streams.get(ref.id) @@ -297,7 +300,8 @@ def _write_nowait(future: asyncio.Future[int]) -> None: runtime = temporalio.workflow._Runtime.current() def done(future: asyncio.Future[int]) -> None: - # No caller awaits the write, so its error fails the Workflow Task. + # No caller awaits the write, so the run raises its error after this callback, as an + # error that the run doesn't catch. if not future.cancelled() and future.exception() is not None: setattr(runtime, "_current_activation_error", future.exception()) @@ -305,13 +309,27 @@ def done(future: asyncio.Future[int]) -> None: async def drain() -> None: - """Wait for the run's writes and closes that nobody awaits, before the run ends.""" - for task in list(_state().tails.values()): + """Wait for the run's writes and closes that nobody awaits, before the run ends. + + A publish error that no caller got raises here, as an error that the run doesn't catch. + """ + state = _state() + for task in list(state.tails.values()): try: await task except BaseException: # A cancelled write already cancelled its callers. pass + for future in state.failed_publishes: + error = future.exception() if _unretrieved(future) else None + if error is not None: + raise error + + +def _unretrieved(future: asyncio.Future[int]) -> bool: + # asyncio sets this flag when the future fails and clears it when a caller gets the error, + # with await, result() or exception(). It is what asyncio logs as never retrieved. + return getattr(future, "_log_traceback", False) def _payload(value: Any) -> Any: @@ -326,30 +344,47 @@ class WorkflowTopic(TopicBase[T]): This API is experimental. """ - async def publish(self, *values: T) -> int: - """Write one Record for each value, in order, and return the last Record's offset. + def publish(self, *values: T) -> asyncio.Future[int]: + """Start the write of one Record for each value, in order, and return its future. + + The write has its place in the Stream's order when the call returns, also when no + caller awaits the future. Writes to one Stream keep the order of the calls. The write + is a Local Activity, so History has its result and not the values. Publishes that wait + behind a write to the same Stream are sent as one Local Activity, and each call gets the + offset of its own last Record. A list is one value. - The write is a Local Activity, so History has its result and not the values. Publishes - that wait behind a write to the same Stream are sent as one Local Activity, and each - call gets the offset of its own last Record. A list is one value. Writes to one Stream - keep the order of the calls. + A caller that awaits the future, or calls its ``result()`` or ``exception()``, gets the + write's error and owns it. An error that no caller gets raises at the end of the run, as + an error of :py:meth:`publish_nowait` does. So a ``StreamError`` fails the run. + + Returns: + The future of the write. It gives the offset of the last Record, or raises + ``StreamError`` when the backend refused the write, such as + ``StreamClosedError`` for a Stream that another caller closed. Raises: - StreamError: The backend refused the write, such as ``StreamClosedError`` for a - Stream that another caller closed. StreamClosedError: This run closed the Stream. ReadOnlyContextError: The code must not change state, such as a query handler. StreamConfigError: The Worker has no Stream backend, or not the Stream's backend. It fails the Workflow Task, not the run. """ - return await self._stream._append(self._name, values) + state = _state() + future = self._stream._append(self._name, values) + + def failed(future: asyncio.Future[int]) -> None: + if _unretrieved(future): + state.failed_publishes.append(future) + + future.add_done_callback(failed) + return future def publish_nowait(self, *values: T) -> None: """Write one Record for each value, in order, without an await. Use it in sync Workflow code, such as ``@workflow.init`` and sync signal and update handlers. It's like ``asyncio.Queue.put_nowait`` for code that can't await ``put``. A - backend error fails the Workflow Task. + backend error raises in the run as an error that the run doesn't catch, so a + ``StreamError`` fails the run. """ _write_nowait(self._stream._append(self._name, values)) @@ -464,7 +499,8 @@ def close_nowait( """Close the Stream without an await, for sync Workflow code. It is to :py:meth:`close` what ``asyncio.Queue.put_nowait`` is to ``put``. It starts the - close, and the end of the run waits for it. A backend error fails the Workflow Task. + close, and the end of the run waits for it. A backend error raises in the run as an + error that the run doesn't catch, so a ``StreamError`` fails the run. """ _write_nowait(self._close(result, failure)) diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 8eb77b71dc..856f517264 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -80,7 +80,8 @@ def create_stream_nowait( Use it in sync Workflow code, such as ``@workflow.init``. It's like ``asyncio.Queue.put_nowait`` for code that can't await ``put``. It takes the options of - :py:func:`create_stream`. A backend error fails the Workflow Task. + :py:func:`create_stream`. A backend error raises in the run as an error that the run + doesn't catch, so a ``StreamError`` fails the run. .. warning:: This API is experimental. diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index d30a0948b7..f32d1fc244 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -18,7 +18,7 @@ WorkflowHandle, ) from temporalio.common import RetryPolicy -from temporalio.common._streams import _writes +from temporalio.common._streams import _workflow, _writes from temporalio.common._streams._backend import _ExtensionCalls from temporalio.common._streams._defn import definition_of from temporalio.contrib import stream as redis @@ -636,6 +636,103 @@ async def test_init_creates_and_publishes_with_no_await(stream_client: Client): assert isinstance(items[-1], common.Closed) and items[-1].result == "done" +@workflow.defn +class PublishesWithNoAwait: + @workflow.run + async def run(self) -> list[int]: + tokens = (await workflow.create_stream(Events)).topic(Events.tokens) + first = tokens.publish("a") + second = tokens.publish("b") + last = await tokens.publish("c") + # Nobody awaits this write, and the run's end still waits for it. + tokens.publish("d") + return [first.result(), second.result(), last] + + +async def test_publish_starts_the_write_and_returns_its_future(stream_client: Client): + async with new_worker(stream_client, PublishesWithNoAwait) as worker: + handle = await stream_client.start_workflow( + PublishesWithNoAwait.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + offsets = await handle.result() + items = await read(handle) + written = [item for item in items if isinstance(item, common.Record)] + assert [r.data for r in written] == ["a", "b", "c", "d"] + assert offsets == [r.offset for r in written[:3]] + + +@workflow.defn +class LeavesAFailedPublish: + @workflow.run + async def run(self, stream_id: str, retrieve: bool) -> str: + # Nobody created the Stream, so the backend refuses the write. + missing = common.WorkflowStream(id=stream_id).topic(Events.tokens) + future = missing.publish("lost") + errors: list[BaseException | None] = [] + if retrieve: + future.add_done_callback(lambda done: errors.append(done.exception())) + await workflow.wait_condition(lambda: bool(errors)) + return "done" + + +async def test_publish_error_that_no_caller_gets_raises_at_the_run_end( + stream_client: Client, +): + # The error raises as if the run didn't catch it, as with publish_nowait. + async with new_worker(stream_client, LeavesAFailedPublish) as worker: + lost = await stream_client.start_workflow( + LeavesAFailedPublish.run, + args=[f"missing-{uuid.uuid4()}", False], + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + with pytest.raises(WorkflowFailureError) as failed: + await lost.result() + assert isinstance(failed.value.cause, ApplicationError) + assert failed.value.cause.type == "StreamNotFoundError" + # A done callback that reads the error owns it, so the run completes. + owned = await stream_client.start_workflow( + LeavesAFailedPublish.run, + args=[f"missing-{uuid.uuid4()}", True], + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await owned.result() == "done" + + +def test_asyncio_flags_a_future_error_that_no_caller_got(): + # The run's end reads this asyncio flag, so pin it for both Future classes. + loop = asyncio.new_event_loop() + try: + for future_class in (asyncio.Future, getattr(asyncio.futures, "_PyFuture")): + unread: asyncio.Future[int] = future_class(loop=loop) + unread.set_exception(ValueError("refused")) + assert _workflow._unretrieved(unread) + unread.exception() + assert not _workflow._unretrieved(unread) + by_result: asyncio.Future[int] = future_class(loop=loop) + by_result.set_exception(ValueError("refused")) + with pytest.raises(ValueError): + by_result.result() + assert not _workflow._unretrieved(by_result) + awaited: asyncio.Future[int] = future_class(loop=loop) + loop.call_soon(awaited.set_exception, ValueError("refused")) + + async def wait(future: asyncio.Future[int]) -> None: + with pytest.raises(ValueError): + await future + + loop.run_until_complete(wait(awaited)) + assert not _workflow._unretrieved(awaited) + cancelled: asyncio.Future[int] = future_class(loop=loop) + cancelled.cancel() + assert not _workflow._unretrieved(cancelled) + finally: + loop.close() + + @workflow.defn class Gathers: @workflow.run