diff --git a/changelog/added/linking-lynx.md b/changelog/added/linking-lynx.md new file mode 100644 index 0000000000..61d7b5e3fc --- /dev/null +++ b/changelog/added/linking-lynx.md @@ -0,0 +1 @@ +Experimental `temporalio.contrib.streams`: a producer writes `LINK` records with `link()` and `join()` to name a child Workflow's stream, and `StreamHandle.follow()` reads a stream and every stream its links open in one order, from client or Activity code. Each batch end gives a `FollowPosition` that resumes every open link with no loss and no repeat. diff --git a/scripts/gen_protos.py b/scripts/gen_protos.py index f7f8edca87..b1924f2fca 100644 --- a/scripts/gen_protos.py +++ b/scripts/gen_protos.py @@ -47,6 +47,9 @@ def find_streams_proto_dir() -> Path: streams_proto_dir = find_streams_proto_dir() +# The stream API that the crate carries until it lands upstream. Core's api_upstream does not have +# it, so its modules go into temporalio.api from here. +streams_api_proto_dir = streams_proto_dir / "api_stream" # Exclude testsrv dependencies protos proto_paths = [ @@ -58,6 +61,7 @@ def find_streams_proto_dir() -> Path: ] # The crate's api_include copy is left out, since api_upstream generates those modules. proto_paths.extend((streams_proto_dir / "temporal").glob("**/*.proto")) +proto_paths.extend(streams_api_proto_dir.glob("**/*.proto")) proto_paths.extend(test_proto_dir.glob("**/*.proto")) proto_paths.extend(additional_proto_dir.glob("**/*.proto")) @@ -216,6 +220,7 @@ def generate_protos(output_dir: Path): sys.executable, "-mgrpc_tools.protoc", f"--proto_path={api_proto_dir}", + f"--proto_path={streams_api_proto_dir}", f"--proto_path={api_cloud_proto_dir}", f"--proto_path={core_proto_dir}", f"--proto_path={testsrv_proto_dir}", diff --git a/temporalio/api/enums/v1/__init__.py b/temporalio/api/enums/v1/__init__.py index 518a2f32bc..cd3b4d20cc 100644 --- a/temporalio/api/enums/v1/__init__.py +++ b/temporalio/api/enums/v1/__init__.py @@ -44,6 +44,7 @@ from .query_pb2 import QueryRejectCondition, QueryResultType from .reset_pb2 import ResetReapplyExcludeType, ResetReapplyType, ResetType from .schedule_pb2 import ScheduleOverlapPolicy +from .stream_pb2 import StreamOwnerKind from .task_queue_pb2 import ( BuildIdTaskReachability, DescribeTaskQueueMode, @@ -120,6 +121,7 @@ "Severity", "SignalExternalWorkflowExecutionFailedCause", "StartChildWorkflowExecutionFailedCause", + "StreamOwnerKind", "SuggestContinueAsNewReason", "TaskQueueKind", "TaskQueueType", diff --git a/temporalio/api/enums/v1/stream_pb2.py b/temporalio/api/enums/v1/stream_pb2.py new file mode 100644 index 0000000000..5571d5c4d8 --- /dev/null +++ b/temporalio/api/enums/v1/stream_pb2.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporal/api/enums/v1/stream.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import enum_type_wrapper + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b"\n\"temporal/api/enums/v1/stream.proto\x12\x15temporal.api.enums.v1*\x9d\x01\n\x0fStreamOwnerKind\x12!\n\x1dSTREAM_OWNER_KIND_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTREAM_OWNER_KIND_WORKFLOW\x10\x01\x12\x1e\n\x1aSTREAM_OWNER_KIND_ACTIVITY\x10\x02\x12'\n#STREAM_OWNER_KIND_WORKFLOW_ACTIVITY\x10\x03\x42\x83\x01\n\x18io.temporal.api.enums.v1B\x0bStreamProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3" +) + +_STREAMOWNERKIND = DESCRIPTOR.enum_types_by_name["StreamOwnerKind"] +StreamOwnerKind = enum_type_wrapper.EnumTypeWrapper(_STREAMOWNERKIND) +STREAM_OWNER_KIND_UNSPECIFIED = 0 +STREAM_OWNER_KIND_WORKFLOW = 1 +STREAM_OWNER_KIND_ACTIVITY = 2 +STREAM_OWNER_KIND_WORKFLOW_ACTIVITY = 3 + + +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b"\n\030io.temporal.api.enums.v1B\013StreamProtoP\001Z!go.temporal.io/api/enums/v1;enums\252\002\027Temporalio.Api.Enums.V1\352\002\032Temporalio::Api::Enums::V1" + _STREAMOWNERKIND._serialized_start = 62 + _STREAMOWNERKIND._serialized_end = 219 +# @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/enums/v1/stream_pb2.pyi b/temporalio/api/enums/v1/stream_pb2.pyi new file mode 100644 index 0000000000..b2eb6b8810 --- /dev/null +++ b/temporalio/api/enums/v1/stream_pb2.pyi @@ -0,0 +1,77 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import builtins +import sys +import typing + +import google.protobuf.descriptor +import google.protobuf.internal.enum_type_wrapper + +if sys.version_info >= (3, 10): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class _StreamOwnerKind: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _StreamOwnerKindEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + _StreamOwnerKind.ValueType + ], + builtins.type, +): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + STREAM_OWNER_KIND_UNSPECIFIED: _StreamOwnerKind.ValueType # 0 + STREAM_OWNER_KIND_WORKFLOW: _StreamOwnerKind.ValueType # 1 + """A Workflow owns the stream. The Workflow closes the stream itself, with a result or a + failure. If the Workflow's run chain ends with no close and callbacks are attached, the + stream notifier closes the stream with a failure within `streamnotifier.ownerCheckInterval`. + """ + STREAM_OWNER_KIND_ACTIVITY: _StreamOwnerKind.ValueType # 2 + """A standalone Activity owns the stream. The Activity closes the stream itself, with a result + or a failure. If the Activity run reaches a terminal status with no close and callbacks + are attached, the stream notifier closes the stream within + `streamnotifier.ownerCheckInterval`. A completed run gives no result. A failed, canceled, + terminated or timed-out run gives the run's own failure. + """ + STREAM_OWNER_KIND_WORKFLOW_ACTIVITY: _StreamOwnerKind.ValueType # 3 + """An Activity that a Workflow scheduled owns the stream. `workflow_id` names the Workflow, + `run_id` the run that scheduled the Activity, and `activity_id` the Activity. The stream + closes when the Workflow sees the Activity end. If the Activity is no longer pending in the + run and no close came, the stream notifier closes the stream with a failure, after + `streamnotifier.ownerStartGrace` from its start and `streamnotifier.ownerCloseGrace` from + the first check that found the Activity gone. + """ + +class StreamOwnerKind(_StreamOwnerKind, metaclass=_StreamOwnerKindEnumTypeWrapper): + """What owns a stream. The owner decides when the stream closes.""" + +STREAM_OWNER_KIND_UNSPECIFIED: StreamOwnerKind.ValueType # 0 +STREAM_OWNER_KIND_WORKFLOW: StreamOwnerKind.ValueType # 1 +"""A Workflow owns the stream. The Workflow closes the stream itself, with a result or a +failure. If the Workflow's run chain ends with no close and callbacks are attached, the +stream notifier closes the stream with a failure within `streamnotifier.ownerCheckInterval`. +""" +STREAM_OWNER_KIND_ACTIVITY: StreamOwnerKind.ValueType # 2 +"""A standalone Activity owns the stream. The Activity closes the stream itself, with a result +or a failure. If the Activity run reaches a terminal status with no close and callbacks +are attached, the stream notifier closes the stream within +`streamnotifier.ownerCheckInterval`. A completed run gives no result. A failed, canceled, +terminated or timed-out run gives the run's own failure. +""" +STREAM_OWNER_KIND_WORKFLOW_ACTIVITY: StreamOwnerKind.ValueType # 3 +"""An Activity that a Workflow scheduled owns the stream. `workflow_id` names the Workflow, +`run_id` the run that scheduled the Activity, and `activity_id` the Activity. The stream +closes when the Workflow sees the Activity end. If the Activity is no longer pending in the +run and no close came, the stream notifier closes the stream with a failure, after +`streamnotifier.ownerStartGrace` from its start and `streamnotifier.ownerCloseGrace` from +the first check that found the Activity gone. +""" +global___StreamOwnerKind = StreamOwnerKind diff --git a/temporalio/api/stream/__init__.py b/temporalio/api/stream/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/temporalio/api/stream/v1/__init__.py b/temporalio/api/stream/v1/__init__.py new file mode 100644 index 0000000000..93bec5bac3 --- /dev/null +++ b/temporalio/api/stream/v1/__init__.py @@ -0,0 +1,5 @@ +from .message_pb2 import StreamReference + +__all__ = [ + "StreamReference", +] diff --git a/temporalio/api/stream/v1/message_pb2.py b/temporalio/api/stream/v1/message_pb2.py new file mode 100644 index 0000000000..d4a6ae42ab --- /dev/null +++ b/temporalio/api/stream/v1/message_pb2.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: temporal/api/stream/v1/message.proto +"""Generated protocol buffer code.""" + +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from temporalio.api.enums.v1 import ( + stream_pb2 as temporal_dot_api_dot_enums_dot_v1_dot_stream__pb2, +) + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n$temporal/api/stream/v1/message.proto\x12\x16temporal.api.stream.v1\x1a"temporal/api/enums/v1/stream.proto"\x96\x01\n\x0fStreamReference\x12:\n\nowner_kind\x18\x01 \x01(\x0e\x32&.temporal.api.enums.v1.StreamOwnerKind\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x0e\n\x06run_id\x18\x03 \x01(\t\x12\r\n\x05topic\x18\x04 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x05 \x01(\tB\x89\x01\n\x19io.temporal.api.stream.v1B\x0cMessageProtoP\x01Z#go.temporal.io/api/stream/v1;stream\xaa\x02\x18Temporalio.Api.Stream.V1\xea\x02\x1bTemporalio::Api::Stream::V1b\x06proto3' +) + + +_STREAMREFERENCE = DESCRIPTOR.message_types_by_name["StreamReference"] +StreamReference = _reflection.GeneratedProtocolMessageType( + "StreamReference", + (_message.Message,), + { + "DESCRIPTOR": _STREAMREFERENCE, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamReference) + }, +) +_sym_db.RegisterMessage(StreamReference) + +if _descriptor._USE_C_DESCRIPTORS == False: + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b"\n\031io.temporal.api.stream.v1B\014MessageProtoP\001Z#go.temporal.io/api/stream/v1;stream\252\002\030Temporalio.Api.Stream.V1\352\002\033Temporalio::Api::Stream::V1" + _STREAMREFERENCE._serialized_start = 101 + _STREAMREFERENCE._serialized_end = 251 +# @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/stream/v1/message_pb2.pyi b/temporalio/api/stream/v1/message_pb2.pyi new file mode 100644 index 0000000000..a36aeef985 --- /dev/null +++ b/temporalio/api/stream/v1/message_pb2.pyi @@ -0,0 +1,75 @@ +""" +@generated by mypy-protobuf. Do not edit manually! +isort:skip_file +""" + +import builtins +import sys + +import google.protobuf.descriptor +import google.protobuf.message + +import temporalio.api.enums.v1.stream_pb2 + +if sys.version_info >= (3, 8): + import typing as typing_extensions +else: + import typing_extensions + +DESCRIPTOR: google.protobuf.descriptor.FileDescriptor + +class StreamReference(google.protobuf.message.Message): + """Names one stream within a namespace: its owner and its topic.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + OWNER_KIND_FIELD_NUMBER: builtins.int + WORKFLOW_ID_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + TOPIC_FIELD_NUMBER: builtins.int + ACTIVITY_ID_FIELD_NUMBER: builtins.int + owner_kind: temporalio.api.enums.v1.stream_pb2.StreamOwnerKind.ValueType + workflow_id: builtins.str + """The owning Workflow, for `STREAM_OWNER_KIND_WORKFLOW`. The Workflow that scheduled the + owning Activity, for `STREAM_OWNER_KIND_WORKFLOW_ACTIVITY`. Empty for other kinds. + """ + run_id: builtins.str + """For `STREAM_OWNER_KIND_WORKFLOW`, the id of the first run in the owning Workflow's run + chain. The stream notifier is keyed by it, so one notifier serves the stream across the + chain's runs, and a stream that reuses the Workflow id in a new chain gets a notifier of its + own. For `STREAM_OWNER_KIND_ACTIVITY`, the id of the owning Activity run. For + `STREAM_OWNER_KIND_WORKFLOW_ACTIVITY`, the id of the run that scheduled the Activity, not + the chain's first run. An empty run id is refused. + """ + topic: builtins.str + """The stream's topic within its owner.""" + activity_id: builtins.str + """The owning standalone Activity, for `STREAM_OWNER_KIND_ACTIVITY`. The owning Activity's id + in the scheduling run, for `STREAM_OWNER_KIND_WORKFLOW_ACTIVITY`. Empty for other kinds. + """ + def __init__( + self, + *, + owner_kind: temporalio.api.enums.v1.stream_pb2.StreamOwnerKind.ValueType = ..., + workflow_id: builtins.str = ..., + run_id: builtins.str = ..., + topic: builtins.str = ..., + activity_id: builtins.str = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "activity_id", + b"activity_id", + "owner_kind", + b"owner_kind", + "run_id", + b"run_id", + "topic", + b"topic", + "workflow_id", + b"workflow_id", + ], + ) -> None: ... + +global___StreamReference = StreamReference diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 5d10f86f54..f941e43ea8 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -2926,7 +2926,7 @@ dependencies = [ [[package]] name = "temporalio-streams" version = "0.1.0" -source = "git+https://github.com/moetemp/sdk-streams?rev=7efd5ad9fb0ba72f14ab60676440346bf1e62092#7efd5ad9fb0ba72f14ab60676440346bf1e62092" +source = "git+https://github.com/moetemp/sdk-streams?rev=dc5d8bfa9b734a6b8fc7f76b2f97344306ea8427#dc5d8bfa9b734a6b8fc7f76b2f97344306ea8427" dependencies = [ "async-trait", "aws-lc-rs", diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index 84317819ef..85c61f93cc 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -91,7 +91,7 @@ url = "2.2" # makes it use the submodule's crates, so the bridge sees one `Connection` type. [dependencies.temporalio-streams] git = "https://github.com/moetemp/sdk-streams" -rev = "7efd5ad9fb0ba72f14ab60676440346bf1e62092" +rev = "dc5d8bfa9b734a6b8fc7f76b2f97344306ea8427" default-features = false features = ["envconfig", "redis"] diff --git a/temporalio/bridge/proto/streams/__init__.py b/temporalio/bridge/proto/streams/__init__.py index be395a8a36..afa31dcddc 100644 --- a/temporalio/bridge/proto/streams/__init__.py +++ b/temporalio/bridge/proto/streams/__init__.py @@ -10,8 +10,13 @@ DeleteOwnerResponse, DescribeStoreRequest, DescribeStoreResponse, + FollowedRecord, + FollowRequest, + FollowResponse, + FollowSource, LatestRequest, LatestResponse, + LinkState, MemoryStoreConfig, NamedProducer, OutputRecord, @@ -56,8 +61,13 @@ "DeleteOwnerResponse", "DescribeStoreRequest", "DescribeStoreResponse", + "FollowRequest", + "FollowResponse", + "FollowSource", + "FollowedRecord", "LatestRequest", "LatestResponse", + "LinkState", "MemoryStoreConfig", "NamedProducer", "OutputRecord", diff --git a/temporalio/bridge/proto/streams/streams_pb2.py b/temporalio/bridge/proto/streams/streams_pb2.py index 28838167eb..29f7ffc39f 100644 --- a/temporalio/bridge/proto/streams/streams_pb2.py +++ b/temporalio/bridge/proto/streams/streams_pb2.py @@ -25,7 +25,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\'temporal/sdk/core/streams/streams.proto\x12\x0f\x63oresdk.streams\x1a\x1egoogle/protobuf/duration.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/sdk/streams/v1/message.proto"\xbc\x01\n\x11StreamStoreConfig\x12\x32\n\x05redis\x18\x01 \x01(\x0b\x32!.coresdk.streams.RedisStoreConfigH\x00\x12\x34\n\x06memory\x18\x02 \x01(\x0b\x32".coresdk.streams.MemoryStoreConfigH\x00\x12\x34\n\x11owner_start_grace\x18\x06 \x01(\x0b\x32\x19.google.protobuf.DurationB\x07\n\x05store"\xd8\x01\n\x10RedisStoreConfig\x12\x0c\n\x04urls\x18\x01 \x03(\t\x12\x0f\n\x07\x63luster\x18\x02 \x01(\x08\x12\x12\n\nkey_prefix\x18\x03 \x01(\t\x12,\n\tretention\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x1f\n\x17\x62locking_reads_per_node\x18\x05 \x01(\r\x12\x33\n\x10response_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\r\n\x05plain\x18\n \x01(\x08"M\n\x11MemoryStoreConfig\x12\x38\n\x0c\x63\x61pabilities\x18\x01 \x01(\x0b\x32".coresdk.streams.StoreCapabilities"\x8c\x01\n\rStreamAddress\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x34\n\nowner_kind\x18\x02 \x01(\x0e\x32 .coresdk.streams.StreamOwnerKind\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\r\n\x05topic\x18\x05 \x01(\t"G\n\x07\x43hainId\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0c\x66irst_run_id\x18\x03 \x01(\t"y\n\rNamedProducer\x12\x13\n\x0bproducer_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\x12\x1d\n\x15task_started_event_id\x18\x03 \x01(\x03\x12#\n\x1b\x63ompleted_started_event_ids\x18\x04 \x03(\x03"]\n\x10\x41\x63tivityProducer\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x05"\x8c\x01\n\x0c\x41ppendRecord\x12\x37\n\x04kind\x18\x01 \x01(\x0e\x32).temporal.sdk.streams.v1.StreamRecordKind\x12-\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x14\n\x0c\x63ontent_hash\x18\x03 \x01(\x0c"\x85\x02\n\rAppendRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress\x12/\n\x05named\x18\x02 \x01(\x0b\x32\x1e.coresdk.streams.NamedProducerH\x00\x12\x35\n\x08\x61\x63tivity\x18\x03 \x01(\x0b\x32!.coresdk.streams.ActivityProducerH\x00\x12\x10\n\x08sequence\x18\x04 \x01(\x03\x12.\n\x07records\x18\x05 \x03(\x0b\x32\x1d.coresdk.streams.AppendRecord\x12\x0e\n\x06\x64igest\x18\x06 \x01(\x0c\x42\n\n\x08producer";\n\x0e\x41ppendResponse\x12\x14\n\x0c\x66irst_cursor\x18\x01 \x01(\t\x12\x13\n\x0blast_cursor\x18\x02 \x01(\t"\x99\x01\n\x0bReadRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress\x12\r\n\x05\x61\x66ter\x18\x02 \x01(\t\x12\'\n\x04wait\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x04 \x01(\r\x12\r\n\x05state\x18\x05 \x01(\x0c"]\n\x0cSupersession\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x13\n\x0bproducer_id\x18\x02 \x01(\t\x12\x18\n\x10previous_attempt\x18\x03 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x03"\xa3\x01\n\nReadRecord\x12\x0e\n\x06\x63ursor\x18\x01 \x01(\t\x12\x37\n\x06stored\x18\x02 \x01(\x0b\x32%.temporal.sdk.streams.v1.StreamRecordH\x00\x12\x33\n\nsuperseded\x18\x03 \x01(\x0b\x32\x1d.coresdk.streams.SupersessionH\x00\x12\r\n\x05stale\x18\x04 \x01(\x08\x42\x08\n\x06record"i\n\x0cReadResponse\x12,\n\x07records\x18\x01 \x03(\x0b\x32\x1b.coresdk.streams.ReadRecord\x12\x0e\n\x06\x63ursor\x18\x02 \x01(\t\x12\r\n\x05state\x18\x03 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x04 \x01(\x08"?\n\rLatestRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress" \n\x0eLatestResponse\x12\x0e\n\x06\x63ursor\x18\x01 \x01(\t">\n\x0c\x43loseRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress"\x0f\n\rCloseResponse"r\n\x12\x44\x65leteOwnerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x34\n\nowner_kind\x18\x02 \x01(\x0e\x32 .coresdk.streams.StreamOwnerKind\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t"&\n\x13\x44\x65leteOwnerResponse\x12\x0f\n\x07\x64\x65leted\x18\x01 \x01(\x04"\xbb\x01\n\x15RetireProducerRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress\x12/\n\x05named\x18\x02 \x01(\x0b\x32\x1e.coresdk.streams.NamedProducerH\x00\x12\x35\n\x08\x61\x63tivity\x18\x03 \x01(\x0b\x32!.coresdk.streams.ActivityProducerH\x00\x42\n\n\x08producer"\x18\n\x16RetireProducerResponse"\x16\n\x14\x44\x65scribeStoreRequest"_\n\x15\x44\x65scribeStoreResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x38\n\x0c\x63\x61pabilities\x18\x02 \x01(\x0b\x32".coresdk.streams.StoreCapabilities"\xc1\x02\n\x11StoreCapabilities\x12\x16\n\x0eproducer_state\x18\x01 \x01(\x08\x12\x11\n\tkept_last\x18\x04 \x01(\x08\x12\r\n\x05holds\x18\x05 \x01(\x08\x12\x13\n\x0b\x63hange_feed\x18\x06 \x01(\x08\x12\x12\n\ncheap_logs\x18\x07 \x01(\x08\x12\x14\n\x0c\x64\x65lete_owner\x18\x08 \x01(\x08\x12\x15\n\rtopic_listing\x18\t \x01(\x08\x12\x11\n\tlong_poll\x18\n \x01(\x08\x12\x17\n\x0f\x64\x65nse_positions\x18\x0b \x01(\x08\x12\x31\n\x0ereplay_horizon\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x18\n\x10max_record_bytes\x18\r \x01(\x04\x12\x17\n\x0fmax_batch_bytes\x18\x0e \x01(\x04J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04"b\n\rStreamFailure\x12\x30\n\x04kind\x18\x01 \x01(\x0e\x32".coresdk.streams.StreamFailureKind\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x0e\n\x06\x63ursor\x18\x03 \x01(\t"\xa5\x01\n\x12StoreAppendRequest\x12\'\n\x05\x63hain\x18\x01 \x01(\x0b\x32\x18.coresdk.streams.ChainId\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x13\n\x0bproducer_id\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x03\x12\x10\n\x08sequence\x18\x05 \x01(\x03\x12\x0e\n\x06\x64igest\x18\x06 \x01(\x0c\x12\x0f\n\x07records\x18\x07 \x03(\x0c"D\n\x13StoreAppendResponse\x12\x16\n\x0e\x66irst_position\x18\x01 \x01(\t\x12\x15\n\rlast_position\x18\x02 \x01(\t"\xa0\x01\n\x10StoreReadRequest\x12\'\n\x05\x63hain\x18\x01 \x01(\x0b\x32\x18.coresdk.streams.ChainId\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x16\n\x0e\x61\x66ter_position\x18\x03 \x01(\t\x12\'\n\x04wait\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x05 \x01(\r"0\n\x0cStoredRecord\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06record\x18\x02 \x01(\x0c"S\n\x11StoreReadResponse\x12.\n\x07records\x18\x01 \x03(\x0b\x32\x1d.coresdk.streams.StoredRecord\x12\x0e\n\x06\x63losed\x18\x02 \x01(\x08"L\n\x12StoreLatestRequest\x12\'\n\x05\x63hain\x18\x01 \x01(\x0b\x32\x18.coresdk.streams.ChainId\x12\r\n\x05topic\x18\x02 \x01(\t"\'\n\x13StoreLatestResponse\x12\x10\n\x08position\x18\x01 \x01(\t"\xab\x03\n\x1aPublishWorkflowTaskRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tworker_id\x18\x02 \x01(\t\x12\x12\n\ntask_queue\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x05 \x01(\t\x12\x0e\n\x06run_id\x18\x06 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x07 \x01(\t\x12\x0c\n\x04\x63ron\x18\x08 \x01(\x08\x12\x18\n\x10has_retry_policy\x18\t \x01(\x08\x12\x1d\n\x15task_started_event_id\x18\n \x01(\x03\x12\x17\n\x0f\x66\x61iled_attempts\x18\x0b \x01(\r\x12#\n\x1b\x63ompleted_started_event_ids\x18\x0c \x03(\x03\x12.\n\x07records\x18\r \x03(\x0b\x32\x1d.coresdk.streams.OutputRecord\x12\x42\n\x10terminal_command\x18\x0e \x01(\x0e\x32(.coresdk.streams.WorkflowTerminalCommand"\x9b\x01\n\x0cOutputRecord\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x37\n\x04kind\x18\x02 \x01(\x0e\x32).temporal.sdk.streams.v1.StreamRecordKind\x12-\n\x04\x62ody\x18\x03 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x14\n\x0c\x63ontent_hash\x18\x04 \x01(\x0c"5\n\x1bPublishWorkflowTaskResponse\x12\x16\n\x0e\x64ropped_topics\x18\x01 \x03(\t"Y\n\x11RunEvictedRequest\x12\x11\n\tworker_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12!\n\x19workflow_execution_ending\x18\x03 \x01(\x08"\x14\n\x12RunEvictedResponse"*\n\x15ShutdownWorkerRequest\x12\x11\n\tworker_id\x18\x01 \x01(\t"\x18\n\x16ShutdownWorkerResponse*T\n\x0fStreamOwnerKind\x12!\n\x1dSTREAM_OWNER_KIND_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTREAM_OWNER_KIND_WORKFLOW\x10\x01*\xea\x03\n\x11StreamFailureKind\x12#\n\x1fSTREAM_FAILURE_KIND_UNSPECIFIED\x10\x00\x12*\n&STREAM_FAILURE_KIND_PRODUCER_DIVERGENT\x10\x01\x12&\n"STREAM_FAILURE_KIND_PRODUCER_STALE\x10\x02\x12\'\n#STREAM_FAILURE_KIND_OUTCOME_UNKNOWN\x10\x03\x12\x1f\n\x1bSTREAM_FAILURE_KIND_REFUSED\x10\x04\x12\x1e\n\x1aSTREAM_FAILURE_KIND_CLOSED\x10\x05\x12\x1e\n\x1aSTREAM_FAILURE_KIND_CURSOR\x10\x06\x12\x1f\n\x1bSTREAM_FAILURE_KIND_EXPIRED\x10\x07\x12!\n\x1dSTREAM_FAILURE_KIND_NOT_FOUND\x10\x08\x12\x1e\n\x1aSTREAM_FAILURE_KIND_RECORD\x10\t\x12\x1f\n\x1bSTREAM_FAILURE_KIND_STORAGE\x10\n\x12#\n\x1fSTREAM_FAILURE_KIND_UNSUPPORTED\x10\x0b\x12(\n$STREAM_FAILURE_KIND_INVALID_ARGUMENT\x10\x0c*\xe5\x01\n\x17WorkflowTerminalCommand\x12)\n%WORKFLOW_TERMINAL_COMMAND_UNSPECIFIED\x10\x00\x12&\n"WORKFLOW_TERMINAL_COMMAND_COMPLETE\x10\x01\x12"\n\x1eWORKFLOW_TERMINAL_COMMAND_FAIL\x10\x02\x12$\n WORKFLOW_TERMINAL_COMMAND_CANCEL\x10\x03\x12-\n)WORKFLOW_TERMINAL_COMMAND_CONTINUE_AS_NEW\x10\x04\x32\xfb\x06\n\rStreamService\x12I\n\x06\x41ppend\x12\x1e.coresdk.streams.AppendRequest\x1a\x1f.coresdk.streams.AppendResponse\x12\x43\n\x04Read\x12\x1c.coresdk.streams.ReadRequest\x1a\x1d.coresdk.streams.ReadResponse\x12I\n\x06Latest\x12\x1e.coresdk.streams.LatestRequest\x1a\x1f.coresdk.streams.LatestResponse\x12\x46\n\x05\x43lose\x12\x1d.coresdk.streams.CloseRequest\x1a\x1e.coresdk.streams.CloseResponse\x12X\n\x0b\x44\x65leteOwner\x12#.coresdk.streams.DeleteOwnerRequest\x1a$.coresdk.streams.DeleteOwnerResponse\x12\x61\n\x0eRetireProducer\x12&.coresdk.streams.RetireProducerRequest\x1a\'.coresdk.streams.RetireProducerResponse\x12^\n\rDescribeStore\x12%.coresdk.streams.DescribeStoreRequest\x1a&.coresdk.streams.DescribeStoreResponse\x12p\n\x13PublishWorkflowTask\x12+.coresdk.streams.PublishWorkflowTaskRequest\x1a,.coresdk.streams.PublishWorkflowTaskResponse\x12U\n\nRunEvicted\x12".coresdk.streams.RunEvictedRequest\x1a#.coresdk.streams.RunEvictedResponse\x12\x61\n\x0eShutdownWorker\x12&.coresdk.streams.ShutdownWorkerRequest\x1a\'.coresdk.streams.ShutdownWorkerResponseB-\xea\x02*Temporalio::Internal::Bridge::Api::Streamsb\x06proto3' + b'\n\'temporal/sdk/core/streams/streams.proto\x12\x0f\x63oresdk.streams\x1a\x1egoogle/protobuf/duration.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/sdk/streams/v1/message.proto"\xbc\x01\n\x11StreamStoreConfig\x12\x32\n\x05redis\x18\x01 \x01(\x0b\x32!.coresdk.streams.RedisStoreConfigH\x00\x12\x34\n\x06memory\x18\x02 \x01(\x0b\x32".coresdk.streams.MemoryStoreConfigH\x00\x12\x34\n\x11owner_start_grace\x18\x06 \x01(\x0b\x32\x19.google.protobuf.DurationB\x07\n\x05store"\xd8\x01\n\x10RedisStoreConfig\x12\x0c\n\x04urls\x18\x01 \x03(\t\x12\x0f\n\x07\x63luster\x18\x02 \x01(\x08\x12\x12\n\nkey_prefix\x18\x03 \x01(\t\x12,\n\tretention\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x1f\n\x17\x62locking_reads_per_node\x18\x05 \x01(\r\x12\x33\n\x10response_timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\r\n\x05plain\x18\n \x01(\x08"M\n\x11MemoryStoreConfig\x12\x38\n\x0c\x63\x61pabilities\x18\x01 \x01(\x0b\x32".coresdk.streams.StoreCapabilities"\x8c\x01\n\rStreamAddress\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x34\n\nowner_kind\x18\x02 \x01(\x0e\x32 .coresdk.streams.StreamOwnerKind\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t\x12\x0e\n\x06run_id\x18\x04 \x01(\t\x12\r\n\x05topic\x18\x05 \x01(\t"G\n\x07\x43hainId\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x02 \x01(\t\x12\x14\n\x0c\x66irst_run_id\x18\x03 \x01(\t"y\n\rNamedProducer\x12\x13\n\x0bproducer_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\x12\x1d\n\x15task_started_event_id\x18\x03 \x01(\x03\x12#\n\x1b\x63ompleted_started_event_ids\x18\x04 \x03(\x03"]\n\x10\x41\x63tivityProducer\x12\x13\n\x0bworkflow_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12\x13\n\x0b\x61\x63tivity_id\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x05"\xbf\x01\n\x0c\x41ppendRecord\x12\x37\n\x04kind\x18\x01 \x01(\x0e\x32).temporal.sdk.streams.v1.StreamRecordKind\x12-\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x14\n\x0c\x63ontent_hash\x18\x03 \x01(\x0c\x12\x31\n\x04link\x18\x04 \x01(\x0b\x32#.temporal.sdk.streams.v1.StreamLink"\x85\x02\n\rAppendRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress\x12/\n\x05named\x18\x02 \x01(\x0b\x32\x1e.coresdk.streams.NamedProducerH\x00\x12\x35\n\x08\x61\x63tivity\x18\x03 \x01(\x0b\x32!.coresdk.streams.ActivityProducerH\x00\x12\x10\n\x08sequence\x18\x04 \x01(\x03\x12.\n\x07records\x18\x05 \x03(\x0b\x32\x1d.coresdk.streams.AppendRecord\x12\x0e\n\x06\x64igest\x18\x06 \x01(\x0c\x42\n\n\x08producer";\n\x0e\x41ppendResponse\x12\x14\n\x0c\x66irst_cursor\x18\x01 \x01(\t\x12\x13\n\x0blast_cursor\x18\x02 \x01(\t"\x99\x01\n\x0bReadRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress\x12\r\n\x05\x61\x66ter\x18\x02 \x01(\t\x12\'\n\x04wait\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x04 \x01(\r\x12\r\n\x05state\x18\x05 \x01(\x0c"]\n\x0cSupersession\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x13\n\x0bproducer_id\x18\x02 \x01(\t\x12\x18\n\x10previous_attempt\x18\x03 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x03"\xa3\x01\n\nReadRecord\x12\x0e\n\x06\x63ursor\x18\x01 \x01(\t\x12\x37\n\x06stored\x18\x02 \x01(\x0b\x32%.temporal.sdk.streams.v1.StreamRecordH\x00\x12\x33\n\nsuperseded\x18\x03 \x01(\x0b\x32\x1d.coresdk.streams.SupersessionH\x00\x12\r\n\x05stale\x18\x04 \x01(\x08\x42\x08\n\x06record"\x7f\n\x0cReadResponse\x12,\n\x07records\x18\x01 \x03(\x0b\x32\x1b.coresdk.streams.ReadRecord\x12\x0e\n\x06\x63ursor\x18\x02 \x01(\t\x12\r\n\x05state\x18\x03 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x04 \x01(\x08\x12\x14\n\x0cstart_cursor\x18\x0c \x01(\t"\xc0\x01\n\rFollowRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress\x12\r\n\x05\x61\x66ter\x18\x02 \x01(\t\x12\r\n\x05state\x18\x03 \x01(\x0c\x12\'\n\x04wait\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x05 \x01(\r\x12\x11\n\tmax_depth\x18\x06 \x01(\r\x12\x10\n\x08max_open\x18\x07 \x01(\r"\x89\x01\n\x0e\x46ollowedRecord\x12\x0e\n\x06source\x18\x01 \x01(\r\x12-\n\x06record\x18\x02 \x01(\x0b\x32\x1b.coresdk.streams.ReadRecordH\x00\x12\x30\n\nlink_state\x18\x03 \x01(\x0b\x32\x1a.coresdk.streams.LinkStateH\x00\x42\x06\n\x04item"\x92\x02\n\tLinkState\x12-\n\x04kind\x18\x01 \x01(\x0e\x32\x1f.coresdk.streams.LinkState.Kind\x12\x31\n\x04link\x18\x02 \x01(\x0b\x32#.temporal.sdk.streams.v1.StreamLink\x12-\n\x05\x65rror\x18\x05 \x01(\x0b\x32\x1e.coresdk.streams.StreamFailure\x12\x0e\n\x06reason\x18\x06 \x01(\t"d\n\x04Kind\x12\x14\n\x10KIND_UNSPECIFIED\x10\x00\x12\x0f\n\x0bKIND_OPENED\x10\x01\x12\r\n\tKIND_DONE\x10\x02\x12\x14\n\x10KIND_UNAVAILABLE\x10\x03\x12\x10\n\x0cKIND_SKIPPED\x10\x04"{\n\x0c\x46ollowSource\x12\x0e\n\x06number\x18\x01 \x01(\r\x12\x0e\n\x06parent\x18\x02 \x01(\r\x12\r\n\x05\x64\x65pth\x18\x03 \x01(\r\x12.\n\x06stream\x18\x04 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress\x12\x0c\n\x04name\x18\x05 \x01(\t"\x8f\x01\n\x0e\x46ollowResponse\x12\x30\n\x07records\x18\x01 \x03(\x0b\x32\x1f.coresdk.streams.FollowedRecord\x12\r\n\x05state\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12.\n\x07sources\x18\x04 \x03(\x0b\x32\x1d.coresdk.streams.FollowSource"?\n\rLatestRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress" \n\x0eLatestResponse\x12\x0e\n\x06\x63ursor\x18\x01 \x01(\t">\n\x0c\x43loseRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress"\x0f\n\rCloseResponse"r\n\x12\x44\x65leteOwnerRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x34\n\nowner_kind\x18\x02 \x01(\x0e\x32 .coresdk.streams.StreamOwnerKind\x12\x13\n\x0bworkflow_id\x18\x03 \x01(\t"&\n\x13\x44\x65leteOwnerResponse\x12\x0f\n\x07\x64\x65leted\x18\x01 \x01(\x04"\xbb\x01\n\x15RetireProducerRequest\x12.\n\x06stream\x18\x01 \x01(\x0b\x32\x1e.coresdk.streams.StreamAddress\x12/\n\x05named\x18\x02 \x01(\x0b\x32\x1e.coresdk.streams.NamedProducerH\x00\x12\x35\n\x08\x61\x63tivity\x18\x03 \x01(\x0b\x32!.coresdk.streams.ActivityProducerH\x00\x42\n\n\x08producer"\x18\n\x16RetireProducerResponse"\x16\n\x14\x44\x65scribeStoreRequest"_\n\x15\x44\x65scribeStoreResponse\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x38\n\x0c\x63\x61pabilities\x18\x02 \x01(\x0b\x32".coresdk.streams.StoreCapabilities"\xc1\x02\n\x11StoreCapabilities\x12\x16\n\x0eproducer_state\x18\x01 \x01(\x08\x12\x11\n\tkept_last\x18\x04 \x01(\x08\x12\r\n\x05holds\x18\x05 \x01(\x08\x12\x13\n\x0b\x63hange_feed\x18\x06 \x01(\x08\x12\x12\n\ncheap_logs\x18\x07 \x01(\x08\x12\x14\n\x0c\x64\x65lete_owner\x18\x08 \x01(\x08\x12\x15\n\rtopic_listing\x18\t \x01(\x08\x12\x11\n\tlong_poll\x18\n \x01(\x08\x12\x17\n\x0f\x64\x65nse_positions\x18\x0b \x01(\x08\x12\x31\n\x0ereplay_horizon\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x18\n\x10max_record_bytes\x18\r \x01(\x04\x12\x17\n\x0fmax_batch_bytes\x18\x0e \x01(\x04J\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04"b\n\rStreamFailure\x12\x30\n\x04kind\x18\x01 \x01(\x0e\x32".coresdk.streams.StreamFailureKind\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x0e\n\x06\x63ursor\x18\x03 \x01(\t"\xa5\x01\n\x12StoreAppendRequest\x12\'\n\x05\x63hain\x18\x01 \x01(\x0b\x32\x18.coresdk.streams.ChainId\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x13\n\x0bproducer_id\x18\x03 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x04 \x01(\x03\x12\x10\n\x08sequence\x18\x05 \x01(\x03\x12\x0e\n\x06\x64igest\x18\x06 \x01(\x0c\x12\x0f\n\x07records\x18\x07 \x03(\x0c"D\n\x13StoreAppendResponse\x12\x16\n\x0e\x66irst_position\x18\x01 \x01(\t\x12\x15\n\rlast_position\x18\x02 \x01(\t"\xa0\x01\n\x10StoreReadRequest\x12\'\n\x05\x63hain\x18\x01 \x01(\x0b\x32\x18.coresdk.streams.ChainId\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x16\n\x0e\x61\x66ter_position\x18\x03 \x01(\t\x12\'\n\x04wait\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bmax_records\x18\x05 \x01(\r"0\n\x0cStoredRecord\x12\x10\n\x08position\x18\x01 \x01(\t\x12\x0e\n\x06record\x18\x02 \x01(\x0c"S\n\x11StoreReadResponse\x12.\n\x07records\x18\x01 \x03(\x0b\x32\x1d.coresdk.streams.StoredRecord\x12\x0e\n\x06\x63losed\x18\x02 \x01(\x08"L\n\x12StoreLatestRequest\x12\'\n\x05\x63hain\x18\x01 \x01(\x0b\x32\x18.coresdk.streams.ChainId\x12\r\n\x05topic\x18\x02 \x01(\t"\'\n\x13StoreLatestResponse\x12\x10\n\x08position\x18\x01 \x01(\t"\xab\x03\n\x1aPublishWorkflowTaskRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x11\n\tworker_id\x18\x02 \x01(\t\x12\x12\n\ntask_queue\x18\x03 \x01(\t\x12\x15\n\rworkflow_type\x18\x04 \x01(\t\x12\x13\n\x0bworkflow_id\x18\x05 \x01(\t\x12\x0e\n\x06run_id\x18\x06 \x01(\t\x12\x1e\n\x16\x66irst_execution_run_id\x18\x07 \x01(\t\x12\x0c\n\x04\x63ron\x18\x08 \x01(\x08\x12\x18\n\x10has_retry_policy\x18\t \x01(\x08\x12\x1d\n\x15task_started_event_id\x18\n \x01(\x03\x12\x17\n\x0f\x66\x61iled_attempts\x18\x0b \x01(\r\x12#\n\x1b\x63ompleted_started_event_ids\x18\x0c \x03(\x03\x12.\n\x07records\x18\r \x03(\x0b\x32\x1d.coresdk.streams.OutputRecord\x12\x42\n\x10terminal_command\x18\x0e \x01(\x0e\x32(.coresdk.streams.WorkflowTerminalCommand"\xce\x01\n\x0cOutputRecord\x12\r\n\x05topic\x18\x01 \x01(\t\x12\x37\n\x04kind\x18\x02 \x01(\x0e\x32).temporal.sdk.streams.v1.StreamRecordKind\x12-\n\x04\x62ody\x18\x03 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x14\n\x0c\x63ontent_hash\x18\x04 \x01(\x0c\x12\x31\n\x04link\x18\x05 \x01(\x0b\x32#.temporal.sdk.streams.v1.StreamLink"5\n\x1bPublishWorkflowTaskResponse\x12\x16\n\x0e\x64ropped_topics\x18\x01 \x03(\t"Y\n\x11RunEvictedRequest\x12\x11\n\tworker_id\x18\x01 \x01(\t\x12\x0e\n\x06run_id\x18\x02 \x01(\t\x12!\n\x19workflow_execution_ending\x18\x03 \x01(\x08"\x14\n\x12RunEvictedResponse"*\n\x15ShutdownWorkerRequest\x12\x11\n\tworker_id\x18\x01 \x01(\t"\x18\n\x16ShutdownWorkerResponse*T\n\x0fStreamOwnerKind\x12!\n\x1dSTREAM_OWNER_KIND_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTREAM_OWNER_KIND_WORKFLOW\x10\x01*\xea\x03\n\x11StreamFailureKind\x12#\n\x1fSTREAM_FAILURE_KIND_UNSPECIFIED\x10\x00\x12*\n&STREAM_FAILURE_KIND_PRODUCER_DIVERGENT\x10\x01\x12&\n"STREAM_FAILURE_KIND_PRODUCER_STALE\x10\x02\x12\'\n#STREAM_FAILURE_KIND_OUTCOME_UNKNOWN\x10\x03\x12\x1f\n\x1bSTREAM_FAILURE_KIND_REFUSED\x10\x04\x12\x1e\n\x1aSTREAM_FAILURE_KIND_CLOSED\x10\x05\x12\x1e\n\x1aSTREAM_FAILURE_KIND_CURSOR\x10\x06\x12\x1f\n\x1bSTREAM_FAILURE_KIND_EXPIRED\x10\x07\x12!\n\x1dSTREAM_FAILURE_KIND_NOT_FOUND\x10\x08\x12\x1e\n\x1aSTREAM_FAILURE_KIND_RECORD\x10\t\x12\x1f\n\x1bSTREAM_FAILURE_KIND_STORAGE\x10\n\x12#\n\x1fSTREAM_FAILURE_KIND_UNSUPPORTED\x10\x0b\x12(\n$STREAM_FAILURE_KIND_INVALID_ARGUMENT\x10\x0c*\xe5\x01\n\x17WorkflowTerminalCommand\x12)\n%WORKFLOW_TERMINAL_COMMAND_UNSPECIFIED\x10\x00\x12&\n"WORKFLOW_TERMINAL_COMMAND_COMPLETE\x10\x01\x12"\n\x1eWORKFLOW_TERMINAL_COMMAND_FAIL\x10\x02\x12$\n WORKFLOW_TERMINAL_COMMAND_CANCEL\x10\x03\x12-\n)WORKFLOW_TERMINAL_COMMAND_CONTINUE_AS_NEW\x10\x04\x32\xc6\x07\n\rStreamService\x12I\n\x06\x41ppend\x12\x1e.coresdk.streams.AppendRequest\x1a\x1f.coresdk.streams.AppendResponse\x12\x43\n\x04Read\x12\x1c.coresdk.streams.ReadRequest\x1a\x1d.coresdk.streams.ReadResponse\x12I\n\x06Latest\x12\x1e.coresdk.streams.LatestRequest\x1a\x1f.coresdk.streams.LatestResponse\x12\x46\n\x05\x43lose\x12\x1d.coresdk.streams.CloseRequest\x1a\x1e.coresdk.streams.CloseResponse\x12X\n\x0b\x44\x65leteOwner\x12#.coresdk.streams.DeleteOwnerRequest\x1a$.coresdk.streams.DeleteOwnerResponse\x12\x61\n\x0eRetireProducer\x12&.coresdk.streams.RetireProducerRequest\x1a\'.coresdk.streams.RetireProducerResponse\x12I\n\x06\x46ollow\x12\x1e.coresdk.streams.FollowRequest\x1a\x1f.coresdk.streams.FollowResponse\x12^\n\rDescribeStore\x12%.coresdk.streams.DescribeStoreRequest\x1a&.coresdk.streams.DescribeStoreResponse\x12p\n\x13PublishWorkflowTask\x12+.coresdk.streams.PublishWorkflowTaskRequest\x1a,.coresdk.streams.PublishWorkflowTaskResponse\x12U\n\nRunEvicted\x12".coresdk.streams.RunEvictedRequest\x1a#.coresdk.streams.RunEvictedResponse\x12\x61\n\x0eShutdownWorker\x12&.coresdk.streams.ShutdownWorkerRequest\x1a\'.coresdk.streams.ShutdownWorkerResponseB-\xea\x02*Temporalio::Internal::Bridge::Api::Streamsb\x06proto3' ) _STREAMOWNERKIND = DESCRIPTOR.enum_types_by_name["StreamOwnerKind"] @@ -70,6 +70,11 @@ _SUPERSESSION = DESCRIPTOR.message_types_by_name["Supersession"] _READRECORD = DESCRIPTOR.message_types_by_name["ReadRecord"] _READRESPONSE = DESCRIPTOR.message_types_by_name["ReadResponse"] +_FOLLOWREQUEST = DESCRIPTOR.message_types_by_name["FollowRequest"] +_FOLLOWEDRECORD = DESCRIPTOR.message_types_by_name["FollowedRecord"] +_LINKSTATE = DESCRIPTOR.message_types_by_name["LinkState"] +_FOLLOWSOURCE = DESCRIPTOR.message_types_by_name["FollowSource"] +_FOLLOWRESPONSE = DESCRIPTOR.message_types_by_name["FollowResponse"] _LATESTREQUEST = DESCRIPTOR.message_types_by_name["LatestRequest"] _LATESTRESPONSE = DESCRIPTOR.message_types_by_name["LatestResponse"] _CLOSEREQUEST = DESCRIPTOR.message_types_by_name["CloseRequest"] @@ -100,6 +105,7 @@ _RUNEVICTEDRESPONSE = DESCRIPTOR.message_types_by_name["RunEvictedResponse"] _SHUTDOWNWORKERREQUEST = DESCRIPTOR.message_types_by_name["ShutdownWorkerRequest"] _SHUTDOWNWORKERRESPONSE = DESCRIPTOR.message_types_by_name["ShutdownWorkerResponse"] +_LINKSTATE_KIND = _LINKSTATE.enum_types_by_name["Kind"] StreamStoreConfig = _reflection.GeneratedProtocolMessageType( "StreamStoreConfig", (_message.Message,), @@ -254,6 +260,61 @@ ) _sym_db.RegisterMessage(ReadResponse) +FollowRequest = _reflection.GeneratedProtocolMessageType( + "FollowRequest", + (_message.Message,), + { + "DESCRIPTOR": _FOLLOWREQUEST, + "__module__": "temporal.sdk.core.streams.streams_pb2", + # @@protoc_insertion_point(class_scope:coresdk.streams.FollowRequest) + }, +) +_sym_db.RegisterMessage(FollowRequest) + +FollowedRecord = _reflection.GeneratedProtocolMessageType( + "FollowedRecord", + (_message.Message,), + { + "DESCRIPTOR": _FOLLOWEDRECORD, + "__module__": "temporal.sdk.core.streams.streams_pb2", + # @@protoc_insertion_point(class_scope:coresdk.streams.FollowedRecord) + }, +) +_sym_db.RegisterMessage(FollowedRecord) + +LinkState = _reflection.GeneratedProtocolMessageType( + "LinkState", + (_message.Message,), + { + "DESCRIPTOR": _LINKSTATE, + "__module__": "temporal.sdk.core.streams.streams_pb2", + # @@protoc_insertion_point(class_scope:coresdk.streams.LinkState) + }, +) +_sym_db.RegisterMessage(LinkState) + +FollowSource = _reflection.GeneratedProtocolMessageType( + "FollowSource", + (_message.Message,), + { + "DESCRIPTOR": _FOLLOWSOURCE, + "__module__": "temporal.sdk.core.streams.streams_pb2", + # @@protoc_insertion_point(class_scope:coresdk.streams.FollowSource) + }, +) +_sym_db.RegisterMessage(FollowSource) + +FollowResponse = _reflection.GeneratedProtocolMessageType( + "FollowResponse", + (_message.Message,), + { + "DESCRIPTOR": _FOLLOWRESPONSE, + "__module__": "temporal.sdk.core.streams.streams_pb2", + # @@protoc_insertion_point(class_scope:coresdk.streams.FollowResponse) + }, +) +_sym_db.RegisterMessage(FollowResponse) + LatestRequest = _reflection.GeneratedProtocolMessageType( "LatestRequest", (_message.Message,), @@ -546,12 +607,12 @@ DESCRIPTOR._serialized_options = ( b"\352\002*Temporalio::Internal::Bridge::Api::Streams" ) - _STREAMOWNERKIND._serialized_start = 4662 - _STREAMOWNERKIND._serialized_end = 4746 - _STREAMFAILUREKIND._serialized_start = 4749 - _STREAMFAILUREKIND._serialized_end = 5239 - _WORKFLOWTERMINALCOMMAND._serialized_start = 5242 - _WORKFLOWTERMINALCOMMAND._serialized_end = 5471 + _STREAMOWNERKIND._serialized_start = 5669 + _STREAMOWNERKIND._serialized_end = 5753 + _STREAMFAILUREKIND._serialized_start = 5756 + _STREAMFAILUREKIND._serialized_end = 6246 + _WORKFLOWTERMINALCOMMAND._serialized_start = 6249 + _WORKFLOWTERMINALCOMMAND._serialized_end = 6478 _STREAMSTORECONFIG._serialized_start = 170 _STREAMSTORECONFIG._serialized_end = 358 _REDISSTORECONFIG._serialized_start = 361 @@ -567,71 +628,83 @@ _ACTIVITYPRODUCER._serialized_start = 997 _ACTIVITYPRODUCER._serialized_end = 1090 _APPENDRECORD._serialized_start = 1093 - _APPENDRECORD._serialized_end = 1233 - _APPENDREQUEST._serialized_start = 1236 - _APPENDREQUEST._serialized_end = 1497 - _APPENDRESPONSE._serialized_start = 1499 - _APPENDRESPONSE._serialized_end = 1558 - _READREQUEST._serialized_start = 1561 - _READREQUEST._serialized_end = 1714 - _SUPERSESSION._serialized_start = 1716 - _SUPERSESSION._serialized_end = 1809 - _READRECORD._serialized_start = 1812 - _READRECORD._serialized_end = 1975 - _READRESPONSE._serialized_start = 1977 - _READRESPONSE._serialized_end = 2082 - _LATESTREQUEST._serialized_start = 2084 - _LATESTREQUEST._serialized_end = 2147 - _LATESTRESPONSE._serialized_start = 2149 - _LATESTRESPONSE._serialized_end = 2181 - _CLOSEREQUEST._serialized_start = 2183 - _CLOSEREQUEST._serialized_end = 2245 - _CLOSERESPONSE._serialized_start = 2247 - _CLOSERESPONSE._serialized_end = 2262 - _DELETEOWNERREQUEST._serialized_start = 2264 - _DELETEOWNERREQUEST._serialized_end = 2378 - _DELETEOWNERRESPONSE._serialized_start = 2380 - _DELETEOWNERRESPONSE._serialized_end = 2418 - _RETIREPRODUCERREQUEST._serialized_start = 2421 - _RETIREPRODUCERREQUEST._serialized_end = 2608 - _RETIREPRODUCERRESPONSE._serialized_start = 2610 - _RETIREPRODUCERRESPONSE._serialized_end = 2634 - _DESCRIBESTOREREQUEST._serialized_start = 2636 - _DESCRIBESTOREREQUEST._serialized_end = 2658 - _DESCRIBESTORERESPONSE._serialized_start = 2660 - _DESCRIBESTORERESPONSE._serialized_end = 2755 - _STORECAPABILITIES._serialized_start = 2758 - _STORECAPABILITIES._serialized_end = 3079 - _STREAMFAILURE._serialized_start = 3081 - _STREAMFAILURE._serialized_end = 3179 - _STOREAPPENDREQUEST._serialized_start = 3182 - _STOREAPPENDREQUEST._serialized_end = 3347 - _STOREAPPENDRESPONSE._serialized_start = 3349 - _STOREAPPENDRESPONSE._serialized_end = 3417 - _STOREREADREQUEST._serialized_start = 3420 - _STOREREADREQUEST._serialized_end = 3580 - _STOREDRECORD._serialized_start = 3582 - _STOREDRECORD._serialized_end = 3630 - _STOREREADRESPONSE._serialized_start = 3632 - _STOREREADRESPONSE._serialized_end = 3715 - _STORELATESTREQUEST._serialized_start = 3717 - _STORELATESTREQUEST._serialized_end = 3793 - _STORELATESTRESPONSE._serialized_start = 3795 - _STORELATESTRESPONSE._serialized_end = 3834 - _PUBLISHWORKFLOWTASKREQUEST._serialized_start = 3837 - _PUBLISHWORKFLOWTASKREQUEST._serialized_end = 4264 - _OUTPUTRECORD._serialized_start = 4267 - _OUTPUTRECORD._serialized_end = 4422 - _PUBLISHWORKFLOWTASKRESPONSE._serialized_start = 4424 - _PUBLISHWORKFLOWTASKRESPONSE._serialized_end = 4477 - _RUNEVICTEDREQUEST._serialized_start = 4479 - _RUNEVICTEDREQUEST._serialized_end = 4568 - _RUNEVICTEDRESPONSE._serialized_start = 4570 - _RUNEVICTEDRESPONSE._serialized_end = 4590 - _SHUTDOWNWORKERREQUEST._serialized_start = 4592 - _SHUTDOWNWORKERREQUEST._serialized_end = 4634 - _SHUTDOWNWORKERRESPONSE._serialized_start = 4636 - _SHUTDOWNWORKERRESPONSE._serialized_end = 4660 - _STREAMSERVICE._serialized_start = 5474 - _STREAMSERVICE._serialized_end = 6365 + _APPENDRECORD._serialized_end = 1284 + _APPENDREQUEST._serialized_start = 1287 + _APPENDREQUEST._serialized_end = 1548 + _APPENDRESPONSE._serialized_start = 1550 + _APPENDRESPONSE._serialized_end = 1609 + _READREQUEST._serialized_start = 1612 + _READREQUEST._serialized_end = 1765 + _SUPERSESSION._serialized_start = 1767 + _SUPERSESSION._serialized_end = 1860 + _READRECORD._serialized_start = 1863 + _READRECORD._serialized_end = 2026 + _READRESPONSE._serialized_start = 2028 + _READRESPONSE._serialized_end = 2155 + _FOLLOWREQUEST._serialized_start = 2158 + _FOLLOWREQUEST._serialized_end = 2350 + _FOLLOWEDRECORD._serialized_start = 2353 + _FOLLOWEDRECORD._serialized_end = 2490 + _LINKSTATE._serialized_start = 2493 + _LINKSTATE._serialized_end = 2767 + _LINKSTATE_KIND._serialized_start = 2667 + _LINKSTATE_KIND._serialized_end = 2767 + _FOLLOWSOURCE._serialized_start = 2769 + _FOLLOWSOURCE._serialized_end = 2892 + _FOLLOWRESPONSE._serialized_start = 2895 + _FOLLOWRESPONSE._serialized_end = 3038 + _LATESTREQUEST._serialized_start = 3040 + _LATESTREQUEST._serialized_end = 3103 + _LATESTRESPONSE._serialized_start = 3105 + _LATESTRESPONSE._serialized_end = 3137 + _CLOSEREQUEST._serialized_start = 3139 + _CLOSEREQUEST._serialized_end = 3201 + _CLOSERESPONSE._serialized_start = 3203 + _CLOSERESPONSE._serialized_end = 3218 + _DELETEOWNERREQUEST._serialized_start = 3220 + _DELETEOWNERREQUEST._serialized_end = 3334 + _DELETEOWNERRESPONSE._serialized_start = 3336 + _DELETEOWNERRESPONSE._serialized_end = 3374 + _RETIREPRODUCERREQUEST._serialized_start = 3377 + _RETIREPRODUCERREQUEST._serialized_end = 3564 + _RETIREPRODUCERRESPONSE._serialized_start = 3566 + _RETIREPRODUCERRESPONSE._serialized_end = 3590 + _DESCRIBESTOREREQUEST._serialized_start = 3592 + _DESCRIBESTOREREQUEST._serialized_end = 3614 + _DESCRIBESTORERESPONSE._serialized_start = 3616 + _DESCRIBESTORERESPONSE._serialized_end = 3711 + _STORECAPABILITIES._serialized_start = 3714 + _STORECAPABILITIES._serialized_end = 4035 + _STREAMFAILURE._serialized_start = 4037 + _STREAMFAILURE._serialized_end = 4135 + _STOREAPPENDREQUEST._serialized_start = 4138 + _STOREAPPENDREQUEST._serialized_end = 4303 + _STOREAPPENDRESPONSE._serialized_start = 4305 + _STOREAPPENDRESPONSE._serialized_end = 4373 + _STOREREADREQUEST._serialized_start = 4376 + _STOREREADREQUEST._serialized_end = 4536 + _STOREDRECORD._serialized_start = 4538 + _STOREDRECORD._serialized_end = 4586 + _STOREREADRESPONSE._serialized_start = 4588 + _STOREREADRESPONSE._serialized_end = 4671 + _STORELATESTREQUEST._serialized_start = 4673 + _STORELATESTREQUEST._serialized_end = 4749 + _STORELATESTRESPONSE._serialized_start = 4751 + _STORELATESTRESPONSE._serialized_end = 4790 + _PUBLISHWORKFLOWTASKREQUEST._serialized_start = 4793 + _PUBLISHWORKFLOWTASKREQUEST._serialized_end = 5220 + _OUTPUTRECORD._serialized_start = 5223 + _OUTPUTRECORD._serialized_end = 5429 + _PUBLISHWORKFLOWTASKRESPONSE._serialized_start = 5431 + _PUBLISHWORKFLOWTASKRESPONSE._serialized_end = 5484 + _RUNEVICTEDREQUEST._serialized_start = 5486 + _RUNEVICTEDREQUEST._serialized_end = 5575 + _RUNEVICTEDRESPONSE._serialized_start = 5577 + _RUNEVICTEDRESPONSE._serialized_end = 5597 + _SHUTDOWNWORKERREQUEST._serialized_start = 5599 + _SHUTDOWNWORKERREQUEST._serialized_end = 5641 + _SHUTDOWNWORKERRESPONSE._serialized_start = 5643 + _SHUTDOWNWORKERRESPONSE._serialized_end = 5667 + _STREAMSERVICE._serialized_start = 6481 + _STREAMSERVICE._serialized_end = 7447 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/streams/streams_pb2.pyi b/temporalio/bridge/proto/streams/streams_pb2.pyi index 80bcba2a05..e14c1a908e 100644 --- a/temporalio/bridge/proto/streams/streams_pb2.pyi +++ b/temporalio/bridge/proto/streams/streams_pb2.pyi @@ -491,29 +491,41 @@ class AppendRecord(google.protobuf.message.Message): KIND_FIELD_NUMBER: builtins.int BODY_FIELD_NUMBER: builtins.int CONTENT_HASH_FIELD_NUMBER: builtins.int + LINK_FIELD_NUMBER: builtins.int kind: temporalio.bridge.proto.streams.v1.message_pb2.StreamRecordKind.ValueType @property def body(self) -> temporalio.api.common.v1.message_pb2.Payload: - """Set on DATA only, after lang's payload codec and external storage.""" + """Set on DATA, and optional on LINK, after lang's payload codec and external storage.""" content_hash: builtins.bytes """The SHA-256 of the body as the payload converter made it, before the codec. A codec that encrypts with a fresh nonce makes a retry's bytes differ, so a retry is recognized by this. Empty when there is no body. """ + @property + def link(self) -> temporalio.bridge.proto.streams.v1.message_pb2.StreamLink: + """Set on LINK only. A LINK's body is optional.""" def __init__( self, *, kind: temporalio.bridge.proto.streams.v1.message_pb2.StreamRecordKind.ValueType = ..., body: temporalio.api.common.v1.message_pb2.Payload | None = ..., content_hash: builtins.bytes = ..., + link: temporalio.bridge.proto.streams.v1.message_pb2.StreamLink | None = ..., ) -> None: ... def HasField( - self, field_name: typing_extensions.Literal["body", b"body"] + self, field_name: typing_extensions.Literal["body", b"body", "link", b"link"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ - "body", b"body", "content_hash", b"content_hash", "kind", b"kind" + "body", + b"body", + "content_hash", + b"content_hash", + "kind", + b"kind", + "link", + b"link", ], ) -> None: ... @@ -776,6 +788,7 @@ class ReadResponse(google.protobuf.message.Message): CURSOR_FIELD_NUMBER: builtins.int STATE_FIELD_NUMBER: builtins.int DONE_FIELD_NUMBER: builtins.int + START_CURSOR_FIELD_NUMBER: builtins.int @property def records( self, @@ -794,6 +807,10 @@ class ReadResponse(google.protobuf.message.Message): """The owner's chain ended and the read delivered every record the stream retains. No later call returns a record. """ + start_cursor: builtins.str + """The cursor this answer reads strictly after, with `$end` resolved. A Follow that cuts a page + at a gate reads it again from here. + """ def __init__( self, *, @@ -801,6 +818,7 @@ class ReadResponse(google.protobuf.message.Message): cursor: builtins.str = ..., state: builtins.bytes = ..., done: builtins.bool = ..., + start_cursor: builtins.str = ..., ) -> None: ... def ClearField( self, @@ -811,6 +829,8 @@ class ReadResponse(google.protobuf.message.Message): b"done", "records", b"records", + "start_cursor", + b"start_cursor", "state", b"state", ], @@ -818,6 +838,294 @@ class ReadResponse(google.protobuf.message.Message): global___ReadResponse = ReadResponse +class FollowRequest(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + STREAM_FIELD_NUMBER: builtins.int + AFTER_FIELD_NUMBER: builtins.int + STATE_FIELD_NUMBER: builtins.int + WAIT_FIELD_NUMBER: builtins.int + MAX_RECORDS_FIELD_NUMBER: builtins.int + MAX_DEPTH_FIELD_NUMBER: builtins.int + MAX_OPEN_FIELD_NUMBER: builtins.int + @property + def stream(self) -> global___StreamAddress: + """The root stream.""" + after: builtins.str + """The root cursor, on the first call. Later calls resume from `state` and ignore it.""" + state: builtins.bytes + """The `state` of the previous answer. Empty on the first call.""" + @property + def wait(self) -> google.protobuf.duration_pb2.Duration: + """How long to wait for a record when no source has one. The service waits at most 60 seconds.""" + max_records: builtins.int + """How many stored records one answer holds at most, across all sources. Zero uses 100. The + service answers with at most 1,000. + """ + max_depth: builtins.int + """How deep links are followed. The root is depth zero. Zero uses 4. The first call's limits + hold for the whole follow, and `state` keeps them. + """ + max_open: builtins.int + """How many linked streams are open at once. Zero uses 16. + Jetty adds fields 8 and 9 for Workflow followers. + """ + def __init__( + self, + *, + stream: global___StreamAddress | None = ..., + after: builtins.str = ..., + state: builtins.bytes = ..., + wait: google.protobuf.duration_pb2.Duration | None = ..., + max_records: builtins.int = ..., + max_depth: builtins.int = ..., + max_open: builtins.int = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal["stream", b"stream", "wait", b"wait"], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "after", + b"after", + "max_depth", + b"max_depth", + "max_open", + b"max_open", + "max_records", + b"max_records", + "state", + b"state", + "stream", + b"stream", + "wait", + b"wait", + ], + ) -> None: ... + +global___FollowRequest = FollowRequest + +class FollowedRecord(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SOURCE_FIELD_NUMBER: builtins.int + RECORD_FIELD_NUMBER: builtins.int + LINK_STATE_FIELD_NUMBER: builtins.int + source: builtins.int + """The source the item belongs to. Zero is the root. A SKIPPED link names the source that holds + the link. + """ + @property + def record(self) -> global___ReadRecord: ... + @property + def link_state(self) -> global___LinkState: ... + def __init__( + self, + *, + source: builtins.int = ..., + record: global___ReadRecord | None = ..., + link_state: global___LinkState | None = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "item", b"item", "link_state", b"link_state", "record", b"record" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "item", + b"item", + "link_state", + b"link_state", + "record", + b"record", + "source", + b"source", + ], + ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["item", b"item"] + ) -> typing_extensions.Literal["record", "link_state"] | None: ... + +global___FollowedRecord = FollowedRecord + +class LinkState(google.protobuf.message.Message): + """What happened to a link, as the follower saw it.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class _Kind: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + + class _KindEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + LinkState._Kind.ValueType + ], + builtins.type, + ): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + KIND_UNSPECIFIED: LinkState._Kind.ValueType # 0 + KIND_OPENED: LinkState._Kind.ValueType # 1 + """The linked stream is open, and its records follow.""" + KIND_DONE: LinkState._Kind.ValueType # 2 + """The linked stream's segment ended: the stream closed, or it reached the JOIN's `until`.""" + KIND_UNAVAILABLE: LinkState._Kind.ValueType # 3 + """The linked stream can't be read. `error` says why. The follow goes on without it.""" + KIND_SKIPPED: LinkState._Kind.ValueType # 4 + """The link was not followed. `reason` says why.""" + + class Kind(_Kind, metaclass=_KindEnumTypeWrapper): ... + KIND_UNSPECIFIED: LinkState.Kind.ValueType # 0 + KIND_OPENED: LinkState.Kind.ValueType # 1 + """The linked stream is open, and its records follow.""" + KIND_DONE: LinkState.Kind.ValueType # 2 + """The linked stream's segment ended: the stream closed, or it reached the JOIN's `until`.""" + KIND_UNAVAILABLE: LinkState.Kind.ValueType # 3 + """The linked stream can't be read. `error` says why. The follow goes on without it.""" + KIND_SKIPPED: LinkState.Kind.ValueType # 4 + """The link was not followed. `reason` says why.""" + + KIND_FIELD_NUMBER: builtins.int + LINK_FIELD_NUMBER: builtins.int + ERROR_FIELD_NUMBER: builtins.int + REASON_FIELD_NUMBER: builtins.int + kind: global___LinkState.Kind.ValueType + @property + def link(self) -> temporalio.bridge.proto.streams.v1.message_pb2.StreamLink: ... + @property + def error(self) -> global___StreamFailure: + """Jetty adds fields 3 and 4 for the close outcome on DONE. + Set on UNAVAILABLE. + """ + reason: builtins.str + """Set on SKIPPED: `depth`, `limit`, `cycle`, `open` or `endpoint`. Set to `withdrawn` on a DONE + that a SUPERSEDED caused, since the attempt that wrote the OPEN was withdrawn. Set to + `resumed` on an OPENED where the same OPEN from a later attempt of that producer reads the + withdrawn segment on from its cursor. + """ + def __init__( + self, + *, + kind: global___LinkState.Kind.ValueType = ..., + link: temporalio.bridge.proto.streams.v1.message_pb2.StreamLink | None = ..., + error: global___StreamFailure | None = ..., + reason: builtins.str = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["error", b"error", "link", b"link"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "error", b"error", "kind", b"kind", "link", b"link", "reason", b"reason" + ], + ) -> None: ... + +global___LinkState = LinkState + +class FollowSource(google.protobuf.message.Message): + """A source that an answer's records name.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + NUMBER_FIELD_NUMBER: builtins.int + PARENT_FIELD_NUMBER: builtins.int + DEPTH_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + number: builtins.int + parent: builtins.int + """The source that holds the source's link. Zero for the root and its links.""" + depth: builtins.int + @property + def stream(self) -> global___StreamAddress: ... + name: builtins.str + """The link's name. Empty for the root.""" + def __init__( + self, + *, + number: builtins.int = ..., + parent: builtins.int = ..., + depth: builtins.int = ..., + stream: global___StreamAddress | None = ..., + name: builtins.str = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "depth", + b"depth", + "name", + b"name", + "number", + b"number", + "parent", + b"parent", + "stream", + b"stream", + ], + ) -> None: ... + +global___FollowSource = FollowSource + +class FollowResponse(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RECORDS_FIELD_NUMBER: builtins.int + STATE_FIELD_NUMBER: builtins.int + DONE_FIELD_NUMBER: builtins.int + SOURCES_FIELD_NUMBER: builtins.int + @property + def records( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___FollowedRecord + ]: ... + state: builtins.bytes + """The composite position: every source's cursor and read state, and the gates. Opaque to lang.""" + done: builtins.bool + """The root ended and every linked segment ended. No later call returns a record.""" + @property + def sources( + self, + ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ + global___FollowSource + ]: + """Each source that `records` names. + Jetty adds fields 5 to 9 for Workflow followers and their replay. + """ + def __init__( + self, + *, + records: collections.abc.Iterable[global___FollowedRecord] | None = ..., + state: builtins.bytes = ..., + done: builtins.bool = ..., + sources: collections.abc.Iterable[global___FollowSource] | None = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "done", + b"done", + "records", + b"records", + "sources", + b"sources", + "state", + b"state", + ], + ) -> None: ... + +global___FollowResponse = FollowResponse + class LatestRequest(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -1496,17 +1804,21 @@ class OutputRecord(google.protobuf.message.Message): KIND_FIELD_NUMBER: builtins.int BODY_FIELD_NUMBER: builtins.int CONTENT_HASH_FIELD_NUMBER: builtins.int + LINK_FIELD_NUMBER: builtins.int topic: builtins.str kind: temporalio.bridge.proto.streams.v1.message_pb2.StreamRecordKind.ValueType - """DATA or FINISH.""" + """DATA, FINISH or LINK.""" @property def body(self) -> temporalio.api.common.v1.message_pb2.Payload: - """The value on DATA, after lang's payload codec. Absent on FINISH.""" + """The value on DATA, after lang's payload codec. Absent on FINISH. Optional on LINK.""" content_hash: builtins.bytes - """The SHA-256 of the body as the payload converter made it, before the codec. Set on DATA. The - service builds each topic's batch digest from these, so a repeat of a batch matches whatever - the codec made of the bodies. + """The SHA-256 of the body as the payload converter made it, before the codec. Set with a body. + The service builds each topic's batch digest from these, so a repeat of a batch matches + whatever the codec made of the bodies. """ + @property + def link(self) -> temporalio.bridge.proto.streams.v1.message_pb2.StreamLink: + """Set on LINK only.""" def __init__( self, *, @@ -1514,9 +1826,10 @@ class OutputRecord(google.protobuf.message.Message): kind: temporalio.bridge.proto.streams.v1.message_pb2.StreamRecordKind.ValueType = ..., body: temporalio.api.common.v1.message_pb2.Payload | None = ..., content_hash: builtins.bytes = ..., + link: temporalio.bridge.proto.streams.v1.message_pb2.StreamLink | None = ..., ) -> None: ... def HasField( - self, field_name: typing_extensions.Literal["body", b"body"] + self, field_name: typing_extensions.Literal["body", b"body", "link", b"link"] ) -> builtins.bool: ... def ClearField( self, @@ -1527,6 +1840,8 @@ class OutputRecord(google.protobuf.message.Message): b"content_hash", "kind", b"kind", + "link", + b"link", "topic", b"topic", ], diff --git a/temporalio/bridge/proto/streams/v1/__init__.py b/temporalio/bridge/proto/streams/v1/__init__.py index fc6311caae..be41c11ddd 100644 --- a/temporalio/bridge/proto/streams/v1/__init__.py +++ b/temporalio/bridge/proto/streams/v1/__init__.py @@ -1,6 +1,7 @@ -from .message_pb2 import StreamRecord, StreamRecordKind +from .message_pb2 import StreamLink, StreamRecord, StreamRecordKind __all__ = [ + "StreamLink", "StreamRecord", "StreamRecordKind", ] diff --git a/temporalio/bridge/proto/streams/v1/message_pb2.py b/temporalio/bridge/proto/streams/v1/message_pb2.py index 747a2452ed..ff07ff7ca6 100644 --- a/temporalio/bridge/proto/streams/v1/message_pb2.py +++ b/temporalio/bridge/proto/streams/v1/message_pb2.py @@ -18,9 +18,12 @@ from temporalio.api.common.v1 import ( message_pb2 as temporal_dot_api_dot_common_dot_v1_dot_message__pb2, ) +from temporalio.api.stream.v1 import ( + message_pb2 as temporal_dot_api_dot_stream_dot_v1_dot_message__pb2, +) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n%temporal/sdk/streams/v1/message.proto\x12\x17temporal.sdk.streams.v1\x1a$temporal/api/common/v1/message.proto"\x8e\x03\n\x0cStreamRecord\x12-\n\x04\x62ody\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x45\n\x08metadata\x18\x02 \x03(\x0b\x32\x33.temporal.sdk.streams.v1.StreamRecord.MetadataEntry\x12\r\n\x05topic\x18\x03 \x01(\t\x12\x37\n\x04kind\x18\x04 \x01(\x0e\x32).temporal.sdk.streams.v1.StreamRecordKind\x12\x13\n\x0bproducer_id\x18\x05 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x06 \x01(\x03\x12\x10\n\x08sequence\x18\x07 \x01(\x03\x12\x1d\n\x15task_started_event_id\x18\t \x01(\x03\x12\x17\n\x0f\x63ompleted_tasks\x18\n \x03(\x03\x1aP\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01*\xba\x01\n\x10StreamRecordKind\x12"\n\x1eSTREAM_RECORD_KIND_UNSPECIFIED\x10\x00\x12\x1b\n\x17STREAM_RECORD_KIND_DATA\x10\x01\x12\x1d\n\x19STREAM_RECORD_KIND_FINISH\x10\x02\x12\x1c\n\x18STREAM_RECORD_KIND_CLOSE\x10\x05\x12\x1c\n\x18STREAM_RECORD_KIND_TOPIC\x10\x06"\x04\x08\x03\x10\x03"\x04\x08\x04\x10\x04\x42\x32\xea\x02/Temporalio::Internal::Bridge::Api::StreamRecordb\x06proto3' + b'\n%temporal/sdk/streams/v1/message.proto\x12\x17temporal.sdk.streams.v1\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/stream/v1/message.proto"\xc1\x03\n\x0cStreamRecord\x12-\n\x04\x62ody\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x45\n\x08metadata\x18\x02 \x03(\x0b\x32\x33.temporal.sdk.streams.v1.StreamRecord.MetadataEntry\x12\r\n\x05topic\x18\x03 \x01(\t\x12\x37\n\x04kind\x18\x04 \x01(\x0e\x32).temporal.sdk.streams.v1.StreamRecordKind\x12\x13\n\x0bproducer_id\x18\x05 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x06 \x01(\x03\x12\x10\n\x08sequence\x18\x07 \x01(\x03\x12\x31\n\x04link\x18\x08 \x01(\x0b\x32#.temporal.sdk.streams.v1.StreamLink\x12\x1d\n\x15task_started_event_id\x18\t \x01(\x03\x12\x17\n\x0f\x63ompleted_tasks\x18\n \x03(\x03\x1aP\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload:\x02\x38\x01"\x90\x02\n\nStreamLink\x12\x38\n\x05\x65vent\x18\x01 \x01(\x0e\x32).temporal.sdk.streams.v1.StreamLink.Event\x12\x37\n\x06stream\x18\x02 \x01(\x0b\x32\'.temporal.api.stream.v1.StreamReference\x12\x0c\n\x04name\x18\x03 \x01(\t\x12\r\n\x05\x61\x66ter\x18\x04 \x01(\t\x12\r\n\x05until\x18\x05 \x01(\t\x12\x11\n\tnamespace\x18\x06 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x07 \x01(\t">\n\x05\x45vent\x12\x15\n\x11\x45VENT_UNSPECIFIED\x10\x00\x12\x0e\n\nEVENT_OPEN\x10\x01\x12\x0e\n\nEVENT_JOIN\x10\x02*\xd1\x01\n\x10StreamRecordKind\x12"\n\x1eSTREAM_RECORD_KIND_UNSPECIFIED\x10\x00\x12\x1b\n\x17STREAM_RECORD_KIND_DATA\x10\x01\x12\x1d\n\x19STREAM_RECORD_KIND_FINISH\x10\x02\x12\x1b\n\x17STREAM_RECORD_KIND_LINK\x10\x04\x12\x1c\n\x18STREAM_RECORD_KIND_CLOSE\x10\x05\x12\x1c\n\x18STREAM_RECORD_KIND_TOPIC\x10\x06"\x04\x08\x03\x10\x03\x42\x32\xea\x02/Temporalio::Internal::Bridge::Api::StreamRecordb\x06proto3' ) _STREAMRECORDKIND = DESCRIPTOR.enum_types_by_name["StreamRecordKind"] @@ -28,12 +31,15 @@ STREAM_RECORD_KIND_UNSPECIFIED = 0 STREAM_RECORD_KIND_DATA = 1 STREAM_RECORD_KIND_FINISH = 2 +STREAM_RECORD_KIND_LINK = 4 STREAM_RECORD_KIND_CLOSE = 5 STREAM_RECORD_KIND_TOPIC = 6 _STREAMRECORD = DESCRIPTOR.message_types_by_name["StreamRecord"] _STREAMRECORD_METADATAENTRY = _STREAMRECORD.nested_types_by_name["MetadataEntry"] +_STREAMLINK = DESCRIPTOR.message_types_by_name["StreamLink"] +_STREAMLINK_EVENT = _STREAMLINK.enum_types_by_name["Event"] StreamRecord = _reflection.GeneratedProtocolMessageType( "StreamRecord", (_message.Message,), @@ -55,6 +61,17 @@ _sym_db.RegisterMessage(StreamRecord) _sym_db.RegisterMessage(StreamRecord.MetadataEntry) +StreamLink = _reflection.GeneratedProtocolMessageType( + "StreamLink", + (_message.Message,), + { + "DESCRIPTOR": _STREAMLINK, + "__module__": "temporal.sdk.streams.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.sdk.streams.v1.StreamLink) + }, +) +_sym_db.RegisterMessage(StreamLink) + if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = ( @@ -62,10 +79,14 @@ ) _STREAMRECORD_METADATAENTRY._options = None _STREAMRECORD_METADATAENTRY._serialized_options = b"8\001" - _STREAMRECORDKIND._serialized_start = 506 - _STREAMRECORDKIND._serialized_end = 692 - _STREAMRECORD._serialized_start = 105 - _STREAMRECORD._serialized_end = 503 - _STREAMRECORD_METADATAENTRY._serialized_start = 423 - _STREAMRECORD_METADATAENTRY._serialized_end = 503 + _STREAMRECORDKIND._serialized_start = 870 + _STREAMRECORDKIND._serialized_end = 1079 + _STREAMRECORD._serialized_start = 143 + _STREAMRECORD._serialized_end = 592 + _STREAMRECORD_METADATAENTRY._serialized_start = 512 + _STREAMRECORD_METADATAENTRY._serialized_end = 592 + _STREAMLINK._serialized_start = 595 + _STREAMLINK._serialized_end = 867 + _STREAMLINK_EVENT._serialized_start = 805 + _STREAMLINK_EVENT._serialized_end = 867 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/bridge/proto/streams/v1/message_pb2.pyi b/temporalio/bridge/proto/streams/v1/message_pb2.pyi index 3a4ded7e14..cd551fafe2 100644 --- a/temporalio/bridge/proto/streams/v1/message_pb2.pyi +++ b/temporalio/bridge/proto/streams/v1/message_pb2.pyi @@ -14,6 +14,7 @@ import google.protobuf.internal.enum_type_wrapper import google.protobuf.message import temporalio.api.common.v1.message_pb2 +import temporalio.api.stream.v1.message_pb2 if sys.version_info >= (3, 10): import typing as typing_extensions @@ -41,6 +42,10 @@ class _StreamRecordKindEnumTypeWrapper( """The producer named by `producer_id` writes nothing more on `topic`. Says nothing about that producer's outcome and does not end the stream. """ + STREAM_RECORD_KIND_LINK: _StreamRecordKind.ValueType # 4 + """A reference to another stream. `link` carries it. `body` may carry a + value, after the payload codec. + """ STREAM_RECORD_KIND_CLOSE: _StreamRecordKind.ValueType # 5 """Ends `topic` for every reader, or every topic of the chain when `topic` is empty. Readers drop every record after the first one. On a store @@ -62,6 +67,10 @@ STREAM_RECORD_KIND_FINISH: StreamRecordKind.ValueType # 2 """The producer named by `producer_id` writes nothing more on `topic`. Says nothing about that producer's outcome and does not end the stream. """ +STREAM_RECORD_KIND_LINK: StreamRecordKind.ValueType # 4 +"""A reference to another stream. `link` carries it. `body` may carry a +value, after the payload codec. +""" STREAM_RECORD_KIND_CLOSE: StreamRecordKind.ValueType # 5 """Ends `topic` for every reader, or every topic of the chain when `topic` is empty. Readers drop every record after the first one. On a store @@ -113,6 +122,7 @@ class StreamRecord(google.protobuf.message.Message): PRODUCER_ID_FIELD_NUMBER: builtins.int ATTEMPT_FIELD_NUMBER: builtins.int SEQUENCE_FIELD_NUMBER: builtins.int + LINK_FIELD_NUMBER: builtins.int TASK_STARTED_EVENT_ID_FIELD_NUMBER: builtins.int COMPLETED_TASKS_FIELD_NUMBER: builtins.int @property @@ -142,6 +152,11 @@ class StreamRecord(google.protobuf.message.Message): """The producer's position within its attempt, starting at one, or zero when it does not number its records. """ + @property + def link(self) -> global___StreamLink: + """Set on LINK only. It stays out of the payload codec, as ids in History + do, so the stream service can follow it. + """ task_started_event_id: builtins.int """Set by a Workflow run's own producer and its Local Activity calls only: the WorkflowTaskStarted event id of the task attempt that wrote the @@ -170,11 +185,12 @@ class StreamRecord(google.protobuf.message.Message): producer_id: builtins.str = ..., attempt: builtins.int = ..., sequence: builtins.int = ..., + link: global___StreamLink | None = ..., task_started_event_id: builtins.int = ..., completed_tasks: collections.abc.Iterable[builtins.int] | None = ..., ) -> None: ... def HasField( - self, field_name: typing_extensions.Literal["body", b"body"] + self, field_name: typing_extensions.Literal["body", b"body", "link", b"link"] ) -> builtins.bool: ... def ClearField( self, @@ -187,6 +203,8 @@ class StreamRecord(google.protobuf.message.Message): b"completed_tasks", "kind", b"kind", + "link", + b"link", "metadata", b"metadata", "producer_id", @@ -201,3 +219,103 @@ class StreamRecord(google.protobuf.message.Message): ) -> None: ... global___StreamRecord = StreamRecord + +class StreamLink(google.protobuf.message.Message): + """A reference from one stream to another, for followers.""" + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + class _Event: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + + class _EventEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + StreamLink._Event.ValueType + ], + builtins.type, + ): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + EVENT_UNSPECIFIED: StreamLink._Event.ValueType # 0 + EVENT_OPEN: StreamLink._Event.ValueType # 1 + """A follower starts reading `stream` strictly after `after`.""" + EVENT_JOIN: StreamLink._Event.ValueType # 2 + """The writer waited for the link named `name` up to `until`, or to + its end when `until` is empty. + """ + + class Event(_Event, metaclass=_EventEnumTypeWrapper): ... + EVENT_UNSPECIFIED: StreamLink.Event.ValueType # 0 + EVENT_OPEN: StreamLink.Event.ValueType # 1 + """A follower starts reading `stream` strictly after `after`.""" + EVENT_JOIN: StreamLink.Event.ValueType # 2 + """The writer waited for the link named `name` up to `until`, or to + its end when `until` is empty. + """ + + EVENT_FIELD_NUMBER: builtins.int + STREAM_FIELD_NUMBER: builtins.int + NAME_FIELD_NUMBER: builtins.int + AFTER_FIELD_NUMBER: builtins.int + UNTIL_FIELD_NUMBER: builtins.int + NAMESPACE_FIELD_NUMBER: builtins.int + ENDPOINT_FIELD_NUMBER: builtins.int + event: global___StreamLink.Event.ValueType + @property + def stream(self) -> temporalio.api.stream.v1.message_pb2.StreamReference: + """The linked stream. Required on OPEN. Empty on JOIN.""" + name: builtins.str + """Names the link within its producer. A JOIN matches the OPEN with the + same name. At most 256 bytes. Required. + """ + after: builtins.str + """A cursor of the linked stream. Empty starts at the oldest retained + record. + """ + until: builtins.str + """On JOIN, a cursor of the linked stream where the segment ends. Empty is + the stream's end. + """ + namespace: builtins.str + """The linked stream's namespace. Empty is the namespace of the stream + that holds the link. + """ + endpoint: builtins.str + """The Nexus endpoint that serves the linked stream. Empty is the endpoint + that served the stream that holds the link. + """ + def __init__( + self, + *, + event: global___StreamLink.Event.ValueType = ..., + stream: temporalio.api.stream.v1.message_pb2.StreamReference | None = ..., + name: builtins.str = ..., + after: builtins.str = ..., + until: builtins.str = ..., + namespace: builtins.str = ..., + endpoint: builtins.str = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["stream", b"stream"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "after", + b"after", + "endpoint", + b"endpoint", + "event", + b"event", + "name", + b"name", + "namespace", + b"namespace", + "stream", + b"stream", + "until", + b"until", + ], + ) -> None: ... + +global___StreamLink = StreamLink diff --git a/temporalio/bridge/streams_generated.py b/temporalio/bridge/streams_generated.py index e2b1f2ced8..130a82eb1a 100644 --- a/temporalio/bridge/streams_generated.py +++ b/temporalio/bridge/streams_generated.py @@ -50,6 +50,14 @@ async def describe_store( "DescribeStore", req, temporalio.bridge.proto.streams.DescribeStoreResponse ) + async def follow( + self, req: temporalio.bridge.proto.streams.FollowRequest + ) -> temporalio.bridge.proto.streams.FollowResponse: + """Invokes the StreamService.follow call.""" + return await self._store.call( + "Follow", req, temporalio.bridge.proto.streams.FollowResponse + ) + async def latest( self, req: temporalio.bridge.proto.streams.LatestRequest ) -> temporalio.bridge.proto.streams.LatestResponse: diff --git a/temporalio/contrib/streams/README.md b/temporalio/contrib/streams/README.md index e481cdb35e..b3cb185ea7 100644 --- a/temporalio/contrib/streams/README.md +++ b/temporalio/contrib/streams/README.md @@ -156,6 +156,79 @@ Known limit: on the topic, the reader can't prove the call's task failed, and its records stand. +## Link records and followers + +A stream can hold a reference to another Workflow's stream, as a `LINK` +record. A follower reads the stream and each stream its links open in one +order. So an agent writes a link per child agent in place of copying the +child's records, and a reader still sees one ordered view. + +```python +# Workflow code (agent). Links go out with the activation, as appended values do. +events = stream_producer(EVENTS) +events.link(StreamRef.for_workflow(child_id), name="sub-3", after=child_cursor) # OPEN +reply = await child.execute_update(...) +events.join(name="sub-3", until=reply.turn_end) # JOIN, the agent waited here + +# Client code. One ordered view of the agent and every stream it links. +async for item in stream_handle(client, agent_id, topic=EVENTS).follow(): + if item.link_state is not None: # OPENED, DONE, UNAVAILABLE or SKIPPED + ui.mark(item.source, item.link_state) + else: + ui.render(item.source, item.record) + if item.position is not None: + saved = item.position # resumes every open link, not only the agent +``` + +- **Writing.** `StreamProducer.link(ref, *, name, after=None, value=None)` + and `WorkflowStreamProducer.link(...)` write an `OPEN`. The follower reads + the linked stream strictly after `after`, or from its oldest record. + `join(*, name, until=None)` writes a `JOIN`. It says the producer waited + for the link up to `until`, or to the linked stream's end. A link names a + Workflow's stream only. Any other ref raises `ValueError`. +- **Mixed batches.** `append` also takes a `StreamLink` next to plain values, + so `append(event, StreamLink(LinkEvent.OPEN, ref, name))` writes the event + and its link as one batch, with one dedupe identity. +- **Following.** `StreamHandle.follow()` and `ActivityStreamHandle.follow()` + read a stream and the streams it links, from client or Activity code. + Workflow code can't follow. +- **Order.** Each stream keeps its own order. A linked stream's records + never come before their `OPEN`. A record that comes after a `JOIN` waits + until the linked stream reached the join point or its end. Other records + come in the order they were read. No clock decides the order. +- **Link reports.** Items with `link_state` say what happened to a link. + `OPENED` when a segment starts. `DONE` when it ends. `UNAVAILABLE` with + the error when it can't be read, for example a Workflow that never + started, after `owner_start_grace`. `SKIPPED` with a reason: `depth` past + `max_depth` (4), `limit` past `max_open` (16), `cycle` for a link back to + a stream on the path, `open` for a stream already open, and `endpoint` + for a stream in another namespace or behind a Nexus endpoint. A linked + stream never fails the follow. +- **Positions.** Each record keeps its own stream's cursor. The last item of + each batch carries a `FollowPosition`, which resumes the root and every + open segment with no loss and no repeat. To resume from an earlier item, + resume from the previous batch end. +- **Retries.** An attempt owns the `OPEN`s it writes. A `SUPERSEDED` ends + each segment the withdrawn attempt opened, with `DONE` and reason + `withdrawn`. The same `OPEN` from a later attempt of that producer, with + the same name, stream and `after`, resumes the segment where it paused, + with `OPENED` and reason `resumed`. So a retry that writes the same `OPEN` + again shows each linked record once. Another `OPEN` of the name reads the + stream from its `after`. +- **Plaintext.** A link's ref, name and cursors are stored as plaintext, as + Workflow ids are in History. A link's value goes through the codec, so put + secrets there. + +Known limits: + +- **Size.** The follow position grows with the open segments. With the + default bounds it stays below 64 KiB. +- **Waits.** An idle follow holds one store wait per open source. +- **Old readers.** A reader of an earlier release reads a `LINK` as a + record it can't decode. +- **Withdrawn joins.** A `JOIN` from a withdrawn attempt stays in effect, + since its segment ended before the `SUPERSEDED`. + ## A Workflow publishes to its own stream `stream_producer` is one function for Workflow code and Activity code. In diff --git a/temporalio/contrib/streams/__init__.py b/temporalio/contrib/streams/__init__.py index 11e7afb0a4..7fce16b106 100644 --- a/temporalio/contrib/streams/__init__.py +++ b/temporalio/contrib/streams/__init__.py @@ -80,6 +80,12 @@ StreamStorageError, StreamUnsupportedError, ) +from temporalio.contrib.streams._follow import ( + FollowedItem, + FollowPosition, + LinkState, + LinkStateKind, +) from temporalio.contrib.streams._handles import ( ActivityStreamHandle, StreamHandle, @@ -90,7 +96,9 @@ from temporalio.contrib.streams._plugin import StreamStorePlugin from temporalio.contrib.streams._record import ( Cursor, + LinkEvent, RecordKind, + StreamLink, StreamRecord, Supersession, ) @@ -111,6 +119,11 @@ "Cursor", "DEFAULT_TOPIC", "END", + "FollowPosition", + "FollowedItem", + "LinkEvent", + "LinkState", + "LinkStateKind", "RecordKind", "StoreCapabilities", "StreamClosedError", @@ -118,6 +131,7 @@ "StreamError", "StreamExpiredError", "StreamHandle", + "StreamLink", "StreamNotFoundError", "StreamOutcomeUnknownError", "StreamOwnerKind", diff --git a/temporalio/contrib/streams/_follow.py b/temporalio/contrib/streams/_follow.py new file mode 100644 index 0000000000..a897795463 --- /dev/null +++ b/temporalio/contrib/streams/_follow.py @@ -0,0 +1,260 @@ +"""Following a stream and the streams its links open, in one order. + +The stream runtime's ``Follow`` keeps the order. It reads the root and each open linked +stream, holds a child's records until its ``OPEN`` and a parent's records +after a ``JOIN`` until the child reached the join point, and reports each +link's state. What stays here is what needs the user's types: the codec of +each source, converting values, and the position a follower resumes from. +""" + +from __future__ import annotations + +import base64 +import enum +from collections.abc import AsyncGenerator +from dataclasses import dataclass +from datetime import timedelta +from typing import TYPE_CHECKING, Any, Generic, TypeVar + +from temporalio.bridge.proto.streams import FollowRequest, FollowResponse +from temporalio.bridge.proto.streams import LinkState as WireLinkState +from temporalio.contrib.streams._body import decode_body +from temporalio.contrib.streams._errors import error_from_failure +from temporalio.contrib.streams._plugin import call +from temporalio.contrib.streams._record import Cursor, StreamLink, StreamRecord +from temporalio.contrib.streams._ref import StreamRef, from_address +from temporalio.contrib.streams._wire import from_wire_link, to_record +from temporalio.converter import ( + DataConverter, + PayloadConverter, + WorkflowSerializationContext, +) + +if TYPE_CHECKING: + from temporalio.contrib.streams._handles import StreamHandle + +__all__ = [ + "FollowPosition", + "FollowedItem", + "LinkState", + "LinkStateKind", +] + +T = TypeVar("T") + +_FOLLOW_WAIT = timedelta(seconds=30) +_MAX_RECORDS = 100 + + +@dataclass(frozen=True) +class FollowPosition: + """Where a follower stands, in the root stream and in each open link. + + Pass it as ``after=`` to resume. It is plain data, so it can cross + Continue-as-New. ``cursor`` is the root's cursor, and ``state`` the stream runtime's + follow state in base64, which holds the cursor of each open linked + stream. + """ + + cursor: str + state: str = "" + + +@enum.unique +class LinkStateKind(enum.IntEnum): + """What happened to a link, as a follower reports it.""" + + OPENED = 1 + """The follower started reading the linked stream. + + With ``reason`` ``resumed``, a later attempt of the producer wrote the same ``OPEN`` + again, and the follower reads the withdrawn segment on from where it paused. + """ + DONE = 2 + """The linked stream ended, or reached the ``JOIN`` point. + + With ``reason`` ``withdrawn``, the attempt that wrote the ``OPEN`` was superseded. The + segment pauses, and the same ``OPEN`` from a later attempt resumes it. + """ + UNAVAILABLE = 3 + """The linked stream can't be read. The follow goes on without it.""" + SKIPPED = 4 + """The follower did not open the link. ``reason`` says why.""" + + +@dataclass(frozen=True) +class LinkState: + """A follower's report on one link.""" + + kind: LinkStateKind + link: StreamLink + error: BaseException | None = None + """On ``UNAVAILABLE``, why the linked stream can't be read.""" + reason: str = "" + """On ``SKIPPED``: ``depth``, ``limit``, ``cycle``, ``open`` or ``endpoint``. + + On ``DONE``, ``withdrawn`` or empty. On ``OPENED``, ``resumed`` or empty. + """ + + +@dataclass(frozen=True) +class FollowedItem(Generic[T]): + """One record or link report of a follow, with the stream it came from. + + Exactly one of ``record`` and ``link_state`` is set. + """ + + source: StreamRef + """The stream the item belongs to.""" + depth: int + """How many links away from the root the source is. The root is 0.""" + record: StreamRecord[T] | None + link_state: LinkState | None + position: FollowPosition | None + """Where to resume after this item. + + Set only on the last item of each batch, since the stream runtime gives the follow + state at each batch end. Other items carry ``None``. To resume from an + earlier item, resume from the previous batch end, which gives that + item and the rest of its batch again.""" + + +@dataclass(frozen=True) +class FollowSourceInfo: + """What a follow knows of one source, by its number.""" + + ref: StreamRef + depth: int + + +def update_sources( + sources: dict[int, FollowSourceInfo], response: FollowResponse +) -> None: + """Add the sources ``response`` names to ``sources``.""" + for source in response.sources: + sources[source.number] = FollowSourceInfo( + from_address(source.stream), source.depth + ) + + +async def decode_follow( + converter: DataConverter, + namespace: str, + root: StreamRef, + response: FollowResponse, +) -> None: + """Run the codec over every body in ``response``, in place. + + Each source decodes in the context of the Workflow that owns it, as that + Workflow encoded it. + """ + contexts: dict[int, DataConverter] = { + 0: converter.with_context( + WorkflowSerializationContext( + namespace=namespace, workflow_id=root.workflow_id + ) + ) + } + for source in response.sources: + stream = source.stream + contexts[source.number] = converter.with_context( + WorkflowSerializationContext( + namespace=stream.namespace or namespace, workflow_id=stream.workflow_id + ) + ) + for followed in response.records: + if not followed.HasField("record"): + continue + stored = followed.record.stored + if followed.record.HasField("stored") and stored.HasField("body"): + context = contexts.get(followed.source, contexts[0]) + stored.body.CopyFrom(await decode_body(context, stored.body)) + + +def follow_items( + payload_converter: PayloadConverter, + sources: dict[int, FollowSourceInfo], + response: FollowResponse, + result_type: type | None, + cursor: str, +) -> tuple[list[FollowedItem[Any]], str]: + """The items of a decoded ``response``, and the root cursor after them. + + The last item carries the position after the batch. + """ + items: list[FollowedItem[Any]] = [] + for followed in response.records: + source = sources[followed.source] + record = link_state = None + if followed.HasField("record"): + record = to_record( + payload_converter, + followed.record, + result_type if followed.source == 0 else None, + ) + if followed.source == 0: + cursor = record.cursor.token + else: + link_state = _link_state(followed.link_state) + items.append(FollowedItem(source.ref, source.depth, record, link_state, None)) + if items: + position = FollowPosition(cursor, base64.b64encode(response.state).decode()) + last = items[-1] + items[-1] = FollowedItem( + last.source, last.depth, last.record, last.link_state, position + ) + return items, cursor + + +def _link_state(wire: WireLinkState) -> LinkState: + return LinkState( + kind=LinkStateKind(wire.kind), + link=from_wire_link(wire.link), + error=error_from_failure(wire.error) if wire.HasField("error") else None, + reason=wire.reason, + ) + + +async def follow( + handle: StreamHandle, + topic: str, + after: Cursor | FollowPosition, + max_depth: int, + max_open: int, + result_type: type | None, +) -> AsyncGenerator[FollowedItem[Any], None]: + """See :meth:`temporalio.contrib.streams.StreamHandle.follow`.""" + service = await handle._plugin._service_for(handle._client) + if isinstance(after, FollowPosition): + cursor, state = after.cursor, base64.b64decode(after.state) + else: + cursor, state = after.token, b"" + request = FollowRequest( + stream=handle._address(topic, run_id=await handle._read_run_id()), + after=cursor, + state=state, + max_records=_MAX_RECORDS, + max_depth=max_depth, + max_open=max_open, + ) + request.wait.FromTimedelta(_FOLLOW_WAIT) + root = handle.ref.with_topic(topic) + sources = {0: FollowSourceInfo(root, 0)} + converter = handle._client.data_converter + namespace = handle._client.namespace + while True: + response: FollowResponse = await call(service.follow(request)) + update_sources(sources, response) + await decode_follow(converter, namespace, root, response) + items, cursor = follow_items( + converter.payload_converter, + sources, + response, + result_type, + cursor, + ) + for item in items: + yield item + if response.done: + return + request.state = response.state diff --git a/temporalio/contrib/streams/_handles.py b/temporalio/contrib/streams/_handles.py index 72e5e40646..e38822defb 100644 --- a/temporalio/contrib/streams/_handles.py +++ b/temporalio/contrib/streams/_handles.py @@ -47,13 +47,20 @@ StreamNotFoundError, StreamUnsupportedError, ) +from temporalio.contrib.streams._follow import FollowedItem, FollowPosition, follow from temporalio.contrib.streams._local_activity import LocalCall, current_local_call from temporalio.contrib.streams._plugin import ( StreamStorePlugin, call, store_for_client, ) -from temporalio.contrib.streams._record import Cursor, RecordKind, StreamRecord +from temporalio.contrib.streams._record import ( + Cursor, + LinkEvent, + RecordKind, + StreamLink, + StreamRecord, +) from temporalio.contrib.streams._ref import StreamRef from temporalio.contrib.streams._topic import StreamTopic, resolve_topic from temporalio.contrib.streams._wire import WireRecord, from_read, to_wire @@ -115,10 +122,14 @@ def attempt(self) -> int: """The attempt this producer writes.""" return self._attempt - async def append(self, *values: T) -> Cursor: + async def append(self, *values: T | StreamLink) -> Cursor: """Append ``values`` as one batch and return the cursor of its last record. - The batch lands whole and in order. The producer moves its sequence + A :class:`temporalio.contrib.streams.StreamLink` among the values is a + ``LINK`` record, as :meth:`link` and :meth:`join` write it, and the + other values are ``DATA`` records. So one batch can hold a value and + the link that goes with it, under one dedupe identity. The batch lands + whole and in order. The producer moves its sequence past the batch only when the append succeeds, so a call that raised :class:`temporalio.contrib.streams.StreamOutcomeUnknownError` can be repeated with the same values: the store answers a repeat of the @@ -144,11 +155,19 @@ async def append(self, *values: T) -> Cursor: StreamRefusedError: The store refused the batch, for example because it is out of memory. Nothing was written. StreamClosedError: The stream refuses appends. + ValueError: A link names nothing, or names a stream that no + Workflow owns. Nothing was written. """ + items: list[tuple[RecordKind, Any, StreamLink | None]] = [ + (RecordKind.LINK, None, v) + if isinstance(v, StreamLink) + else (RecordKind.DATA, v, None) + for v in values + ] async with self._lock: - if not values: + if not items: return self._last - return await self._write(RecordKind.DATA, values) + return await self._write(items) async def finish(self) -> Cursor: """Write ``FINISH`` for this producer on this topic and return its cursor. @@ -157,9 +176,50 @@ async def finish(self) -> Cursor: work behind it succeeded, and it does not end anyone's read. """ async with self._lock: - return await self._write(RecordKind.FINISH, [None]) + return await self._write([(RecordKind.FINISH, None, None)]) + + async def link( + self, + ref: StreamRef, + *, + name: str, + after: Cursor | None = None, + value: Any = None, + ) -> Cursor: + """Write a ``LINK`` that opens ``ref``'s stream, and return its cursor. + + A follower reads the linked stream from strictly after ``after``, or + from its oldest record, in one order with this stream. ``name`` names + the link within this producer, and :meth:`join` names it again. + ``ref`` is stored as plaintext. ``value``, when given, goes through + the converter and codec as a ``DATA`` value does. + + Raises: + ValueError: ``name`` is empty or over 256 bytes, or ``ref`` names + a stream that no Workflow owns. + StreamError: As :meth:`append` raises. + """ + link = StreamLink(LinkEvent.OPEN, ref, name, after=after) + async with self._lock: + return await self._write([(RecordKind.LINK, value, link)]) + + async def join(self, *, name: str, until: Cursor | None = None) -> Cursor: + """Write a ``LINK`` that says this producer waited for link ``name``. + + A follower holds this stream's later records until the linked stream + reached ``until``, or its end when ``until`` is ``None``. + + Raises: + ValueError: ``name`` is empty or over 256 bytes. + StreamError: As :meth:`append` raises. + """ + link = StreamLink(LinkEvent.JOIN, None, name, until=until) + async with self._lock: + return await self._write([(RecordKind.LINK, None, link)]) - async def _write(self, kind: RecordKind, values: Any) -> Cursor: + async def _write( + self, items: list[tuple[RecordKind, Any, StreamLink | None]] + ) -> Cursor: converter = self._handle._converter wires = [ to_wire( @@ -170,8 +230,9 @@ async def _write(self, kind: RecordKind, values: Any) -> Cursor: producer_id=self._producer_id, attempt=self._attempt, sequence=self._sequence + index, + link=link, ) - for index, value in enumerate(values) + for index, (kind, value, link) in enumerate(items) ] request = AppendRequest( stream=self._handle._address(self._topic), @@ -208,6 +269,8 @@ async def _append_records( records = [] for wire in wires: record = AppendRecord(kind=wire.kind) + if wire.HasField("link"): + record.link.CopyFrom(wire.link) if wire.HasField("body"): record.body.CopyFrom(next(bodies)) record.content_hash = content_hash(wire.body) @@ -364,6 +427,53 @@ async def _read( request.after = response.cursor request.state = response.state + def follow( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor | FollowPosition = BEGINNING, + max_depth: int = 4, + max_open: int = 16, + result_type: type | None = None, + ) -> AsyncGenerator[FollowedItem[Any], None]: + """Yield the records of ``topic`` and of the streams its links open, in one order. + + Each ``LINK`` with event ``OPEN`` opens a segment of the linked + stream, and the follower reads it in place. A child's records never + come before its ``OPEN``. A record that comes after a ``JOIN`` waits + until the child reached the join point or its end. Other records + come in the order they were read. No clock decides the order. Each + item names its source stream, and keeps that stream's own cursor. + + Link reports come as items with ``link_state`` set. ``OPENED`` when a + segment starts, ``DONE`` when it ends, ``UNAVAILABLE`` with the error + when the child can't be read, and ``SKIPPED`` for a link past + ``max_depth`` or ``max_open``, to a stream already open, to an + ancestor, or to another namespace or Nexus endpoint. A child never + fails the follow. The link records themselves come too, as records of + their stream. + + ``result_type`` applies to the root's records. A child's records + decode with the default rules, since a child's topic may hold + another type. + + Only the last item of each batch carries ``position``, which resumes + the root and every open segment with no loss and no repeat. An + earlier item carries ``None``. To resume from it, resume from the + previous batch end, which repeats the rest of that batch. The follow + ends when the root is done and every segment ended. + + Raises: + ValueError: A bound is below one, or ``result_type`` was passed + with a topic definition. + StreamError: As :meth:`read` raises for the root. Only a + ``STORAGE`` failure of a child fails the follow. + """ + if max_depth < 1 or max_open < 1: + raise ValueError("max_depth and max_open must be at least 1") + name, result_type = self._resolve(topic, result_type) + return follow(self, name, after, max_depth, max_open, result_type) + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: """The cursor of the newest record on ``topic``, or ``BEGINNING`` when empty. @@ -536,6 +646,24 @@ def read( topic=topic, after=after, result_type=result_type, idle_wait=idle_wait ) + def follow( + self, + *, + topic: str | StreamTopic[Any] | None = None, + after: Cursor | FollowPosition = BEGINNING, + max_depth: int = 4, + max_open: int = 16, + result_type: type | None = None, + ) -> AsyncGenerator[FollowedItem[Any], None]: + """See :meth:`temporalio.contrib.streams.StreamHandle.follow`.""" + return self._inner.follow( + topic=topic, + after=after, + max_depth=max_depth, + max_open=max_open, + result_type=result_type, + ) + async def latest(self, *, topic: str | StreamTopic[Any] | None = None) -> Cursor: """See :meth:`temporalio.contrib.streams.StreamHandle.latest`.""" return await self._inner.latest(topic=topic) diff --git a/temporalio/contrib/streams/_record.py b/temporalio/contrib/streams/_record.py index 6a91f7d454..3563e76e9c 100644 --- a/temporalio/contrib/streams/_record.py +++ b/temporalio/contrib/streams/_record.py @@ -8,11 +8,16 @@ import enum from dataclasses import dataclass -from typing import Generic, TypeVar +from typing import TYPE_CHECKING, Generic, TypeVar + +if TYPE_CHECKING: + from temporalio.contrib.streams._ref import StreamRef __all__ = [ "Cursor", + "LinkEvent", "RecordKind", + "StreamLink", "StreamRecord", "Supersession", ] @@ -64,7 +69,14 @@ class RecordKind(enum.IntEnum): earlier attempt's records twice is harmless and missing it is not. """ - # 4 is kept for a kind a later release stores. + LINK = 4 + """A reference to another stream, in :attr:`StreamRecord.link`. + + A follower reads the linked stream in place, in one order with this + stream. ``value`` is an optional value the writer put on the link, such + as a label. A reader that follows nothing sees it as a record of its + own. + """ CLOSE = 5 """The topic in ``topic`` is closed, or every topic of the run chain when empty. @@ -86,6 +98,23 @@ class RecordKind(enum.IntEnum): """ +@enum.unique +class LinkEvent(enum.IntEnum): + """What a :attr:`RecordKind.LINK` record says about its linked stream. + + The values match ``temporal.sdk.streams.v1.StreamLink.Event``. + """ + + OPEN = 1 + """A follower starts reading the linked stream strictly after ``after``.""" + + JOIN = 2 + """The writer waited for the link named ``name`` up to ``until``. + + With no ``until``, it waited for the linked stream's end. + """ + + @dataclass(frozen=True) class Cursor: """A position in one stream. @@ -119,13 +148,37 @@ class Supersession: attempt: int +@dataclass(frozen=True) +class StreamLink: + """The reference a :attr:`RecordKind.LINK` record holds. + + It is stored as plaintext, outside the payload codec, as Workflow and + Activity ids are in History. Put secrets in the record's value. + """ + + event: LinkEvent + ref: StreamRef | None + """The linked stream. Set on ``OPEN``, ``None`` on ``JOIN``.""" + name: str + """Names the link within its producer. A ``JOIN`` matches the ``OPEN`` with this name.""" + after: Cursor | None = None + """On ``OPEN``, a cursor of the linked stream. ``None`` starts at its oldest record.""" + until: Cursor | None = None + """On ``JOIN``, a cursor of the linked stream. ``None`` is the stream's end.""" + namespace: str = "" + """The linked stream's namespace. Empty is the namespace of this stream.""" + endpoint: str = "" + """The Nexus endpoint of the linked stream. Empty is the one that served this stream.""" + + @dataclass(frozen=True) class StreamRecord(Generic[T]): """One record as a reader sees it. - ``value`` is set on a :attr:`RecordKind.DATA` record and - ``supersession`` on a :attr:`RecordKind.SUPERSEDED` one. Other kinds - carry neither, so a consumer narrows on ``kind`` first. + ``value`` is set on a :attr:`RecordKind.DATA` record, + ``supersession`` on a :attr:`RecordKind.SUPERSEDED` one, and ``link`` on + a :attr:`RecordKind.LINK` one, whose ``value`` is optional. Other kinds + carry none, so a consumer narrows on ``kind`` first. """ kind: RecordKind @@ -145,9 +198,11 @@ class StreamRecord(Generic[T]): scheduling run is in its ``producer_id``, as ``@``. """ value: T | None = None - """The published value. Set on ``DATA`` only.""" + """The published value. Set on ``DATA``, and on ``LINK`` when the writer gave one.""" supersession: Supersession | None = None """The attempt change being reported. Set on ``SUPERSEDED`` only.""" + link: StreamLink | None = None + """The reference to another stream. Set on ``LINK`` only.""" stale: bool = False """An older attempt wrote it after this read delivered a newer attempt. diff --git a/temporalio/contrib/streams/_ref.py b/temporalio/contrib/streams/_ref.py index 27bebd8af8..b39dc44281 100644 --- a/temporalio/contrib/streams/_ref.py +++ b/temporalio/contrib/streams/_ref.py @@ -13,6 +13,9 @@ from dataclasses import dataclass from typing import Any, Literal +from temporalio.api.enums.v1 import StreamOwnerKind as _ApiOwnerKind +from temporalio.api.stream.v1 import StreamReference +from temporalio.bridge.proto.streams import StreamAddress from temporalio.contrib.streams._errors import StreamUnsupportedError from temporalio.contrib.streams._topic import ( DEFAULT_TOPIC, @@ -109,3 +112,51 @@ def with_topic(self, topic: str | StreamTopic[Any] | None) -> StreamRef: """ name, _ = resolve_topic(topic) return dataclasses.replace(self, topic=name) + + +def from_address(address: StreamAddress) -> StreamRef: + """The ref of the Workflow stream the stream runtime's ``address`` names.""" + return StreamRef.for_workflow( + address.workflow_id, topic=address.topic, chain_run_id=address.run_id or None + ) + + +def to_reference(ref: StreamRef) -> StreamReference: + """``ref`` as the API's stream reference, which a link record stores. + + Raises: + ValueError: ``ref`` names a stream that no Workflow owns. A link + in this release names only a Workflow's stream. + """ + if ref.kind != "workflow": + raise ValueError( + f"a link names a Workflow's stream, not one owned by kind {ref.kind!r}" + ) + return StreamReference( + owner_kind=_ApiOwnerKind.STREAM_OWNER_KIND_WORKFLOW, + workflow_id=ref.workflow_id, + run_id=ref.chain_run_id or "", + topic=ref.topic, + ) + + +def from_reference(reference: StreamReference) -> StreamRef: + """The ref an API stream reference names. + + An owner kind this release lacks decodes as ``"kind_"``, which a + handle refuses where it opens. + + Raises: + ValueError: The reference names no stream. + """ + kind = ( + "workflow" + if reference.owner_kind == _ApiOwnerKind.STREAM_OWNER_KIND_WORKFLOW + else f"kind_{int(reference.owner_kind)}" + ) + return StreamRef( + kind, + reference.workflow_id, + topic=reference.topic, + chain_run_id=reference.run_id or None, + ) diff --git a/temporalio/contrib/streams/_wire.py b/temporalio/contrib/streams/_wire.py index d46173bd30..ab1b0de596 100644 --- a/temporalio/contrib/streams/_wire.py +++ b/temporalio/contrib/streams/_wire.py @@ -12,18 +12,34 @@ import temporalio.converter from temporalio.bridge.proto.streams import ReadRecord +from temporalio.bridge.proto.streams.v1 import StreamLink as WireLink from temporalio.bridge.proto.streams.v1 import StreamRecord as WireRecord from temporalio.bridge.proto.streams.v1 import StreamRecordKind from temporalio.contrib.streams._body import decode_body from temporalio.contrib.streams._errors import StreamRecordError from temporalio.contrib.streams._record import ( Cursor, + LinkEvent, RecordKind, + StreamLink, StreamRecord, Supersession, ) +from temporalio.contrib.streams._ref import from_reference, to_reference -__all__ = ["RUN_ID_KEY", "WireRecord", "from_read", "to_wire"] +__all__ = [ + "RUN_ID_KEY", + "WireRecord", + "from_read", + "from_wire_link", + "to_record", + "to_wire", + "to_wire_link", +] + +# The most bytes a link's name and its other strings may take, as the stream runtime checks them. +_MAX_LINK_NAME_BYTES = 256 +_MAX_LINK_FIELD_BYTES = 1024 RUN_ID_KEY = "temporal.io/run-id" """The record metadata key the producing run id is stored under. @@ -44,18 +60,22 @@ def to_wire( producer_id: str = "", attempt: int = 0, sequence: int = 0, + link: StreamLink | None = None, ) -> WireRecord: """The record as the converter produced it, before the codec. - Only a ``DATA`` record carries a body; the converter encodes ``value`` - into it. + A ``DATA`` record carries a body, and the converter encodes ``value`` + into it. A ``LINK`` record carries ``link``, and a body only when + ``value`` is not ``None``. Raises: ValueError: ``kind`` is ``SUPERSEDED`` or ``UNSPECIFIED``, which no - writer stores. + writer stores, or a ``LINK`` has no valid ``link``. """ - if kind not in (RecordKind.DATA, RecordKind.FINISH): - raise ValueError(f"a writer stores DATA or FINISH records, not {kind.name}") + if kind not in (RecordKind.DATA, RecordKind.FINISH, RecordKind.LINK): + raise ValueError( + f"a writer stores DATA, FINISH or LINK records, not {kind.name}" + ) record = WireRecord( topic=topic, kind=StreamRecordKind.ValueType(int(kind)), @@ -63,11 +83,71 @@ def to_wire( attempt=attempt, sequence=sequence, ) - if kind is RecordKind.DATA: + if kind is RecordKind.LINK: + if link is None: + raise ValueError("a LINK record needs a link") + record.link.CopyFrom(to_wire_link(link)) + if value is not None: + record.body.CopyFrom(converter.to_payloads([value])[0]) + elif kind is RecordKind.DATA: record.body.CopyFrom(converter.to_payloads([value])[0]) return record +def to_wire_link(link: StreamLink) -> WireLink: + """``link`` as the stream runtime stores it. + + Raises: + ValueError: The name is empty or over 256 bytes, a cursor, the + namespace or the endpoint is over 1024 bytes, an ``OPEN`` has no + ref or has ``until``, or a ``JOIN`` has a ref or ``after``. + """ + if not link.name or len(link.name.encode()) > _MAX_LINK_NAME_BYTES: + raise ValueError( + f"a link name needs 1 to {_MAX_LINK_NAME_BYTES} bytes, got {link.name!r}" + ) + if (link.event is LinkEvent.OPEN) != (link.ref is not None): + raise ValueError("an OPEN link needs a ref, and a JOIN link has none") + if link.event is LinkEvent.OPEN and link.until is not None: + raise ValueError("an OPEN link has no until") + if link.event is LinkEvent.JOIN and link.after is not None: + raise ValueError("a JOIN link has no after") + wire = WireLink( + event=WireLink.Event.ValueType(int(link.event)), + name=link.name, + after=link.after.token if link.after is not None else "", + until=link.until.token if link.until is not None else "", + namespace=link.namespace, + endpoint=link.endpoint, + ) + for name, text in ( + ("after", wire.after), + ("until", wire.until), + ("namespace", wire.namespace), + ("endpoint", wire.endpoint), + ): + if len(text.encode()) > _MAX_LINK_FIELD_BYTES: + raise ValueError( + f"a link's {name} takes at most {_MAX_LINK_FIELD_BYTES} bytes" + ) + if link.ref is not None: + wire.stream.CopyFrom(to_reference(link.ref)) + return wire + + +def from_wire_link(wire: WireLink) -> StreamLink: + """The link the stream runtime stored in ``wire``.""" + return StreamLink( + event=LinkEvent(wire.event), + ref=from_reference(wire.stream) if wire.HasField("stream") else None, + name=wire.name, + after=Cursor(wire.after) if wire.after else None, + until=Cursor(wire.until) if wire.until else None, + namespace=wire.namespace, + endpoint=wire.endpoint, + ) + + async def from_read( converter: temporalio.converter.DataConverter, read: ReadRecord, @@ -79,6 +159,34 @@ async def from_read( StreamRecordError: The body did not decode or convert, or the record holds a kind this SDK does not know. """ + if read.HasField("stored") and read.stored.HasField("body"): + try: + body = await decode_body(converter, read.stored.body) + except Exception as error: + raise StreamRecordError( + f"stream record at {read.cursor} could not be decoded: {error}", + Cursor(read.cursor), + ) from error + decoded = ReadRecord() + decoded.CopyFrom(read) + decoded.stored.body.CopyFrom(body) + read = decoded + return to_record(converter.payload_converter, read, result_type) + + +def to_record( + payload_converter: temporalio.converter.PayloadConverter, + read: ReadRecord, + result_type: type | None, +) -> StreamRecord[Any]: + """The record a reader yields for a record whose body the codec already decoded. + + A follower reads with it, since it decodes each source in its own context. + + Raises: + StreamRecordError: The body did not convert, or the record holds a + kind this SDK does not know. + """ cursor = Cursor(read.cursor) if read.HasField("superseded"): superseded = read.superseded @@ -98,10 +206,18 @@ async def from_read( if kind is RecordKind.SUPERSEDED: raise ValueError("a SUPERSEDED record is synthesized, never stored") value: Any = None - if kind in (RecordKind.DATA, RecordKind.UNSPECIFIED) and wire.HasField("body"): - body = await decode_body(converter, wire.body) - hints = [result_type] if result_type is not None else None - value = converter.payload_converter.from_payloads([body], hints)[0] + if kind in ( + RecordKind.DATA, + RecordKind.UNSPECIFIED, + RecordKind.LINK, + ) and wire.HasField("body"): + hints = ( + [result_type] + if result_type is not None and kind is not RecordKind.LINK + else None + ) + value = payload_converter.from_payloads([wire.body], hints)[0] + link = from_wire_link(wire.link) if kind is RecordKind.LINK else None except Exception as error: raise StreamRecordError( f"stream record at {cursor} could not be decoded: {error}", cursor @@ -117,5 +233,6 @@ async def from_read( if RUN_ID_KEY in wire.metadata else "", value=value, + link=link, stale=read.stale, ) diff --git a/temporalio/contrib/streams/_workflow_producer.py b/temporalio/contrib/streams/_workflow_producer.py index 6e8b63effc..e7af199d50 100644 --- a/temporalio/contrib/streams/_workflow_producer.py +++ b/temporalio/contrib/streams/_workflow_producer.py @@ -28,7 +28,10 @@ from temporalio.contrib.streams._errors import StreamError, StreamUnsupportedError from temporalio.contrib.streams._handles import activity_handle from temporalio.contrib.streams._local_activity import _WorkflowInbound +from temporalio.contrib.streams._record import Cursor, LinkEvent, StreamLink +from temporalio.contrib.streams._ref import StreamRef from temporalio.contrib.streams._topic import StreamTopic, resolve_topic +from temporalio.contrib.streams._wire import to_wire_link __all__ = [ "WorkflowStreamProducer", @@ -115,19 +118,25 @@ def topic(self) -> str: """The name of the topic this producer is bound to.""" return self._topic - def append(self, value: T) -> None: - """Append ``value`` on this topic and return at once. - - The value is converted here, with the Workflow's payload converter. A - :class:`temporalio.common.RawValue` passes through pre-encoded. The - record is visible when the activation that appended it completes, + def append(self, *values: T | StreamLink) -> None: + """Append ``values`` on this topic, in order, and return at once. + + A :class:`temporalio.contrib.streams.StreamLink` among the values is a + ``LINK`` record, as :meth:`link` and :meth:`join` write it, and the + other values are ``DATA`` records. Every value is converted here, with + the Workflow's payload converter, before any record is added, so a + bad value adds none of them. A :class:`temporalio.common.RawValue` + passes through pre-encoded. The records go with the activation, in + one batch with the activation's other records. A record is visible + when the activation that appended it completes, which is before the Workflow Task completes. If the task then fails or times out, the record stays, and the task's rerun appends again under a higher attempt, which readers see as ``SUPERSEDED``. Raises: - ValueError: The topic was already finished in this run. - temporalio.contrib.streams.StreamError: The value's payload is + ValueError: The topic was already finished in this run, or a link + names nothing or names a stream that no Workflow owns. + temporalio.contrib.streams.StreamError: A value's payload is over 1.5 MiB. Nothing is appended. temporalio.workflow.ReadOnlyContextError: Called from a query or an update validator. @@ -135,20 +144,91 @@ def append(self, value: T) -> None: _check_writable("append to") if self._topic in self._producers.finished: raise ValueError(f"topic {self._topic!r} was already finished") + records = [ + self._link_record(value, None) + if isinstance(value, StreamLink) + else self._data_record(value) + for value in values + ] + for record in records: + self._producers.add(record) + + def _data_record(self, value: Any) -> OutputRecord: payload = workflow.payload_converter().to_payloads([value])[0] if payload.ByteSize() > _MAX_RECORD_BYTES: raise StreamError( f"a record of {payload.ByteSize()} bytes on {self._topic!r} is over " f"the per-record limit of {_MAX_RECORD_BYTES} bytes" ) - self._producers.add( - OutputRecord( - topic=self._topic, - kind=StreamRecordKind.STREAM_RECORD_KIND_DATA, - body=payload, - content_hash=content_hash(payload), - ) + return OutputRecord( + topic=self._topic, + kind=StreamRecordKind.STREAM_RECORD_KIND_DATA, + body=payload, + content_hash=content_hash(payload), + ) + + def link( + self, + ref: StreamRef, + *, + name: str, + after: Cursor | None = None, + value: Any = None, + ) -> None: + """Append a ``LINK`` that opens ``ref``'s stream on this topic. + + A follower reads the linked stream from strictly after ``after``, or + from its oldest record, in one order with this stream. ``name`` names + the link within this run, and :meth:`join` names it again. ``ref`` + is stored as plaintext. ``value``, when given, is converted as + :meth:`append` converts it. The record goes with this activation, as + :meth:`append` sends it. + + Raises: + ValueError: The topic was already finished in this run, + ``name`` is empty or over 256 bytes, or ``ref`` names a + stream that no Workflow owns. + temporalio.workflow.ReadOnlyContextError: Called from a query or + an update validator. + """ + self._add_link(StreamLink(LinkEvent.OPEN, ref, name, after=after), value) + + def join(self, *, name: str, until: Cursor | None = None) -> None: + """Append a ``LINK`` that says this run waited for link ``name``. + + A follower holds this stream's later records until the linked stream + reached ``until``, or its end when ``until`` is ``None``. + + Raises: + ValueError: The topic was already finished in this run, or + ``name`` is empty or over 256 bytes. + temporalio.workflow.ReadOnlyContextError: Called from a query or + an update validator. + """ + self._add_link(StreamLink(LinkEvent.JOIN, None, name, until=until), None) + + def _add_link(self, link: StreamLink, value: Any) -> None: + _check_writable("link on") + if self._topic in self._producers.finished: + raise ValueError(f"topic {self._topic!r} was already finished") + self._producers.add(self._link_record(link, value)) + + def _link_record(self, link: StreamLink, value: Any) -> OutputRecord: + record = OutputRecord( + topic=self._topic, + kind=StreamRecordKind.STREAM_RECORD_KIND_LINK, + link=to_wire_link(link), ) + if value is not None: + payload = workflow.payload_converter().to_payloads([value])[0] + if payload.ByteSize() > _MAX_RECORD_BYTES: + raise StreamError( + f"a link value of {payload.ByteSize()} bytes on {self._topic!r} " + f"is over the per-record limit of {_MAX_RECORD_BYTES} bytes" + ) + record.body.CopyFrom(payload) + record.content_hash = content_hash(payload) + return record async def flush(self) -> None: """Return at once, since each record goes with its activation. diff --git a/temporalio/contrib/streams/docs/wire-contract.md b/temporalio/contrib/streams/docs/wire-contract.md index 27569821da..3699773a81 100644 --- a/temporalio/contrib/streams/docs/wire-contract.md +++ b/temporalio/contrib/streams/docs/wire-contract.md @@ -52,10 +52,11 @@ use it: | `body` | 1 | The published value as a `temporal.api.common.v1.Payload`, after the payload codec | | `metadata` | 2 | Map of string to `Payload`, see below | | `topic` | 3 | The topic name | -| `kind` | 4 | `STREAM_RECORD_KIND_DATA` (1), `STREAM_RECORD_KIND_FINISH` (2), `STREAM_RECORD_KIND_CLOSE` (5) or `STREAM_RECORD_KIND_TOPIC` (6). 0 reads as DATA. 3 is reserved, since readers synthesize `SUPERSEDED` with it, and 4 is kept for links between streams | +| `kind` | 4 | `STREAM_RECORD_KIND_DATA` (1), `STREAM_RECORD_KIND_FINISH` (2), `STREAM_RECORD_KIND_LINK` (4), `STREAM_RECORD_KIND_CLOSE` (5) or `STREAM_RECORD_KIND_TOPIC` (6). 0 reads as DATA. 3 is reserved, since readers synthesize `SUPERSEDED` with it | | `producer_id` | 5 | Who wrote it. An Activity writes as `@`, a Local Activity as `__temporal_local_@`, and a Workflow as `@` | | `attempt` | 6 | The producer's attempt. A Local Activity's is `task attempt << 14 \| min(Local Activity attempt, 16383)`, with the task attempt of the Workflow Task in which Core dispatched it, below 2^46. A Workflow's is `prefix << 16 \| retry`: `prefix` is the WorkflowTaskStarted event id of the run's last completed Workflow Task (0 before one), and `retry` counts the reruns of the task since then, from 1 | | `sequence` | 7 | The record's position in its attempt, starting at 1 | +| `link` | 8 | On `LINK` only, a `temporal.sdk.streams.v1.StreamLink`: the event (`OPEN` or `JOIN`), the linked Workflow's stream as a `temporal.api.stream.v1.StreamReference` on `OPEN`, the link's name, and the `after` or `until` cursor. It is plaintext, outside the payload codec. A `LINK` may carry a `body` | | `task_started_event_id` | 9 | On a Workflow's records, the WorkflowTaskStarted event id of the task that wrote it. On a Local Activity's, that of the task in which Core dispatched it | | `completed_tasks` | 10 | On the first record of each Workflow or Local Activity batch, the WorkflowTaskStarted ids of the run's last completed tasks, newest first, at most 8, with a trailing 0 when the list reaches back to the run's start | diff --git a/tests/contrib/streams/test_follow.py b/tests/contrib/streams/test_follow.py new file mode 100644 index 0000000000..b69725582d --- /dev/null +++ b/tests/contrib/streams/test_follow.py @@ -0,0 +1,342 @@ +"""Following a Workflow's stream and the streams its links open, in one order.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncGenerator +from datetime import timedelta +from typing import Any + +import pytest + +from temporalio import activity, workflow +from temporalio.client import Client +from temporalio.contrib.streams import ( + BEGINNING, + FollowedItem, + FollowPosition, + LinkStateKind, + RecordKind, + StreamError, + StreamRef, + _follow, + activity_handle, + stream_handle, + stream_producer, + topic, +) +from temporalio.contrib.streams.memory import MemoryStreams +from tests.contrib.streams._support import connect_with, new_workflow_id, read_all +from tests.contrib.streams.test_outside_handles import Waits, running_owner +from tests.helpers import new_worker + +EVENTS = topic("events", str) + + +@workflow.defn +class Grandchild: + @workflow.run + async def run(self) -> None: + events = stream_producer(EVENTS) + events.append("g1") + events.append("g2") + + +@workflow.defn +class Child: + @workflow.run + async def run(self, name: str, grandchild: bool) -> None: + events = stream_producer(EVENTS) + events.append(f"{name}1") + if grandchild: + child_id = f"{workflow.info().workflow_id}-g" + events.link(StreamRef.for_workflow(child_id, topic=EVENTS), name="g") + await workflow.execute_child_workflow(Grandchild.run, id=child_id) + events.join(name="g") + events.append(f"{name}2") + + +@workflow.defn +class Agent: + @workflow.run + async def run(self) -> None: + events = stream_producer(EVENTS) + agent_id = workflow.info().workflow_id + events.append("d1") + children = {} + for name, grandchild in (("one", False), ("two", True)): + child_id = f"{agent_id}-{name}" + events.link(StreamRef.for_workflow(child_id, topic=EVENTS), name=name) + children[name] = await workflow.start_child_workflow( + Child.run, args=[name, grandchild], id=child_id + ) + events.append("d2") + await children["one"] + events.join(name="one") + events.append("d3") + await children["two"] + events.join(name="two") + events.append("d4") + + +def shape(items: list[FollowedItem[Any]]) -> list[str]: + """Each item as a word: a value, a link record's event, or a link report.""" + out = [] + for item in items: + if item.link_state is not None: + state = item.link_state + reason = f":{state.reason}" if state.reason else "" + out.append(f"{state.kind.name}:{state.link.name}{reason}") + elif item.record is not None and item.record.kind is RecordKind.LINK: + assert item.record.link is not None + out.append(f"{item.record.link.event.name}:{item.record.link.name}") + elif item.record is not None and item.record.kind is RecordKind.DATA: + out.append(str(item.record.value)) + return out + + +def check_gates(order: list[str]) -> None: + """Order within each stream, and the open and close gates.""" + assert len(set(order)) == len(order), order + at = order.index + assert at("OPENED:one") < at("one1") + assert at("OPENED:two") < at("two1") + assert at("OPENED:g") < at("g1") + assert at("DONE:one") < at("JOIN:one") < at("d3") + assert at("DONE:g") < at("JOIN:g") < at("two2") + assert at("DONE:two") < at("JOIN:two") < at("d4") + for a, b in ( + ("d1", "d2"), + ("d2", "d3"), + ("d3", "d4"), + ("one1", "one2"), + ("two1", "two2"), + ("g1", "g2"), + ): + assert at(a) < at(b), (a, b, order) + + +async def collect( + follow: AsyncGenerator[FollowedItem[Any], None], +) -> list[FollowedItem[Any]]: + async def pull() -> list[FollowedItem[Any]]: + return [item async for item in follow] + + return await asyncio.wait_for(pull(), 20) + + +async def run_agent(client: Client) -> str: + workflow_id = new_workflow_id() + async with new_worker(client, Agent, Child, Grandchild) as worker: + await client.execute_workflow( + Agent.run, id=workflow_id, task_queue=worker.task_queue + ) + return workflow_id + + +async def test_a_follow_reads_the_agent_and_its_children_in_one_order(client: Client): + streams = await connect_with(client, MemoryStreams()) + agent_id = await run_agent(streams) + items = await collect(stream_handle(streams, agent_id, topic=EVENTS).follow()) + order = shape(items) + check_gates(order) + assert set(order) >= {"d1", "d2", "d3", "d4", "one1", "one2", "two1", "two2"} + sources = {item.source.workflow_id: item.depth for item in items} + assert sources == { + agent_id: 0, + f"{agent_id}-one": 1, + f"{agent_id}-two": 1, + f"{agent_id}-two-g": 2, + } + # A child's records keep the cursors a plain read of the child gives. + one = stream_handle(streams, f"{agent_id}-one", topic=EVENTS) + assert [ + item.record.cursor + for item in items + if item.record is not None and item.source == one.ref + ] == [record.cursor for record in await read_all(one.read())] + assert items[-1].position is not None + + +async def test_a_follow_resumed_at_each_batch_end_loses_and_repeats_nothing( + client: Client, monkeypatch: pytest.MonkeyPatch +): + streams = await connect_with(client, MemoryStreams()) + agent_id = await run_agent(streams) + handle = stream_handle(streams, agent_id, topic=EVENTS) + # One stored record per batch, so every record ends a batch. + monkeypatch.setattr(_follow, "_MAX_RECORDS", 1) + whole = shape(await collect(handle.follow())) + check_gates(whole) + + resumed: list[FollowedItem[Any]] = [] + after: FollowPosition | None = None + batches = 0 + while True: + follow = handle.follow(after=after or BEGINNING) + ended = True + async for item in follow: + resumed.append(item) + if item.position is not None: + after = item.position + ended = False + break + await follow.aclose() + if ended: + break + batches += 1 + assert batches < 200 + assert batches > len(whole) // 2 + assert shape(resumed) == whole + + +async def test_a_follow_skips_a_link_past_its_depth(client: Client): + streams = await connect_with(client, MemoryStreams()) + agent_id = await run_agent(streams) + handle = stream_handle(streams, agent_id, topic=EVENTS) + order = shape(await collect(handle.follow(max_depth=1))) + assert "SKIPPED:g:depth" in order + assert not any(word.startswith("g") for word in order) + # A JOIN of a skipped link waits for nothing. + assert order.index("JOIN:g") < order.index("two2") + with pytest.raises(ValueError, match="at least 1"): + handle.follow(max_depth=0) + with pytest.raises(ValueError, match="at least 1"): + handle.follow(max_open=0) + + +async def test_a_link_back_to_an_open_stream_is_a_cycle(client: Client): + streams = await connect_with(client, MemoryStreams()) + async with new_worker(streams, Waits) as worker: + agent_id = await running_owner(streams, worker.task_queue) + child_id = await running_owner(streams, worker.task_queue) + agent = stream_handle(streams, agent_id, topic=EVENTS) + child = stream_handle(streams, child_id, topic=EVENTS) + writes = child.producer(producer_id="c", attempt=1) + await writes.append("c1") + await writes.link(agent.ref, name="back") + await writes.append("c2") + writes = agent.producer(producer_id="a", attempt=1) + await writes.link(child.ref, name="child") + await writes.join(name="child") + await writes.append("d1") + for workflow_id in (child_id, agent_id): + owner = streams.get_workflow_handle(workflow_id) + await owner.signal(Waits.finish) + await owner.result() + order = shape(await collect(agent.follow())) + assert order == [ + "OPEN:child", + "OPENED:child", + "c1", + "OPEN:back", + "SKIPPED:back:cycle", + "c2", + "DONE:child", + "JOIN:child", + "d1", + ] + + +async def test_a_retry_that_writes_the_same_open_resumes_the_child(client: Client): + streams = await connect_with(client, MemoryStreams()) + async with new_worker(streams, Waits) as worker: + agent_id = await running_owner(streams, worker.task_queue) + child_id = await running_owner(streams, worker.task_queue) + agent = stream_handle(streams, agent_id, topic=EVENTS) + child = stream_handle(streams, child_id, topic=EVENTS) + await child.producer(producer_id="c", attempt=1).append("c1", "c2") + for attempt in (1, 2): + writes = agent.producer(producer_id="tool", attempt=attempt) + await writes.append(f"m{attempt}") + await writes.link(child.ref, name="child") + for workflow_id in (child_id, agent_id): + owner = streams.get_workflow_handle(workflow_id) + await owner.signal(Waits.finish) + await owner.result() + items = await collect(agent.follow()) + assert shape(items) == [ + "m1", + "OPEN:child", + "OPENED:child", + "DONE:child:withdrawn", + "m2", + "OPEN:child", + "OPENED:child:resumed", + "c1", + "c2", + "DONE:child", + ] + states = [item for item in items if item.link_state is not None] + assert [item.link_state.reason for item in states if item.link_state] == [ + "", + "withdrawn", + "resumed", + "", + ] + assert len({item.source for item in states}) == 1, "the child keeps its source" + assert any( + item.record and item.record.kind is RecordKind.SUPERSEDED for item in items + ) + + +async def test_a_missing_child_is_unavailable_after_the_grace(client: Client): + streams = await connect_with(client, MemoryStreams(owner_start_grace=timedelta(0))) + async with new_worker(streams, Waits) as worker: + agent_id = await running_owner(streams, worker.task_queue) + agent = stream_handle(streams, agent_id, topic=EVENTS) + writes = agent.producer(producer_id="a", attempt=1) + gone = StreamRef.for_workflow(f"{agent_id}-never-started", topic=EVENTS) + await writes.link(gone, name="gone") + await writes.append("d1") + owner = streams.get_workflow_handle(agent_id) + await owner.signal(Waits.finish) + await owner.result() + items = await collect(agent.follow()) + assert shape(items) == ["OPEN:gone", "OPENED:gone", "d1", "UNAVAILABLE:gone"] + unavailable = items[-1].link_state + assert unavailable is not None and unavailable.kind is LinkStateKind.UNAVAILABLE + assert isinstance(unavailable.error, StreamError) + assert unavailable.link.ref == gone + + +@activity.defn +async def follow_own_stream() -> list[str]: + # The scheduling Workflow is still running, so the follow stops at the last word. + items = [] + async for item in activity_handle().follow(topic=EVENTS): + items.append(item) + if shape(items)[-1:] == ["end"]: + break + return shape(items) + + +@workflow.defn +class FollowsFromAnActivity: + @workflow.run + async def run(self) -> list[str]: + events = stream_producer(EVENTS) + child_id = f"{workflow.info().workflow_id}-g" + events.link(StreamRef.for_workflow(child_id, topic=EVENTS), name="g") + await workflow.execute_child_workflow(Grandchild.run, id=child_id) + events.join(name="g") + events.append("end") + return await workflow.execute_activity( + follow_own_stream, start_to_close_timeout=timedelta(seconds=20) + ) + + +async def test_an_activity_follows_its_workflows_stream(client: Client): + streams = await connect_with(client, MemoryStreams()) + async with new_worker( + streams, + FollowsFromAnActivity, + Grandchild, + activities=[follow_own_stream], + ) as worker: + order = await streams.execute_workflow( + FollowsFromAnActivity.run, + id=new_workflow_id(), + task_queue=worker.task_queue, + ) + assert order == ["OPEN:g", "OPENED:g", "g1", "g2", "DONE:g", "JOIN:g", "end"] diff --git a/tests/contrib/streams/test_records_wire.py b/tests/contrib/streams/test_records_wire.py index 59dc83f060..61dd692595 100644 --- a/tests/contrib/streams/test_records_wire.py +++ b/tests/contrib/streams/test_records_wire.py @@ -2,16 +2,28 @@ from __future__ import annotations +import hashlib +import json import uuid from collections.abc import Sequence +from typing import Any import pytest from temporalio.api.common.v1 import Payload +from temporalio.api.stream.v1 import StreamReference from temporalio.bridge.proto.streams import ReadRecord, Supersession from temporalio.bridge.proto.streams.v1 import StreamRecordKind from temporalio.common import RawValue -from temporalio.contrib.streams import Cursor, RecordKind, StreamRecordError +from temporalio.contrib.streams import ( + Cursor, + LinkEvent, + RecordKind, + StreamLink, + StreamRecordError, + StreamRef, + StreamUnsupportedError, +) from temporalio.contrib.streams import Supersession as ReportedSupersession from temporalio.contrib.streams._body import ( batch_digest, @@ -19,8 +31,16 @@ decode_body, encode_bodies, ) -from temporalio.contrib.streams._wire import RUN_ID_KEY, WireRecord, from_read, to_wire +from temporalio.contrib.streams._ref import from_reference, to_reference +from temporalio.contrib.streams._wire import ( + RUN_ID_KEY, + WireRecord, + from_read, + to_wire, + to_wire_link, +) from temporalio.converter import DataConverter, PayloadCodec +from tests.contrib.streams._support import streams_crate_dir CONVERTER = DataConverter.default @@ -193,3 +213,164 @@ async def test_the_codec_runs_on_the_body_and_never_on_the_hashes(): str, ) assert record.value == "a" + + +@pytest.mark.parametrize( + "link", + [ + StreamLink( + LinkEvent.OPEN, + StreamRef.for_workflow("child", topic="tokens"), + "sub-1", + after=Cursor("c7"), + ), + StreamLink( + LinkEvent.OPEN, + StreamRef.for_workflow("child", chain_run_id="r"), + "sub-3", + namespace="other", + endpoint="streams", + ), + StreamLink(LinkEvent.JOIN, None, "sub-3", until=Cursor("c9")), + StreamLink(LinkEvent.JOIN, None, "sub-1"), + ], + ids=["workflow-after", "workflow-remote", "join-until", "join-end"], +) +async def test_a_link_reads_back_as_it_was_written(link: StreamLink): + wire = to_wire( + CONVERTER.payload_converter, + topic="events", + kind=RecordKind.LINK, + link=link, + producer_id="p", + attempt=1, + sequence=4, + ) + assert wire.kind == StreamRecordKind.STREAM_RECORD_KIND_LINK + assert not wire.HasField("body") + record = await from_read(CONVERTER, stored(wire, "c4"), None) + assert record.kind is RecordKind.LINK + assert record.link == link + assert record.value is None + + +async def test_a_link_carries_an_optional_value_through_the_codec(): + link = StreamLink(LinkEvent.OPEN, StreamRef.for_workflow("child"), "sub") + wire = to_wire( + CONVERTER.payload_converter, + topic="events", + kind=RecordKind.LINK, + link=link, + value={"label": "search"}, + ) + record = await from_read(CONVERTER, stored(wire), dict) + assert record.value == {"label": "search"} + assert record.link == link + + +@pytest.mark.parametrize( + "link, message", + [ + (StreamLink(LinkEvent.OPEN, None, "x"), "needs a ref"), + (StreamLink(LinkEvent.JOIN, StreamRef.for_workflow("wf"), "x"), "needs a ref"), + (StreamLink(LinkEvent.OPEN, StreamRef.for_workflow("wf"), ""), "1 to 256"), + ( + StreamLink(LinkEvent.OPEN, StreamRef.for_workflow("wf"), "x" * 257), + "1 to 256", + ), + ( + StreamLink( + LinkEvent.OPEN, StreamRef.for_workflow("wf"), "x", until=Cursor("c") + ), + "no until", + ), + (StreamLink(LinkEvent.JOIN, None, "x", after=Cursor("c")), "no after"), + ( + StreamLink( + LinkEvent.OPEN, StreamRef.for_workflow("wf"), "x", endpoint="e" * 1025 + ), + "endpoint takes at most 1024", + ), + ( + StreamLink(LinkEvent.OPEN, StreamRef("activity", "", topic="t"), "x"), + "names a Workflow's stream", + ), + ], +) +def test_a_link_that_names_nothing_is_refused(link: StreamLink, message: str): + with pytest.raises(ValueError, match=message): + to_wire(CONVERTER.payload_converter, topic="t", kind=RecordKind.LINK, link=link) + with pytest.raises(ValueError, match="needs a link"): + to_wire(CONVERTER.payload_converter, topic="t", kind=RecordKind.LINK) + + +def test_a_ref_of_a_later_owner_kind_decodes_from_a_link(): + reference = to_reference(StreamRef.for_workflow("wf")) + reference.owner_kind = 9 # type: ignore[assignment] + ref = from_reference(reference) + assert ref.kind == "kind_9" + with pytest.raises(StreamUnsupportedError): + ref._require_supported() + + +def _workflow_batch_digest(topic: str, records: list[tuple[int, bytes, Any]]) -> str: + # The stream runtime's rule for a Workflow batch, so a link the SDK writes keeps the digest + # that a rerun of its task must repeat. + digest = hashlib.sha256() + for kind, hashed, link in records: + fields: list[bytes] = [] + if link is not None: + stream = link.stream + fields = [ + link.event.to_bytes(4, "big"), + stream.owner_kind.to_bytes(4, "big"), + stream.workflow_id.encode(), + stream.run_id.encode(), + stream.topic.encode(), + stream.activity_id.encode(), + link.name.encode(), + link.after.encode(), + link.until.encode(), + link.namespace.encode(), + link.endpoint.encode(), + ] + body = ( + len(topic).to_bytes(8, "big") + + topic.encode() + + kind.to_bytes(4, "big") + + len(hashed).to_bytes(8, "big") + + hashed + + b"".join(len(f).to_bytes(8, "big") + f for f in fields) + ) + digest.update(len(body).to_bytes(8, "big") + body) + return digest.hexdigest() + + +def test_a_link_keeps_the_batch_digest_of_the_shared_vectors(): + path = streams_crate_dir() / "testdata/workflow_attempts.json" + vectors = json.loads(path.read_text()) + if "batch_digests" not in vectors: + pytest.skip("the pinned sdk-streams commit predates the batch_digests vector") + cases = vectors["batch_digests"] + assert any("link" in r for case in cases for r in case["records"]) + for case in cases: + records: list[tuple[int, bytes, Any]] = [] + for record in case["records"]: + link = None + if "link" in record: + spec = record["link"] + stream = spec.get("stream") + link = to_wire_link( + StreamLink( + LinkEvent(spec["event"]), + from_reference(StreamReference(**stream)) if stream else None, + spec["name"], + after=Cursor(spec["after"]) if spec.get("after") else None, + until=Cursor(spec["until"]) if spec.get("until") else None, + ) + ) + hashed = bytes.fromhex(record.get("content_hash", "")) + records.append((record["kind"], hashed, link)) + assert _workflow_batch_digest(case["topic"], records) == case["digest"], case[ + "label" + ] diff --git a/tests/contrib/streams/test_supersession.py b/tests/contrib/streams/test_supersession.py index 1c98555618..915d072bb6 100644 --- a/tests/contrib/streams/test_supersession.py +++ b/tests/contrib/streams/test_supersession.py @@ -12,8 +12,11 @@ from temporalio.client import Client from temporalio.contrib.streams import ( Cursor, + LinkEvent, RecordKind, + StreamLink, StreamRecord, + StreamRef, Supersession, SupersessionTracker, stream_handle, @@ -387,3 +390,21 @@ async def test_a_reader_sees_each_value_once_after_a_task_reruns(client: Client) if record.kind is RecordKind.DATA and not record.stale: shown.append(record.value) assert shown == ["before", "a", "b"] + + +def test_a_rerun_withdraws_the_links_of_its_earlier_attempt(): + tracker: SupersessionTracker[str] = SupersessionTracker() + link: StreamRecord[str] = StreamRecord( + kind=RecordKind.LINK, + cursor=Cursor("l1"), + topic="t", + producer_id="tool@run", + attempt=1, + link=StreamLink(LinkEvent.OPEN, StreamRef.for_workflow("child"), "sub"), + ) + assert tracker.observe(link) == [] + assert tracker.observe(data("tool@run", 1, "a")) == [] + assert tracker.observe(superseded("tool@run", 1, 2)) == [ + link, + data("tool@run", 1, "a"), + ] diff --git a/tests/contrib/streams/test_workflow_producer.py b/tests/contrib/streams/test_workflow_producer.py index 5f1494d954..8cfda68398 100644 --- a/tests/contrib/streams/test_workflow_producer.py +++ b/tests/contrib/streams/test_workflow_producer.py @@ -34,8 +34,12 @@ WorkflowUpdateFailedError, ) from temporalio.contrib.streams import ( + Cursor, + LinkEvent, RecordKind, StreamError, + StreamLink, + StreamRef, StreamStorePlugin, StreamUnsupportedError, SupersessionTracker, @@ -52,6 +56,7 @@ from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker from temporalio.worker.workflow_sandbox import SandboxedWorkflowRunner from tests.contrib.streams._support import connect_with, new_workflow_id, read_all +from tests.contrib.streams.test_outside_handles import Waits, running_owner from tests.helpers import new_worker STEPS = topic("steps", str) @@ -628,3 +633,171 @@ async def test_a_history_of_the_local_activity_writer_does_not_replay(): await Replayer( workflows=[FlushesOnce], plugins=[MemoryStreams()] ).replay_workflow(history) + + +@activity.defn +async def latest_link() -> str: + # The Worker appends the activation's records before Core dispatches this call. + return (await activity_handle().latest(topic=STEPS)).token + + +@workflow.defn +class LinksAndJoins: + @workflow.run + async def run(self) -> tuple[str, str, str]: + steps = stream_producer(STEPS) + steps.append("d1") + steps.link( + StreamRef.for_workflow("child", chain_run_id="r"), + name="sub-1", + after=Cursor("c3"), + value="search", + ) + landed = await workflow.execute_local_activity( + latest_link, start_to_close_timeout=timedelta(seconds=10) + ) + steps.join(name="sub-1", until=Cursor("c9")) + steps.append("d2") + unnamed = owner = "" + try: + steps.link(StreamRef.for_workflow("child"), name="") + except ValueError as error: + unnamed = str(error) + try: + steps.link(StreamRef("activity", "", topic="t"), name="x") + except ValueError as error: + owner = str(error) + return landed, unnamed, owner + + +async def test_a_workflow_link_lands_with_its_activation(client: Client): + streams = await connect_with(client, MemoryStreams()) + workflow_id = new_workflow_id() + async with new_worker(streams, LinksAndJoins, activities=[latest_link]) as worker: + handle = await streams.start_workflow( + LinksAndJoins.run, id=workflow_id, task_queue=worker.task_queue + ) + landed, unnamed, owner = await handle.result() + records = await read_all(stream_handle(streams, workflow_id).read(topic=STEPS)) + assert [r.kind for r in records] == [ + RecordKind.DATA, + RecordKind.LINK, + RecordKind.LINK, + RecordKind.DATA, + ] + # The link went out with the activation that wrote it, before the Local + # Activity ran. + assert landed == records[1].cursor.token + assert records[1].link == StreamLink( + LinkEvent.OPEN, + StreamRef.for_workflow("child", chain_run_id="r"), + "sub-1", + after=Cursor("c3"), + ) + assert records[1].value == "search" + assert records[2].link == StreamLink( + LinkEvent.JOIN, None, "sub-1", until=Cursor("c9") + ) + assert records[2].value is None + assert "1 to 256" in unnamed + assert "names a Workflow's stream" in owner + await Replayer( + workflows=[LinksAndJoins], plugins=[MemoryStreams()] + ).replay_workflow(await handle.fetch_history()) + + +async def test_a_producer_links_and_joins(client: Client): + streams = await connect_with(client, MemoryStreams()) + async with new_worker(streams, Waits) as worker: + workflow_id = await running_owner(streams, worker.task_queue) + stream = stream_handle(streams, workflow_id, topic=STEPS) + producer = stream.producer(producer_id="agent", attempt=1) + child = StreamRef.for_workflow(f"{workflow_id}-child", topic=STEPS) + await producer.append("d1") + opened = await producer.link(child, name="sub-1", value={"label": "search"}) + await producer.join(name="sub-1") + with pytest.raises(ValueError, match="names a Workflow's stream"): + await producer.link(StreamRef("activity", "", topic="t"), name="x") + await streams.get_workflow_handle(workflow_id).signal(Waits.finish) + records = await read_all(stream.read()) + assert [r.cursor for r in records][1] == opened + assert records[1].link == StreamLink(LinkEvent.OPEN, child, "sub-1") + assert records[1].value == {"label": "search"} + assert records[2].link == StreamLink(LinkEvent.JOIN, None, "sub-1") + assert [r.sequence for r in records] == [1, 2, 3] + + +@workflow.defn +class AppendsMixed: + @workflow.run + async def run(self) -> str: + steps = stream_producer(STEPS) + child = StreamRef.for_workflow("child") + steps.append("d1", StreamLink(LinkEvent.OPEN, child, "sub-1"), "d2") + try: + steps.append("lost", StreamLink(LinkEvent.OPEN, child, "")) + except ValueError as error: + return str(error) + return "" + + +async def test_a_workflow_appends_values_and_links_in_one_call(client: Client): + streams = await connect_with(client, MemoryStreams()) + workflow_id = new_workflow_id() + async with new_worker(streams, AppendsMixed) as worker: + refused = await streams.execute_workflow( + AppendsMixed.run, id=workflow_id, task_queue=worker.task_queue + ) + records = await read_all(stream_handle(streams, workflow_id).read(topic=STEPS)) + assert [(r.kind, r.value) for r in records] == [ + (RecordKind.DATA, "d1"), + (RecordKind.LINK, None), + (RecordKind.DATA, "d2"), + ] + assert records[1].link == StreamLink( + LinkEvent.OPEN, StreamRef.for_workflow("child"), "sub-1" + ) + # A bad link adds none of the call's records. + assert "1 to 256" in refused + + +async def mixed_batch_appends_once(streams: Client) -> None: + async with new_worker(streams, Waits) as worker: + workflow_id = await running_owner(streams, worker.task_queue) + stream = stream_handle(streams, workflow_id, topic=STEPS) + child = StreamRef.for_workflow(f"{workflow_id}-child", topic=STEPS) + batch = ( + "sent", + StreamLink(LinkEvent.OPEN, child, "sub-1"), + StreamLink(LinkEvent.JOIN, None, "sub-1"), + "received", + ) + first = stream.producer(producer_id="agent", attempt=1) + last = await first.append(*batch) + # A resend of the same batch from a fresh producer object dedupes. + again = stream.producer(producer_id="agent", attempt=1) + assert await again.append(*batch) == last + with pytest.raises(ValueError, match="1 to 256"): + await first.append("x", StreamLink(LinkEvent.OPEN, child, "")) + await first.append("after") + await streams.get_workflow_handle(workflow_id).signal(Waits.finish) + records = await read_all(stream.read()) + assert [r.kind for r in records] == [ + RecordKind.DATA, + RecordKind.LINK, + RecordKind.LINK, + RecordKind.DATA, + RecordKind.DATA, + ] + assert records[3].cursor == last + assert records[1].link == StreamLink(LinkEvent.OPEN, child, "sub-1") + assert records[2].link == StreamLink(LinkEvent.JOIN, None, "sub-1") + assert [r.sequence for r in records] == [1, 2, 3, 4, 5] + + +async def test_a_producer_appends_values_and_links_as_one_batch(client: Client): + await mixed_batch_appends_once(await connect_with(client, MemoryStreams())) + + +async def test_a_resent_mixed_batch_dedupes_on_redis(client: Client): + await mixed_batch_appends_once(await connect_with(client, redis_store()))