diff --git a/changelog/added/namespace-tagged-breadcrumbs.md b/changelog/added/namespace-tagged-breadcrumbs.md new file mode 100644 index 0000000000..4da8dd5798 --- /dev/null +++ b/changelog/added/namespace-tagged-breadcrumbs.md @@ -0,0 +1 @@ +Add a `temporalNamespace` attribute to all spans created by the `temporalio.contrib.opentelemetry` tracing interceptors, set to the client namespace for client spans and the workflow, activity, or Nexus operation namespace for worker spans. diff --git a/temporalio/contrib/opentelemetry/_interceptor.py b/temporalio/contrib/opentelemetry/_interceptor.py index f503d5ac55..2b9f8dea0d 100644 --- a/temporalio/contrib/opentelemetry/_interceptor.py +++ b/temporalio/contrib/opentelemetry/_interceptor.py @@ -33,6 +33,7 @@ import temporalio.client import temporalio.converter import temporalio.exceptions +import temporalio.nexus import temporalio.nexus.system.workflow_service.models import temporalio.worker import temporalio.workflow @@ -60,6 +61,24 @@ _ContextT = TypeVar("_ContextT", bound=nexusrpc.handler.OperationContext) +def _client_namespace(next: temporalio.client.OutboundInterceptor) -> str | None: + # Client interceptors are not given the client, so walk the chain down to + # the SDK's terminal implementation, which holds it. + current: object = next + while current is not None: + client = getattr(current, "_client", None) + if isinstance(client, temporalio.client.Client): + return client.namespace + current = getattr(current, "next", None) + return None + + +def _namespace_attributes( + namespace: str | None, +) -> dict[str, opentelemetry.util.types.AttributeValue]: + return {"temporalNamespace": namespace} if namespace else {} + + class TracingInterceptor(temporalio.client.Interceptor, temporalio.worker.Interceptor): """Interceptor that supports client and worker OpenTelemetry span creation and propagation. @@ -269,6 +288,7 @@ def __init__( ) -> None: super().__init__(next) self.root = root + self._namespace_attributes = _namespace_attributes(_client_namespace(next)) async def start_workflow( self, input: temporalio.client.StartWorkflowInput @@ -278,7 +298,10 @@ async def start_workflow( ) with self.root._start_as_current_span( f"{prefix}:{input.workflow}", - attributes={"temporalWorkflowID": input.id}, + attributes={ + "temporalWorkflowID": input.id, + **self._namespace_attributes, + }, input_with_headers=input, kind=opentelemetry.trace.SpanKind.CLIENT, ): @@ -287,7 +310,10 @@ async def start_workflow( async def query_workflow(self, input: temporalio.client.QueryWorkflowInput) -> Any: with self.root._start_as_current_span( f"QueryWorkflow:{input.query}", - attributes={"temporalWorkflowID": input.id}, + attributes={ + "temporalWorkflowID": input.id, + **self._namespace_attributes, + }, input_with_headers=input, kind=opentelemetry.trace.SpanKind.CLIENT, ): @@ -298,7 +324,10 @@ async def signal_workflow( ) -> None: with self.root._start_as_current_span( f"SignalWorkflow:{input.signal}", - attributes={"temporalWorkflowID": input.id}, + attributes={ + "temporalWorkflowID": input.id, + **self._namespace_attributes, + }, input_with_headers=input, kind=opentelemetry.trace.SpanKind.CLIENT, ): @@ -309,7 +338,10 @@ async def start_workflow_update( ) -> temporalio.client.WorkflowUpdateHandle[Any]: with self.root._start_as_current_span( f"StartWorkflowUpdate:{input.update}", - attributes={"temporalWorkflowID": input.id}, + attributes={ + "temporalWorkflowID": input.id, + **self._namespace_attributes, + }, input_with_headers=input, kind=opentelemetry.trace.SpanKind.CLIENT, ): @@ -318,8 +350,9 @@ async def start_workflow_update( async def start_update_with_start_workflow( self, input: temporalio.client.StartWorkflowUpdateWithStartInput ) -> temporalio.client.WorkflowUpdateHandle[Any]: - attrs = { + attrs: dict[str, opentelemetry.util.types.AttributeValue] = { "temporalWorkflowID": input.start_workflow_input.id, + **self._namespace_attributes, } if input.update_workflow_input.update_id is not None: attrs["temporalUpdateID"] = input.update_workflow_input.update_id @@ -347,6 +380,7 @@ async def start_activity( attributes={ "temporalActivityID": input.id, "temporalActivityType": input.activity_type, + **self._namespace_attributes, }, input_with_headers=input, kind=opentelemetry.trace.SpanKind.CLIENT, @@ -367,7 +401,10 @@ async def execute_activity( self, input: temporalio.worker.ExecuteActivityInput ) -> Any: info = temporalio.activity.info() - attributes: dict[str, str] = {"temporalActivityID": info.activity_id} + attributes: dict[str, str] = { + "temporalActivityID": info.activity_id, + "temporalNamespace": info.namespace, + } if info.workflow_id: attributes["temporalWorkflowID"] = info.workflow_id if info.workflow_run_id: @@ -404,7 +441,7 @@ async def execute_nexus_operation_start( with self._root._start_as_current_span( f"RunStartNexusOperationHandler:{input.ctx.service}/{input.ctx.operation}", context=self._context_from_nexus_headers(input.ctx.headers), - attributes={}, + attributes=_nexus_namespace_attributes(), input_with_ctx=input, kind=opentelemetry.trace.SpanKind.SERVER, ): @@ -416,13 +453,19 @@ async def execute_nexus_operation_cancel( with self._root._start_as_current_span( f"RunCancelNexusOperationHandler:{input.ctx.service}/{input.ctx.operation}", context=self._context_from_nexus_headers(input.ctx.headers), - attributes={}, + attributes=_nexus_namespace_attributes(), input_with_ctx=input, kind=opentelemetry.trace.SpanKind.SERVER, ): return await self.next.execute_nexus_operation_cancel(input) +def _nexus_namespace_attributes() -> dict[str, opentelemetry.util.types.AttributeValue]: + if not temporalio.nexus.in_operation(): + return {} + return _namespace_attributes(temporalio.nexus.info().namespace) + + class _InputWithHeaders(Protocol): headers: Mapping[str, temporalio.api.common.v1.Payload] @@ -708,6 +751,7 @@ def _completed_span( attributes: dict[str, opentelemetry.util.types.AttributeValue] = { "temporalWorkflowID": info.workflow_id, "temporalRunID": info.run_id, + "temporalNamespace": info.namespace, } if additional_attributes: @@ -718,7 +762,7 @@ def _completed_span( _CompletedWorkflowSpanParams( context=new_context_carrier, name=span_name, - # Always set span attributes as workflow ID and run ID + # Always set span attributes as workflow ID, run ID, and namespace attributes=attributes, time_ns=temporalio.workflow.time_ns(), link_context=link_context_carrier, diff --git a/temporalio/contrib/opentelemetry/_otel_interceptor.py b/temporalio/contrib/opentelemetry/_otel_interceptor.py index 5207095dc3..c77b09c166 100644 --- a/temporalio/contrib/opentelemetry/_otel_interceptor.py +++ b/temporalio/contrib/opentelemetry/_otel_interceptor.py @@ -36,6 +36,11 @@ import temporalio.worker import temporalio.workflow from temporalio.contrib.opentelemetry._context import attached_context +from temporalio.contrib.opentelemetry._interceptor import ( + _client_namespace, + _namespace_attributes, + _nexus_namespace_attributes, +) from temporalio.contrib.opentelemetry._tracer_provider import ( ReplaySafeTracerProvider, ) @@ -223,6 +228,7 @@ def __init__( ) -> None: super().__init__(next) self._add_temporal_spans = add_temporal_spans + self._namespace_attributes = _namespace_attributes(_client_namespace(next)) async def start_workflow( self, input: temporalio.client.StartWorkflowInput @@ -234,7 +240,10 @@ async def start_workflow( get_tracer(__name__), f"{prefix}:{input.workflow}", add_temporal_spans=self._add_temporal_spans, - attributes={"temporalWorkflowID": input.id}, + attributes={ + "temporalWorkflowID": input.id, + **self._namespace_attributes, + }, kind=opentelemetry.trace.SpanKind.CLIENT, ): input.headers = _context_to_headers(input.headers) @@ -245,7 +254,10 @@ async def query_workflow(self, input: temporalio.client.QueryWorkflowInput) -> A get_tracer(__name__), f"QueryWorkflow:{input.query}", add_temporal_spans=self._add_temporal_spans, - attributes={"temporalWorkflowID": input.id}, + attributes={ + "temporalWorkflowID": input.id, + **self._namespace_attributes, + }, kind=opentelemetry.trace.SpanKind.CLIENT, ): input.headers = _context_to_headers(input.headers) @@ -258,7 +270,10 @@ async def signal_workflow( get_tracer(__name__), f"SignalWorkflow:{input.signal}", add_temporal_spans=self._add_temporal_spans, - attributes={"temporalWorkflowID": input.id}, + attributes={ + "temporalWorkflowID": input.id, + **self._namespace_attributes, + }, kind=opentelemetry.trace.SpanKind.CLIENT, ): input.headers = _context_to_headers(input.headers) @@ -271,7 +286,10 @@ async def start_workflow_update( get_tracer(__name__), f"StartWorkflowUpdate:{input.update}", add_temporal_spans=self._add_temporal_spans, - attributes={"temporalWorkflowID": input.id}, + attributes={ + "temporalWorkflowID": input.id, + **self._namespace_attributes, + }, kind=opentelemetry.trace.SpanKind.CLIENT, ): input.headers = _context_to_headers(input.headers) @@ -280,8 +298,9 @@ async def start_workflow_update( async def start_update_with_start_workflow( self, input: temporalio.client.StartWorkflowUpdateWithStartInput ) -> temporalio.client.WorkflowUpdateHandle[Any]: - attrs = { + attrs: dict[str, opentelemetry.util.types.AttributeValue] = { "temporalWorkflowID": input.start_workflow_input.id, + **self._namespace_attributes, } if input.update_workflow_input.update_id is not None: attrs["temporalUpdateID"] = input.update_workflow_input.update_id @@ -311,6 +330,7 @@ async def start_activity( attributes={ "temporalActivityID": input.id, "temporalActivityType": input.activity_type, + **self._namespace_attributes, }, kind=opentelemetry.trace.SpanKind.CLIENT, ): @@ -341,6 +361,7 @@ async def execute_activity( "temporalWorkflowID": info.workflow_id or "", "temporalRunID": info.workflow_run_id or "", "temporalActivityID": info.activity_id, + "temporalNamespace": info.namespace, }, kind=opentelemetry.trace.SpanKind.SERVER, ): @@ -375,7 +396,7 @@ async def execute_nexus_operation_start( get_tracer(__name__), f"RunStartNexusOperationHandler:{input.ctx.service}/{input.ctx.operation}", add_temporal_spans=self._add_temporal_spans, - attributes={}, + attributes=_nexus_namespace_attributes(), kind=opentelemetry.trace.SpanKind.SERVER, ): return await self.next.execute_nexus_operation_start(input) @@ -388,7 +409,7 @@ async def execute_nexus_operation_cancel( get_tracer(__name__), f"RunCancelNexusOperationHandler:{input.ctx.service}/{input.ctx.operation}", add_temporal_spans=self._add_temporal_spans, - attributes={}, + attributes=_nexus_namespace_attributes(), kind=opentelemetry.trace.SpanKind.SERVER, ): return await self.next.execute_nexus_operation_cancel(input) @@ -421,6 +442,7 @@ def _workflow_maybe_span(self, name: str) -> Iterator[None]: attributes: dict[str, opentelemetry.util.types.AttributeValue] = { "temporalWorkflowID": info.workflow_id, "temporalRunID": info.run_id, + "temporalNamespace": info.namespace, } with _maybe_span( get_tracer(__name__), @@ -513,6 +535,7 @@ def _workflow_maybe_span( attributes: dict[str, opentelemetry.util.types.AttributeValue] = { "temporalWorkflowID": info.workflow_id, "temporalRunID": info.run_id, + "temporalNamespace": info.namespace, } with _maybe_span( get_tracer(__name__), diff --git a/temporalio/contrib/opentelemetry/workflow.py b/temporalio/contrib/opentelemetry/workflow.py index e872979a48..4bb16d1ac8 100644 --- a/temporalio/contrib/opentelemetry/workflow.py +++ b/temporalio/contrib/opentelemetry/workflow.py @@ -34,8 +34,8 @@ def completed_span( Args: name: Name of the span. - attributes: Attributes to set on the span if any. Workflow ID and - run ID are automatically added. + attributes: Attributes to set on the span if any. Workflow ID, + run ID, and namespace are automatically added. exception: Optional exception to record on the span. """ if interceptor := TracingWorkflowInboundInterceptor._from_context(): diff --git a/tests/contrib/opentelemetry/test_opentelemetry.py b/tests/contrib/opentelemetry/test_opentelemetry.py index 88fd2b4a7a..ade3b213e0 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry.py +++ b/tests/contrib/opentelemetry/test_opentelemetry.py @@ -25,7 +25,13 @@ from opentelemetry.trace import StatusCode, get_tracer from temporalio import activity, nexus, workflow -from temporalio.client import Client, WithStartWorkflowOperation, WorkflowUpdateStage +from temporalio.client import ( + Client, + OutboundInterceptor, + WithStartWorkflowOperation, + WorkflowUpdateStage, +) +from temporalio.client import Interceptor as ClientInterceptor from temporalio.common import RetryPolicy, WorkflowIDConflictPolicy from temporalio.contrib.opentelemetry import ( OpenTelemetryInterceptor, @@ -43,6 +49,7 @@ ApplicationErrorCategory, NexusOperationError, ) +from temporalio.service import RPCError from temporalio.testing import WorkflowEnvironment from temporalio.worker import ( ExecuteWorkflowInput, @@ -440,6 +447,7 @@ async def test_opentelemetry_tracing(client: Client, env: WorkflowEnvironment): "SignalWorkflow:signal", "StartWorkflowUpdate:update", ] + assert_namespace_on_spans(exporter.get_finished_spans(), client.namespace) async def test_opentelemetry_tracing_update_with_start( @@ -533,6 +541,7 @@ async def test_opentelemetry_tracing_update_with_start( " MyCustomSpan", " CompleteWorkflow:TracingWorkflow", ] + assert_namespace_on_spans(exporter.get_finished_spans(), client.namespace) @pytest.mark.requires_local_server @@ -595,6 +604,72 @@ async def test_opentelemetry_tracing_nexus(client: Client, env: WorkflowEnvironm " RunCancelNexusOperationHandler:InterceptedNexusService/intercepted_operation", " CompleteWorkflow:TracingWorkflow", ] + assert_namespace_on_spans(exporter.get_finished_spans(), client.namespace) + + +def assert_namespace_on_spans( + spans: Iterable[ReadableSpan], + namespace: str, + *, + only: Callable[[ReadableSpan], bool] = lambda _: True, +) -> None: + checked = [span for span in spans if only(span)] + assert checked + mismatched = { + span.name: (span.attributes or {}).get("temporalNamespace") + for span in checked + if (span.attributes or {}).get("temporalNamespace") != namespace + } + assert not mismatched, ( + f"Spans without temporalNamespace={namespace!r}: {mismatched}" + ) + + +class _PassthroughClientInterceptor(ClientInterceptor): + def intercept_client(self, next: OutboundInterceptor) -> OutboundInterceptor: + return OutboundInterceptor(next) + + +@pytest.mark.parametrize("interceptor_type", ["tracing", "otel"]) +async def test_opentelemetry_client_span_namespace_from_client( + client: Client, + interceptor_type: str, + reset_otel_tracer_provider: Any, # type: ignore[reportUnusedParameter] +): + # Uses a namespace that does not exist so the call fails after the client + # span is created, which avoids needing to register a namespace. + namespace = f"namespace-{uuid.uuid4()}" + exporter = InMemorySpanExporter() + if interceptor_type == "tracing": + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracing_interceptor: ClientInterceptor = TracingInterceptor( + get_tracer(__name__, tracer_provider=provider) + ) + else: + replay_safe_provider = create_tracer_provider() + replay_safe_provider.add_span_processor(SimpleSpanProcessor(exporter)) + opentelemetry.trace.set_tracer_provider(replay_safe_provider) + tracing_interceptor = OpenTelemetryInterceptor(add_temporal_spans=True) + client_config = client.config() + client_config["namespace"] = namespace + # Another interceptor below the tracing one must not hide the namespace + client_config["interceptors"] = [ + tracing_interceptor, + _PassthroughClientInterceptor(), + ] + client = Client(**client_config) + + with pytest.raises(RPCError): + await client.start_workflow( + SimpleWorkflow.run, + id=f"workflow_{uuid.uuid4()}", + task_queue=f"task_queue_{uuid.uuid4()}", + ) + + spans = exporter.get_finished_spans() + assert [span.name for span in spans] == ["StartWorkflow:SimpleWorkflow"] + assert_namespace_on_spans(spans, namespace) def dump_spans( @@ -1056,6 +1131,7 @@ async def test_opentelemetry_standalone_activity_tracing( assert start_activity_span.attributes is not None assert start_activity_span.attributes["temporalActivityID"] == activity_id assert start_activity_span.attributes["temporalActivityType"] == "tracing_activity" + assert_namespace_on_spans(finished_spans, client.namespace) def _v1_workflow_context(success_is_complete: bool = True) -> Any: diff --git a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py index 29cbde33d9..1dfb7e0043 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py +++ b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py @@ -30,7 +30,10 @@ from temporalio.testing import WorkflowEnvironment # Import the dump_spans function from the original opentelemetry test -from tests.contrib.opentelemetry.test_opentelemetry import dump_spans +from tests.contrib.opentelemetry.test_opentelemetry import ( + assert_namespace_on_spans, + dump_spans, +) from tests.helpers import assert_eventually, new_worker from tests.helpers.nexus import make_nexus_endpoint_name @@ -457,6 +460,10 @@ async def test_opentelemetry_comprehensive_tracing( assert actual_hierarchy == expected_hierarchy, ( f"Span hierarchy mismatch.\nExpected:\n{expected_hierarchy}\nActual:\n{actual_hierarchy}" ) + # Only spans created by the interceptor, user spans are untouched + assert_namespace_on_spans( + spans, new_client.namespace, only=lambda span: ":" in span.name + ) async def test_otel_tracing_with_added_spans(