Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/added/namespace-tagged-breadcrumbs.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add a `temporalNamespace` attribute to all spans created by the `temporalio.contrib.opentelemetry` tracing interceptors, set to the client namespace for client spans and the workflow, activity, or Nexus operation namespace for worker spans.
62 changes: 53 additions & 9 deletions temporalio/contrib/opentelemetry/_interceptor.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import temporalio.client
import temporalio.converter
import temporalio.exceptions
import temporalio.nexus
import temporalio.nexus.system.workflow_service.models
import temporalio.worker
import temporalio.workflow
Expand Down Expand Up @@ -60,6 +61,24 @@
_ContextT = TypeVar("_ContextT", bound=nexusrpc.handler.OperationContext)


def _client_namespace(next: temporalio.client.OutboundInterceptor) -> str | None:
# Client interceptors are not given the client, so walk the chain down to
# the SDK's terminal implementation, which holds it.
current: object = next
while current is not None:
client = getattr(current, "_client", None)
if isinstance(client, temporalio.client.Client):
return client.namespace
current = getattr(current, "next", None)
return None


def _namespace_attributes(
namespace: str | None,
) -> dict[str, opentelemetry.util.types.AttributeValue]:
return {"temporalNamespace": namespace} if namespace else {}


class TracingInterceptor(temporalio.client.Interceptor, temporalio.worker.Interceptor):
"""Interceptor that supports client and worker OpenTelemetry span creation
and propagation.
Expand Down Expand Up @@ -269,6 +288,7 @@ def __init__(
) -> None:
super().__init__(next)
self.root = root
self._namespace_attributes = _namespace_attributes(_client_namespace(next))

async def start_workflow(
self, input: temporalio.client.StartWorkflowInput
Expand All @@ -278,7 +298,10 @@ async def start_workflow(
)
with self.root._start_as_current_span(
f"{prefix}:{input.workflow}",
attributes={"temporalWorkflowID": input.id},
attributes={
"temporalWorkflowID": input.id,
**self._namespace_attributes,
},
input_with_headers=input,
kind=opentelemetry.trace.SpanKind.CLIENT,
):
Expand All @@ -287,7 +310,10 @@ async def start_workflow(
async def query_workflow(self, input: temporalio.client.QueryWorkflowInput) -> Any:
with self.root._start_as_current_span(
f"QueryWorkflow:{input.query}",
attributes={"temporalWorkflowID": input.id},
attributes={
"temporalWorkflowID": input.id,
**self._namespace_attributes,
},
input_with_headers=input,
kind=opentelemetry.trace.SpanKind.CLIENT,
):
Expand All @@ -298,7 +324,10 @@ async def signal_workflow(
) -> None:
with self.root._start_as_current_span(
f"SignalWorkflow:{input.signal}",
attributes={"temporalWorkflowID": input.id},
attributes={
"temporalWorkflowID": input.id,
**self._namespace_attributes,
},
input_with_headers=input,
kind=opentelemetry.trace.SpanKind.CLIENT,
):
Expand All @@ -309,7 +338,10 @@ async def start_workflow_update(
) -> temporalio.client.WorkflowUpdateHandle[Any]:
with self.root._start_as_current_span(
f"StartWorkflowUpdate:{input.update}",
attributes={"temporalWorkflowID": input.id},
attributes={
"temporalWorkflowID": input.id,
**self._namespace_attributes,
},
input_with_headers=input,
kind=opentelemetry.trace.SpanKind.CLIENT,
):
Expand All @@ -318,8 +350,9 @@ async def start_workflow_update(
async def start_update_with_start_workflow(
self, input: temporalio.client.StartWorkflowUpdateWithStartInput
) -> temporalio.client.WorkflowUpdateHandle[Any]:
attrs = {
attrs: dict[str, opentelemetry.util.types.AttributeValue] = {
"temporalWorkflowID": input.start_workflow_input.id,
**self._namespace_attributes,
}
if input.update_workflow_input.update_id is not None:
attrs["temporalUpdateID"] = input.update_workflow_input.update_id
Expand Down Expand Up @@ -347,6 +380,7 @@ async def start_activity(
attributes={
"temporalActivityID": input.id,
"temporalActivityType": input.activity_type,
**self._namespace_attributes,
},
input_with_headers=input,
kind=opentelemetry.trace.SpanKind.CLIENT,
Expand All @@ -367,7 +401,10 @@ async def execute_activity(
self, input: temporalio.worker.ExecuteActivityInput
) -> Any:
info = temporalio.activity.info()
attributes: dict[str, str] = {"temporalActivityID": info.activity_id}
attributes: dict[str, str] = {
"temporalActivityID": info.activity_id,
"temporalNamespace": info.namespace,
}
if info.workflow_id:
attributes["temporalWorkflowID"] = info.workflow_id
if info.workflow_run_id:
Expand Down Expand Up @@ -404,7 +441,7 @@ async def execute_nexus_operation_start(
with self._root._start_as_current_span(
f"RunStartNexusOperationHandler:{input.ctx.service}/{input.ctx.operation}",
context=self._context_from_nexus_headers(input.ctx.headers),
attributes={},
attributes=_nexus_namespace_attributes(),
input_with_ctx=input,
kind=opentelemetry.trace.SpanKind.SERVER,
):
Expand All @@ -416,13 +453,19 @@ async def execute_nexus_operation_cancel(
with self._root._start_as_current_span(
f"RunCancelNexusOperationHandler:{input.ctx.service}/{input.ctx.operation}",
context=self._context_from_nexus_headers(input.ctx.headers),
attributes={},
attributes=_nexus_namespace_attributes(),
input_with_ctx=input,
kind=opentelemetry.trace.SpanKind.SERVER,
):
return await self.next.execute_nexus_operation_cancel(input)


def _nexus_namespace_attributes() -> dict[str, opentelemetry.util.types.AttributeValue]:
if not temporalio.nexus.in_operation():
return {}
return _namespace_attributes(temporalio.nexus.info().namespace)


class _InputWithHeaders(Protocol):
headers: Mapping[str, temporalio.api.common.v1.Payload]

Expand Down Expand Up @@ -708,6 +751,7 @@ def _completed_span(
attributes: dict[str, opentelemetry.util.types.AttributeValue] = {
"temporalWorkflowID": info.workflow_id,
"temporalRunID": info.run_id,
"temporalNamespace": info.namespace,
}

if additional_attributes:
Expand All @@ -718,7 +762,7 @@ def _completed_span(
_CompletedWorkflowSpanParams(
context=new_context_carrier,
name=span_name,
# Always set span attributes as workflow ID and run ID
# Always set span attributes as workflow ID, run ID, and namespace
attributes=attributes,
time_ns=temporalio.workflow.time_ns(),
link_context=link_context_carrier,
Expand Down
37 changes: 30 additions & 7 deletions temporalio/contrib/opentelemetry/_otel_interceptor.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@
import temporalio.worker
import temporalio.workflow
from temporalio.contrib.opentelemetry._context import attached_context
from temporalio.contrib.opentelemetry._interceptor import (
_client_namespace,
_namespace_attributes,
_nexus_namespace_attributes,
)
from temporalio.contrib.opentelemetry._tracer_provider import (
ReplaySafeTracerProvider,
)
Expand Down Expand Up @@ -223,6 +228,7 @@ def __init__(
) -> None:
super().__init__(next)
self._add_temporal_spans = add_temporal_spans
self._namespace_attributes = _namespace_attributes(_client_namespace(next))

async def start_workflow(
self, input: temporalio.client.StartWorkflowInput
Expand All @@ -234,7 +240,10 @@ async def start_workflow(
get_tracer(__name__),
f"{prefix}:{input.workflow}",
add_temporal_spans=self._add_temporal_spans,
attributes={"temporalWorkflowID": input.id},
attributes={
"temporalWorkflowID": input.id,
**self._namespace_attributes,
},
kind=opentelemetry.trace.SpanKind.CLIENT,
):
input.headers = _context_to_headers(input.headers)
Expand All @@ -245,7 +254,10 @@ async def query_workflow(self, input: temporalio.client.QueryWorkflowInput) -> A
get_tracer(__name__),
f"QueryWorkflow:{input.query}",
add_temporal_spans=self._add_temporal_spans,
attributes={"temporalWorkflowID": input.id},
attributes={
"temporalWorkflowID": input.id,
**self._namespace_attributes,
},
kind=opentelemetry.trace.SpanKind.CLIENT,
):
input.headers = _context_to_headers(input.headers)
Expand All @@ -258,7 +270,10 @@ async def signal_workflow(
get_tracer(__name__),
f"SignalWorkflow:{input.signal}",
add_temporal_spans=self._add_temporal_spans,
attributes={"temporalWorkflowID": input.id},
attributes={
"temporalWorkflowID": input.id,
**self._namespace_attributes,
},
kind=opentelemetry.trace.SpanKind.CLIENT,
):
input.headers = _context_to_headers(input.headers)
Expand All @@ -271,7 +286,10 @@ async def start_workflow_update(
get_tracer(__name__),
f"StartWorkflowUpdate:{input.update}",
add_temporal_spans=self._add_temporal_spans,
attributes={"temporalWorkflowID": input.id},
attributes={
"temporalWorkflowID": input.id,
**self._namespace_attributes,
},
kind=opentelemetry.trace.SpanKind.CLIENT,
):
input.headers = _context_to_headers(input.headers)
Expand All @@ -280,8 +298,9 @@ async def start_workflow_update(
async def start_update_with_start_workflow(
self, input: temporalio.client.StartWorkflowUpdateWithStartInput
) -> temporalio.client.WorkflowUpdateHandle[Any]:
attrs = {
attrs: dict[str, opentelemetry.util.types.AttributeValue] = {
"temporalWorkflowID": input.start_workflow_input.id,
**self._namespace_attributes,
}
if input.update_workflow_input.update_id is not None:
attrs["temporalUpdateID"] = input.update_workflow_input.update_id
Expand Down Expand Up @@ -311,6 +330,7 @@ async def start_activity(
attributes={
"temporalActivityID": input.id,
"temporalActivityType": input.activity_type,
**self._namespace_attributes,
},
kind=opentelemetry.trace.SpanKind.CLIENT,
):
Expand Down Expand Up @@ -341,6 +361,7 @@ async def execute_activity(
"temporalWorkflowID": info.workflow_id or "",
"temporalRunID": info.workflow_run_id or "",
"temporalActivityID": info.activity_id,
"temporalNamespace": info.namespace,
},
kind=opentelemetry.trace.SpanKind.SERVER,
):
Expand Down Expand Up @@ -375,7 +396,7 @@ async def execute_nexus_operation_start(
get_tracer(__name__),
f"RunStartNexusOperationHandler:{input.ctx.service}/{input.ctx.operation}",
add_temporal_spans=self._add_temporal_spans,
attributes={},
attributes=_nexus_namespace_attributes(),
kind=opentelemetry.trace.SpanKind.SERVER,
):
return await self.next.execute_nexus_operation_start(input)
Expand All @@ -388,7 +409,7 @@ async def execute_nexus_operation_cancel(
get_tracer(__name__),
f"RunCancelNexusOperationHandler:{input.ctx.service}/{input.ctx.operation}",
add_temporal_spans=self._add_temporal_spans,
attributes={},
attributes=_nexus_namespace_attributes(),
kind=opentelemetry.trace.SpanKind.SERVER,
):
return await self.next.execute_nexus_operation_cancel(input)
Expand Down Expand Up @@ -421,6 +442,7 @@ def _workflow_maybe_span(self, name: str) -> Iterator[None]:
attributes: dict[str, opentelemetry.util.types.AttributeValue] = {
"temporalWorkflowID": info.workflow_id,
"temporalRunID": info.run_id,
"temporalNamespace": info.namespace,
}
with _maybe_span(
get_tracer(__name__),
Expand Down Expand Up @@ -513,6 +535,7 @@ def _workflow_maybe_span(
attributes: dict[str, opentelemetry.util.types.AttributeValue] = {
"temporalWorkflowID": info.workflow_id,
"temporalRunID": info.run_id,
"temporalNamespace": info.namespace,
}
with _maybe_span(
get_tracer(__name__),
Expand Down
4 changes: 2 additions & 2 deletions temporalio/contrib/opentelemetry/workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ def completed_span(

Args:
name: Name of the span.
attributes: Attributes to set on the span if any. Workflow ID and
run ID are automatically added.
attributes: Attributes to set on the span if any. Workflow ID,
run ID, and namespace are automatically added.
exception: Optional exception to record on the span.
"""
if interceptor := TracingWorkflowInboundInterceptor._from_context():
Expand Down
Loading