OpenTelemetry Collector Cookbook

pip install "agentrust-sdk[otel]"

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 5s
    send_batch_size: 512
  memory_limiter:
    check_interval: 1s
    limit_mib: 256
  resource:
    attributes:
      - key: deployment.environment
        value: ${AGENTRUST_ENV}
        action: upsert

# Override exporters section per backend below
exporters:
  logging:
    verbosity: normal

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, resource, batch]
      exporters: [logging]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, resource, batch]
      exporters: [logging]

# docker-compose.override.yml — place in agentrust-edge/
services:
  jaeger:
    image: jaegertracing/all-in-one:1.57
    ports:
      - "16686:16686"   # Jaeger UI
      - "4317:4317"     # OTLP gRPC
      - "4318:4318"     # OTLP HTTP
    environment:
      COLLECTOR_OTLP_ENABLED: "true"

  gateway:
    environment:
      OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger:4317"
      OTEL_SERVICE_NAME: "agentrust-gateway"

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter

provider = TracerProvider()
provider.add_span_processor(
    BatchSpanProcessor(OTLPSpanExporter(endpoint="http://localhost:4317", insecure=True))
)
trace.set_tracer_provider(provider)

# Now import agentrust_sdk — it will find the tracer
from agentrust_sdk import harness, embed_gateway

# Gateway
OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-gateway-prod-eu-west-0.grafana.net/otlp
OTEL_EXPORTER_OTLP_HEADERS=Authorization=Basic <base64(instance_id:api_key)>
OTEL_SERVICE_NAME=agentrust-gateway

# SDK side (Python)
export OTEL_EXPORTER_OTLP_ENDPOINT=https://otlp-gateway-prod-eu-west-0.grafana.net/otlp
export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic <base64(instance_id:api_key)>"

from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry import trace
import os

provider = TracerProvider()
provider.add_span_processor(
    BatchSpanProcessor(
        OTLPSpanExporter(
            endpoint=os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] + "/v1/traces",
            headers={"Authorization": os.environ["OTEL_EXPORTER_OTLP_HEADERS"].split("=", 1)[1]},
        )
    )
)
trace.set_tracer_provider(provider)

# docker-compose.override.yml
services:
  datadog-agent:
    image: gcr.io/datadoghq/agent:7
    environment:
      DD_API_KEY: "${DD_API_KEY}"
      DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT: "0.0.0.0:4317"
      DD_SITE: "datadoghq.com"
    ports:
      - "4317:4317"

  gateway:
    environment:
      OTEL_EXPORTER_OTLP_ENDPOINT: "http://datadog-agent:4317"
      OTEL_SERVICE_NAME: "agentrust-gateway"

from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
# ... same TracerProvider setup as Option A, pointing at localhost:4317

apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-collector-config
  namespace: agentrust
data:
  config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
    processors:
      batch:
        timeout: 5s
      memory_limiter:
        limit_mib: 256
    exporters:
      otlp:
        endpoint: "<your-backend-endpoint>"
    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: [memory_limiter, batch]
          exporters: [otlp]
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector
  namespace: agentrust
spec:
  selector:
    matchLabels:
      app: otel-collector
  template:
    metadata:
      labels:
        app: otel-collector
    spec:
      containers:
        - name: otel-collector
          image: otel/opentelemetry-collector-contrib:0.100.0
          args: ["--config=/conf/config.yaml"]
          ports:
            - containerPort: 4317
              name: otlp-grpc
          volumeMounts:
            - name: config
              mountPath: /conf
          resources:
            limits:
              memory: 256Mi
              cpu: 250m
      volumes:
        - name: config
          configMap:
            name: otel-collector-config
---
apiVersion: v1
kind: Service
metadata:
  name: otel-collector
  namespace: agentrust
spec:
  selector:
    app: otel-collector
  ports:
    - port: 4317
      targetPort: 4317
      name: otlp-grpc

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm install otel-operator open-telemetry/opentelemetry-operator \
  --namespace observability --create-namespace

# my-values.yaml
gateway:
  env:
    OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.agentrust.svc.cluster.local:4317"
    OTEL_SERVICE_NAME: "agentrust-edge-gateway"
    OTEL_RESOURCE_ATTRIBUTES: "deployment.environment=production,k8s.namespace.name=agentrust"

helm upgrade agentrust-edge ./agentrust-edge/k8s/helm/agentrust-edge -f my-values.yaml

# 1. Send a validation request
curl -X POST http://localhost:8000/v1/runtime/validate \
  -H "Content-Type: application/json" \
  -d '{"agent_id":"otel-test","framework":"REST",
       "request":{"user":"u","input":"hello"},
       "execution":{"model":"gpt-4o","tools_called":[],"latency_ms":50,"tokens":10},
       "output":{"text":"ok"}}'

# 2. Verify spans in Jaeger
open http://localhost:16686
# Select service: agentrust-edge-gateway → Find Traces

# 3. Verify spans in Grafana Tempo
# Explore → Tempo → Tag: agentrust.agent_id = otel-test

# 4. Verify in Datadog
# APM → Services → agentrust-edge-gateway → Traces

# 5. Check collector received spans (debug logging exporter)
docker logs otel-collector 2>&1 | grep agentrust

pip install "agentrust-sdk"   # no [otel] — no traces, no metrics, no errors