diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..40b375b8b02 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +_layout +_dotnetsdk +_downloads +_work +_package +_diag +ghalistener +**/bin/Debug +**/obj diff --git a/docs/adrs/4366-native-otel-export.md b/docs/adrs/4366-native-otel-export.md new file mode 100644 index 00000000000..380fc1c2e60 --- /dev/null +++ b/docs/adrs/4366-native-otel-export.md @@ -0,0 +1,326 @@ +# ADR 4366: Native OpenTelemetry export + +**Date**: 2026-06-17 (revised 2026-06-28) + +**Status**: Accepted + +## Context + +The runner already collects the timing and result data the GitHub Actions +timeline API exposes (job and per-step start/finish/result via `TimelineRecord`). +Operators who want this data in their own observability stack today have to poll +the timeline/REST API after the fact and translate it to OpenTelemetry out of +band — post-hoc, second-granularity, and without runner-host context. + +We want the runner to emit this telemetry **natively as OpenTelemetry**, in real +time, with sub-second precision, so it can be sent straight to any OTLP collector +(the OpenTelemetry Collector, Grafana/Tempo, Jaeger, Honeycomb, etc.). + +## Decision + +Add an opt-in, best-effort **OTLP/HTTP exporter** — `IOTelTraceExporter` +(`OTelTraceExporter`, a `RunnerService` resolved via `HostContext.GetService<>()`) +— implemented entirely with the .NET base class library (`System.Text.Json`, +`System.Security.Cryptography`, `System.Net.Http`). It emits all three OTLP +signals for each job the runner executes. + +### Signals + +| Signal | Endpoint | Payload | +|---------|---------------|---------| +| Traces | `/v1/traces` | One **job span** (trace root) + one **step span** per top-level step + child spans for action downloads, per-container pull/create, and the `run:` step's process (with semconv `process.exit.code` as a typed int). Job/step spans are `INTERNAL` (semconv task runs); network-shaped child spans (downloads, registry/daemon ops) are `CLIENT`. | +| Metrics | `/v1/metrics` | `cicd.pipeline.run.errors` (semconv monotonic counter, failures only), `github.pipeline.job.duration` (histogram), `github.pipeline.task.duration` (histogram, one point per job/step). Semconv names are adopted where the semantics match; the duration metrics stay vendor-namespaced because the worker observes one *job*, not the pipeline run — semconv `cicd.pipeline.run.duration` (whole-run duration by `cicd.pipeline.run.state`) can only be emitted correctly by the control plane, and semconv defines no task-level duration metric. | +| Logs | `/v1/logs` | Job- and step-level annotations (`::warning::` / `::error::` / `::notice::`), each correlated to its span via `traceId`/`spanId`. | + +Runner identity is attached once as the OTLP **Resource**: +`service.name=github-actions-runner`, `service.version`, `host.name`, `host.arch`, +`os.type` (arch/OS mapped to the semconv enum values — `amd64`/`arm64`, +`linux`/`darwin`/`windows`), semconv `cicd.worker.{name,id}` + +`cicd.system.component=agent`, +`service.instance.id`, and `github.runner.{group,ephemeral}`. The standard +`OTEL_RESOURCE_ATTRIBUTES` env var is honored and merged (so ARC can attach +`k8s.*` via the Downward API), with explicitly-set keys taking precedence. + +Spans and steps carry GitHub/VCS context (`github.*`, `vcs.*`) and CI/CD +semantic-convention attributes (`cicd.pipeline.*`). On failure the span gets +`error.type` (low-cardinality classifier), an ERROR status carrying the error +message as status `message`, and — when a message exists — a message-only +`exception` event (a CI conclusion is not an exception class, so +`exception.type` is omitted rather than faked). + +### Trace shape — the job span is the trace root + +The job span is the **root of the runner's trace** (`parentSpanId` omitted). The +runner owns the *job*, not the *workflow run*, so it does **not** parent the job +to a run/workflow span it never emits — doing so left a dangling `parentSpanId` +that broke trace assembly (regression-guarded by `JobSpan_IsRoot_NoDanglingParent`). + +Two relationships replace that link: + +- **The authoritative run span comes from the GitHub API path** (a separate + consumer that reconstructs the run from the timeline/REST API) and is merged + downstream by the **deterministic ID contract** (see below), so the run span + and the runner's job span land in the same trace without the runner emitting + the run span itself. +- **An inbound scheduler context is attached as a span LINK, not a parent.** If an + upstream controller (e.g. ARC) injects a W3C `traceparent` via + `ACTIONS_RUNNER_PARENT_TRACEPARENT` (+ `…_TRACESTATE`), `AddInboundParentLink` + parses it and adds a link (`cicd.system.component=controller`) to the job span. + This expresses cross-trace causality without re-rooting the runner's own + deterministic trace; `tracestate` and the inbound sampled flag ride on the link. + +### Deterministic, content-derived span IDs + +Span/trace IDs are derived only from identifiers the runner always has — run id, +run attempt, and job/step **display names** — via **SHA-256 of a UTF-8 input +string, truncated** (16 bytes for a trace id, 8 for a span id). This lets a +consumer that also reconstructs the trace from the GitHub API compute the same +IDs and merge the two sources (server-side queue time from the API + runner-native +step precision). + +``` +traceId = sha256("{run_id}-{run_attempt}")[:16] (32 hex) +job span = sha256("job-{run_id}-{run_attempt}-{job_name}")[:8] (16 hex) +step span = sha256("step-{run_id}-{run_attempt}-{job_name}-{step_number}-{step_name}")[:8] +child span = sha256("{type}-{run_id}-{run_attempt}-{job_name}-{name}-{startNano}")[:8] +``` + +SHA-256 (not MD5) because MD5 throws on a FIPS-enabled host. The IDs are +intentionally deterministic, so the W3C Level-2 randomness flag is never set. The +full normative spec, rationale, and golden conformance vectors live in +[`docs/otel-id-contract.md`](../otel-id-contract.md) — that document, not any +external repo, is the source of truth for the contract. + +Only **top-level** steps get spans; composite/embedded sub-steps are skipped +(their display names aren't unique and have no counterpart in the +API-reconstructed trace). + +### Transport + +- **OTLP/JSON over HTTP**, built with `Utf8JsonWriter` (correct escaping — a + control char in a step name can't invalidate the batch). Attribute values are + serialized with the correct OTLP `AnyValue` wire types (string/bool/int/ + double/array/bytes), not stringified. No protobuf, no gRPC, no SDK (see + *Alternatives* below). Requests identify themselves with a + `GitHubActionsRunner-OTLP-Exporter/{version}` `User-Agent`, so collector + operators can attribute and rate-limit the traffic. +- **gzip** `Content-Encoding`. A job's spans/logs are highly repetitive, so this + typically shrinks the body ~10x (see `docs/otel-benchmarks.md`); + `CompressionLevel.Fastest` keeps CPU negligible. Always-on with no opt-out: the + OTLP spec requires every server component to accept gzip request bodies. +- **Chunked POSTs.** Spans/logs are sent in batches of at most `MaxItemsPerPost` + (1,000) per request — ~4 MB uncompressed even at the 10k buffer cap, safely + under the OTel Collector's 20 MiB default decompressed-body limit — so a 413 + (non-retryable per OTLP) can never drop a whole signal, and per-POST + serialization memory stays bounded. All chunks share the one flush deadline. +- **One retry** on a transient failure (exactly the OTLP/HTTP retryable codes — + 429/502/503/504 — or a transport exception), bounded by the flush deadline. + The server's `Retry-After` is honored when it fits the deadline (skipping the + retry when it can't); otherwise the delay is a jittered 100–400 ms so fleets + don't retry in lockstep. With this deliberate single-retry bound, the spec's + exponential-backoff SHOULD degenerates to that one jittered wait. Per-request + HTTP timeout is 5 s. +- Sent over the runner's **proxy-aware** `HostContext.CreateHttpClientHandler()`, + honoring the runner's existing proxy config. For self-signed/private-CA + collectors, `ACTIONS_RUNNER_OTLP_CERTIFICATE` points at a PEM CA bundle that + is trusted for the collector connection (hostname still checked; a bad bundle + fails closed). `ACTIONS_RUNNER_OTLP_INSECURE=true` instead disables **all** + server-certificate validation — a MITM can then read the export traffic, + including any `ACTIONS_RUNNER_OTLP_HEADERS` credential — so enabling it logs + an explicit warning; prefer the CA bundle. +- **Per-buffer memory caps** (`MaxBufferedSpans`/`Logs`/`TaskMetrics` = 10 000 + each). A pathological job (e.g. unbounded `::warning::` annotations) can't grow + the buffers without bound; excess is dropped and counted, never OOMing the + worker. The job span — the trace root, recorded last — bypasses the span cap + (one per job, worst case cap+1): dropping it would orphan every exported step + span, which all parent to its deterministic ID. +- **Secret masking at flush.** Every exported string — span names, attribute and + resource values, log bodies — is run through the runner's `SecretMasker` before + serialization, the same scrubbing applied to all other off-box telemetry. + Collector auth-header values from `ACTIONS_RUNNER_OTLP_HEADERS` are registered + with the masker too (deliberately with **no minimum-length floor** — a short + credential is still a credential), and the variable itself is scrubbed from + the worker's process env right after it is read — host step processes inherit + that env, so leaving it set would hand the raw credential to every step. + +### Lifecycle + +1. `JobRunner.RunAsync` → `SetResource(...)` captures runner identity. +2. `ExecutionContext.InitializeJob` → `SetJobInfo(...)` captures run/job + identifiers once, so every span ID and parent link stays consistent. +3. `ExecutionContext.Complete()` → `RecordStepCompletion` / `RecordJobCompletion` + build pending spans/logs/metrics from the timeline record. +4. `JobRunner.CompleteJobAsync` → `FlushAsync()` POSTs all three signals. + +### Gating + +Two independent gates, both must allow export: + +| Gate | Controlled by | Default | +|------|---------------|---------| +| `ACTIONS_RUNNER_OTLP_ENDPOINT` set | Runner operator (env) | unset → **off** | +| `actions_runner_otel_export` feature flag | GitHub (server) | unprovisioned → **on** | + +``` +enabled = endpoint configured && (feature flag ?? true) +``` + +The endpoint env var is the operator's explicit opt-in. The feature flag is a +server-side **kill switch** layered on top. The flag defaults **on** when +unprovisioned: the operator already opted in by configuring an endpoint, and a +`?? false` default would silently break that opt-in on self-hosted / GHES runners +where the flag isn't provisioned (the primary audience). On github.com the flag +is always provisioned, so GitHub retains full control by sending `false`; the +default only matters where the server doesn't know the flag, and there "on" is +correct. **Export still requires the operator to set an endpoint** — an +unprovisioned flag alone exports nothing. + +**Known convention deviation (needs maintainer sign-off):** every other +`Constants.Runner.Features` gate in this repo defaults off (`?? false`); this +is the sole `?? true`, for the reason above. If the conventional polarity is +required, the equivalent is a negative flag (e.g. +`actions_runner_otel_export_disabled`, `?? false`) — same behavior everywhere, +conventional default. We'd rather take that rename in review than silently +break the self-hosted/GHES opt-in with a positive `?? false`. + +## Alternatives considered: OpenTelemetry .NET SDK + +The first question a maintainer will ask is *"why not just use the +OpenTelemetry .NET SDK?"* We hand-rolled the exporter deliberately: + +- **Zero new dependencies.** The runner ships with **no OpenTelemetry NuGet + packages** (confirmed: `grep -i opentelemetry src/Runner.Worker/Runner.Worker.csproj` + → none). The runner is deliberately dependency-averse — it is widely deployed, + security-sensitive, and trim/AOT-sensitive. Pulling in the SDK + an OTLP + exporter + their transitive graph (gRPC/protobuf or the HTTP exporter stack) + onto every runner is a large, hard-to-justify supply-chain and binary-size + cost for what is ~1 file of BCL code. +- **Content-derived IDs.** The design needs **deterministic, content-derived + span IDs** (SHA-256 of identifier strings) so runner spans dedupe/merge with + API-reconstructed spans. The SDK's `Activity`/`IdGenerator` model assumes + random IDs and has no clean hook to derive an ID from span content at creation. +- **Post-hoc reconstruction, not live tracing.** Spans are reconstructed *after* + the fact from `TimelineRecord`s (start/end/result already known), not traced + live through an `Activity`/`ActivityListener`. The SDK is built around live, + in-process activities; bending it to emit fully-formed historical spans fights + the grain. +- **Native primitives already exist.** The runner already has a native + `SecretMasker` (mandatory for anything leaving the box) and a proxy-aware + `HttpClientHandler`. The SDK exporter would need to be taught both. + +OTLP wire choices, with rationale: + +- **JSON over protobuf** — no protobuf dependency or codegen; trivially + inspectable; serialized safely with `Utf8JsonWriter`. The ~size cost is erased + by gzip. +- **HTTP over gRPC** — every OTLP collector accepts OTLP/HTTP; it traverses the + runner's existing HTTP proxy stack; no HTTP/2 / gRPC dependency. + +Sub-alternatives rejected: + +- **SDK exporter only** (hand-build `Activity`s, export via the SDK's OTLP + exporter) — still drags in the SDK + exporter dependency graph and still has no + clean content-derived-ID hook; saves little over a full hand-roll. +- **Sidecar / collector-only** (runner does nothing; an external agent scrapes + the API) — that's exactly today's status quo we're replacing: post-hoc, + second-granularity, no runner-host context, and extra infrastructure for every + operator. + +## Trade-offs (review-flagged, documented honestly) + +- **(a) Flush is on the job-completion critical path — by design.** `FlushAsync` + is awaited *before* the completion report. Reordering (flush after, or racing + the two) was considered and rejected: the completion report is what lets + ephemeral runners (e.g. ARC) tear the worker down, so flushing after it races + pod deletion and nondeterministically drops the job's telemetry — on exactly + the fleets this feature targets. The cost is bounded to a **4 s overall + deadline** across all three signals (including the retry), so a hung or + unreachable collector can delay job completion by at most that (fast-fail + errors like connection-refused cost far less); healthy-path cost is + milliseconds. It is **best-effort**: every exporter entry point + (`Initialize`, `SetResource`, `SetJobInfo`, `Record*`, `StepPropagationEnv`, + `FlushAsync`) is guarded — telemetry can never throw into the job. Failure is + not silent either: when any export POST fails or buffered data is dropped at + the caps, one end-of-job **Warning** summary says exactly what was lost, so + operators grep one line instead of correlating per-signal `_diag` entries. A + worker crash before flush loses that job's un-flushed telemetry — acceptable, + since telemetry must never gate or fail real work. +- **(b) Sampling/cardinality is server/collector-controlled.** The runner emits + **everything** for an enabled job (no client-side sampling); volume is governed + by the server feature flag + the operator's endpoint opt-in, and any + sampling/aggregation is the collector's job. This keeps the runner simple and + puts the cost/volume policy where the operator controls it. +- **(c) The CICD + VCS semantic conventions are experimental.** The `cicd.*` and + `vcs.*` attributes are **Development/experimental** in semconv and subject to + breaking change. `schema_url` is **pinned** (`…/schemas/1.34.0`, audited so every emitted attribute exists in that release) on every + resource/scope so consumers can detect the version and adapt. +- **(d) Flag default + endpoint requirement.** The feature flag defaults **on** + when unprovisioned, but export still requires an operator-set endpoint, so the + on-by-default flag never causes unexpected egress on its own. + +## Configuration + +| Env var | Required | Description | +|---------|----------|-------------| +| `ACTIONS_RUNNER_OTLP_ENDPOINT` | Yes | OTLP/HTTP base URL, e.g. `http://collector:4318`. Signals POST to `{endpoint}/v1/{traces,metrics,logs}`. A URL already ending in `/v1/traces` is accepted. | +| `ACTIONS_RUNNER_OTLP_HEADERS` | No | Comma-separated `key=value` headers for collector auth, e.g. `authorization=Bearer xyz,x-api-key=abc`. Values are registered with the secret masker. | +| `ACTIONS_RUNNER_OTLP_CERTIFICATE` | No | Path to a PEM CA bundle trusted for the collector connection (self-signed/private-CA collectors). Mirrors `OTEL_EXPORTER_OTLP_CERTIFICATE`. | +| `ACTIONS_RUNNER_OTLP_INSECURE` | No | `true` disables **all** TLS certificate validation (not just self-signed) — exposes the connection, including auth headers, to MITM. Logs a warning; prefer `…_OTLP_CERTIFICATE`. | +| `ACTIONS_RUNNER_OTLP_PROPAGATE` | No | `true` to inject `TRACEPARENT` + `OTEL_*` into each step's env so in-job tools nest under the step span. | +| `ACTIONS_RUNNER_PARENT_TRACEPARENT` / `…_TRACESTATE` | No | Inbound W3C context from an upstream scheduler; attached to the job span as a link. | +| `OTEL_RESOURCE_ATTRIBUTES` | No | Standard OTel env var; merged into the Resource (e.g. ARC `k8s.*`). | + +These are runner-namespaced (`ACTIONS_RUNNER_OTLP_*`) rather than the standard +`OTEL_*` export variables, so enabling runner export does not clobber the `OTEL_*` +config a workflow's own steps may rely on for their application telemetry. + +### Step propagation (`StepPropagationEnv`) + +When `ACTIONS_RUNNER_OTLP_PROPAGATE=true`, each step's env is given +(`run:` script steps, JS actions, and Docker container actions; composite +actions receive it through their embedded steps): + +- `TRACEPARENT = 00-{traceId}-{stepSpanId}-01` (matches the exporter's IDs, so + in-job OTel tools nest under the correct step span), +- `OTEL_EXPORTER_OTLP_ENDPOINT` (the base URL, with any URL userinfo credential + — `user:token@` — stripped; the userinfo is also registered with the secret + masker so it can never appear raw in diag logs), +- `OTEL_RESOURCE_ATTRIBUTES` (run/job/repo identity). + +**Precedence: workflow wins.** Each variable is injected only when the step's +env does not already define it — a step that sets its own `TRACEPARENT` or +`OTEL_*` (for its application's telemetry) is never clobbered by the runner +(guarded by `OTelPropagation_NeverClobbersWorkflowSetEnv`). + +The endpoint is propagated from the **runner host's perspective**: for +`container:` jobs and container actions the collector must be reachable from +inside the job container — `http://localhost:4318` on the host does not resolve +there (use a host-gateway or cluster-reachable address instead). + +It **deliberately does not** propagate `OTEL_EXPORTER_OTLP_HEADERS`: that header +carries the collector credential, and handing it to user step processes would let +any step read and exfiltrate it. Steps needing collector auth must get a separate, +scoped credential out of band (guarded by `StepPropagationEnv_DoesNotLeakAuthHeaders`). + +### Viewing locally + +```bash +# Point the runner at any OTLP/HTTP collector (the OpenTelemetry Collector, +# Jaeger :4318, Grafana Alloy, ...): +export ACTIONS_RUNNER_OTLP_ENDPOINT=http://localhost:4318 +./run.sh +``` + +## Consequences + +- Operators get real-time, sub-second job/step traces, metrics, and logs with + runner-host context, no API polling. +- The deterministic ID contract means runner spans merge cleanly with + API-reconstructed traces, and re-runs need no special handling: attempt _N_ + lands in trace `sha256("{run_id}-{N}")[:16]`, the same trace the API path + reconstructs for that attempt. +- The exporter is self-contained: if the OTLP wire format must evolve, only + `OTelTraceExporter` (serialization) and `OTelHttpTransport` (the extracted, + wire-tested HTTP component) change — no dependency churn. +- The runner stays dependency-free of the OTel SDK, preserving its trim/AOT and + supply-chain posture. diff --git a/docs/otel-benchmarks.md b/docs/otel-benchmarks.md new file mode 100644 index 00000000000..f40e8b73168 --- /dev/null +++ b/docs/otel-benchmarks.md @@ -0,0 +1,93 @@ +# Native OTel export — performance & memory benchmarks + +Overhead of the native OpenTelemetry export (PR #4366). Goal: prove the +instrumentation is cheap enough to run on every job and bounded in memory. + +## Micro-benchmark (emit + serialize) + +`src/Test/L0/Worker/OTelTraceExporterBenchmark.cs` — reuses the exporter's internal +hooks; measures per-step record cost, end-of-job OTLP/JSON serialize cost, payload +size, and allocations. Run: + +``` +dotnet test src/Test/Test.csproj -c Release \ + --filter "FullyQualifiedName~OTelTraceExporterBenchmark" \ + --logger "console;verbosity=detailed" +``` + +Results (Release, net8.0, Apple M-series; representative): + +| spans | emit (ns/step) | alloc (B/step) | retained (B/span) | serialize | payload (B/span) | +|-------:|---------------:|---------------:|------------------:|----------:|-----------------:| +| 10 | ~6,500 | ~4,200 | ~5,700 | 0.09 ms | 1,829 | +| 100 | ~2,800 | ~4,120 | ~4,280 | 0.83 ms | 1,806 | +| 1,000 | ~2,300 | ~4,110 | ~4,150 | 11.2 ms | 1,807 | +| 10,000 | ~3,500 | ~4,150 | ~3,230 | 97 ms | 1,811 | + +### Reading it +- **Emit: ~2–6 µs per step.** Negligible next to real step durations (seconds–minutes). + Cost is 3× SHA-256 (deterministic IDs) + ~30 attribute sets per span. +- **Memory: ~4 KB allocated and retained per span**, held until the single end-of-job + flush. Linear and freed at job end: + - 50 steps ≈ 0.2 MB · 200 ≈ 0.8 MB · 1,000 ≈ 4 MB · **10,000 ≈ ~32 MB** + - This is the case for a per-buffer cap (see below): a pathological job (100k + steps/annotations) would retain hundreds of MB. +- **Serialize: ~10 µs/span** (one `Utf8JsonWriter` pass at flush). 11 ms @ 1k, ~100 ms @ 10k. +- **Payload: ~1.8 KB/span** of OTLP/JSON, gzipped and chunked into POSTs of at most + `MaxItemsPerPost` (1,000) spans/logs each (→ ~4 MB uncompressed per request even + at the 10k buffer cap, safely under the OTel Collector's 20 MiB default body limit). + +## Macro-benchmark (job wall-time, ON vs OFF) — measured + +A 50-trivial-step workflow (`run: ":"`) on the local self-hosted runner, OTel ON +(healthy collector) vs OFF (endpoint unset → exporter disabled), job wall-time from +the GitHub job `started_at`→`completed_at`: + +| | run 1 | run 2 | run 3 | +|---|---|---|---| +| **OTel ON** | 18 s | 25 s | 18 s | +| **OTel OFF** | 18 s | 25 s | 25 s | + +The two distributions **fully overlap**: job-to-job variance (~7 s, from runner +orchestration/scheduling) dwarfs any OTel signal. **Caveat: n=3 per arm.** With +samples this small the honest claim is bounded, not statistical: any ON/OFF delta +is below the ~7 s orchestration noise floor, consistent with the micro numbers +(µs/step emit + a single bounded, gzipped flush POST at job end). The hard +guarantee comes from the failure-mode bounds below, not from these three runs. + +## Failure modes (collector down) — measured + +The job-end flush is the only place a bad collector can cost wall time. Both +failure classes are measured in L0 (run on every CI pass, not just benchmarked +once), timing a real `FlushAsync` against a real local socket: + +| collector failure | test | measured job-end delay | +|---|---|---| +| Connection refused (down, NXDOMAIN-like fast-fail) | `Flush_UnreachableEndpoint_DoesNotThrow` | **< 1 s** (fail + one 200 ms retry) | +| Accepts TCP, then stalls (blackhole / slow-loris) | `Flush_HangingCollector_BoundedByOverallDeadline` | **~4 s**, asserted < 6 s | + +The stall case is the worst case: nothing fast-fails, so the 4 s overall flush +deadline (`flushCts.CancelAfter(4 s)`, one deadline across all three signals +including the retry) is what bounds it — and the test asserts that bound holds +against a genuinely hung socket. DNS failure and blackholed IPs collapse to the +same two classes: either the socket op errors fast (row 1) or the cancellation +deadline fires (row 2). Flush is best-effort and swallowed — it never fails the +job; the loss is reported in one end-of-job Warning summary. + +## Sampling / scale model + +Decision: **server-controlled**. The runner emits every span/metric/log +unconditionally (gated only by the server feature flag + the operator's endpoint +opt-in). Sampling and metric-cardinality control are the **collector's / server's** +responsibility (tail sampling, cardinality limits, rate limiting) — not the runner's. +This keeps the runner simple, preserves tail visibility (all failures), and matches +how export is already gated. Memory is bounded at the source by the per-buffer caps +(see code: `MaxBufferedSpans/Logs/TaskMetrics`). + +## Acceptable-overhead thresholds (proposed) +- Disabled (common case): job wall-time delta < 0.1 %; RSS delta within noise. +- Enabled, healthy collector: added job-end wall-time < ~100 ms. +- Enabled, collector down: added job-end wall-time ≤ ~4 s (the flush deadline; + measured above, asserted in L0). +- Per-step emit < ~50 µs. Serialize < ~10 ms for 200 spans. +- Peak RSS delta a few KB/span (cap buffers so a runaway job can't OOM the worker). diff --git a/docs/otel-id-contract.md b/docs/otel-id-contract.md new file mode 100644 index 00000000000..990ab4bf485 --- /dev/null +++ b/docs/otel-id-contract.md @@ -0,0 +1,153 @@ +# OpenTelemetry deterministic ID contract + +**Status**: Normative. This document is the source of truth for the deterministic +trace/span IDs the runner's native OTLP exporter emits (see +[ADR 4366](adrs/4366-native-otel-export.md)). It is owned by this repository. + +Any independent consumer that reconstructs a GitHub Actions trace from the public +API (timeline / REST) MUST compute IDs by this spec to merge with runner-emitted +spans. [`stefanpenner/otel-explorer`](https://github.com/stefanpenner/otel-explorer) +is **one example consumer** of this published contract, not a byte-compatibility +obligation: this document, not that repo, defines the contract. + +The reference implementation is `OTelTraceExporter` in +`src/Runner.Worker/OTelTraceExporter.cs` (functions `NewTraceID`, `NewJobSpanID`, +`NewStepSpanID`, `NewSpanIDFromString`, `NewSpanID`). + +## Why deterministic IDs + +Two systems describe the same run from different vantage points: + +- the **runner**, post-hoc, from its `TimelineRecord`s (sub-second step timing, + runner-host context), and +- the **GitHub API path**, which reconstructs the run/workflow span and + server-side queue time. + +Deriving every ID from stable identifiers both sides already know means the two +sets of spans **dedupe and merge into one trace** with no shared state, no +coordination, and no W3C propagation between them. The same property makes +**re-runs** trivial: each attempt is its own trace. + +## Algorithm + +``` +id = truncate( SHA-256( UTF-8( input_string ) ), N bytes ) → lowercase hex +``` + +- **SHA-256**, not MD5. MD5 throws on a FIPS-enabled host; SHA-256 gives the same + deterministic-from-`run_id` behavior. Only the **leading bytes** of the digest + are used. +- **Trace ID**: leading **16 bytes** → **32 hex chars**. +- **Span ID**: leading **8 bytes** → **16 hex chars**. +- Hex is lowercase, zero-padded per byte (`x2`). +- `run_attempt` of `0` (or absent) is normalized to `1` before hashing. +- IDs are intentionally **non-random**, so the W3C trace-context Level-2 + randomness flag is never set. + +## Input strings + +| ID | Bytes | Input string (UTF-8) | +|----|-------|----------------------| +| Trace ID | 16 | `"{run_id}-{run_attempt}"` | +| Job span | 8 | `"job-{run_id}-{run_attempt}-{job_name}"` | +| Step span | 8 | `"step-{run_id}-{run_attempt}-{job_name}-{step_number}-{step_name}"` | +| Child span | 8 | `"{span_type}-{run_id}-{run_attempt}-{job_name}-{name}-{start_unix_nano}"` | + +Field definitions: + +- **`run_id`** — GitHub `github.run_id` (the workflow run id). +- **`run_attempt`** — `github.run_attempt`, 1-based (`0`/absent → `1`). +- **`job_name`** — the job's **display name** (matches the GitHub API `job.name`), + not the `github.job` context key. +- **`step_number`** — the **1-based** top-level step number, matching the GitHub + API `step.number`. It disambiguates two top-level steps that share a display + name while keeping the ID mergeable with the API trace. +- **`step_name`** — the step's display name. +- **`span_type`**, **`name`**, **`start_unix_nano`** — for generic child spans + (e.g. action download). The start time is included so repeated same-named + operations don't collide. Child spans are runner-only; the API path has no + counterpart, so they are not a merge point. + +### Workflow / run span ID (API-path only) + +The trace also contains a **run/workflow span** whose ID is the **big-endian +8-byte encoding of `run_id`** (`NewSpanID(run_id)`), *not* a hash. The runner +**does not emit this span** and **does not parent its job span to it** — the job +span is the trace root (`parentSpanId` omitted). The encoding is retained in the +runner and published here only so the **API path** can emit the run span into the +same trace; the runner's job span merges by sharing the trace ID. + +``` +run/workflow span = bigendian_uint64(run_id) → 16 hex +``` + +## Trace shape + +``` +trace sha256("{run_id}-{run_attempt}")[:16] +└─ (run/workflow span bigendian(run_id)) ← emitted by the API path, not the runner + └─ job span sha256("job-…")[:8] ← runner: trace ROOT (no parent) + ├─ step span sha256("step-…-{n}-…")[:8] + │ └─ child span sha256("{type}-…")[:8] + └─ … +``` + +An inbound scheduler `traceparent` (if present) is attached to the job span as a +**link**, not a parent — it does not change any ID above. + +### Span kinds + +Per the CI/CD span conventions (cicd-spans), **task-run spans are `INTERNAL`** +and **pipeline-run spans are `SERVER`** (and carry `cicd.pipeline.result`). The +runner's job and step spans are task runs (`cicd.pipeline.task.*`), so the +runner emits them as `INTERNAL`; the `SERVER` pipeline-run span belongs to the +API path with the run/workflow span above. Span kind is therefore a structural +discriminator: `SERVER` = the run, `INTERNAL` = tasks. + +## Attribute scope: cicd.worker.* + +`cicd.worker.name` and `cicd.worker.id` identify the runner host that executed +a job. Their OTel **scope** differs between the runner and the ARC listener: + +| Component | Scope | Rationale | +|-----------|-------|-----------| +| Runner (`OTelTraceExporter.cs`) | **Resource** attribute | One worker process per runner host; the identity is constant for the process lifetime. | +| ARC listener (`cmd/ghalistener/metrics/otel.go`) | **Span** attribute | One listener process serves many runners; the worker identity varies per job span. | + +Both carry the same key names and value semantics. A TraceQL query that must +match spans from either component should check both scopes: + +``` +span.cicd.worker.name != "" || resource.cicd.worker.name != "" +``` + +## On enumerability of trace IDs + +Trace IDs derive from the **public, enumerable `run_id`** (a known, guessable +property of a run). This is **acceptable for CI traces**: the data is workflow +metadata, the collector is operator-controlled, and the merge property requires a +shared, reproducible trace ID. It is a deliberate trade-off, not an oversight — do +not treat a runner trace ID as a secret. + +## Conformance vectors + +Golden values from the L0 suite (`src/Test/L0/Worker/OTelTraceExporterL0.cs`). A +conforming implementation MUST reproduce these exactly. + +| Function | Input | Output | +|----------|-------|--------| +| Trace ID | `run_id=99999, run_attempt=1` → `sha256("99999-1")[:16]` | `acad1e2a107636235fd56bb742499bd0` | +| Job span | `99999, 1, "build"` → `sha256("job-99999-1-build")[:8]` | `81606d47848a59c0` | +| Step span | `99999, 1, "build", 3, "Run tests"` → `sha256("step-99999-1-build-3-Run tests")[:8]` | `7a4c67339b7bb8a7` | +| Run/workflow span | `bigendian(99999)` | `000000000001869f` | +| Run/workflow span | `bigendian(42)` | `000000000000002a` | + +Derived check — the propagated `TRACEPARENT` for that step is +`00-acad1e2a107636235fd56bb742499bd0-7a4c67339b7bb8a7-01` +(`00-{traceId}-{stepSpanId}-01`). + +Normalization checks: + +- `run_attempt = 0` produces the same IDs as `run_attempt = 1`. +- The same inputs always produce the same IDs; different `run_attempt` produces a + different trace ID. diff --git a/otel-runner.Dockerfile b/otel-runner.Dockerfile new file mode 100644 index 00000000000..ca1a909833e --- /dev/null +++ b/otel-runner.Dockerfile @@ -0,0 +1,39 @@ +# Build an ARC-compatible runner image carrying the native-OTel runner +# (actions/runner#4366). We build the linux layout from source, then overlay it +# onto the official actions-runner image (keeping its runner user, k8s hooks, +# and entrypoint). +# +# docker build -f otel-runner.Dockerfile -t otel-runner:dev . +# kind load docker-image otel-runner:dev --name gha-runner + +# ---- builder: compile the runner layout from source (linux/arm64 native) ---- +FROM mcr.microsoft.com/dotnet/sdk:8.0-noble AS builder +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl git unzip ca-certificates \ + && rm -rf /var/lib/apt/lists/* +ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 +WORKDIR /runner +COPY . . +WORKDIR /runner/src +# The reported runner version must be >= GitHub's current minimum, else the +# Actions service denies the ephemeral/JIT runner (AccessDenied -> exit code 7), +# which ARC interprets as "Outdated" and tears the runner down in a loop. +# PR#4366 branches off 2.333.0; bump the reported version to a current one. +ARG RUNNER_VERSION_OVERRIDE=2.335.1 +RUN printf '%s' "${RUNNER_VERSION_OVERRIDE}" > runnerversion +# dev.sh bootstraps its own pinned dotnet SDK and downloads externals (node). +RUN ./dev.sh layout Release + +# ---- runtime: official ARC runner image + our instrumented binaries ---- +FROM ghcr.io/actions/actions-runner:latest +# Merge the source-built layout over the official runner home. This replaces +# bin/externals/run.sh/config.sh/env.sh with our OTel build while keeping the +# official image's k8s hooks, run-helper.sh, safe_sleep.sh, and entrypoint. +COPY --from=builder --chown=runner:runner /runner/_layout/. /home/runner/ + +# Bake the runner launch command into the image. ARC normally sets +# `command: [/home/runner/run.sh]` on the pod, but doing so in chart values +# triggers an "Outdated" hash-reconcile loop in gha-runner-scale-set 0.14.1. +# Baking it here lets the values stay command-free (loop-safe). +WORKDIR /home/runner +CMD ["/home/runner/run.sh"] diff --git a/releaseNote.md b/releaseNote.md index 6d348dadfc8..812398e6472 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -27,6 +27,7 @@ * Add new env var to allow single-prefix multiline logs on stdout by @nuclearpidgeon in https://github.com/actions/runner/pull/4424 * Bump Microsoft.DevTunnels.Connections from 1.3.39 to 1.3.48 by @dependabot[bot] in https://github.com/actions/runner/pull/4441 * Bump System.Formats.Asn1 and System.Security.Cryptography.Pkcs by @dependabot[bot] in https://github.com/actions/runner/pull/4369 +* Native OpenTelemetry export (opt-in) by @stefanpenner in https://github.com/actions/runner/pull/4366 — with `ACTIONS_RUNNER_OTLP_ENDPOINT` set, the runner emits OTLP/HTTP traces, metrics, and logs per job: job/step/child spans with deterministic (SHA-256) IDs, `cicd.*`/`vcs.*` semconv attributes (schema 1.34.0), annotation logs, and duration/error metrics. Best-effort and bounded: secret-masked output, capped buffers, gzip + chunked POSTs with a single bounded retry, 4 s job-end flush deadline, one Warning summary on export loss. Optional `ACTIONS_RUNNER_OTLP_PROPAGATE=true` injects `TRACEPARENT`/`OTEL_*` into step env (workflow-set values always win). Server kill switch: `actions_runner_otel_export`. See docs/adrs/4366-native-otel-export.md and docs/otel-id-contract.md. ## New Contributors * @GitPaulo made their first contribution in https://github.com/actions/runner/pull/4383 diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 8536e942a27..8621b91f3fd 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -180,6 +180,7 @@ public static class Features public static readonly string BatchActionResolution = "actions_batch_action_resolution"; public static readonly string UseBearerTokenForCodeload = "actions_use_bearer_token_for_codeload"; public static readonly string OverrideDebuggerWelcomeMessage = "actions_runner_override_debugger_welcome_message"; + public static readonly string RunnerOtelExport = "actions_runner_otel_export"; } // Node version migration related constants @@ -312,6 +313,20 @@ public static class Agent public static readonly string ActionArchiveCacheDirectory = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE"; public static readonly string SymlinkCachedActions = "ACTIONS_RUNNER_SYMLINK_CACHED_ACTIONS"; public static readonly string EmitCompositeMarkers = "ACTIONS_RUNNER_EMIT_COMPOSITE_MARKERS"; + + // Native OpenTelemetry trace export (see docs/adrs/4366-native-otel-export.md). + public static readonly string OtlpEndpoint = "ACTIONS_RUNNER_OTLP_ENDPOINT"; + public static readonly string OtlpHeaders = "ACTIONS_RUNNER_OTLP_HEADERS"; + // Path to a PEM CA bundle trusted for the collector connection (the safe + // primitive for self-signed collectors; mirrors OTEL_EXPORTER_OTLP_CERTIFICATE). + public static readonly string OtlpCertificate = "ACTIONS_RUNNER_OTLP_CERTIFICATE"; + public static readonly string OtlpInsecure = "ACTIONS_RUNNER_OTLP_INSECURE"; + public static readonly string OtlpPropagate = "ACTIONS_RUNNER_OTLP_PROPAGATE"; + // Inbound W3C trace context from an upstream scheduler (e.g. ARC): the job + // span gets a span LINK to this context (cross-trace causality), keeping the + // runner's deterministic IDs intact. + public static readonly string OtlpParentTraceparent = "ACTIONS_RUNNER_PARENT_TRACEPARENT"; + public static readonly string OtlpParentTracestate = "ACTIONS_RUNNER_PARENT_TRACESTATE"; } public static class System diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 49f9d3e739e..2323cd20517 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -228,11 +228,10 @@ public sealed class ActionManager : RunnerService, IActionManager { throw new Exception($"Missing download info for {lookupKey}"); } - Exception downloadFailure = null; try { - await DownloadRepositoryActionAsync(executionContext, downloadInfo); + await DownloadActionWithOtelSpan(executionContext, downloadInfo); } catch (Exception ex) { @@ -463,7 +462,7 @@ public sealed class ActionManager : RunnerService, IActionManager Exception downloadFailure = null; try { - await DownloadRepositoryActionAsync(executionContext, downloadInfo); + await DownloadActionWithOtelSpan(executionContext, downloadInfo); } catch (Exception ex) { @@ -1079,6 +1078,40 @@ private async Task ResolveNewActionsAsync(IExecutionContext executionContext, Li } } + private async Task DownloadActionWithOtelSpan(IExecutionContext executionContext, WebApi.ActionDownloadInfo downloadInfo) + { + var start = DateTime.UtcNow; + var result = "success"; + try + { + await DownloadRepositoryActionAsync(executionContext, downloadInfo); + } + catch + { + result = "failure"; + throw; + } + finally + { + HostContext.GetService().RecordSpan( + $"Resolve {downloadInfo.NameWithOwner}@{downloadInfo.Ref}", + "action_download", + start, + DateTime.UtcNow, + new Dictionary + { + ["github.action"] = downloadInfo.NameWithOwner, + ["github.action_ref"] = downloadInfo.Ref, + // Give action-resolution spans a result like every other task span. + ["cicd.pipeline.task.run.result"] = result, + }, + // Action resolution runs inside this step (e.g. "Set up job") — nest under it. + parentStepName: executionContext.StepDisplayName, + parentStepNumber: executionContext.StepOrder, + spanKind: 3); // CLIENT — a network download + } + } + private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, WebApi.ActionDownloadInfo downloadInfo) { Trace.Entering(); diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index c5cccb77ef0..8edd95cb96a 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -163,6 +163,27 @@ public async Task StopContainersAsync(IExecutionContext executionContext, object await RemoveContainerNetworkAsync(executionContext, containers.First().ContainerNetwork); } + // Sub-span for a container operation (pull/create), nested under the + // "Initialize containers" step so per-container image-pull/create latency is visible. + private void RecordContainerSpan(IExecutionContext executionContext, string name, DateTime start, int exitCode, string image) + { + HostContext.GetService().RecordSpan( + name, + "container", + start, + DateTime.UtcNow, + new Dictionary + { + ["container.image.name"] = image, + // semconv registers process.exit.code (dots) as an int attribute. + ["process.exit.code"] = (long)exitCode, + ["cicd.pipeline.task.run.result"] = exitCode == 0 ? "success" : "failure", + }, + parentStepName: executionContext.StepDisplayName, + parentStepNumber: executionContext.StepOrder, + spanKind: 3); // CLIENT — container registry/daemon ops + } + private async Task StartContainerAsync(IExecutionContext executionContext, ContainerInfo container) { Trace.Entering(); @@ -196,6 +217,7 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta var configLocation = await ContainerRegistryLogin(executionContext, container); // Pull down docker image with retry up to 3 times + var pullStart = DateTime.UtcNow; int retryCount = 0; int pullExitCode = 0; while (retryCount < 3) @@ -219,6 +241,7 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta // Remove credentials after pulling ContainerRegistryLogout(configLocation); + RecordContainerSpan(executionContext, $"pull image: {container.ContainerImage}", pullStart, pullExitCode, container.ContainerImage); if (retryCount == 3 && pullExitCode != 0) { @@ -230,11 +253,13 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta MountWellKnownDirectories(executionContext, container); } + var createStart = DateTime.UtcNow; container.ContainerId = await _dockerManager.DockerCreate(executionContext, container); ArgUtil.NotNullOrEmpty(container.ContainerId, nameof(container.ContainerId)); // Start container int startExitCode = await _dockerManager.DockerStart(executionContext, container.ContainerId); + RecordContainerSpan(executionContext, $"create container: {container.ContainerImage}", createStart, startExitCode, container.ContainerImage); if (startExitCode != 0) { throw new InvalidOperationException($"Docker start fail with exit code {startExitCode}"); diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 6d7698fdd9c..d44f0d1af65 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -44,6 +44,9 @@ public interface IExecutionContext : IRunnerService string ScopeName { get; } string SiblingScopeName { get; } string ContextName { get; } + // Timeline display name + order of this step's record (used to key its OTel step span). + string StepDisplayName { get; } + int StepOrder { get; } ActionRunStage Stage { get; } Task ForceCompleted { get; } TaskResult? Result { get; set; } @@ -107,6 +110,9 @@ public interface IExecutionContext : IRunnerService void SetRunnerContext(string name, string value); string GetGitHubContext(string name); void SetGitHubContext(string name, string value); + // W3C trace context + OTEL_* env to propagate into this step's process so + // in-job tools/actions emit spans parented to the step. Empty unless opted in. + IDictionary GetOTelStepEnv(); void SetOutput(string name, string value, out string reference); void SetTimeout(TimeSpan? timeout); @@ -201,6 +207,8 @@ private ExecutionContext(ExecutionContext parent, bool embedded) public string ScopeName { get; private set; } public string SiblingScopeName { get; private set; } public string ContextName { get; private set; } + public string StepDisplayName => _record.Name; + public int StepOrder => _record.Order ?? 0; public ActionRunStage Stage { get; private set; } public Task ForceCompleted => _forceCompleted.Task; public CancellationToken CancellationToken => _cancellationTokenSource.Token; @@ -593,6 +601,47 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = }); Global.StepsResult.Add(stepResult); + + var otel = HostContext.GetService(); + var firstError = _record.Issues?.FirstOrDefault(i => i.Type == IssueType.Error)?.Message; + otel.RecordStepCompletion( + stepName: _record.Name, + stepNumber: _record.Order, + startTime: _record.StartTime, + endTime: _record.FinishTime, + conclusion: _record.Result, + stepType: StepTelemetry?.Type, + actionName: StepTelemetry?.Action, + actionRef: StepTelemetry?.Ref, + isEmbedded: IsEmbedded, + errorMessage: firstError, + stepStage: StepTelemetry?.Stage); + + // Mirror this step's errors/warnings as OTel logs correlated to the step span. + if (!IsEmbedded) + { + _record.Issues?.ForEach(issue => + { + otel.RecordStepLog(_record.Name, _record.Order, issue.Type.ToString(), issue.Message); + }); + } + } + else if (_record.RecordType == ExecutionContextType.Job) + { + var jobOtel = HostContext.GetService(); + jobOtel.RecordJobCompletion( + startTime: _record.StartTime, + endTime: _record.FinishTime, + conclusion: _record.Result, + throttlingDelayMs: Interlocked.Read(ref _totalThrottlingDelayInMilliseconds), + errorMessage: _record.Issues?.FirstOrDefault(i => i.Type == IssueType.Error)?.Message); + + // Mirror job-level issues/annotations as OTel logs correlated to the job span + // (step issues are forwarded above; job-level ones were otherwise dropped). + _record.Issues?.ForEach(issue => + { + jobOtel.RecordJobLog(issue.Type.ToString(), issue.Message); + }); } if (Global.Variables.GetBoolean(Constants.Runner.Features.SendJobLevelAnnotations) ?? false) @@ -676,6 +725,23 @@ public void SetGitHubContext(string name, string value) githubContext[name] = new StringContextData(value); } + public IDictionary GetOTelStepEnv() + { + return HostContext.GetService().StepPropagationEnv(_record.Name, _record.Order); + } + + // PR number for pull_request events, parsed from github.ref (refs/pull/N/merge). + private string GetPullRequestNumber() + { + var eventName = GetGitHubContext("event_name"); + if (eventName == null || !eventName.StartsWith("pull_request", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + var match = System.Text.RegularExpressions.Regex.Match(GetGitHubContext("ref") ?? "", @"refs/pull/(\d+)/"); + return match.Success ? match.Groups[1].Value : null; + } + public string GetGitHubContext(string name) { ArgUtil.NotNullOrEmpty(name, nameof(name)); @@ -1036,6 +1102,26 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation } ExpressionValues["github"] = githubContext; + // Capture job-level identifiers once so native OTel job/step spans + // share consistent deterministic IDs and parent links. + HostContext.GetService().SetJobInfo( + runId: GetGitHubContext("run_id"), + runAttempt: GetGitHubContext("run_attempt"), + jobName: message.JobDisplayName, + jobKey: githubJob, + repository: GetGitHubContext("repository"), + workflow: GetGitHubContext("workflow"), + eventName: GetGitHubContext("event_name"), + serverUrl: GetGitHubContext("server_url"), + // Server-side kill switch; defaults on so the endpoint opt-in works + // where the flag isn't provisioned (self-hosted/GHES). + featureEnabled: Global.Variables.GetBoolean(Constants.Runner.Features.RunnerOtelExport) ?? true, + sha: GetGitHubContext("sha"), + refName: GetGitHubContext("head_ref") ?? GetGitHubContext("ref_name"), + actor: GetGitHubContext("actor"), + baseRef: GetGitHubContext("base_ref"), + changeId: GetPullRequestNumber()); + Trace.Info("Initialize Env context"); #if OS_WINDOWS ExpressionValues["env"] = new DictionaryContextData(); diff --git a/src/Runner.Worker/Handlers/ContainerActionHandler.cs b/src/Runner.Worker/Handlers/ContainerActionHandler.cs index 1f67c9dd360..76983a5f03e 100644 --- a/src/Runner.Worker/Handlers/ContainerActionHandler.cs +++ b/src/Runner.Worker/Handlers/ContainerActionHandler.cs @@ -247,6 +247,8 @@ public async Task RunAsync(ActionRunStage stage) } } + AddOTelPropagationToEnvironment(); + foreach (var variable in this.Environment) { container.ContainerEnvironmentVariables[variable.Key] = container.TranslateToContainerPath(variable.Value); diff --git a/src/Runner.Worker/Handlers/Handler.cs b/src/Runner.Worker/Handlers/Handler.cs index 5d77a051768..35fb1c0378c 100644 --- a/src/Runner.Worker/Handlers/Handler.cs +++ b/src/Runner.Worker/Handlers/Handler.cs @@ -202,6 +202,25 @@ protected void AddEnvironmentVariable(string key, string value) #endif } + protected void AddOTelPropagationToEnvironment() + { + // Propagate W3C trace context + OTEL_* so OTel-instrumented tools in this + // step emit spans/logs parented to the step span (opt-in, no-op otherwise). + // Workflow-set env always wins: a step that configures its own TRACEPARENT + // or OTEL_* (e.g. for its application's telemetry) is never clobbered. + var otelStepEnv = ExecutionContext.GetOTelStepEnv(); + if (otelStepEnv != null) + { + foreach (var kv in otelStepEnv) + { + if (!Environment.ContainsKey(kv.Key)) + { + Environment[kv.Key] = kv.Value; + } + } + } + } + protected void AddPrependPathToEnvironment() { // Validate args. diff --git a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs index 29def039f99..a52663999ca 100644 --- a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs +++ b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs @@ -86,6 +86,8 @@ public async Task RunAsync(ActionRunStage stage) } } + AddOTelPropagationToEnvironment(); + // Resolve the target script. string target = null; if (stage == ActionRunStage.Main) diff --git a/src/Runner.Worker/Handlers/ScriptHandler.cs b/src/Runner.Worker/Handlers/ScriptHandler.cs index b898f051ded..038efda85e7 100644 --- a/src/Runner.Worker/Handlers/ScriptHandler.cs +++ b/src/Runner.Worker/Handlers/ScriptHandler.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; @@ -326,6 +327,8 @@ public async Task RunAsync(ActionRunStage stage) } } + AddOTelPropagationToEnvironment(); + ExecutionContext.Debug($"{fileName} {arguments}"); Inputs.TryGetValue("standardInInput", out var standardInInput); @@ -336,7 +339,11 @@ public async Task RunAsync(ActionRunStage stage) StepHost.ErrorDataReceived += stderrManager.OnDataReceived; // Execute - int exitCode = await StepHost.ExecuteAsync(ExecutionContext, + var processStart = DateTime.UtcNow; + int exitCode = -1; + try + { + exitCode = await StepHost.ExecuteAsync(ExecutionContext, workingDirectory: StepHost.ResolvePathForStepHost(ExecutionContext, workingDirectory), fileName: fileName, arguments: arguments, @@ -347,6 +354,26 @@ public async Task RunAsync(ActionRunStage stage) inheritConsoleHandler: !ExecutionContext.Global.Variables.Retain_Default_Encoding, standardInInput: standardInInput, cancellationToken: ExecutionContext.CancellationToken); + } + finally + { + // Sub-span for the actual command process, nested under the step span — + // splits real command time from runner overhead within the step. + HostContext.GetService().RecordSpan( + $"process: {Path.GetFileName(fileName)}", + "process", + processStart, + DateTime.UtcNow, + new Dictionary + { + ["process.executable.name"] = Path.GetFileName(fileName), + // semconv registers process.exit.code (dots) as an int attribute. + ["process.exit.code"] = (long)exitCode, + ["cicd.pipeline.task.run.result"] = exitCode == 0 ? "success" : "failure", + }, + parentStepName: ExecutionContext.StepDisplayName, + parentStepNumber: ExecutionContext.StepOrder); + } // Error if (exitCode != 0) diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 3c4799a29e0..fd43c2ae01a 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -45,6 +45,18 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat DateTime jobStartTimeUtc = DateTime.UtcNow; _runnerSettings = HostContext.GetService().GetSettings(); + + // Describe the runner itself as the OTLP Resource for native OTel export. + HostContext.GetService().SetResource( + runnerName: _runnerSettings.AgentName, + runnerId: _runnerSettings.AgentId.ToString(), + runnerGroup: _runnerSettings.PoolName, + runnerVersion: BuildConstants.RunnerPackage.Version, + osType: VarUtil.OS, + arch: VarUtil.OSArchitecture, + machineName: Environment.MachineName, + ephemeral: _runnerSettings.Ephemeral); + IRunnerService server = null; // add orchestration id to useragent for better correlation. @@ -322,6 +334,13 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution telemetry = jobContext.Global.JobTelemetry.Select(x => new Telemetry { Type = x.Type.ToString(), Message = x.Message, }).ToList(); } + // Flush OTel spans BEFORE reporting job completion — deliberately. + // Completion is the signal that lets ephemeral runners (e.g. ARC) tear + // this worker down; flushing after it races pod deletion and drops the + // job's telemetry. Cost is bounded: no-op when OTel is off, and a hard + // 4 s overall flush deadline when the collector is down (see ADR 4366). + await HostContext.GetService().FlushAsync(default); + Trace.Info($"Raising job completed against run service"); var completeJobRetryLimit = 5; var exceptions = new List(); @@ -403,6 +422,10 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution // Make sure we don't submit secrets as telemetry MaskTelemetrySecrets(jobContext.Global.JobTelemetry); + // Flush OTel spans BEFORE reporting job completion — deliberately; see + // the comment on the run-service path above (ephemeral teardown race). + await HostContext.GetService().FlushAsync(default); + Trace.Info($"Raising job completed event"); var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, jobContext.JobOutputs, jobContext.ActionsEnvironment, jobContext.Global.StepsTelemetry, jobContext.Global.JobTelemetry); diff --git a/src/Runner.Worker/OTelHttpTransport.cs b/src/Runner.Worker/OTelHttpTransport.cs new file mode 100644 index 00000000000..90ab58746e4 --- /dev/null +++ b/src/Runner.Worker/OTelHttpTransport.cs @@ -0,0 +1,191 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace GitHub.Runner.Worker +{ + /// + /// OTLP/HTTP transport for the native OTel exporter: gzip + one bounded retry on + /// transient failures + OTLP partial-success detection. Separated from the recorder + /// and constructor-injectable (HttpMessageHandler) so the wire path is unit-testable + /// in isolation. Best-effort: never throws; logs via the supplied callback. + /// + internal sealed class OTelHttpTransport : IDisposable + { + private readonly HttpClient _httpClient; + private readonly Action _log; + + // Test seam: lets retry-timing tests observe delays instead of sleeping for real. + internal Func DelayAsync = Task.Delay; + + public OTelHttpTransport(HttpMessageHandler handler, IReadOnlyList> headers = null, Action log = null, string userAgent = null) + { + _httpClient = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(5) }; + _log = log ?? (_ => { }); + if (!string.IsNullOrEmpty(userAgent)) + { + // OTLP exporter spec SHOULD: identify the exporter and its version so + // collector operators can attribute and police runner-fleet traffic. + _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", userAgent); + } + if (headers != null) + { + foreach (var h in headers) + { + _httpClient.DefaultRequestHeaders.TryAddWithoutValidation(h.Key, h.Value); + } + } + } + + // gzip the OTLP/JSON body. A job's spans/logs are highly repetitive, so this + // typically shrinks the payload ~10x — less bandwidth and faster posts. + // Always-on with no opt-out: the OTLP spec requires every server to accept it + // ("All server components MUST support ... Gzip compression"). + internal static byte[] Gzip(byte[] raw) + { + using var ms = new MemoryStream(); + using (var gz = new GZipStream(ms, CompressionLevel.Fastest, leaveOpen: true)) + { + gz.Write(raw, 0, raw.Length); + } + return ms.ToArray(); + } + + internal static byte[] GzipUtf8(string s) => Gzip(Encoding.UTF8.GetBytes(s)); + + // Exactly the OTLP/HTTP failures table: 429, 502, 503, 504 SHOULD be retried; + // "All other 4xx or 5xx response status codes MUST NOT be retried" (so no 408/500). + private static bool IsTransientStatus(int status) => + status == 429 || status == 502 || status == 503 || status == 504; + + // Longest Retry-After the transport will honor: flush runs under a hard 4 s + // overall deadline, so a server asking to wait longer than this can't be + // satisfied in time — the retry is skipped instead of thrown at a collector + // that just said it's overloaded. + internal static readonly TimeSpan MaxRetryAfter = TimeSpan.FromSeconds(2); + + // 100-400 ms random delay so a fleet of runners flushing at once doesn't + // retry in lockstep against the same collector. + private static TimeSpan Jitter() => TimeSpan.FromMilliseconds(Random.Shared.Next(100, 401)); + + // Delay before the single retry. Honors Retry-After (OTLP spec SHOULD) when the + // server sent one; without it, falls back to a short jittered delay — under the + // deliberate one-retry bound (see the ADR transport section) the spec's + // exponential-backoff SHOULD degenerates to this single jittered wait. Returns + // false when Retry-After is longer than the flush deadline could honor: that + // retry would be wasted, so it is skipped entirely. + internal static bool TryGetRetryDelay(HttpResponseMessage response, out TimeSpan delay) + { + var retryAfter = response?.Headers?.RetryAfter; + var requested = retryAfter?.Delta; + if (requested == null && retryAfter?.Date != null) + { + requested = retryAfter.Date.Value - DateTimeOffset.UtcNow; + } + if (requested == null) + { + delay = Jitter(); + return true; + } + if (requested.Value > MaxRetryAfter) + { + delay = default; + return false; + } + delay = requested.Value > TimeSpan.Zero ? requested.Value : TimeSpan.Zero; + return true; + } + + // OTLP partial success: a 200 response may carry + // {"partialSuccess":{"rejectedSpans":"N",...,"errorMessage":"..."}}. Returns a + // short description when items were rejected, else null. Best-effort; never throws. + internal static string DescribePartialSuccess(string responseBody) + { + try + { + if (string.IsNullOrEmpty(responseBody) || !responseBody.Contains("partialSuccess")) return null; + using var doc = JsonDocument.Parse(responseBody); + if (!doc.RootElement.TryGetProperty("partialSuccess", out var ps)) return null; + long rejected = 0; + foreach (var key in new[] { "rejectedSpans", "rejectedDataPoints", "rejectedLogRecords" }) + { + if (ps.TryGetProperty(key, out var v)) + { + if (v.ValueKind == JsonValueKind.Number) rejected += v.GetInt64(); + else if (v.ValueKind == JsonValueKind.String && long.TryParse(v.GetString(), out var n)) rejected += n; + } + } + var msg = ps.TryGetProperty("errorMessage", out var em) ? em.GetString() : null; + if (rejected > 0 || !string.IsNullOrEmpty(msg)) return $"{rejected} rejected: {msg}"; + return null; + } + catch { return null; } + } + + private static async Task ReadPartialSuccessAsync(HttpResponseMessage response, CancellationToken ct) + { + try { return DescribePartialSuccess(await response.Content.ReadAsStringAsync(ct)); } + catch { return null; } + } + + // POST one OTLP/JSON payload (UTF-8 bytes), gzipped, with a single transient retry + // bounded by the caller's deadline. Returns true if delivered (incl. partial + // success), false otherwise. + public async Task PostAsync(string url, byte[] utf8Json, string what, CancellationToken cancellationToken) + { + try + { + var body = Gzip(utf8Json); + for (var attempt = 1; attempt <= 2; attempt++) + { + try + { + using var content = new ByteArrayContent(body); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + content.Headers.ContentEncoding.Add("gzip"); + using var response = await _httpClient.PostAsync(url, content, cancellationToken); + if (response.IsSuccessStatusCode) + { + var partial = await ReadPartialSuccessAsync(response, cancellationToken); + _log(string.IsNullOrEmpty(partial) + ? $"Exported {what} to {url} ({body.Length} B gzip)" + : $"OTel export partially rejected (best-effort, ignored): {partial}"); + return true; + } + if (attempt < 2 && IsTransientStatus((int)response.StatusCode) && !cancellationToken.IsCancellationRequested) + { + if (TryGetRetryDelay(response, out var delay)) + { + await DelayAsync(delay, cancellationToken); + continue; + } + _log($"OTel export throttled, Retry-After exceeds flush deadline (best-effort, ignored): HTTP {(int)response.StatusCode}"); + return false; + } + _log($"OTel export rejected by collector (best-effort, ignored): HTTP {(int)response.StatusCode}"); + return false; + } + catch (Exception ex) when (attempt < 2 && ex is not OperationCanceledException && !cancellationToken.IsCancellationRequested) + { + _log($"OTel export attempt {attempt} failed, retrying: {ex.Message}"); + await DelayAsync(Jitter(), cancellationToken); + } + } + } + catch (Exception ex) + { + _log($"OTel export failed (best-effort, ignored): {ex.Message}"); + } + return false; + } + + public void Dispose() => _httpClient?.Dispose(); + } +} diff --git a/src/Runner.Worker/OTelTraceExporter.Serialization.cs b/src/Runner.Worker/OTelTraceExporter.Serialization.cs new file mode 100644 index 00000000000..ec7c2de1651 --- /dev/null +++ b/src/Runner.Worker/OTelTraceExporter.Serialization.cs @@ -0,0 +1,483 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; +using System.Text.Json; + +namespace GitHub.Runner.Worker +{ + public sealed partial class OTelTraceExporter + { + // OTLP/JSON, serialized with Utf8JsonWriter for correct escaping (a control + // character in a step name must not be able to invalidate the whole batch). + // Builders return the UTF-8 bytes directly — no bytes -> string -> bytes + // round-trip — so flush never triple-materializes a large payload. + private byte[] BuildOTLPSpansJson(List spans, List> resource) + { + using var stream = new MemoryStream(); + using (var w = new Utf8JsonWriter(stream, s_jsonOptions)) + { + w.WriteStartObject(); + w.WriteStartArray("resourceSpans"); + w.WriteStartObject(); + + w.WriteStartObject("resource"); + w.WriteStartArray("attributes"); + WriteAttributes(w, resource); + w.WriteEndArray(); + w.WriteEndObject(); + + w.WriteStartArray("scopeSpans"); + w.WriteStartObject(); + w.WriteStartObject("scope"); + w.WriteString("name", "github.actions.runner"); + if (!string.IsNullOrEmpty(_serviceVersion)) w.WriteString("version", _serviceVersion); + w.WriteEndObject(); + + w.WriteStartArray("spans"); + foreach (var s in spans) + { + w.WriteStartObject(); + w.WriteString("traceId", s.TraceId); + w.WriteString("spanId", s.SpanId); + if (!string.IsNullOrEmpty(s.ParentSpanId)) + { + w.WriteString("parentSpanId", s.ParentSpanId); + } + w.WriteString("name", Mask(s.Name) ?? ""); + w.WriteNumber("kind", s.Kind); + w.WriteString("startTimeUnixNano", s.StartTimeUnixNano.ToString(CultureInfo.InvariantCulture)); + w.WriteString("endTimeUnixNano", s.EndTimeUnixNano.ToString(CultureInfo.InvariantCulture)); + w.WriteStartArray("attributes"); + WriteAttributes(w, s.Attributes); + w.WriteEndArray(); + if (s.Links.Count > 0) + { + w.WriteStartArray("links"); + foreach (var lk in s.Links) + { + w.WriteStartObject(); + w.WriteString("traceId", lk.TraceId); + w.WriteString("spanId", lk.SpanId); + if (!string.IsNullOrEmpty(lk.TraceState)) + { + w.WriteString("traceState", lk.TraceState); + } + if (lk.Flags != 0) + { + w.WriteNumber("flags", lk.Flags); + } + w.WriteStartArray("attributes"); + WriteAttributes(w, lk.Attributes); + w.WriteEndArray(); + w.WriteEndObject(); + } + w.WriteEndArray(); + } + if (s.Events.Count > 0) + { + w.WriteStartArray("events"); + foreach (var ev in s.Events) + { + w.WriteStartObject(); + w.WriteString("name", ev.Name); + w.WriteString("timeUnixNano", ev.TimeUnixNano.ToString(CultureInfo.InvariantCulture)); + w.WriteStartArray("attributes"); + WriteAttributes(w, ev.Attributes); + w.WriteEndArray(); + w.WriteEndObject(); + } + w.WriteEndArray(); + } + w.WriteStartObject("status"); + if (s.StatusCode != 0) + { + w.WriteNumber("code", s.StatusCode); + if (!string.IsNullOrEmpty(s.StatusMessage)) + { + w.WriteString("message", Mask(s.StatusMessage)); + } + } + w.WriteEndObject(); + w.WriteEndObject(); + } + w.WriteEndArray(); // spans + w.WriteString("schemaUrl", SchemaUrl); + w.WriteEndObject(); // scopeSpans[0] + w.WriteEndArray(); // scopeSpans + w.WriteString("schemaUrl", SchemaUrl); + w.WriteEndObject(); // resourceSpans[0] + w.WriteEndArray(); // resourceSpans + w.WriteEndObject(); + } + return stream.ToArray(); + } + + private void WriteAttributes(Utf8JsonWriter w, List> attrs) + { + foreach (var kv in attrs) + { + w.WriteStartObject(); + w.WriteString("key", kv.Key); + WriteAnyValue(w, "value", kv.Value); + w.WriteEndObject(); + } + } + + // OTLP AnyValue: each CLR type must land on its own wire type — silently + // stringifying a double (or an array) gives downstream numeric queries the + // wrong type with no error anywhere. Strings are masked; genuinely + // unsupported types fall back to their masked string form (documented, + // not silent: this is the only stringify arm). + private void WriteAnyValue(Utf8JsonWriter w, string propertyName, object value) + { + w.WriteStartObject(propertyName); + switch (value) + { + case string s: // before IEnumerable — string is IEnumerable + w.WriteString("stringValue", Mask(s) ?? ""); + break; + case byte[] bytes: + w.WriteString("bytesValue", Convert.ToBase64String(bytes)); + break; + case System.Collections.IEnumerable items: + w.WriteStartObject("arrayValue"); + w.WriteStartArray("values"); + foreach (var item in items) + { + w.WriteStartObject(); + WriteScalarAnyValueBody(w, item); + w.WriteEndObject(); + } + w.WriteEndArray(); + w.WriteEndObject(); + break; + default: + WriteScalarAnyValueBody(w, value); + break; + } + w.WriteEndObject(); + } + + // Scalar AnyValue fields, written into the current object (also used for + // arrayValue elements, which have no wrapper key). Nested arrays are + // stringified — one level is all the attribute API promises. + private void WriteScalarAnyValueBody(Utf8JsonWriter w, object value) + { + switch (value) + { + case bool b: + w.WriteBoolean("boolValue", b); + break; + case long l: + w.WriteString("intValue", l.ToString(CultureInfo.InvariantCulture)); + break; + case int n: + w.WriteString("intValue", n.ToString(CultureInfo.InvariantCulture)); + break; + case double d: + w.WriteNumber("doubleValue", d); + break; + case float f: + w.WriteNumber("doubleValue", f); + break; + default: + w.WriteString("stringValue", Mask(value?.ToString()) ?? ""); + break; + } + } + + private byte[] BuildOTLPLogsJson(List logs, List> resource) + { + using var stream = new MemoryStream(); + using (var w = new Utf8JsonWriter(stream, s_jsonOptions)) + { + w.WriteStartObject(); + w.WriteStartArray("resourceLogs"); + w.WriteStartObject(); + w.WriteStartObject("resource"); + w.WriteStartArray("attributes"); + WriteAttributes(w, resource); + w.WriteEndArray(); + w.WriteEndObject(); + w.WriteStartArray("scopeLogs"); + w.WriteStartObject(); + w.WriteStartObject("scope"); + w.WriteString("name", "github.actions.runner"); + if (!string.IsNullOrEmpty(_serviceVersion)) w.WriteString("version", _serviceVersion); + w.WriteEndObject(); + w.WriteStartArray("logRecords"); + foreach (var l in logs) + { + w.WriteStartObject(); + w.WriteString("timeUnixNano", l.TimeUnixNano.ToString(CultureInfo.InvariantCulture)); + w.WriteString("observedTimeUnixNano", l.TimeUnixNano.ToString(CultureInfo.InvariantCulture)); + w.WriteNumber("severityNumber", l.SeverityNumber); + w.WriteString("severityText", l.SeverityText); + w.WriteStartObject("body"); + w.WriteString("stringValue", Mask(l.Body) ?? ""); + w.WriteEndObject(); + w.WriteString("traceId", l.TraceId); + w.WriteString("spanId", l.SpanId); + w.WriteStartArray("attributes"); + WriteAttributes(w, l.Attributes); + w.WriteEndArray(); + w.WriteEndObject(); + } + w.WriteEndArray(); + w.WriteString("schemaUrl", SchemaUrl); + w.WriteEndObject(); + w.WriteEndArray(); + w.WriteString("schemaUrl", SchemaUrl); + w.WriteEndObject(); + w.WriteEndArray(); + w.WriteEndObject(); + } + return stream.ToArray(); + } + + private sealed class OTelLog + { + public long TimeUnixNano; + public int SeverityNumber; + public string SeverityText; + public string Body; + public string TraceId; + public string SpanId; + public readonly List> Attributes = new(); + } + + private sealed class JobMetrics + { + public long StartNano; + public long EndNano; + public double DurationSeconds; + public string PipelineName; + public string Attempt; + public string Result; + public int Errors; + } + + // Per-task (job or step) duration observation -> cicd.pipeline.task.duration. + private sealed class TaskMetric + { + public string Name; + public string Type; // "job" | "step" + public string Result; // semconv result (success|failure|skip|...) + public string Attempt; + public double DurationSeconds; + public long StartNano; + public long EndNano; + } + + // Explicit duration buckets (seconds) so the collector can aggregate real + // percentiles across runs, instead of an empty-bounds (single implicit bucket) + // histogram. Mirrors what the collector's spanmetrics connector would emit. + private static readonly double[] s_durationBucketsSeconds = { 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1800 }; + + // Write a histogram dataPoint body for ONE duration observation: places it in the + // right explicit bucket so cumulative aggregation in the collector is meaningful. + private static void WriteSingleObservationHistogram(Utf8JsonWriter w, double seconds, long startNano, long endNano) + { + var bounds = s_durationBucketsSeconds; + var bucket = bounds.Length; // overflow bucket unless an upper bound matches + for (var i = 0; i < bounds.Length; i++) + { + if (seconds <= bounds[i]) { bucket = i; break; } + } + w.WriteString("startTimeUnixNano", startNano.ToString(CultureInfo.InvariantCulture)); + w.WriteString("timeUnixNano", endNano.ToString(CultureInfo.InvariantCulture)); + w.WriteString("count", "1"); + w.WriteNumber("sum", seconds); + w.WriteStartArray("bucketCounts"); + for (var i = 0; i <= bounds.Length; i++) { w.WriteStringValue(i == bucket ? "1" : "0"); } + w.WriteEndArray(); + w.WriteStartArray("explicitBounds"); + foreach (var b in bounds) { w.WriteNumberValue(b); } + w.WriteEndArray(); + w.WriteNumber("min", seconds); + w.WriteNumber("max", seconds); + } + + // OTLP/JSON metrics: cicd.pipeline.run.errors (semconv counter, failures only) + + // github.pipeline.job.duration + github.pipeline.task.duration (histograms). + // Naming: semconv names are adopted where the semantics match — a failed job IS + // "an error encountered in a pipeline run" (cicd.pipeline.run.errors). The duration + // metrics keep the vendor (github.*) namespace because the worker observes only its + // own JOB: semconv cicd.pipeline.run.duration means the whole run's duration grouped + // by cicd.pipeline.run.state (queued/executing/finalizing), which only the control + // plane can emit correctly (a 10-job run would otherwise contribute 10 bogus "run + // duration" points), and semconv defines no task-level duration metric. + // CUMULATIVE temporality is valid: each per-run process exports one point with its own + // startTimeUnixNano; the collector aggregates across runs. + private byte[] BuildOTLPMetricsJson(JobMetrics m, List tasks, List> resource) + { + using var stream = new MemoryStream(); + using (var w = new Utf8JsonWriter(stream, s_jsonOptions)) + { + w.WriteStartObject(); + w.WriteStartArray("resourceMetrics"); + w.WriteStartObject(); + w.WriteStartObject("resource"); + w.WriteStartArray("attributes"); + WriteAttributes(w, resource); + w.WriteEndArray(); + w.WriteEndObject(); + w.WriteStartArray("scopeMetrics"); + w.WriteStartObject(); + w.WriteStartObject("scope"); + w.WriteString("name", "github.actions.runner"); + if (!string.IsNullOrEmpty(_serviceVersion)) w.WriteString("version", _serviceVersion); + w.WriteEndObject(); + w.WriteStartArray("metrics"); + + if (m != null) + { + // github.pipeline.job.duration — histogram with a single observation. + w.WriteStartObject(); + w.WriteString("name", "github.pipeline.job.duration"); + w.WriteString("unit", "s"); + w.WriteStartObject("histogram"); + w.WriteNumber("aggregationTemporality", 2); // CUMULATIVE + w.WriteStartArray("dataPoints"); + w.WriteStartObject(); + w.WriteStartArray("attributes"); + WriteAttributes(w, new List> + { + new("cicd.pipeline.name", m.PipelineName), + new("github.run_attempt", m.Attempt), + new("cicd.pipeline.result", m.Result), + }); + w.WriteEndArray(); + WriteSingleObservationHistogram(w, m.DurationSeconds, m.StartNano, m.EndNano); + w.WriteEndObject(); + w.WriteEndArray(); + w.WriteEndObject(); // histogram + w.WriteEndObject(); // metric + + if (m.Errors > 0) + { + w.WriteStartObject(); + w.WriteString("name", "cicd.pipeline.run.errors"); + w.WriteString("unit", "{error}"); + w.WriteStartObject("sum"); + w.WriteNumber("aggregationTemporality", 2); + w.WriteBoolean("isMonotonic", true); + w.WriteStartArray("dataPoints"); + w.WriteStartObject(); + w.WriteStartArray("attributes"); + WriteAttributes(w, new List> + { + new("cicd.pipeline.name", m.PipelineName), + new("github.run_attempt", m.Attempt), + new("error.type", "failure"), + }); + w.WriteEndArray(); + w.WriteString("startTimeUnixNano", m.StartNano.ToString(CultureInfo.InvariantCulture)); + w.WriteString("timeUnixNano", m.EndNano.ToString(CultureInfo.InvariantCulture)); + w.WriteString("asInt", m.Errors.ToString(CultureInfo.InvariantCulture)); + w.WriteEndObject(); + w.WriteEndArray(); + w.WriteEndObject(); + w.WriteEndObject(); + } + } + + // github.pipeline.task.duration — histogram with one observation per job/step. + if (tasks != null && tasks.Count > 0) + { + w.WriteStartObject(); + w.WriteString("name", "github.pipeline.task.duration"); + w.WriteString("unit", "s"); + w.WriteStartObject("histogram"); + w.WriteNumber("aggregationTemporality", 2); // CUMULATIVE + w.WriteStartArray("dataPoints"); + foreach (var task in tasks) + { + w.WriteStartObject(); + w.WriteStartArray("attributes"); + WriteAttributes(w, new List> + { + new("cicd.pipeline.task.name", task.Name), + new("cicd.pipeline.task.run.result", task.Result), + new("github.run_attempt", task.Attempt), + new("github.record_type", task.Type), + }); + w.WriteEndArray(); + WriteSingleObservationHistogram(w, task.DurationSeconds, task.StartNano, task.EndNano); + w.WriteEndObject(); + } + w.WriteEndArray(); + w.WriteEndObject(); // histogram + w.WriteEndObject(); // metric + } + + w.WriteEndArray(); // metrics + w.WriteString("schemaUrl", SchemaUrl); + w.WriteEndObject(); // scopeMetrics[0] + w.WriteEndArray(); + w.WriteString("schemaUrl", SchemaUrl); + w.WriteEndObject(); // resourceMetrics[0] + w.WriteEndArray(); + w.WriteEndObject(); + } + return stream.ToArray(); + } + + private sealed class OTelEvent + { + public string Name; + public long TimeUnixNano; + public readonly List> Attributes = new(); + } + + // A span link expresses cross-trace causality (W3C). Used for the inbound + // scheduler context (e.g. ARC) so the job links to its "schedule" span + // without re-rooting the runner's deterministic trace. + private sealed class OTelLink + { + public string TraceId; + public string SpanId; + public string TraceState; + public int Flags; + public readonly List> Attributes = new(); + } + + private sealed class OTelSpan + { + public string TraceId; + public string SpanId; + public string ParentSpanId; + public string Name; + public int Kind = 1; + public long StartTimeUnixNano; + public long EndTimeUnixNano; + public int StatusCode; // 0 unset, 1 ok, 2 error + public string StatusMessage; // error description; only written with an error code + public readonly List Links = new(); + public readonly List> Attributes = new(); + public readonly List Events = new(); + + public void Set(string key, object value) + { + Attributes.Add(new KeyValuePair(key, value)); + } + + // semconv exception event, message-only: a CI conclusion ("failure") is not + // an exception class, so exception.type is omitted rather than faked — that + // would bucket every failed step into one meaningless error group. Semconv + // requires at least one of type/message, so no message means no event. + public void AddException(string message, long timeNano) + { + if (string.IsNullOrEmpty(message)) + { + return; + } + var ev = new OTelEvent { Name = "exception", TimeUnixNano = timeNano }; + ev.Attributes.Add(new("exception.message", message)); + Events.Add(ev); + } + } + } +} diff --git a/src/Runner.Worker/OTelTraceExporter.cs b/src/Runner.Worker/OTelTraceExporter.cs new file mode 100644 index 00000000000..c12f4f713ba --- /dev/null +++ b/src/Runner.Worker/OTelTraceExporter.cs @@ -0,0 +1,1204 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Sdk; + +namespace GitHub.Runner.Worker +{ + [ServiceLocator(Default = typeof(OTelTraceExporter))] + public interface IOTelTraceExporter : IRunnerService + { + void SetResource(string runnerName, string runnerId, string runnerGroup, string runnerVersion, string osType, string arch, string machineName, bool ephemeral); + void SetJobInfo(string runId, string runAttempt, string jobName, string jobKey, string repository, string workflow, string eventName, string serverUrl, bool featureEnabled = true, string sha = null, string refName = null, string actor = null, string baseRef = null, string changeId = null); + void RecordStepCompletion(string stepName, int? stepNumber, DateTime? startTime, DateTime? endTime, TaskResult? conclusion, string stepType, string actionName, string actionRef, bool isEmbedded = false, string errorMessage = null, string stepStage = null); + // OTel log record correlated to the JOB span (job-level issues/annotations). + void RecordJobLog(string severityText, string message); + void RecordJobCompletion(DateTime? startTime, DateTime? endTime, TaskResult? conclusion, long throttlingDelayMs = 0, string errorMessage = null); + // Generic child span for finer-grained timing, e.g. action download. Parents to the + // given step (when the operation runs inside one, e.g. "Set up job"), else the job. + // Attribute values keep their type on the wire: bool/int/long serialize as + // boolValue/intValue (semconv int attributes like process.exit.code), else stringValue. + void RecordSpan(string name, string spanType, DateTime startTime, DateTime endTime, IDictionary attributes = null, string parentStepName = null, int parentStepNumber = 0, int spanKind = 1); + // OTel log record correlated to a step span (step issues/annotations). + void RecordStepLog(string stepName, int? stepNumber, string severityText, string message); + // W3C trace context + OTEL_* to inject into a step's env so in-job tools nest under the step span. + IDictionary StepPropagationEnv(string stepName, int? stepNumber); + Task FlushAsync(CancellationToken cancellationToken = default); + } + + /// + /// Emits native OpenTelemetry trace spans for a job and its steps, plus runner + /// identity as an OTLP Resource. Spans use deterministic IDs derived only from + /// identifiers the runner always has (run id, run attempt, job display name, + /// step name) so they merge with a GitHub Actions trace reconstruction. + /// + /// Opt-in: set ACTIONS_RUNNER_OTLP_ENDPOINT to an OTLP/HTTP base URL; spans are + /// POSTed to {endpoint}/v1/traces. Export is best-effort — failures are logged + /// and swallowed, never affecting job execution. + /// + /// The job span is the trace root; steps are its children. The deterministic-ID + /// contract (SHA-256 of run/job/step identifiers, truncated to 16/8 bytes) is + /// specified normatively in docs/otel-id-contract.md so consumers can reconstruct + /// and de-dupe the same trace; otel-explorer is one example consumer. + /// + public sealed partial class OTelTraceExporter : RunnerService, IOTelTraceExporter + { + private static readonly JsonWriterOptions s_jsonOptions = new() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; + // Declares which semantic-convention version this telemetry conforms to. + // Attribute set audited against the v1.34.0 registry: it is the earliest + // release that defines every emitted key (vcs.provider.name, vcs.owner.name, + // vcs.repository.name, cicd.worker.id/name, cicd.system.component, + // cicd.pipeline.run.url.full, cicd.pipeline.task.run.result, + // cicd.pipeline.result, process.exit.code, ...). Bump only after re-auditing. + private const string SchemaUrl = "https://opentelemetry.io/schemas/1.34.0"; + private string _serviceVersion = ""; + + private readonly object _lock = new(); + private readonly List _pendingSpans = new(); + private readonly List _pendingLogs = new(); + // Hard caps so a pathological job (e.g. unbounded ::warning:: annotations) can't + // grow these buffers without limit and OOM the worker — the export must never + // affect the job. ~4 KB/span retained until flush (see docs/otel-benchmarks.md); + // 10k keeps worst-case retention bounded to tens of MB. Excess is dropped + counted. + internal const int MaxBufferedSpans = 10000; + internal const int MaxBufferedLogs = 10000; + internal const int MaxBufferedTaskMetrics = 10000; + // Max spans/logs per OTLP POST. At ~4 KB/span (docs/otel-benchmarks.md) each + // request stays ~4 MB uncompressed — well under the OTel Collector's 20 MiB + // default decompressed-body limit — so one 413 can never drop a whole signal, + // and per-POST serialization memory stays bounded. Mirrors OTel SDK batch + // sizing (BatchSpanProcessor exports 512-item batches; we trade a few larger + // requests for staying inside the single 4 s flush deadline). + internal const int MaxItemsPerPost = 1000; + private long _droppedSpans; + private long _droppedLogs; + private long _droppedTaskMetrics; + private string _endpoint; + private string _baseUrl; + // _baseUrl with any URL userinfo (user:password@) removed — the only form + // that may be handed to step processes or written to logs. + private string _propagationBaseUrl; + private string _tracesUrl; + private string _logsUrl; + private string _metricsUrl; + private string _rawHeaders; + private JobMetrics _jobMetrics; + private readonly List _taskMetrics = new(); + private bool _enabled; + // Inject W3C TRACEPARENT + OTEL_* into each step's env so in-job tools/actions + // emit spans parented to the step. Opt-in (ACTIONS_RUNNER_OTLP_PROPAGATE). + private bool _propagate; + // Server-side kill switch, captured at job init. Defaults to true so the + // operator's endpoint opt-in works on self-hosted/GHES where the flag isn't + // provisioned; GitHub can send false to disable export fleet-wide. + private bool _featureEnabled = true; + private OTelHttpTransport _transport; + private JobInfo _jobInfo; + // Last flush's loss summary (null when everything was delivered); also + // logged at Warning so fleet operators see export failure without diffing + // per-signal Info lines. + private string _lastFlushSummary; + private List> _resource = DefaultResource(); + + private sealed class JobInfo + { + public long RunId; + public long RunAttempt; + public string RunIdRaw; + public string RunAttemptRaw; + public string JobName; // job display name, matches GitHub API job.name + public string JobKey; // github.job context key (e.g. "build") + public string Repository; + public string Workflow; + public string EventName; + public string ServerUrl; + public string Sha; + public string RefName; // branch/tag (vcs.ref.head.name) + public string Actor; + public string BaseRef; // base branch for PRs (vcs.ref.base.name) + public string ChangeId; // PR number (vcs.change.id) + } + + public override void Initialize(IHostContext hostContext) + { + base.Initialize(hostContext); + // Telemetry must NEVER throw into the job path. The first GetService() + // (which runs Initialize) happens before JobRunner's try block, so any + // unexpected failure here must disable export, not escape into the job. + try + { + InitializeInternal(); + } + catch (Exception ex) + { + _enabled = false; + Trace.Info($"OTel export disabled, initialization failed (best-effort): {ex.Message}"); + } + } + + private void InitializeInternal() + { + // Capture the collector credential, then scrub it from the process env + // immediately: host step processes inherit the worker env (ProcessInvoker + // copies it), so leaving it set would hand the raw header to every + // untrusted step — the exact leak StepPropagationEnv deliberately avoids. + // Same pattern as CommandSettings removing env-provided config after read. + _rawHeaders = Environment.GetEnvironmentVariable(Constants.Variables.Agent.OtlpHeaders); + if (!string.IsNullOrEmpty(_rawHeaders)) + { + Environment.SetEnvironmentVariable(Constants.Variables.Agent.OtlpHeaders, null); + } + _endpoint = Environment.GetEnvironmentVariable(Constants.Variables.Agent.OtlpEndpoint)?.TrimEnd('/'); + _enabled = !string.IsNullOrEmpty(_endpoint); + if (!_enabled) + { + return; + } + + // Allow either a base URL or one that already includes the signal path. + _tracesUrl = _endpoint.EndsWith("/v1/traces", StringComparison.OrdinalIgnoreCase) + ? _endpoint + : $"{_endpoint}/v1/traces"; + _baseUrl = _endpoint.EndsWith("/v1/traces", StringComparison.OrdinalIgnoreCase) + ? _endpoint.Substring(0, _endpoint.Length - "/v1/traces".Length) + : _endpoint; + _logsUrl = $"{_baseUrl}/v1/logs"; + _metricsUrl = $"{_baseUrl}/v1/metrics"; + // A basic-auth-in-URL collector (https://user:token@...) embeds a credential + // in the endpoint. Register it with the masker (so diag/transport log lines + // can never carry it raw) and keep a stripped base URL for step propagation — + // the same non-leak posture as withholding OTLP headers from steps. + if (Uri.TryCreate(_endpoint, UriKind.Absolute, out var endpointUri) && !string.IsNullOrEmpty(endpointUri.UserInfo)) + { + HostContext.SecretMasker.AddValue(endpointUri.UserInfo); + var colon = endpointUri.UserInfo.IndexOf(':'); + if (colon >= 0 && colon < endpointUri.UserInfo.Length - 1) + { + HostContext.SecretMasker.AddValue(endpointUri.UserInfo.Substring(colon + 1)); + } + } + _propagationBaseUrl = StripUserInfo(_baseUrl); + _propagate = StringUtil.ConvertToBoolean( + Environment.GetEnvironmentVariable(Constants.Variables.Agent.OtlpPropagate)); + + var insecure = StringUtil.ConvertToBoolean( + Environment.GetEnvironmentVariable(Constants.Variables.Agent.OtlpInsecure)); + var caFile = Environment.GetEnvironmentVariable(Constants.Variables.Agent.OtlpCertificate); + // Proxy-aware handler so export honors the runner's proxy configuration. + var handler = HostContext.CreateHttpClientHandler(); + if (!string.IsNullOrEmpty(caFile)) + { + // Trust exactly the CA(s) in this PEM bundle for the collector connection — + // the safe primitive for self-signed collectors (hostname is still checked). + // A bad bundle throws here, which disables export (fail-closed) rather than + // silently falling back to weaker validation. + var trustedRoots = new System.Security.Cryptography.X509Certificates.X509Certificate2Collection(); + trustedRoots.ImportFromPemFile(caFile); + handler.ServerCertificateCustomValidationCallback = + (request, cert, chain, errors) => + (errors & System.Net.Security.SslPolicyErrors.RemoteCertificateNameMismatch) == 0 + && ValidateWithCustomTrustRoots(cert, trustedRoots); + } + else if (insecure) + { + // This flag disables ALL server-certificate validation, not just "allow + // self-signed" — any MITM can read the export traffic, including the + // ACTIONS_RUNNER_OTLP_HEADERS credential sent on every POST. Never do + // this silently; prefer ACTIONS_RUNNER_OTLP_CERTIFICATE (CA bundle). + handler.ServerCertificateCustomValidationCallback = + HttpClientHandler.DangerousAcceptAnyServerCertificateValidator; + Trace.Warning($"{Constants.Variables.Agent.OtlpInsecure}=true disables ALL TLS certificate validation for OTel export" + + (string.IsNullOrEmpty(_rawHeaders) ? "" : " — the OTLP auth header is exposed to any man-in-the-middle") + + $"; prefer {Constants.Variables.Agent.OtlpCertificate} with a CA bundle."); + } + // Optional collector auth headers (read + scrubbed above), e.g. + // ACTIONS_RUNNER_OTLP_HEADERS="authorization=Bearer xyz,x-api-key=abc" + var headers = new List>(); + if (!string.IsNullOrEmpty(_rawHeaders)) + { + foreach (var pair in _rawHeaders.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var eq = pair.IndexOf('='); + if (eq <= 0) + { + continue; + } + var value = pair.Substring(eq + 1).Trim(); + // Header values are commonly auth tokens; register so they're masked if + // ever logged. Deliberately no name filter or length floor (same fail-safe + // posture as the add-mask command): a heuristic would leave short tokens or + // non-standard auth headers unmasked, and over-masking a short non-secret + // value only cosmetically redacts logs on this one runner. + HostContext.SecretMasker.AddValue(value); + headers.Add(new KeyValuePair(pair.Substring(0, eq).Trim(), value)); + } + } + _transport = new OTelHttpTransport(handler, headers, msg => Trace.Info(msg), + userAgent: $"GitHubActionsRunner-OTLP-Exporter/{BuildConstants.RunnerPackage.Version}"); + Trace.Info($"Native OTel export enabled, endpoint: {StripUserInfo(_endpoint)}"); + } + + public bool IsEnabled => _enabled && _featureEnabled; + + public void SetResource(string runnerName, string runnerId, string runnerGroup, string runnerVersion, string osType, string arch, string machineName, bool ephemeral) + { + // Best-effort like every other hook: called from JobRunner before its try + // block, so a throw here would fail the job for telemetry's sake. + try + { + SetResourceInternal(runnerName, runnerId, runnerGroup, runnerVersion, osType, arch, machineName, ephemeral); + } + catch (Exception ex) + { + Trace.Info($"Skipping OTel resource (best-effort): {ex.Message}"); + } + } + + private void SetResourceInternal(string runnerName, string runnerId, string runnerGroup, string runnerVersion, string osType, string arch, string machineName, bool ephemeral) + { + var attrs = DefaultResource(); + void Add(string k, object v) + { + if (v is string s && string.IsNullOrEmpty(s)) + { + return; + } + attrs.Add(new KeyValuePair(k, v)); + } + _serviceVersion = runnerVersion ?? ""; + Add("service.version", runnerVersion); + Add("host.name", machineName); + Add("host.arch", ToSemconvHostArch(arch)); + Add("os.type", ToSemconvOsType(osType)); + // os.name is free-form (unlike the os.type enum); keep the runner's raw value. + Add("os.name", osType); + // semconv cicd.worker.* identifies the executor (no bespoke github.runner.name/id + // dupes); group/ephemeral have no semconv equivalent so they stay github.runner.*. + Add("cicd.worker.name", runnerName); + Add("cicd.worker.id", runnerId); + Add("service.instance.id", runnerId); + Add("github.runner.group", runnerGroup); + Add("github.runner.ephemeral", ephemeral); + // The runner is the CI/CD agent executing tasks (semconv cicd.system.component). + Add("cicd.system.component", "agent"); + + // Honor the standard OTEL_RESOURCE_ATTRIBUTES env var (OTel spec): a + // comma-separated list of key=value pairs with percent-encoded values. This is + // the spec-native way for a deployment (e.g. ARC via the Kubernetes Downward + // API) to attach k8s.pod.name / k8s.namespace.name / k8s.node.name and any other + // resource attributes — no runner-specific env names required. Keys set + // explicitly above take precedence over the env var. + var present = new HashSet(); + foreach (var kv in attrs) + { + present.Add(kv.Key); + } + foreach (var kv in ParseOtelResourceAttributes(Environment.GetEnvironmentVariable("OTEL_RESOURCE_ATTRIBUTES"))) + { + if (present.Add(kv.Key)) + { + attrs.Add(new KeyValuePair(kv.Key, kv.Value)); + } + } + lock (_lock) + { + _resource = attrs; + } + } + + // Chain-validate a server certificate against ONLY the supplied trust roots + // (ACTIONS_RUNNER_OTLP_CERTIFICATE bundle) — a self-signed collector cert or a + // private CA validates; anything else, e.g. a MITM's cert, does not. + internal static bool ValidateWithCustomTrustRoots( + System.Security.Cryptography.X509Certificates.X509Certificate2 cert, + System.Security.Cryptography.X509Certificates.X509Certificate2Collection trustedRoots) + { + if (cert == null) + { + return false; + } + using var chain = new System.Security.Cryptography.X509Certificates.X509Chain(); + chain.ChainPolicy.TrustMode = System.Security.Cryptography.X509Certificates.X509ChainTrustMode.CustomRootTrust; + chain.ChainPolicy.CustomTrustStore.AddRange(trustedRoots); + chain.ChainPolicy.RevocationMode = System.Security.Cryptography.X509Certificates.X509RevocationMode.NoCheck; + return chain.Build(cert); + } + + // Remove URL userinfo (user:password@) from an absolute URL; returns non-URL + // strings unchanged. Used before an endpoint is propagated to step env or logged. + internal static string StripUserInfo(string url) + { + if (string.IsNullOrEmpty(url) || !url.Contains('@')) + { + return url; + } + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || string.IsNullOrEmpty(uri.UserInfo)) + { + return url; + } + var stripped = uri.GetComponents(UriComponents.AbsoluteUri & ~UriComponents.UserInfo, UriFormat.UriEscaped); + // Uri normalization appends "/" to an empty path; keep the caller's shape. + return url.EndsWith("/", StringComparison.Ordinal) ? stripped : stripped.TrimEnd('/'); + } + + // Map VarUtil.OS ("Linux"/"macOS"/"Windows") to the semconv os.type enum, which is + // lowercase and has no macOS value — Apple platforms are "darwin". Standard + // dashboards filter on these exact values, so the raw names would never match. + internal static string ToSemconvOsType(string os) + { + return os switch + { + "Linux" => "linux", + "macOS" => "darwin", + "Windows" => "windows", + _ => os?.ToLowerInvariant(), + }; + } + + // Map VarUtil.OSArchitecture ("X86"/"X64"/"ARM"/"ARM64") to the semconv + // host.arch enum (x86|amd64|arm32|arm64). + internal static string ToSemconvHostArch(string arch) + { + return arch switch + { + "X64" => "amd64", + "X86" => "x86", + "ARM" => "arm32", + "ARM64" => "arm64", + _ => arch?.ToLowerInvariant(), + }; + } + + // ParseOtelResourceAttributes parses the W3C-Baggage-style value of the standard + // OTEL_RESOURCE_ATTRIBUTES env var: comma-separated key=value pairs, values + // percent-encoded. Malformed entries are skipped. + internal static IEnumerable> ParseOtelResourceAttributes(string raw) + { + if (string.IsNullOrEmpty(raw)) + { + yield break; + } + foreach (var pair in raw.Split(',')) + { + var trimmed = pair.Trim(); + var eq = trimmed.IndexOf('='); + if (eq <= 0) + { + continue; + } + var key = trimmed.Substring(0, eq).Trim(); + if (key.Length == 0) + { + continue; + } + var value = trimmed.Substring(eq + 1).Trim(); + yield return new KeyValuePair(key, Uri.UnescapeDataString(value)); + } + } + + public void SetJobInfo(string runId, string runAttempt, string jobName, string jobKey, string repository, string workflow, string eventName, string serverUrl, bool featureEnabled = true, string sha = null, string refName = null, string actor = null, string baseRef = null, string changeId = null) + { + if (!_enabled) + { + lock (_lock) { _featureEnabled = featureEnabled; } + return; + } + + try + { + SetJobInfoInternal(runId, runAttempt, jobName, jobKey, repository, workflow, eventName, serverUrl, featureEnabled, sha, refName, actor, baseRef, changeId); + } + catch (Exception ex) + { + Trace.Info($"Skipping OTel job info (best-effort): {ex.Message}"); + } + } + + private void SetJobInfoInternal(string runId, string runAttempt, string jobName, string jobKey, string repository, string workflow, string eventName, string serverUrl, bool featureEnabled, string sha, string refName, string actor, string baseRef, string changeId) + { + long.TryParse(runId, NumberStyles.Integer, CultureInfo.InvariantCulture, out var runIdNum); + long.TryParse(runAttempt, NumberStyles.Integer, CultureInfo.InvariantCulture, out var runAttemptNum); + if (runAttemptNum == 0) + { + runAttemptNum = 1; + } + + lock (_lock) + { + _featureEnabled = featureEnabled; + _jobInfo = new JobInfo + { + RunId = runIdNum, + RunAttempt = runAttemptNum, + RunIdRaw = runId ?? "", + RunAttemptRaw = string.IsNullOrEmpty(runAttempt) ? "1" : runAttempt, + JobName = jobName ?? "", + JobKey = jobKey ?? "", + Repository = repository ?? "", + Workflow = workflow ?? "", + EventName = eventName ?? "", + ServerUrl = string.IsNullOrEmpty(serverUrl) ? "https://github.com" : serverUrl, + Sha = sha ?? "", + RefName = refName ?? "", + Actor = actor ?? "", + BaseRef = baseRef ?? "", + ChangeId = changeId ?? "", + }; + } + } + + // Common GitHub/VCS context shared by job and step spans. + private static void AddCommonContext(OTelSpan span, JobInfo job) + { + span.Set("github.repository", job.Repository); + span.Set("github.workflow", job.Workflow); + span.Set("github.event_name", job.EventName); + span.Set("github.run_id", job.RunIdRaw); + span.Set("github.run_attempt", job.RunAttemptRaw); + span.Set("github.job", job.JobKey); + span.Set("cicd.pipeline.name", job.Workflow); + span.Set("cicd.pipeline.run.id", job.RunIdRaw); + span.Set("cicd.pipeline.run.url.full", $"{job.ServerUrl}/{job.Repository}/actions/runs/{job.RunIdRaw}/attempts/{job.RunAttemptRaw}"); + span.Set("vcs.repository.url.full", $"{job.ServerUrl}/{job.Repository}"); + span.Set("vcs.provider.name", "github"); + if (!string.IsNullOrEmpty(job.Sha)) span.Set("vcs.ref.head.revision", job.Sha); + if (!string.IsNullOrEmpty(job.RefName)) span.Set("vcs.ref.head.name", job.RefName); + if (!string.IsNullOrEmpty(job.BaseRef)) span.Set("vcs.ref.base.name", job.BaseRef); + if (!string.IsNullOrEmpty(job.ChangeId)) span.Set("vcs.change.id", job.ChangeId); + if (!string.IsNullOrEmpty(job.Repository)) + { + var slash = job.Repository.IndexOf('/'); + if (slash > 0) + { + span.Set("vcs.owner.name", job.Repository.Substring(0, slash)); + span.Set("vcs.repository.name", job.Repository.Substring(slash + 1)); + } + } + if (!string.IsNullOrEmpty(job.Actor)) span.Set("github.actor", job.Actor); + } + + public void RecordStepCompletion(string stepName, int? stepNumber, DateTime? startTime, DateTime? endTime, TaskResult? conclusion, string stepType, string actionName, string actionRef, bool isEmbedded = false, string errorMessage = null, string stepStage = null) + { + // Embedded/composite sub-steps are not top-level timeline steps: their + // display names aren't unique (would collide as span IDs) and they have no + // counterpart in the API-reconstructed trace. Only emit top-level steps. + if (!IsEnabled || isEmbedded) + { + return; + } + + try + { + JobInfo job; + lock (_lock) { job = _jobInfo; } + if (job == null || job.RunId == 0 || string.IsNullOrEmpty(stepName)) + { + return; + } + + var ghConclusion = NormalizeConclusion(conclusion); + var span = new OTelSpan + { + TraceId = NewTraceID(job.RunId, job.RunAttempt), + SpanId = NewStepSpanID(job.RunId, job.RunAttempt, job.JobName, stepNumber ?? 0, stepName), + ParentSpanId = NewJobSpanID(job.RunId, job.RunAttempt, job.JobName), + Name = stepName, + Kind = 1, // INTERNAL + StartTimeUnixNano = ToUnixNano(startTime ?? DateTime.UtcNow), + EndTimeUnixNano = ToUnixNano(endTime ?? DateTime.UtcNow), + }; + + var runUrl = $"{job.ServerUrl}/{job.Repository}/actions/runs/{job.RunIdRaw}"; + var stepUrl = $"{runUrl}/attempts/{job.RunAttemptRaw}#step:{stepNumber ?? 0}:1"; + + span.Set("github.record_type", "step"); + AddCommonContext(span, job); + span.Set("github.step_number", (long)(stepNumber ?? 0)); + span.Set("github.conclusion", ghConclusion); + if (!string.IsNullOrEmpty(stepType)) span.Set("github.step_type", stepType); + if (!string.IsNullOrEmpty(stepStage)) span.Set("github.action_stage", stepStage); + if (!string.IsNullOrEmpty(actionName)) span.Set("github.action", actionName); + if (!string.IsNullOrEmpty(actionRef)) span.Set("github.action_ref", actionRef); + span.Set("cicd.pipeline.task.name", stepName); + span.Set("cicd.pipeline.task.run.id", span.SpanId); + span.Set("cicd.pipeline.task.run.result", ToSemconvResult(ghConclusion)); + span.Set("cicd.pipeline.task.run.url.full", stepUrl); + var taskType = InferTaskType(stepName, actionName); + if (taskType != null) span.Set("cicd.pipeline.task.type", taskType); + + ApplyStatus(span, ghConclusion, errorMessage); + if (ghConclusion == "failure") + { + span.AddException(errorMessage, span.EndTimeUnixNano); + } + + var stepMetric = new TaskMetric + { + Name = stepName, + Type = "step", + Result = ToSemconvResult(ghConclusion), + Attempt = job.RunAttemptRaw, + StartNano = span.StartTimeUnixNano, + EndNano = span.EndTimeUnixNano, + DurationSeconds = Math.Max(0, (span.EndTimeUnixNano - span.StartTimeUnixNano) / 1_000_000_000.0), + }; + + lock (_lock) + { + AddBounded(_pendingSpans, span, MaxBufferedSpans, ref _droppedSpans); + AddBounded(_taskMetrics, stepMetric, MaxBufferedTaskMetrics, ref _droppedTaskMetrics); + } + } + catch (Exception ex) + { + Trace.Info($"Skipping OTel step span (best-effort): {ex.Message}"); + } + } + + // Best-effort cicd.pipeline.task.type (semconv enum: build|test|deploy); omit when unknown. + internal static string InferTaskType(string name, string actionName) + { + var s = $"{name} {actionName}".ToLowerInvariant(); + if (s.Contains("test") || s.Contains("lint") || s.Contains("spec")) return "test"; + if (s.Contains("deploy") || s.Contains("release") || s.Contains("publish")) return "deploy"; + if (s.Contains("build") || s.Contains("compile") || s.Contains("package")) return "build"; + return null; + } + + public void RecordJobCompletion(DateTime? startTime, DateTime? endTime, TaskResult? conclusion, long throttlingDelayMs = 0, string errorMessage = null) + { + if (!IsEnabled) + { + return; + } + + try + { + JobInfo job; + lock (_lock) { job = _jobInfo; } + if (job == null || job.RunId == 0) + { + return; + } + + var ghConclusion = NormalizeConclusion(conclusion); + var jobSpanId = NewJobSpanID(job.RunId, job.RunAttempt, job.JobName); + var span = new OTelSpan + { + TraceId = NewTraceID(job.RunId, job.RunAttempt), + SpanId = jobSpanId, + // Root of the runner's trace. The runner owns the job, not the + // workflow run, so we do NOT parent to a run span we never emit + // (that left a dangling parentSpanId). The authoritative run span + // comes from the GitHub API path and is merged downstream; an + // upstream scheduler, when present, is attached as a span link below. + ParentSpanId = null, + Name = job.JobName, + // INTERNAL: the job is a cicd task run (it carries cicd.pipeline.task.*), + // and cicd-spans says task-run spans SHOULD be INTERNAL. The SERVER + // pipeline-run span (with cicd.pipeline.result) is the API-side + // consumer's to emit — see the trace-shape note above. + Kind = 1, // INTERNAL + StartTimeUnixNano = ToUnixNano(startTime ?? DateTime.UtcNow), + EndTimeUnixNano = ToUnixNano(endTime ?? DateTime.UtcNow), + }; + + span.Set("github.record_type", "job"); + AddInboundParentLink(span); + AddCommonContext(span, job); + span.Set("github.conclusion", ghConclusion); + span.Set("cicd.pipeline.task.name", job.JobName); + span.Set("cicd.pipeline.task.run.id", span.SpanId); + span.Set("cicd.pipeline.task.run.result", ToSemconvResult(ghConclusion)); + span.Set("cicd.pipeline.task.run.url.full", $"{job.ServerUrl}/{job.Repository}/actions/runs/{job.RunIdRaw}/attempts/{job.RunAttemptRaw}"); + if (throttlingDelayMs > 0) + { + // Time the runner spent blocked on server throttling during this job. + span.Set("github.throttling_delay_ms", throttlingDelayMs); + } + + ApplyStatus(span, ghConclusion, errorMessage); + if (ghConclusion == "failure") + { + span.AddException(errorMessage, span.EndTimeUnixNano); + } + + var durationSeconds = Math.Max(0, (span.EndTimeUnixNano - span.StartTimeUnixNano) / 1_000_000_000.0); + var metrics = new JobMetrics + { + StartNano = span.StartTimeUnixNano, + EndNano = span.EndTimeUnixNano, + DurationSeconds = durationSeconds, + PipelineName = job.Workflow, + Attempt = job.RunAttemptRaw, + Result = ToSemconvResult(ghConclusion), + Errors = ghConclusion == "failure" ? 1 : 0, + }; + var jobMetric = new TaskMetric + { + Name = job.JobName, + Type = "job", + Result = ToSemconvResult(ghConclusion), + Attempt = job.RunAttemptRaw, + StartNano = span.StartTimeUnixNano, + EndNano = span.EndTimeUnixNano, + DurationSeconds = durationSeconds, + }; + + lock (_lock) + { + // The job span is the trace ROOT and is recorded last: at the buffer + // cap the drop-newest policy would evict exactly it, orphaning every + // already-buffered step span (they all parent to its deterministic + // ID). One root per job, so it bypasses the cap (worst case cap+1). + _pendingSpans.Add(span); + _jobMetrics = metrics; + AddBounded(_taskMetrics, jobMetric, MaxBufferedTaskMetrics, ref _droppedTaskMetrics); + } + } + catch (Exception ex) + { + Trace.Info($"Skipping OTel job span (best-effort): {ex.Message}"); + } + } + + public void RecordSpan(string name, string spanType, DateTime startTime, DateTime endTime, IDictionary attributes = null, string parentStepName = null, int parentStepNumber = 0, int spanKind = 1) + { + if (!IsEnabled || string.IsNullOrEmpty(name)) + { + return; + } + + try + { + JobInfo job; + lock (_lock) { job = _jobInfo; } + if (job == null || job.RunId == 0) + { + return; + } + + // Nest under the owning step (e.g. action resolution runs inside "Set up job"), + // falling back to the job span when no step is supplied. + var parentSpanId = string.IsNullOrEmpty(parentStepName) + ? NewJobSpanID(job.RunId, job.RunAttempt, job.JobName) + : NewStepSpanID(job.RunId, job.RunAttempt, job.JobName, parentStepNumber, parentStepName); + + var startNano = ToUnixNano(startTime); + var span = new OTelSpan + { + TraceId = NewTraceID(job.RunId, job.RunAttempt), + // Include start time so repeated same-named operations don't collide. + SpanId = NewSpanIDFromString($"{spanType}-{job.RunId}-{job.RunAttempt}-{job.JobName}-{name}-{startNano}"), + ParentSpanId = parentSpanId, + Name = name, + Kind = spanKind, + StartTimeUnixNano = startNano, + EndTimeUnixNano = ToUnixNano(endTime), + }; + span.Set("github.record_type", spanType); + AddCommonContext(span, job); + // task.name marks this as a child task (not a root pipeline) for enrichers. + span.Set("cicd.pipeline.task.name", name); + if (attributes != null) + { + foreach (var kv in attributes) + { + span.Set(kv.Key, kv.Value); + } + } + lock (_lock) { AddBounded(_pendingSpans, span, MaxBufferedSpans, ref _droppedSpans); } + } + catch (Exception ex) + { + Trace.Info($"Skipping OTel span '{name}' (best-effort): {ex.Message}"); + } + } + + public void RecordJobLog(string severityText, string message) + { + if (!IsEnabled || string.IsNullOrEmpty(message)) + { + return; + } + try + { + JobInfo job; + lock (_lock) { job = _jobInfo; } + if (job == null || job.RunId == 0) + { + return; + } + var log = new OTelLog + { + TimeUnixNano = ToUnixNano(DateTime.UtcNow), + SeverityText = string.IsNullOrEmpty(severityText) ? "INFO" : severityText.ToUpperInvariant(), + SeverityNumber = SeverityNumber(severityText), + Body = message, + TraceId = NewTraceID(job.RunId, job.RunAttempt), + SpanId = NewJobSpanID(job.RunId, job.RunAttempt, job.JobName), + }; + log.Attributes.Add(new("cicd.pipeline.task.name", job.JobName)); + lock (_lock) { AddBounded(_pendingLogs, log, MaxBufferedLogs, ref _droppedLogs); } + } + catch (Exception ex) + { + Trace.Info($"Skipping OTel job log (best-effort): {ex.Message}"); + } + } + + public void RecordStepLog(string stepName, int? stepNumber, string severityText, string message) + { + if (!IsEnabled || string.IsNullOrEmpty(message)) + { + return; + } + try + { + JobInfo job; + lock (_lock) { job = _jobInfo; } + if (job == null || job.RunId == 0 || string.IsNullOrEmpty(stepName)) + { + return; + } + var log = new OTelLog + { + TimeUnixNano = ToUnixNano(DateTime.UtcNow), + SeverityText = string.IsNullOrEmpty(severityText) ? "INFO" : severityText.ToUpperInvariant(), + SeverityNumber = SeverityNumber(severityText), + Body = message, + TraceId = NewTraceID(job.RunId, job.RunAttempt), + SpanId = NewStepSpanID(job.RunId, job.RunAttempt, job.JobName, stepNumber ?? 0, stepName), + }; + log.Attributes.Add(new("cicd.pipeline.task.name", stepName)); + lock (_lock) { AddBounded(_pendingLogs, log, MaxBufferedLogs, ref _droppedLogs); } + } + catch (Exception ex) + { + Trace.Info($"Skipping OTel log (best-effort): {ex.Message}"); + } + } + + public IDictionary StepPropagationEnv(string stepName, int? stepNumber) + { + var env = new Dictionary(); + try + { + if (!IsEnabled || !_propagate) + { + return env; + } + JobInfo job; + lock (_lock) { job = _jobInfo; } + if (job == null || job.RunId == 0 || string.IsNullOrEmpty(stepName)) + { + return env; + } + var traceId = NewTraceID(job.RunId, job.RunAttempt); + var spanId = NewStepSpanID(job.RunId, job.RunAttempt, job.JobName, stepNumber ?? 0, stepName); + env["TRACEPARENT"] = $"00-{traceId}-{spanId}-01"; + // Base endpoint (the OTel SDK appends /v1/traces, /v1/logs itself), + // with any URL userinfo credential stripped. + env["OTEL_EXPORTER_OTLP_ENDPOINT"] = _propagationBaseUrl; + // NOTE: deliberately do NOT propagate OTEL_EXPORTER_OTLP_HEADERS. It carries the + // collector credential; handing it to user step processes lets any step read and + // exfiltrate it. Steps that need to authenticate to the collector must be given a + // separate, scoped credential out of band. + env["OTEL_RESOURCE_ATTRIBUTES"] = + $"service.name=github-actions-job,github.run_id={job.RunIdRaw},github.run_attempt={job.RunAttemptRaw},github.job={job.JobKey},github.repository={job.Repository}"; + return env; + } + catch (Exception ex) + { + // Called per step from the handler's run path — never let telemetry + // env assembly throw into step execution. + Trace.Info($"Skipping OTel step env (best-effort): {ex.Message}"); + env.Clear(); + return env; + } + } + + private static int SeverityNumber(string text) + { + return (text?.ToUpperInvariant()) switch + { + "ERROR" => 17, + "WARN" or "WARNING" => 13, + "NOTICE" => 10, + "DEBUG" => 5, + _ => 9, // INFO + }; + } + + // Append under a buffer cap; excess is dropped and counted (call under _lock). + private static void AddBounded(List list, T item, int cap, ref long dropped) + { + if (list.Count >= cap) + { + dropped++; + return; + } + list.Add(item); + } + + public async Task FlushAsync(CancellationToken cancellationToken = default) + { + if (!IsEnabled) + { + return; + } + + // Export is best-effort and must NEVER affect the job: serialization or any + // other failure here is contained so it can't escape into job completion. + try + { + List spans; + List logs; + JobMetrics metrics; + List tasks; + List> resource; + long droppedSpans, droppedLogs, droppedTasks; + lock (_lock) + { + spans = new List(_pendingSpans); + logs = new List(_pendingLogs); + metrics = _jobMetrics; + tasks = new List(_taskMetrics); + droppedSpans = _droppedSpans; + droppedLogs = _droppedLogs; + droppedTasks = _droppedTaskMetrics; + _pendingSpans.Clear(); + _pendingLogs.Clear(); + _jobMetrics = null; + _taskMetrics.Clear(); + _droppedSpans = _droppedLogs = _droppedTaskMetrics = 0; + resource = _resource; + } + + // One overall deadline across all signals (incl. retries) so flush never + // delays job completion by more than this, even if the collector is slow. + using var flushCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + flushCts.CancelAfter(TimeSpan.FromSeconds(4)); + var flushToken = flushCts.Token; + + var totalPosts = 0; + var failedPosts = 0; + async Task PostCountedAsync(string url, byte[] payload, string what) + { + totalPosts++; + if (!await _transport.PostAsync(url, payload, what, flushToken)) + { + failedPosts++; + } + } + + // Chunked POSTs: a whole job in one request can exceed a collector's + // body limit (413 is non-retryable per OTLP — the entire signal would + // be dropped), and serializing per chunk bounds peak memory at flush. + for (var i = 0; i < spans.Count; i += MaxItemsPerPost) + { + var batch = spans.GetRange(i, Math.Min(MaxItemsPerPost, spans.Count - i)); + await PostCountedAsync(_tracesUrl, BuildOTLPSpansJson(batch, resource), $"{batch.Count} span(s)"); + } + for (var i = 0; i < logs.Count; i += MaxItemsPerPost) + { + var batch = logs.GetRange(i, Math.Min(MaxItemsPerPost, logs.Count - i)); + await PostCountedAsync(_logsUrl, BuildOTLPLogsJson(batch, resource), $"{batch.Count} log(s)"); + } + if (metrics != null || tasks.Count > 0) + { + await PostCountedAsync(_metricsUrl, BuildOTLPMetricsJson(metrics, tasks, resource), "metrics"); + } + + // ONE end-of-job summary at Warning when anything was lost (failed POSTs + // or cap-drops) — greppable at fleet scale, unlike per-signal Info lines. + var summary = BuildExportSummary(totalPosts, failedPosts, droppedSpans, droppedLogs, droppedTasks); + if (summary != null) + { + _lastFlushSummary = summary; + Trace.Warning(summary); + } + } + catch (Exception ex) + { + _lastFlushSummary = $"OTel export summary: flush failed entirely ({ex.Message}) — this job's telemetry was lost (best-effort, job unaffected)"; + Trace.Warning(_lastFlushSummary); + } + } + + // Null when everything was delivered and nothing was dropped; otherwise one + // operator-facing line describing exactly what was lost. + internal static string BuildExportSummary(int totalPosts, int failedPosts, long droppedSpans, long droppedLogs, long droppedTasks) + { + if (failedPosts == 0 && droppedSpans + droppedLogs + droppedTasks == 0) + { + return null; + } + return $"OTel export summary: {failedPosts}/{totalPosts} POST(s) failed; dropped over buffer cap: " + + $"{droppedSpans} span(s), {droppedLogs} log(s), {droppedTasks} task-metric(s) (best-effort, job unaffected)"; + } + + // ---- shared deterministic ID contract (pure, mirrored in otel-explorer) ---- + // + // SHA-256 (truncated) — NOT MD5: MD5 is disallowed under FIPS, so a FIPS-enabled + // host would throw. SHA-256 gives the same deterministic-from-run-id behavior; + // we take the leading 16 bytes for a trace ID and 8 for a span ID. These IDs are + // intentionally deterministic (not random), so the W3C Level-2 randomness flag is + // never set on the trace. + + internal static string NewTraceID(long runId, long runAttempt) + { + if (runAttempt == 0) runAttempt = 1; + return BytesToHex(SHA256.HashData(Encoding.UTF8.GetBytes($"{runId}-{runAttempt}")), 16); + } + + internal static string NewSpanID(long id) + { + var bytes = new byte[8]; + for (int i = 0; i < 8; i++) + { + bytes[i] = (byte)(id >> (56 - 8 * i)); + } + return BytesToHex(bytes); + } + + internal static string NewSpanIDFromString(string s) + { + return BytesToHex(SHA256.HashData(Encoding.UTF8.GetBytes(s)), 8); + } + + internal static string NewJobSpanID(long runId, long runAttempt, string jobName) + { + if (runAttempt == 0) runAttempt = 1; + return NewSpanIDFromString($"job-{runId}-{runAttempt}-{jobName}"); + } + + internal static string NewStepSpanID(long runId, long runAttempt, string jobName, int stepNumber, string stepName) + { + if (runAttempt == 0) runAttempt = 1; + // Step number disambiguates two top-level steps that share a display name; + // it matches the GitHub API's 1-based step.number so the IDs still merge. + return NewSpanIDFromString($"step-{runId}-{runAttempt}-{jobName}-{stepNumber}-{stepName}"); + } + + // ---- helpers ---- + + private static void ApplyStatus(OTelSpan span, string ghConclusion, string errorMessage = null) + { + if (ghConclusion == "failure") + { + span.StatusCode = 2; // ERROR + span.StatusMessage = errorMessage; // status.message per Recording Errors guidance + span.Set("error.type", "failure"); // low-cardinality classifier + } + } + + internal static string NormalizeConclusion(TaskResult? result) + { + return result switch + { + TaskResult.Succeeded => "success", + TaskResult.SucceededWithIssues => "success", + TaskResult.Failed => "failure", + TaskResult.Abandoned => "failure", + TaskResult.Canceled => "cancelled", + TaskResult.Skipped => "skipped", + _ => "unknown", + }; + } + + internal static string ToSemconvResult(string ghConclusion) + { + // semconv cicd.pipeline.task.run.result enum: success|failure|cancellation|skip|timeout|error + return ghConclusion switch + { + "success" => "success", + "failure" => "failure", + "cancelled" => "cancellation", + "skipped" => "skip", + _ => "error", + }; + } + + private static List> DefaultResource() + { + return new List> + { + new("service.name", "github-actions-runner"), + }; + } + + // If an upstream scheduler injected a W3C traceparent, attach it to the job span + // as a span LINK (cross-trace causality) — we do NOT re-root the job's own + // deterministic trace. tracestate is preserved and the inbound sampled flag is + // carried on the link rather than overridden. + private void AddInboundParentLink(OTelSpan span) + { + var traceparent = Environment.GetEnvironmentVariable(Constants.Variables.Agent.OtlpParentTraceparent); + if (!TryParseTraceparent(traceparent, out var traceId, out var spanId, out var flags)) + { + return; + } + var link = new OTelLink { TraceId = traceId, SpanId = spanId, Flags = flags }; + var tracestate = Environment.GetEnvironmentVariable(Constants.Variables.Agent.OtlpParentTracestate); + if (!string.IsNullOrEmpty(tracestate)) + { + link.TraceState = tracestate.Trim(); + } + // The linked span belongs to the controller that scheduled this job (semconv). + link.Attributes.Add(new KeyValuePair("cicd.system.component", "controller")); + span.Links.Add(link); + } + + // TryParseTraceparent validates a W3C traceparent ("version-traceid-spanid-flags"). + // Rejects malformed input and the forbidden "ff" version; tolerates unknown future + // versions by reading only the first four fields. + internal static bool TryParseTraceparent(string traceparent, out string traceId, out string spanId, out byte flags) + { + traceId = null; + spanId = null; + flags = 0; + if (string.IsNullOrEmpty(traceparent)) + { + return false; + } + var parts = traceparent.Trim().Split('-'); + if (parts.Length < 4) + { + return false; + } + var version = parts[0]; + if (version.Length != 2 || !IsHex(version) || version == "ff") + { + return false; + } + // Version 00 is exactly four fields; unknown versions may carry more. + if (version == "00" && parts.Length != 4) + { + return false; + } + var tid = parts[1]; + var sid = parts[2]; + var flg = parts[3]; + if (tid.Length != 32 || !IsHex(tid) || IsAllZeroHex(tid)) + { + return false; + } + if (sid.Length != 16 || !IsHex(sid) || IsAllZeroHex(sid)) + { + return false; + } + if (flg.Length != 2 || !IsHex(flg)) + { + return false; + } + traceId = tid.ToLowerInvariant(); + spanId = sid.ToLowerInvariant(); + flags = Convert.ToByte(flg, 16); + return true; + } + + private static bool IsHex(string s) + { + foreach (var c in s) + { + var hex = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); + if (!hex) + { + return false; + } + } + return true; + } + + private static bool IsAllZeroHex(string s) + { + foreach (var c in s) + { + if (c != '0') + { + return false; + } + } + return true; + } + + private static string BytesToHex(byte[] bytes, int length = 0) + { + if (length <= 0) length = bytes.Length; + var sb = new StringBuilder(length * 2); + for (int i = 0; i < length; i++) + sb.Append(bytes[i].ToString("x2")); + return sb.ToString(); + } + + private static long ToUnixNano(DateTime dt) + { + // Timeline timestamps are UTC; treat an Unspecified kind as UTC rather than + // letting ToUniversalTime() shift it by the local offset. + if (dt.Kind == DateTimeKind.Unspecified) + { + dt = DateTime.SpecifyKind(dt, DateTimeKind.Utc); + } + return (dt.ToUniversalTime().Ticks - 621355968000000000L) * 100; + } + + private string Mask(string s) + { + if (string.IsNullOrEmpty(s)) + { + return s; + } + return HostContext?.SecretMasker?.MaskSecrets(s) ?? s; + } + + internal string LastFlushSummaryForTest => _lastFlushSummary; + + internal int PendingSpanCountForTest + { + get { lock (_lock) { return _pendingSpans.Count; } } + } + + internal long DroppedSpanCountForTest + { + get { lock (_lock) { return _droppedSpans; } } + } + + internal int PendingLogCountForTest + { + get { lock (_lock) { return _pendingLogs.Count; } } + } + + internal string BuildPendingOtlpJsonForTest() + { + lock (_lock) { return Encoding.UTF8.GetString(BuildOTLPSpansJson(new List(_pendingSpans), _resource)); } + } + + internal string BuildPendingOtlpLogsJsonForTest() + { + lock (_lock) { return Encoding.UTF8.GetString(BuildOTLPLogsJson(new List(_pendingLogs), _resource)); } + } + + internal string BuildPendingOtlpMetricsJsonForTest() + { + lock (_lock) + { + return _jobMetrics == null && _taskMetrics.Count == 0 + ? null + : Encoding.UTF8.GetString(BuildOTLPMetricsJson(_jobMetrics, _taskMetrics, _resource)); + } + } + } +} diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index c1cf692204f..e018bf73c44 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -18,6 +18,12 @@ public sealed class TestHostContext : IHostContext, IDisposable { private readonly ConcurrentDictionary> _serviceInstances = new(); private readonly ConcurrentDictionary _serviceSingletons = new(); + // See GetService: only these types auto-instantiate their ServiceLocator + // default when unregistered; everything else keeps the fail-loud throw. + private static readonly HashSet _serviceLocatorFallbackAllowlist = new() + { + typeof(GitHub.Runner.Worker.IOTelTraceExporter), + }; private readonly ITraceManager _traceManager; private readonly Terminal _term; private readonly SecretMasker _secretMasker; @@ -129,11 +135,24 @@ public T GetService() where T : class, IRunnerService { _trace.Verbose($"Get service: '{typeof(T).Name}'"); - // Get the registered singleton instance. + // Return the registered singleton (mock or explicitly-set instance) if present. object service; if (!_serviceSingletons.TryGetValue(typeof(T), out service)) { - throw new Exception($"Singleton instance not registered for type '{typeof(T).FullName}'."); + // Unregistered services fail loud: a test that forgets to mock a + // dependency must throw here, not silently run the real implementation. + // The only exception is a short allowlist of leaf telemetry services + // that production code resolves lazily on many paths and that are inert + // unless explicitly configured via env vars; those mirror HostContext + // and instantiate their ServiceLocator default. + var defaultType = _serviceLocatorFallbackAllowlist.Contains(typeof(T)) + ? GetServiceLocatorDefault(typeof(T)) + : null; + if (defaultType == null) + { + throw new Exception($"Singleton instance not registered for type '{typeof(T).FullName}'."); + } + service = _serviceSingletons.GetOrAdd(typeof(T), Activator.CreateInstance(defaultType)); } T s = service as T; @@ -141,6 +160,25 @@ public T GetService() where T : class, IRunnerService return s; } + private static Type GetServiceLocatorDefault(Type interfaceType) + { + foreach (CustomAttributeData attr in interfaceType.GetTypeInfo().CustomAttributes) + { + if (attr.AttributeType != typeof(ServiceLocatorAttribute)) + { + continue; + } + foreach (CustomAttributeNamedArgument arg in attr.NamedArguments) + { + if (string.Equals(arg.MemberName, ServiceLocatorAttribute.DefaultPropertyName, StringComparison.Ordinal)) + { + return arg.TypedValue.Value as Type; + } + } + } + return null; + } + public void EnqueueInstance(T instance) where T : class, IRunnerService { // Enqueue a service instance to be returned by CreateService. diff --git a/src/Test/L0/TestHostContextL0.cs b/src/Test/L0/TestHostContextL0.cs new file mode 100644 index 00000000000..9b382bb19c4 --- /dev/null +++ b/src/Test/L0/TestHostContextL0.cs @@ -0,0 +1,37 @@ +using System; +using GitHub.Runner.Worker; +using Xunit; + +namespace GitHub.Runner.Common.Tests +{ + public sealed class TestHostContextL0 + { + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void GetService_UnregisteredService_FailsLoud() + { + using (var hc = new TestHostContext(this)) + { + // A test that forgets to mock a service dependency must fail loudly, + // not silently exercise the real implementation. + Assert.Throws(() => hc.GetService()); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void GetService_OTelTraceExporter_ResolvesDefaultWithoutRegistration() + { + using (var hc = new TestHostContext(this)) + { + // Allowlisted leaf telemetry service: production code resolves it + // lazily on many paths, and it is inert unless its env vars are set. + var exporter = hc.GetService(); + Assert.NotNull(exporter); + Assert.Same(exporter, hc.GetService()); + } + } + } +} diff --git a/src/Test/L0/Worker/HandlerL0.cs b/src/Test/L0/Worker/HandlerL0.cs index f9dbd67c757..9f5d054a653 100644 --- a/src/Test/L0/Worker/HandlerL0.cs +++ b/src/Test/L0/Worker/HandlerL0.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Threading.Tasks; using GitHub.Actions.RunService.WebApi; using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.WebApi; @@ -85,5 +87,75 @@ public void PrepareExecution_PopulateTelemetry_DockerActions() Assert.Equal("ubuntu:20.04", _stepTelemetry.Action); } } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void OTelPropagation_NeverClobbersWorkflowSetEnv() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange: the workflow's step env: block deliberately set its own values. + var handler = new TestOTelEnvHandler(); + handler.Initialize(hc); + handler.ExecutionContext = _ec.Object; + handler.Environment = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["TRACEPARENT"] = "00-11111111111111111111111111111111-1111111111111111-01", + ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "http://app-collector:4318", + }; + _ec.Setup(x => x.GetOTelStepEnv()).Returns(new Dictionary + { + ["TRACEPARENT"] = "00-22222222222222222222222222222222-2222222222222222-01", + ["OTEL_EXPORTER_OTLP_ENDPOINT"] = "http://runner-collector:4318", + ["OTEL_RESOURCE_ATTRIBUTES"] = "service.name=github-actions-job", + }); + + // Act. + handler.ApplyOTelPropagation(); + + // Assert: workflow-set values win; only missing keys are injected. + Assert.Equal("00-11111111111111111111111111111111-1111111111111111-01", handler.Environment["TRACEPARENT"]); + Assert.Equal("http://app-collector:4318", handler.Environment["OTEL_EXPORTER_OTLP_ENDPOINT"]); + Assert.Equal("service.name=github-actions-job", handler.Environment["OTEL_RESOURCE_ATTRIBUTES"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void OTelPropagation_InjectsWhenAbsent_NullSafe() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange. + var handler = new TestOTelEnvHandler(); + handler.Initialize(hc); + handler.ExecutionContext = _ec.Object; + handler.Environment = new Dictionary(StringComparer.OrdinalIgnoreCase); + _ec.Setup(x => x.GetOTelStepEnv()).Returns(new Dictionary + { + ["TRACEPARENT"] = "00-22222222222222222222222222222222-2222222222222222-01", + }); + + // Act. + handler.ApplyOTelPropagation(); + + // Assert. + Assert.Equal("00-22222222222222222222222222222222-2222222222222222-01", handler.Environment["TRACEPARENT"]); + + // A null step env (e.g. mocked-out execution context) must be a no-op, not a throw. + _ec.Setup(x => x.GetOTelStepEnv()).Returns((IDictionary)null); + handler.ApplyOTelPropagation(); + Assert.Single(handler.Environment); + } + } + + // Minimal concrete Handler exposing the shared OTel env injection for test. + private sealed class TestOTelEnvHandler : Handler, IHandler + { + public Task RunAsync(ActionRunStage stage) => Task.CompletedTask; + public void ApplyOTelPropagation() => AddOTelPropagationToEnvironment(); + } } } diff --git a/src/Test/L0/Worker/OTelHttpTransportL0.cs b/src/Test/L0/Worker/OTelHttpTransportL0.cs new file mode 100644 index 00000000000..97198c278a3 --- /dev/null +++ b/src/Test/L0/Worker/OTelHttpTransportL0.cs @@ -0,0 +1,228 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Runner.Worker; +using Xunit; + +namespace GitHub.Runner.Common.Tests.Worker +{ + // Wire-path tests for the OTLP/HTTP transport using a fake HttpMessageHandler: + // URL routing, Content-Type, gzip Content-Encoding, gunzip -> original body, auth + // header, retry behavior, and partial-success — none of which had coverage before. + public sealed class OTelHttpTransportL0 + { + private sealed class FakeHandler : HttpMessageHandler + { + public readonly List Requests = new(); + public readonly List Bodies = new(); + private readonly Queue> _responses; + + public FakeHandler(params Func[] responses) => + _responses = new Queue>(responses); + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken ct) + { + Requests.Add(request); + Bodies.Add(request.Content != null ? await request.Content.ReadAsByteArrayAsync(ct) : Array.Empty()); + return _responses.Count > 0 ? _responses.Dequeue()() : new HttpResponseMessage(HttpStatusCode.OK); + } + } + + private static HttpResponseMessage Resp(HttpStatusCode code, string body = "") => + new(code) { Content = new StringContent(body) }; + + private static string Gunzip(byte[] b) + { + using var ms = new MemoryStream(b); + using var gz = new GZipStream(ms, CompressionMode.Decompress); + using var sr = new StreamReader(gz); + return sr.ReadToEnd(); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task Post_RoutesGzipsAndAttachesAuthHeader() + { + var handler = new FakeHandler(() => Resp(HttpStatusCode.OK)); + using var transport = new OTelHttpTransport(handler, + new[] { new KeyValuePair("authorization", "Bearer xyz") }); + + var ok = await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{\"resourceSpans\":[]}"), "spans", default); + + Assert.True(ok); + var req = handler.Requests.Single(); + Assert.Equal("http://collector/v1/traces", req.RequestUri.ToString()); + Assert.Equal("application/json", req.Content.Headers.ContentType.MediaType); + Assert.Contains("gzip", req.Content.Headers.ContentEncoding); + Assert.Equal("Bearer xyz", req.Headers.GetValues("authorization").Single()); + Assert.Equal("{\"resourceSpans\":[]}", Gunzip(handler.Bodies[0])); // gunzips to the original OTLP + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task Post_SendsIdentifyingUserAgent() + { + // OTLP exporter spec SHOULD: a User-Agent identifying the exporter + version, + // so collector operators can attribute runner-fleet traffic. + var handler = new FakeHandler(() => Resp(HttpStatusCode.OK)); + using var transport = new OTelHttpTransport(handler, userAgent: "GitHubActionsRunner-OTLP-Exporter/2.333.0"); + + await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{}"), "spans", default); + + Assert.Equal("GitHubActionsRunner-OTLP-Exporter/2.333.0", + string.Join(" ", handler.Requests.Single().Headers.GetValues("User-Agent"))); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task Post_RetriesOnTransientThenSucceeds() + { + var handler = new FakeHandler(() => Resp(HttpStatusCode.ServiceUnavailable), () => Resp(HttpStatusCode.OK)); + using var transport = new OTelHttpTransport(handler); + + var ok = await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{}"), "spans", default); + + Assert.True(ok); + Assert.Equal(2, handler.Requests.Count); // 503 then retried -> 200 + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task Post_DoesNotRetryOnClientError() + { + var handler = new FakeHandler(() => Resp(HttpStatusCode.BadRequest)); + using var transport = new OTelHttpTransport(handler); + + var ok = await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{}"), "spans", default); + + Assert.False(ok); + Assert.Single(handler.Requests); // 4xx is not retried + } + + [Theory] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + [InlineData(HttpStatusCode.RequestTimeout)] // 408 + [InlineData(HttpStatusCode.InternalServerError)] // 500 + public async Task Post_DoesNotRetryNonRetryableStatuses(HttpStatusCode code) + { + // OTLP/HTTP failures table: only 429/502/503/504 are retryable; + // all other 4xx/5xx MUST NOT be retried. + var handler = new FakeHandler(() => Resp(code)); + using var transport = new OTelHttpTransport(handler); + + var ok = await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{}"), "spans", default); + + Assert.False(ok); + Assert.Single(handler.Requests); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task Post_HonorsRetryAfterOnThrottle() + { + // OTLP spec SHOULD: wait as long as the server's Retry-After before retrying. + var handler = new FakeHandler( + () => + { + var r = Resp(HttpStatusCode.TooManyRequests); + r.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromSeconds(1)); + return r; + }, + () => Resp(HttpStatusCode.OK)); + using var transport = new OTelHttpTransport(handler); + var delays = new List(); + transport.DelayAsync = (d, _) => { delays.Add(d); return Task.CompletedTask; }; + + var ok = await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{}"), "spans", default); + + Assert.True(ok); + Assert.Equal(2, handler.Requests.Count); + Assert.Equal(TimeSpan.FromSeconds(1), delays.Single()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task Post_SkipsRetryWhenRetryAfterExceedsFlushDeadline() + { + // A Retry-After the 4 s flush deadline can't honor: don't burn the retry. + var handler = new FakeHandler(() => + { + var r = Resp(HttpStatusCode.TooManyRequests); + r.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromSeconds(30)); + return r; + }); + using var transport = new OTelHttpTransport(handler); + var delays = new List(); + transport.DelayAsync = (d, _) => { delays.Add(d); return Task.CompletedTask; }; + + var ok = await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{}"), "spans", default); + + Assert.False(ok); + Assert.Single(handler.Requests); + Assert.Empty(delays); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task Post_JittersRetryDelayWithoutRetryAfter() + { + // No Retry-After -> short jittered delay (100-400 ms), not a fixed interval, + // so a fleet of runners flushing at once doesn't retry in lockstep. + var handler = new FakeHandler(() => Resp(HttpStatusCode.ServiceUnavailable), () => Resp(HttpStatusCode.OK)); + using var transport = new OTelHttpTransport(handler); + var delays = new List(); + transport.DelayAsync = (d, _) => { delays.Add(d); return Task.CompletedTask; }; + + var ok = await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{}"), "spans", default); + + Assert.True(ok); + Assert.Equal(2, handler.Requests.Count); + Assert.InRange(delays.Single().TotalMilliseconds, 100, 400); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task Post_PartialSuccessIsDeliveredAndLogged() + { + var handler = new FakeHandler(() => Resp(HttpStatusCode.OK, "{\"partialSuccess\":{\"rejectedSpans\":\"2\",\"errorMessage\":\"dropped\"}}")); + string logged = null; + using var transport = new OTelHttpTransport(handler, null, m => logged = m); + + var ok = await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{}"), "spans", default); + + Assert.True(ok); + Assert.Contains("partially rejected", logged); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task Post_ExpiredDeadlineDoesNotThrowOrRetryForever() + { + var handler = new FakeHandler(() => Resp(HttpStatusCode.ServiceUnavailable)); + using var transport = new OTelHttpTransport(handler); + using var cts = new CancellationTokenSource(); + cts.Cancel(); // deadline already blown + + var ok = await transport.PostAsync("http://collector/v1/traces", Encoding.UTF8.GetBytes("{}"), "spans", cts.Token); + + Assert.False(ok); // best-effort: swallowed, never throws + } + } +} diff --git a/src/Test/L0/Worker/OTelTraceExporterBenchmark.cs b/src/Test/L0/Worker/OTelTraceExporterBenchmark.cs new file mode 100644 index 00000000000..26b28048514 --- /dev/null +++ b/src/Test/L0/Worker/OTelTraceExporterBenchmark.cs @@ -0,0 +1,90 @@ +using System; +using System.Diagnostics; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common.Tests; +using GitHub.Runner.Worker; +using Xunit; +using Xunit.Abstractions; + +namespace GitHub.Runner.Common.Tests.Worker +{ + // Micro-benchmark for the native OTel exporter's hot paths: per-step emit cost, + // end-of-job serialize cost, payload size, and allocations. Tagged "Benchmark" so + // it stays out of the normal L0 run. Invoke explicitly: + // dotnet test --filter "FullyQualifiedName~OTelTraceExporterBenchmark" + public sealed class OTelTraceExporterBenchmark : IDisposable + { + private const string EndpointEnv = "ACTIONS_RUNNER_OTLP_ENDPOINT"; + private readonly string _originalEndpoint = Environment.GetEnvironmentVariable(EndpointEnv); + private readonly ITestOutputHelper _out; + + public OTelTraceExporterBenchmark(ITestOutputHelper output) => _out = output; + + public void Dispose() => Environment.SetEnvironmentVariable(EndpointEnv, _originalEndpoint); + + private OTelTraceExporter Enabled(TestHostContext hc) + { + hc.SetSingleton(new HttpClientHandlerFactory()); + Environment.SetEnvironmentVariable(EndpointEnv, "http://localhost:4318"); + var exporter = new OTelTraceExporter(); + exporter.Initialize(hc); + return exporter; + } + + [Theory] + [Trait("Category", "Benchmark")] + [InlineData(1)] + [InlineData(10)] + [InlineData(100)] + [InlineData(1000)] + [InlineData(10000)] + public void EmitAndSerialize(int n) + { + var start = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + // Warm up JIT + secret masker + first-step allocations. + using (var warm = new TestHostContext(this)) + { + var w = Enabled(warm); + w.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + for (var i = 0; i < 50; i++) + { + w.RecordStepCompletion($"warm {i}", i, start, start.AddSeconds(1), TaskResult.Succeeded, "node20", null, null); + } + _ = w.BuildPendingOtlpJsonForTest(); + } + + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + + // --- emit (per-step record) --- + GC.Collect(); + GC.WaitForPendingFinalizers(); + var heapBefore = GC.GetTotalMemory(true); + var allocBefore = GC.GetAllocatedBytesForCurrentThread(); + var sw = Stopwatch.StartNew(); + for (var i = 0; i < n; i++) + { + exporter.RecordStepCompletion($"step {i}", i, start, start.AddSeconds(1), TaskResult.Succeeded, "node20", "actions/checkout", "v4"); + } + sw.Stop(); + var emitAllocPerStep = (GC.GetAllocatedBytesForCurrentThread() - allocBefore) / (double)n; + var emitNsPerStep = sw.Elapsed.TotalMilliseconds * 1_000_000.0 / n; + var retainedPerSpan = (GC.GetTotalMemory(false) - heapBefore) / (double)n; // spans held until flush + + // --- serialize (one OTLP/JSON build over all accumulated spans) --- + var swSer = Stopwatch.StartNew(); + var json = exporter.BuildPendingOtlpJsonForTest(); + swSer.Stop(); + var bytesPerSpan = json.Length / (double)n; + + _out.WriteLine( + $"n={n,6} | emit {emitNsPerStep,8:F0} ns/step | emitAlloc {emitAllocPerStep,8:F0} B/step | " + + $"retained {retainedPerSpan,8:F0} B/span | serialize {swSer.Elapsed.TotalMilliseconds,8:F2} ms | " + + $"payload {json.Length,9} B ({bytesPerSpan,6:F0} B/span)"); + + Assert.Equal(n, exporter.PendingSpanCountForTest); + } + } +} diff --git a/src/Test/L0/Worker/OTelTraceExporterL0.cs b/src/Test/L0/Worker/OTelTraceExporterL0.cs new file mode 100644 index 00000000000..3b0c3d4109f --- /dev/null +++ b/src/Test/L0/Worker/OTelTraceExporterL0.cs @@ -0,0 +1,1364 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Sdk; +using GitHub.Runner.Worker; +using Moq; +using Xunit; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class OTelTraceExporterL0 : IDisposable + { + private const string EndpointEnv = "ACTIONS_RUNNER_OTLP_ENDPOINT"; + private const string PropagateEnv = "ACTIONS_RUNNER_OTLP_PROPAGATE"; + private const string HeadersEnv = "ACTIONS_RUNNER_OTLP_HEADERS"; + private readonly string _originalEndpoint = Environment.GetEnvironmentVariable(EndpointEnv); + private readonly string _originalPropagate = Environment.GetEnvironmentVariable(PropagateEnv); + private readonly string _originalHeaders = Environment.GetEnvironmentVariable(HeadersEnv); + + public void Dispose() + { + Environment.SetEnvironmentVariable(EndpointEnv, _originalEndpoint); + Environment.SetEnvironmentVariable(PropagateEnv, _originalPropagate); + Environment.SetEnvironmentVariable(HeadersEnv, _originalHeaders); + } + + private OTelTraceExporter Enabled(TestHostContext hc, string endpoint = "http://localhost:4318") + { + // An enabled exporter builds a proxy-aware HttpClientHandler via the factory. + hc.SetSingleton(new HttpClientHandlerFactory()); + Environment.SetEnvironmentVariable(EndpointEnv, endpoint); + var exporter = new OTelTraceExporter(); + exporter.Initialize(hc); + return exporter; + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void StepPropagationEnv_DoesNotLeakAuthHeaders() + { + Environment.SetEnvironmentVariable(HeadersEnv, "authorization=Bearer super-secret-token"); + Environment.SetEnvironmentVariable(PropagateEnv, "true"); + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + + var env = exporter.StepPropagationEnv("Build", 1); + + // The trace context + endpoint are safe to hand to step processes. + Assert.True(env.ContainsKey("TRACEPARENT")); + Assert.True(env.ContainsKey("OTEL_EXPORTER_OTLP_ENDPOINT")); + // The collector auth header is a credential and must NOT reach user step env. + Assert.False(env.ContainsKey("OTEL_EXPORTER_OTLP_HEADERS")); + Assert.DoesNotContain(env.Values, v => v != null && v.Contains("super-secret-token")); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Initialize_ScrubsOtlpHeadersFromProcessEnv() + { + // Host step processes inherit the worker's env (ProcessInvoker copies it), + // so leaving the raw collector credential in the process env would hand it + // to every untrusted step — the exact leak StepPropagationEnv guards against. + Environment.SetEnvironmentVariable(HeadersEnv, "authorization=Bearer super-secret-token"); + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + Assert.Null(Environment.GetEnvironmentVariable(HeadersEnv)); + // ... and the exporter still captured the header before scrubbing. + Assert.True(exporter.IsEnabled); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Initialize_ScrubsOtlpHeaders_EvenWhenExportDisabled() + { + // A credential configured without an endpoint is inert for export but + // still a credential — it must not linger for step processes to inherit. + Environment.SetEnvironmentVariable(EndpointEnv, null); + Environment.SetEnvironmentVariable(HeadersEnv, "authorization=Bearer super-secret-token"); + using var hc = new TestHostContext(this); + var exporter = new OTelTraceExporter(); + exporter.Initialize(hc); + Assert.Null(Environment.GetEnvironmentVariable(HeadersEnv)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Metrics_CarryRunAttempt() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "2", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordStepCompletion("Build", 1, t, t.AddSeconds(2), TaskResult.Succeeded, "node20", null, null); + exporter.RecordJobCompletion(t, t.AddSeconds(9), TaskResult.Succeeded); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpMetricsJsonForTest()); + var metrics = doc.RootElement.GetProperty("resourceMetrics")[0].GetProperty("scopeMetrics")[0].GetProperty("metrics"); + foreach (var m in metrics.EnumerateArray()) + { + var name = m.GetProperty("name").GetString(); + var pts = m.TryGetProperty("histogram", out var h) ? h.GetProperty("dataPoints") + : m.GetProperty("sum").GetProperty("dataPoints"); + foreach (var dp in pts.EnumerateArray()) + { + var a = ReadAttrsFrom(dp); + Assert.True(a.ContainsKey("github.run_attempt"), $"{name} missing run.attempt"); + Assert.Equal("2", a["github.run_attempt"]); + } + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void DurationHistogram_HasExplicitBucketsNotEmptyBounds() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordJobCompletion(t, t.AddSeconds(9), TaskResult.Succeeded); // 9s -> (5,10] bucket + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpMetricsJsonForTest()); + var metrics = doc.RootElement.GetProperty("resourceMetrics")[0].GetProperty("scopeMetrics")[0].GetProperty("metrics"); + Assert.Equal("github.pipeline.job.duration", metrics[0].GetProperty("name").GetString()); + var dp = metrics[0].GetProperty("histogram").GetProperty("dataPoints")[0]; + var bounds = dp.GetProperty("explicitBounds"); + var buckets = dp.GetProperty("bucketCounts"); + Assert.True(bounds.GetArrayLength() > 0, "duration histogram must have explicit bounds (not empty)"); + Assert.Equal(bounds.GetArrayLength() + 1, buckets.GetArrayLength()); // OTLP histogram invariant + var total = 0; + foreach (var b in buckets.EnumerateArray()) total += int.Parse(b.GetString()); + Assert.Equal(1, total); // the single observation, bucketed + Assert.Equal(9.0, dp.GetProperty("sum").GetDouble()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Buffers_AreCappedToBoundMemory() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var over = OTelTraceExporter.MaxBufferedSpans + 50; + for (var i = 0; i < over; i++) + { + exporter.RecordStepCompletion($"step {i}", i, t, t.AddSeconds(1), TaskResult.Succeeded, "node20", null, null); + } + // A runaway job can't grow the buffer past the cap; excess is dropped + counted. + Assert.Equal(OTelTraceExporter.MaxBufferedSpans, exporter.PendingSpanCountForTest); + Assert.True(exporter.DroppedSpanCountForTest >= 50, $"expected >=50 dropped, got {exporter.DroppedSpanCountForTest}"); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void JobRootSpan_IsNeverDroppedAtBufferCap() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + for (var i = 0; i < OTelTraceExporter.MaxBufferedSpans; i++) + { + exporter.RecordStepCompletion($"step {i}", i, t, t.AddSeconds(1), TaskResult.Succeeded, "node20", null, null); + } + Assert.Equal(OTelTraceExporter.MaxBufferedSpans, exporter.PendingSpanCountForTest); + + // The job span is the trace ROOT and is recorded LAST. If the cap evicted + // it, every exported step span would parent to a span that never arrives. + exporter.RecordJobCompletion(t, t.AddSeconds(9), TaskResult.Succeeded); + + Assert.Equal(OTelTraceExporter.MaxBufferedSpans + 1, exporter.PendingSpanCountForTest); + Assert.Contains("\"spanId\":\"81606d47848a59c0\"", exporter.BuildPendingOtlpJsonForTest()); // job root present + } + + // Captures OTLP POSTs at the HttpClientHandler layer — exactly what the + // exporter's transport puts on the wire. + private sealed class CapturingClientHandler : HttpClientHandler + { + public readonly List<(Uri Uri, byte[] Body)> Requests = new(); + + protected override async System.Threading.Tasks.Task SendAsync(HttpRequestMessage request, CancellationToken ct) + { + Requests.Add((request.RequestUri, await request.Content.ReadAsByteArrayAsync(ct))); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") }; + } + } + + private static string GunzipString(byte[] body) + { + using var ms = new System.IO.MemoryStream(body); + using var gs = new System.IO.Compression.GZipStream(ms, System.IO.Compression.CompressionMode.Decompress); + using var sr = new System.IO.StreamReader(gs); + return sr.ReadToEnd(); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async System.Threading.Tasks.Task Flush_ChunksOversizedSignalsAcrossPosts() + { + using var hc = new TestHostContext(this); + var handler = new CapturingClientHandler(); + var factory = new Mock(); + factory.Setup(f => f.CreateClientHandler(It.IsAny())).Returns(handler); + hc.SetSingleton(factory.Object); + Environment.SetEnvironmentVariable(EndpointEnv, "http://localhost:4318"); + var exporter = new OTelTraceExporter(); + exporter.Initialize(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var total = OTelTraceExporter.MaxItemsPerPost + 1; + for (var i = 0; i < total; i++) + { + exporter.RecordStepCompletion($"step {i}", i, t, t.AddSeconds(1), TaskResult.Succeeded, "node20", null, null); + } + + await exporter.FlushAsync(default); + + // One giant POST can exceed a collector's body limit (413, non-retryable -> + // the whole signal silently dropped); bounded chunks keep every request small. + var tracePosts = handler.Requests.Where(r => r.Uri.AbsolutePath == "/v1/traces").ToList(); + Assert.Equal(2, tracePosts.Count); + var counts = tracePosts.Select(r => + { + using var doc = JsonDocument.Parse(GunzipString(r.Body)); + return doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("scopeSpans")[0].GetProperty("spans").GetArrayLength(); + }).ToList(); + Assert.Equal(total, counts.Sum()); // nothing lost across chunks + Assert.All(counts, c => Assert.InRange(c, 1, OTelTraceExporter.MaxItemsPerPost)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void GzipUtf8_RoundTripsAndCompresses() + { + // Repetitive payload like real OTLP/JSON spans. + var json = string.Concat(System.Linq.Enumerable.Repeat( + "{\"key\":\"cicd.pipeline.task.name\",\"value\":{\"stringValue\":\"Build\"}},", 500)); + var gz = OTelHttpTransport.GzipUtf8(json); + + Assert.True(gz.Length < System.Text.Encoding.UTF8.GetByteCount(json) / 2, "expected >2x compression"); + using var ms = new System.IO.MemoryStream(gz); + using var gs = new System.IO.Compression.GZipStream(ms, System.IO.Compression.CompressionMode.Decompress); + using var sr = new System.IO.StreamReader(gs); + Assert.Equal(json, sr.ReadToEnd()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void RecordSpan_HonorsClientSpanKind() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordSpan("pull image: alpine", "container", t, t.AddSeconds(1), null, spanKind: 3); // CLIENT + Assert.Equal(3, SpanAt(exporter, 0).GetProperty("kind").GetInt32()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Attributes_SerializeWithCorrectAnyValueWireTypes() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordSpan("pull image: alpine", "container", t, t.AddSeconds(1), new Dictionary + { + ["d"] = 0.75, // double -> doubleValue, NOT stringValue + ["f"] = 0.5f, // float -> doubleValue + ["b"] = true, + ["l"] = 42L, + ["s"] = "text", + ["arr"] = new object[] { "a", 1L, 2.5 }, // array -> arrayValue with typed elements + }); + + var raw = new Dictionary(); + foreach (var a in SpanAt(exporter, 0).GetProperty("attributes").EnumerateArray()) + { + raw[a.GetProperty("key").GetString()] = a.GetProperty("value").Clone(); + } + + // OTLP AnyValue: each CLR type must land on its own wire type — a double + // silently coerced to stringValue breaks numeric queries downstream. + Assert.Equal(0.75, raw["d"].GetProperty("doubleValue").GetDouble()); + Assert.Equal(0.5, raw["f"].GetProperty("doubleValue").GetDouble()); + Assert.True(raw["b"].GetProperty("boolValue").GetBoolean()); + Assert.Equal("42", raw["l"].GetProperty("intValue").GetString()); + Assert.Equal("text", raw["s"].GetProperty("stringValue").GetString()); + var values = raw["arr"].GetProperty("arrayValue").GetProperty("values"); + Assert.Equal(3, values.GetArrayLength()); + Assert.Equal("a", values[0].GetProperty("stringValue").GetString()); + Assert.Equal("1", values[1].GetProperty("intValue").GetString()); + Assert.Equal(2.5, values[2].GetProperty("doubleValue").GetDouble()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void DescribePartialSuccess_FlagsRejectedItems() + { + Assert.Null(OTelHttpTransport.DescribePartialSuccess("{}")); + Assert.Null(OTelHttpTransport.DescribePartialSuccess("{\"partialSuccess\":{}}")); + var s = OTelHttpTransport.DescribePartialSuccess("{\"partialSuccess\":{\"rejectedSpans\":\"3\",\"errorMessage\":\"bad batch\"}}"); + Assert.NotNull(s); + Assert.Contains("3 rejected", s); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Logs_HaveSchemaUrlAndObservedTime() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepLog("Run tests", 3, "Error", "boom"); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpLogsJsonForTest()); + var scopeLogs = doc.RootElement.GetProperty("resourceLogs")[0].GetProperty("scopeLogs")[0]; + // 1.34.0 is the earliest semconv release defining every emitted attribute + // (vcs.provider.name, cicd.worker.*, cicd.pipeline.task.run.result, ...); + // declaring an older schema would misdirect schema-aware transformations. + Assert.Equal("https://opentelemetry.io/schemas/1.34.0", scopeLogs.GetProperty("schemaUrl").GetString()); + var rec = scopeLogs.GetProperty("logRecords")[0]; + Assert.True(rec.TryGetProperty("observedTimeUnixNano", out _)); + } + + // ---- shared deterministic ID contract (golden values mirrored in otel-explorer) ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void TraceID_GoldenAndDeterministic() + { + Assert.Equal("acad1e2a107636235fd56bb742499bd0", OTelTraceExporter.NewTraceID(99999, 1)); + Assert.Equal(OTelTraceExporter.NewTraceID(99999, 1), OTelTraceExporter.NewTraceID(99999, 1)); + Assert.NotEqual(OTelTraceExporter.NewTraceID(99999, 1), OTelTraceExporter.NewTraceID(99999, 2)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void TraceID_CrossLanguageSha256() + { + // Trace ID = leading 16 bytes of SHA-256("99999-1"); otel-explorer truncates + // the same digest identically. (MD5 is avoided — it throws under FIPS.) + var full = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes("99999-1")); + var expectedHex = BitConverter.ToString(full, 0, 16).Replace("-", "").ToLowerInvariant(); + Assert.Equal(expectedHex, OTelTraceExporter.NewTraceID(99999, 1)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void DefaultAttemptZero_TreatedAsOne() + { + Assert.Equal(OTelTraceExporter.NewTraceID(99999, 1), OTelTraceExporter.NewTraceID(99999, 0)); + Assert.Equal(OTelTraceExporter.NewJobSpanID(99999, 1, "build"), OTelTraceExporter.NewJobSpanID(99999, 0, "build")); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SpanID_BigEndian_MatchesOtelExplorer() + { + Assert.Equal("000000000000002a", OTelTraceExporter.NewSpanID(42)); + Assert.Equal("000000000001869f", OTelTraceExporter.NewSpanID(99999)); // workflow span for run 99999 + Assert.Equal(16, OTelTraceExporter.NewSpanID(42).Length); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void JobSpanID_Golden() + { + Assert.Equal("81606d47848a59c0", OTelTraceExporter.NewJobSpanID(99999, 1, "build")); // sha256("job-99999-1-build")[:8] + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void StepSpanID_Golden() + { + Assert.Equal("7a4c67339b7bb8a7", OTelTraceExporter.NewStepSpanID(99999, 1, "build", 3, "Run tests")); // sha256("step-99999-1-build-3-Run tests")[:8] + } + + // ---- inbound W3C trace context -> job span link ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Traceparent_ParsesValidAndRejectsMalformed() + { + Assert.True(OTelTraceExporter.TryParseTraceparent( + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", out var t, out var s, out var f)); + Assert.Equal("0af7651916cd43dd8448eb211c80319c", t); + Assert.Equal("b7ad6b7169203331", s); + Assert.Equal(1, f); + // Unknown future version: tolerate, reading only the first four fields. + Assert.True(OTelTraceExporter.TryParseTraceparent( + "cc-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01-extra", out _, out _, out _)); + // Rejections. + Assert.False(OTelTraceExporter.TryParseTraceparent(null, out _, out _, out _)); + Assert.False(OTelTraceExporter.TryParseTraceparent("garbage", out _, out _, out _)); + Assert.False(OTelTraceExporter.TryParseTraceparent( // forbidden version ff + "ff-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", out _, out _, out _)); + Assert.False(OTelTraceExporter.TryParseTraceparent( // all-zero trace id + "00-00000000000000000000000000000000-b7ad6b7169203331-01", out _, out _, out _)); + Assert.False(OTelTraceExporter.TryParseTraceparent( // all-zero span id + "00-0af7651916cd43dd8448eb211c80319c-0000000000000000-01", out _, out _, out _)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void JobSpan_LinksToInboundParentContext() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + var prevTp = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_PARENT_TRACEPARENT"); + var prevTs = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_PARENT_TRACESTATE"); + try + { + Environment.SetEnvironmentVariable("ACTIONS_RUNNER_PARENT_TRACEPARENT", + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"); + Environment.SetEnvironmentVariable("ACTIONS_RUNNER_PARENT_TRACESTATE", "arc=abc123"); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordJobCompletion(DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpJsonForTest()); + var spans = doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("scopeSpans")[0].GetProperty("spans"); + JsonElement job = default; + var found = false; + foreach (var sp in spans.EnumerateArray()) + { + var a = ReadAttrs(sp); + if (a.TryGetValue("github.record_type", out var ty) && ty == "job") { job = sp; found = true; break; } + } + Assert.True(found); + // The runner's own trace stays deterministic; the upstream context is a LINK. + Assert.Equal("acad1e2a107636235fd56bb742499bd0", job.GetProperty("traceId").GetString()); + var links = job.GetProperty("links"); + Assert.Equal(1, links.GetArrayLength()); + Assert.Equal("0af7651916cd43dd8448eb211c80319c", links[0].GetProperty("traceId").GetString()); + Assert.Equal("b7ad6b7169203331", links[0].GetProperty("spanId").GetString()); + Assert.Equal("arc=abc123", links[0].GetProperty("traceState").GetString()); + } + finally + { + Environment.SetEnvironmentVariable("ACTIONS_RUNNER_PARENT_TRACEPARENT", prevTp); + Environment.SetEnvironmentVariable("ACTIONS_RUNNER_PARENT_TRACESTATE", prevTs); + } + } + + // ---- enable / disable ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void IsEnabled_FalseByDefault() + { + using var hc = new TestHostContext(this); + Environment.SetEnvironmentVariable(EndpointEnv, null); + var exporter = new OTelTraceExporter(); + exporter.Initialize(hc); + Assert.False(exporter.IsEnabled); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void IsEnabled_TrueWhenEndpointSet() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + Assert.True(exporter.IsEnabled); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void FeatureFlagOff_DisablesEvenWithEndpoint() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com", featureEnabled: false); + Assert.False(exporter.IsEnabled); + exporter.RecordStepCompletion("Run tests", 3, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, "node20", "actions/checkout", "v4"); + Assert.Equal(0, exporter.PendingSpanCountForTest); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Disabled_RecordsNothing() + { + using var hc = new TestHostContext(this); + Environment.SetEnvironmentVariable(EndpointEnv, null); + var exporter = new OTelTraceExporter(); + exporter.Initialize(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepCompletion("Run tests", 3, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, "node20", "actions/checkout", "v4"); + Assert.Equal(0, exporter.PendingSpanCountForTest); + } + + // ---- step span content + semconv ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void StepSpan_LinksToJobAndCarriesSemconv() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var start = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordStepCompletion("Run tests", 3, start, start.AddSeconds(2), TaskResult.Succeeded, "node20", "actions/checkout", "v4"); + + Assert.Equal(1, exporter.PendingSpanCountForTest); + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpJsonForTest()); + var span = doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("scopeSpans")[0].GetProperty("spans")[0]; + + Assert.Equal("acad1e2a107636235fd56bb742499bd0", span.GetProperty("traceId").GetString()); + Assert.Equal("7a4c67339b7bb8a7", span.GetProperty("spanId").GetString()); + Assert.Equal("81606d47848a59c0", span.GetProperty("parentSpanId").GetString()); // job span + Assert.Equal("Run tests", span.GetProperty("name").GetString()); + + var attrs = ReadAttrs(span); + Assert.Equal("step", attrs["github.record_type"]); + Assert.False(attrs.ContainsKey("source")); // runner identified by scope, not a custom attr + Assert.Equal("Run tests", attrs["cicd.pipeline.task.name"]); + Assert.Equal("success", attrs["cicd.pipeline.task.run.result"]); + Assert.Equal("7a4c67339b7bb8a7", attrs["cicd.pipeline.task.run.id"]); + Assert.True(attrs.ContainsKey("cicd.pipeline.task.run.url.full")); + Assert.Equal("https://github.com/octo/repo", attrs["vcs.repository.url.full"]); + Assert.Equal("3", attrs["github.step_number"]); + Assert.Equal("actions/checkout", attrs["github.action"]); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void StepSpan_FailureSetsErrorAndStatus() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepCompletion("Run tests", 3, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Failed, null, null, null); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpJsonForTest()); + var span = doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("scopeSpans")[0].GetProperty("spans")[0]; + + Assert.Equal(2, span.GetProperty("status").GetProperty("code").GetInt32()); // ERROR + var attrs = ReadAttrs(span); + Assert.Equal("failure", attrs["github.conclusion"]); + Assert.Equal("failure", attrs["cicd.pipeline.task.run.result"]); + Assert.Equal("failure", attrs["error.type"]); + } + + // ---- job span ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void JobSpan_IsRoot_NoDanglingParent() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordJobCompletion(DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpJsonForTest()); + var span = doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("scopeSpans")[0].GetProperty("spans")[0]; + + Assert.Equal("81606d47848a59c0", span.GetProperty("spanId").GetString()); // job + // The job is the root of the runner's trace: no parentSpanId pointing at a + // workflow/run span the runner never emits (that was a dangling parent). + Assert.False(span.TryGetProperty("parentSpanId", out _)); + Assert.Equal("job", ReadAttrs(span)["github.record_type"]); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void EmbeddedStep_IsNotRecorded() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + // Composite/embedded sub-steps would collide by name and have no API counterpart. + exporter.RecordStepCompletion("Build", 1, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, null, null, null, isEmbedded: true); + Assert.Equal(0, exporter.PendingSpanCountForTest); + + // ... but a top-level step with the same name still records. + exporter.RecordStepCompletion("Build", 1, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, null, null, null, isEmbedded: false); + Assert.Equal(1, exporter.PendingSpanCountForTest); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Json_ControlCharsInName_ProducesValidJson() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + // A raw control char + quote + backslash must not break the batch. + exporter.RecordStepCompletion("step\u0001name", 1, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, null, null, null); + + // Parses without throwing == valid JSON, and round-trips the name. + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpJsonForTest()); + var span = doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("scopeSpans")[0].GetProperty("spans")[0]; + Assert.Equal("step\u0001name", span.GetProperty("name").GetString()); + } + + // ---- best-effort resilience ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async System.Threading.Tasks.Task Flush_UnreachableEndpoint_DoesNotThrow() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc, "http://127.0.0.1:0"); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepCompletion("Run tests", 3, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, null, null, null); + + var ex = await Record.ExceptionAsync(() => exporter.FlushAsync(default)); + Assert.Null(ex); + // Failure is not silent: one end-of-job summary (logged at Warning) says + // what was lost, so a fleet operator can grep it instead of correlating + // per-signal Info lines across _diag files. + Assert.NotNull(exporter.LastFlushSummaryForTest); + Assert.Contains("failed", exporter.LastFlushSummaryForTest); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async System.Threading.Tasks.Task Flush_HangingCollector_BoundedByOverallDeadline() + { + // Worst case for job-end latency: a collector that ACCEPTS the TCP + // connection and then stalls (slow-loris). Nothing fast-fails, so only + // the 4 s overall flush deadline bounds the wait — measure that it does. + using var listener = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0); + listener.Start(); + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + var held = new List(); + _ = System.Threading.Tasks.Task.Run(async () => + { + while (true) + { + held.Add(await listener.AcceptTcpClientAsync()); // accept, never respond + } + }); + + using var hc = new TestHostContext(this); + var exporter = Enabled(hc, $"http://127.0.0.1:{port}"); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepCompletion("Run tests", 3, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, null, null, null); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var ex = await Record.ExceptionAsync(() => exporter.FlushAsync(default)); + sw.Stop(); + + Assert.Null(ex); + // Hard bound: 4 s overall deadline (+ scheduling slack). This is the + // measured worst-case job-end delay a hung collector can cause. + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(6), $"flush took {sw.Elapsed}, expected < 6 s"); + Assert.NotNull(exporter.LastFlushSummaryForTest); + Assert.Contains("failed", exporter.LastFlushSummaryForTest); + listener.Stop(); + held.ForEach(c => c.Dispose()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void ExportSummary_NullWhenCleanAndDescriptiveOnLoss() + { + // Clean flush -> no summary line at all. + Assert.Null(OTelTraceExporter.BuildExportSummary(3, 0, 0, 0, 0)); + // Failed POSTs and cap-drops each produce the single summary. + var s = OTelTraceExporter.BuildExportSummary(3, 2, 7, 0, 1); + Assert.Contains("2/3", s); + Assert.Contains("7 span(s)", s); + Assert.Contains("1 task-metric(s)", s); + Assert.NotNull(OTelTraceExporter.BuildExportSummary(3, 0, 0, 5, 0)); // drops alone warn too + } + + // ---- TLS: custom CA trust (the safe primitive for self-signed collectors) ---- + + private static System.Security.Cryptography.X509Certificates.X509Certificate2 NewSelfSignedCert(string cn) + { + using var rsa = System.Security.Cryptography.RSA.Create(2048); + var req = new System.Security.Cryptography.X509Certificates.CertificateRequest( + $"CN={cn}", rsa, System.Security.Cryptography.HashAlgorithmName.SHA256, + System.Security.Cryptography.RSASignaturePadding.Pkcs1); + return req.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(1)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void CustomCaTrust_AcceptsOnlyCertsFromTheBundle() + { + using var collectorCert = NewSelfSignedCert("collector.internal"); + using var otherCert = NewSelfSignedCert("mitm.example"); + var trusted = new System.Security.Cryptography.X509Certificates.X509Certificate2Collection { collectorCert }; + + // The bundled CA (here: the self-signed collector cert itself) validates ... + Assert.True(OTelTraceExporter.ValidateWithCustomTrustRoots(collectorCert, trusted)); + // ... any other cert — e.g. a MITM's — does not. + Assert.False(OTelTraceExporter.ValidateWithCustomTrustRoots(otherCert, trusted)); + Assert.False(OTelTraceExporter.ValidateWithCustomTrustRoots(null, trusted)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Initialize_LoadsCustomCaBundle() + { + var caFile = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"otel-ca-{Guid.NewGuid():N}.pem"); + var prev = Environment.GetEnvironmentVariable("ACTIONS_RUNNER_OTLP_CERTIFICATE"); + try + { + using var cert = NewSelfSignedCert("collector.internal"); + System.IO.File.WriteAllText(caFile, cert.ExportCertificatePem()); + Environment.SetEnvironmentVariable("ACTIONS_RUNNER_OTLP_CERTIFICATE", caFile); + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + Assert.True(exporter.IsEnabled); // a valid PEM CA bundle loads cleanly + } + finally + { + Environment.SetEnvironmentVariable("ACTIONS_RUNNER_OTLP_CERTIFICATE", prev); + System.IO.File.Delete(caFile); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Initialize_Failure_DisablesExportInsteadOfThrowing() + { + // Telemetry must never throw into the job path: Initialize runs before + // JobRunner's try block, so a failure here must disable export, not escape. + using var hc = new TestHostContext(this); + var factory = new Mock(); + factory.Setup(f => f.CreateClientHandler(It.IsAny())) + .Throws(new InvalidOperationException("boom")); + hc.SetSingleton(factory.Object); + Environment.SetEnvironmentVariable(EndpointEnv, "http://localhost:4318"); + Environment.SetEnvironmentVariable(PropagateEnv, "true"); + + var exporter = new OTelTraceExporter(); + Assert.Null(Record.Exception(() => exporter.Initialize(hc))); + Assert.False(exporter.IsEnabled); + + // Every later hook is a safe no-op on the disabled exporter. + Assert.Null(Record.Exception(() => + { + exporter.SetResource("my-runner", "42", "default", "2.333.0", "Linux", "X64", "host-1", ephemeral: false); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepCompletion("Build", 1, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, "node20", null, null); + })); + Assert.Empty(exporter.StepPropagationEnv("Build", 1)); + Assert.Equal(0, exporter.PendingSpanCountForTest); + } + + // ---- secret masking (uses the host's real SecretMasker) ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SecretMasker_ScrubsSpanStrings() + { + using var hc = new TestHostContext(this); + hc.SecretMasker.AddValue("s3cr3t"); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepCompletion("deploy s3cr3t", 3, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, null, "octo/deploy s3cr3t", null); + + var json = exporter.BuildPendingOtlpJsonForTest(); + Assert.DoesNotContain("s3cr3t", json); + Assert.Contains("***", json); + } + + // ---- resource (runner info) ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Resource_CarriesRunnerInfo() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetResource("my-runner", "42", "default", "2.333.0", "Linux", "X64", "host-1", ephemeral: true); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordJobCompletion(DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpJsonForTest()); + var resourceAttrs = ReadAttrsFrom(doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("resource")); + Assert.Equal("github-actions-runner", resourceAttrs["service.name"]); + Assert.Equal("my-runner", resourceAttrs["cicd.worker.name"]); // semconv (no github.runner.name dupe) + Assert.False(resourceAttrs.ContainsKey("github.runner.name")); + Assert.Equal("2.333.0", resourceAttrs["service.version"]); + Assert.Equal("linux", resourceAttrs["os.type"]); // semconv enum value + Assert.Equal("amd64", resourceAttrs["host.arch"]); // semconv enum value + Assert.Equal("true", resourceAttrs["github.runner.ephemeral"]); // boolValue + Assert.Equal("agent", resourceAttrs["cicd.system.component"]); // semconv: runner is the agent + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Resource_OsTypeAndHostArch_UseSemconvEnumValues() + { + // semconv os.type enum is lowercase and has no "macOS" (Apple is "darwin"); + // host.arch enum is amd64|arm32|arm64|x86 — not the runner's "X64"/"ARM" names. + Assert.Equal("linux", OTelTraceExporter.ToSemconvOsType("Linux")); + Assert.Equal("darwin", OTelTraceExporter.ToSemconvOsType("macOS")); + Assert.Equal("windows", OTelTraceExporter.ToSemconvOsType("Windows")); + Assert.Equal("amd64", OTelTraceExporter.ToSemconvHostArch("X64")); + Assert.Equal("x86", OTelTraceExporter.ToSemconvHostArch("X86")); + Assert.Equal("arm32", OTelTraceExporter.ToSemconvHostArch("ARM")); + Assert.Equal("arm64", OTelTraceExporter.ToSemconvHostArch("ARM64")); + + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetResource("my-runner", "42", "default", "2.333.0", "macOS", "ARM64", "host-1", ephemeral: false); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordJobCompletion(DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpJsonForTest()); + var resourceAttrs = ReadAttrsFrom(doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("resource")); + Assert.Equal("darwin", resourceAttrs["os.type"]); + Assert.Equal("arm64", resourceAttrs["host.arch"]); + Assert.Equal("macOS", resourceAttrs["os.name"]); // raw value preserved (os.name is free-form) + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Resource_MergesOtelResourceAttributesEnv() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + var prev = Environment.GetEnvironmentVariable("OTEL_RESOURCE_ATTRIBUTES"); + try + { + // Standard OTel env var (spec): comma-separated, values percent-encoded. + // This is how ARC/Downward-API attaches k8s.* — no runner-specific env names. + Environment.SetEnvironmentVariable("OTEL_RESOURCE_ATTRIBUTES", + "k8s.pod.name=runner-abc123,k8s.namespace.name=actions,k8s.node.name=node%2D3"); + exporter.SetResource("my-runner", "42", "default", "2.333.0", "Linux", "X64", "host-1", ephemeral: true); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordJobCompletion(DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpJsonForTest()); + var resourceAttrs = ReadAttrsFrom(doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("resource")); + Assert.Equal("runner-abc123", resourceAttrs["k8s.pod.name"]); + Assert.Equal("actions", resourceAttrs["k8s.namespace.name"]); + Assert.Equal("node-3", resourceAttrs["k8s.node.name"]); // percent-decoded + // Explicitly-set keys win over the env var. + Assert.Equal("github-actions-runner", resourceAttrs["service.name"]); + } + finally + { + Environment.SetEnvironmentVariable("OTEL_RESOURCE_ATTRIBUTES", prev); + } + } + + // ---- enrichments (#13): vcs, actor, run url, throttling ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Span_CarriesVcsAndActorContext() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com", + sha: "abc123", refName: "main", actor: "octocat"); + exporter.RecordStepCompletion("Run tests", 3, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, null, null, null); + + var attrs = ReadAttrs(SpanAt(exporter, 0)); + Assert.Equal("abc123", attrs["vcs.ref.head.revision"]); // semconv name (not vcs.revision) + Assert.Equal("main", attrs["vcs.ref.head.name"]); + Assert.Equal("github", attrs["vcs.provider.name"]); + Assert.Equal("octo", attrs["vcs.owner.name"]); + Assert.Equal("repo", attrs["vcs.repository.name"]); + Assert.Equal("octocat", attrs["github.actor"]); + Assert.Equal("https://github.com/octo/repo/actions/runs/99999/attempts/1", attrs["cicd.pipeline.run.url.full"]); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SemconvResult_SkippedMapsToSkipEnum() + { + // semconv cicd.pipeline.task.run.result enum uses "skip", not "skipped". + Assert.Equal("skip", OTelTraceExporter.ToSemconvResult("skipped")); + Assert.Equal("cancellation", OTelTraceExporter.ToSemconvResult("cancelled")); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Resource_HasCicdWorkerAndSchemaUrl() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetResource("my-runner", "42", "default", "2.333.0", "Linux", "X64", "host-1", ephemeral: true); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordJobCompletion(DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded); + + var json = exporter.BuildPendingOtlpJsonForTest(); + Assert.Contains("https://opentelemetry.io/schemas/", json); // schema_url declared + using var doc = JsonDocument.Parse(json); + var rs = doc.RootElement.GetProperty("resourceSpans")[0]; + var resAttrs = ReadAttrsFrom(rs.GetProperty("resource")); + Assert.Equal("my-runner", resAttrs["cicd.worker.name"]); + Assert.Equal("42", resAttrs["cicd.worker.id"]); + Assert.Equal("2.333.0", rs.GetProperty("scopeSpans")[0].GetProperty("scope").GetProperty("version").GetString()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void JobSpan_CarriesThrottlingDelay() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordJobCompletion(DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded, throttlingDelayMs: 4200); + + Assert.Equal("4200", ReadAttrs(SpanAt(exporter, 0))["github.throttling_delay_ms"]); + } + + // ---- generic child spans (#14) ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void RecordSpan_ParentsToJobAndCarriesType() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordSpan("Resolve actions/checkout@v4", "action_download", t, t.AddSeconds(1), + new System.Collections.Generic.Dictionary + { + ["github.action"] = "actions/checkout", + ["cicd.pipeline.task.run.result"] = "success", + }); + + var span = SpanAt(exporter, 0); + Assert.Equal("81606d47848a59c0", span.GetProperty("parentSpanId").GetString()); // job span (no parent step given) + var attrs = ReadAttrs(span); + Assert.Equal("action_download", attrs["github.record_type"]); + Assert.Equal("actions/checkout", attrs["github.action"]); + // Action-resolution spans now carry a result like every other task span. + Assert.Equal("success", attrs["cicd.pipeline.task.run.result"]); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void RecordSpan_ParentsToStep_WhenStepGiven() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + // Action resolution happens during "Set up job", so the span nests under that step. + exporter.RecordSpan("Resolve actions/checkout@v4", "action_download", t, t.AddSeconds(1), + new System.Collections.Generic.Dictionary { ["github.action"] = "actions/checkout" }, + parentStepName: "Set up job", parentStepNumber: 1); + + var span = SpanAt(exporter, 0); + var expected = OTelTraceExporter.NewStepSpanID(99999, 1, "build", 1, "Set up job"); + Assert.Equal(expected, span.GetProperty("parentSpanId").GetString()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void RecordSpan_ProcessExitCode_IsSemconvIntAttribute() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + // Same shape as the ScriptHandler/ContainerOperationProvider hook sites: + // the registered semconv key is process.exit.code (dots, type int) — a + // negative exit code must serialize culture-invariantly as an intValue. + exporter.RecordSpan("process: bash", "process", t, t.AddSeconds(1), + new System.Collections.Generic.Dictionary + { + ["process.executable.name"] = "bash", + ["process.exit.code"] = (long)-1, + }); + + var span = SpanAt(exporter, 0); + var found = false; + foreach (var a in span.GetProperty("attributes").EnumerateArray()) + { + if (a.GetProperty("key").GetString() == "process.exit.code") + { + found = true; + // OTLP/JSON int64 is a string-encoded intValue, never a stringValue. + Assert.Equal("-1", a.GetProperty("value").GetProperty("intValue").GetString()); + Assert.False(a.GetProperty("value").TryGetProperty("stringValue", out _)); + } + } + Assert.True(found, "expected process.exit.code attribute"); + } + + // ---- trace-context propagation (#15) ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void StepPropagationEnv_EmptyUnlessOptedIn() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); // propagate flag not set + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + Assert.Empty(exporter.StepPropagationEnv("Run tests", 3)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void StepPropagationEnv_TraceparentMatchesStepSpan() + { + using var hc = new TestHostContext(this); + Environment.SetEnvironmentVariable(PropagateEnv, "true"); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + + var env = exporter.StepPropagationEnv("Run tests", 3); + // 00-{trace}-{step span}-01, matching the step span IDs the exporter emits. + Assert.Equal("00-acad1e2a107636235fd56bb742499bd0-7a4c67339b7bb8a7-01", env["TRACEPARENT"]); + Assert.Equal("http://localhost:4318", env["OTEL_EXPORTER_OTLP_ENDPOINT"]); + Assert.Contains("github.run_id=99999", env["OTEL_RESOURCE_ATTRIBUTES"]); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void StripUserInfo_RemovesUrlCredentials() + { + Assert.Equal("http://localhost:4318", OTelTraceExporter.StripUserInfo("http://localhost:4318")); + Assert.Equal("http://localhost:4318", OTelTraceExporter.StripUserInfo("http://user:t0ps3cret@localhost:4318")); + Assert.Equal("https://otlp.example.com/custom", OTelTraceExporter.StripUserInfo("https://user@otlp.example.com/custom")); + Assert.Equal("not a url", OTelTraceExporter.StripUserInfo("not a url")); // pass-through + Assert.Null(OTelTraceExporter.StripUserInfo(null)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void StepPropagationEnv_StripsEndpointUserinfo() + { + // Basic-auth-in-URL collectors put a credential in the endpoint itself; + // it must never reach untrusted step processes (headers are already withheld). + Environment.SetEnvironmentVariable(PropagateEnv, "true"); + using var hc = new TestHostContext(this); + var exporter = Enabled(hc, "http://runner:t0ps3cret@localhost:4318"); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + + var env = exporter.StepPropagationEnv("Run tests", 3); + Assert.Equal("http://localhost:4318", env["OTEL_EXPORTER_OTLP_ENDPOINT"]); + Assert.DoesNotContain(env.Values, v => v != null && v.Contains("t0ps3cret")); + + // ... and the credential is registered with the masker so the endpoint + // can never appear raw in diag/transport logs. + Assert.DoesNotContain("t0ps3cret", hc.SecretMasker.MaskSecrets("posting to http://runner:t0ps3cret@localhost:4318/v1/traces")); + } + + // ---- OTel logs (#16) ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void RecordStepLog_CorrelatesToStepSpan() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepLog("Run tests", 3, "Error", "boom: test failed"); + + Assert.Equal(1, exporter.PendingLogCountForTest); + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpLogsJsonForTest()); + var rec = doc.RootElement.GetProperty("resourceLogs")[0].GetProperty("scopeLogs")[0].GetProperty("logRecords")[0]; + Assert.Equal("acad1e2a107636235fd56bb742499bd0", rec.GetProperty("traceId").GetString()); + Assert.Equal("7a4c67339b7bb8a7", rec.GetProperty("spanId").GetString()); // same as step span + Assert.Equal("ERROR", rec.GetProperty("severityText").GetString()); + Assert.Equal(17, rec.GetProperty("severityNumber").GetInt32()); + Assert.Equal("boom: test failed", rec.GetProperty("body").GetProperty("stringValue").GetString()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void StepSpan_CarriesActionStage() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordStepCompletion("Post Checkout", 14, t, t.AddSeconds(1), TaskResult.Succeeded, "node20", "actions/checkout", "v4", stepStage: "Post"); + + Assert.Equal("Post", ReadAttrs(SpanAt(exporter, 0))["github.action_stage"]); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void RecordJobLog_CorrelatesToJobSpan() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordJobLog("Warning", "the runner is shutting down"); + + Assert.Equal(1, exporter.PendingLogCountForTest); + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpLogsJsonForTest()); + var rec = doc.RootElement.GetProperty("resourceLogs")[0].GetProperty("scopeLogs")[0].GetProperty("logRecords")[0]; + Assert.Equal("81606d47848a59c0", rec.GetProperty("spanId").GetString()); // same as the job span + Assert.Equal("WARNING", rec.GetProperty("severityText").GetString()); + Assert.Equal("the runner is shutting down", rec.GetProperty("body").GetProperty("stringValue").GetString()); + } + + // ---- exception events + PR/task.type (#17/#18) ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void FailedStep_EmitsExceptionEvent_AndStatusMessage() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepCompletion("Run tests", 3, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Failed, null, null, null, + errorMessage: "assertion failed: 2 != 3"); + + var span = SpanAt(exporter, 0); + var ev = span.GetProperty("events")[0]; + Assert.Equal("exception", ev.GetProperty("name").GetString()); + var attrs = ReadAttrsFrom(ev); + // A CI conclusion is not an exception class; semconv exception.type must be + // a real type or omitted (message-only exception events are valid). + Assert.False(attrs.ContainsKey("exception.type")); + Assert.Equal("assertion failed: 2 != 3", attrs["exception.message"]); + // Recording Errors guidance: the error description rides on span status. + var status = span.GetProperty("status"); + Assert.Equal(2, status.GetProperty("code").GetInt32()); + Assert.Equal("assertion failed: 2 != 3", status.GetProperty("message").GetString()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void FailedStep_WithoutErrorMessage_HasNoEmptyExceptionEvent() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordStepCompletion("Run tests", 3, DateTime.UtcNow, DateTime.UtcNow, TaskResult.Failed, null, null, null); + + var span = SpanAt(exporter, 0); + // An exception event with neither type nor message is invalid semconv. + Assert.False(span.TryGetProperty("events", out _)); + var status = span.GetProperty("status"); + Assert.Equal(2, status.GetProperty("code").GetInt32()); + Assert.False(status.TryGetProperty("message", out _)); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void TaskType_InferredFromName() + { + Assert.Equal("test", OTelTraceExporter.InferTaskType("Run unit tests", null)); + Assert.Equal("deploy", OTelTraceExporter.InferTaskType("Release to prod", null)); + Assert.Equal("build", OTelTraceExporter.InferTaskType("Compile", null)); + Assert.Null(OTelTraceExporter.InferTaskType("Greet", "actions/checkout")); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void PullRequest_EmitsVcsChangeAndBase() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "pull_request", "https://github.com", + sha: "abc", refName: "feature", actor: "octocat", baseRef: "main", changeId: "42"); + exporter.RecordJobCompletion(DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded); + + var attrs = ReadAttrs(SpanAt(exporter, 0)); + Assert.Equal("42", attrs["vcs.change.id"]); + Assert.Equal("main", attrs["vcs.ref.base.name"]); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void JobAndStepSpans_AreInternalTaskRuns() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordStepCompletion("Build", 1, t, t.AddSeconds(2), TaskResult.Succeeded, "node20", null, null); + exporter.RecordJobCompletion(t, t.AddSeconds(9), TaskResult.Succeeded); + + // cicd-spans: task-run spans SHOULD be INTERNAL. Both the job and its steps + // carry cicd.pipeline.task.* attributes — they are task runs; the SERVER + // pipeline-run span belongs to the API-side consumer, not the runner. + Assert.Equal(1, SpanAt(exporter, 0).GetProperty("kind").GetInt32()); // step + Assert.Equal(1, SpanAt(exporter, 1).GetProperty("kind").GetInt32()); // job + } + + // ---- metrics (#1) ---- + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void JobMetrics_DurationHistogramAndErrorCounter() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var start = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordJobCompletion(start, start.AddSeconds(9), TaskResult.Failed); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpMetricsJsonForTest()); + var metrics = doc.RootElement.GetProperty("resourceMetrics")[0].GetProperty("scopeMetrics")[0].GetProperty("metrics"); + var dur = metrics[0]; + // Vendor name: the worker observes one JOB's wall time, not the pipeline + // run's, so semconv cicd.pipeline.run.duration would be semantically wrong. + Assert.Equal("github.pipeline.job.duration", dur.GetProperty("name").GetString()); + var dp = dur.GetProperty("histogram").GetProperty("dataPoints")[0]; + Assert.Equal(9.0, dp.GetProperty("sum").GetDouble()); + var durAttrs = ReadAttrsFrom(dp); + Assert.Equal("failure", durAttrs["cicd.pipeline.result"]); + Assert.Equal("CI", durAttrs["cicd.pipeline.name"]); + + var err = metrics[1]; + // semconv name: a failed job IS "an error encountered in a pipeline run". + Assert.Equal("cicd.pipeline.run.errors", err.GetProperty("name").GetString()); + var errDp = err.GetProperty("sum").GetProperty("dataPoints")[0]; + Assert.Equal("1", errDp.GetProperty("asInt").GetString()); + var errAttrs = ReadAttrsFrom(errDp); + Assert.Equal("CI", errAttrs["cicd.pipeline.name"]); // required by semconv + Assert.Equal("failure", errAttrs["error.type"]); // required by semconv + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void JobMetrics_SuccessHasNoErrorCounter() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + exporter.RecordJobCompletion(DateTime.UtcNow, DateTime.UtcNow, TaskResult.Succeeded); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpMetricsJsonForTest()); + var metrics = doc.RootElement.GetProperty("resourceMetrics")[0].GetProperty("scopeMetrics")[0].GetProperty("metrics"); + // job.duration + the job's task.duration (no errors counter on success). + Assert.Equal(2, metrics.GetArrayLength()); + Assert.Equal("github.pipeline.job.duration", metrics[0].GetProperty("name").GetString()); + Assert.Equal("github.pipeline.task.duration", metrics[1].GetProperty("name").GetString()); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void TaskMetrics_DurationPerStepAndJob() + { + using var hc = new TestHostContext(this); + var exporter = Enabled(hc); + exporter.SetJobInfo("99999", "1", "build", "build", "octo/repo", "CI", "push", "https://github.com"); + var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + exporter.RecordStepCompletion("Build", 1, t, t.AddSeconds(2), TaskResult.Succeeded, "node20", null, null); + exporter.RecordStepCompletion("Unit tests", 2, t.AddSeconds(2), t.AddSeconds(5), TaskResult.Failed, "node20", null, null); + exporter.RecordJobCompletion(t, t.AddSeconds(9), TaskResult.Failed); + + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpMetricsJsonForTest()); + var metrics = doc.RootElement.GetProperty("resourceMetrics")[0].GetProperty("scopeMetrics")[0].GetProperty("metrics"); + JsonElement taskDur = default; var found = false; + foreach (var m in metrics.EnumerateArray()) + { + if (m.GetProperty("name").GetString() == "github.pipeline.task.duration") { taskDur = m; found = true; } + } + Assert.True(found, "expected github.pipeline.task.duration metric"); + + var byName = new System.Collections.Generic.Dictionary(); + foreach (var dp in taskDur.GetProperty("histogram").GetProperty("dataPoints").EnumerateArray()) + { + var a = ReadAttrsFrom(dp); + byName[a["cicd.pipeline.task.name"]] = (a["github.record_type"], a["cicd.pipeline.task.run.result"], dp.GetProperty("sum").GetDouble()); + } + Assert.Equal(3, byName.Count); // 2 steps + the job + Assert.Equal(("step", "success", 2.0), byName["Build"]); + Assert.Equal(("step", "failure", 3.0), byName["Unit tests"]); + Assert.Equal(("job", "failure", 9.0), byName["build"]); + } + + // ---- helpers ---- + + private static JsonElement SpanAt(OTelTraceExporter exporter, int i) + { + using var doc = JsonDocument.Parse(exporter.BuildPendingOtlpJsonForTest()); + return doc.RootElement.GetProperty("resourceSpans")[0].GetProperty("scopeSpans")[0].GetProperty("spans")[i].Clone(); + } + + + private static System.Collections.Generic.Dictionary ReadAttrs(JsonElement span) => ReadAttrsFrom(span); + + private static System.Collections.Generic.Dictionary ReadAttrsFrom(JsonElement parent) + { + var dict = new System.Collections.Generic.Dictionary(); + foreach (var a in parent.GetProperty("attributes").EnumerateArray()) + { + var key = a.GetProperty("key").GetString(); + var val = a.GetProperty("value"); + string str = val.TryGetProperty("stringValue", out var sv) ? sv.GetString() + : val.TryGetProperty("intValue", out var iv) ? iv.GetString() + : val.TryGetProperty("boolValue", out var bv) ? (bv.GetBoolean() ? "true" : "false") + : null; + dict[key] = str; + } + return dict; + } + } +}