Feature/surface pod errors

controllers/actions.github.com/constants.go

@@ -83,3 +83,20 @@ const (
 	ReasonTooManyPodFailures = "TooManyPodFailures"
 	ReasonInvalidPodFailure  = "InvalidPod"
 )
+
+// Container waiting state reasons that indicate unrecoverable pod errors.
+// These are surfaced in EphemeralRunner.Status.Message for easier diagnosis.
+var terminalContainerWaitingReasons = map[string]string{
+	"ImagePullBackOff":           "ImagePull",
+	"ErrImagePull":               "ImagePull",
+	"InvalidImageName":           "ImagePull",
+	"RegistryUnavailable":        "ImagePull",
+	"CreateContainerConfigError": "ContainerConfig",
+	"CrashLoopBackOff":           "ContainerCrash",
+	"RunContainerError":          "ContainerCrash",
+}
+
+// Container terminated reasons that indicate unrecoverable pod errors.
+var terminalContainerTerminatedReasons = map[string]string{
+	"OOMKilled": "ResourceLimit",
+}

controllers/actions.github.com/ephemeralrunner_controller.go

@@ -32,6 +32,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/events"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -46,8 +47,10 @@ const (
 // EphemeralRunnerReconciler reconciles a EphemeralRunner object
 type EphemeralRunnerReconciler struct {
 	client.Client
-	Log    logr.Logger
-	Scheme *runtime.Scheme
+	Log             logr.Logger
+	Scheme          *runtime.Scheme
+	Recorder        events.EventRecorder
+	GitHubAnnotator GitHubAnnotator
 	ResourceBuilder
 }
 
@@ -232,6 +235,13 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 
 	if len(ephemeralRunner.Status.Failures) > maxFailures {
 		log.Info(fmt.Sprintf("EphemeralRunner has failed more than %d times. Deleting ephemeral runner so it can be re-created", maxFailures))
+		msg := fmt.Sprintf("Runner pod failed %d times and will not be retried", maxFailures)
+		if ephemeralRunner.Status.Message != "" {
+			msg = fmt.Sprintf("%s. Last error: %s", msg, ephemeralRunner.Status.Message)
+		}
+		r.Recorder.Eventf(&ephemeralRunner, nil, corev1.EventTypeWarning, ReasonTooManyPodFailures, "MaxRetriesExceeded", msg)
+		annotateRunnerFailure(ctx, r.GitHubAnnotator, &ephemeralRunner, log)
+
 		if err := r.Delete(ctx, &ephemeralRunner); err != nil {
 			log.Error(fmt.Errorf("failed to delete ephemeral runner after %d failures: %w", maxFailures, err), "Failed to delete ephemeral runner")
 			return ctrl.Result{}, err
@@ -365,11 +375,28 @@ func (r *EphemeralRunnerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, &ephemeralRunner, pod, log)
 
 	case cs == nil:
-		// starting, no container state yet
+		// Runner container status not yet available. Check if any container is in a
+		// terminal error state (e.g. ImagePullBackOff) that prevents the pod from progressing.
+		if containerErrors := extractPodContainerErrors(pod); len(containerErrors) > 0 {
+			errMsg := formatPodContainerErrors(containerErrors)
+			log.Info("Pod has container errors while waiting for runner container", "errors", errMsg)
+			r.Recorder.Eventf(&ephemeralRunner, nil, corev1.EventTypeWarning, "ContainerError", "PodStartupFailure", errMsg)
+			return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, &ephemeralRunner, pod, log)
+		}
 		log.Info("Waiting for runner container status to be available")
 		return ctrl.Result{}, nil
 
-	case cs.State.Terminated == nil: // container is not terminated and pod phase is not failed, so runner is still running
+	case cs.State.Terminated == nil:
+		// Container is not terminated and pod phase is not failed. Check for waiting errors
+		// (e.g. CrashLoopBackOff on the runner container itself).
+		if cs.State.Waiting != nil {
+			if _, isTerminal := terminalContainerWaitingReasons[cs.State.Waiting.Reason]; isTerminal {
+				errMsg := fmt.Sprintf("container %q: %s: %s", cs.Name, cs.State.Waiting.Reason, cs.State.Waiting.Message)
+				log.Info("Runner container is in terminal waiting state", "reason", cs.State.Waiting.Reason)
+				r.Recorder.Eventf(&ephemeralRunner, nil, corev1.EventTypeWarning, "ContainerError", "RunnerContainerWaiting", errMsg)
+				return ctrl.Result{}, r.deleteEphemeralRunnerOrPod(ctx, &ephemeralRunner, pod, log)
+			}
+		}
 		log.Info("Runner container is still running; updating ephemeral runner status")
 		if err := r.updateRunStatusFromPod(ctx, &ephemeralRunner, pod, log); err != nil {
 			log.Info("Failed to update ephemeral runner status. Requeue to not miss this event")
@@ -634,6 +661,17 @@ func (r *EphemeralRunnerReconciler) deletePodAsFailed(ctx context.Context, ephem
 	ephemeralRunner.Status.Reason = pod.Status.Reason
 	ephemeralRunner.Status.Message = pod.Status.Message
 
+	// When pod-level reason/message are empty, extract details from container statuses.
+	// This surfaces errors like ImagePullBackOff that only appear at the container level.
+	if ephemeralRunner.Status.Message == "" {
+		if containerErrors := extractPodContainerErrors(pod); len(containerErrors) > 0 {
+			ephemeralRunner.Status.Message = formatPodContainerErrors(containerErrors)
+			if ephemeralRunner.Status.Reason == "" {
+				ephemeralRunner.Status.Reason = containerErrors[0].Category
+			}
+		}
+	}
+
 	if err := r.Status().Patch(ctx, ephemeralRunner, client.MergeFrom(original)); err != nil {
 		return fmt.Errorf("failed to update ephemeral runner status with failure count: %w", err)
 	}
@@ -859,6 +897,9 @@ func (r *EphemeralRunnerReconciler) deleteRunnerFromService(ctx context.Context,
 // SetupWithManager sets up the controller with the Manager.
 func (r *EphemeralRunnerReconciler) SetupWithManager(mgr ctrl.Manager, opts ...Option) error {
 	r.ResourceBuilder.setSchemeIfUnset(r.Scheme)
+	if r.Recorder == nil {
+		r.Recorder = mgr.GetEventRecorder("ephemeral-runner-controller")
+	}
 
 	return builderWithOptions(
 		ctrl.NewControllerManagedBy(mgr).

controllers/actions.github.com/github_annotations.go

@@ -0,0 +1,75 @@
+package actionsgithubcom
+
+import (
+	"context"
+	"fmt"
+
+	v1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
+	"github.com/actions/actions-runner-controller/github/actions"
+	"github.com/go-logr/logr"
+)
+
+// GitHubAnnotator creates Check Run annotations on GitHub to surface pod errors in the Actions UI.
+// Implementations must handle authentication and API calls to GitHub.
+type GitHubAnnotator interface {
+	CreateErrorAnnotation(ctx context.Context, opts ErrorAnnotationOpts) error
+}
+
+// ErrorAnnotationOpts contains the information needed to create a GitHub error annotation.
+type ErrorAnnotationOpts struct {
+	Owner         string
+	Repository    string
+	RunnerName    string
+	Namespace     string
+	Message       string
+	Reason        string
+	WorkflowRunID int64
+	JobID         string
+
+	// runner is the EphemeralRunner object, used internally by DynamicCheckRunAnnotator
+	// to resolve credentials via SecretResolver.
+	runner *v1alpha1.EphemeralRunner
+}
+
+// annotateRunnerFailure attempts to create a GitHub Check Run annotation for a failed runner.
+// It is a best-effort operation — errors are logged but do not block the reconciliation.
+func annotateRunnerFailure(ctx context.Context, annotator GitHubAnnotator, ephemeralRunner *v1alpha1.EphemeralRunner, log logr.Logger) {
+	if annotator == nil {
+		return
+	}
+
+	owner, repo, err := parseOwnerRepo(ephemeralRunner.Spec.GitHubConfigURL)
+	if err != nil {
+		log.Info("Cannot annotate GitHub: failed to parse config URL", "error", err)
+		return
+	}
+
+	opts := ErrorAnnotationOpts{
+		Owner:         owner,
+		Repository:    repo,
+		RunnerName:    ephemeralRunner.Name,
+		Namespace:     ephemeralRunner.Namespace,
+		Message:       ephemeralRunner.Status.Message,
+		Reason:        ephemeralRunner.Status.Reason,
+		WorkflowRunID: ephemeralRunner.Status.WorkflowRunID,
+		JobID:         ephemeralRunner.Status.JobID,
+		runner:        ephemeralRunner,
+	}
+
+	if err := annotator.CreateErrorAnnotation(ctx, opts); err != nil {
+		log.Error(err, "Failed to create GitHub error annotation (best-effort)")
+	}
+}
+
+func parseOwnerRepo(githubConfigURL string) (string, string, error) {
+	ghConfig, err := actions.ParseGitHubConfigFromURL(githubConfigURL)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to parse GitHub config URL: %w", err)
+	}
+
+	if ghConfig.Organization == "" {
+		return "", "", fmt.Errorf("cannot determine owner from config URL %q", githubConfigURL)
+	}
+
+	return ghConfig.Organization, ghConfig.Repository, nil
+}

controllers/actions.github.com/github_annotations_test.go

@@ -0,0 +1,69 @@
+package actionsgithubcom
+
+import (
+	"context"
+	"testing"
+
+	v1alpha1 "github.com/actions/actions-runner-controller/apis/actions.github.com/v1alpha1"
+	"github.com/go-logr/logr"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseOwnerRepo_RepoLevel(t *testing.T) {
+	owner, repo, err := parseOwnerRepo("https://github.com/my-org/my-repo")
+	require.NoError(t, err)
+	assert.Equal(t, "my-org", owner)
+	assert.Equal(t, "my-repo", repo)
+}
+
+func TestParseOwnerRepo_OrgLevel(t *testing.T) {
+	owner, repo, err := parseOwnerRepo("https://github.com/my-org")
+	require.NoError(t, err)
+	assert.Equal(t, "my-org", owner)
+	assert.Equal(t, "", repo)
+}
+
+func TestParseOwnerRepo_InvalidURL(t *testing.T) {
+	// URL with no path resolves to an empty organization, which we reject
+	_, _, err := parseOwnerRepo("https://github.com")
+	assert.Error(t, err)
+}
+
+type fakeAnnotator struct {
+	called bool
+	opts   ErrorAnnotationOpts
+	err    error
+}
+
+func (f *fakeAnnotator) CreateErrorAnnotation(_ context.Context, opts ErrorAnnotationOpts) error {
+	f.called = true
+	f.opts = opts
+	return f.err
+}
+
+func TestAnnotateRunnerFailure_CallsAnnotator(t *testing.T) {
+	annotator := &fakeAnnotator{}
+	runner := &v1alpha1.EphemeralRunner{}
+	runner.Name = "test-runner"
+	runner.Namespace = "test-ns"
+	runner.Spec.GitHubConfigURL = "https://github.com/my-org/my-repo"
+	runner.Status.Message = "container \"runner\": ImagePullBackOff: image not found"
+	runner.Status.Reason = "ImagePull"
+
+	annotateRunnerFailure(context.Background(), annotator, runner, logr.Discard())
+
+	assert.True(t, annotator.called)
+	assert.Equal(t, "my-org", annotator.opts.Owner)
+	assert.Equal(t, "my-repo", annotator.opts.Repository)
+	assert.Equal(t, "test-runner", annotator.opts.RunnerName)
+	assert.Contains(t, annotator.opts.Message, "ImagePullBackOff")
+}
+
+func TestAnnotateRunnerFailure_NilAnnotator(t *testing.T) {
+	runner := &v1alpha1.EphemeralRunner{}
+	runner.Spec.GitHubConfigURL = "https://github.com/my-org/my-repo"
+
+	// Should not panic
+	annotateRunnerFailure(context.Background(), nil, runner, logr.Discard())
+}