From 0cc44d28b54428fd78eca91fc83a7653a961fbd0 Mon Sep 17 00:00:00 2001 From: "Victor U." Date: Mon, 11 Nov 2024 17:17:24 +0200 Subject: [PATCH 1/8] feat: [SRE-457] Add Conclusions --- pkg/metrics/get_workflow_runs_from_github.go | 37 +++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/pkg/metrics/get_workflow_runs_from_github.go b/pkg/metrics/get_workflow_runs_from_github.go index 4e376f1..59c161f 100644 --- a/pkg/metrics/get_workflow_runs_from_github.go +++ b/pkg/metrics/get_workflow_runs_from_github.go @@ -45,6 +45,8 @@ func getFieldValue(repo string, run github.WorkflowRun, field string) string { return *run.Event case "status": return *run.Status + case "conclusion": + return run.GetConclusion() } log.Printf("Tried to fetch invalid field '%s'", field) return "" @@ -109,21 +111,41 @@ func getWorkflowRunsFromGithub() { for _, repo := range repositories { r := strings.Split(repo, "/") runs := getRecentWorkflowRuns(r[0], r[1]) - for _, run := range runs { var s float64 = 0 - if run.GetConclusion() == "success" { + switch run.GetConclusion() { + case "completed": s = 1 - } else if run.GetConclusion() == "skipped" { + case "action_required": s = 2 - } else if run.GetConclusion() == "in_progress" { + case "cancelled": s = 3 - } else if run.GetConclusion() == "queued" { + case "failure": s = 4 + case "neutral": + s = 5 + case "skipped": + s = 6 + case "stale": + s = 7 + case "success": + s = 8 + case "timed_out": + s = 9 + case "in_progress": + s = 10 + case "queued": + s = 11 + case "requested": + s = 12 + case "waiting": + s = 13 + case "pending": + s = 14 + default: + s = 0 // unknown status } - fields := getRelevantFields(repo, run) - workflowRunStatusGauge.WithLabelValues(fields...).Set(s) var run_usage *github.WorkflowRunUsage = nil @@ -140,7 +162,6 @@ func getWorkflowRunsFromGithub() { } } } - time.Sleep(time.Duration(config.Github.Refresh) * time.Second) } } From 8b42ebabe9686ab549a9eb610ddd3bdffdc78a3c Mon Sep 17 00:00:00 2001 From: Viktor Avd Date: Fri, 6 Jun 2025 16:33:03 +0100 Subject: [PATCH 2/8] feat: export job-level metrics for GH Actions --- README.md | 43 +++++++ pkg/config/config.go | 8 ++ pkg/metrics/get_workflow_jobs_from_github.go | 112 +++++++++++++++++++ pkg/metrics/metrics.go | 19 ++++ 4 files changed, 182 insertions(+) create mode 100644 pkg/metrics/get_workflow_jobs_from_github.go diff --git a/README.md b/README.md index 6f09d6b..be35533 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Authentication can either via a Github Token or the Github App Authentication 3 | Github Api URL | github_api_url, url | GITHUB_API_URL | api.github.com | Github API URL (primarily for Github Enterprise usage) | | Github Enterprise Name | enterprise_name | ENTERPRISE_NAME | "" | Enterprise name. Needed for enterprise endpoints (/enterprises/{ENTERPRISE_NAME}/*). Currently used to get Enterprise level tunners status | | Fields to export | export_fields | EXPORT_FIELDS | repo,id,node_id,head_branch,head_sha,run_number,workflow_id,workflow,event,status | A comma separated list of fields for workflow metrics that should be exported | +| Export job fields | export_job_fields | EXPORT_JOB_FIELDS | repo,workflow,job_name,conclusion,event | A comma-separated list of fields for job-level metrics that should be exported | ## Exported stats @@ -94,6 +95,48 @@ Gauge type | workflow | Workflow Name | | status | Workflow status (completed/in_progress) | +### github_workflow_job_status +Gauge type + +**Result possibility** + +| ID | Description | +|---|---| +| 0 | Failure | +| 1 | Success | +| 2 | Skipped | +| 3 | In Progress | +| 4 | Queued | + +**Fields** + +| Name | Description | +|---|---| +| repo | Repository like \/\ | +| workflow | Workflow Name | +| job_name | Name of the job | +| conclusion | Job conclusion (success/failure/skipped/cancelled) | +| event | Event type like push/pull_request/... | + +### github_workflow_job_duration_seconds +Gauge type + +**Result possibility** + +| Gauge | Description | +|---|---| +| seconds | Number of seconds that a specific workflow job took time to complete. | + +**Fields** + +| Name | Description | +|---|---| +| repo | Repository like \/\ | +| workflow | Workflow Name | +| job_name | Name of the job | +| conclusion | Job conclusion (success/failure/skipped/cancelled) | +| event | Event type like push/pull_request/... | + ### github_job > :warning: **This is a duplicate of the `github_workflow_run_status` metric that will soon be deprecated, do not use anymore.** diff --git a/pkg/config/config.go b/pkg/config/config.go index 3a9fc9e..7fa41e7 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -22,6 +22,7 @@ var ( Debug bool EnterpriseName string WorkflowFields string + WorkflowJobFields string ) // InitConfiguration - set configuration from env vars or command parameters @@ -113,6 +114,13 @@ func InitConfiguration() []cli.Flag { Value: "repo,id,node_id,head_branch,head_sha,run_number,workflow_id,workflow,event,status", Destination: &WorkflowFields, }, + &cli.StringFlag{ + Name: "export_job_fields", + EnvVars: []string{"EXPORT_JOB_FIELDS"}, + Usage: "A comma separated list of fields for job metrics that should be exported", + Value: "repo,workflow,job_name,conclusion,event", + Destination: &WorkflowJobFields, + }, &cli.BoolFlag{ Name: "fetch_workflow_run_usage", EnvVars: []string{"FETCH_WORKFLOW_RUN_USAGE"}, diff --git a/pkg/metrics/get_workflow_jobs_from_github.go b/pkg/metrics/get_workflow_jobs_from_github.go new file mode 100644 index 0000000..cd4075a --- /dev/null +++ b/pkg/metrics/get_workflow_jobs_from_github.go @@ -0,0 +1,112 @@ +package metrics + +import ( + "context" + "log" + "strings" + "time" + + "github.com/google/go-github/v45/github" + "github.com/spendesk/github-actions-exporter/pkg/config" +) + +// getJobFieldValue - returns a value for a given field from a GitHub workflow job +func getJobFieldValue(repo string, workflow string, event string, job *github.WorkflowJob, field string) string { + switch field { + case "repo": + return repo + case "workflow": + return workflow + case "job_name": + return job.GetName() + case "conclusion": + return job.GetConclusion() + case "event": + return event + } + log.Printf("Tried to fetch invalid job field '%s'", field) + return "" +} + +func getRelevantJobFields(repo string, workflow string, event string, job *github.WorkflowJob) []string { + relevantFields := strings.Split(config.WorkflowJobFields, ",") + result := make([]string, len(relevantFields)) + for i, field := range relevantFields { + result[i] = getJobFieldValue(repo, workflow, event,job, field) + } + return result +} + +func getJobsForRun(owner string, repo string, runID int64) []*github.WorkflowJob { + opt := &github.ListWorkflowJobsOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + var jobs []*github.WorkflowJob + for { + resp, rr, err := client.Actions.ListWorkflowJobs(context.Background(), owner, repo, runID, opt) + if rl_err, ok := err.(*github.RateLimitError); ok { + log.Printf("ListWorkflowJobs ratelimited. Pausing until %s", rl_err.Rate.Reset.Time.String()) + time.Sleep(time.Until(rl_err.Rate.Reset.Time)) + continue + } else if err != nil { + log.Printf("ListWorkflowJobs error for run %d in repo %s/%s: %s", runID, owner, repo, err.Error()) + return jobs + } + jobs = append(jobs, resp.Jobs...) + if rr.NextPage == 0 { + break + } + opt.Page = rr.NextPage + } + return jobs +} + +// getWorkflowJobsFromGithub - fetch jobs for each workflow run and emit metrics +func getWorkflowJobsFromGithub() { + for { + for _, repo := range repositories { + r := strings.Split(repo, "/") + runs := getRecentWorkflowRuns(r[0], r[1]) + + // Create a map of run IDs to their events + runEventMap := make(map[int64]string) + for _, run := range runs { + runEventMap[run.GetID()] = run.GetEvent() + } + + for _, run := range runs { + event := runEventMap[run.GetID()] + workflowName := getFieldValue(repo, *run, "workflow") + jobs := getJobsForRun(r[0], r[1], run.GetID()) + + for _, job := range jobs { + fields := getRelevantJobFields(repo, workflowName, event, job) + + var status float64 = 0 + switch job.GetConclusion() { + case "success": + status = 1 + case "skipped": + status = 2 + case "in_progress": + status = 3 + case "queued": + status = 4 + } + workflowJobStatusGauge.WithLabelValues(fields...).Set(status) + + start := job.GetStartedAt() + end := job.GetCompletedAt() + + if !start.IsZero() && !end.IsZero() { + duration := end.Time.Sub(start.Time).Seconds() + workflowJobDurationGauge.WithLabelValues(fields...).Set(duration) + } else { + log.Printf("Skipping duration metric for job %s in run %d (start or end time missing)", job.GetName(), job.GetRunID()) + } + } + } + } + time.Sleep(time.Duration(config.Github.Refresh) * time.Second) + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index e803203..85b6dd8 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -23,6 +23,8 @@ var ( err error workflowRunStatusGauge *prometheus.GaugeVec workflowRunDurationGauge *prometheus.GaugeVec + workflowJobStatusGauge *prometheus.GaugeVec + workflowJobDurationGauge *prometheus.GaugeVec ) // InitMetrics - register metrics in prometheus lib and start func for monitor @@ -41,10 +43,26 @@ func InitMetrics() { }, strings.Split(config.WorkflowFields, ","), ) + workflowJobStatusGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "github_workflow_job_status", + Help: "Status of each job inside GitHub workflow run", + }, + strings.Split(config.WorkflowJobFields, ","), + ) + workflowJobDurationGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "github_workflow_job_duration_seconds", + Help: "Duration of each GitHub workflow job in seconds", + }, + strings.Split(config.WorkflowJobFields, ","), + ) prometheus.MustRegister(runnersGauge) prometheus.MustRegister(runnersOrganizationGauge) prometheus.MustRegister(workflowRunStatusGauge) prometheus.MustRegister(workflowRunDurationGauge) + prometheus.MustRegister(workflowJobStatusGauge) + prometheus.MustRegister(workflowJobDurationGauge) prometheus.MustRegister(workflowBillGauge) prometheus.MustRegister(runnersEnterpriseGauge) @@ -65,6 +83,7 @@ func InitMetrics() { go getRunnersFromGithub() go getRunnersOrganizationFromGithub() go getWorkflowRunsFromGithub() + go getWorkflowJobsFromGithub() go getRunnersEnterpriseFromGithub() } From 981e24bb8404a1aedb41f68f492fa7eb77e495b0 Mon Sep 17 00:00:00 2001 From: Viktor Avd Date: Fri, 6 Jun 2025 17:06:02 +0100 Subject: [PATCH 3/8] Update Helm chart values.yaml with new configuration option --- charts/github-actions-exporter/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/github-actions-exporter/values.yaml b/charts/github-actions-exporter/values.yaml index 8f75ba9..6b55812 100644 --- a/charts/github-actions-exporter/values.yaml +++ b/charts/github-actions-exporter/values.yaml @@ -16,6 +16,7 @@ env: GITHUB_API_URL: api.github.com # ENTERPRISE_NAME: "" # Needed for enterprise endpoints (/enterprises/{ENTERPRISE_NAME}/*) # EXPORT_FIELDS: "repo,id,node_id,head_branch,head_sha,run_number,workflow_id,workflow,event,status" # A comma separated list of fields for workflow metrics that should be exported +# EXPORT_JOB_FIELDS: "repo,workflow,job_name,conclusion,event" # A comma separated list of fields for job metrics that should be exported # For the github authentications need to create a secret by default called actions-exporter # for authentication via github personal token # key: github_token From 79ad984ff92e6048c34e4b9dd51f16c2acc7d2da Mon Sep 17 00:00:00 2001 From: "Victor U." Date: Mon, 9 Jun 2025 14:30:17 +0200 Subject: [PATCH 4/8] update registry --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4b150b..5ed949a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ permissions: packages: write env: - REGISTRY: ghcr.io + REGISTRY: docker.io IMAGE: ${{ github.repository }} APP_NAME: github-actions-exporter @@ -38,7 +38,7 @@ jobs: - name: Build app run: CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags="-X 'main.version=${{ steps.version.outputs.full }}'" -o bin/${{ env.APP_NAME }} . - + - name: Generate MD5 run: md5sum bin/${{ env.APP_NAME }} > bin/${{ env.APP_NAME }}.md5 @@ -80,4 +80,4 @@ jobs: file: ./Dockerfile push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file + labels: ${{ steps.meta.outputs.labels }} From 50277f949fe8b3a914a24e5dd0d61b06cc00d254 Mon Sep 17 00:00:00 2001 From: "Victor U." Date: Mon, 9 Jun 2025 14:35:00 +0200 Subject: [PATCH 5/8] Add docker login action --- .github/workflows/release.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ed949a..81195d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,6 +52,12 @@ jobs: bin/${{ env.APP_NAME }}.md5 prerelease: ${{ steps.version.outputs.prerelease }} + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Docker meta id: meta uses: docker/metadata-action@v5 From 49658852714d5d1894b1d04dea242e12500a1a25 Mon Sep 17 00:00:00 2001 From: Viktor Avd Date: Thu, 12 Jun 2025 12:49:46 +0100 Subject: [PATCH 6/8] add more values to metrics --- pkg/config/config.go | 2 +- pkg/metrics/get_workflow_jobs_from_github.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7fa41e7..76fc488 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -118,7 +118,7 @@ func InitConfiguration() []cli.Flag { Name: "export_job_fields", EnvVars: []string{"EXPORT_JOB_FIELDS"}, Usage: "A comma separated list of fields for job metrics that should be exported", - Value: "repo,workflow,job_name,conclusion,event", + Value: "repo,workflow,job_name,conclusion,event,run_id,job_id", Destination: &WorkflowJobFields, }, &cli.BoolFlag{ diff --git a/pkg/metrics/get_workflow_jobs_from_github.go b/pkg/metrics/get_workflow_jobs_from_github.go index cd4075a..64118c3 100644 --- a/pkg/metrics/get_workflow_jobs_from_github.go +++ b/pkg/metrics/get_workflow_jobs_from_github.go @@ -4,6 +4,7 @@ import ( "context" "log" "strings" + "strconv" "time" "github.com/google/go-github/v45/github" @@ -23,6 +24,10 @@ func getJobFieldValue(repo string, workflow string, event string, job *github.Wo return job.GetConclusion() case "event": return event + case "run_id": + return strconv.FormatInt(job.GetRunID(), 10) + case "job_id": + return strconv.FormatInt(job.GetID(), 10) } log.Printf("Tried to fetch invalid job field '%s'", field) return "" From e71aaf15610fcc8f64f3c3316dde08e243ecc3c8 Mon Sep 17 00:00:00 2001 From: "Victor U." Date: Mon, 9 Jun 2025 14:41:19 +0200 Subject: [PATCH 7/8] Update action --- .github/workflows/release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 81195d9..151d017 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ permissions: env: REGISTRY: docker.io - IMAGE: ${{ github.repository }} + IMAGE: juroapp APP_NAME: github-actions-exporter jobs: @@ -54,6 +54,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 + if: github.event_name != 'pull_request' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -71,13 +72,12 @@ jobs: type=semver,pattern={{major}} type=sha - - name: Log in to GitHub Container Registry + - name: Login to Docker Hub uses: docker/login-action@v3 if: github.event_name != 'pull_request' with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v5 From fcf32d893da2b541b6188e1fc0391d05efd89e7c Mon Sep 17 00:00:00 2001 From: "Victor U." Date: Mon, 9 Jun 2025 14:48:00 +0200 Subject: [PATCH 8/8] Update action add image repo --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 151d017..579bd34 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -63,7 +63,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE }} + images: ${{ env.REGISTRY }}/${{ env.IMAGE }}/${{ env.APP_NAME }} tags: | type=ref,event=branch type=ref,event=pr