From 480e8ccb1bae1a495e960b55997769cb77399792 Mon Sep 17 00:00:00 2001 From: "prath.shenoy" Date: Tue, 18 Aug 2026 00:21:58 +0000 Subject: [PATCH] feat(orchestrator): Consume hook events --- .../orchestrator/server/BUILD.bazel | 1 + .../submitqueue/orchestrator/server/main.go | 2 + submitqueue/orchestrator/BUILD.bazel | 20 ++++- submitqueue/orchestrator/pipeline.go | 25 +++++++ submitqueue/orchestrator/pipeline_test.go | 73 +++++++++++++++++++ 5 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 submitqueue/orchestrator/pipeline_test.go diff --git a/service/submitqueue/orchestrator/server/BUILD.bazel b/service/submitqueue/orchestrator/server/BUILD.bazel index c11a1f67..186bc4cd 100644 --- a/service/submitqueue/orchestrator/server/BUILD.bazel +++ b/service/submitqueue/orchestrator/server/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "//platform/extension/consumergate/noop:go_default_library", "//platform/extension/counter:go_default_library", "//platform/extension/counter/mysql:go_default_library", + "//platform/extension/hook/noop:go_default_library", "//platform/extension/messagequeue/mysql:go_default_library", "//platform/githubactions:go_default_library", "//platform/http:go_default_library", diff --git a/service/submitqueue/orchestrator/server/main.go b/service/submitqueue/orchestrator/server/main.go index a3a1d21a..13861a43 100644 --- a/service/submitqueue/orchestrator/server/main.go +++ b/service/submitqueue/orchestrator/server/main.go @@ -37,6 +37,7 @@ import ( consumergatenoop "github.com/uber/submitqueue/platform/extension/consumergate/noop" "github.com/uber/submitqueue/platform/extension/counter" mysqlcounter "github.com/uber/submitqueue/platform/extension/counter/mysql" + hooknoop "github.com/uber/submitqueue/platform/extension/hook/noop" queueMySQL "github.com/uber/submitqueue/platform/extension/messagequeue/mysql" "github.com/uber/submitqueue/platform/pipeline" "github.com/uber/submitqueue/submitqueue/core/changeset" @@ -198,6 +199,7 @@ func run() error { Analyzer: profiles.AnalyzerFactory(), Speculator: profiles.SpeculatorFactory(), Validator: validatorFactory{}, + Hook: hooknoop.New(), } // Assemble the pipeline: one call builds the topic registry, creates diff --git a/submitqueue/orchestrator/BUILD.bazel b/submitqueue/orchestrator/BUILD.bazel index 82e60a9d..dab1e92e 100644 --- a/submitqueue/orchestrator/BUILD.bazel +++ b/submitqueue/orchestrator/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_go//go:def.bzl", "go_library") +load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -6,9 +6,12 @@ go_library( importpath = "github.com/uber/submitqueue/submitqueue/orchestrator", visibility = ["//visibility:public"], deps = [ + "//api/base/hook:go_default_library", "//api/runway/messagequeue:go_default_library", "//platform/consumer:go_default_library", "//platform/extension/counter:go_default_library", + "//platform/extension/hook:go_default_library", + "//platform/hook:go_default_library", "//platform/pipeline:go_default_library", "//submitqueue/core/topickey:go_default_library", "//submitqueue/extension/buildrunner:go_default_library", @@ -34,3 +37,18 @@ go_library( "@org_uber_go_zap//:go_default_library", ], ) + +go_test( + name = "go_default_test", + srcs = ["pipeline_test.go"], + embed = [":go_default_library"], + deps = [ + "//api/base/hook:go_default_library", + "//platform/extension/hook/noop:go_default_library", + "//platform/pipeline:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + "@com_github_uber_go_tally//:go_default_library", + "@org_uber_go_zap//zaptest:go_default_library", + ], +) diff --git a/submitqueue/orchestrator/pipeline.go b/submitqueue/orchestrator/pipeline.go index 60cfef5e..326fc8b6 100644 --- a/submitqueue/orchestrator/pipeline.go +++ b/submitqueue/orchestrator/pipeline.go @@ -18,10 +18,15 @@ package orchestrator import ( + "fmt" + "github.com/uber-go/tally" + basehook "github.com/uber/submitqueue/api/base/hook" runwaymq "github.com/uber/submitqueue/api/runway/messagequeue" "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/extension/counter" + hookext "github.com/uber/submitqueue/platform/extension/hook" + platformhook "github.com/uber/submitqueue/platform/hook" "github.com/uber/submitqueue/platform/pipeline" "github.com/uber/submitqueue/submitqueue/core/topickey" "github.com/uber/submitqueue/submitqueue/extension/buildrunner" @@ -77,6 +82,10 @@ type Deps struct { // Validator resolves the validator for each queue. Validator validator.Factory + + // Hook receives lifecycle events for fire-and-forget side effects. Wire + // noop when the deployment has no integrations. + Hook hookext.Hook } // Stages is the orchestrator's pipeline topology as a typed table. @@ -211,6 +220,22 @@ var Stages = []pipeline.Stage[Deps]{ return dlq.NewDLQBatchController(d.Logger, d.Scope, d.Storage, sc.Registry, sc.TopicKey, sc.ConsumerGroup), nil }, }, + // Any stage can publish here and this one publishes nothing onward, so the + // row sits outside the flow the rest of the table is ordered by. + { + Key: basehook.TopicKeyHook, + Name: "submitqueue-hook", + ConsumerGroup: "orchestrator", + New: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { + if d.Hook == nil { + return nil, fmt.Errorf("hook is required; wire noop when the deployment has no integrations") + } + return platformhook.NewDispatcher(d.Logger, d.Scope, d.Hook, sc.TopicKey, sc.ConsumerGroup), nil + }, + DLQ: func(d Deps, sc pipeline.StageContext) (consumer.Controller, error) { + return platformhook.NewDLQController(d.Logger, d.Scope, sc.TopicKey, sc.ConsumerGroup), nil + }, + }, } // PublishOnlyTopics declares topics the orchestrator publishes to but does diff --git a/submitqueue/orchestrator/pipeline_test.go b/submitqueue/orchestrator/pipeline_test.go new file mode 100644 index 00000000..e523fb56 --- /dev/null +++ b/submitqueue/orchestrator/pipeline_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package orchestrator + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/uber-go/tally" + basehook "github.com/uber/submitqueue/api/base/hook" + hooknoop "github.com/uber/submitqueue/platform/extension/hook/noop" + "github.com/uber/submitqueue/platform/pipeline" + "go.uber.org/zap/zaptest" +) + +func hookStage(t *testing.T) pipeline.Stage[Deps] { + t.Helper() + for _, s := range Stages { + if s.Key == basehook.TopicKeyHook { + return s + } + } + require.FailNow(t, "no stage is registered for the hook topic key") + return pipeline.Stage[Deps]{} +} + +func TestHookStage(t *testing.T) { + deps := func(t *testing.T) Deps { + return Deps{ + Logger: zaptest.NewLogger(t).Sugar(), + Scope: tally.NoopScope, + Hook: hooknoop.New(), + } + } + stageContext := func(key string) pipeline.StageContext { + return pipeline.StageContext{ + TopicKey: basehook.TopicKey(key), + ConsumerGroup: "orchestrator", + } + } + + t.Run("dispatcher subscribes to the key the engine assigns", func(t *testing.T) { + controller, err := hookStage(t).New(deps(t), stageContext("hook")) + require.NoError(t, err) + assert.Equal(t, basehook.TopicKeyHook, controller.TopicKey()) + }) + + t.Run("reconciler subscribes to the dead-letter key the engine derives", func(t *testing.T) { + controller, err := hookStage(t).DLQ(deps(t), stageContext("hook_dlq")) + require.NoError(t, err) + assert.Equal(t, basehook.TopicKey("hook_dlq"), controller.TopicKey()) + }) + + t.Run("a host that leaves the hook unwired fails to construct", func(t *testing.T) { + d := deps(t) + d.Hook = nil + _, err := hookStage(t).New(d, stageContext("hook")) + require.Error(t, err) + }) +}