diff --git a/backend/Makefile b/backend/Makefile index 45ac60e6..4676c402 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -4,7 +4,6 @@ OUTPUT=type=docker,dest=$(HOME)/tmp/mcai_server.tar GOCACHE=/root/.cache/go-build GOMODCACHE?=/go/pkg/mod REGISTRY=chaitin-registry.cn-hangzhou.cr.aliyuncs.com/monkeycode - # make server PLATFORM= TAG= OUTPUT_SERVER= GOCACHE= image: docker buildx build \ diff --git a/backend/biz/host/handler/v1/internal.go b/backend/biz/host/handler/v1/internal.go index 1e03e12e..bc1a7888 100644 --- a/backend/biz/host/handler/v1/internal.go +++ b/backend/biz/host/handler/v1/internal.go @@ -32,6 +32,7 @@ import ( type InternalHostHandler struct { logger *slog.Logger repo domain.HostRepo + taskRepo taskLogStoreRepo teamRepo domain.TeamHostRepo redis *redis.Client getAgentToken agentTokenGetter @@ -46,6 +47,10 @@ type InternalHostHandler struct { tokenProvider *gituc.TokenProvider } +type taskLogStoreRepo interface { + GetLogStore(ctx context.Context, id uuid.UUID) (consts.LogStore, error) +} + func NewInternalHostHandler(i *do.Injector) (*InternalHostHandler, error) { w := do.MustInvoke[*web.Web](i) tf := do.MustInvoke[taskflow.Clienter](i) @@ -54,6 +59,7 @@ func NewInternalHostHandler(i *do.Injector) (*InternalHostHandler, error) { h := &InternalHostHandler{ logger: do.MustInvoke[*slog.Logger](i).With("module", "InternalHostHandler"), repo: do.MustInvoke[domain.HostRepo](i), + taskRepo: do.MustInvoke[domain.TaskRepo](i), teamRepo: do.MustInvoke[domain.TeamHostRepo](i), redis: rdb, getAgentToken: defaultAgentTokenGetter(rdb), @@ -78,6 +84,7 @@ func NewInternalHostHandler(i *do.Injector) (*InternalHostHandler, error) { g.POST("/coding-config", web.BindHandler(h.GetCodingConfig)) g.POST("/git-credential", web.BindHandler(h.GitCredential)) g.GET("/vm/list", web.BaseHandler(h.VMList)) + g.POST("/task-log-store", web.BindHandler(h.GetTaskLogStore)) g.POST("/task-stream-ips", web.BindHandler(h.GetTaskStreamIPs)) return h, nil @@ -186,6 +193,19 @@ func (h *InternalHostHandler) CheckToken(c *web.Context, req taskflow.CheckToken return c.Success(tk) } +func (h *InternalHostHandler) GetTaskLogStore(c *web.Context, req taskflow.GetTaskLogStoreReq) error { + store, err := h.taskRepo.GetLogStore(c.Request().Context(), req.TaskID) + if err != nil { + return err + } + if store == "" { + store = consts.LogStoreLoki + } + return c.Success(taskflow.GetTaskLogStoreResp{ + LogStore: string(store), + }) +} + func (h *InternalHostHandler) agentAuth(ctx context.Context, token, mid string) (*taskflow.Token, error) { // 1) 优先从 Redis 读取一次性 agent token，并清除 key := fmt.Sprintf("agent:token:%s", token) diff --git a/backend/biz/host/handler/v1/internal_auth_test.go b/backend/biz/host/handler/v1/internal_auth_test.go index 54d85f35..8f581310 100644 --- a/backend/biz/host/handler/v1/internal_auth_test.go +++ b/backend/biz/host/handler/v1/internal_auth_test.go @@ -130,7 +130,7 @@ func TestAgentAuthSoftDeletedRecycledVMStillTriggersDelete(t *testing.T) { type internalHostRepoStub struct { vm *db.VirtualMachine assertSkipMarker bool - skipMarkerKey interface{} + skipMarkerKey any skipMarkerValue string } diff --git a/backend/biz/host/handler/v1/internal_logstore_test.go b/backend/biz/host/handler/v1/internal_logstore_test.go new file mode 100644 index 00000000..c7d96180 --- /dev/null +++ b/backend/biz/host/handler/v1/internal_logstore_test.go @@ -0,0 +1,114 @@ +package v1 + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/GoYoko/web" + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/consts" + taskflowpkg "github.com/chaitin/MonkeyCode/backend/pkg/taskflow" +) + +type taskLogStoreRepoStub struct { + store consts.LogStore + err error +} + +func (s *taskLogStoreRepoStub) GetLogStore(context.Context, uuid.UUID) (consts.LogStore, error) { + return s.store, s.err +} + +func TestInternalHostHandler_GetTaskLogStore_EmptyMeansLoki(t *testing.T) { + h := &InternalHostHandler{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + taskRepo: &taskLogStoreRepoStub{}, + } + req := taskflowpkg.GetTaskLogStoreReq{TaskID: uuid.New()} + resp := callGetTaskLogStore(t, h, req) + if resp.LogStore != string(consts.LogStoreLoki) { + t.Fatalf("log_store = %q, want %q", resp.LogStore, consts.LogStoreLoki) + } +} + +func TestInternalHostHandler_GetTaskLogStore_ClickHousePassthrough(t *testing.T) { + h := &InternalHostHandler{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + taskRepo: &taskLogStoreRepoStub{store: consts.LogStoreClickHouse}, + } + req := taskflowpkg.GetTaskLogStoreReq{TaskID: uuid.New()} + resp := callGetTaskLogStore(t, h, req) + if resp.LogStore != string(consts.LogStoreClickHouse) { + t.Fatalf("log_store = %q, want %q", resp.LogStore, consts.LogStoreClickHouse) + } +} + +func TestInternalHostHandler_GetTaskLogStore_RepoError(t *testing.T) { + h := &InternalHostHandler{ + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + taskRepo: &taskLogStoreRepoStub{err: errors.New("boom")}, + } + req := taskflowpkg.GetTaskLogStoreReq{TaskID: uuid.New()} + body, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + w := web.New() + w.POST("/internal/task-log-store", web.BindHandler(h.GetTaskLogStore)) + rec := httptest.NewRecorder() + httpReq := httptest.NewRequest(http.MethodPost, "/internal/task-log-store", strings.NewReader(string(body))) + httpReq.Header.Set("Content-Type", "application/json") + w.Echo().ServeHTTP(rec, httpReq) + + var resp web.Resp + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal resp: %v", err) + } + if resp.Code == 0 { + t.Fatalf("response = %+v, want error", resp) + } +} + +func callGetTaskLogStore(t *testing.T, h *InternalHostHandler, req taskflowpkg.GetTaskLogStoreReq) taskflowpkg.GetTaskLogStoreResp { + t.Helper() + + body, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + + w := web.New() + w.POST("/internal/task-log-store", web.BindHandler(h.GetTaskLogStore)) + + rec := httptest.NewRecorder() + httpReq := httptest.NewRequest(http.MethodPost, "/internal/task-log-store", strings.NewReader(string(body))) + httpReq.Header.Set("Content-Type", "application/json") + w.Echo().ServeHTTP(rec, httpReq) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + + var resp web.Resp + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal resp: %v", err) + } + data, err := json.Marshal(resp.Data) + if err != nil { + t.Fatalf("marshal resp data: %v", err) + } + + var out taskflowpkg.GetTaskLogStoreResp + if err := json.Unmarshal(data, &out); err != nil { + t.Fatalf("unmarshal typed resp: %v", err) + } + return out +} diff --git a/backend/biz/host/usecase/host_test.go b/backend/biz/host/usecase/host_test.go index f5d0f0bb..bf16848c 100644 --- a/backend/biz/host/usecase/host_test.go +++ b/backend/biz/host/usecase/host_test.go @@ -116,6 +116,17 @@ func (s *hostTaskRepoStub) GetByID(ctx context.Context, id uuid.UUID) (*db.Task, return s.client.Task.Get(ctx, id) } +func (s *hostTaskRepoStub) GetLogStore(ctx context.Context, id uuid.UUID) (consts.LogStore, error) { + tk, err := s.client.Task.Get(ctx, id) + if err != nil { + return "", err + } + if tk.LogStore == nil { + return "", nil + } + return *tk.LogStore, nil +} + func (s *hostTaskRepoStub) Stat(context.Context, uuid.UUID) (*domain.TaskStats, error) { panic("unexpected call to Stat") } diff --git a/backend/biz/task/handler/v1/task.go b/backend/biz/task/handler/v1/task.go index 968fef8a..5ad7ae49 100644 --- a/backend/biz/task/handler/v1/task.go +++ b/backend/biz/task/handler/v1/task.go @@ -10,7 +10,6 @@ import ( "net/http" "os" "path/filepath" - "strconv" "strings" "time" @@ -25,13 +24,13 @@ import ( "github.com/chaitin/MonkeyCode/backend/domain" "github.com/chaitin/MonkeyCode/backend/errcode" "github.com/chaitin/MonkeyCode/backend/middleware" - "github.com/chaitin/MonkeyCode/backend/pkg/loki" "github.com/chaitin/MonkeyCode/backend/pkg/nls" "github.com/chaitin/MonkeyCode/backend/pkg/taskflow" + "github.com/chaitin/MonkeyCode/backend/pkg/tasklog" "github.com/chaitin/MonkeyCode/backend/pkg/ws" ) -var errRoundEnded = errors.New("round ended") +var errTurnEnded = errors.New("turn ended") // TaskHandler 任务处理器 type TaskHandler struct { @@ -41,7 +40,7 @@ type TaskHandler struct { pubhost domain.PublicHostUsecase logger *slog.Logger taskflow taskflow.Clienter - loki *loki.Client + tasklog *tasklog.Gateway nls *nls.NLS taskConns *ws.TaskConn controlConns *ws.ControlConn @@ -59,7 +58,7 @@ func NewTaskHandler(i *do.Injector) (*TaskHandler, error) { uuc := do.MustInvoke[domain.UserUsecase](i) logger := do.MustInvoke[*slog.Logger](i) tf := do.MustInvoke[taskflow.Clienter](i) - lok := do.MustInvoke[*loki.Client](i) + gw := do.MustInvoke[*tasklog.Gateway](i) auth := do.MustInvoke[*middleware.AuthMiddleware](i) targetActive := do.MustInvoke[*middleware.TargetActiveMiddleware](i) tc := do.MustInvoke[*ws.TaskConn](i) @@ -88,7 +87,7 @@ func NewTaskHandler(i *do.Injector) (*TaskHandler, error) { pubhost: pubhost, logger: logger.With("handler", "task.handler"), taskflow: tf, - loki: lok, + tasklog: gw, nls: nlsSvc, taskConns: tc, controlConns: cc, @@ -110,7 +109,7 @@ func NewTaskHandler(i *do.Injector) (*TaskHandler, error) { v1.GET("/:id", web.BindHandler(h.Info)) v1.GET("/stream", web.BindHandler(h.Stream)) v1.GET("/control", web.BindHandler(h.Control)) - v1.GET("/rounds", web.BindHandler(h.TaskRounds)) + v1.GET("/rounds", web.BindHandler(h.TaskTurns)) v1.POST("", web.BindHandler(h.Create)) v1.PUT("/stop", web.BindHandler(h.Stop)) v1.DELETE("/:id", web.BindHandler(h.Delete)) @@ -336,20 +335,20 @@ func (h *TaskHandler) PublicStream(c *web.Context, req domain.IDReq[uuid.UUID]) // @Description - user-input: 用户输入 // @Description - user-cancel: 取消当前操作，不会终止任务 // @Description - reply-question: 回复 AI 的提问 -// @Description - cursor: 历史游标，用于通过 /rounds 接口加载更早的论次 +// @Description - cursor: 历史游标，用于通过 /rounds 接口加载更早的轮次 // @Description // @Description cursor 消息结构： // @Description ```json -// @Description { "type": "cursor", "data": { "cursor": "", "has_more": true }, "timestamp": 0 } +// @Description { "type": "cursor", "data": { "cursor": "", "has_more": true }, "timestamp": 0 } // @Description ``` -// @Description - cursor: 当前论次 task-started 的时间戳（Unix 纳秒），作为 GET /rounds 接口的 cursor 参数向前翻页 -// @Description - has_more: 是否存在更早的论次。为 false 时表示当前论次即为第一论次，无需再翻页 +// @Description - cursor: 当前分页游标，作为 GET /rounds 接口的 cursor 参数向前翻页 +// @Description - has_more: 是否存在更早的轮次。为 false 时表示当前轮次即为第一轮，无需再翻页 // @Tags 【用户】任务管理 // @Accept json // @Produce json // @Security MonkeyCodeAIAuth // @Param id query string true "任务 ID" -// @Param mode query string false "模式：new(等待用户输入)|attach(仅拉取当前论次)，默认 new" +// @Param mode query string false "模式：new(等待用户输入)|attach(仅拉取当前轮次)，默认 new" // @Success 200 {object} web.Resp{} "成功" // @Failure 500 {object} web.Resp "服务器内部错误" // @Router /api/v1/users/tasks/stream [get] @@ -424,15 +423,13 @@ func (h *TaskHandler) attachStream(ctx context.Context, cancel context.CancelCau }() attachNow := time.Now().UTC() - roundStart, err := h.loki.FindLatestRoundStart(ctx, taskID, taskCreatedAt, attachNow) + latestTurn, err := h.tasklog.QueryLatestTurn(ctx, task.ID, taskCreatedAt, attachNow, task.LogStore) if err != nil { - return fmt.Errorf("find latest round start: %w", err) + return fmt.Errorf("query latest turn: %w", err) } - hasMore := roundStart.After(taskCreatedAt) - h.writeCursor(wsConn, roundStart, hasMore) + h.writeCursor(wsConn, latestTurn.NextCursor, latestTurn.HasMore) - // 读最新论次的 loki 历史窗口 - ended, err := h.replayLatestRoundHistory(ctx, wsConn, logger, taskID, roundStart, attachNow) + ended, err := h.replayLatestTurnHistory(wsConn, latestTurn.Entries) if err != nil { return err } @@ -446,26 +443,18 @@ func (h *TaskHandler) attachStream(ctx context.Context, cancel context.CancelCau return nil } -func buildTaskStreamsFromHistoryEntries(entries []loki.LogEntry, logger *slog.Logger) ([]domain.TaskStream, bool) { +func buildTaskStreamsFromLogEntries(entries []tasklog.Entry, logger *slog.Logger) ([]domain.TaskStream, bool) { streams := make([]domain.TaskStream, 0, len(entries)) ended := false - for _, l := range entries { - if l.Line == "" { - continue - } - var chunk taskflow.TaskChunk - if err := json.Unmarshal([]byte(l.Line), &chunk); err != nil { - logger.Error("failed to unmarshal log entry", "line", l.Line, "error", err) - continue - } + for _, entry := range entries { streams = append(streams, domain.TaskStream{ - Type: consts.TaskStreamType(chunk.Event), - Data: chunk.Data, - Kind: chunk.Kind, - Timestamp: l.Timestamp.UnixMilli(), + Type: consts.TaskStreamType(entry.Event), + Data: []byte(entry.Data), + Kind: entry.Kind, + Timestamp: entry.TS.UnixMilli(), }) - if chunk.Event == "task-ended" { + if entry.Event == "task-ended" { ended = true } } @@ -473,13 +462,8 @@ func buildTaskStreamsFromHistoryEntries(entries []loki.LogEntry, logger *slog.Lo return streams, ended } -func (h *TaskHandler) replayLatestRoundHistory(ctx context.Context, wsConn *ws.WebsocketManager, logger *slog.Logger, taskID string, start, end time.Time) (bool, error) { - entries, err := h.loki.QueryWindowByTaskID(ctx, taskID, start, end) - if err != nil { - return false, fmt.Errorf("query latest round history: %w", err) - } - - streams, ended := buildTaskStreamsFromHistoryEntries(entries, logger) +func (h *TaskHandler) replayLatestTurnHistory(wsConn *ws.WebsocketManager, entries []tasklog.Entry) (bool, error) { + streams, ended := buildTaskStreamsFromLogEntries(entries, h.logger) for _, stream := range streams { if err := wsConn.WriteJSON(stream); err != nil { return false, err @@ -513,7 +497,7 @@ func (h *TaskHandler) consumeLiveStream(ctx context.Context, cancel context.Canc return } if chunk.Event == "task-ended" { - cancel(errRoundEnded) + cancel(errTurnEnded) return } } @@ -532,13 +516,13 @@ func (h *TaskHandler) subscribeRealtimeStream(ctx context.Context, cancel contex } if chunk.Event == "task-ended" { - cancel(errRoundEnded) - return errRoundEnded + cancel(errTurnEnded) + return errTurnEnded } return nil }) - if err != nil && !errors.Is(err, errRoundEnded) { + if err != nil && !errors.Is(err, errTurnEnded) { logger.ErrorContext(ctx, "realtime stream failed", "error", err) h.writeError(wsConn, fmt.Errorf("failed to subscribe realtime stream: %w", err)) cancel(fmt.Errorf("failed to subscribe realtime stream: %w", err)) @@ -643,6 +627,7 @@ func (h *TaskHandler) handleReplyQuestion(ctx context.Context, logger *slog.Logg return } req.TaskId = task.ID.String() + req.LogStore = string(task.LogStore) if err := h.taskflow.TaskManager().AskUserQuestion(ctx, req); err != nil { logger.With("error", err).WarnContext(ctx, "failed to send ask user question") } @@ -669,13 +654,11 @@ func (h *TaskHandler) writeError(wsConn *ws.WebsocketManager, err error) { }) } -// writeCursor 向 WebSocket 发送 cursor 消息，通知前端可以通过 /rounds 接口加载更早的历史 -func (h *TaskHandler) writeCursor(wsConn *ws.WebsocketManager, indexTime time.Time, hasMore bool) { - if indexTime.IsZero() { +// writeCursor 向 WebSocket 发送 cursor 消息，通知前端可以通过 /rounds 接口加载更早的历史轮次 +func (h *TaskHandler) writeCursor(wsConn *ws.WebsocketManager, cursor string, hasMore bool) { + if cursor == "" { return } - - cursor := strconv.FormatInt(indexTime.UnixNano()-1, 10) data, _ := json.Marshal(map[string]any{ "cursor": cursor, "has_more": hasMore, @@ -687,22 +670,22 @@ func (h *TaskHandler) writeCursor(wsConn *ws.WebsocketManager, indexTime time.Ti }) } -// TaskRounds 查询任务历史论次（原始 TaskChunk，向前翻页） +// TaskTurns 查询任务历史轮次（原始 TaskChunk，向前翻页） // -// @Summary 查询任务历史论次 -// @Description 根据 cursor 向前翻页查询任务的历史论次。limit 为论次数（非条目数）， -// @Description limit=2 表示返回 2 论的完整消息。返回的 chunks 按时间倒序排列（最新在前）。 +// @Summary 查询任务历史轮次 +// @Description 根据 cursor 向前翻页查询任务的历史轮次。limit 为轮次数（非条目数）， +// @Description limit=2 表示返回 2 轮的完整消息。返回的 chunks 按时间倒序排列（最新在前）。 // @Tags 【用户】任务管理 // @Accept json // @Produce json // @Security MonkeyCodeAIAuth // @Param id query string true "任务 ID" -// @Param cursor query string false "游标（时间戳 Unix ns）" -// @Param limit query int false "论次数（默认 2，上限 10）" +// @Param cursor query string false "分页游标" +// @Param limit query int false "轮次数（默认 2，上限 10）" // @Success 200 {object} web.Resp{data=domain.TaskRoundsResp} "成功" // @Failure 500 {object} web.Resp "服务器内部错误" // @Router /api/v1/users/tasks/rounds [get] -func (h *TaskHandler) TaskRounds(c *web.Context, req domain.TaskRoundsReq) error { +func (h *TaskHandler) TaskTurns(c *web.Context, req domain.TaskRoundsReq) error { ctx := c.Request().Context() user := middleware.GetUser(c) @@ -712,21 +695,12 @@ func (h *TaskHandler) TaskRounds(c *web.Context, req domain.TaskRoundsReq) error return err } - // 确定查询时间范围：从 cursor 往前查 - end := time.Now() - if req.Cursor != "" { - ns, err := strconv.ParseInt(req.Cursor, 10, 64) - if err != nil { - return errcode.ErrBadRequest.Wrap(fmt.Errorf("invalid cursor: %w", err)) - } - end = time.Unix(0, ns) - } start := time.Unix(task.CreatedAt, 0) - result, err := h.loki.QueryRounds(ctx, task.ID.String(), start, end, req.Limit) + result, err := h.tasklog.QueryTurns(ctx, task.ID, start, req.Cursor, req.Limit, task.LogStore) if err != nil { - h.logger.With("error", err, "task_id", task.ID).ErrorContext(ctx, "failed to query rounds") - return errcode.ErrInternalServer.Wrap(fmt.Errorf("failed to query rounds: %w", err)) + h.logger.With("error", err, "task_id", task.ID).ErrorContext(ctx, "failed to query turns") + return errcode.ErrInternalServer.Wrap(fmt.Errorf("failed to query turns: %w", err)) } chunks := make([]*domain.TaskChunkEntry, 0, len(result.Chunks)+1) @@ -756,8 +730,8 @@ func (h *TaskHandler) TaskRounds(c *web.Context, req domain.TaskRoundsReq) error Chunks: chunks, HasMore: result.HasMore, } - if result.HasMore && result.NextTS > 0 { - resp.NextCursor = strconv.FormatInt(result.NextTS, 10) + if result.HasMore && result.NextCursor != "" { + resp.NextCursor = result.NextCursor } return c.Success(resp) diff --git a/backend/biz/task/handler/v1/task_attach_test.go b/backend/biz/task/handler/v1/task_attach_test.go index de918309..6b0b079b 100644 --- a/backend/biz/task/handler/v1/task_attach_test.go +++ b/backend/biz/task/handler/v1/task_attach_test.go @@ -6,16 +6,18 @@ import ( "testing" "time" - "github.com/chaitin/MonkeyCode/backend/pkg/loki" + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/pkg/tasklog" ) -func TestBuildTaskStreamsFromHistoryEntriesStopsWhenEnded(t *testing.T) { +func TestBuildTaskStreamsFromLogEntriesStopsWhenEnded(t *testing.T) { base := time.Unix(1_700_000_000, 0).UTC() logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - streams, ended := buildTaskStreamsFromHistoryEntries([]loki.LogEntry{ - {Timestamp: base, Line: `{"event":"task-started","kind":"acp_event"}`}, - {Timestamp: base.Add(time.Second), Line: `{"event":"task-ended","kind":"acp_event"}`}, + streams, ended := buildTaskStreamsFromLogEntries([]tasklog.Entry{ + {TaskID: uuid.Nil, TS: base, Event: "task-started", Kind: "acp_event"}, + {TaskID: uuid.Nil, TS: base.Add(time.Second), Event: "task-ended", Kind: "acp_event"}, }, logger) if !ended { @@ -26,13 +28,13 @@ func TestBuildTaskStreamsFromHistoryEntriesStopsWhenEnded(t *testing.T) { } } -func TestBuildTaskStreamsFromHistoryEntriesKeepsStreamingWhenNotEnded(t *testing.T) { +func TestBuildTaskStreamsFromLogEntriesKeepsStreamingWhenNotEnded(t *testing.T) { base := time.Unix(1_700_000_000, 0).UTC() logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - streams, ended := buildTaskStreamsFromHistoryEntries([]loki.LogEntry{ - {Timestamp: base, Line: `{"event":"task-started","kind":"acp_event"}`}, - {Timestamp: base.Add(time.Second), Line: `{"event":"task-running","kind":"agent_message_chunk"}`}, + streams, ended := buildTaskStreamsFromLogEntries([]tasklog.Entry{ + {TaskID: uuid.Nil, TS: base, Event: "task-started", Kind: "acp_event"}, + {TaskID: uuid.Nil, TS: base.Add(time.Second), Event: "task-running", Kind: "agent_message_chunk"}, }, logger) if ended { diff --git a/backend/biz/task/handler/v1/task_control.go b/backend/biz/task/handler/v1/task_control.go index edb9d56d..0a88f0bb 100644 --- a/backend/biz/task/handler/v1/task_control.go +++ b/backend/biz/task/handler/v1/task_control.go @@ -351,6 +351,7 @@ func (h *TaskHandler) handleControlCall(ctx context.Context, wsConn *ws.Websocke } req.ID = task.ID req.ExecutionConfig = nil + req.LogStore = string(task.LogStore) requestID = req.RequestId result, err = h.taskflow.TaskManager().Restart(ctx, req) diff --git a/backend/biz/task/repo/task.go b/backend/biz/task/repo/task.go index b70e739b..ac026599 100644 --- a/backend/biz/task/repo/task.go +++ b/backend/biz/task/repo/task.go @@ -127,6 +127,22 @@ func (t *TaskRepo) GetByID(ctx context.Context, id uuid.UUID) (*db.Task, error) First(ctx) } +func (t *TaskRepo) GetLogStore(ctx context.Context, id uuid.UUID) (consts.LogStore, error) { + var rows []struct { + LogStore *string `json:"log_store"` + } + if err := t.db.Task.Query(). + Where(task.ID(id)). + Select(task.FieldLogStore). + Scan(ctx, &rows); err != nil { + return "", err + } + if len(rows) == 0 || rows[0].LogStore == nil { + return "", nil + } + return consts.LogStore(*rows[0].LogStore), nil +} + // Info implements domain.TaskRepo. func (t *TaskRepo) Info(ctx context.Context, u *domain.User, id uuid.UUID, isPrivileged bool) (*db.Task, error) { q := t.db.Task.Query(). @@ -414,6 +430,7 @@ func (t *TaskRepo) Create(ctx context.Context, u *domain.User, req domain.Create SetContent(req.Content). SetUserID(u.ID). SetStatus(consts.TaskStatusPending). + SetLogStore(consts.LogStoreClickHouse). Save(ctx) if err != nil { return err diff --git a/backend/biz/task/service/tasksummary.go b/backend/biz/task/service/tasksummary.go index 2f3eacbd..73c3ec7b 100644 --- a/backend/biz/task/service/tasksummary.go +++ b/backend/biz/task/service/tasksummary.go @@ -7,7 +7,7 @@ import ( "errors" "fmt" "log/slog" - "slices" + "sort" "strings" "sync" "time" @@ -19,10 +19,9 @@ import ( "github.com/chaitin/MonkeyCode/backend/consts" "github.com/chaitin/MonkeyCode/backend/db" "github.com/chaitin/MonkeyCode/backend/db/task" - "github.com/chaitin/MonkeyCode/backend/domain" "github.com/chaitin/MonkeyCode/backend/pkg/delayqueue" "github.com/chaitin/MonkeyCode/backend/pkg/llm" - "github.com/chaitin/MonkeyCode/backend/pkg/loki" + "github.com/chaitin/MonkeyCode/backend/pkg/tasklog" ) var ( @@ -31,27 +30,43 @@ var ( // TaskSummaryService 任务摘要生成服务 type TaskSummaryService struct { - cfg *config.Config - db *db.Client - loki *loki.Client - llm *llm.Client - summaryQueue *delayqueue.TaskSummaryQueue - logger *slog.Logger - taskRepo domain.TaskRepo + cfg *config.Config + db *db.Client + llm *llm.Client + summaryQueue *delayqueue.TaskSummaryQueue + logger *slog.Logger + conversationReader ConversationReader // 生命周期管理 cancel context.CancelFunc wg sync.WaitGroup } +type tasklogGateway interface { + QueryTurns(ctx context.Context, taskID uuid.UUID, taskCreatedAt time.Time, cursor string, limit int, store consts.LogStore) (*tasklog.QueryTurnsResp, error) +} + +type ConversationReader interface { + Fetch(ctx context.Context, taskID uuid.UUID, createdAt time.Time, store consts.LogStore, initialContent string, maxRounds int) ([]llm.Message, error) +} + +type tasklogConversationReader struct { + gateway tasklogGateway + logger *slog.Logger +} + +func newTasklogConversationReader(gateway tasklogGateway, logger *slog.Logger) *tasklogConversationReader { + return &tasklogConversationReader{gateway: gateway, logger: logger} +} + // NewTaskSummaryService 创建任务摘要生成服务 func NewTaskSummaryService(i *do.Injector) (*TaskSummaryService, error) { cfg := do.MustInvoke[*config.Config](i) d := do.MustInvoke[*db.Client](i) - lok := do.MustInvoke[*loki.Client](i) + tlg := do.MustInvoke[*tasklog.Gateway](i) sq := do.MustInvoke[*delayqueue.TaskSummaryQueue](i) l := do.MustInvoke[*slog.Logger](i) - tr := do.MustInvoke[domain.TaskRepo](i) + logger := l.With("module", "TaskSummaryService") // 使用 task_summary 自己的 LLM 配置，不依赖全局 LLM Client llmClient := llm.NewClient(llm.Config{ @@ -62,13 +77,12 @@ func NewTaskSummaryService(i *do.Injector) (*TaskSummaryService, error) { }) s := &TaskSummaryService{ - cfg: cfg, - db: d, - loki: lok, - llm: llmClient, - summaryQueue: sq, - logger: l.With("module", "TaskSummaryService"), - taskRepo: tr, + cfg: cfg, + db: d, + llm: llmClient, + summaryQueue: sq, + logger: logger, + conversationReader: newTasklogConversationReader(tlg, logger), } // 启动消费者 @@ -146,7 +160,7 @@ func (s *TaskSummaryService) GenerateSummaryNow(ctx context.Context, taskID stri return "", fmt.Errorf("failed to get task: %w", err) } - conversation, err := s.fetchConversation(ctx, taskID, t.CreatedAt) + conversation, err := s.fetchConversation(ctx, taskUUID, t.CreatedAt, normalizeSummaryLogStore(t.LogStore), t.Content) if err != nil { if errors.Is(err, errNoConversation) { return "", nil @@ -234,9 +248,9 @@ func (s *TaskSummaryService) handleJob(ctx context.Context, job *delayqueue.Job[ } createdAt := t.CreatedAt - logger.DebugContext(ctx, "fetching conversation from loki", "created_at", createdAt) + logger.DebugContext(ctx, "fetching conversation", "created_at", createdAt) - conversation, err := s.fetchConversation(ctx, taskID, createdAt) + conversation, err := s.fetchConversation(ctx, taskUUID, createdAt, normalizeSummaryLogStore(t.LogStore), t.Content) if err != nil { if errors.Is(err, errNoConversation) { logger.InfoContext(ctx, "no conversation found, skip") @@ -262,99 +276,101 @@ func (s *TaskSummaryService) handleJob(ctx context.Context, job *delayqueue.Job[ return nil } -// fetchConversation 从 Loki 获取历史对话，只保留最近 N 轮对话（user-input / reply-question 及其对应的 agent 回复）。 -// 使用倒序查询，从最新的日志往前查，收集到足够轮用户消息后立即停止，避免遍历全部历史。 -func (s *TaskSummaryService) fetchConversation(ctx context.Context, taskID string, createdAt time.Time) ([]llm.Message, error) { +func (s *TaskSummaryService) fetchConversation(ctx context.Context, taskID uuid.UUID, createdAt time.Time, store consts.LogStore, initialContent string) ([]llm.Message, error) { + if s.conversationReader == nil { + return nil, errors.New("task summary conversation reader is nil") + } maxRounds := s.cfg.TaskSummary.MaxRounds if maxRounds <= 0 { maxRounds = 3 } - const pageSize = 200 - taskUUID, err := uuid.Parse(taskID) - if err != nil { - return nil, fmt.Errorf("failed to parse task id: %w", err) - } + return s.conversationReader.Fetch(ctx, taskID, createdAt, store, initialContent, maxRounds) +} - t, err := s.taskRepo.GetByID(ctx, taskUUID) - if err != nil { - return nil, fmt.Errorf("failed to get task: %w", err) +func (r *tasklogConversationReader) Fetch(ctx context.Context, taskID uuid.UUID, createdAt time.Time, store consts.LogStore, initialContent string, maxRounds int) ([]llm.Message, error) { + if r.gateway == nil { + return nil, errors.New("tasklog gateway is nil") } + if maxRounds <= 0 { + maxRounds = 3 + } + logRounds := maxRounds + if strings.TrimSpace(initialContent) != "" { + logRounds-- + } + if logRounds < 0 { + logRounds = 0 + } + const pageSize = 20 - // 从后往前分页查 Loki，收集到足够的用户轮次后停止 - start := createdAt - end := time.Now() - var tailEntries []loki.LogEntry + var chunks []*tasklog.TurnChunk userRoundCount := 0 + cursor := "" - for { - entries, err := s.loki.QueryByTaskID(ctx, taskID, start, end, pageSize, "backward") + for logRounds > 0 { + resp, err := r.gateway.QueryTurns(ctx, taskID, createdAt, cursor, pageSize, store) if err != nil { - return nil, fmt.Errorf("failed to fetch loki history: %w", err) + return nil, fmt.Errorf("failed to fetch task log history: %w", err) + } + if resp == nil { + break } - done := false - for _, entry := range entries { - tailEntries = append(tailEntries, entry) - - if entry.Line == "" { + stopPaging := false + for _, chunk := range resp.Chunks { + if chunk == nil { continue } - var lokiEnt lokiEntry - if err := json.Unmarshal([]byte(entry.Line), &lokiEnt); err != nil { - continue + if (chunk.Event == "user-input" || chunk.Event == "reply-question") && userRoundCount >= logRounds { + stopPaging = true + break } - if lokiEnt.Event == "user-input" || lokiEnt.Event == "reply-question" { + chunks = append(chunks, chunk) + if chunk.Event == "user-input" || chunk.Event == "reply-question" { userRoundCount++ - if userRoundCount >= maxRounds { - done = true - break - } } } - if done || len(entries) < pageSize { + if stopPaging || userRoundCount >= logRounds || !resp.HasMore || resp.NextCursor == "" { break } - // 向更早的时间翻页 - end = entries[len(entries)-1].Timestamp.Add(-time.Nanosecond) + cursor = resp.NextCursor } - // 反转为时间正序 - slices.Reverse(tailEntries) + return buildSummaryConversation(ctx, r.logger, taskID, chunks, userRoundCount, maxRounds, initialContent) +} + +func buildSummaryConversation(ctx context.Context, logger *slog.Logger, taskID uuid.UUID, chunks []*tasklog.TurnChunk, userRoundCount, maxRounds int, initialContent string) ([]llm.Message, error) { + sort.Slice(chunks, func(i, j int) bool { + a := chunks[i] + b := chunks[j] + if a == nil { + return b != nil + } + if b == nil { + return false + } + return a.Timestamp < b.Timestamp + }) - // 按正序解析为 messages var messages []llm.Message - // 如果 Loki 中用户轮次不足 3 轮，补上初始任务内容 - if userRoundCount < maxRounds { - messages = append(messages, llm.Message{Role: "user", Content: t.Content}) + if initialContent != "" { + messages = append(messages, llm.Message{Role: "user", Content: initialContent}) } agentMsg := []string{} - for _, entry := range tailEntries { - if entry.Line == "" { - continue - } - - s.logger.DebugContext(ctx, "loki entry", "entry", entry.Line) - - var lokiEnt lokiEntry - if err := json.Unmarshal([]byte(entry.Line), &lokiEnt); err != nil { - s.logger.ErrorContext(ctx, "failed to unmarshal loki entry", "task_id", taskID, "error", err) - continue - } - - if lokiEnt.Data == "" { + for _, chunk := range chunks { + if chunk == nil || len(chunk.Data) == 0 { continue } - decoded, err := base64.StdEncoding.DecodeString(lokiEnt.Data) + decoded, err := base64.StdEncoding.DecodeString(string(chunk.Data)) if err != nil { - s.logger.ErrorContext(ctx, "failed to decode base64 data", "task_id", taskID, "error", err) - continue + decoded = chunk.Data } - switch lokiEnt.Event { + switch chunk.Event { case "user-input", "reply-question": var userInputText string var ur userReply @@ -392,18 +408,30 @@ func (s *TaskSummaryService) fetchConversation(ctx context.Context, taskID strin messages = append(messages, llm.Message{Role: "assistant", Content: agentContent}) } - s.logger.DebugContext(ctx, "conversation", "messages_count", len(messages), "messages", messages) + if logger != nil { + logger.DebugContext(ctx, "conversation", "task_id", taskID, "messages_count", len(messages), "messages", messages) + } return messages, nil } +func normalizeSummaryLogStore(store *consts.LogStore) consts.LogStore { + if store == nil || strings.TrimSpace(string(*store)) == "" { + return consts.LogStoreLoki + } + return *store +} + // generateSummary 调用 LLM 生成摘要 func (s *TaskSummaryService) generateSummary(ctx context.Context, conversation []llm.Message) (string, error) { - systemPrompt := `你是一个对话标题生成器，专门为用户与 AI 助手的对话生成简短、具体的标题。你只输出标题本身，不做任何解释。` - maxChars := s.cfg.TaskSummary.MaxChars if maxChars <= 0 { maxChars = 300 } + if summary, ok := fallbackSummaryFromConversation(conversation, maxChars); ok { + return summary, nil + } + + systemPrompt := `你是一个对话标题生成器，专门为用户与 AI 助手的对话生成简短、具体的标题。你只输出标题本身，不做任何解释。` userPrompt := fmt.Sprintf(`请根据以上对话，总结用户的核心意图，生成一个简短标题。 @@ -411,6 +439,8 @@ func (s *TaskSummaryService) generateSummary(ctx context.Context, conversation [ - 不超过%d字 - 不要标点结尾 - 只输出标题，不要解释 +- 第一条用户消息是任务原始需求，必须优先依据它生成标题 +- 后续对话只作为补充上下文，不要把上下文交接、压缩摘要或运行状态总结当成任务标题 - 重点关注用户想要完成什么目标，而不是 AI 问了什么问题 - 标题要具体，让人一看就知道用户想做什么 - 如果是开发任务：说明做的是什么应用/功能（如"开发五子棋游戏"） @@ -436,3 +466,46 @@ func (s *TaskSummaryService) generateSummary(ctx context.Context, conversation [ return strings.TrimSpace(resp.Content), nil } + +func fallbackSummaryFromConversation(conversation []llm.Message, maxChars int) (string, bool) { + userInputs := make([]string, 0, len(conversation)) + hasAssistant := false + for _, msg := range conversation { + switch msg.Role { + case "user": + content := strings.TrimSpace(msg.Content) + if content != "" { + userInputs = append(userInputs, content) + } + case "assistant": + if strings.TrimSpace(msg.Content) != "" { + hasAssistant = true + } + } + } + if hasAssistant || len(userInputs) != 1 || !isLowInformationInput(userInputs[0]) { + return "", false + } + return truncateSummary(userInputs[0], maxChars), true +} + +func isLowInformationInput(input string) bool { + normalized := strings.ToLower(strings.TrimSpace(input)) + normalized = strings.Trim(normalized, " \t\r\n.!?。！？~～,，") + switch normalized { + case "hi", "hello", "hey", "你好", "您好", "嗨", "哈喽", "hello there", "ok", "okay", "嗯", "嗯嗯", "额": + return true + } + return false +} + +func truncateSummary(s string, maxChars int) string { + if maxChars <= 0 { + return s + } + runes := []rune(s) + if len(runes) <= maxChars { + return s + } + return string(runes[:maxChars]) +} diff --git a/backend/biz/task/service/tasksummary_test.go b/backend/biz/task/service/tasksummary_test.go new file mode 100644 index 00000000..f084b2e6 --- /dev/null +++ b/backend/biz/task/service/tasksummary_test.go @@ -0,0 +1,179 @@ +package service + +import ( + "context" + "encoding/base64" + "encoding/json" + "io" + "log/slog" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/consts" + "github.com/chaitin/MonkeyCode/backend/pkg/llm" + "github.com/chaitin/MonkeyCode/backend/pkg/tasklog" +) + +type fakeTasklogGateway struct { + responses []*tasklog.QueryTurnsResp + stores []consts.LogStore + calls int +} + +func (f *fakeTasklogGateway) QueryTurns(ctx context.Context, taskID uuid.UUID, taskCreatedAt time.Time, cursor string, limit int, store consts.LogStore) (*tasklog.QueryTurnsResp, error) { + f.calls++ + f.stores = append(f.stores, store) + if len(f.responses) == 0 { + return &tasklog.QueryTurnsResp{}, nil + } + resp := f.responses[0] + f.responses = f.responses[1:] + return resp, nil +} + +func TestTasklogConversationReaderFetchClickHouseChunks(t *testing.T) { + taskID := uuid.New() + createdAt := time.Date(2026, 4, 24, 10, 0, 0, 0, time.UTC) + runningData := mustJSON(t, wsData{Update: wsUpdate{SessionUpdate: "agent_message_chunk", Content: wsContent{Text: "好的"}}}) + gateway := &fakeTasklogGateway{responses: []*tasklog.QueryTurnsResp{{ + Chunks: []*tasklog.TurnChunk{ + {Event: "task-running", Data: []byte(base64.StdEncoding.EncodeToString(runningData)), Timestamp: 2}, + {Event: "user-input", Data: []byte(base64.StdEncoding.EncodeToString([]byte("请帮我修复测试"))), Timestamp: 1}, + }, + }}} + reader := newTasklogConversationReader(gateway, slog.New(slog.NewTextHandler(io.Discard, nil))) + + messages, err := reader.Fetch(context.Background(), taskID, createdAt, consts.LogStoreClickHouse, "初始任务", 3) + if err != nil { + t.Fatalf("Fetch() error = %v", err) + } + if len(messages) != 3 { + t.Fatalf("len(messages) = %d, want 3: %#v", len(messages), messages) + } + assertMessage(t, messages[0], "user", "初始任务") + assertMessage(t, messages[1], "user", "请帮我修复测试") + assertMessage(t, messages[2], "assistant", "好的") + if len(gateway.stores) != 1 || gateway.stores[0] != consts.LogStoreClickHouse { + t.Fatalf("gateway stores = %#v, want [%q]", gateway.stores, consts.LogStoreClickHouse) + } +} + +func TestNormalizeSummaryLogStoreEmptyMeansLoki(t *testing.T) { + if got := normalizeSummaryLogStore(nil); got != consts.LogStoreLoki { + t.Fatalf("normalizeSummaryLogStore(nil) = %q, want %q", got, consts.LogStoreLoki) + } + empty := consts.LogStore("") + if got := normalizeSummaryLogStore(&empty); got != consts.LogStoreLoki { + t.Fatalf("normalizeSummaryLogStore(empty) = %q, want %q", got, consts.LogStoreLoki) + } + clickhouse := consts.LogStoreClickHouse + if got := normalizeSummaryLogStore(&clickhouse); got != consts.LogStoreClickHouse { + t.Fatalf("normalizeSummaryLogStore(clickhouse) = %q, want %q", got, consts.LogStoreClickHouse) + } +} + +func TestTasklogConversationReaderStopsWhenMaxRoundsReached(t *testing.T) { + runningData := mustJSON(t, wsData{Update: wsUpdate{SessionUpdate: "agent_message_chunk", Content: wsContent{Text: "助手回复"}}}) + gateway := &fakeTasklogGateway{responses: []*tasklog.QueryTurnsResp{ + { + Chunks: []*tasklog.TurnChunk{ + {Event: "user-input", Data: []byte(base64.StdEncoding.EncodeToString([]byte("第一轮"))), Timestamp: 1}, + {Event: "task-running", Data: []byte(base64.StdEncoding.EncodeToString(runningData)), Timestamp: 2}, + }, + HasMore: true, + NextCursor: "next", + }, + { + Chunks: []*tasklog.TurnChunk{{Event: "user-input", Data: []byte(base64.StdEncoding.EncodeToString([]byte("不应读取"))), Timestamp: 0}}, + }, + }} + reader := newTasklogConversationReader(gateway, slog.New(slog.NewTextHandler(io.Discard, nil))) + + messages, err := reader.Fetch(context.Background(), uuid.New(), time.Now(), consts.LogStoreLoki, "", 1) + if err != nil { + t.Fatalf("Fetch() error = %v", err) + } + if gateway.calls != 1 { + t.Fatalf("gateway calls = %d, want 1", gateway.calls) + } + if len(messages) != 2 { + t.Fatalf("len(messages) = %d, want 2: %#v", len(messages), messages) + } + assertMessage(t, messages[0], "user", "第一轮") + assertMessage(t, messages[1], "assistant", "助手回复") +} + +func TestTasklogConversationReaderKeepsOnlyRecentMaxRoundsInSinglePage(t *testing.T) { + latestRunningData := mustJSON(t, wsData{Update: wsUpdate{SessionUpdate: "agent_message_chunk", Content: wsContent{Text: "最新助手"}}}) + olderRunningData := mustJSON(t, wsData{Update: wsUpdate{SessionUpdate: "agent_message_chunk", Content: wsContent{Text: "更旧助手"}}}) + gateway := &fakeTasklogGateway{responses: []*tasklog.QueryTurnsResp{{ + Chunks: []*tasklog.TurnChunk{ + {Event: "user-input", Data: []byte(base64.StdEncoding.EncodeToString([]byte("最新用户"))), Timestamp: 1}, + {Event: "task-running", Data: []byte(base64.StdEncoding.EncodeToString(latestRunningData)), Timestamp: 2}, + {Event: "user-input", Data: []byte(base64.StdEncoding.EncodeToString([]byte("更旧用户"))), Timestamp: 3}, + {Event: "task-running", Data: []byte(base64.StdEncoding.EncodeToString(olderRunningData)), Timestamp: 4}, + }, + }}} + reader := newTasklogConversationReader(gateway, slog.New(slog.NewTextHandler(io.Discard, nil))) + + messages, err := reader.Fetch(context.Background(), uuid.New(), time.Now(), consts.LogStoreClickHouse, "", 1) + if err != nil { + t.Fatalf("Fetch() error = %v", err) + } + if len(messages) != 2 { + t.Fatalf("len(messages) = %d, want 2: %#v", len(messages), messages) + } + assertMessage(t, messages[0], "user", "最新用户") + assertMessage(t, messages[1], "assistant", "最新助手") +} + +func TestTasklogConversationReaderAnchorsSummaryToInitialContent(t *testing.T) { + gateway := &fakeTasklogGateway{responses: []*tasklog.QueryTurnsResp{{ + Chunks: []*tasklog.TurnChunk{ + {Event: "user-input", Data: []byte(base64.StdEncoding.EncodeToString([]byte("不相干的上下文摘要"))), Timestamp: 1}, + }, + }}} + reader := newTasklogConversationReader(gateway, slog.New(slog.NewTextHandler(io.Discard, nil))) + + messages, err := reader.Fetch(context.Background(), uuid.New(), time.Now(), consts.LogStoreClickHouse, "修复 ClickHouse 日志查询失败", 1) + if err != nil { + t.Fatalf("Fetch() error = %v", err) + } + if gateway.calls != 0 { + t.Fatalf("gateway calls = %d, want 0", gateway.calls) + } + if len(messages) != 1 { + t.Fatalf("len(messages) = %d, want 1: %#v", len(messages), messages) + } + assertMessage(t, messages[0], "user", "修复 ClickHouse 日志查询失败") +} + +func TestFallbackSummaryForLowInformationGreeting(t *testing.T) { + summary, ok := fallbackSummaryFromConversation([]llm.Message{ + {Role: "user", Content: "hi"}, + }, 300) + if !ok { + t.Fatal("expected fallback summary") + } + if summary != "hi" { + t.Fatalf("summary = %q, want hi", summary) + } +} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + data, err := json.Marshal(v) + if err != nil { + t.Fatalf("json marshal: %v", err) + } + return data +} + +func assertMessage(t *testing.T, got llm.Message, role, content string) { + t.Helper() + if got.Role != role || got.Content != content { + t.Fatalf("message = {Role:%q Content:%q}, want {Role:%q Content:%q}", got.Role, got.Content, role, content) + } +} diff --git a/backend/biz/task/usecase/gittask.go b/backend/biz/task/usecase/gittask.go index f210deb1..f2936f4c 100644 --- a/backend/biz/task/usecase/gittask.go +++ b/backend/biz/task/usecase/gittask.go @@ -80,8 +80,9 @@ func (g *GitTaskUsecase) Create(ctx context.Context, req domain.CreateGitTaskReq BaseURL: m.BaseURL, Model: m.Model, }, - Cores: fmt.Sprintf("%d", g.cfg.Task.Core), - Memory: g.cfg.Task.Memory, + Cores: fmt.Sprintf("%d", g.cfg.Task.Core), + Memory: g.cfg.Task.Memory, + LogStore: normalizeTaskLogStore(t.LogStore), }) if err != nil { return nil, err @@ -114,7 +115,8 @@ func (g *GitTaskUsecase) Create(ctx context.Context, req domain.CreateGitTaskReq BaseURL: m.BaseURL, Model: m.Model, }, - Env: req.Env, + Env: req.Env, + LogStore: normalizeTaskLogStore(t.LogStore), } b, err := json.Marshal(createTaskReq) if err != nil { diff --git a/backend/biz/task/usecase/logstore.go b/backend/biz/task/usecase/logstore.go new file mode 100644 index 00000000..336ef728 --- /dev/null +++ b/backend/biz/task/usecase/logstore.go @@ -0,0 +1,10 @@ +package usecase + +import "github.com/chaitin/MonkeyCode/backend/consts" + +func normalizeTaskLogStore(store *consts.LogStore) string { + if store == nil || *store == "" { + return string(consts.LogStoreLoki) + } + return string(*store) +} diff --git a/backend/biz/task/usecase/task.go b/backend/biz/task/usecase/task.go index 73aa2dbc..6774a1a2 100644 --- a/backend/biz/task/usecase/task.go +++ b/backend/biz/task/usecase/task.go @@ -199,6 +199,7 @@ func (a *TaskUsecase) SwitchModel(ctx context.Context, user *domain.User, taskID ID: taskID, RequestId: req.RequestID, LoadSession: req.LoadSession, + LogStore: string(t.LogStore), ExecutionConfig: &taskflow.TaskExecutionConfig{ Envs: envs, ConfigFiles: configs, @@ -340,7 +341,8 @@ func (a *TaskUsecase) Cancel(ctx context.Context, user *domain.User, id uuid.UUI EnvironmentID: tk.VirtualMachine.EnvironmentID, }, Task: &taskflow.Task{ - ID: id, + ID: id, + LogStore: string(tk.LogStore), }, }); err != nil { return err @@ -363,8 +365,9 @@ func (a *TaskUsecase) Continue(ctx context.Context, user *domain.User, id uuid.U EnvironmentID: tk.VirtualMachine.EnvironmentID, }, Task: &taskflow.Task{ - ID: id, - Text: content, + ID: id, + Text: content, + LogStore: string(tk.LogStore), }, }); err != nil { return err @@ -514,9 +517,10 @@ func (a *TaskUsecase) Create(ctx context.Context, user *domain.User, req domain. BaseURL: m.BaseURL, Model: m.Model, }, - Cores: "2", - Memory: 8 << 30, - Envs: env, + Cores: "2", + Memory: 8 << 30, + Envs: env, + LogStore: normalizeTaskLogStore(t.LogStore), }) if err != nil { return nil, err @@ -544,6 +548,7 @@ func (a *TaskUsecase) Create(ctx context.Context, user *domain.User, req domain. }, Configs: configs, McpConfigs: mcps, + LogStore: normalizeTaskLogStore(t.LogStore), } b, err := json.Marshal(createTaskReq) if err != nil { diff --git a/backend/biz/task/usecase/task_switch_model_test.go b/backend/biz/task/usecase/task_switch_model_test.go index 69a84f19..e0fddfbf 100644 --- a/backend/biz/task/usecase/task_switch_model_test.go +++ b/backend/biz/task/usecase/task_switch_model_test.go @@ -27,12 +27,14 @@ func TestSwitchModelRestartsWithExecutionConfigAndUpdatesModel(t *testing.T) { toModelID := uuid.MustParse("55555555-5555-5555-5555-555555555555") switchID := uuid.MustParse("66666666-6666-6666-6666-666666666666") restartTaskID := uuid.MustParse("77777777-7777-7777-7777-777777777777") + logStore := consts.LogStoreClickHouse repo := &switchModelTaskRepo{ task: &db.Task{ ID: taskID, UserID: userID, Status: consts.TaskStatusProcessing, + LogStore: &logStore, CreatedAt: time.Now(), LastActiveAt: time.Now(), Edges: db.TaskEdges{ @@ -134,6 +136,9 @@ func TestSwitchModelRestartsWithExecutionConfigAndUpdatesModel(t *testing.T) { if taskMgr.restartReq.ID != taskID { t.Fatalf("restart task id = %s, want %s", taskMgr.restartReq.ID, taskID) } + if taskMgr.restartReq.LogStore != string(consts.LogStoreClickHouse) { + t.Fatalf("restart log_store = %q, want %q", taskMgr.restartReq.LogStore, consts.LogStoreClickHouse) + } if taskMgr.restartReq.ExecutionConfig == nil { t.Fatal("restart execution_config is nil") } @@ -281,6 +286,12 @@ type switchModelTaskRepo struct { func (r *switchModelTaskRepo) GetByID(context.Context, uuid.UUID) (*db.Task, error) { return nil, errors.New("unused") } +func (r *switchModelTaskRepo) GetLogStore(context.Context, uuid.UUID) (consts.LogStore, error) { + if r.task.LogStore == nil { + return consts.LogStoreLoki, nil + } + return *r.task.LogStore, nil +} func (r *switchModelTaskRepo) Stat(context.Context, uuid.UUID) (*domain.TaskStats, error) { return nil, nil } diff --git a/backend/config/config.go b/backend/config/config.go index 7e67977c..1fd0bd42 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -49,6 +49,7 @@ type Config struct { Task Task `mapstructure:"task"` TaskSummary TaskSummary `mapstructure:"task_summary"` Loki Loki `mapstructure:"loki"` + ClickHouse ClickHouse `mapstructure:"clickhouse"` LLM LLM `mapstructure:"llm"` Notify Notify `mapstructure:"notify"` VMIdle VMIdle `mapstructure:"vm_idle"` @@ -126,6 +127,19 @@ type Loki struct { Addr string `mapstructure:"addr"` // Loki 服务地址 } +type ClickHouse struct { + Addr string `mapstructure:"addr"` + Database string `mapstructure:"database"` + Table string `mapstructure:"table"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + ReadUsername string `mapstructure:"read_username"` + ReadPassword string `mapstructure:"read_password"` + MaxOpenConns int `mapstructure:"max_open_conns"` + MaxIdleConns int `mapstructure:"max_idle_conns"` + ConnMaxLifetime int `mapstructure:"conn_max_lifetime"` +} + // LLM 大语言模型配置 type LLM struct { BaseURL string `mapstructure:"base_url"` @@ -175,6 +189,16 @@ func Init(dir string) (*Config, error) { v.SetDefault("server.addr", ":8888") v.SetDefault("server.base_url", "http://localhost:8888") v.SetDefault("loki.addr", "http://monkeycode-ai-loki:3100") + v.SetDefault("clickhouse.addr", "") + v.SetDefault("clickhouse.database", "") + v.SetDefault("clickhouse.table", "task_logs") + v.SetDefault("clickhouse.username", "") + v.SetDefault("clickhouse.password", "") + v.SetDefault("clickhouse.read_username", "") + v.SetDefault("clickhouse.read_password", "") + v.SetDefault("clickhouse.max_open_conns", 64) + v.SetDefault("clickhouse.max_idle_conns", 32) + v.SetDefault("clickhouse.conn_max_lifetime", 3600) v.SetDefault("database.master", "") v.SetDefault("database.slave", "") v.SetDefault("database.max_open_conns", 100) diff --git a/backend/consts/tasklog.go b/backend/consts/tasklog.go new file mode 100644 index 00000000..1a3ce989 --- /dev/null +++ b/backend/consts/tasklog.go @@ -0,0 +1,8 @@ +package consts + +type LogStore string + +const ( + LogStoreLoki LogStore = "loki" + LogStoreClickHouse LogStore = "clickhouse" +) diff --git a/backend/db/migrate/schema.go b/backend/db/migrate/schema.go index 13832efb..7fb836cc 100644 --- a/backend/db/migrate/schema.go +++ b/backend/db/migrate/schema.go @@ -743,6 +743,7 @@ var ( {Name: "title", Type: field.TypeString, Nullable: true, Size: 2147483647}, {Name: "summary", Type: field.TypeString, Nullable: true, Size: 2147483647}, {Name: "status", Type: field.TypeString}, + {Name: "log_store", Type: field.TypeString, Nullable: true}, {Name: "created_at", Type: field.TypeTime}, {Name: "last_active_at", Type: field.TypeTime}, {Name: "updated_at", Type: field.TypeTime}, @@ -757,7 +758,7 @@ var ( ForeignKeys: []*schema.ForeignKey{ { Symbol: "tasks_users_tasks", - Columns: []*schema.Column{TasksColumns[12]}, + Columns: []*schema.Column{TasksColumns[13]}, RefColumns: []*schema.Column{UsersColumns[0]}, OnDelete: schema.NoAction, }, diff --git a/backend/db/mutation.go b/backend/db/mutation.go index ba2651c5..d1ca635c 100644 --- a/backend/db/mutation.go +++ b/backend/db/mutation.go @@ -23764,6 +23764,7 @@ type TaskMutation struct { title *string summary *string status *consts.TaskStatus + log_store *consts.LogStore created_at *time.Time last_active_at *time.Time updated_at *time.Time @@ -24235,6 +24236,55 @@ func (m *TaskMutation) ResetStatus() { m.status = nil } +// SetLogStore sets the "log_store" field. +func (m *TaskMutation) SetLogStore(cs consts.LogStore) { + m.log_store = &cs +} + +// LogStore returns the value of the "log_store" field in the mutation. +func (m *TaskMutation) LogStore() (r consts.LogStore, exists bool) { + v := m.log_store + if v == nil { + return + } + return *v, true +} + +// OldLogStore returns the old "log_store" field's value of the Task entity. +// If the Task object wasn't provided to the builder, the object is fetched from the database. +// An error is returned if the mutation operation is not UpdateOne, or the database query fails. +func (m *TaskMutation) OldLogStore(ctx context.Context) (v *consts.LogStore, err error) { + if !m.op.Is(OpUpdateOne) { + return v, errors.New("OldLogStore is only allowed on UpdateOne operations") + } + if m.id == nil || m.oldValue == nil { + return v, errors.New("OldLogStore requires an ID field in the mutation") + } + oldValue, err := m.oldValue(ctx) + if err != nil { + return v, fmt.Errorf("querying old value for OldLogStore: %w", err) + } + return oldValue.LogStore, nil +} + +// ClearLogStore clears the value of the "log_store" field. +func (m *TaskMutation) ClearLogStore() { + m.log_store = nil + m.clearedFields[task.FieldLogStore] = struct{}{} +} + +// LogStoreCleared returns if the "log_store" field was cleared in this mutation. +func (m *TaskMutation) LogStoreCleared() bool { + _, ok := m.clearedFields[task.FieldLogStore] + return ok +} + +// ResetLogStore resets all changes to the "log_store" field. +func (m *TaskMutation) ResetLogStore() { + m.log_store = nil + delete(m.clearedFields, task.FieldLogStore) +} + // SetCreatedAt sets the "created_at" field. func (m *TaskMutation) SetCreatedAt(t time.Time) { m.created_at = &t @@ -24723,7 +24773,7 @@ func (m *TaskMutation) Type() string { // order to get all numeric fields that were incremented/decremented, call // AddedFields(). func (m *TaskMutation) Fields() []string { - fields := make([]string, 0, 12) + fields := make([]string, 0, 13) if m.deleted_at != nil { fields = append(fields, task.FieldDeletedAt) } @@ -24748,6 +24798,9 @@ func (m *TaskMutation) Fields() []string { if m.status != nil { fields = append(fields, task.FieldStatus) } + if m.log_store != nil { + fields = append(fields, task.FieldLogStore) + } if m.created_at != nil { fields = append(fields, task.FieldCreatedAt) } @@ -24784,6 +24837,8 @@ func (m *TaskMutation) Field(name string) (ent.Value, bool) { return m.Summary() case task.FieldStatus: return m.Status() + case task.FieldLogStore: + return m.LogStore() case task.FieldCreatedAt: return m.CreatedAt() case task.FieldLastActiveAt: @@ -24817,6 +24872,8 @@ func (m *TaskMutation) OldField(ctx context.Context, name string) (ent.Value, er return m.OldSummary(ctx) case task.FieldStatus: return m.OldStatus(ctx) + case task.FieldLogStore: + return m.OldLogStore(ctx) case task.FieldCreatedAt: return m.OldCreatedAt(ctx) case task.FieldLastActiveAt: @@ -24890,6 +24947,13 @@ func (m *TaskMutation) SetField(name string, value ent.Value) error { } m.SetStatus(v) return nil + case task.FieldLogStore: + v, ok := value.(consts.LogStore) + if !ok { + return fmt.Errorf("unexpected type %T for field %s", value, name) + } + m.SetLogStore(v) + return nil case task.FieldCreatedAt: v, ok := value.(time.Time) if !ok { @@ -24960,6 +25024,9 @@ func (m *TaskMutation) ClearedFields() []string { if m.FieldCleared(task.FieldSummary) { fields = append(fields, task.FieldSummary) } + if m.FieldCleared(task.FieldLogStore) { + fields = append(fields, task.FieldLogStore) + } if m.FieldCleared(task.FieldCompletedAt) { fields = append(fields, task.FieldCompletedAt) } @@ -24989,6 +25056,9 @@ func (m *TaskMutation) ClearField(name string) error { case task.FieldSummary: m.ClearSummary() return nil + case task.FieldLogStore: + m.ClearLogStore() + return nil case task.FieldCompletedAt: m.ClearCompletedAt() return nil @@ -25024,6 +25094,9 @@ func (m *TaskMutation) ResetField(name string) error { case task.FieldStatus: m.ResetStatus() return nil + case task.FieldLogStore: + m.ResetLogStore() + return nil case task.FieldCreatedAt: m.ResetCreatedAt() return nil diff --git a/backend/db/runtime/runtime.go b/backend/db/runtime/runtime.go index 5b28010a..8eb66af6 100644 --- a/backend/db/runtime/runtime.go +++ b/backend/db/runtime/runtime.go @@ -649,15 +649,15 @@ func init() { // task.ContentValidator is a validator for the "content" field. It is called by the builders before save. task.ContentValidator = taskDescContent.Validators[0].(func(string) error) // taskDescCreatedAt is the schema descriptor for created_at field. - taskDescCreatedAt := taskFields[8].Descriptor() + taskDescCreatedAt := taskFields[9].Descriptor() // task.DefaultCreatedAt holds the default value on creation for the created_at field. task.DefaultCreatedAt = taskDescCreatedAt.Default.(func() time.Time) // taskDescLastActiveAt is the schema descriptor for last_active_at field. - taskDescLastActiveAt := taskFields[9].Descriptor() + taskDescLastActiveAt := taskFields[10].Descriptor() // task.DefaultLastActiveAt holds the default value on creation for the last_active_at field. task.DefaultLastActiveAt = taskDescLastActiveAt.Default.(func() time.Time) // taskDescUpdatedAt is the schema descriptor for updated_at field. - taskDescUpdatedAt := taskFields[10].Descriptor() + taskDescUpdatedAt := taskFields[11].Descriptor() // task.DefaultUpdatedAt holds the default value on creation for the updated_at field. task.DefaultUpdatedAt = taskDescUpdatedAt.Default.(func() time.Time) // task.UpdateDefaultUpdatedAt holds the default value on update for the updated_at field. diff --git a/backend/db/task.go b/backend/db/task.go index 739d2b65..20b42f92 100644 --- a/backend/db/task.go +++ b/backend/db/task.go @@ -36,6 +36,8 @@ type Task struct { Summary string `json:"summary,omitempty"` // Status holds the value of the "status" field. Status consts.TaskStatus `json:"status,omitempty"` + // LogStore holds the value of the "log_store" field. + LogStore *consts.LogStore `json:"log_store,omitempty"` // CreatedAt holds the value of the "created_at" field. CreatedAt time.Time `json:"created_at,omitempty"` // LastActiveAt holds the value of the "last_active_at" field. @@ -130,7 +132,7 @@ func (*Task) scanValues(columns []string) ([]any, error) { values := make([]any, len(columns)) for i := range columns { switch columns[i] { - case task.FieldKind, task.FieldSubType, task.FieldContent, task.FieldTitle, task.FieldSummary, task.FieldStatus: + case task.FieldKind, task.FieldSubType, task.FieldContent, task.FieldTitle, task.FieldSummary, task.FieldStatus, task.FieldLogStore: values[i] = new(sql.NullString) case task.FieldDeletedAt, task.FieldCreatedAt, task.FieldLastActiveAt, task.FieldUpdatedAt, task.FieldCompletedAt: values[i] = new(sql.NullTime) @@ -205,6 +207,13 @@ func (_m *Task) assignValues(columns []string, values []any) error { } else if value.Valid { _m.Status = consts.TaskStatus(value.String) } + case task.FieldLogStore: + if value, ok := values[i].(*sql.NullString); !ok { + return fmt.Errorf("unexpected type %T for field log_store", values[i]) + } else if value.Valid { + _m.LogStore = new(consts.LogStore) + *_m.LogStore = consts.LogStore(value.String) + } case task.FieldCreatedAt: if value, ok := values[i].(*sql.NullTime); !ok { return fmt.Errorf("unexpected type %T for field created_at", values[i]) @@ -319,6 +328,11 @@ func (_m *Task) String() string { builder.WriteString("status=") builder.WriteString(fmt.Sprintf("%v", _m.Status)) builder.WriteString(", ") + if v := _m.LogStore; v != nil { + builder.WriteString("log_store=") + builder.WriteString(fmt.Sprintf("%v", *v)) + } + builder.WriteString(", ") builder.WriteString("created_at=") builder.WriteString(_m.CreatedAt.Format(time.ANSIC)) builder.WriteString(", ") diff --git a/backend/db/task/task.go b/backend/db/task/task.go index 593318c0..f1c2b166 100644 --- a/backend/db/task/task.go +++ b/backend/db/task/task.go @@ -31,6 +31,8 @@ const ( FieldSummary = "summary" // FieldStatus holds the string denoting the status field in the database. FieldStatus = "status" + // FieldLogStore holds the string denoting the log_store field in the database. + FieldLogStore = "log_store" // FieldCreatedAt holds the string denoting the created_at field in the database. FieldCreatedAt = "created_at" // FieldLastActiveAt holds the string denoting the last_active_at field in the database. @@ -106,6 +108,7 @@ var Columns = []string{ FieldTitle, FieldSummary, FieldStatus, + FieldLogStore, FieldCreatedAt, FieldLastActiveAt, FieldUpdatedAt, @@ -196,6 +199,11 @@ func ByStatus(opts ...sql.OrderTermOption) OrderOption { return sql.OrderByField(FieldStatus, opts...).ToFunc() } +// ByLogStore orders the results by the log_store field. +func ByLogStore(opts ...sql.OrderTermOption) OrderOption { + return sql.OrderByField(FieldLogStore, opts...).ToFunc() +} + // ByCreatedAt orders the results by the created_at field. func ByCreatedAt(opts ...sql.OrderTermOption) OrderOption { return sql.OrderByField(FieldCreatedAt, opts...).ToFunc() diff --git a/backend/db/task/where.go b/backend/db/task/where.go index 6ca5888b..17d50d60 100644 --- a/backend/db/task/where.go +++ b/backend/db/task/where.go @@ -100,6 +100,12 @@ func Status(v consts.TaskStatus) predicate.Task { return predicate.Task(sql.FieldEQ(FieldStatus, vc)) } +// LogStore applies equality check predicate on the "log_store" field. It's identical to LogStoreEQ. +func LogStore(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldEQ(FieldLogStore, vc)) +} + // CreatedAt applies equality check predicate on the "created_at" field. It's identical to CreatedAtEQ. func CreatedAt(v time.Time) predicate.Task { return predicate.Task(sql.FieldEQ(FieldCreatedAt, v)) @@ -667,6 +673,100 @@ func StatusContainsFold(v consts.TaskStatus) predicate.Task { return predicate.Task(sql.FieldContainsFold(FieldStatus, vc)) } +// LogStoreEQ applies the EQ predicate on the "log_store" field. +func LogStoreEQ(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldEQ(FieldLogStore, vc)) +} + +// LogStoreNEQ applies the NEQ predicate on the "log_store" field. +func LogStoreNEQ(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldNEQ(FieldLogStore, vc)) +} + +// LogStoreIn applies the In predicate on the "log_store" field. +func LogStoreIn(vs ...consts.LogStore) predicate.Task { + v := make([]any, len(vs)) + for i := range v { + v[i] = string(vs[i]) + } + return predicate.Task(sql.FieldIn(FieldLogStore, v...)) +} + +// LogStoreNotIn applies the NotIn predicate on the "log_store" field. +func LogStoreNotIn(vs ...consts.LogStore) predicate.Task { + v := make([]any, len(vs)) + for i := range v { + v[i] = string(vs[i]) + } + return predicate.Task(sql.FieldNotIn(FieldLogStore, v...)) +} + +// LogStoreGT applies the GT predicate on the "log_store" field. +func LogStoreGT(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldGT(FieldLogStore, vc)) +} + +// LogStoreGTE applies the GTE predicate on the "log_store" field. +func LogStoreGTE(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldGTE(FieldLogStore, vc)) +} + +// LogStoreLT applies the LT predicate on the "log_store" field. +func LogStoreLT(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldLT(FieldLogStore, vc)) +} + +// LogStoreLTE applies the LTE predicate on the "log_store" field. +func LogStoreLTE(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldLTE(FieldLogStore, vc)) +} + +// LogStoreContains applies the Contains predicate on the "log_store" field. +func LogStoreContains(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldContains(FieldLogStore, vc)) +} + +// LogStoreHasPrefix applies the HasPrefix predicate on the "log_store" field. +func LogStoreHasPrefix(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldHasPrefix(FieldLogStore, vc)) +} + +// LogStoreHasSuffix applies the HasSuffix predicate on the "log_store" field. +func LogStoreHasSuffix(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldHasSuffix(FieldLogStore, vc)) +} + +// LogStoreIsNil applies the IsNil predicate on the "log_store" field. +func LogStoreIsNil() predicate.Task { + return predicate.Task(sql.FieldIsNull(FieldLogStore)) +} + +// LogStoreNotNil applies the NotNil predicate on the "log_store" field. +func LogStoreNotNil() predicate.Task { + return predicate.Task(sql.FieldNotNull(FieldLogStore)) +} + +// LogStoreEqualFold applies the EqualFold predicate on the "log_store" field. +func LogStoreEqualFold(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldEqualFold(FieldLogStore, vc)) +} + +// LogStoreContainsFold applies the ContainsFold predicate on the "log_store" field. +func LogStoreContainsFold(v consts.LogStore) predicate.Task { + vc := string(v) + return predicate.Task(sql.FieldContainsFold(FieldLogStore, vc)) +} + // CreatedAtEQ applies the EQ predicate on the "created_at" field. func CreatedAtEQ(v time.Time) predicate.Task { return predicate.Task(sql.FieldEQ(FieldCreatedAt, v)) diff --git a/backend/db/task_create.go b/backend/db/task_create.go index bafeb7a8..ff3cec0e 100644 --- a/backend/db/task_create.go +++ b/backend/db/task_create.go @@ -111,6 +111,20 @@ func (_c *TaskCreate) SetStatus(v consts.TaskStatus) *TaskCreate { return _c } +// SetLogStore sets the "log_store" field. +func (_c *TaskCreate) SetLogStore(v consts.LogStore) *TaskCreate { + _c.mutation.SetLogStore(v) + return _c +} + +// SetNillableLogStore sets the "log_store" field if the given value is not nil. +func (_c *TaskCreate) SetNillableLogStore(v *consts.LogStore) *TaskCreate { + if v != nil { + _c.SetLogStore(*v) + } + return _c +} + // SetCreatedAt sets the "created_at" field. func (_c *TaskCreate) SetCreatedAt(v time.Time) *TaskCreate { _c.mutation.SetCreatedAt(v) @@ -409,6 +423,10 @@ func (_c *TaskCreate) createSpec() (*Task, *sqlgraph.CreateSpec) { _spec.SetField(task.FieldStatus, field.TypeString, value) _node.Status = value } + if value, ok := _c.mutation.LogStore(); ok { + _spec.SetField(task.FieldLogStore, field.TypeString, value) + _node.LogStore = &value + } if value, ok := _c.mutation.CreatedAt(); ok { _spec.SetField(task.FieldCreatedAt, field.TypeTime, value) _node.CreatedAt = value @@ -698,6 +716,24 @@ func (u *TaskUpsert) UpdateStatus() *TaskUpsert { return u } +// SetLogStore sets the "log_store" field. +func (u *TaskUpsert) SetLogStore(v consts.LogStore) *TaskUpsert { + u.Set(task.FieldLogStore, v) + return u +} + +// UpdateLogStore sets the "log_store" field to the value that was provided on create. +func (u *TaskUpsert) UpdateLogStore() *TaskUpsert { + u.SetExcluded(task.FieldLogStore) + return u +} + +// ClearLogStore clears the value of the "log_store" field. +func (u *TaskUpsert) ClearLogStore() *TaskUpsert { + u.SetNull(task.FieldLogStore) + return u +} + // SetCreatedAt sets the "created_at" field. func (u *TaskUpsert) SetCreatedAt(v time.Time) *TaskUpsert { u.Set(task.FieldCreatedAt, v) @@ -940,6 +976,27 @@ func (u *TaskUpsertOne) UpdateStatus() *TaskUpsertOne { }) } +// SetLogStore sets the "log_store" field. +func (u *TaskUpsertOne) SetLogStore(v consts.LogStore) *TaskUpsertOne { + return u.Update(func(s *TaskUpsert) { + s.SetLogStore(v) + }) +} + +// UpdateLogStore sets the "log_store" field to the value that was provided on create. +func (u *TaskUpsertOne) UpdateLogStore() *TaskUpsertOne { + return u.Update(func(s *TaskUpsert) { + s.UpdateLogStore() + }) +} + +// ClearLogStore clears the value of the "log_store" field. +func (u *TaskUpsertOne) ClearLogStore() *TaskUpsertOne { + return u.Update(func(s *TaskUpsert) { + s.ClearLogStore() + }) +} + // SetCreatedAt sets the "created_at" field. func (u *TaskUpsertOne) SetCreatedAt(v time.Time) *TaskUpsertOne { return u.Update(func(s *TaskUpsert) { @@ -1358,6 +1415,27 @@ func (u *TaskUpsertBulk) UpdateStatus() *TaskUpsertBulk { }) } +// SetLogStore sets the "log_store" field. +func (u *TaskUpsertBulk) SetLogStore(v consts.LogStore) *TaskUpsertBulk { + return u.Update(func(s *TaskUpsert) { + s.SetLogStore(v) + }) +} + +// UpdateLogStore sets the "log_store" field to the value that was provided on create. +func (u *TaskUpsertBulk) UpdateLogStore() *TaskUpsertBulk { + return u.Update(func(s *TaskUpsert) { + s.UpdateLogStore() + }) +} + +// ClearLogStore clears the value of the "log_store" field. +func (u *TaskUpsertBulk) ClearLogStore() *TaskUpsertBulk { + return u.Update(func(s *TaskUpsert) { + s.ClearLogStore() + }) +} + // SetCreatedAt sets the "created_at" field. func (u *TaskUpsertBulk) SetCreatedAt(v time.Time) *TaskUpsertBulk { return u.Update(func(s *TaskUpsert) { diff --git a/backend/db/task_update.go b/backend/db/task_update.go index 454265fc..67deab93 100644 --- a/backend/db/task_update.go +++ b/backend/db/task_update.go @@ -173,6 +173,26 @@ func (_u *TaskUpdate) SetNillableStatus(v *consts.TaskStatus) *TaskUpdate { return _u } +// SetLogStore sets the "log_store" field. +func (_u *TaskUpdate) SetLogStore(v consts.LogStore) *TaskUpdate { + _u.mutation.SetLogStore(v) + return _u +} + +// SetNillableLogStore sets the "log_store" field if the given value is not nil. +func (_u *TaskUpdate) SetNillableLogStore(v *consts.LogStore) *TaskUpdate { + if v != nil { + _u.SetLogStore(*v) + } + return _u +} + +// ClearLogStore clears the value of the "log_store" field. +func (_u *TaskUpdate) ClearLogStore() *TaskUpdate { + _u.mutation.ClearLogStore() + return _u +} + // SetCreatedAt sets the "created_at" field. func (_u *TaskUpdate) SetCreatedAt(v time.Time) *TaskUpdate { _u.mutation.SetCreatedAt(v) @@ -529,6 +549,12 @@ func (_u *TaskUpdate) sqlSave(ctx context.Context) (_node int, err error) { if value, ok := _u.mutation.Status(); ok { _spec.SetField(task.FieldStatus, field.TypeString, value) } + if value, ok := _u.mutation.LogStore(); ok { + _spec.SetField(task.FieldLogStore, field.TypeString, value) + } + if _u.mutation.LogStoreCleared() { + _spec.ClearField(task.FieldLogStore, field.TypeString) + } if value, ok := _u.mutation.CreatedAt(); ok { _spec.SetField(task.FieldCreatedAt, field.TypeTime, value) } @@ -968,6 +994,26 @@ func (_u *TaskUpdateOne) SetNillableStatus(v *consts.TaskStatus) *TaskUpdateOne return _u } +// SetLogStore sets the "log_store" field. +func (_u *TaskUpdateOne) SetLogStore(v consts.LogStore) *TaskUpdateOne { + _u.mutation.SetLogStore(v) + return _u +} + +// SetNillableLogStore sets the "log_store" field if the given value is not nil. +func (_u *TaskUpdateOne) SetNillableLogStore(v *consts.LogStore) *TaskUpdateOne { + if v != nil { + _u.SetLogStore(*v) + } + return _u +} + +// ClearLogStore clears the value of the "log_store" field. +func (_u *TaskUpdateOne) ClearLogStore() *TaskUpdateOne { + _u.mutation.ClearLogStore() + return _u +} + // SetCreatedAt sets the "created_at" field. func (_u *TaskUpdateOne) SetCreatedAt(v time.Time) *TaskUpdateOne { _u.mutation.SetCreatedAt(v) @@ -1354,6 +1400,12 @@ func (_u *TaskUpdateOne) sqlSave(ctx context.Context) (_node *Task, err error) { if value, ok := _u.mutation.Status(); ok { _spec.SetField(task.FieldStatus, field.TypeString, value) } + if value, ok := _u.mutation.LogStore(); ok { + _spec.SetField(task.FieldLogStore, field.TypeString, value) + } + if _u.mutation.LogStoreCleared() { + _spec.ClearField(task.FieldLogStore, field.TypeString) + } if value, ok := _u.mutation.CreatedAt(); ok { _spec.SetField(task.FieldCreatedAt, field.TypeTime, value) } diff --git a/backend/docs/swagger.json b/backend/docs/swagger.json index 1f24eaeb..49222a73 100644 --- a/backend/docs/swagger.json +++ b/backend/docs/swagger.json @@ -4529,58 +4529,6 @@ } } }, - "/api/v1/users/mcp/tools": { - "get": { - "security": [ - { - "MonkeyCodeAIAuth": [] - } - ], - "description": "获取当前登录用户可使用的 MCP Tool 列表及其启用状态", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "【用户】MCP 配置" - ], - "summary": "获取当前用户可见的 MCP Tool 列表", - "responses": { - "200": { - "description": "成功", - "schema": { - "allOf": [ - { - "$ref": "#/definitions/web.Resp" - }, - { - "type": "object", - "properties": { - "data": { - "$ref": "#/definitions/domain.ListUserMCPToolsResp" - } - } - } - ] - } - }, - "401": { - "description": "未授权", - "schema": { - "$ref": "#/definitions/web.Resp" - } - }, - "500": { - "description": "服务器内部错误", - "schema": { - "$ref": "#/definitions/web.Resp" - } - } - } - } - }, "/api/v1/users/mcp/tools/{id}": { "put": { "security": [ @@ -7311,7 +7259,7 @@ "MonkeyCodeAIAuth": [] } ], - "description": "根据 cursor 向前翻页查询任务的历史论次。limit 为论次数（非条目数），\nlimit=2 表示返回 2 论的完整消息。返回的 chunks 按时间倒序排列（最新在前）。", + "description": "根据 cursor 向前翻页查询任务的历史轮次。limit 为轮次数（非条目数），\nlimit=2 表示返回 2 轮的完整消息。返回的 chunks 按时间倒序排列（最新在前）。", "consumes": [ "application/json" ], @@ -7321,7 +7269,7 @@ "tags": [ "【用户】任务管理" ], - "summary": "查询任务历史论次", + "summary": "查询任务历史轮次", "parameters": [ { "type": "string", @@ -7332,13 +7280,13 @@ }, { "type": "string", - "description": "游标（时间戳 Unix ns）", + "description": "分页游标", "name": "cursor", "in": "query" }, { "type": "integer", - "description": "论次数（默认 2，上限 10）", + "description": "轮次数（默认 2，上限 10）", "name": "limit", "in": "query" } @@ -7457,7 +7405,7 @@ "MonkeyCodeAIAuth": [] } ], - "description": "功能定位：该接口通过 WebSocket 仅做 Agent ↔ 前端 的数据代理与转发，不进行任何包体解析或改写。所有数据以原始格式透传并存储。\n数据格式约定：当前仅支持文本帧透传。服务端将 Agent 的原始文本数据包装为如下结构返回给前端（对应 domain.TaskStream）：\n```json\n{ \"type\": \"string\", \"data\": \"string\", \"kind\": \"string\", \"timestamp\": 0 }\n```\ntype 字段说明：\n- task-started: 本轮任务启动\n- task-ended: 本轮任务结束\n- task-error: 本轮任务发生错误\n- task-running: 任务正在运行\n- task-event: 任务临时事件, 不持久化\n- file-change: 文件变动事件\n- permission-resp: 用户的权限响应\n- auto-approve: 开启自动批准\n- disable-auto-approve: 关闭自动批准\n- user-input: 用户输入\n- user-cancel: 取消当前操作，不会终止任务\n- reply-question: 回复 AI 的提问\n- cursor: 历史游标，用于通过 /rounds 接口加载更早的论次\n\ncursor 消息结构：\n```json\n{ \"type\": \"cursor\", \"data\": { \"cursor\": \"\u003clastTaskStartedTS_ns\u003e\", \"has_more\": true }, \"timestamp\": 0 }\n```\n- cursor: 当前论次 task-started 的时间戳（Unix 纳秒），作为 GET /rounds 接口的 cursor 参数向前翻页\n- has_more: 是否存在更早的论次。为 false 时表示当前论次即为第一论次，无需再翻页", + "description": "功能定位：该接口通过 WebSocket 仅做 Agent ↔ 前端 的数据代理与转发，不进行任何包体解析或改写。所有数据以原始格式透传并存储。\n数据格式约定：当前仅支持文本帧透传。服务端将 Agent 的原始文本数据包装为如下结构返回给前端（对应 domain.TaskStream）：\n```json\n{ \"type\": \"string\", \"data\": \"string\", \"kind\": \"string\", \"timestamp\": 0 }\n```\ntype 字段说明：\n- task-started: 本轮任务启动\n- task-ended: 本轮任务结束\n- task-error: 本轮任务发生错误\n- task-running: 任务正在运行\n- task-event: 任务临时事件, 不持久化\n- file-change: 文件变动事件\n- permission-resp: 用户的权限响应\n- auto-approve: 开启自动批准\n- disable-auto-approve: 关闭自动批准\n- user-input: 用户输入\n- user-cancel: 取消当前操作，不会终止任务\n- reply-question: 回复 AI 的提问\n- cursor: 历史游标，用于通过 /rounds 接口加载更早的轮次\n\ncursor 消息结构：\n```json\n{ \"type\": \"cursor\", \"data\": { \"cursor\": \"\u003cnextCursor\u003e\", \"has_more\": true }, \"timestamp\": 0 }\n```\n- cursor: 当前分页游标，作为 GET /rounds 接口的 cursor 参数向前翻页\n- has_more: 是否存在更早的轮次。为 false 时表示当前轮次即为第一轮，无需再翻页", "consumes": [ "application/json" ], @@ -7478,7 +7426,7 @@ }, { "type": "string", - "description": "模式：new(等待用户输入)|attach(仅拉取当前论次)，默认 new", + "description": "模式：new(等待用户输入)|attach(仅拉取当前轮次)，默认 new", "name": "mode", "in": "query" } @@ -7699,6 +7647,17 @@ "InterfaceTypeAnthropic" ] }, + "consts.LogStore": { + "type": "string", + "enum": [ + "loki", + "clickhouse" + ], + "x-enum-varnames": [ + "LogStoreLoki", + "LogStoreClickHouse" + ] + }, "consts.ModelProvider": { "type": "string", "enum": [ @@ -7959,11 +7918,13 @@ "type": "string", "enum": [ "active", - "inactive" + "inactive", + "banded" ], "x-enum-varnames": [ "UserStatusActive", - "UserStatusInactive" + "UserStatusInactive", + "UserStatusBanded" ] }, "db.Cursor": { @@ -9134,17 +9095,6 @@ } } }, - "domain.ListUserMCPToolsResp": { - "type": "object", - "properties": { - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/domain.MCPTool" - } - } - } - }, "domain.ListUserMCPUpstreamsResp": { "type": "object", "properties": { @@ -9196,7 +9146,7 @@ "type": "integer" }, "scope": { - "type": "string" + "$ref": "#/definitions/mcptool.Scope" } } }, @@ -9234,7 +9184,7 @@ "type": "string" }, "scope": { - "type": "string" + "$ref": "#/definitions/mcpupstream.Scope" }, "slug": { "type": "string" @@ -9324,6 +9274,9 @@ }, "updated_at": { "type": "integer" + }, + "weight": { + "type": "integer" } } }, @@ -9630,6 +9583,12 @@ "image": { "$ref": "#/definitions/domain.Image" }, + "last_active_at": { + "type": "integer" + }, + "log_store": { + "$ref": "#/definitions/consts.LogStore" + }, "model": { "$ref": "#/definitions/domain.Model" }, @@ -9908,6 +9867,12 @@ "image": { "$ref": "#/definitions/domain.Image" }, + "last_active_at": { + "type": "integer" + }, + "log_store": { + "$ref": "#/definitions/consts.LogStore" + }, "model": { "$ref": "#/definitions/domain.Model" }, @@ -10016,7 +9981,7 @@ "type": "boolean" }, "next_cursor": { - "description": "下一页游标（最早条目的时间戳 ns）", + "description": "下一页游标", "type": "string" } } @@ -10802,6 +10767,28 @@ } } }, + "mcptool.Scope": { + "type": "string", + "enum": [ + "user", + "platform" + ], + "x-enum-varnames": [ + "ScopeUser", + "ScopePlatform" + ] + }, + "mcpupstream.Scope": { + "type": "string", + "enum": [ + "user", + "platform" + ], + "x-enum-varnames": [ + "ScopeUser", + "ScopePlatform" + ] + }, "taskflow.File": { "type": "object", "properties": { diff --git a/backend/domain/task.go b/backend/domain/task.go index b42bcf35..024b97e0 100644 --- a/backend/domain/task.go +++ b/backend/domain/task.go @@ -35,6 +35,7 @@ type TaskUsecase interface { // TaskRepo 任务数据访问接口 type TaskRepo interface { GetByID(ctx context.Context, id uuid.UUID) (*db.Task, error) + GetLogStore(ctx context.Context, id uuid.UUID) (consts.LogStore, error) Stat(ctx context.Context, id uuid.UUID) (*TaskStats, error) StatByIDs(ctx context.Context, ids []uuid.UUID) (map[uuid.UUID]*TaskStats, error) Info(ctx context.Context, user *User, id uuid.UUID, isPrivileged bool) (*db.Task, error) @@ -211,6 +212,7 @@ type Task struct { Title string `json:"title"` Summary string `json:"summary"` Status consts.TaskStatus `json:"status"` + LogStore consts.LogStore `json:"log_store"` VirtualMachine *VirtualMachine `json:"virtualmachine"` CreatedAt int64 `json:"created_at"` LastActiveAt int64 `json:"last_active_at"` @@ -248,6 +250,12 @@ func (t *Task) From(src *db.Task) *Task { t.Title = src.Title t.Summary = src.Summary t.Status = src.Status + if src.LogStore != nil { + t.LogStore = *src.LogStore + } + if t.LogStore == "" { + t.LogStore = consts.LogStoreLoki + } t.CreatedAt = src.CreatedAt.Unix() t.LastActiveAt = src.LastActiveAt.Unix() t.CompletedAt = src.CompletedAt.Unix() @@ -301,17 +309,17 @@ type TaskControlReq struct { ID uuid.UUID `json:"id" query:"id" validate:"required"` // 任务 id } -// TaskRoundsReq 查询任务历史论次请求（向前翻页） +// TaskRoundsReq 查询任务历史轮次请求（向前翻页） type TaskRoundsReq struct { ID uuid.UUID `json:"id" query:"id" validate:"required"` // 任务 ID - Cursor string `json:"cursor" query:"cursor"` // 游标（时间戳 Unix ns，从此时间点往前查询） - Limit int `json:"limit" query:"limit"` // 返回的论次数（默认 2，上限 10） + Cursor string `json:"cursor" query:"cursor"` // 分页游标 + Limit int `json:"limit" query:"limit"` // 返回的轮次数（默认 2，上限 10） } -// TaskRoundsResp 查询任务历史论次响应 +// TaskRoundsResp 查询任务历史轮次响应 type TaskRoundsResp struct { Chunks []*TaskChunkEntry `json:"chunks"` - NextCursor string `json:"next_cursor,omitempty"` // 下一页游标（最早条目的时间戳 ns） + NextCursor string `json:"next_cursor,omitempty"` // 下一页游标 HasMore bool `json:"has_more"` } diff --git a/backend/domain/task_logstore_test.go b/backend/domain/task_logstore_test.go new file mode 100644 index 00000000..4fb51c35 --- /dev/null +++ b/backend/domain/task_logstore_test.go @@ -0,0 +1,50 @@ +package domain_test + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/consts" + "github.com/chaitin/MonkeyCode/backend/db" + "github.com/chaitin/MonkeyCode/backend/domain" + "github.com/chaitin/MonkeyCode/backend/pkg/taskflow" +) + +func TestTaskFromIncludesLogStore(t *testing.T) { + src := &db.Task{ + ID: uuid.MustParse("11111111-1111-1111-1111-111111111111"), + UserID: uuid.MustParse("22222222-2222-2222-2222-222222222222"), + Kind: consts.TaskTypeDevelop, + Status: consts.TaskStatusPending, + Content: "init task", + CreatedAt: time.Unix(1710000000, 0), + } + store := consts.LogStoreClickHouse + src.LogStore = &store + + got := (&domain.Task{}).From(src) + if got.LogStore != consts.LogStoreClickHouse { + t.Fatalf("log store = %q, want %q", got.LogStore, consts.LogStoreClickHouse) + } +} + +func TestCreateTaskReqIncludesLogStore(t *testing.T) { + req := taskflow.CreateTaskReq{ + ID: uuid.MustParse("33333333-3333-3333-3333-333333333333"), + VMID: "vm-1", + Text: "hello", + LogStore: "clickhouse", + } + + b, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), `"log_store":"clickhouse"`) { + t.Fatalf("marshaled request missing log_store: %s", string(b)) + } +} diff --git a/backend/ent/schema/task.go b/backend/ent/schema/task.go index d697660c..c26b60e4 100644 --- a/backend/ent/schema/task.go +++ b/backend/ent/schema/task.go @@ -43,6 +43,7 @@ func (Task) Fields() []ent.Field { field.Text("title").Optional(), field.Text("summary").Optional(), field.String("status").GoType(consts.TaskStatus("")), + field.String("log_store").GoType(consts.LogStore("")).Optional().Nillable(), field.Time("created_at").Default(time.Now), field.Time("last_active_at").Default(time.Now), field.Time("updated_at").Default(time.Now).UpdateDefault(time.Now), diff --git a/backend/go.mod b/backend/go.mod index 7602e127..6a95a9e6 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -4,6 +4,8 @@ go 1.25.0 require ( entgo.io/ent v0.14.5 + github.com/ClickHouse/clickhouse-go/v2 v2.45.0 + github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/GoYoko/web v1.6.0 github.com/ackcoder/go-cap v1.1.3 github.com/alicebob/miniredis/v2 v2.35.0 @@ -32,8 +34,10 @@ require ( require ( ariga.io/atlas v0.32.1-0.20250325101103-175b25e1c1b9 // indirect github.com/BurntSushi/toml v1.6.0 // indirect + github.com/ClickHouse/ch-go v0.71.0 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/aliyun/alibaba-cloud-sdk-go v1.61.1376 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/bradleyfalzon/ghinstallation/v2 v2.17.0 // indirect @@ -41,6 +45,8 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/inflect v0.19.0 // indirect @@ -62,6 +68,7 @@ require ( github.com/hashicorp/hcl/v2 v2.18.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect @@ -70,12 +77,16 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nicksnyder/go-i18n/v2 v2.6.1 // indirect + github.com/paulmach/orb v0.12.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/rs/zerolog v1.34.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/satori/go.uuid v1.2.0 // indirect + github.com/segmentio/asm v1.2.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed // indirect github.com/shurcooL/graphql v0.0.0-20240915155400-7ee5256398cf // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect diff --git a/backend/go.sum b/backend/go.sum index 17672e86..9e0b5650 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -6,8 +6,12 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= -github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoyRM= +github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw= +github.com/ClickHouse/clickhouse-go/v2 v2.45.0 h1:iHt15nA4iYhfde5bDQAcLAat9BAh7B5ksPRNRa4UI7s= +github.com/ClickHouse/clickhouse-go/v2 v2.45.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/GoYoko/web v1.6.0 h1:gwnErVfMSDKc8XwJIW9iiMBNuzwx1E3QwqPiGwEW76U= github.com/GoYoko/web v1.6.0/go.mod h1:MNOw+4KjmtRzUabIMqWK3t59yibnO1sDCp3EcLCmJVc= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -22,6 +26,8 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.61.1376 h1:lExo7heZgdFn5AbaNJEllbA0KSJ github.com/aliyun/alibaba-cloud-sdk-go v1.61.1376/go.mod h1:9CMdKNL3ynIGPpfTcdwTvIm8SGuAZYYC4jFVSSvE1YQ= github.com/aliyun/alibabacloud-nls-go-sdk v1.1.1 h1:LjItoNZuu5xHlsByFo+kr3nGa4LRIESCGWhfurayxBg= github.com/aliyun/alibabacloud-nls-go-sdk v1.1.1/go.mod h1:4BDMUKpEaP/Ct79w0ozR0nbnEj49g1k3mrgX/IKG5I4= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= @@ -50,10 +56,10 @@ github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -66,6 +72,10 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -93,6 +103,10 @@ github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXe github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -139,6 +153,10 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -179,20 +197,26 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/nicksnyder/go-i18n/v2 v2.6.1 h1:JDEJraFsQE17Dut9HFDHzCoAWGEQJom5s0TRd17NIEQ= github.com/nicksnyder/go-i18n/v2 v2.6.1/go.mod h1:Vee0/9RD3Quc/NmwEjzzD7VTZ+Ir7QbXocrkhOzmUKA= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/palantir/go-githubapp v0.38.1 h1:Pkb3yVErM0G85piU2DbJ9c1jlKc32bUZ/LMQqFqY6qQ= github.com/palantir/go-githubapp v0.38.1/go.mod h1:5MYLd1/cAhKITPQUDbdZ/vR8SsM7uGgohLCixOEbnGU= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= +github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -214,8 +238,12 @@ github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TV github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed h1:KT7hI8vYXgU0s2qaMkrfq9tCA1w/iEPgfredVP+4Tzw= github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= github.com/shurcooL/graphql v0.0.0-20240915155400-7ee5256398cf h1:o1uxfymjZ7jZ4MsgCErcwWGtVKSiNAXtS59Lhs6uI/g= @@ -234,15 +262,23 @@ github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= @@ -255,6 +291,7 @@ github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= gitlab.com/gitlab-org/api/client-go v1.46.0 h1:YxBWFZIFYKcGESCb9fpkwzouo+apyB9pr/XTWzNoL24= gitlab.com/gitlab-org/api/client-go v1.46.0/go.mod h1:FtgyU6g2HS5+fMhw6nLK96GBEEBx5MzntOiJWfIaiN8= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.67.0 h1:0FKdyaoWXDmSCpQuv3m2UiJIRNxb1CK1mILy5QyKxc4= @@ -282,6 +319,7 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -293,6 +331,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= @@ -300,18 +339,25 @@ golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -327,6 +373,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/backend/migration/clickhouse/task_logs.sql b/backend/migration/clickhouse/task_logs.sql new file mode 100644 index 00000000..cab822f8 --- /dev/null +++ b/backend/migration/clickhouse/task_logs.sql @@ -0,0 +1,20 @@ +CREATE TABLE IF NOT EXISTS task_logs +ON CLUSTER mcai_cluster +( + task_id UUID, + ts DateTime64(9, 'UTC'), + event LowCardinality(String), + kind LowCardinality(String), + turn_seq UInt32, + data String CODEC(ZSTD(3)), + msg_seq_start UInt64, + msg_seq_end UInt64, + source LowCardinality(String), + log_version UInt16, + ingest_id UUID +) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/mcai/task_logs', '{replica}') +PARTITION BY toYYYYMM(ts) +ORDER BY (task_id, turn_seq, ts, msg_seq_start, ingest_id) +TTL ts + INTERVAL 60 DAY TO VOLUME 'warm' +SETTINGS storage_policy = 'hot_warm'; diff --git a/backend/pkg/clickhouse/client.go b/backend/pkg/clickhouse/client.go new file mode 100644 index 00000000..a8718294 --- /dev/null +++ b/backend/pkg/clickhouse/client.go @@ -0,0 +1,158 @@ +package clickhouse + +import ( + "context" + "database/sql" + "fmt" + "log/slog" + "net/url" + "regexp" + "strings" + "time" + + _ "github.com/ClickHouse/clickhouse-go/v2" + + "github.com/chaitin/MonkeyCode/backend/config" +) + +const TaskLogTable = "task_logs" + +var clickHouseIdentifierRE = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +type Client struct { + db *sql.DB + table string +} + +func New(cfg config.ClickHouse, logger *slog.Logger) (*Client, error) { + if strings.TrimSpace(cfg.Addr) == "" { + return nil, nil + } + table, err := NormalizeTable(cfg.Table) + if err != nil { + return nil, err + } + + dsn, err := buildDSN(cfg) + if err != nil { + return nil, err + } + + db, err := sql.Open("clickhouse", dsn) + if err != nil { + return nil, err + } + applyPoolOptions(db, cfg) + if err := db.Ping(); err != nil { + _ = db.Close() + return nil, err + } + if logger != nil { + logger.With("component", "clickhouse").Info("clickhouse connection established") + } + return NewWithDBAndTable(db, table), nil +} + +func NewWithDB(db *sql.DB) *Client { + return &Client{db: db, table: TaskLogTable} +} + +func NewWithDBAndTable(db *sql.DB, table string) *Client { + table, err := NormalizeTable(table) + if err != nil { + table = TaskLogTable + } + return &Client{db: db, table: table} +} + +func (c *Client) Table() string { + if c == nil || c.table == "" { + return TaskLogTable + } + return c.table +} + +func validateClickHouseIdentifier(name, label string) error { + if !clickHouseIdentifierRE.MatchString(name) { + return fmt.Errorf("invalid clickhouse %s: %q", label, name) + } + return nil +} + +func NormalizeTable(table string) (string, error) { + table = strings.TrimSpace(table) + if table == "" { + table = TaskLogTable + } + if err := validateClickHouseIdentifier(table, "table"); err != nil { + return "", err + } + return table, nil +} + +func (c *Client) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) { + return c.db.QueryContext(ctx, query, args...) +} + +func (c *Client) QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row { + return c.db.QueryRowContext(ctx, query, args...) +} + +func (c *Client) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + return c.db.ExecContext(ctx, query, args...) +} + +func applyPoolOptions(db *sql.DB, cfg config.ClickHouse) { + if cfg.MaxOpenConns > 0 { + db.SetMaxOpenConns(cfg.MaxOpenConns) + } + if cfg.MaxIdleConns > 0 { + db.SetMaxIdleConns(cfg.MaxIdleConns) + } + if lifetime := connMaxLifetime(cfg.ConnMaxLifetime); lifetime > 0 { + db.SetConnMaxLifetime(lifetime) + } +} + +func connMaxLifetime(seconds int) time.Duration { + if seconds <= 0 { + return 0 + } + return time.Duration(seconds) * time.Second +} + +func buildDSN(cfg config.ClickHouse) (string, error) { + username, password := readCredentials(cfg) + return buildDSNWithCredentials(cfg, username, password) +} + +func readCredentials(cfg config.ClickHouse) (string, string) { + username := strings.TrimSpace(cfg.ReadUsername) + password := cfg.ReadPassword + if username == "" { + username = cfg.Username + password = cfg.Password + } + return username, password +} + +func buildDSNWithCredentials(cfg config.ClickHouse, username, password string) (string, error) { + addr := strings.TrimSpace(cfg.Addr) + if addr == "" { + return "", fmt.Errorf("clickhouse addr is empty") + } + if !strings.Contains(addr, "://") { + addr = "clickhouse://" + addr + } + u, err := url.Parse(addr) + if err != nil { + return "", err + } + if username != "" { + u.User = url.UserPassword(username, password) + } + if cfg.Database != "" { + u.Path = "/" + strings.TrimPrefix(cfg.Database, "/") + } + return u.String(), nil +} diff --git a/backend/pkg/clickhouse/client_test.go b/backend/pkg/clickhouse/client_test.go new file mode 100644 index 00000000..1fc2cb27 --- /dev/null +++ b/backend/pkg/clickhouse/client_test.go @@ -0,0 +1,111 @@ +package clickhouse + +import ( + "database/sql" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + + "github.com/chaitin/MonkeyCode/backend/config" +) + +func TestApplyPoolOptions(t *testing.T) { + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + applyPoolOptions(db, config.ClickHouse{ + MaxOpenConns: 64, + MaxIdleConns: 32, + ConnMaxLifetime: 30, + }) + + stats := db.Stats() + if stats.MaxOpenConnections != 64 { + t.Fatalf("max open connections = %d, want 64", stats.MaxOpenConnections) + } +} + +func TestConnMaxLifetime(t *testing.T) { + if got := connMaxLifetime(30); got != 30*time.Second { + t.Fatalf("conn max lifetime = %s, want 30s", got) + } + if got := connMaxLifetime(0); got != 0 { + t.Fatalf("zero conn max lifetime = %s, want 0", got) + } +} + +func TestNormalizeTableUsesConfiguredTable(t *testing.T) { + table, err := NormalizeTable("task_logs_test") + if err != nil { + t.Fatal(err) + } + if table != "task_logs_test" { + t.Fatalf("table = %q, want task_logs_test", table) + } +} + +func TestNormalizeTableDefaultsToTaskLogTable(t *testing.T) { + table, err := NormalizeTable("") + if err != nil { + t.Fatal(err) + } + if table != TaskLogTable { + t.Fatalf("table = %q, want %s", table, TaskLogTable) + } +} + +func TestNormalizeTableRejectsUnsafeTableName(t *testing.T) { + _, err := NormalizeTable("task_logs; DROP TABLE task_logs") + if err == nil { + t.Fatal("expected unsafe table name error") + } +} + +func TestBuildDSNUsesSingleChproxyEndpoint(t *testing.T) { + dsn, err := buildDSN(config.ClickHouse{ + Addr: "chproxy:9000", + Database: "monkeycode", + ReadUsername: "mc_reader", + ReadPassword: "reader-secret", + }) + if err != nil { + t.Fatal(err) + } + if dsn != "clickhouse://mc_reader:reader-secret@chproxy:9000/monkeycode" { + t.Fatalf("dsn = %q, want chproxy endpoint", dsn) + } +} + +func TestBuildDSNPreservesHTTPChproxyEndpoint(t *testing.T) { + dsn, err := buildDSN(config.ClickHouse{ + Addr: "http://chproxy:8123", + Database: "mcai", + ReadUsername: "mc_reader", + ReadPassword: "reader-secret", + }) + if err != nil { + t.Fatal(err) + } + if dsn != "http://mc_reader:reader-secret@chproxy:8123/mcai" { + t.Fatalf("dsn = %q, want http chproxy endpoint", dsn) + } +} + +func TestBuildDSNFallsBackToLegacyCredentials(t *testing.T) { + dsn, err := buildDSN(config.ClickHouse{ + Addr: "chproxy:9000", + Database: "monkeycode", + Username: "legacy", + Password: "legacy-secret", + }) + if err != nil { + t.Fatal(err) + } + if dsn != "clickhouse://legacy:legacy-secret@chproxy:9000/monkeycode" { + t.Fatalf("dsn = %q, want legacy credentials", dsn) + } +} diff --git a/backend/pkg/lifecycle/taskhook.go b/backend/pkg/lifecycle/taskhook.go index 614a6705..c750654b 100644 --- a/backend/pkg/lifecycle/taskhook.go +++ b/backend/pkg/lifecycle/taskhook.go @@ -122,6 +122,9 @@ func (h *TaskHook) handleProcessing(ctx context.Context, id uuid.UUID, metadata h.logger.With("task_id", id, "error", err).ErrorContext(ctx, "failed to unmarshal CreateTaskReq") return fmt.Errorf("failed to unmarshal CreateTaskReq: %w", err) } + if t.LogStore != nil { + createReq.LogStore = string(*t.LogStore) + } h.logger.With("task_id", id).InfoContext(ctx, "creating taskflow task") if err := h.taskflow.TaskManager().Create(ctx, createReq); err != nil { diff --git a/backend/pkg/lifecycle/taskhook_test.go b/backend/pkg/lifecycle/taskhook_test.go index 15867dbd..5ec11669 100644 --- a/backend/pkg/lifecycle/taskhook_test.go +++ b/backend/pkg/lifecycle/taskhook_test.go @@ -87,6 +87,17 @@ func (s *taskHookRepoStub) GetByID(ctx context.Context, id uuid.UUID) (*db.Task, return s.client.Task.Get(ctx, id) } +func (s *taskHookRepoStub) GetLogStore(ctx context.Context, id uuid.UUID) (consts.LogStore, error) { + tk, err := s.client.Task.Get(ctx, id) + if err != nil { + return "", err + } + if tk.LogStore == nil { + return "", nil + } + return *tk.LogStore, nil +} + func (s *taskHookRepoStub) Stat(context.Context, uuid.UUID) (*domain.TaskStats, error) { panic("unexpected call to Stat") } diff --git a/backend/pkg/register.go b/backend/pkg/register.go index 72a35a4b..24b97b73 100644 --- a/backend/pkg/register.go +++ b/backend/pkg/register.go @@ -14,6 +14,7 @@ import ( "github.com/chaitin/MonkeyCode/backend/domain" "github.com/chaitin/MonkeyCode/backend/middleware" "github.com/chaitin/MonkeyCode/backend/pkg/captcha" + "github.com/chaitin/MonkeyCode/backend/pkg/clickhouse" "github.com/chaitin/MonkeyCode/backend/pkg/delayqueue" "github.com/chaitin/MonkeyCode/backend/pkg/email" "github.com/chaitin/MonkeyCode/backend/pkg/lifecycle" @@ -28,6 +29,7 @@ import ( "github.com/chaitin/MonkeyCode/backend/pkg/store" "github.com/chaitin/MonkeyCode/backend/pkg/tasker" "github.com/chaitin/MonkeyCode/backend/pkg/taskflow" + "github.com/chaitin/MonkeyCode/backend/pkg/tasklog" "github.com/chaitin/MonkeyCode/backend/pkg/ws" ) @@ -126,6 +128,22 @@ func RegisterInfra(i *do.Injector, w ...*web.Web) error { return loki.NewClient(cfg.Loki.Addr), nil }) + do.Provide(i, func(i *do.Injector) (*clickhouse.Client, error) { + cfg := do.MustInvoke[*config.Config](i) + l := do.MustInvoke[*slog.Logger](i) + return clickhouse.New(cfg.ClickHouse, l) + }) + + do.Provide(i, func(i *do.Injector) (*tasklog.Gateway, error) { + lokiClient := do.MustInvoke[*loki.Client](i) + clickhouseClient := do.MustInvoke[*clickhouse.Client](i) + + return &tasklog.Gateway{ + Loki: tasklog.NewLokiProvider(lokiClient), + ClickHouse: tasklog.NewClickHouseProvider(clickhouseClient), + }, nil + }) + // TaskSummary Queue do.Provide(i, func(i *do.Injector) (*delayqueue.TaskSummaryQueue, error) { r := do.MustInvoke[*redis.Client](i) diff --git a/backend/pkg/taskflow/types.go b/backend/pkg/taskflow/types.go index f8ba1151..f7a40617 100644 --- a/backend/pkg/taskflow/types.go +++ b/backend/pkg/taskflow/types.go @@ -133,6 +133,7 @@ type CreateVirtualMachineReq struct { Memory uint64 `json:"memory"` InstallCodingAgents bool `json:"install_coding_agents"` Envs []string `json:"envs,omitempty"` + LogStore string `json:"log_store,omitempty"` } // Git 仓库信息 @@ -278,6 +279,14 @@ type CheckTokenReq struct { MachineID string `json:"machine_id"` } +type GetTaskLogStoreReq struct { + TaskID uuid.UUID `json:"task_id" validate:"required"` +} + +type GetTaskLogStoreResp struct { + LogStore string `json:"log_store"` +} + // TokenUser token 中的用户信息 type TokenUser struct { ID string `json:"id"` @@ -359,6 +368,7 @@ type AskUserQuestionResponse struct { RequestId string `json:"request_id,omitempty"` AnswersJson string `json:"answers_json,omitempty"` Cancelled bool `json:"cancelled,omitempty"` + LogStore string `json:"log_store,omitempty"` } // ApplyWebClientIPReq 同步 Web 客户端 IP 请求 @@ -494,6 +504,7 @@ type RestartTaskReq struct { RequestId string `json:"request_id,omitempty"` LoadSession bool `json:"load_session"` ExecutionConfig *TaskExecutionConfig `json:"execution_config,omitempty"` + LogStore string `json:"log_store,omitempty"` } // RestartTaskResp 重启任务响应 @@ -519,9 +530,10 @@ type TaskReq struct { // Task 任务信息 type Task struct { - ID uuid.UUID `json:"id"` - Text string `json:"text"` - Image string `json:"image"` + ID uuid.UUID `json:"id"` + Text string `json:"text"` + Image string `json:"image"` + LogStore string `json:"log_store,omitempty"` } // ==================== CreateTask 类型 ==================== @@ -587,6 +599,7 @@ type CreateTaskReq struct { Configs []ConfigFile `json:"configs,omitzero"` McpConfigs []McpServerConfig `json:"mcp_configs,omitzero"` Env map[string]string `json:"env,omitempty"` + LogStore string `json:"log_store,omitempty"` } // ==================== VirtualMachine 查询类型 ==================== diff --git a/backend/pkg/taskflow/types_test.go b/backend/pkg/taskflow/types_test.go index 2e6cadfa..4afc991b 100644 --- a/backend/pkg/taskflow/types_test.go +++ b/backend/pkg/taskflow/types_test.go @@ -2,6 +2,7 @@ package taskflow import ( "encoding/json" + "strings" "testing" "github.com/google/uuid" @@ -62,3 +63,70 @@ func TestRestartTaskReqMarshalExecutionConfig(t *testing.T) { t.Fatalf("config file mode = %v, want %d", file["mode"], mode) } } + +func TestCreateVirtualMachineReqIncludesLogStore(t *testing.T) { + req := CreateVirtualMachineReq{ + UserID: "u-1", + HostID: "h-1", + TaskID: uuid.MustParse("11111111-1111-1111-1111-111111111111"), + LogStore: "clickhouse", + } + + b, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), `"log_store":"clickhouse"`) { + t.Fatalf("marshaled request missing log_store: %s", string(b)) + } +} + +func TestRestartTaskReqIncludesLogStore(t *testing.T) { + req := RestartTaskReq{ + ID: uuid.MustParse("22222222-2222-2222-2222-222222222222"), + RequestId: "req-1", + LogStore: "clickhouse", + } + + b, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), `"log_store":"clickhouse"`) { + t.Fatalf("marshaled request missing log_store: %s", string(b)) + } +} + +func TestAskUserQuestionResponseIncludesLogStore(t *testing.T) { + req := AskUserQuestionResponse{ + TaskId: "task-1", + RequestId: "req-1", + AnswersJson: `{"ok":true}`, + LogStore: "clickhouse", + } + + b, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), `"log_store":"clickhouse"`) { + t.Fatalf("marshaled request missing log_store: %s", string(b)) + } +} + +func TestTaskReqIncludesLogStore(t *testing.T) { + req := TaskReq{ + Task: &Task{ + ID: uuid.MustParse("33333333-3333-3333-3333-333333333333"), + LogStore: "clickhouse", + }, + } + + b, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), `"log_store":"clickhouse"`) { + t.Fatalf("marshaled request missing log_store: %s", string(b)) + } +} diff --git a/backend/pkg/tasklog/clickhouse_provider.go b/backend/pkg/tasklog/clickhouse_provider.go new file mode 100644 index 00000000..261004d0 --- /dev/null +++ b/backend/pkg/tasklog/clickhouse_provider.go @@ -0,0 +1,240 @@ +package tasklog + +import ( + "context" + "database/sql" + "fmt" + "strconv" + "time" + + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/pkg/clickhouse" +) + +type ClickHouseProvider struct { + client *clickhouse.Client +} + +func NewClickHouseProvider(client *clickhouse.Client) *ClickHouseProvider { + return &ClickHouseProvider{client: client} +} + +func (p *ClickHouseProvider) Name() string { + return "clickhouse" +} + +func (p *ClickHouseProvider) QueryLatestTurn(ctx context.Context, taskID uuid.UUID, taskCreatedAt, end time.Time) (*QueryLatestTurnResp, error) { + if p.client == nil { + return nil, ErrProviderUnavailable + } + table := p.client.Table() + + qTurn := fmt.Sprintf(` + SELECT max(turn_seq) + FROM %s + WHERE task_id = ? AND ts >= ? AND ts <= ?`, table) + + var latestTurn sql.NullInt64 + if err := p.client.QueryRowContext(ctx, qTurn, taskID, taskCreatedAt, end).Scan(&latestTurn); err != nil { + return nil, err + } + if !latestTurn.Valid || latestTurn.Int64 <= 0 { + return &QueryLatestTurnResp{}, nil + } + + entries, err := p.queryEntriesByTurn(ctx, taskID, uint32(latestTurn.Int64), taskCreatedAt, end) + if err != nil { + return nil, err + } + + resp := &QueryLatestTurnResp{ + Entries: entries, + } + hasMore, err := p.hasLowerTurn(ctx, taskID, uint32(latestTurn.Int64)) + if err != nil { + return nil, err + } + resp.HasMore = hasMore + if hasMore { + resp.NextCursor = strconv.FormatUint(uint64(latestTurn.Int64), 10) + } + return resp, nil +} + +func (p *ClickHouseProvider) QueryTurns(ctx context.Context, taskID uuid.UUID, _ time.Time, cursor string, limit int) (*QueryTurnsResp, error) { + if p.client == nil { + return nil, ErrProviderUnavailable + } + table := p.client.Table() + if limit <= 0 { + limit = 2 + } + if limit > 10 { + limit = 10 + } + + cursorFilter := "" + args := []any{taskID, taskID} + if cursor != "" { + turn, err := strconv.ParseUint(cursor, 10, 32) + if err != nil { + return nil, err + } + cursorFilter = "AND turn_seq < ?" + args = append(args, uint32(turn)) + } + args = append(args, limit+1) + + q := fmt.Sprintf(` +SELECT ts, event, kind, data, turn_seq +FROM %[1]s +WHERE task_id = ? AND turn_seq IN ( + SELECT DISTINCT turn_seq + FROM %[1]s + WHERE task_id = ? + %[2]s + ORDER BY turn_seq DESC + LIMIT ? +) +ORDER BY turn_seq DESC, ts ASC, msg_seq_start ASC, ingest_id ASC +`, table, cursorFilter) + + rows, err := p.client.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + chunks := make([]*TurnChunk, 0) + seenTurns := make(map[uint32]struct{}, limit+1) + turns := make([]uint32, 0, limit+1) + for rows.Next() { + var ( + ts time.Time + event string + kind string + data string + turnSeq uint32 + ) + if err := rows.Scan(&ts, &event, &kind, &data, &turnSeq); err != nil { + return nil, err + } + if _, ok := seenTurns[turnSeq]; !ok { + seenTurns[turnSeq] = struct{}{} + turns = append(turns, turnSeq) + } + if len(turns) > limit { + continue + } + chunks = append(chunks, &TurnChunk{ + Data: []byte(data), + Event: event, + Kind: kind, + Timestamp: ts.UTC().UnixNano(), + }) + } + if err := rows.Err(); err != nil { + return nil, err + } + if len(turns) == 0 { + return &QueryTurnsResp{}, nil + } + + hasMore := len(turns) > limit + if hasMore { + turns = turns[:limit] + } + + resp := &QueryTurnsResp{ + Chunks: chunks, + HasMore: hasMore, + } + if hasMore { + oldest := turns[len(turns)-1] + resp.NextCursor = strconv.FormatUint(uint64(oldest), 10) + } + return resp, nil +} + +func (p *ClickHouseProvider) hasLowerTurn(ctx context.Context, taskID uuid.UUID, turnSeq uint32) (bool, error) { + if turnSeq == 0 { + return false, nil + } + table := p.client.Table() + + q := fmt.Sprintf(` +SELECT turn_seq + FROM %s + WHERE task_id = ? AND turn_seq < ? + GROUP BY turn_seq + ORDER BY turn_seq DESC + LIMIT ?`, table) + + rows, err := p.client.QueryContext(ctx, q, taskID, turnSeq, 1) + if err != nil { + return false, err + } + defer rows.Close() + + return rows.Next(), rows.Err() +} + +func (p *ClickHouseProvider) queryEntriesByTurn(ctx context.Context, taskID uuid.UUID, turnSeq uint32, start, end time.Time) ([]Entry, error) { + table := p.client.Table() + q := fmt.Sprintf(` +SELECT task_id, ts, event, kind, turn_seq, data, msg_seq_start, msg_seq_end + FROM %s + WHERE task_id = ? AND turn_seq = ? AND ts >= ? AND ts <= ? + ORDER BY turn_seq ASC, ts ASC, msg_seq_start ASC, ingest_id ASC`, table) + + rows, err := p.client.QueryContext(ctx, q, taskID, turnSeq, start, end) + if err != nil { + return nil, err + } + defer rows.Close() + + entries := make([]Entry, 0) + for rows.Next() { + var ( + id string + ts time.Time + event string + kind string + seq uint32 + data string + msgSeqStart uint64 + msgSeqEnd uint64 + ) + if err := rows.Scan(&id, &ts, &event, &kind, &seq, &data, &msgSeqStart, &msgSeqEnd); err != nil { + return nil, err + } + parsedID, err := uuid.Parse(id) + if err != nil { + return nil, err + } + entries = append(entries, Entry{ + TaskID: parsedID, + TS: ts.UTC(), + Event: event, + Kind: kind, + TurnSeq: seq, + Data: data, + MsgSeq: formatMsgSeqRange(msgSeqStart, msgSeqEnd), + }) + } + if err := rows.Err(); err != nil { + return nil, err + } + return entries, nil +} + +func formatMsgSeqRange(start, end uint64) string { + if start == 0 && end == 0 { + return "" + } + if start == end { + return strconv.FormatUint(start, 10) + } + return fmt.Sprintf("%d-%d", start, end) +} diff --git a/backend/pkg/tasklog/clickhouse_provider_test.go b/backend/pkg/tasklog/clickhouse_provider_test.go new file mode 100644 index 00000000..803b88cf --- /dev/null +++ b/backend/pkg/tasklog/clickhouse_provider_test.go @@ -0,0 +1,180 @@ +package tasklog_test + +import ( + "context" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/pkg/clickhouse" + "github.com/chaitin/MonkeyCode/backend/pkg/tasklog" +) + +func TestClickHouseProviderQueryLatestTurnUsesTurnSeqCursor(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + taskID := uuid.MustParse("11111111-1111-1111-1111-111111111111") + start := time.Unix(1_710_000_000, 0).UTC() + end := start.Add(time.Minute) + + mock.ExpectQuery("SELECT max\\(turn_seq\\)[\\s\\S]*WHERE task_id = \\? AND ts >= \\? AND ts <= \\?\\s*$"). + WithArgs(taskID, start, end). + WillReturnRows(sqlmock.NewRows([]string{"max"}).AddRow(2)) + + rows := sqlmock.NewRows([]string{"task_id", "ts", "event", "kind", "turn_seq", "data", "msg_seq_start", "msg_seq_end"}). + AddRow(taskID.String(), start.Add(10*time.Second), "user-input", "", 2, "hello", uint64(0), uint64(0)). + AddRow(taskID.String(), start.Add(11*time.Second), "task-running", "acp_event", 2, `{"text":"world"}`, uint64(2), uint64(4)) + + mock.ExpectQuery("SELECT task_id, ts, event, kind, turn_seq, data, msg_seq_start, msg_seq_end[\\s\\S]*ORDER BY turn_seq ASC, ts ASC, msg_seq_start ASC, ingest_id ASC\\s*$"). + WithArgs(taskID, 2, start, end). + WillReturnRows(rows) + + mock.ExpectQuery("SELECT turn_seq[\\s\\S]*turn_seq < \\?[\\s\\S]*LIMIT \\?\\s*$"). + WithArgs(taskID, uint32(2), 1). + WillReturnRows(sqlmock.NewRows([]string{"turn_seq"}).AddRow(1)) + + provider := tasklog.NewClickHouseProvider(clickhouse.NewWithDBAndTable(db, "task_logs_test")) + resp, err := provider.QueryLatestTurn(context.Background(), taskID, start, end) + if err != nil { + t.Fatal(err) + } + if len(resp.Entries) != 2 { + t.Fatalf("len(entries) = %d, want 2", len(resp.Entries)) + } + if resp.Entries[0].MsgSeq != "" { + t.Fatalf("entry[0].msg_seq = %q, want empty", resp.Entries[0].MsgSeq) + } + if resp.Entries[1].MsgSeq != "2-4" { + t.Fatalf("entry[1].msg_seq = %q, want 2-4", resp.Entries[1].MsgSeq) + } + if !resp.HasMore { + t.Fatal("expected has_more=true") + } + if resp.NextCursor != "2" { + t.Fatalf("next_cursor = %q, want 2", resp.NextCursor) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestClickHouseProviderQueryLatestTurnHandlesSparseTurnsWithoutMore(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + taskID := uuid.MustParse("11111111-1111-1111-1111-111111111111") + start := time.Unix(1_710_000_000, 0).UTC() + end := start.Add(time.Minute) + + mock.ExpectQuery("SELECT max\\(turn_seq\\)[\\s\\S]*WHERE task_id = \\? AND ts >= \\? AND ts <= \\?\\s*$"). + WithArgs(taskID, start, end). + WillReturnRows(sqlmock.NewRows([]string{"max"}).AddRow(5)) + + rows := sqlmock.NewRows([]string{"task_id", "ts", "event", "kind", "turn_seq", "data", "msg_seq_start", "msg_seq_end"}). + AddRow(taskID.String(), start.Add(10*time.Second), "user-input", "", 5, "hello", uint64(0), uint64(0)) + + mock.ExpectQuery("SELECT task_id, ts, event, kind, turn_seq, data, msg_seq_start, msg_seq_end[\\s\\S]*ORDER BY turn_seq ASC, ts ASC, msg_seq_start ASC, ingest_id ASC\\s*$"). + WithArgs(taskID, 5, start, end). + WillReturnRows(rows) + + mock.ExpectQuery("SELECT turn_seq[\\s\\S]*turn_seq < \\?[\\s\\S]*LIMIT \\?\\s*$"). + WithArgs(taskID, uint32(5), 1). + WillReturnRows(sqlmock.NewRows([]string{"turn_seq"})) + + provider := tasklog.NewClickHouseProvider(clickhouse.NewWithDBAndTable(db, "task_logs_test")) + resp, err := provider.QueryLatestTurn(context.Background(), taskID, start, end) + if err != nil { + t.Fatal(err) + } + if resp.HasMore { + t.Fatal("expected has_more=false") + } + if resp.NextCursor != "" { + t.Fatalf("next_cursor = %q, want empty", resp.NextCursor) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestClickHouseProviderQueryTurnsUsesSparseTurnCursor(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + taskID := uuid.MustParse("11111111-1111-1111-1111-111111111111") + chunkRows := sqlmock.NewRows([]string{"ts", "event", "kind", "data", "turn_seq"}). + AddRow(time.Unix(1_710_000_010, 0).UTC(), "user-input", "", "latest", uint32(1)) + + mock.ExpectQuery("SELECT ts, event, kind, data, turn_seq[\\s\\S]*FROM task_logs_test[\\s\\S]*turn_seq IN \\([\\s\\S]*SELECT DISTINCT turn_seq[\\s\\S]*FROM task_logs_test[\\s\\S]*turn_seq < \\?[\\s\\S]*LIMIT \\?[\\s\\S]*ORDER BY turn_seq DESC, ts ASC, msg_seq_start ASC, ingest_id ASC\\s*$"). + WithArgs(taskID, taskID, uint32(2), 2). + WillReturnRows(chunkRows) + + provider := tasklog.NewClickHouseProvider(clickhouse.NewWithDBAndTable(db, "task_logs_test")) + resp, err := provider.QueryTurns(context.Background(), taskID, time.Time{}, "2", 1) + if err != nil { + t.Fatal(err) + } + if len(resp.Chunks) != 1 { + t.Fatalf("len(chunks) = %d, want 1", len(resp.Chunks)) + } + if resp.HasMore { + t.Fatal("expected has_more=false") + } + if resp.NextCursor != "" { + t.Fatalf("next_cursor = %q, want empty", resp.NextCursor) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestClickHouseProviderQueryTurnsSkipsExtraTurnFromLimitPlusOne(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + taskID := uuid.MustParse("11111111-1111-1111-1111-111111111111") + now := time.Unix(1_710_000_010, 0).UTC() + chunkRows := sqlmock.NewRows([]string{"ts", "event", "kind", "data", "turn_seq"}). + AddRow(now, "task-running", "acp_event", "turn-3", uint32(3)). + AddRow(now.Add(time.Second), "task-running", "acp_event", "turn-2", uint32(2)) + + mock.ExpectQuery("SELECT ts, event, kind, data, turn_seq[\\s\\S]*FROM task_logs_test[\\s\\S]*turn_seq IN \\([\\s\\S]*SELECT DISTINCT turn_seq[\\s\\S]*FROM task_logs_test[\\s\\S]*ORDER BY turn_seq DESC[\\s\\S]*LIMIT \\?[\\s\\S]*ORDER BY turn_seq DESC, ts ASC, msg_seq_start ASC, ingest_id ASC\\s*$"). + WithArgs(taskID, taskID, 2). + WillReturnRows(chunkRows) + + provider := tasklog.NewClickHouseProvider(clickhouse.NewWithDBAndTable(db, "task_logs_test")) + resp, err := provider.QueryTurns(context.Background(), taskID, time.Time{}, "", 1) + if err != nil { + t.Fatal(err) + } + if len(resp.Chunks) != 1 { + t.Fatalf("len(chunks) = %d, want 1", len(resp.Chunks)) + } + if string(resp.Chunks[0].Data) != "turn-3" { + t.Fatalf("chunk data = %q, want turn-3", string(resp.Chunks[0].Data)) + } + if !resp.HasMore { + t.Fatal("expected has_more=true") + } + if resp.NextCursor != "3" { + t.Fatalf("next_cursor = %q, want 3", resp.NextCursor) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/backend/pkg/tasklog/gateway.go b/backend/pkg/tasklog/gateway.go new file mode 100644 index 00000000..ff761a4e --- /dev/null +++ b/backend/pkg/tasklog/gateway.go @@ -0,0 +1,52 @@ +package tasklog + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/consts" +) + +type Gateway struct { + Loki Provider + ClickHouse Provider +} + +func (g *Gateway) QueryLatestTurn(ctx context.Context, taskID uuid.UUID, taskCreatedAt, end time.Time, store consts.LogStore) (*QueryLatestTurnResp, error) { + p, err := g.providerByStore(store) + if err != nil { + return nil, err + } + return p.QueryLatestTurn(ctx, taskID, taskCreatedAt, end) +} + +func (g *Gateway) QueryTurns(ctx context.Context, taskID uuid.UUID, taskCreatedAt time.Time, cursor string, limit int, store consts.LogStore) (*QueryTurnsResp, error) { + p, err := g.providerByStore(store) + if err != nil { + return nil, err + } + return p.QueryTurns(ctx, taskID, taskCreatedAt, cursor, limit) +} + +func (g *Gateway) providerByStore(store consts.LogStore) (Provider, error) { + s := consts.LogStore(strings.TrimSpace(string(store))) + switch s { + case "", consts.LogStoreLoki: + return providerOrUnavailable(g.Loki, string(consts.LogStoreLoki)) + case consts.LogStoreClickHouse: + return providerOrUnavailable(g.ClickHouse, string(consts.LogStoreClickHouse)) + default: + return nil, fmt.Errorf("unsupported task log store: %q", store) + } +} + +func providerOrUnavailable(p Provider, name string) (Provider, error) { + if p == nil { + return nil, fmt.Errorf("%w: %s", ErrProviderUnavailable, name) + } + return p, nil +} diff --git a/backend/pkg/tasklog/gateway_test.go b/backend/pkg/tasklog/gateway_test.go new file mode 100644 index 00000000..c24a6b0c --- /dev/null +++ b/backend/pkg/tasklog/gateway_test.go @@ -0,0 +1,124 @@ +package tasklog + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/consts" +) + +type gatewayProviderStub struct { + name string + queryLatestTurnCalled bool + queryTurnsCalled bool +} + +func (s *gatewayProviderStub) Name() string { + return s.name +} + +func (s *gatewayProviderStub) QueryLatestTurn(context.Context, uuid.UUID, time.Time, time.Time) (*QueryLatestTurnResp, error) { + s.queryLatestTurnCalled = true + return &QueryLatestTurnResp{}, nil +} + +func (s *gatewayProviderStub) QueryTurns(context.Context, uuid.UUID, time.Time, string, int) (*QueryTurnsResp, error) { + s.queryTurnsCalled = true + return &QueryTurnsResp{}, nil +} + +func TestGatewayEmptyStoreUsesLoki(t *testing.T) { + loki := &gatewayProviderStub{name: "loki"} + clickHouse := &gatewayProviderStub{name: "clickhouse"} + gateway := &Gateway{Loki: loki, ClickHouse: clickHouse} + + _, err := gateway.QueryTurns(context.Background(), uuid.New(), time.Now(), "", 10, "") + if err != nil { + t.Fatalf("QueryTurns returned error: %v", err) + } + if !loki.queryTurnsCalled { + t.Fatal("expected Loki QueryTurns to be called") + } + if clickHouse.queryTurnsCalled { + t.Fatal("expected ClickHouse QueryTurns not to be called") + } +} + +func TestGatewayClickHouseStoreUsesClickHouse(t *testing.T) { + loki := &gatewayProviderStub{name: "loki"} + clickHouse := &gatewayProviderStub{name: "clickhouse"} + gateway := &Gateway{Loki: loki, ClickHouse: clickHouse} + + _, err := gateway.QueryLatestTurn(context.Background(), uuid.New(), time.Now(), time.Now(), consts.LogStoreClickHouse) + if err != nil { + t.Fatalf("QueryLatestTurn returned error: %v", err) + } + if !clickHouse.queryLatestTurnCalled { + t.Fatal("expected ClickHouse QueryLatestTurn to be called") + } + if loki.queryLatestTurnCalled { + t.Fatal("expected Loki QueryLatestTurn not to be called") + } +} + +func TestGatewayUnknownStoreReturnsError(t *testing.T) { + loki := &gatewayProviderStub{name: "loki"} + clickHouse := &gatewayProviderStub{name: "clickhouse"} + gateway := &Gateway{Loki: loki, ClickHouse: clickHouse} + + _, err := gateway.QueryTurns(context.Background(), uuid.New(), time.Now(), "", 10, consts.LogStore("bad-store")) + if err == nil { + t.Fatal("expected QueryTurns to return error") + } + if !strings.Contains(err.Error(), "unsupported task log store") { + t.Fatalf("expected unsupported store error, got: %v", err) + } + if loki.queryTurnsCalled || clickHouse.queryTurnsCalled { + t.Fatal("expected no provider to be called") + } +} + +func TestGatewayNilLokiProviderReturnsError(t *testing.T) { + clickHouse := &gatewayProviderStub{name: "clickhouse"} + gateway := &Gateway{ClickHouse: clickHouse} + + _, err := gateway.QueryTurns(context.Background(), uuid.New(), time.Now(), "", 10, "") + if err == nil { + t.Fatal("expected QueryTurns to return error") + } + if !errors.Is(err, ErrProviderUnavailable) { + t.Fatalf("expected provider unavailable error, got: %v", err) + } + if !strings.Contains(err.Error(), "loki") { + t.Fatalf("expected error to contain provider name, got: %v", err) + } + if clickHouse.queryTurnsCalled { + t.Fatal("expected ClickHouse QueryTurns not to be called") + } +} + +func TestGatewayNilClickHouseProviderReturnsError(t *testing.T) { + loki := &gatewayProviderStub{name: "loki"} + gateway := &Gateway{Loki: loki} + + _, err := gateway.QueryLatestTurn(context.Background(), uuid.New(), time.Now(), time.Now(), consts.LogStoreClickHouse) + if err == nil { + t.Fatal("expected QueryLatestTurn to return error") + } + if !errors.Is(err, ErrProviderUnavailable) { + t.Fatalf("expected provider unavailable error, got: %v", err) + } + if !strings.Contains(err.Error(), "clickhouse") { + t.Fatalf("expected error to contain provider name, got: %v", err) + } + if loki.queryLatestTurnCalled { + t.Fatal("expected Loki QueryLatestTurn not to be called") + } +} + +var _ Provider = (*gatewayProviderStub)(nil) diff --git a/backend/pkg/tasklog/loki_provider.go b/backend/pkg/tasklog/loki_provider.go new file mode 100644 index 00000000..5e2a9295 --- /dev/null +++ b/backend/pkg/tasklog/loki_provider.go @@ -0,0 +1,110 @@ +package tasklog + +import ( + "context" + "encoding/json" + "strconv" + "time" + + "github.com/google/uuid" + + "github.com/chaitin/MonkeyCode/backend/pkg/loki" + "github.com/chaitin/MonkeyCode/backend/pkg/taskflow" +) + +type LokiProvider struct { + client *loki.Client +} + +func NewLokiProvider(client *loki.Client) *LokiProvider { + return &LokiProvider{client: client} +} + +func (p *LokiProvider) Name() string { + return "loki" +} + +func (p *LokiProvider) QueryWindow(ctx context.Context, taskID uuid.UUID, start, end time.Time) ([]Entry, error) { + if p.client == nil { + return nil, ErrProviderUnavailable + } + entries, err := p.client.QueryWindowByTaskID(ctx, taskID.String(), start, end) + if err != nil { + return nil, err + } + + out := make([]Entry, 0, len(entries)) + for _, entry := range entries { + var chunk taskflow.TaskChunk + if err := json.Unmarshal([]byte(entry.Line), &chunk); err != nil { + continue + } + out = append(out, Entry{ + TaskID: taskID, + TS: entry.Timestamp.UTC(), + Event: chunk.Event, + Kind: chunk.Kind, + Data: string(chunk.Data), + MsgSeq: entry.Labels["msg_seq"], + Labels: entry.Labels, + }) + } + return out, nil +} + +func (p *LokiProvider) QueryLatestTurn(ctx context.Context, taskID uuid.UUID, taskCreatedAt, end time.Time) (*QueryLatestTurnResp, error) { + if p.client == nil { + return nil, ErrProviderUnavailable + } + turnStart, err := p.client.FindLatestRoundStart(ctx, taskID.String(), taskCreatedAt, end) + if err != nil { + return nil, err + } + entries, err := p.QueryWindow(ctx, taskID, turnStart, end) + if err != nil { + return nil, err + } + resp := &QueryLatestTurnResp{ + Entries: entries, + HasMore: turnStart.After(taskCreatedAt), + } + if resp.HasMore { + resp.NextCursor = strconv.FormatInt(turnStart.UnixNano()-1, 10) + } + return resp, nil +} + +func (p *LokiProvider) QueryTurns(ctx context.Context, taskID uuid.UUID, taskCreatedAt time.Time, cursor string, limit int) (*QueryTurnsResp, error) { + if p.client == nil { + return nil, ErrProviderUnavailable + } + end := time.Now() + if cursor != "" { + ns, err := strconv.ParseInt(cursor, 10, 64) + if err != nil { + return nil, err + } + end = time.Unix(0, ns) + } + resp, err := p.client.QueryRounds(ctx, taskID.String(), taskCreatedAt, end, limit) + if err != nil { + return nil, err + } + out := &QueryTurnsResp{ + Chunks: make([]*TurnChunk, 0, len(resp.Chunks)), + HasMore: resp.HasMore, + } + if resp.HasMore && resp.NextTS > 0 { + out.NextCursor = strconv.FormatInt(resp.NextTS, 10) + } + for _, chunk := range resp.Chunks { + out.Chunks = append(out.Chunks, &TurnChunk{ + Data: chunk.Data, + Event: chunk.Event, + Kind: chunk.Kind, + Timestamp: chunk.Timestamp, + Labels: chunk.Labels, + }) + } + return out, nil +} diff --git a/backend/pkg/tasklog/provider.go b/backend/pkg/tasklog/provider.go new file mode 100644 index 00000000..cce5ab41 --- /dev/null +++ b/backend/pkg/tasklog/provider.go @@ -0,0 +1,14 @@ +package tasklog + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +type Provider interface { + Name() string + QueryLatestTurn(ctx context.Context, taskID uuid.UUID, taskCreatedAt, end time.Time) (*QueryLatestTurnResp, error) + QueryTurns(ctx context.Context, taskID uuid.UUID, taskCreatedAt time.Time, cursor string, limit int) (*QueryTurnsResp, error) +} diff --git a/backend/pkg/tasklog/types.go b/backend/pkg/tasklog/types.go new file mode 100644 index 00000000..265736b7 --- /dev/null +++ b/backend/pkg/tasklog/types.go @@ -0,0 +1,44 @@ +package tasklog + +import ( + "errors" + "time" + + "github.com/google/uuid" +) + +var ( + ErrProviderUnavailable = errors.New("tasklog provider unavailable") + ErrUnsupported = errors.New("tasklog operation unsupported") +) + +type Entry struct { + TaskID uuid.UUID + TS time.Time + Event string + Kind string + TurnSeq uint32 + Data string + MsgSeq string + Labels map[string]string +} + +type QueryLatestTurnResp struct { + Entries []Entry + HasMore bool + NextCursor string +} + +type TurnChunk struct { + Data []byte + Event string + Kind string + Timestamp int64 + Labels map[string]string +} + +type QueryTurnsResp struct { + Chunks []*TurnChunk + HasMore bool + NextCursor string +}