Files
gitea/routers/api/actions/runner/interceptor.go
Giteabot 64d559a8e6 perf(actions): debounce runner heartbeat writes and throttle task picks (#38281) (#38368)
Backport #38281 by @bircni

3 reductions in the DB load generated by many runners polling `FetchTask`:

**1. Debounce runner heartbeat writes**
Every poll wrote `last_online`, and every `UpdateTask`/`UpdateLog` wrote
`last_active` — while a runner streams logs that is many writes per
second per runner. These are now persisted only when stale enough to
actually affect the active/offline status (`ShouldPersistLastOnline` /
`ShouldPersistLastActive`), using the existing columns.

**2. Throttle concurrent task picks**
A new in-process semaphore (`MAX_CONCURRENT_TASK_PICKS`) bounds how many
runners run the task-assignment transaction at once, so a fleet polling
together cannot stampede the query. Throttled polls retry on their next
poll without advancing the runner's tasks version.

**3. Paginate the task-pick query**
`CreateTaskForRunner` previously loaded every waiting job in the
runner's scope into memory on each poll (no `LIMIT`). Now it pages
through the waiting backlog oldest-first with `LIMIT`, claiming the
first label-matching job.

Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: Zettat123 <zettat123@gmail.com>
2026-07-07 16:27:00 -07:00

94 lines
2.8 KiB
Go

// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package runner
import (
"context"
"crypto/subtle"
"errors"
"strings"
"time"
actions_model "gitea.dev/models/actions"
auth_model "gitea.dev/models/auth"
"gitea.dev/modules/log"
"gitea.dev/modules/timeutil"
"gitea.dev/modules/util"
"connectrpc.com/connect"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
uuidHeaderKey = "x-runner-uuid"
tokenHeaderKey = "x-runner-token"
)
var withRunner = connect.WithInterceptors(connect.UnaryInterceptorFunc(func(unaryFunc connect.UnaryFunc) connect.UnaryFunc {
return func(ctx context.Context, request connect.AnyRequest) (connect.AnyResponse, error) {
methodName := getMethodName(request)
if methodName == "Register" {
return unaryFunc(ctx, request)
}
uuid := request.Header().Get(uuidHeaderKey)
token := request.Header().Get(tokenHeaderKey)
runner, err := actions_model.GetRunnerByUUID(ctx, uuid)
if err != nil {
if errors.Is(err, util.ErrNotExist) {
return nil, status.Error(codes.Unauthenticated, "unregistered runner")
}
return nil, status.Error(codes.Internal, err.Error())
}
if subtle.ConstantTimeCompare([]byte(runner.TokenHash), []byte(auth_model.HashToken(token, runner.TokenSalt))) != 1 {
return nil, status.Error(codes.Unauthenticated, "unregistered runner")
}
now := time.Now()
cols := make([]string, 0, 2)
// Debounce last_active too: while a runner streams logs, UpdateLog fires
// many times per second and writing on each is a major source of DB load.
// Persist only when stale enough to affect the active/idle status.
if (methodName == "UpdateTask" || methodName == "UpdateLog") &&
actions_model.ShouldPersistLastActive(runner.LastActive, now) {
runner.LastActive = timeutil.TimeStamp(now.Unix())
cols = append(cols, "last_active")
}
// Debounce last_online: writing on every poll is a major source of DB load
// with many runners. Persist only when stale enough to affect offline status.
if actions_model.ShouldPersistLastOnline(runner.LastOnline, now) {
runner.LastOnline = timeutil.TimeStamp(now.Unix())
cols = append(cols, "last_online")
}
if len(cols) > 0 {
if err := actions_model.UpdateRunner(ctx, runner, cols...); err != nil {
log.Error("can't update runner status: %v", err)
}
}
ctx = context.WithValue(ctx, runnerCtxKey{}, runner)
return unaryFunc(ctx, request)
}
}))
func getMethodName(req connect.AnyRequest) string {
splits := strings.Split(req.Spec().Procedure, "/")
if len(splits) > 0 {
return splits[len(splits)-1]
}
return ""
}
type runnerCtxKey struct{}
func GetRunner(ctx context.Context) *actions_model.ActionRunner {
if v := ctx.Value(runnerCtxKey{}); v != nil {
if r, ok := v.(*actions_model.ActionRunner); ok {
return r
}
}
return nil
}