mirror of
https://github.com/go-gitea/gitea.git
synced 2026-07-19 15:41:15 +00:00
perf(actions): debounce runner heartbeat writes and throttle task picks (#38281)
3 reductions in the DB load generated by many runners polling `FetchTask`: **1. Debounce runner heartbeat writes** Every poll wrote `last_online`, and every `UpdateTask`/`UpdateLog` wrote `last_active` — while a runner streams logs that is many writes per second per runner. These are now persisted only when stale enough to actually affect the active/offline status (`ShouldPersistLastOnline` / `ShouldPersistLastActive`), using the existing columns. **2. Throttle concurrent task picks** A new in-process semaphore (`MAX_CONCURRENT_TASK_PICKS`) bounds how many runners run the task-assignment transaction at once, so a fleet polling together cannot stampede the query. Throttled polls retry on their next poll without advancing the runner's tasks version. **3. Paginate the task-pick query** `CreateTaskForRunner` previously loaded every waiting job in the runner's scope into memory on each poll (no `LIMIT`). Now it pages through the waiting backlog oldest-first with `LIMIT`, claiming the first label-matching job. --------- Co-authored-by: Zettat123 <zettat123@gmail.com>
This commit is contained in:
@@ -7,16 +7,47 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
runnerv1 "gitea.dev/actions-proto-go/runner/v1"
|
||||
actions_model "gitea.dev/models/actions"
|
||||
"gitea.dev/models/db"
|
||||
secret_model "gitea.dev/models/secret"
|
||||
"gitea.dev/modules/log"
|
||||
"gitea.dev/modules/setting"
|
||||
|
||||
"google.golang.org/protobuf/types/known/structpb"
|
||||
)
|
||||
|
||||
var (
|
||||
taskPickSem chan struct{}
|
||||
taskPickSemOnce sync.Once
|
||||
)
|
||||
|
||||
func taskPickLimiter() chan struct{} {
|
||||
taskPickSemOnce.Do(func() {
|
||||
taskPickSem = make(chan struct{}, setting.Actions.MaxConcurrentTaskPicks)
|
||||
})
|
||||
return taskPickSem
|
||||
}
|
||||
|
||||
// TryPickTask attempts to assign a task to the runner, bounding the number of
|
||||
// concurrent assignment transactions to avoid a thundering herd when many
|
||||
// runners poll at once. When the concurrency limit is reached it returns
|
||||
// throttled=true without touching the DB, so the caller can let the runner
|
||||
// retry on its next poll instead of advancing its tasks version.
|
||||
func TryPickTask(ctx context.Context, runner *actions_model.ActionRunner) (task *runnerv1.Task, ok, throttled bool, err error) {
|
||||
sem := taskPickLimiter()
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
defer func() { <-sem }()
|
||||
default:
|
||||
return nil, false, true, nil
|
||||
}
|
||||
task, ok, err = PickTask(ctx, runner)
|
||||
return task, ok, false, err
|
||||
}
|
||||
|
||||
func PickTask(ctx context.Context, runner *actions_model.ActionRunner) (*runnerv1.Task, bool, error) {
|
||||
var (
|
||||
task *runnerv1.Task
|
||||
|
||||
34
services/actions/task_test.go
Normal file
34
services/actions/task_test.go
Normal file
@@ -0,0 +1,34 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package actions
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
actions_model "gitea.dev/models/actions"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestTryPickTaskThrottled(t *testing.T) {
|
||||
sem := taskPickLimiter()
|
||||
|
||||
// Saturate every assignment slot so the next attempt must be throttled.
|
||||
for range cap(sem) {
|
||||
sem <- struct{}{}
|
||||
}
|
||||
defer func() {
|
||||
for range cap(sem) {
|
||||
<-sem
|
||||
}
|
||||
}()
|
||||
|
||||
// No DB access happens on the throttled path, so this is safe without fixtures.
|
||||
task, ok, throttled, err := TryPickTask(t.Context(), &actions_model.ActionRunner{})
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, task)
|
||||
assert.False(t, ok)
|
||||
assert.True(t, throttled)
|
||||
}
|
||||
Reference in New Issue
Block a user