perf(actions): debounce runner heartbeat writes and throttle task picks (#38281)

3 reductions in the DB load generated by many runners polling
`FetchTask`:

**1. Debounce runner heartbeat writes**
Every poll wrote `last_online`, and every `UpdateTask`/`UpdateLog` wrote
`last_active` — while a runner streams logs that is many writes per
second per runner. These are now persisted only when stale enough to
actually affect the active/offline status (`ShouldPersistLastOnline` /
`ShouldPersistLastActive`), using the existing columns.

**2. Throttle concurrent task picks**
A new in-process semaphore (`MAX_CONCURRENT_TASK_PICKS`) bounds how many
runners run the task-assignment transaction at once, so a fleet polling
together cannot stampede the query. Throttled polls retry on their next
poll without advancing the runner's tasks version.

**3. Paginate the task-pick query**
`CreateTaskForRunner` previously loaded every waiting job in the
runner's scope into memory on each poll (no `LIMIT`). Now it pages
through the waiting backlog oldest-first with `LIMIT`, claiming the
first label-matching job.

---------

Co-authored-by: Zettat123 <zettat123@gmail.com>

models/actions/runner.go

@@ -75,8 +75,28 @@ type ActionRunner struct {
 const (
 	RunnerOfflineTime = time.Minute
 	RunnerIdleTime    = 10 * time.Second
+	// RunnerHeartbeatInterval is how often last_online is persisted on poll.
+	// Must stay well below RunnerOfflineTime so runners don't flap to offline.
+	RunnerHeartbeatInterval = 30 * time.Second
+	// RunnerActiveInterval is how often last_active is persisted while a runner
+	// streams task updates and logs. Must stay well below RunnerIdleTime so a
+	// busy runner keeps showing ACTIVE without a DB write on every RPC.
+	RunnerActiveInterval = 5 * time.Second
 )
 
+// ShouldPersistLastOnline reports whether last_online is stale enough to be
+// worth writing back. Avoids a DB write on every runner poll.
+func ShouldPersistLastOnline(last timeutil.TimeStamp, now time.Time) bool {
+	return now.Sub(last.AsTime()) >= RunnerHeartbeatInterval
+}
+
+// ShouldPersistLastActive reports whether last_active is stale enough to be
+// worth writing back. Avoids a DB write on every UpdateTask/UpdateLog RPC while
+// a runner is actively streaming logs.
+func ShouldPersistLastActive(last timeutil.TimeStamp, now time.Time) bool {
+	return now.Sub(last.AsTime()) >= RunnerActiveInterval
+}
+
 // BelongsToOwnerName before calling, should guarantee that all attributes are loaded
 func (r *ActionRunner) BelongsToOwnerName() string {
 	if r.RepoID != 0 {

models/actions/runner_test.go

@@ -6,14 +6,86 @@ package actions
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"gitea.dev/models/db"
 	"gitea.dev/models/unittest"
+	"gitea.dev/modules/timeutil"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func TestShouldPersistLastOnline(t *testing.T) {
+	now := time.Now()
+	tests := []struct {
+		name string
+		last timeutil.TimeStamp
+		want bool
+	}{
+		{
+			name: "fresh, skip write",
+			last: timeutil.TimeStamp(now.Add(-5 * time.Second).Unix()),
+			want: false,
+		},
+		{
+			name: "exactly at interval, write",
+			last: timeutil.TimeStamp(now.Add(-RunnerHeartbeatInterval).Unix()),
+			want: true,
+		},
+		{
+			name: "stale, write",
+			last: timeutil.TimeStamp(now.Add(-2 * RunnerHeartbeatInterval).Unix()),
+			want: true,
+		},
+		{
+			name: "zero (never seen), write",
+			last: 0,
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, ShouldPersistLastOnline(tt.last, now))
+		})
+	}
+}
+
+func TestShouldPersistLastActive(t *testing.T) {
+	now := time.Now()
+	tests := []struct {
+		name string
+		last timeutil.TimeStamp
+		want bool
+	}{
+		{
+			name: "fresh, skip write",
+			last: timeutil.TimeStamp(now.Add(-1 * time.Second).Unix()),
+			want: false,
+		},
+		{
+			name: "exactly at interval, write",
+			last: timeutil.TimeStamp(now.Add(-RunnerActiveInterval).Unix()),
+			want: true,
+		},
+		{
+			name: "stale, write",
+			last: timeutil.TimeStamp(now.Add(-2 * RunnerActiveInterval).Unix()),
+			want: true,
+		},
+		{
+			name: "zero (never seen), write",
+			last: 0,
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.want, ShouldPersistLastActive(tt.last, now))
+		})
+	}
+}
+
 func TestFindRunnerOptions_ToOrders_StableTiebreaker(t *testing.T) {
 	// Sorts on a non-unique column must end with the unique id tiebreaker so
 	// pagination is deterministic; without it, runners sharing the same

models/actions/task.go

@@ -231,6 +231,11 @@ func makeTaskStepDisplayName(step *jobparser.Step, limit int) (name string) {
 // another runner won the optimistic-lock race; it is never returned to callers.
 var errJobAlreadyClaimed = errors.New("job already claimed by another runner")
 
+// pickTaskBatchSize bounds how many waiting jobs each CreateTaskForRunner query loads,
+// so a large backlog is not fetched into memory on every runner poll.
+// It is a var only so tests can shrink it to exercise pagination cheaply.
+var pickTaskBatchSize = 100
+
 // CreateTaskForRunner finds a waiting job that matches the runner's labels and
 // atomically claims it. It iterates through all matching jobs so that a
 // concurrent claim by another runner (which would lose the optimistic lock on
@@ -249,31 +254,51 @@ func CreateTaskForRunner(ctx context.Context, runner *ActionRunner) (*ActionTask
 			Join("INNER", "repo_unit", "`repository`.id = `repo_unit`.repo_id").
 			Where(builder.Eq{"`repository`.owner_id": runner.OwnerID, "`repo_unit`.type": unit.TypeActions}))
 	}
-	if jobCond.IsValid() {
-		jobCond = builder.In("run_id", builder.Select("id").From("action_run").Where(jobCond))
-	}
-
-	var jobs []*ActionRunJob
-	if err := e.Where("task_id=? AND status=? AND is_reusable_caller=?", 0, StatusWaiting, false).And(jobCond).Asc("updated", "id").Find(&jobs); err != nil {
-		return nil, false, err
-	}
+	baseCond := builder.Eq{"task_id": 0, "status": StatusWaiting, "is_reusable_caller": false}.And(jobCond)
 
 	// TODO: a more efficient way to filter labels
 	log.Trace("runner labels: %v", runner.AgentLabels)
-	for _, v := range jobs {
-		if !runner.CanMatchLabels(v.RunsOn) {
-			continue
+
+	// Page through the waiting jobs oldest-first instead of loading the whole backlog into memory on every poll.
+	// Keyset pagination on (updated, id) is safe under concurrent claims:
+	// updated only moves forward, so the advancing cursor never skips a still-waiting job even as claimed jobs drop out.
+	var cursorUpdated timeutil.TimeStamp
+	var cursorID int64
+	for {
+		cond := baseCond
+		if cursorID > 0 {
+			cond = cond.And(builder.Or(
+				builder.Gt{"updated": cursorUpdated},
+				builder.And(builder.Eq{"updated": cursorUpdated}, builder.Gt{"id": cursorID}),
+			))
 		}
-		task, ok, err := claimJobForRunner(ctx, runner, v)
-		if err != nil {
+
+		var jobs []*ActionRunJob
+		if err := e.Where(cond).Asc("updated", "id").Limit(pickTaskBatchSize).Find(&jobs); err != nil {
 			return nil, false, err
 		}
-		if ok {
-			return task, true, nil
+
+		for _, v := range jobs {
+			if !runner.CanMatchLabels(v.RunsOn) {
+				continue
+			}
+			task, ok, err := claimJobForRunner(ctx, runner, v)
+			if err != nil {
+				return nil, false, err
+			}
+			if ok {
+				return task, true, nil
+			}
+			// Another runner claimed this job concurrently; try the next one.
 		}
-		// Another runner claimed this job concurrently; try the next one.
+
+		// A short page means no waiting jobs remain beyond it.
+		if len(jobs) < pickTaskBatchSize {
+			return nil, false, nil
+		}
+		last := jobs[len(jobs)-1]
+		cursorUpdated, cursorID = last.Updated, last.ID
 	}
-	return nil, false, nil
 }
 
 // claimJobForRunner attempts to atomically claim job for runner inside its own

models/actions/task_test.go

@@ -371,3 +371,74 @@ func TestReleaseTaskForRunner(t *testing.T) {
 	unittest.AssertNotExistsBean(t, &ActionTask{ID: task.ID})
 	unittest.AssertNotExistsBean(t, &ActionTaskStep{TaskID: task.ID})
 }
+
+// TestCreateTaskForRunnerPagination verifies that a job sitting beyond the first page is still claimed
+func TestCreateTaskForRunnerPagination(t *testing.T) {
+	require.NoError(t, unittest.PrepareTestDatabase())
+
+	defer func(orig int) { pickTaskBatchSize = orig }(pickTaskBatchSize)
+	pickTaskBatchSize = 2
+
+	run := &ActionRun{
+		Title:         "pagination-test-run",
+		RepoID:        1,
+		OwnerID:       2,
+		WorkflowID:    "test.yaml",
+		Index:         9903,
+		TriggerUserID: 2,
+		Ref:           "refs/heads/main",
+		CommitSHA:     "c2d72f548424103f01ee1dc02889c1e2bff816b0",
+		Event:         "push",
+		TriggerEvent:  "push",
+		Status:        StatusWaiting,
+	}
+	require.NoError(t, db.Insert(t.Context(), run))
+
+	// Five waiting jobs the runner cannot run, then one it can.
+	// With a page size of 2 the matching job only appears on the third page.
+	for i := range 5 {
+		mismatch := &ActionRunJob{
+			RunID:     run.ID,
+			RepoID:    run.RepoID,
+			OwnerID:   run.OwnerID,
+			CommitSHA: run.CommitSHA,
+			Name:      "mismatch-" + string(rune('a'+i)),
+			Attempt:   1,
+			JobID:     "mismatch-" + string(rune('a'+i)),
+			Status:    StatusWaiting,
+			RunsOn:    []string{"windows-latest"},
+		}
+		require.NoError(t, db.Insert(t.Context(), mismatch))
+	}
+
+	target := &ActionRunJob{
+		RunID:           run.ID,
+		RepoID:          run.RepoID,
+		OwnerID:         run.OwnerID,
+		CommitSHA:       run.CommitSHA,
+		Name:            "target-job",
+		Attempt:         1,
+		JobID:           "target-job",
+		Status:          StatusWaiting,
+		RunsOn:          []string{"ubuntu-latest"},
+		WorkflowPayload: []byte("on: push\njobs:\n  target-job:\n    runs-on: ubuntu-latest\n    steps:\n      - run: echo hi\n"),
+	}
+	require.NoError(t, db.Insert(t.Context(), target))
+
+	runner := &ActionRunner{
+		UUID:        "pagination-runner-uuid",
+		Name:        "pagination-runner",
+		AgentLabels: []string{"ubuntu-latest"},
+	}
+	runner.GenerateAndFillToken()
+	require.NoError(t, db.Insert(t.Context(), runner))
+
+	task, ok, err := CreateTaskForRunner(t.Context(), runner)
+	require.NoError(t, err)
+	require.True(t, ok)
+	require.NotNil(t, task)
+
+	claimed := unittest.AssertExistsAndLoadBean(t, &ActionRunJob{ID: target.ID})
+	assert.Equal(t, StatusRunning, claimed.Status)
+	assert.Equal(t, task.ID, claimed.TaskID)
+}