perf(actions): debounce runner heartbeat writes and throttle task picks (#38281)

3 reductions in the DB load generated by many runners polling
`FetchTask`:

**1. Debounce runner heartbeat writes**
Every poll wrote `last_online`, and every `UpdateTask`/`UpdateLog` wrote
`last_active` — while a runner streams logs that is many writes per
second per runner. These are now persisted only when stale enough to
actually affect the active/offline status (`ShouldPersistLastOnline` /
`ShouldPersistLastActive`), using the existing columns.

**2. Throttle concurrent task picks**
A new in-process semaphore (`MAX_CONCURRENT_TASK_PICKS`) bounds how many
runners run the task-assignment transaction at once, so a fleet polling
together cannot stampede the query. Throttled polls retry on their next
poll without advancing the runner's tasks version.

**3. Paginate the task-pick query**
`CreateTaskForRunner` previously loaded every waiting job in the
runner's scope into memory on each poll (no `LIMIT`). Now it pages
through the waiting backlog oldest-first with `LIMIT`, claiming the
first label-matching job.

---------

Co-authored-by: Zettat123 <zettat123@gmail.com>

routers/api/actions/runner/interceptor.go

@@ -8,6 +8,7 @@ import (
 	"crypto/subtle"
 	"errors"
 	"strings"
+	"time"
 
 	actions_model "gitea.dev/models/actions"
 	auth_model "gitea.dev/models/auth"
@@ -45,14 +46,26 @@ var withRunner = connect.WithInterceptors(connect.UnaryInterceptorFunc(func(unar
 			return nil, status.Error(codes.Unauthenticated, "unregistered runner")
 		}
 
-		cols := []string{"last_online"}
-		runner.LastOnline = timeutil.TimeStampNow()
-		if methodName == "UpdateTask" || methodName == "UpdateLog" {
-			runner.LastActive = timeutil.TimeStampNow()
+		now := time.Now()
+		cols := make([]string, 0, 2)
+		// Debounce last_active too: while a runner streams logs, UpdateLog fires
+		// many times per second and writing on each is a major source of DB load.
+		// Persist only when stale enough to affect the active/idle status.
+		if (methodName == "UpdateTask" || methodName == "UpdateLog") &&
+			actions_model.ShouldPersistLastActive(runner.LastActive, now) {
+			runner.LastActive = timeutil.TimeStamp(now.Unix())
 			cols = append(cols, "last_active")
 		}
-		if err := actions_model.UpdateRunner(ctx, runner, cols...); err != nil {
-			log.Error("can't update runner status: %v", err)
+		// Debounce last_online: writing on every poll is a major source of DB load
+		// with many runners. Persist only when stale enough to affect offline status.
+		if actions_model.ShouldPersistLastOnline(runner.LastOnline, now) {
+			runner.LastOnline = timeutil.TimeStamp(now.Unix())
+			cols = append(cols, "last_online")
+		}
+		if len(cols) > 0 {
+			if err := actions_model.UpdateRunner(ctx, runner, cols...); err != nil {
+				log.Error("can't update runner status: %v", err)
+			}
 		}
 
 		ctx = context.WithValue(ctx, runnerCtxKey{}, runner)

routers/api/actions/runner/runner.go

@@ -194,9 +194,15 @@ func (s *Service) FetchTask(
 		// if the task version in request is not equal to the version in db,
 		// it means there may still be some tasks that haven't been assigned.
 		// try to pick a task for the runner that send the request.
-		if t, ok, err := actions_service.PickTask(ctx, freshRunner); err != nil {
+		if t, ok, throttled, err := actions_service.TryPickTask(ctx, freshRunner); err != nil {
 			log.Error("pick task failed: %v", err)
 			return nil, status.Errorf(codes.Internal, "pick task: %v", err)
+		} else if throttled {
+			// Concurrency limit reached: don't advance the runner's tasks version,
+			// so it retries on its next poll instead of sleeping until the next bump.
+			latestVersion = tasksVersion
+			//  A steady stream here means MAX_CONCURRENT_TASK_PICKS is too low for the fleet.
+			log.Debug("task pick throttled for runner %q (id %d); it will retry on its next poll", freshRunner.Name, freshRunner.ID)
 		} else if ok {
 			task = t
 		}