feat(actions)!: add RUN_RETENTION_DAYS to delete old action runs (#38855)

Gitea keeps completed Actions runs forever. Artifacts and logs expire on
their own schedule, but the run rows never go away, so `action_run` and
its child tables grow without bound.

Adds `RUN_RETENTION_DAYS` to delete completed runs along with their
jobs, tasks and anything the earlier expiries left behind. It defaults
to 400 days, matching how long GitHub keeps run history browsable. A
dedicated `cleanup_action_runs` cron task performs the cleanup, so
admins can schedule it separately from the nightly artifact and log
sweep.

`0` now means "keep forever" for all three retention settings, where
`LOG_RETENTION_DAYS` and `ARTIFACT_RETENTION_DAYS` previously took it
literally and deleted everything at the next sweep.

Docs: https://gitea.com/gitea/docs/pulls/502

----

## ⚠️ BREAKING ⚠️

`RUN_RETENTION_DAYS` defaults to 400, so completed runs older than that
are deleted when the cron task next runs at midnight. Set
`RUN_RETENTION_DAYS = 0` before upgrading to keep all runs.

---------

Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: silverwind <me@silverwind.io>

routers/api/actions/artifacts.go

@@ -75,9 +75,11 @@ import (
 	"gitea.dev/modules/httplib"
 	"gitea.dev/modules/json"
 	"gitea.dev/modules/log"
+	"gitea.dev/modules/optional"
 	"gitea.dev/modules/reqctx"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/storage"
+	"gitea.dev/modules/timeutil"
 	"gitea.dev/modules/util"
 	"gitea.dev/modules/web"
 	web_types "gitea.dev/modules/web/types"
@@ -245,25 +247,12 @@ func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) {
 		return
 	}
 
-	// get upload file size
 	fileRealTotalSize := getUploadFileSize(ctx)
-
-	// get artifact retention days
-	expiredDays := setting.Actions.ArtifactRetentionDays
-	if queryRetentionDays := ctx.Req.URL.Query().Get("retentionDays"); queryRetentionDays != "" {
-		var err error
-		expiredDays, err = strconv.ParseInt(queryRetentionDays, 10, 64)
-		if err != nil {
-			log.Error("Error parse retention days: %v", err)
-			ctx.HTTPError(http.StatusBadRequest, "Error parse retention days")
-			return
-		}
+	var expiry optional.Option[timeutil.TimeStamp]
+	if days := ctx.FormOptionalInt64("retentionDays"); days.Has() {
+		expiry = optional.Some(timeutil.TimeStampNow().Add(timeutil.Day * days.Value()))
 	}
-	log.Debug("[artifact] upload chunk, name: %s, path: %s, size: %d, retention days: %d",
-		artifactName, artifactPath, fileRealTotalSize, expiredDays)
-
-	// create or get artifact with name and path
-	artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiredDays)
+	artifact, err := actions.CreateArtifact(ctx, task, artifactName, artifactPath, expiry)
 	if err != nil {
 		log.Error("Error create or get artifact: %v", err)
 		ctx.HTTPError(http.StatusInternalServerError, "Error create or get artifact")
@@ -288,7 +277,7 @@ func (ar artifactRoutes) uploadArtifact(ctx *ArtifactContext) {
 		artifact.FileSize = fileRealTotalSize
 		artifact.FileCompressedSize = chunksTotalSize
 		artifact.ContentEncodingOrType = ctx.Req.Header.Get("Content-Encoding")
-		if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
+		if err := actions.UpdateArtifact(ctx, artifact, "file_size", "file_compressed_size", "content_encoding"); err != nil {
 			log.Error("Error update artifact: %v", err)
 			ctx.HTTPError(http.StatusInternalServerError, "Error update artifact")
 			return
@@ -349,7 +338,7 @@ func (ar artifactRoutes) listArtifacts(ctx *ArtifactContext) {
 	artifacts, err := actions.FindReadableArtifacts(ctx, actions.FindArtifactsOptions{
 		RunID:         runID,
 		RunAttemptIDs: attemptIDs,
-		Status:        int(actions.ArtifactStatusUploadConfirmed),
+		Status:        actions.ArtifactStatusUploadConfirmed,
 	})
 	if err != nil {
 		log.Error("Error getting artifacts: %v", err)
@@ -421,7 +410,7 @@ func (ar artifactRoutes) getDownloadArtifactURL(ctx *ArtifactContext) {
 		RunID:         runID,
 		RunAttemptIDs: attemptIDs,
 		ArtifactName:  itemPath,
-		Status:        int(actions.ArtifactStatusUploadConfirmed),
+		Status:        actions.ArtifactStatusUploadConfirmed,
 	})
 	if err != nil {
 		log.Error("Error getting artifacts: %v", err)

routers/api/actions/artifacts_chunks.go

@@ -279,9 +279,18 @@ func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID, ru
 			log.Debug("artifact %d chunks not found", art.ID)
 			continue
 		}
-		if err := mergeChunksForArtifact(ctx, chunks, st, art, ""); err != nil {
+		storagePath, err := mergeChunksForArtifact(chunks, st, art, "")
+		if err != nil {
 			return err
 		}
+		if storagePath == "" {
+			continue
+		}
+		art.StoragePath = storagePath
+		art.Status = actions.ArtifactStatusUploadConfirmed
+		if err := actions.UpdateArtifact(ctx, art, "storage_path", "status"); err != nil {
+			return fmt.Errorf("update artifact error: %v", err)
+		}
 	}
 	return nil
 }
@@ -297,7 +306,9 @@ func generateArtifactStoragePath(artifact *actions.ActionArtifact) string {
 	return fmt.Sprintf("%d/%d/%d.%s", artifact.RunID%255, artifact.ID%255, time.Now().UnixNano(), extension)
 }
 
-func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st storage.ObjectStorage, artifact *actions.ActionArtifact, checksum string) error {
+// mergeChunksForArtifact merges the uploaded chunks into one stored object and returns its path.
+// An empty path means the chunks are not complete yet and nothing was merged.
+func mergeChunksForArtifact(chunks []*chunkFileItem, st storage.ObjectStorage, artifact *actions.ActionArtifact, checksum string) (string, error) {
 	sort.Slice(chunks, func(i, j int) bool {
 		return chunks[i].Start < chunks[j].Start
 	})
@@ -316,7 +327,7 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
 	// if the last chunk.End + 1 is not equal to chunk.ChunkLength, means chunks are not uploaded completely
 	if startAt+1 != artifact.FileCompressedSize {
 		log.Debug("[artifact] chunks are not uploaded completely, artifact_id: %d", artifact.ID)
-		return nil
+		return "", nil
 	}
 	// use multiReader
 	readers := make([]io.Reader, 0, len(allChunks))
@@ -331,7 +342,7 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
 		var readCloser io.ReadCloser
 		var err error
 		if readCloser, err = st.Open(c.Path); err != nil {
-			return fmt.Errorf("open chunk error: %v, %s", err, c.Path)
+			return "", fmt.Errorf("open chunk error: %v, %s", err, c.Path)
 		}
 		readers = append(readers, readCloser)
 	}
@@ -351,10 +362,10 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
 	storagePath := generateArtifactStoragePath(artifact)
 	written, err := st.Save(storagePath, mergedReader, artifact.FileCompressedSize)
 	if err != nil {
-		return fmt.Errorf("save merged file error: %v", err)
+		return "", fmt.Errorf("save merged file error: %v", err)
 	}
 	if written != artifact.FileCompressedSize {
-		return errors.New("merged file size is not equal to chunk length")
+		return "", errors.New("merged file size is not equal to chunk length")
 	}
 
 	defer func() {
@@ -371,7 +382,7 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
 		rawChecksum := hashSha256.Sum(nil)
 		actualChecksum := hex.EncodeToString(rawChecksum)
 		if !strings.HasSuffix(checksum, actualChecksum) {
-			return fmt.Errorf("update artifact error checksum is invalid %v vs %v", checksum, actualChecksum)
+			return "", fmt.Errorf("update artifact error checksum is invalid %v vs %v", checksum, actualChecksum)
 		}
 	}
 
@@ -384,11 +395,5 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
 		}
 	}
 
-	artifact.StoragePath = storagePath
-	artifact.Status = actions.ArtifactStatusUploadConfirmed
-	if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
-		return fmt.Errorf("update artifact error: %v", err)
-	}
-
-	return nil
+	return storagePath, nil
 }

routers/api/actions/artifactsv4.go

@@ -107,8 +107,10 @@ import (
 	actions_module "gitea.dev/modules/actions"
 	"gitea.dev/modules/httplib"
 	"gitea.dev/modules/log"
+	"gitea.dev/modules/optional"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/storage"
+	"gitea.dev/modules/timeutil"
 	"gitea.dev/modules/util"
 	"gitea.dev/modules/web"
 	"gitea.dev/services/actions"
@@ -332,9 +334,9 @@ func (r *artifactV4Routes) createArtifact(ctx *ArtifactContext) {
 
 	artifactName := req.Name
 
-	retentionDays := setting.Actions.ArtifactRetentionDays
+	var expiry optional.Option[timeutil.TimeStamp]
 	if req.ExpiresAt != nil {
-		retentionDays = int64(time.Until(req.ExpiresAt.AsTime()).Hours() / 24)
+		expiry = optional.Some(timeutil.TimeStamp(req.ExpiresAt.AsTime().Unix()))
 	}
 	encoding := req.GetMimeType().GetValue()
 	// Validate media type
@@ -347,7 +349,7 @@ func (r *artifactV4Routes) createArtifact(ctx *ArtifactContext) {
 		fileName = artifactName + ".zip"
 	}
 	// create or get artifact with name and path
-	artifact, err := actions_model.CreateArtifact(ctx, ctx.ActionTask, artifactName, fileName, retentionDays)
+	artifact, err := actions_model.CreateArtifact(ctx, ctx.ActionTask, artifactName, fileName, expiry)
 	if err != nil {
 		log.Error("Error create or get artifact: %v", err)
 		ctx.HTTPError(http.StatusInternalServerError, "Error create or get artifact")
@@ -383,7 +385,8 @@ func (r *artifactV4Routes) createArtifact(ctx *ArtifactContext) {
 		}
 	}
 
-	if err := actions_model.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
+	if err := actions_model.UpdateArtifact(ctx, artifact,
+		"content_encoding", "file_size", "file_compressed_size", "storage_path", "status"); err != nil {
 		log.Error("Error UpdateArtifactByID: %v", err)
 		ctx.HTTPError(http.StatusInternalServerError, "Error UpdateArtifactByID")
 		return
@@ -418,7 +421,7 @@ func (r *artifactV4Routes) uploadArtifact(ctx *ArtifactContext) {
 			}
 			artifact.FileCompressedSize += uploadedLength
 			artifact.FileSize += uploadedLength
-			if err := actions_model.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
+			if err := actions_model.UpdateArtifact(ctx, artifact, "file_size", "file_compressed_size"); err != nil {
 				log.Error("Error UpdateArtifactByID: %v", err)
 				ctx.HTTPError(http.StatusInternalServerError, "Error UpdateArtifactByID")
 				return
@@ -450,10 +453,6 @@ type BlockList struct {
 	Latest []string `xml:"Latest"`
 }
 
-type Latest struct {
-	Value string `xml:",chardata"`
-}
-
 func (r *artifactV4Routes) readBlockList(runID, artifactID int64) (*BlockList, error) {
 	blockListName := fmt.Sprintf("%s/%d-%d-blocklist", makeTmpPathNameV4(runID), runID, artifactID)
 	s, err := r.fs.Open(blockListName)
@@ -531,11 +530,24 @@ func (r *artifactV4Routes) finalizeDefaultArtifact(ctx *ArtifactContext, req *Fi
 		return
 	}
 
-	if err := mergeChunksForArtifact(ctx, chunks, r.fs, artifact, req.GetHash().GetValue()); err != nil {
+	storagePath, err := mergeChunksForArtifact(chunks, r.fs, artifact, req.GetHash().GetValue())
+	if err != nil {
 		log.Error("Error merge chunks: %v", err)
 		ctx.HTTPError(http.StatusInternalServerError, "Error merge chunks")
 		return
 	}
+	if storagePath == "" {
+		return
+	}
+
+	artifact.StoragePath = storagePath
+	artifact.Status = actions_model.ArtifactStatusUploadConfirmed
+	if err := actions_model.UpdateArtifact(ctx, artifact,
+		"storage_path", "status", "file_size", "file_compressed_size"); err != nil {
+		log.Error("Error UpdateArtifact: %v", err)
+		ctx.HTTPError(http.StatusInternalServerError, "Error UpdateArtifact")
+		return
+	}
 }
 
 func (r *artifactV4Routes) finalizeAzureServeDirect(ctx *ArtifactContext, req *FinalizeArtifactRequest, artifact *actions_model.ActionArtifact) {
@@ -583,7 +595,7 @@ func (r *artifactV4Routes) finalizeAzureServeDirect(ctx *ArtifactContext, req *F
 	artifact.FileSize = actualLength
 	artifact.FileCompressedSize = actualLength
 	artifact.Status = actions_model.ArtifactStatusUploadConfirmed
-	if err := actions_model.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
+	if err := actions_model.UpdateArtifact(ctx, artifact, "file_size", "file_compressed_size", "status"); err != nil {
 		log.Error("Error UpdateArtifactByID: %v", err)
 		ctx.HTTPError(http.StatusInternalServerError, "Error UpdateArtifactByID")
 		return
@@ -608,7 +620,7 @@ func (r *artifactV4Routes) listArtifacts(ctx *ArtifactContext) {
 	artifacts, err := actions_model.FindReadableArtifacts(ctx, actions_model.FindArtifactsOptions{
 		RunID:                runID,
 		RunAttemptIDs:        attemptIDs,
-		Status:               int(actions_model.ArtifactStatusUploadConfirmed),
+		Status:               actions_model.ArtifactStatusUploadConfirmed,
 		FinalizedArtifactsV4: true,
 	})
 	if err != nil {