gitea/services/repository/generate.go

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package repository

import (
	"bufio"
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	git_model "gitea.dev/models/git"
	repo_model "gitea.dev/models/repo"
	"gitea.dev/modules/git"
	"gitea.dev/modules/glob"
	"gitea.dev/modules/log"
	repo_module "gitea.dev/modules/repository"
	"gitea.dev/modules/setting"
	"gitea.dev/modules/util"

	"github.com/huandu/xstrings"
)

type transformer struct {
	Name      string
	Transform func(string) string
}

type expansion struct {
	Name         string
	Value        string
	Transformers []transformer
}

var globalVars = sync.OnceValue(func() (ret struct {
	defaultTransformers    []transformer
	fileNameSanitizeRegexp *regexp.Regexp
},
) {
	ret.defaultTransformers = []transformer{
		{Name: "SNAKE", Transform: xstrings.ToSnakeCase},
		{Name: "KEBAB", Transform: xstrings.ToKebabCase},
		{Name: "CAMEL", Transform: xstrings.ToCamelCase},
		{Name: "PASCAL", Transform: xstrings.ToPascalCase},
		{Name: "LOWER", Transform: strings.ToLower},
		{Name: "UPPER", Transform: strings.ToUpper},
		{Name: "TITLE", Transform: util.ToTitleCase},
	}

	// invalid filename contents, based on https://github.com/sindresorhus/filename-reserved-regex
	// "COM10" needs to be opened with UNC "\\.\COM10" on Windows, so itself is valid
	ret.fileNameSanitizeRegexp = regexp.MustCompile(`(?i)[<>:"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`)
	return ret
})

func generateExpansion(ctx context.Context, src string, templateRepo, generateRepo *repo_model.Repository) string {
	transformers := globalVars().defaultTransformers
	year, month, day := time.Now().Date()
	expansions := []expansion{
		{Name: "YEAR", Value: strconv.Itoa(year), Transformers: nil},
		{Name: "MONTH", Value: fmt.Sprintf("%02d", int(month)), Transformers: nil},
		{Name: "MONTH_ENGLISH", Value: month.String(), Transformers: transformers},
		{Name: "DAY", Value: fmt.Sprintf("%02d", day), Transformers: nil},
		{Name: "REPO_NAME", Value: generateRepo.Name, Transformers: transformers},
		{Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: transformers},
		{Name: "REPO_DESCRIPTION", Value: generateRepo.Description, Transformers: nil},
		{Name: "TEMPLATE_DESCRIPTION", Value: templateRepo.Description, Transformers: nil},
		{Name: "REPO_OWNER", Value: generateRepo.OwnerName, Transformers: transformers},
		{Name: "TEMPLATE_OWNER", Value: templateRepo.OwnerName, Transformers: transformers},
		{Name: "REPO_LINK", Value: generateRepo.Link(), Transformers: nil},
		{Name: "TEMPLATE_LINK", Value: templateRepo.Link(), Transformers: nil},
		{Name: "REPO_HTTPS_URL", Value: generateRepo.CloneLinkGeneral(ctx).HTTPS, Transformers: nil},
		{Name: "TEMPLATE_HTTPS_URL", Value: templateRepo.CloneLinkGeneral(ctx).HTTPS, Transformers: nil},
		{Name: "REPO_SSH_URL", Value: generateRepo.CloneLinkGeneral(ctx).SSH, Transformers: nil},
		{Name: "TEMPLATE_SSH_URL", Value: templateRepo.CloneLinkGeneral(ctx).SSH, Transformers: nil},
	}

	expansionMap := make(map[string]string)
	for _, e := range expansions {
		expansionMap[e.Name] = e.Value
		for _, tr := range e.Transformers {
			expansionMap[fmt.Sprintf("%s_%s", e.Name, tr.Name)] = tr.Transform(e.Value)
		}
	}

	return os.Expand(src, func(key string) string {
		if val, ok := expansionMap[key]; ok {
			return val
		}
		return key
	})
}

type giteaTemplateFileMatcher struct {
	relPath      string
	globsExpand  []glob.Glob
	globsExclude []glob.Glob
}

func newGiteaTemplateFileMatcher(relPath string, content []byte) *giteaTemplateFileMatcher {
	gt := &giteaTemplateFileMatcher{relPath: relPath}
	scanner := bufio.NewScanner(bytes.NewReader(content))
	addGlob := func(globs *[]glob.Glob, pattern string) {
		g, err := glob.Compile(pattern, '/')
		if err != nil {
			log.Debug("Invalid gitea template glob expression %q (skipped): %v", pattern, err)
			return
		}
		*globs = append(*globs, g)
	}
	curGlobs := &gt.globsExpand
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
			switch line {
			case "[expand]":
				curGlobs = &gt.globsExpand
			case "[exclude]":
				curGlobs = &gt.globsExclude
			default:
				log.Debug("Invalid gitea template glob section %q", line)
			}
			continue
		}
		addGlob(curGlobs, line)
	}
	return gt
}

func (gt *giteaTemplateFileMatcher) hasAnyRules() bool {
	return len(gt.globsExpand) != 0 || len(gt.globsExclude) != 0
}

func (gt *giteaTemplateFileMatcher) matchRules(globs []glob.Glob, s string) bool {
	for _, g := range globs {
		if g.Match(s) {
			return true
		}
	}
	return false
}

func readGiteaTemplateFile(tmpDir string) (*giteaTemplateFileMatcher, error) {
	templateRelPath := filepath.Join(".gitea", "template")
	content, err := util.ReadRegularPathFile(tmpDir, templateRelPath, 1024*1024)
	if err != nil {
		return nil, util.Iif(errors.Is(err, util.ErrNotRegularPathFile), os.ErrNotExist, err)
	}
	return newGiteaTemplateFileMatcher(templateRelPath, content), nil
}

func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, templateRepo, generateRepo *repo_model.Repository) error {
	content, err := util.ReadRegularPathFile(tmpDir, tmpDirSubPath, 1024*1024)
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return nil
		}
		return err
	}
	if err := os.Remove(util.FilePathJoinAbs(tmpDir, tmpDirSubPath)); err != nil {
		return err
	}

	generatedContent := generateExpansion(ctx, string(content), templateRepo, generateRepo)
	substSubPath := filePathSanitize(generateExpansion(ctx, tmpDirSubPath, templateRepo, generateRepo))
	return util.WriteRegularPathFile(tmpDir, substSubPath, []byte(generatedContent), 0o755, 0o644)
}

// processGiteaTemplateFile processes and removes the .gitea/template file,
// does file exclusion and variable expansion for template files, and save the processed files to the filesystem.
// It returns a list of skipped files that are not regular paths.
func processGiteaTemplateFile(ctx context.Context, tmpDir string, templateRepo, generateRepo *repo_model.Repository, fileMatcher *giteaTemplateFileMatcher) (skippedFiles []string, _ error) {
	// Why not use "os.Root" here: symlink is unsafe even in the same root but "os.Root" can't help, it's more difficult to use "os.Root" to do the WalkDir.
	if err := os.Remove(util.FilePathJoinAbs(tmpDir, fileMatcher.relPath)); err != nil {
		return nil, fmt.Errorf("unable to remove .gitea/template: %w", err)
	}
	if !fileMatcher.hasAnyRules() {
		return skippedFiles, nil // Avoid walking tree if there are no globs
	}

	err := filepath.WalkDir(tmpDir, func(fullPath string, d os.DirEntry, walkErr error) error {
		if walkErr != nil {
			return walkErr
		}
		relPath, err := filepath.Rel(tmpDir, fullPath)
		if err != nil {
			return err
		}
		if relPath == "." {
			// WalkDir always visits the root "." first, don't process it (don't "exclude" to remove, or "expand" to subst)
			return nil
		}

		treePath := filepath.ToSlash(relPath)

		// try to "exclude" (remove) first
		if fileMatcher.matchRules(fileMatcher.globsExclude, treePath) {
			isDir := d.IsDir()
			// if the target is a symlink, only the symlink is unlinked, so it is safe
			if err := os.RemoveAll(fullPath); err != nil {
				return err
			}
			return util.Iif(isDir, filepath.SkipDir, nil)
		}

		if d.IsDir() {
			return nil
		}
		// try to expand variables for regular files
		if fileMatcher.matchRules(fileMatcher.globsExpand, treePath) {
			err := substGiteaTemplateFile(ctx, tmpDir, relPath, templateRepo, generateRepo)
			if errors.Is(err, util.ErrNotRegularPathFile) {
				skippedFiles = append(skippedFiles, relPath)
			} else if err != nil {
				return err
			}
		}
		return nil
	}) // end: WalkDir
	if err != nil {
		return nil, err
	}
	if err = util.RemoveAllWithRetry(util.FilePathJoinAbs(tmpDir, ".git")); err != nil {
		return nil, err
	}
	return skippedFiles, nil
}

func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *repo_model.Repository, tmpDir string) error {
	// set default branch based on whether it's specified in the newly generated repo or not
	repo.DefaultBranch = util.IfZero(repo.DefaultBranch, util.IfZero(templateRepo.DefaultBranch, setting.Repository.DefaultBranch))

	// Clone to temporary path and do the init commit.
	if err := git.CloneRepoToLocal(ctx, templateRepo, tmpDir, git.CloneRepoOptions{
		Depth:  1,
		Branch: templateRepo.DefaultBranch,
	}); err != nil {
		return fmt.Errorf("git clone: %w", err)
	}

	// Get active submodules from the template
	submodules, err := git.GetTemplateSubmoduleCommits(ctx, templateRepo)
	if err != nil {
		return fmt.Errorf("GetTemplateSubmoduleCommits: %w", err)
	}

	if err = util.RemoveAllWithRetry(filepath.Join(tmpDir, ".git")); err != nil {
		return fmt.Errorf("remove git dir: %w", err)
	}

	// Variable expansion
	fileMatcher, err := readGiteaTemplateFile(tmpDir)
	if err == nil {
		_, err = processGiteaTemplateFile(ctx, tmpDir, templateRepo, generateRepo, fileMatcher)
		if err != nil {
			return fmt.Errorf("processGiteaTemplateFile: %w", err)
		}
	} else if errors.Is(err, fs.ErrNotExist) {
		log.Debug("skip processing repo template files: no available .gitea/template")
	} else {
		return fmt.Errorf("readGiteaTemplateFile: %w", err)
	}

	if err = git.InitRepositoryLocal(ctx, tmpDir, false, templateRepo.ObjectFormatName); err != nil {
		return err
	}

	if err = git.AddTemplateSubmoduleIndexes(ctx, tmpDir, submodules); err != nil {
		return fmt.Errorf("failed to add submodules: %v", err)
	}

	return initRepoCommit(ctx, tmpDir, repo, repo.Owner)
}

// GenerateGitContent generates git content from a template repository
func GenerateGitContent(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) (err error) {
	tmpDir, cleanup, err := setting.AppDataTempDir("git-repo-content").MkdirTempRandom("gitea-" + generateRepo.Name)
	if err != nil {
		return fmt.Errorf("failed to create temp dir for repository %s: %w", generateRepo.FullName(), err)
	}
	defer cleanup()

	if err = generateRepoCommit(ctx, generateRepo, templateRepo, generateRepo, tmpDir); err != nil {
		return fmt.Errorf("generateRepoCommit: %w", err)
	}
	if err = git.SetDefaultBranch(ctx, generateRepo, generateRepo.DefaultBranch); err != nil {
		return fmt.Errorf("setDefaultBranch: %w", err)
	}
	if err = repo_model.UpdateRepositoryColsNoAutoTime(ctx, generateRepo, "default_branch"); err != nil {
		return fmt.Errorf("updateRepository: %w", err)
	}

	if err := repo_module.UpdateRepoSize(ctx, generateRepo); err != nil {
		return fmt.Errorf("failed to update size for repository: %w", err)
	}

	if err := git_model.CopyLFS(ctx, generateRepo, templateRepo); err != nil {
		return fmt.Errorf("failed to copy LFS: %w", err)
	}

	if _, err := repo_module.SyncRepoBranches(ctx, generateRepo.ID, 0); err != nil {
		return fmt.Errorf("SyncRepoBranches: %w", err)
	}
	return nil
}

// GenerateRepoOptions contains the template units to generate
type GenerateRepoOptions struct {
	Name            string
	DefaultBranch   string
	Description     string
	Private         bool
	GitContent      bool
	Topics          bool
	GitHooks        bool
	Webhooks        bool
	Avatar          bool
	IssueLabels     bool
	ProtectedBranch bool
}

// IsValid checks whether at least one option is chosen for generation
func (gro GenerateRepoOptions) IsValid() bool {
	return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar ||
		gro.IssueLabels || gro.ProtectedBranch // or other items as they are added
}

func filePathSanitize(s string) string {
	fields := strings.Split(filepath.ToSlash(s), "/")
	for i, field := range fields {
		field = strings.TrimSpace(strings.TrimSpace(globalVars().fileNameSanitizeRegexp.ReplaceAllString(field, "_")))
		if strings.HasPrefix(field, "..") {
			field = "__" + field[2:]
		}
		if strings.EqualFold(field, ".git") {
			field = "_" + field[1:]
		}
		fields[i] = field
	}
	return filepath.Clean(filepath.FromSlash(strings.Trim(strings.Join(fields, "/"), "/")))
}