From fa3780268c899b5bd2a239c734dc988653b9dc3c Mon Sep 17 00:00:00 2001 From: Paarth <76951252+paarth-k2002@users.noreply.github.com> Date: Thu, 23 Jul 2026 01:44:18 +0530 Subject: [PATCH] feat(repo): support file exclusion logic in .gitea/template in template generation (#38064) This PR adds support for file exclusion into `.gitea/template` Docs: https://gitea.com/gitea/docs/pulls/470 ### Usage Template owners add a `.gitea/template` file to their template repository: ``` # Files to include for variable expansion *.go text/*.txt **/modules/* # Files/directories to exclude from the generated repo [exclude] vendor/* node_modules/ src/build/ ``` Resolves #37202 --------- Co-authored-by: wxiaoguang --- modules/test/file.go | 87 +++++++++++++++++ services/repository/generate.go | 77 ++++++++++----- services/repository/generate_test.go | 135 +++++++++++++++++---------- 3 files changed, 226 insertions(+), 73 deletions(-) create mode 100644 modules/test/file.go diff --git a/modules/test/file.go b/modules/test/file.go new file mode 100644 index 0000000000..65362feec7 --- /dev/null +++ b/modules/test/file.go @@ -0,0 +1,87 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package test + +import ( + "os" + "path/filepath" + + "gitea.dev/modules/util" +) + +type FileHelper struct { + baseDir string +} + +func NewFileHelper(baseDir string) *FileHelper { + return &FileHelper{baseDir: baseDir} +} + +func (h *FileHelper) fullPath(path string) string { + return filepath.Join(h.baseDir, path) +} + +func (h *FileHelper) AssertExists(t TestingT, path string, expected bool) { + t.Helper() + _, err := os.Stat(h.fullPath(path)) + errIsNotExist := os.IsNotExist(err) + if expected != !errIsNotExist { + t.Fatalf("file %s existence expected %v", path, expected) + } +} + +func (h *FileHelper) AssertFileExists(t TestingT, path string, expected *string) { + t.Helper() + if expected == nil { + h.AssertExists(t, path, false) + return + } + h.AssertFileContent(t, path, *expected) +} + +func (h *FileHelper) AssertFileContent(t TestingT, path, expected string) { + t.Helper() + data, err := os.ReadFile(h.fullPath(path)) + if err != nil { + t.Fatalf("failed to read file %s: %v", path, err) + } + if string(data) != expected { + t.Fatalf("file %s expected %q but got %q", path, expected, string(data)) + } +} + +func (h *FileHelper) AssertSymLink(t TestingT, path, expected string) { + t.Helper() + link, err := os.Readlink(h.fullPath(path)) + if err != nil { + t.Fatalf("failed to read symlink %s: %v", path, err) + } + if link != h.fullPath(expected) { + t.Fatalf("symlink %s expected %q but got %q", path, expected, link) + } +} + +func (h *FileHelper) WriteFile(t TestingT, path, content string, optMode ...os.FileMode) { + t.Helper() + err := os.WriteFile(h.fullPath(path), []byte(content), util.OptionalArg(optMode, 0o644)) + if err != nil { + t.Fatalf("failed to write file %s: %v", path, err) + } +} + +func (h *FileHelper) MkdirAll(t TestingT, path string, optMode ...os.FileMode) { + t.Helper() + err := os.MkdirAll(h.fullPath(path), util.OptionalArg(optMode, 0o755)) + if err != nil { + t.Fatalf("failed to mkdir all %s: %v", path, err) + } +} + +func (h *FileHelper) Symlink(t TestingT, oldName, newName string) { + t.Helper() + err := os.Symlink(h.fullPath(oldName), h.fullPath(newName)) + if err != nil { + t.Fatalf("failed to symlink from %s to %s: %v", oldName, newName, err) + } +} diff --git a/services/repository/generate.go b/services/repository/generate.go index 5f55ccfb8c..eedcffe5bd 100644 --- a/services/repository/generate.go +++ b/services/repository/generate.go @@ -100,37 +100,51 @@ func generateExpansion(ctx context.Context, src string, templateRepo, generateRe }) } -// giteaTemplateFileMatcher holds information about a .gitea/template file type giteaTemplateFileMatcher struct { - relPath string - globs []glob.Glob + relPath string + globsExpand []glob.Glob + globsExclude []glob.Glob } func newGiteaTemplateFileMatcher(relPath string, content []byte) *giteaTemplateFileMatcher { gt := &giteaTemplateFileMatcher{relPath: relPath} - gt.globs = make([]glob.Glob, 0) scanner := bufio.NewScanner(bytes.NewReader(content)) + addGlob := func(globs *[]glob.Glob, pattern string) { + g, err := glob.Compile(pattern, '/') + if err != nil { + log.Debug("Invalid gitea template glob expression %q (skipped): %v", pattern, err) + return + } + *globs = append(*globs, g) + } + curGlobs := >.globsExpand for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } - g, err := glob.Compile(line, '/') - if err != nil { - log.Debug("Invalid glob expression '%s' (skipped): %v", line, err) + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + switch line { + case "[expand]": + curGlobs = >.globsExpand + case "[exclude]": + curGlobs = >.globsExclude + default: + log.Debug("Invalid gitea template glob section %q", line) + } continue } - gt.globs = append(gt.globs, g) + addGlob(curGlobs, line) } return gt } -func (gt *giteaTemplateFileMatcher) HasRules() bool { - return len(gt.globs) != 0 +func (gt *giteaTemplateFileMatcher) hasAnyRules() bool { + return len(gt.globsExpand) != 0 || len(gt.globsExclude) != 0 } -func (gt *giteaTemplateFileMatcher) Match(s string) bool { - for _, g := range gt.globs { +func (gt *giteaTemplateFileMatcher) matchRules(globs []glob.Glob, s string) bool { + for _, g := range globs { if g.Match(s) { return true } @@ -164,14 +178,15 @@ func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, t return util.WriteRegularPathFile(tmpDir, substSubPath, []byte(generatedContent), 0o755, 0o644) } -// processGiteaTemplateFile processes and removes the .gitea/template file, does variable expansion for template files -// and save the processed files to the filesystem. It returns a list of skipped files that are not regular paths. +// processGiteaTemplateFile processes and removes the .gitea/template file, +// does file exclusion and variable expansion for template files, and save the processed files to the filesystem. +// It returns a list of skipped files that are not regular paths. func processGiteaTemplateFile(ctx context.Context, tmpDir string, templateRepo, generateRepo *repo_model.Repository, fileMatcher *giteaTemplateFileMatcher) (skippedFiles []string, _ error) { // Why not use "os.Root" here: symlink is unsafe even in the same root but "os.Root" can't help, it's more difficult to use "os.Root" to do the WalkDir. if err := os.Remove(util.FilePathJoinAbs(tmpDir, fileMatcher.relPath)); err != nil { return nil, fmt.Errorf("unable to remove .gitea/template: %w", err) } - if !fileMatcher.HasRules() { + if !fileMatcher.hasAnyRules() { return skippedFiles, nil // Avoid walking tree if there are no globs } @@ -179,17 +194,35 @@ func processGiteaTemplateFile(ctx context.Context, tmpDir string, templateRepo, if walkErr != nil { return walkErr } - if d.IsDir() { - return nil - } - tmpDirSubPath, err := filepath.Rel(tmpDir, fullPath) + relPath, err := filepath.Rel(tmpDir, fullPath) if err != nil { return err } - if fileMatcher.Match(filepath.ToSlash(tmpDirSubPath)) { - err := substGiteaTemplateFile(ctx, tmpDir, tmpDirSubPath, templateRepo, generateRepo) + if relPath == "." { + // WalkDir always visits the root "." first, don't process it (don't "exclude" to remove, or "expand" to subst) + return nil + } + + treePath := filepath.ToSlash(relPath) + + // try to "exclude" (remove) first + if fileMatcher.matchRules(fileMatcher.globsExclude, treePath) { + isDir := d.IsDir() + // if the target is a symlink, only the symlink is unlinked, so it is safe + if err := os.RemoveAll(fullPath); err != nil { + return err + } + return util.Iif(isDir, filepath.SkipDir, nil) + } + + if d.IsDir() { + return nil + } + // try to expand variables for regular files + if fileMatcher.matchRules(fileMatcher.globsExpand, treePath) { + err := substGiteaTemplateFile(ctx, tmpDir, relPath, templateRepo, generateRepo) if errors.Is(err, util.ErrNotRegularPathFile) { - skippedFiles = append(skippedFiles, tmpDirSubPath) + skippedFiles = append(skippedFiles, relPath) } else if err != nil { return err } diff --git a/services/repository/generate_test.go b/services/repository/generate_test.go index d484f7360a..b1245c4385 100644 --- a/services/repository/generate_test.go +++ b/services/repository/generate_test.go @@ -10,6 +10,8 @@ import ( "testing" repo_model "gitea.dev/models/repo" + "gitea.dev/modules/test" + "gitea.dev/modules/util" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -27,29 +29,36 @@ text/*.txt # All files in modules folders **/modules/* + +# Exclude some files +[exclude] +**/modules/*.tmp `) gt := newGiteaTemplateFileMatcher("", giteaTemplate) - assert.Len(t, gt.globs, 3) + assert.Len(t, gt.globsExpand, 3) tt := []struct { - Path string - Match bool + Path string + Expand bool + Exclude bool }{ - {Path: "main.go", Match: true}, - {Path: "sub/sub/foo.go", Match: true}, + {Path: "main.go", Expand: true}, + {Path: "sub/sub/foo.go", Expand: true}, - {Path: "a.txt", Match: false}, - {Path: "text/a.txt", Match: true}, - {Path: "sub/text/a.txt", Match: false}, - {Path: "text/a.json", Match: false}, + {Path: "a.txt"}, + {Path: "text/a.txt", Expand: true}, + {Path: "sub/text/a.txt"}, + {Path: "text/a.json"}, - {Path: "a/b/c/modules/README.md", Match: true}, - {Path: "a/b/c/modules/d/README.md", Match: false}, + {Path: "a/b/c/modules/README.md", Expand: true}, + {Path: "a/b/c/modules/README.md.tmp", Expand: true, Exclude: true}, + {Path: "a/b/c/modules/d/README.md"}, } for _, tc := range tt { - assert.Equal(t, tc.Match, gt.Match(tc.Path), "path: %s", tc.Path) + assert.Equal(t, tc.Expand, gt.matchRules(gt.globsExpand, tc.Path), "should expand: %s", tc.Path) + assert.Equal(t, tc.Exclude, gt.matchRules(gt.globsExclude, tc.Path), "should exclude: %s", tc.Path) } } @@ -76,26 +85,7 @@ func TestFilePathSanitize(t *testing.T) { func TestProcessGiteaTemplateFileGenerate(t *testing.T) { tmpDir := filepath.Join(t.TempDir(), "gitea-template-test") - - assertFileContent := func(path, expected string) { - data, err := os.ReadFile(filepath.Join(tmpDir, path)) - if expected == "" { - assert.ErrorIs(t, err, os.ErrNotExist) - return - } - require.NoError(t, err) - assert.Equal(t, expected, string(data), "file content mismatch for %s", path) - } - - assertSymLink := func(path, expected string) { - link, err := os.Readlink(filepath.Join(tmpDir, path)) - if expected == "" { - assert.ErrorIs(t, err, os.ErrNotExist) - return - } - require.NoError(t, err) - assert.Equal(t, expected, link, "symlink target mismatch for %s", path) - } + fh := test.NewFileHelper(tmpDir) require.NoError(t, os.MkdirAll(tmpDir+"/.git", 0o755)) require.NoError(t, os.WriteFile(tmpDir+"/.git/config", []byte("git-config-dummy"), 0o644)) @@ -125,9 +115,10 @@ func TestProcessGiteaTemplateFileGenerate(t *testing.T) { // case-4 assertSubstTemplateName := func(normalContent, toLinkContent, fromLinkContent string) { - assertFileContent("subst-${TEMPLATE_NAME}-normal", normalContent) - assertFileContent("subst-${TEMPLATE_NAME}-to-link", toLinkContent) - assertFileContent("subst-${TEMPLATE_NAME}-from-link", fromLinkContent) + // empty as non-existing + fh.AssertFileExists(t, "subst-${TEMPLATE_NAME}-normal", util.Iif(normalContent == "", nil, new(normalContent))) + fh.AssertFileExists(t, "subst-${TEMPLATE_NAME}-to-link", util.Iif(toLinkContent == "", nil, new(toLinkContent))) + fh.AssertFileExists(t, "subst-${TEMPLATE_NAME}-from-link", util.Iif(fromLinkContent == "", nil, new(fromLinkContent))) } // case-5 @@ -155,7 +146,7 @@ func TestProcessGiteaTemplateFileGenerate(t *testing.T) { { templateRepo := &repo_model.Repository{Name: "TemplateRepoName"} generatedRepo := &repo_model.Repository{Name: "/../.gIt/name"} - assertFileContent(".git/config", "git-config-dummy") + fh.AssertFileContent(t, ".git/config", "git-config-dummy") fileMatcher, _ := readGiteaTemplateFile(tmpDir) skippedFiles, err := processGiteaTemplateFile(t.Context(), tmpDir, templateRepo, generatedRepo, fileMatcher) require.NoError(t, err) @@ -167,47 +158,47 @@ func TestProcessGiteaTemplateFileGenerate(t *testing.T) { "subst-${TEMPLATE_NAME}-to-link", "subst-TemplateRepoName-to-link", }, skippedFiles) - assertFileContent(".git/config", "") - assertFileContent(".gitea/template", "") - assertFileContent("include/foo/bar/test.txt", "include subdir TemplateRepoName") + fh.AssertExists(t, ".git/config", false) + fh.AssertExists(t, ".gitea/template", false) + fh.AssertFileContent(t, "include/foo/bar/test.txt", "include subdir TemplateRepoName") } // the lin target should never be modified, and since it is in a subdirectory, it is not affected by the template either - assertFileContent("sub/link-target", "link target content from ${TEMPLATE_NAME}") + fh.AssertFileContent(t, "sub/link-target", "link target content from ${TEMPLATE_NAME}") // case-1 { - assertFileContent("no-such", "") - assertFileContent("normal", "normal content") - assertFileContent("template", "template from TemplateRepoName") + fh.AssertExists(t, "no-such", false) + fh.AssertFileContent(t, "normal", "normal content") + fh.AssertFileContent(t, "template", "template from TemplateRepoName") } // case-2 { // symlink with templates should be preserved (not read or write) - assertSymLink("link", tmpDir+"/sub/link-target") + fh.AssertSymLink(t, "link", "sub/link-target") } // case-3 { - assertFileContent("subst-${REPO_NAME}", "") - assertFileContent("subst-/__/_gIt/name", "dummy subst repo name") + fh.AssertExists(t, "subst-${REPO_NAME}", false) + fh.AssertFileContent(t, "subst-/__/_gIt/name", "dummy subst repo name") } // case-4 { // the paths with templates should have been removed, subst to a regular file, succeed, the link is preserved assertSubstTemplateName("", "", "link target content from ${TEMPLATE_NAME}") - assertFileContent("subst-TemplateRepoName-normal", "dummy subst template name normal") + fh.AssertFileContent(t, "subst-TemplateRepoName-normal", "dummy subst template name normal") // subst to a link, skip, and the target is unchanged - assertSymLink("subst-TemplateRepoName-to-link", tmpDir+"/sub/link-target") + fh.AssertSymLink(t, "subst-TemplateRepoName-to-link", "sub/link-target") // subst from a link, skip, and the target is unchanged - assertSymLink("subst-${TEMPLATE_NAME}-from-link", tmpDir+"/sub/link-target") + fh.AssertSymLink(t, "subst-${TEMPLATE_NAME}-from-link", "sub/link-target") } // case-5 { - assertFileContent("real-dir/real-file", "origin content") + fh.AssertFileContent(t, "real-dir/real-file", "origin content") } } @@ -234,7 +225,49 @@ func TestProcessGiteaTemplateFileRead(t *testing.T) { require.Equal(t, "test-data-regular", string(content)) fm, err := readGiteaTemplateFile(tmpDir) // regular template file require.NoError(t, err) - assert.Len(t, fm.globs, 1) + assert.Len(t, fm.globsExpand, 1) +} + +func TestProcessGiteaTemplateFileExclusion(t *testing.T) { + tmpDir := t.TempDir() + fh := test.NewFileHelper(tmpDir) + + require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, ".gitea"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".gitea", "template"), []byte(` +[expand] +*.md +*.txt + +[exclude] +*.log +`), 0o644)) + fh.WriteFile(t, "test.md", "from ${TEMPLATE_NAME}") + fh.WriteFile(t, "test.go", "package test") + fh.WriteFile(t, "test.log", "log-content") + fh.MkdirAll(t, "subdir") + fh.WriteFile(t, "subdir/foo", "bar") + fh.Symlink(t, "test.go", "symlink.go") + fh.Symlink(t, "test.log", "symlink.log") + fh.Symlink(t, "subdir", "symlink-dir.log") + + templateRepo := &repo_model.Repository{Name: "MyTemplate"} + generatedRepo := &repo_model.Repository{Name: "MyRepo"} + + fm, err := readGiteaTemplateFile(tmpDir) + require.NoError(t, err) + require.Len(t, fm.globsExpand, 2) + require.Len(t, fm.globsExclude, 1) + + _, err = processGiteaTemplateFile(t.Context(), tmpDir, templateRepo, generatedRepo, fm) + require.NoError(t, err) + + fh.AssertFileContent(t, "test.md", "from MyTemplate") + fh.AssertFileContent(t, "test.go", "package test") + fh.AssertExists(t, "test.log", false) + fh.AssertFileContent(t, "subdir/foo", "bar") + fh.AssertSymLink(t, "symlink.go", "test.go") + fh.AssertExists(t, "symlink.log", false) + fh.AssertExists(t, "symlink-dir.log", false) } func TestTransformers(t *testing.T) {