mirror of
https://github.com/go-gitea/gitea.git
synced 2026-07-23 17:32:39 +00:00
feat(repo): support file exclusion logic in .gitea/template in template generation (#38064)
This PR adds support for file exclusion into `.gitea/template` Docs: https://gitea.com/gitea/docs/pulls/470 ### Usage Template owners add a `.gitea/template` file to their template repository: ``` # Files to include for variable expansion *.go text/*.txt **/modules/* # Files/directories to exclude from the generated repo [exclude] vendor/* node_modules/ src/build/ ``` Resolves #37202 --------- Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Paarth
87
modules/test/file.go
Normal file
87
modules/test/file.go
Normal file
@@ -0,0 +1,87 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"gitea.dev/modules/util"
|
||||
)
|
||||
|
||||
type FileHelper struct {
|
||||
baseDir string
|
||||
}
|
||||
|
||||
func NewFileHelper(baseDir string) *FileHelper {
|
||||
return &FileHelper{baseDir: baseDir}
|
||||
}
|
||||
|
||||
func (h *FileHelper) fullPath(path string) string {
|
||||
return filepath.Join(h.baseDir, path)
|
||||
}
|
||||
|
||||
func (h *FileHelper) AssertExists(t TestingT, path string, expected bool) {
|
||||
t.Helper()
|
||||
_, err := os.Stat(h.fullPath(path))
|
||||
errIsNotExist := os.IsNotExist(err)
|
||||
if expected != !errIsNotExist {
|
||||
t.Fatalf("file %s existence expected %v", path, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FileHelper) AssertFileExists(t TestingT, path string, expected *string) {
|
||||
t.Helper()
|
||||
if expected == nil {
|
||||
h.AssertExists(t, path, false)
|
||||
return
|
||||
}
|
||||
h.AssertFileContent(t, path, *expected)
|
||||
}
|
||||
|
||||
func (h *FileHelper) AssertFileContent(t TestingT, path, expected string) {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(h.fullPath(path))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file %s: %v", path, err)
|
||||
}
|
||||
if string(data) != expected {
|
||||
t.Fatalf("file %s expected %q but got %q", path, expected, string(data))
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FileHelper) AssertSymLink(t TestingT, path, expected string) {
|
||||
t.Helper()
|
||||
link, err := os.Readlink(h.fullPath(path))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read symlink %s: %v", path, err)
|
||||
}
|
||||
if link != h.fullPath(expected) {
|
||||
t.Fatalf("symlink %s expected %q but got %q", path, expected, link)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FileHelper) WriteFile(t TestingT, path, content string, optMode ...os.FileMode) {
|
||||
t.Helper()
|
||||
err := os.WriteFile(h.fullPath(path), []byte(content), util.OptionalArg(optMode, 0o644))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write file %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FileHelper) MkdirAll(t TestingT, path string, optMode ...os.FileMode) {
|
||||
t.Helper()
|
||||
err := os.MkdirAll(h.fullPath(path), util.OptionalArg(optMode, 0o755))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to mkdir all %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FileHelper) Symlink(t TestingT, oldName, newName string) {
|
||||
t.Helper()
|
||||
err := os.Symlink(h.fullPath(oldName), h.fullPath(newName))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to symlink from %s to %s: %v", oldName, newName, err)
|
||||
}
|
||||
}
|
||||
@@ -100,37 +100,51 @@ func generateExpansion(ctx context.Context, src string, templateRepo, generateRe
|
||||
})
|
||||
}
|
||||
|
||||
// giteaTemplateFileMatcher holds information about a .gitea/template file
|
||||
type giteaTemplateFileMatcher struct {
|
||||
relPath string
|
||||
globs []glob.Glob
|
||||
relPath string
|
||||
globsExpand []glob.Glob
|
||||
globsExclude []glob.Glob
|
||||
}
|
||||
|
||||
func newGiteaTemplateFileMatcher(relPath string, content []byte) *giteaTemplateFileMatcher {
|
||||
gt := &giteaTemplateFileMatcher{relPath: relPath}
|
||||
gt.globs = make([]glob.Glob, 0)
|
||||
scanner := bufio.NewScanner(bytes.NewReader(content))
|
||||
addGlob := func(globs *[]glob.Glob, pattern string) {
|
||||
g, err := glob.Compile(pattern, '/')
|
||||
if err != nil {
|
||||
log.Debug("Invalid gitea template glob expression %q (skipped): %v", pattern, err)
|
||||
return
|
||||
}
|
||||
*globs = append(*globs, g)
|
||||
}
|
||||
curGlobs := >.globsExpand
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
g, err := glob.Compile(line, '/')
|
||||
if err != nil {
|
||||
log.Debug("Invalid glob expression '%s' (skipped): %v", line, err)
|
||||
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
|
||||
switch line {
|
||||
case "[expand]":
|
||||
curGlobs = >.globsExpand
|
||||
case "[exclude]":
|
||||
curGlobs = >.globsExclude
|
||||
default:
|
||||
log.Debug("Invalid gitea template glob section %q", line)
|
||||
}
|
||||
continue
|
||||
}
|
||||
gt.globs = append(gt.globs, g)
|
||||
addGlob(curGlobs, line)
|
||||
}
|
||||
return gt
|
||||
}
|
||||
|
||||
func (gt *giteaTemplateFileMatcher) HasRules() bool {
|
||||
return len(gt.globs) != 0
|
||||
func (gt *giteaTemplateFileMatcher) hasAnyRules() bool {
|
||||
return len(gt.globsExpand) != 0 || len(gt.globsExclude) != 0
|
||||
}
|
||||
|
||||
func (gt *giteaTemplateFileMatcher) Match(s string) bool {
|
||||
for _, g := range gt.globs {
|
||||
func (gt *giteaTemplateFileMatcher) matchRules(globs []glob.Glob, s string) bool {
|
||||
for _, g := range globs {
|
||||
if g.Match(s) {
|
||||
return true
|
||||
}
|
||||
@@ -164,14 +178,15 @@ func substGiteaTemplateFile(ctx context.Context, tmpDir, tmpDirSubPath string, t
|
||||
return util.WriteRegularPathFile(tmpDir, substSubPath, []byte(generatedContent), 0o755, 0o644)
|
||||
}
|
||||
|
||||
// processGiteaTemplateFile processes and removes the .gitea/template file, does variable expansion for template files
|
||||
// and save the processed files to the filesystem. It returns a list of skipped files that are not regular paths.
|
||||
// processGiteaTemplateFile processes and removes the .gitea/template file,
|
||||
// does file exclusion and variable expansion for template files, and save the processed files to the filesystem.
|
||||
// It returns a list of skipped files that are not regular paths.
|
||||
func processGiteaTemplateFile(ctx context.Context, tmpDir string, templateRepo, generateRepo *repo_model.Repository, fileMatcher *giteaTemplateFileMatcher) (skippedFiles []string, _ error) {
|
||||
// Why not use "os.Root" here: symlink is unsafe even in the same root but "os.Root" can't help, it's more difficult to use "os.Root" to do the WalkDir.
|
||||
if err := os.Remove(util.FilePathJoinAbs(tmpDir, fileMatcher.relPath)); err != nil {
|
||||
return nil, fmt.Errorf("unable to remove .gitea/template: %w", err)
|
||||
}
|
||||
if !fileMatcher.HasRules() {
|
||||
if !fileMatcher.hasAnyRules() {
|
||||
return skippedFiles, nil // Avoid walking tree if there are no globs
|
||||
}
|
||||
|
||||
@@ -179,17 +194,35 @@ func processGiteaTemplateFile(ctx context.Context, tmpDir string, templateRepo,
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
tmpDirSubPath, err := filepath.Rel(tmpDir, fullPath)
|
||||
relPath, err := filepath.Rel(tmpDir, fullPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fileMatcher.Match(filepath.ToSlash(tmpDirSubPath)) {
|
||||
err := substGiteaTemplateFile(ctx, tmpDir, tmpDirSubPath, templateRepo, generateRepo)
|
||||
if relPath == "." {
|
||||
// WalkDir always visits the root "." first, don't process it (don't "exclude" to remove, or "expand" to subst)
|
||||
return nil
|
||||
}
|
||||
|
||||
treePath := filepath.ToSlash(relPath)
|
||||
|
||||
// try to "exclude" (remove) first
|
||||
if fileMatcher.matchRules(fileMatcher.globsExclude, treePath) {
|
||||
isDir := d.IsDir()
|
||||
// if the target is a symlink, only the symlink is unlinked, so it is safe
|
||||
if err := os.RemoveAll(fullPath); err != nil {
|
||||
return err
|
||||
}
|
||||
return util.Iif(isDir, filepath.SkipDir, nil)
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
// try to expand variables for regular files
|
||||
if fileMatcher.matchRules(fileMatcher.globsExpand, treePath) {
|
||||
err := substGiteaTemplateFile(ctx, tmpDir, relPath, templateRepo, generateRepo)
|
||||
if errors.Is(err, util.ErrNotRegularPathFile) {
|
||||
skippedFiles = append(skippedFiles, tmpDirSubPath)
|
||||
skippedFiles = append(skippedFiles, relPath)
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"testing"
|
||||
|
||||
repo_model "gitea.dev/models/repo"
|
||||
"gitea.dev/modules/test"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -27,29 +29,36 @@ text/*.txt
|
||||
|
||||
# All files in modules folders
|
||||
**/modules/*
|
||||
|
||||
# Exclude some files
|
||||
[exclude]
|
||||
**/modules/*.tmp
|
||||
`)
|
||||
|
||||
gt := newGiteaTemplateFileMatcher("", giteaTemplate)
|
||||
assert.Len(t, gt.globs, 3)
|
||||
assert.Len(t, gt.globsExpand, 3)
|
||||
|
||||
tt := []struct {
|
||||
Path string
|
||||
Match bool
|
||||
Path string
|
||||
Expand bool
|
||||
Exclude bool
|
||||
}{
|
||||
{Path: "main.go", Match: true},
|
||||
{Path: "sub/sub/foo.go", Match: true},
|
||||
{Path: "main.go", Expand: true},
|
||||
{Path: "sub/sub/foo.go", Expand: true},
|
||||
|
||||
{Path: "a.txt", Match: false},
|
||||
{Path: "text/a.txt", Match: true},
|
||||
{Path: "sub/text/a.txt", Match: false},
|
||||
{Path: "text/a.json", Match: false},
|
||||
{Path: "a.txt"},
|
||||
{Path: "text/a.txt", Expand: true},
|
||||
{Path: "sub/text/a.txt"},
|
||||
{Path: "text/a.json"},
|
||||
|
||||
{Path: "a/b/c/modules/README.md", Match: true},
|
||||
{Path: "a/b/c/modules/d/README.md", Match: false},
|
||||
{Path: "a/b/c/modules/README.md", Expand: true},
|
||||
{Path: "a/b/c/modules/README.md.tmp", Expand: true, Exclude: true},
|
||||
{Path: "a/b/c/modules/d/README.md"},
|
||||
}
|
||||
|
||||
for _, tc := range tt {
|
||||
assert.Equal(t, tc.Match, gt.Match(tc.Path), "path: %s", tc.Path)
|
||||
assert.Equal(t, tc.Expand, gt.matchRules(gt.globsExpand, tc.Path), "should expand: %s", tc.Path)
|
||||
assert.Equal(t, tc.Exclude, gt.matchRules(gt.globsExclude, tc.Path), "should exclude: %s", tc.Path)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -76,26 +85,7 @@ func TestFilePathSanitize(t *testing.T) {
|
||||
|
||||
func TestProcessGiteaTemplateFileGenerate(t *testing.T) {
|
||||
tmpDir := filepath.Join(t.TempDir(), "gitea-template-test")
|
||||
|
||||
assertFileContent := func(path, expected string) {
|
||||
data, err := os.ReadFile(filepath.Join(tmpDir, path))
|
||||
if expected == "" {
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expected, string(data), "file content mismatch for %s", path)
|
||||
}
|
||||
|
||||
assertSymLink := func(path, expected string) {
|
||||
link, err := os.Readlink(filepath.Join(tmpDir, path))
|
||||
if expected == "" {
|
||||
assert.ErrorIs(t, err, os.ErrNotExist)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, expected, link, "symlink target mismatch for %s", path)
|
||||
}
|
||||
fh := test.NewFileHelper(tmpDir)
|
||||
|
||||
require.NoError(t, os.MkdirAll(tmpDir+"/.git", 0o755))
|
||||
require.NoError(t, os.WriteFile(tmpDir+"/.git/config", []byte("git-config-dummy"), 0o644))
|
||||
@@ -125,9 +115,10 @@ func TestProcessGiteaTemplateFileGenerate(t *testing.T) {
|
||||
|
||||
// case-4
|
||||
assertSubstTemplateName := func(normalContent, toLinkContent, fromLinkContent string) {
|
||||
assertFileContent("subst-${TEMPLATE_NAME}-normal", normalContent)
|
||||
assertFileContent("subst-${TEMPLATE_NAME}-to-link", toLinkContent)
|
||||
assertFileContent("subst-${TEMPLATE_NAME}-from-link", fromLinkContent)
|
||||
// empty as non-existing
|
||||
fh.AssertFileExists(t, "subst-${TEMPLATE_NAME}-normal", util.Iif(normalContent == "", nil, new(normalContent)))
|
||||
fh.AssertFileExists(t, "subst-${TEMPLATE_NAME}-to-link", util.Iif(toLinkContent == "", nil, new(toLinkContent)))
|
||||
fh.AssertFileExists(t, "subst-${TEMPLATE_NAME}-from-link", util.Iif(fromLinkContent == "", nil, new(fromLinkContent)))
|
||||
}
|
||||
|
||||
// case-5
|
||||
@@ -155,7 +146,7 @@ func TestProcessGiteaTemplateFileGenerate(t *testing.T) {
|
||||
{
|
||||
templateRepo := &repo_model.Repository{Name: "TemplateRepoName"}
|
||||
generatedRepo := &repo_model.Repository{Name: "/../.gIt/name"}
|
||||
assertFileContent(".git/config", "git-config-dummy")
|
||||
fh.AssertFileContent(t, ".git/config", "git-config-dummy")
|
||||
fileMatcher, _ := readGiteaTemplateFile(tmpDir)
|
||||
skippedFiles, err := processGiteaTemplateFile(t.Context(), tmpDir, templateRepo, generatedRepo, fileMatcher)
|
||||
require.NoError(t, err)
|
||||
@@ -167,47 +158,47 @@ func TestProcessGiteaTemplateFileGenerate(t *testing.T) {
|
||||
"subst-${TEMPLATE_NAME}-to-link",
|
||||
"subst-TemplateRepoName-to-link",
|
||||
}, skippedFiles)
|
||||
assertFileContent(".git/config", "")
|
||||
assertFileContent(".gitea/template", "")
|
||||
assertFileContent("include/foo/bar/test.txt", "include subdir TemplateRepoName")
|
||||
fh.AssertExists(t, ".git/config", false)
|
||||
fh.AssertExists(t, ".gitea/template", false)
|
||||
fh.AssertFileContent(t, "include/foo/bar/test.txt", "include subdir TemplateRepoName")
|
||||
}
|
||||
|
||||
// the lin target should never be modified, and since it is in a subdirectory, it is not affected by the template either
|
||||
assertFileContent("sub/link-target", "link target content from ${TEMPLATE_NAME}")
|
||||
fh.AssertFileContent(t, "sub/link-target", "link target content from ${TEMPLATE_NAME}")
|
||||
|
||||
// case-1
|
||||
{
|
||||
assertFileContent("no-such", "")
|
||||
assertFileContent("normal", "normal content")
|
||||
assertFileContent("template", "template from TemplateRepoName")
|
||||
fh.AssertExists(t, "no-such", false)
|
||||
fh.AssertFileContent(t, "normal", "normal content")
|
||||
fh.AssertFileContent(t, "template", "template from TemplateRepoName")
|
||||
}
|
||||
|
||||
// case-2
|
||||
{
|
||||
// symlink with templates should be preserved (not read or write)
|
||||
assertSymLink("link", tmpDir+"/sub/link-target")
|
||||
fh.AssertSymLink(t, "link", "sub/link-target")
|
||||
}
|
||||
|
||||
// case-3
|
||||
{
|
||||
assertFileContent("subst-${REPO_NAME}", "")
|
||||
assertFileContent("subst-/__/_gIt/name", "dummy subst repo name")
|
||||
fh.AssertExists(t, "subst-${REPO_NAME}", false)
|
||||
fh.AssertFileContent(t, "subst-/__/_gIt/name", "dummy subst repo name")
|
||||
}
|
||||
|
||||
// case-4
|
||||
{
|
||||
// the paths with templates should have been removed, subst to a regular file, succeed, the link is preserved
|
||||
assertSubstTemplateName("", "", "link target content from ${TEMPLATE_NAME}")
|
||||
assertFileContent("subst-TemplateRepoName-normal", "dummy subst template name normal")
|
||||
fh.AssertFileContent(t, "subst-TemplateRepoName-normal", "dummy subst template name normal")
|
||||
// subst to a link, skip, and the target is unchanged
|
||||
assertSymLink("subst-TemplateRepoName-to-link", tmpDir+"/sub/link-target")
|
||||
fh.AssertSymLink(t, "subst-TemplateRepoName-to-link", "sub/link-target")
|
||||
// subst from a link, skip, and the target is unchanged
|
||||
assertSymLink("subst-${TEMPLATE_NAME}-from-link", tmpDir+"/sub/link-target")
|
||||
fh.AssertSymLink(t, "subst-${TEMPLATE_NAME}-from-link", "sub/link-target")
|
||||
}
|
||||
|
||||
// case-5
|
||||
{
|
||||
assertFileContent("real-dir/real-file", "origin content")
|
||||
fh.AssertFileContent(t, "real-dir/real-file", "origin content")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,7 +225,49 @@ func TestProcessGiteaTemplateFileRead(t *testing.T) {
|
||||
require.Equal(t, "test-data-regular", string(content))
|
||||
fm, err := readGiteaTemplateFile(tmpDir) // regular template file
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, fm.globs, 1)
|
||||
assert.Len(t, fm.globsExpand, 1)
|
||||
}
|
||||
|
||||
func TestProcessGiteaTemplateFileExclusion(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
fh := test.NewFileHelper(tmpDir)
|
||||
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(tmpDir, ".gitea"), 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, ".gitea", "template"), []byte(`
|
||||
[expand]
|
||||
*.md
|
||||
*.txt
|
||||
|
||||
[exclude]
|
||||
*.log
|
||||
`), 0o644))
|
||||
fh.WriteFile(t, "test.md", "from ${TEMPLATE_NAME}")
|
||||
fh.WriteFile(t, "test.go", "package test")
|
||||
fh.WriteFile(t, "test.log", "log-content")
|
||||
fh.MkdirAll(t, "subdir")
|
||||
fh.WriteFile(t, "subdir/foo", "bar")
|
||||
fh.Symlink(t, "test.go", "symlink.go")
|
||||
fh.Symlink(t, "test.log", "symlink.log")
|
||||
fh.Symlink(t, "subdir", "symlink-dir.log")
|
||||
|
||||
templateRepo := &repo_model.Repository{Name: "MyTemplate"}
|
||||
generatedRepo := &repo_model.Repository{Name: "MyRepo"}
|
||||
|
||||
fm, err := readGiteaTemplateFile(tmpDir)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fm.globsExpand, 2)
|
||||
require.Len(t, fm.globsExclude, 1)
|
||||
|
||||
_, err = processGiteaTemplateFile(t.Context(), tmpDir, templateRepo, generatedRepo, fm)
|
||||
require.NoError(t, err)
|
||||
|
||||
fh.AssertFileContent(t, "test.md", "from MyTemplate")
|
||||
fh.AssertFileContent(t, "test.go", "package test")
|
||||
fh.AssertExists(t, "test.log", false)
|
||||
fh.AssertFileContent(t, "subdir/foo", "bar")
|
||||
fh.AssertSymLink(t, "symlink.go", "test.go")
|
||||
fh.AssertExists(t, "symlink.log", false)
|
||||
fh.AssertExists(t, "symlink-dir.log", false)
|
||||
}
|
||||
|
||||
func TestTransformers(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user