From 8220e50b56cf7bf9cdfff29a287c5721c3949464 Mon Sep 17 00:00:00 2001 From: Kyle D Date: Tue, 20 Jun 2023 17:14:47 -0400 Subject: [PATCH] Substitute variables in path names of template repos too (#25294) ### Summary Extend the template variable substitution to replace file paths. This can be helpful for setting up log files & directories that should match the repository name. ### PR Changes - Move files matching glob pattern when setting up repos from template - For security, added ~escaping~ sanitization for cross-platform support and to prevent directory traversal (thanks @silverwind for the reference) - Added unit testing for escaping function - Fixed the integration tests for repo template generation by passing the repo_template_id - Updated the integration testfiles to add some variable substitution & assert the outputs I had to fix the existing repo template integration test and extend it to add a check for variable substitutions. Example: ![image](https://github.com/go-gitea/gitea/assets/12700993/621feb09-0ef3-460e-afa8-da74cd84fa4e) --- .../doc/usage/template-repositories.en-us.md | 2 + modules/repository/generate.go | 32 ++++++++++++++- modules/repository/generate_test.go | 11 +++++ .../2a/83b349fa234131fc5db6f2a0498d3f4d3d6038 | 2 + .../3d/0bc64f2521cfc7ffce6c175c1c846c88eb6df7 | Bin 0 -> 192 bytes .../83/77b2196e99ac8635aae79df3db76959ccd1094 | Bin 0 -> 53 bytes .../99/45b93bcb5b70af06e0322bd2caa6180680991f | Bin 0 -> 28 bytes .../af/f5b10402b4e0479d1e76bc41a42d29fe7f28fa | Bin 0 -> 106 bytes .../b9/04864fd6cd0c8e9054351fd39a980bfd214229 | Bin 0 -> 90 bytes .../c5/10abf4c7c3e0dc4bf07db9344c61c4e6ee7cbc | Bin 0 -> 50 bytes .../e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 | Bin 0 -> 15 bytes .../user27/template1.git/refs/heads/master | 2 +- tests/integration/repo_generate_test.go | 38 ++++++++++++++---- 13 files changed, 77 insertions(+), 10 deletions(-) create mode 100644 tests/gitea-repositories-meta/user27/template1.git/objects/2a/83b349fa234131fc5db6f2a0498d3f4d3d6038 create mode 100644 tests/gitea-repositories-meta/user27/template1.git/objects/3d/0bc64f2521cfc7ffce6c175c1c846c88eb6df7 create mode 100644 tests/gitea-repositories-meta/user27/template1.git/objects/83/77b2196e99ac8635aae79df3db76959ccd1094 create mode 100644 tests/gitea-repositories-meta/user27/template1.git/objects/99/45b93bcb5b70af06e0322bd2caa6180680991f create mode 100644 tests/gitea-repositories-meta/user27/template1.git/objects/af/f5b10402b4e0479d1e76bc41a42d29fe7f28fa create mode 100644 tests/gitea-repositories-meta/user27/template1.git/objects/b9/04864fd6cd0c8e9054351fd39a980bfd214229 create mode 100644 tests/gitea-repositories-meta/user27/template1.git/objects/c5/10abf4c7c3e0dc4bf07db9344c61c4e6ee7cbc create mode 100644 tests/gitea-repositories-meta/user27/template1.git/objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/docs/content/doc/usage/template-repositories.en-us.md b/docs/content/doc/usage/template-repositories.en-us.md index 0c278648b3..5687861b8c 100644 --- a/docs/content/doc/usage/template-repositories.en-us.md +++ b/docs/content/doc/usage/template-repositories.en-us.md @@ -51,6 +51,8 @@ a/b/c/d.json In any file matched by the above globs, certain variables will be expanded. +Matching filenames and paths can also be expanded, and are conservatively sanitized to support cross-platform filesystems. + All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}` | Variable | Expands To | Transformable | diff --git a/modules/repository/generate.go b/modules/repository/generate.go index 31d5ebbb11..102c5af1c9 100644 --- a/modules/repository/generate.go +++ b/modules/repository/generate.go @@ -11,6 +11,7 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" "time" @@ -48,7 +49,7 @@ var defaultTransformers = []transformer{ {Name: "TITLE", Transform: util.ToTitleCase}, } -func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository) string { +func generateExpansion(src string, templateRepo, generateRepo *repo_model.Repository, sanitizeFileName bool) string { expansions := []expansion{ {Name: "REPO_NAME", Value: generateRepo.Name, Transformers: defaultTransformers}, {Name: "TEMPLATE_NAME", Value: templateRepo.Name, Transformers: defaultTransformers}, @@ -74,6 +75,9 @@ func generateExpansion(src string, templateRepo, generateRepo *repo_model.Reposi return os.Expand(src, func(key string) string { if expansion, ok := expansionMap[key]; ok { + if sanitizeFileName { + return fileNameSanitize(expansion) + } return expansion } return key @@ -191,10 +195,24 @@ func generateRepoCommit(ctx context.Context, repo, templateRepo, generateRepo *r } if err := os.WriteFile(path, - []byte(generateExpansion(string(content), templateRepo, generateRepo)), + []byte(generateExpansion(string(content), templateRepo, generateRepo, false)), 0o644); err != nil { return err } + + substPath := filepath.FromSlash(filepath.Join(tmpDirSlash, + generateExpansion(base, templateRepo, generateRepo, true))) + + // Create parent subdirectories if needed or continue silently if it exists + if err := os.MkdirAll(filepath.Dir(substPath), 0o755); err != nil { + return err + } + + // Substitute filename variables + if err := os.Rename(path, substPath); err != nil { + return err + } + break } } @@ -353,3 +371,13 @@ func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templ return generateRepo, nil } + +// Sanitize user input to valid OS filenames +// +// Based on https://github.com/sindresorhus/filename-reserved-regex +// Adds ".." to prevent directory traversal +func fileNameSanitize(s string) string { + re := regexp.MustCompile(`(?i)\.\.|[<>:\"/\\|?*\x{0000}-\x{001F}]|^(con|prn|aux|nul|com\d|lpt\d)$`) + + return strings.TrimSpace(re.ReplaceAllString(s, "_")) +} diff --git a/modules/repository/generate_test.go b/modules/repository/generate_test.go index 1cb9a50f67..b0f97d0ffb 100644 --- a/modules/repository/generate_test.go +++ b/modules/repository/generate_test.go @@ -54,3 +54,14 @@ func TestGiteaTemplate(t *testing.T) { }) } } + +func TestFileNameSanitize(t *testing.T) { + assert.Equal(t, "test_CON", fileNameSanitize("test_CON")) + assert.Equal(t, "test CON", fileNameSanitize("test CON ")) + assert.Equal(t, "__traverse__", fileNameSanitize("../traverse/..")) + assert.Equal(t, "http___localhost_3003_user_test.git", fileNameSanitize("http://localhost:3003/user/test.git")) + assert.Equal(t, "_", fileNameSanitize("CON")) + assert.Equal(t, "_", fileNameSanitize("con")) + assert.Equal(t, "_", fileNameSanitize("\u0000")) + assert.Equal(t, "目标", fileNameSanitize("目标")) +} diff --git a/tests/gitea-repositories-meta/user27/template1.git/objects/2a/83b349fa234131fc5db6f2a0498d3f4d3d6038 b/tests/gitea-repositories-meta/user27/template1.git/objects/2a/83b349fa234131fc5db6f2a0498d3f4d3d6038 new file mode 100644 index 0000000000..ab167ceeaf --- /dev/null +++ b/tests/gitea-repositories-meta/user27/template1.git/objects/2a/83b349fa234131fc5db6f2a0498d3f4d3d6038 @@ -0,0 +1,2 @@ +xAJ0a9\@Ij2Cw"hi޷q~{_ +c)M* rȉSD&M*lpm*5fE_P8DQCɕao?+\>f۸OHH9G"x{w;8 +is09/ IH \ No newline at end of file diff --git a/tests/gitea-repositories-meta/user27/template1.git/objects/3d/0bc64f2521cfc7ffce6c175c1c846c88eb6df7 b/tests/gitea-repositories-meta/user27/template1.git/objects/3d/0bc64f2521cfc7ffce6c175c1c846c88eb6df7 new file mode 100644 index 0000000000000000000000000000000000000000..4912a5a99c66b1a5cd5e2e0f62c85efb62b1301b GIT binary patch literal 192 zcmV;x06+hD0V^p=O;s?ouw*bX00IS->LAwu|9Fp(kbvO$&>){$hNA+jzZ^gO;Ewl) z+MOmoiASEjtJz~{U|?oq0#oeg=<8ammy@5)Fw=FX_37w>^=uD}v@e}nCc)M)QyyZj zUV3IpY9d2(`6kJ{nQPiiS3RHm`F7dVIcEi?Kn)9Wb#(D{)yqv`*vZo7f9)(!--HlT u`OC9raQ{_w(u9~&l3HBCaMx0A^1LOJvsEG$=ggTUuNqQw>NNmC;!MvTR$=b| literal 0 HcmV?d00001 diff --git a/tests/gitea-repositories-meta/user27/template1.git/objects/83/77b2196e99ac8635aae79df3db76959ccd1094 b/tests/gitea-repositories-meta/user27/template1.git/objects/83/77b2196e99ac8635aae79df3db76959ccd1094 new file mode 100644 index 0000000000000000000000000000000000000000..6538644ee823b93faae19d141bb1e708f7b3e297 GIT binary patch literal 53 zcmV-50LuS(0V^p=O;s>9V=y!@Ff%bxC`rvN$Vn_oWmx}pBMZ}(2kvv_%Jw)e(bfD{ Luki~2NDvU>=yDYn literal 0 HcmV?d00001 diff --git a/tests/gitea-repositories-meta/user27/template1.git/objects/99/45b93bcb5b70af06e0322bd2caa6180680991f b/tests/gitea-repositories-meta/user27/template1.git/objects/99/45b93bcb5b70af06e0322bd2caa6180680991f new file mode 100644 index 0000000000000000000000000000000000000000..4af172516fad14786ad551fbdaf843eb8b63399c GIT binary patch literal 28 kcmb6vS3hFa4aausZ?;w%qdANQV7Xc@J%erPRTFN0|})Tb7|>C z=laG*r?|v&DJuk}7UU!*rz)fYrScMUQx$R(OOi7(^U@Wx^m6jkfr>#SUUm8