From 61ad4c607b09f102151298af98757dbae2c2fa88 Mon Sep 17 00:00:00 2001 From: FuXiaoHei Date: Sun, 14 May 2023 06:33:25 +0800 Subject: [PATCH] fix minio storage iterator path (#24691) minio storage iterator shows different behavior with local fs iterator. in local fs storage: ``` go s.IterateObjects("prefix", func(path,obj) println(path) // show "prefix/xxx.file" }) ``` in minio storage: ```go s.IterateObjects("prefix", func(path,obj) println(path) // show "xxx.file" }) ``` I think local fs is correct, minio use wrong `basePath` to trim storage path prefix. --------- Co-authored-by: Giteabot --- .github/workflows/pull-db-tests.yml | 7 +++++ modules/storage/local.go | 4 +-- modules/storage/local_test.go | 37 +--------------------- modules/storage/minio.go | 15 ++++++--- modules/storage/minio_test.go | 18 +++++++++++ modules/storage/storage_test.go | 49 +++++++++++++++++++++++++++++ 6 files changed, 87 insertions(+), 43 deletions(-) create mode 100644 modules/storage/minio_test.go create mode 100644 modules/storage/storage_test.go diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 4011b4201b..3446b71155 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -102,6 +102,13 @@ jobs: --health-retries 10 ports: - 6379:6379 + minio: + image: bitnami/minio:2021.3.17 + env: + MINIO_ACCESS_KEY: 123456 + MINIO_SECRET_KEY: 12345678 + ports: + - "9000:9000" steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 diff --git a/modules/storage/local.go b/modules/storage/local.go index d22974a65a..73ef306979 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -133,8 +133,8 @@ func (l *LocalStorage) URL(path, name string) (*url.URL, error) { } // IterateObjects iterates across the objects in the local storage -func (l *LocalStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error { - dir := l.buildLocalPath(prefix) +func (l *LocalStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error { + dir := l.buildLocalPath(dirName) return filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { if err != nil { return err diff --git a/modules/storage/local_test.go b/modules/storage/local_test.go index 0873f8e14e..1c4b856ab6 100644 --- a/modules/storage/local_test.go +++ b/modules/storage/local_test.go @@ -4,8 +4,6 @@ package storage import ( - "bytes" - "context" "os" "path/filepath" "testing" @@ -57,38 +55,5 @@ func TestBuildLocalPath(t *testing.T) { func TestLocalStorageIterator(t *testing.T) { dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir") - l, err := NewLocalStorage(context.Background(), LocalStorageConfig{Path: dir}) - assert.NoError(t, err) - - testFiles := [][]string{ - {"a/1.txt", "a1"}, - {"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim - {"b/1.txt", "b1"}, - {"b/2.txt", "b2"}, - {"b/3.txt", "b3"}, - {"b/x 4.txt", "bx4"}, - } - for _, f := range testFiles { - _, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1) - assert.NoError(t, err) - } - - expectedList := map[string][]string{ - "a": {"a/1.txt"}, - "b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, - "": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, - "/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, - "a/b/../../a": {"a/1.txt"}, - } - for dir, expected := range expectedList { - count := 0 - err = l.IterateObjects(dir, func(path string, f Object) error { - defer f.Close() - assert.Contains(t, expected, path) - count++ - return nil - }) - assert.NoError(t, err) - assert.Len(t, expected, count) - } + testStorageIterator(t, string(LocalStorageType), LocalStorageConfig{Path: dir}) } diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 250f17827f..c78f351e9c 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -129,7 +129,11 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error } func (m *MinioStorage) buildMinioPath(p string) string { - return util.PathJoinRelX(m.basePath, p) + p = util.PathJoinRelX(m.basePath, p) + if p == "." { + p = "" // minio doesn't use dot as relative path + } + return p } // Open opens a file @@ -224,14 +228,15 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) { } // IterateObjects iterates across the objects in the miniostorage -func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error { +func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error { opts := minio.GetObjectOptions{} lobjectCtx, cancel := context.WithCancel(m.ctx) defer cancel() basePath := m.basePath - if prefix != "" { - basePath = m.buildMinioPath(prefix) + if dirName != "" { + // ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo" + basePath = m.buildMinioPath(dirName) + "/" } for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{ @@ -244,7 +249,7 @@ func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Ob } if err := func(object *minio.Object, fn func(path string, obj Object) error) error { defer object.Close() - return fn(strings.TrimPrefix(mObjInfo.Key, basePath), &minioObject{object}) + return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object}) }(object, fn); err != nil { return convertMinioErr(err) } diff --git a/modules/storage/minio_test.go b/modules/storage/minio_test.go new file mode 100644 index 0000000000..bee1b86318 --- /dev/null +++ b/modules/storage/minio_test.go @@ -0,0 +1,18 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "testing" +) + +func TestMinioStorageIterator(t *testing.T) { + testStorageIterator(t, string(MinioStorageType), MinioStorageConfig{ + Endpoint: "127.0.0.1:9000", + AccessKeyID: "123456", + SecretAccessKey: "12345678", + Bucket: "gitea", + Location: "us-east-1", + }) +} diff --git a/modules/storage/storage_test.go b/modules/storage/storage_test.go new file mode 100644 index 0000000000..b517a9e71a --- /dev/null +++ b/modules/storage/storage_test.go @@ -0,0 +1,49 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package storage + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func testStorageIterator(t *testing.T, typStr string, cfg interface{}) { + l, err := NewStorage(typStr, cfg) + assert.NoError(t, err) + + testFiles := [][]string{ + {"a/1.txt", "a1"}, + {"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim + {"ab/1.txt", "ab1"}, + {"b/1.txt", "b1"}, + {"b/2.txt", "b2"}, + {"b/3.txt", "b3"}, + {"b/x 4.txt", "bx4"}, + } + for _, f := range testFiles { + _, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1) + assert.NoError(t, err) + } + + expectedList := map[string][]string{ + "a": {"a/1.txt"}, + "b": {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"}, + "": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, + "/": {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"}, + "a/b/../../a": {"a/1.txt"}, + } + for dir, expected := range expectedList { + count := 0 + err = l.IterateObjects(dir, func(path string, f Object) error { + defer f.Close() + assert.Contains(t, expected, path) + count++ + return nil + }) + assert.NoError(t, err) + assert.Len(t, expected, count) + } +}