From 7ff783b7320bdeda89ed6a021c304e64f9d9170e Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 1 Oct 2019 21:40:17 +0800 Subject: [PATCH] Move mirror to a standalone package from models (#7486) * move mirror to a standalone package * fix mirror address in template * fix tests * fix lint * fix comment * fix tests * fix tests * fix vendor * fix fmt * fix lint * remove wrong file submitted * fix conflict * remove unrelated changes * fix go mod * fix tests * clean go mod * make vendor work * make vendor work * fix tests * remove duplicated test --- go.sum | 6 + models/repo.go | 6 + models/repo_mirror.go | 395 +-------------------------- modules/cron/cron.go | 5 +- modules/templates/helper.go | 3 + routers/api/v1/repo/repo.go | 4 +- routers/init.go | 3 +- routers/repo/setting.go | 6 +- services/mirror/mirror.go | 395 +++++++++++++++++++++++++++ services/mirror/mirror_test.go | 78 ++++++ services/release/release_test.go | 59 ---- templates/repo/header.tmpl | 2 +- templates/repo/settings/options.tmpl | 2 +- 13 files changed, 507 insertions(+), 457 deletions(-) create mode 100644 services/mirror/mirror.go create mode 100644 services/mirror/mirror_test.go diff --git a/go.sum b/go.sum index 87446bd56b..477199616c 100644 --- a/go.sum +++ b/go.sum @@ -302,6 +302,7 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -447,9 +448,11 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/oliamb/cutter v0.2.2 h1:Lfwkya0HHNU1YLnGv2hTkzHfasrSMkgv4Dn+5rmlk3k= github.com/oliamb/cutter v0.2.2/go.mod h1:4BenG2/4GuRBDbVm/OPahDVqbrOemzpPiG5mi1iryBU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -527,6 +530,7 @@ github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -560,6 +564,7 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/syndtr/goleveldb v0.0.0-20190203031304-2f17a3356c66 h1:AwmkkZT+TucFotNCL+aNJ/0KCMsRtlXN9fs8uoOMSRk= github.com/syndtr/goleveldb v0.0.0-20190203031304-2f17a3356c66/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -745,6 +750,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.0 h1:Tfd7cKwKbFRsI8RMAD3oqqw7JPFRrvFlOsfbgVkjOOw= google.golang.org/appengine v1.6.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= diff --git a/models/repo.go b/models/repo.go index ffd1c30606..eb7f286fec 100644 --- a/models/repo.go +++ b/models/repo.go @@ -1720,6 +1720,12 @@ func UpdateRepository(repo *Repository, visibilityChanged bool) (err error) { return sess.Commit() } +// UpdateRepositoryUpdatedTime updates a repository's updated time +func UpdateRepositoryUpdatedTime(repoID int64, updateTime time.Time) error { + _, err := x.Exec("UPDATE repository SET updated_unix = ? WHERE id = ?", updateTime.Unix(), repoID) + return err +} + // UpdateRepositoryUnits updates a repository's units func UpdateRepositoryUnits(repo *Repository, units []RepoUnit) (err error) { sess := x.NewSession() diff --git a/models/repo_mirror.go b/models/repo_mirror.go index e1c1aefc39..4e91ea296a 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -6,28 +6,14 @@ package models import ( - "fmt" - "net/url" - "strings" "time" - "code.gitea.io/gitea/modules/cache" - "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/sync" "code.gitea.io/gitea/modules/timeutil" - "code.gitea.io/gitea/modules/util" "github.com/go-xorm/xorm" - "github.com/mcuadros/go-version" - "github.com/unknwon/com" ) -// MirrorQueue holds an UniqueQueue object of the mirror -var MirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength) - // Mirror represents mirror information of a repository. type Mirror struct { ID int64 `xorm:"pk autoincr"` @@ -39,7 +25,7 @@ type Mirror struct { UpdatedUnix timeutil.TimeStamp `xorm:"INDEX"` NextUpdateUnix timeutil.TimeStamp `xorm:"INDEX"` - address string `xorm:"-"` + Address string `xorm:"-"` } // BeforeInsert will be invoked by XORM before inserting a record @@ -72,258 +58,6 @@ func (m *Mirror) ScheduleNextUpdate() { } } -func remoteAddress(repoPath string) (string, error) { - var cmd *git.Command - binVersion, err := git.BinVersion() - if err != nil { - return "", err - } - if version.Compare(binVersion, "2.7", ">=") { - cmd = git.NewCommand("remote", "get-url", "origin") - } else { - cmd = git.NewCommand("config", "--get", "remote.origin.url") - } - - result, err := cmd.RunInDir(repoPath) - if err != nil { - if strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return "", nil - } - return "", err - } - if len(result) > 0 { - return result[:len(result)-1], nil - } - return "", nil -} - -func (m *Mirror) readAddress() { - if len(m.address) > 0 { - return - } - var err error - m.address, err = remoteAddress(m.Repo.RepoPath()) - if err != nil { - log.Error("remoteAddress: %v", err) - } -} - -// sanitizeOutput sanitizes output of a command, replacing occurrences of the -// repository's remote address with a sanitized version. -func sanitizeOutput(output, repoPath string) (string, error) { - remoteAddr, err := remoteAddress(repoPath) - if err != nil { - // if we're unable to load the remote address, then we're unable to - // sanitize. - return "", err - } - return util.SanitizeMessage(output, remoteAddr), nil -} - -// Address returns mirror address from Git repository config with credentials censored. -func (m *Mirror) Address() string { - m.readAddress() - return util.SanitizeURLCredentials(m.address, false) -} - -// FullAddress returns mirror address from Git repository config. -func (m *Mirror) FullAddress() string { - m.readAddress() - return m.address -} - -// AddressNoCredentials returns mirror address from Git repository config without credentials. -func (m *Mirror) AddressNoCredentials() string { - m.readAddress() - u, err := url.Parse(m.address) - if err != nil { - // this shouldn't happen but just return it unsanitised - return m.address - } - u.User = nil - return u.String() -} - -// Username returns the mirror address username -func (m *Mirror) Username() string { - m.readAddress() - u, err := url.Parse(m.address) - if err != nil { - // this shouldn't happen but if it does return "" - return "" - } - return u.User.Username() -} - -// Password returns the mirror address password -func (m *Mirror) Password() string { - m.readAddress() - u, err := url.Parse(m.address) - if err != nil { - // this shouldn't happen but if it does return "" - return "" - } - password, _ := u.User.Password() - return password -} - -// SaveAddress writes new address to Git repository config. -func (m *Mirror) SaveAddress(addr string) error { - repoPath := m.Repo.RepoPath() - // Remove old origin - _, err := git.NewCommand("remote", "remove", "origin").RunInDir(repoPath) - if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { - return err - } - - _, err = git.NewCommand("remote", "add", "origin", "--mirror=fetch", addr).RunInDir(repoPath) - return err -} - -// gitShortEmptySha Git short empty SHA -const gitShortEmptySha = "0000000" - -// mirrorSyncResult contains information of a updated reference. -// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. -// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. -type mirrorSyncResult struct { - refName string - oldCommitID string - newCommitID string -} - -// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream. -func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { - results := make([]*mirrorSyncResult, 0, 3) - lines := strings.Split(output, "\n") - for i := range lines { - // Make sure reference name is presented before continue - idx := strings.Index(lines[i], "-> ") - if idx == -1 { - continue - } - - refName := lines[i][idx+3:] - - switch { - case strings.HasPrefix(lines[i], " * "): // New reference - results = append(results, &mirrorSyncResult{ - refName: refName, - oldCommitID: gitShortEmptySha, - }) - case strings.HasPrefix(lines[i], " - "): // Delete reference - results = append(results, &mirrorSyncResult{ - refName: refName, - newCommitID: gitShortEmptySha, - }) - case strings.HasPrefix(lines[i], " "): // New commits of a reference - delimIdx := strings.Index(lines[i][3:], " ") - if delimIdx == -1 { - log.Error("SHA delimiter not found: %q", lines[i]) - continue - } - shas := strings.Split(lines[i][3:delimIdx+3], "..") - if len(shas) != 2 { - log.Error("Expect two SHAs but not what found: %q", lines[i]) - continue - } - results = append(results, &mirrorSyncResult{ - refName: refName, - oldCommitID: shas[0], - newCommitID: shas[1], - }) - - default: - log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i]) - } - } - return results -} - -// runSync returns true if sync finished without error. -func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) { - repoPath := m.Repo.RepoPath() - wikiPath := m.Repo.WikiPath() - timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second - - gitArgs := []string{"remote", "update"} - if m.EnablePrune { - gitArgs = append(gitArgs, "--prune") - } - - _, stderr, err := process.GetManager().ExecDir( - timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath), - git.GitExecutable, gitArgs...) - if err != nil { - // sanitize the output, since it may contain the remote address, which may - // contain a password - message, err := sanitizeOutput(stderr, repoPath) - if err != nil { - log.Error("sanitizeOutput: %v", err) - return nil, false - } - desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message) - log.Error(desc) - if err = CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return nil, false - } - output := stderr - - gitRepo, err := git.OpenRepository(repoPath) - if err != nil { - log.Error("OpenRepository: %v", err) - return nil, false - } - if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil { - log.Error("Failed to synchronize tags to releases for repository: %v", err) - } - - if err := m.Repo.UpdateSize(); err != nil { - log.Error("Failed to update size for mirror repository: %v", err) - } - - if m.Repo.HasWiki() { - if _, stderr, err := process.GetManager().ExecDir( - timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath), - git.GitExecutable, "remote", "update", "--prune"); err != nil { - // sanitize the output, since it may contain the remote address, which may - // contain a password - message, err := sanitizeOutput(stderr, wikiPath) - if err != nil { - log.Error("sanitizeOutput: %v", err) - return nil, false - } - desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message) - log.Error(desc) - if err = CreateRepositoryNotice(desc); err != nil { - log.Error("CreateRepositoryNotice: %v", err) - } - return nil, false - } - } - - branches, err := m.Repo.GetBranches() - if err != nil { - log.Error("GetBranches: %v", err) - return nil, false - } - - for i := range branches { - cache.Remove(m.Repo.GetCommitsCountCacheKey(branches[i].Name, true)) - } - - m.UpdatedUnix = timeutil.TimeStampNow() - return parseRemoteUpdateOutput(output), true -} - -// RunMirrorSync will invoke Mirror's runSync -func RunMirrorSync(mirror *Mirror) bool { - _, ok := mirror.runSync() - return ok -} - func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) { m := &Mirror{RepoID: repoID} has, err := e.Get(m) @@ -356,129 +90,10 @@ func DeleteMirrorByRepoID(repoID int64) error { return err } -// MirrorUpdate checks and updates mirror repositories. -func MirrorUpdate() { - log.Trace("Doing: MirrorUpdate") - - if err := x. +// MirrorsIterate iterates all mirror repositories. +func MirrorsIterate(f func(idx int, bean interface{}) error) error { + return x. Where("next_update_unix<=?", time.Now().Unix()). And("next_update_unix!=0"). - Iterate(new(Mirror), func(idx int, bean interface{}) error { - m := bean.(*Mirror) - if m.Repo == nil { - log.Error("Disconnected mirror repository found: %d", m.ID) - return nil - } - - MirrorQueue.Add(m.RepoID) - return nil - }); err != nil { - log.Error("MirrorUpdate: %v", err) - } -} - -// SyncMirrors checks and syncs mirrors. -// TODO: sync more mirrors at same time. -func SyncMirrors() { - sess := x.NewSession() - defer sess.Close() - // Start listening on new sync requests. - for repoID := range MirrorQueue.Queue() { - log.Trace("SyncMirrors [repo_id: %v]", repoID) - MirrorQueue.Remove(repoID) - - m, err := GetMirrorByRepoID(com.StrTo(repoID).MustInt64()) - if err != nil { - log.Error("GetMirrorByRepoID [%s]: %v", repoID, err) - continue - } - - results, ok := m.runSync() - if !ok { - continue - } - - m.ScheduleNextUpdate() - if err = updateMirror(sess, m); err != nil { - log.Error("UpdateMirror [%s]: %v", repoID, err) - continue - } - - var gitRepo *git.Repository - if len(results) == 0 { - log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID) - } else { - gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) - if err != nil { - log.Error("OpenRepository [%d]: %v", m.RepoID, err) - continue - } - } - - for _, result := range results { - // Discard GitHub pull requests, i.e. refs/pull/* - if strings.HasPrefix(result.refName, "refs/pull/") { - continue - } - - // Create reference - if result.oldCommitID == gitShortEmptySha { - if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil { - log.Error("MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err) - } - continue - } - - // Delete reference - if result.newCommitID == gitShortEmptySha { - if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil { - log.Error("MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err) - } - continue - } - - // Push commits - oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID) - if err != nil { - log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) - continue - } - newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID) - if err != nil { - log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) - continue - } - commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) - if err != nil { - log.Error("CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err) - continue - } - if err = MirrorSyncPushAction(m.Repo, MirrorSyncPushActionOptions{ - RefName: result.refName, - OldCommitID: oldCommitID, - NewCommitID: newCommitID, - Commits: ListToPushCommits(commits), - }); err != nil { - log.Error("MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err) - continue - } - } - - // Get latest commit date and update to current repository updated time - commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) - if err != nil { - log.Error("GetLatestCommitDate [%d]: %v", m.RepoID, err) - continue - } - - if _, err = sess.Exec("UPDATE repository SET updated_unix = ? WHERE id = ?", commitDate.Unix(), m.RepoID); err != nil { - log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err) - continue - } - } -} - -// InitSyncMirrors initializes a go routine to sync the mirrors -func InitSyncMirrors() { - go SyncMirrors() + Iterate(new(Mirror), f) } diff --git a/modules/cron/cron.go b/modules/cron/cron.go index fe87f86e02..089f0fa767 100644 --- a/modules/cron/cron.go +++ b/modules/cron/cron.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/sync" + mirror_service "code.gitea.io/gitea/services/mirror" "github.com/gogs/cron" ) @@ -51,14 +52,14 @@ func NewContext() { err error ) if setting.Cron.UpdateMirror.Enabled { - entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, WithUnique(mirrorUpdate, models.MirrorUpdate)) + entry, err = c.AddFunc("Update mirrors", setting.Cron.UpdateMirror.Schedule, WithUnique(mirrorUpdate, mirror_service.Update)) if err != nil { log.Fatal("Cron[Update mirrors]: %v", err) } if setting.Cron.UpdateMirror.RunAtStart { entry.Prev = time.Now() entry.ExecTimes++ - go WithUnique(mirrorUpdate, models.MirrorUpdate)() + go WithUnique(mirrorUpdate, mirror_service.Update)() } } if setting.Cron.RepoHealthCheck.Enabled { diff --git a/modules/templates/helper.go b/modules/templates/helper.go index ba61dd5eef..b40f7117f5 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -28,6 +28,7 @@ import ( "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/gitdiff" + mirror_service "code.gitea.io/gitea/services/mirror" "gopkg.in/editorconfig/editorconfig-core-go.v1" ) @@ -233,6 +234,8 @@ func NewFuncMap() []template.FuncMap { return float32(n) * 100 / float32(sum) }, "CommentMustAsDiff": gitdiff.CommentMustAsDiff, + "MirrorAddress": mirror_service.Address, + "MirrorFullAddress": mirror_service.AddressNoCredentials, }} } diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index c838ba7271..513e7a37b3 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -20,6 +20,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers/api/v1/convert" + mirror_service "code.gitea.io/gitea/services/mirror" ) var searchOrderByMap = map[string]map[string]models.SearchOrderBy{ @@ -869,6 +870,7 @@ func MirrorSync(ctx *context.APIContext) { ctx.Error(403, "MirrorSync", "Must have write access") } - go models.MirrorQueue.Add(repo.ID) + mirror_service.StartToMirror(repo.ID) + ctx.Status(200) } diff --git a/routers/init.go b/routers/init.go index 5c8741f244..1efddcfaa6 100644 --- a/routers/init.go +++ b/routers/init.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" "code.gitea.io/gitea/services/mailer" + mirror_service "code.gitea.io/gitea/services/mirror" "gitea.com/macaron/macaron" ) @@ -98,7 +99,7 @@ func GlobalInit() { log.Fatal("Failed to initialize issue indexer: %v", err) } models.InitRepoIndexer() - models.InitSyncMirrors() + mirror_service.InitSyncMirrors() models.InitDeliverHooks() models.InitTestPullRequests() } diff --git a/routers/repo/setting.go b/routers/repo/setting.go index b0af396e7d..c74b273420 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -25,6 +25,7 @@ import ( "code.gitea.io/gitea/modules/validation" "code.gitea.io/gitea/routers/utils" "code.gitea.io/gitea/services/mailer" + mirror_service "code.gitea.io/gitea/services/mirror" "github.com/unknwon/com" "mvdan.cc/xurls/v2" @@ -190,7 +191,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { address = u.String() - if err := ctx.Repo.Mirror.SaveAddress(address); err != nil { + if err := mirror_service.SaveAddress(ctx.Repo.Mirror, address); err != nil { ctx.ServerError("SaveAddress", err) return } @@ -204,7 +205,8 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { return } - go models.MirrorQueue.Add(repo.ID) + mirror_service.StartToMirror(repo.ID) + ctx.Flash.Info(ctx.Tr("repo.settings.mirror_sync_in_progress")) ctx.Redirect(repo.Link() + "/settings") diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go new file mode 100644 index 0000000000..3339f72329 --- /dev/null +++ b/services/mirror/mirror.go @@ -0,0 +1,395 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package mirror + +import ( + "fmt" + "net/url" + "strings" + "time" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/sync" + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/util" + + "github.com/mcuadros/go-version" + "github.com/unknwon/com" +) + +// mirrorQueue holds an UniqueQueue object of the mirror +var mirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength) + +func readAddress(m *models.Mirror) { + if len(m.Address) > 0 { + return + } + var err error + m.Address, err = remoteAddress(m.Repo.RepoPath()) + if err != nil { + log.Error("remoteAddress: %v", err) + } +} + +func remoteAddress(repoPath string) (string, error) { + var cmd *git.Command + binVersion, err := git.BinVersion() + if err != nil { + return "", err + } + if version.Compare(binVersion, "2.7", ">=") { + cmd = git.NewCommand("remote", "get-url", "origin") + } else { + cmd = git.NewCommand("config", "--get", "remote.origin.url") + } + + result, err := cmd.RunInDir(repoPath) + if err != nil { + if strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return "", nil + } + return "", err + } + if len(result) > 0 { + return result[:len(result)-1], nil + } + return "", nil +} + +// sanitizeOutput sanitizes output of a command, replacing occurrences of the +// repository's remote address with a sanitized version. +func sanitizeOutput(output, repoPath string) (string, error) { + remoteAddr, err := remoteAddress(repoPath) + if err != nil { + // if we're unable to load the remote address, then we're unable to + // sanitize. + return "", err + } + return util.SanitizeMessage(output, remoteAddr), nil +} + +// AddressNoCredentials returns mirror address from Git repository config without credentials. +func AddressNoCredentials(m *models.Mirror) string { + readAddress(m) + u, err := url.Parse(m.Address) + if err != nil { + // this shouldn't happen but just return it unsanitised + return m.Address + } + u.User = nil + return u.String() +} + +// SaveAddress writes new address to Git repository config. +func SaveAddress(m *models.Mirror, addr string) error { + repoPath := m.Repo.RepoPath() + // Remove old origin + _, err := git.NewCommand("remote", "remove", "origin").RunInDir(repoPath) + if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") { + return err + } + + _, err = git.NewCommand("remote", "add", "origin", "--mirror=fetch", addr).RunInDir(repoPath) + return err +} + +// gitShortEmptySha Git short empty SHA +const gitShortEmptySha = "0000000" + +// mirrorSyncResult contains information of a updated reference. +// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. +// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. +type mirrorSyncResult struct { + refName string + oldCommitID string + newCommitID string +} + +// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream. +func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { + results := make([]*mirrorSyncResult, 0, 3) + lines := strings.Split(output, "\n") + for i := range lines { + // Make sure reference name is presented before continue + idx := strings.Index(lines[i], "-> ") + if idx == -1 { + continue + } + + refName := lines[i][idx+3:] + + switch { + case strings.HasPrefix(lines[i], " * "): // New reference + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " - "): // Delete reference + results = append(results, &mirrorSyncResult{ + refName: refName, + newCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " "): // New commits of a reference + delimIdx := strings.Index(lines[i][3:], " ") + if delimIdx == -1 { + log.Error("SHA delimiter not found: %q", lines[i]) + continue + } + shas := strings.Split(lines[i][3:delimIdx+3], "..") + if len(shas) != 2 { + log.Error("Expect two SHAs but not what found: %q", lines[i]) + continue + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: shas[0], + newCommitID: shas[1], + }) + + default: + log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i]) + } + } + return results +} + +// runSync returns true if sync finished without error. +func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) { + repoPath := m.Repo.RepoPath() + wikiPath := m.Repo.WikiPath() + timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second + + gitArgs := []string{"remote", "update"} + if m.EnablePrune { + gitArgs = append(gitArgs, "--prune") + } + + _, stderr, err := process.GetManager().ExecDir( + timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath), + git.GitExecutable, gitArgs...) + if err != nil { + // sanitize the output, since it may contain the remote address, which may + // contain a password + message, err := sanitizeOutput(stderr, repoPath) + if err != nil { + log.Error("sanitizeOutput: %v", err) + return nil, false + } + desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message) + log.Error(desc) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return nil, false + } + output := stderr + + gitRepo, err := git.OpenRepository(repoPath) + if err != nil { + log.Error("OpenRepository: %v", err) + return nil, false + } + if err = models.SyncReleasesWithTags(m.Repo, gitRepo); err != nil { + log.Error("Failed to synchronize tags to releases for repository: %v", err) + } + + if err := m.Repo.UpdateSize(); err != nil { + log.Error("Failed to update size for mirror repository: %v", err) + } + + if m.Repo.HasWiki() { + if _, stderr, err := process.GetManager().ExecDir( + timeout, wikiPath, fmt.Sprintf("Mirror.runSync: %s", wikiPath), + git.GitExecutable, "remote", "update", "--prune"); err != nil { + // sanitize the output, since it may contain the remote address, which may + // contain a password + message, err := sanitizeOutput(stderr, wikiPath) + if err != nil { + log.Error("sanitizeOutput: %v", err) + return nil, false + } + desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message) + log.Error(desc) + if err = models.CreateRepositoryNotice(desc); err != nil { + log.Error("CreateRepositoryNotice: %v", err) + } + return nil, false + } + } + + branches, err := m.Repo.GetBranches() + if err != nil { + log.Error("GetBranches: %v", err) + return nil, false + } + + for i := range branches { + cache.Remove(m.Repo.GetCommitsCountCacheKey(branches[i].Name, true)) + } + + m.UpdatedUnix = timeutil.TimeStampNow() + return parseRemoteUpdateOutput(output), true +} + +// Address returns mirror address from Git repository config without credentials. +func Address(m *models.Mirror) string { + readAddress(m) + return util.SanitizeURLCredentials(m.Address, false) +} + +// Username returns the mirror address username +func Username(m *models.Mirror) string { + readAddress(m) + u, err := url.Parse(m.Address) + if err != nil { + // this shouldn't happen but if it does return "" + return "" + } + return u.User.Username() +} + +// Password returns the mirror address password +func Password(m *models.Mirror) string { + readAddress(m) + u, err := url.Parse(m.Address) + if err != nil { + // this shouldn't happen but if it does return "" + return "" + } + password, _ := u.User.Password() + return password +} + +// Update checks and updates mirror repositories. +func Update() { + log.Trace("Doing: Update") + + if err := models.MirrorsIterate(func(idx int, bean interface{}) error { + m := bean.(*models.Mirror) + if m.Repo == nil { + log.Error("Disconnected mirror repository found: %d", m.ID) + return nil + } + + mirrorQueue.Add(m.RepoID) + return nil + }); err != nil { + log.Error("Update: %v", err) + } +} + +// SyncMirrors checks and syncs mirrors. +// TODO: sync more mirrors at same time. +func SyncMirrors() { + // Start listening on new sync requests. + for repoID := range mirrorQueue.Queue() { + log.Trace("SyncMirrors [repo_id: %v]", repoID) + mirrorQueue.Remove(repoID) + + m, err := models.GetMirrorByRepoID(com.StrTo(repoID).MustInt64()) + if err != nil { + log.Error("GetMirrorByRepoID [%s]: %v", repoID, err) + continue + } + + results, ok := runSync(m) + if !ok { + continue + } + + m.ScheduleNextUpdate() + if err = models.UpdateMirror(m); err != nil { + log.Error("UpdateMirror [%s]: %v", repoID, err) + continue + } + + var gitRepo *git.Repository + if len(results) == 0 { + log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID) + } else { + gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) + if err != nil { + log.Error("OpenRepository [%d]: %v", m.RepoID, err) + continue + } + } + + for _, result := range results { + // Discard GitHub pull requests, i.e. refs/pull/* + if strings.HasPrefix(result.refName, "refs/pull/") { + continue + } + + // Create reference + if result.oldCommitID == gitShortEmptySha { + if err = models.MirrorSyncCreateAction(m.Repo, result.refName); err != nil { + log.Error("MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err) + } + continue + } + + // Delete reference + if result.newCommitID == gitShortEmptySha { + if err = models.MirrorSyncDeleteAction(m.Repo, result.refName); err != nil { + log.Error("MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err) + } + continue + } + + // Push commits + oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID) + if err != nil { + log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID) + if err != nil { + log.Error("GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) + if err != nil { + log.Error("CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err) + continue + } + if err = models.MirrorSyncPushAction(m.Repo, models.MirrorSyncPushActionOptions{ + RefName: result.refName, + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + Commits: models.ListToPushCommits(commits), + }); err != nil { + log.Error("MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err) + continue + } + } + + // Get latest commit date and update to current repository updated time + commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) + if err != nil { + log.Error("GetLatestCommitDate [%d]: %v", m.RepoID, err) + continue + } + + if err = models.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil { + log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err) + continue + } + } +} + +// InitSyncMirrors initializes a go routine to sync the mirrors +func InitSyncMirrors() { + go SyncMirrors() +} + +// StartToMirror adds repoID to mirror queue +func StartToMirror(repoID int64) { + go mirrorQueue.Add(repoID) +} diff --git a/services/mirror/mirror_test.go b/services/mirror/mirror_test.go new file mode 100644 index 0000000000..76bd4c72f7 --- /dev/null +++ b/services/mirror/mirror_test.go @@ -0,0 +1,78 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package mirror + +import ( + "path/filepath" + "testing" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/git" + release_service "code.gitea.io/gitea/services/release" + + "github.com/stretchr/testify/assert" +) + +func TestMain(m *testing.M) { + models.MainTest(m, filepath.Join("..", "..")) +} + +func TestRelease_MirrorDelete(t *testing.T) { + assert.NoError(t, models.PrepareTestDatabase()) + + user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) + repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) + repoPath := models.RepoPath(user.Name, repo.Name) + + migrationOptions := models.MigrateRepoOptions{ + Name: "test_mirror", + Description: "Test mirror", + IsPrivate: false, + IsMirror: true, + RemoteAddr: repoPath, + Wiki: true, + SyncReleasesWithTags: true, + } + mirror, err := models.MigrateRepository(user, user, migrationOptions) + assert.NoError(t, err) + + gitRepo, err := git.OpenRepository(repoPath) + assert.NoError(t, err) + + findOptions := models.FindReleasesOptions{IncludeDrafts: true, IncludeTags: true} + initCount, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) + assert.NoError(t, err) + + assert.NoError(t, release_service.CreateRelease(gitRepo, &models.Release{ + RepoID: repo.ID, + PublisherID: user.ID, + TagName: "v0.2", + Target: "master", + Title: "v0.2 is released", + Note: "v0.2 is released", + IsDraft: false, + IsPrerelease: false, + IsTag: true, + }, nil)) + + err = mirror.GetMirror() + assert.NoError(t, err) + + _, ok := runSync(mirror.Mirror) + assert.True(t, ok) + + count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) + assert.EqualValues(t, initCount+1, count) + + release, err := models.GetRelease(repo.ID, "v0.2") + assert.NoError(t, err) + assert.NoError(t, release_service.DeleteReleaseByID(release.ID, user, true)) + + _, ok = runSync(mirror.Mirror) + assert.True(t, ok) + + count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions) + assert.EqualValues(t, initCount, count) +} diff --git a/services/release/release_test.go b/services/release/release_test.go index 97febeecb9..d30dfee286 100644 --- a/services/release/release_test.go +++ b/services/release/release_test.go @@ -100,62 +100,3 @@ func TestRelease_Create(t *testing.T) { IsTag: true, }, nil)) } - -func TestRelease_MirrorDelete(t *testing.T) { - assert.NoError(t, models.PrepareTestDatabase()) - - user := models.AssertExistsAndLoadBean(t, &models.User{ID: 2}).(*models.User) - repo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) - repoPath := models.RepoPath(user.Name, repo.Name) - migrationOptions := models.MigrateRepoOptions{ - Name: "test_mirror", - Description: "Test mirror", - IsPrivate: false, - IsMirror: true, - RemoteAddr: repoPath, - Wiki: true, - SyncReleasesWithTags: true, - } - mirror, err := models.MigrateRepository(user, user, migrationOptions) - assert.NoError(t, err) - - gitRepo, err := git.OpenRepository(repoPath) - assert.NoError(t, err) - - findOptions := models.FindReleasesOptions{IncludeDrafts: true, IncludeTags: true} - initCount, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) - assert.NoError(t, err) - - assert.NoError(t, CreateRelease(gitRepo, &models.Release{ - RepoID: repo.ID, - PublisherID: user.ID, - TagName: "v0.2", - Target: "master", - Title: "v0.2 is released", - Note: "v0.2 is released", - IsDraft: false, - IsPrerelease: false, - IsTag: true, - }, nil)) - - err = mirror.GetMirror() - assert.NoError(t, err) - - ok := models.RunMirrorSync(mirror.Mirror) - assert.True(t, ok) - - count, err := models.GetReleaseCountByRepoID(mirror.ID, findOptions) - assert.NoError(t, err) - assert.EqualValues(t, initCount+1, count) - - release, err := models.GetRelease(repo.ID, "v0.2") - assert.NoError(t, err) - assert.NoError(t, DeleteReleaseByID(release.ID, user, true)) - - ok = models.RunMirrorSync(mirror.Mirror) - assert.True(t, ok) - - count, err = models.GetReleaseCountByRepoID(mirror.ID, findOptions) - assert.NoError(t, err) - assert.EqualValues(t, initCount, count) -} diff --git a/templates/repo/header.tmpl b/templates/repo/header.tmpl index e657042b62..fc7f1b660c 100644 --- a/templates/repo/header.tmpl +++ b/templates/repo/header.tmpl @@ -13,7 +13,7 @@ {{.Name}} {{if and .RelAvatarLink .IsPrivate}}{{end}} {{if .IsArchived}}{{end}} - {{if .IsMirror}}
{{$.i18n.Tr "repo.mirror_from"}} {{$.Mirror.Address}}
{{end}} + {{if .IsMirror}}
{{$.i18n.Tr "repo.mirror_from"}} {{MirrorAddress $.Mirror}}
{{end}} {{if .IsFork}}
{{$.i18n.Tr "repo.forked_from"}} {{SubStr .BaseRepo.RelLink 1 -1}}
{{end}}
diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 87e23b6937..4f7d32479e 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -80,7 +80,7 @@
- +

{{.i18n.Tr "repo.mirror_address_desc"}}