diff --git a/.golangci.yml b/.golangci.yml index 0248d54494..52f36c5614 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -106,9 +106,9 @@ issues: - gosec - unparam - staticcheck - - path: services/f3/driver/driver.go + - path: services/f3/driver/base.go linters: - - typecheck + - gosimple - path: models/migrations/v linters: - gocyclo diff --git a/cmd/f3.go b/cmd/f3.go index 91163671e3..de58702bcf 100644 --- a/cmd/f3.go +++ b/cmd/f3.go @@ -6,6 +6,7 @@ import ( "context" "fmt" + auth_model "code.gitea.io/gitea/models/auth" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/services/f3/util" @@ -37,6 +38,11 @@ var CmdF3 = cli.Command{ Value: "", Usage: "The name of the repository", }, + cli.StringFlag{ + Name: "authentication-source", + Value: "", + Usage: "The name of the authentication source matching the forge of origin", + }, cli.BoolFlag{ Name: "no-pull-request", Usage: "Do not dump pull requests", @@ -67,6 +73,17 @@ func runF3(ctx *cli.Context) error { return RunF3(stdCtx, ctx) } +func getAuthenticationSource(ctx context.Context, authenticationSource string) (*auth_model.Source, error) { + source, err := auth_model.GetSourceByName(ctx, authenticationSource) + if err != nil { + if auth_model.IsErrSourceNotExist(err) { + return nil, nil + } + return nil, err + } + return source, nil +} + func RunF3(stdCtx context.Context, ctx *cli.Context) error { doer, err := user_model.GetAdminUser() if err != nil { @@ -78,7 +95,17 @@ func RunF3(stdCtx context.Context, ctx *cli.Context) error { features.PullRequests = false } - forgejo := util.ForgejoForgeRoot(features, doer) + var sourceID int64 + sourceName := ctx.String("authentication-source") + source, err := getAuthenticationSource(stdCtx, sourceName) + if err != nil { + return fmt.Errorf("error retrieving the authentication-source %s %v", sourceName, err) + } + if source != nil { + sourceID = source.ID + } + + forgejo := util.ForgejoForgeRoot(features, doer, sourceID) f3 := util.F3ForgeRoot(features, ctx.String("directory")) if ctx.Bool("export") { diff --git a/go.mod b/go.mod index dd44293336..07471edd5a 100644 --- a/go.mod +++ b/go.mod @@ -119,7 +119,7 @@ require ( gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v3 v3.0.1 - lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230601123105-50a6e740ac04 + lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230701182935-ce3394d54c1e mvdan.cc/xurls/v2 v2.4.0 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 xorm.io/builder v0.3.12 diff --git a/go.sum b/go.sum index ccc3ff194d..dd447d554e 100644 --- a/go.sum +++ b/go.sum @@ -1806,8 +1806,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230601123105-50a6e740ac04 h1:JdNHyMEVNixsOvNw3XqrkWi/RqVLN+wjrdeL6NVk2jE= -lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230601123105-50a6e740ac04/go.mod h1:yIlQydnn+pym6OH20iQ7fbe2TjLfnlOTtEOqvjFaC70= +lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230701182935-ce3394d54c1e h1:dcD+UGLSrgeHYyEWZ+1mZvxQ2BXKyEhjhHcB7a6XgeA= +lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230701182935-ce3394d54c1e/go.mod h1:yIlQydnn+pym6OH20iQ7fbe2TjLfnlOTtEOqvjFaC70= lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.33.6/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= diff --git a/services/f3/driver/driver.go b/services/f3/driver/driver.go index 4f58e5bc5a..34088d751d 100644 --- a/services/f3/driver/driver.go +++ b/services/f3/driver/driver.go @@ -18,7 +18,8 @@ import ( type Options struct { gof3.Options - Doer *user_model.User + AuthenticationSource int64 + Doer *user_model.User } type Forgejo struct { @@ -59,6 +60,10 @@ func (o *Forgejo) GetDoer() *user_model.User { return o.options.Doer } +func (o *Forgejo) GetAuthenticationSource() int64 { + return o.options.AuthenticationSource +} + func (o *Forgejo) GetNewMigrationHTTPClient() gof3.NewMigrationHTTPClientFun { return migrations.NewMigrationHTTPClient } diff --git a/services/f3/driver/user.go b/services/f3/driver/user.go index 0d1a7719e4..bef7906ed2 100644 --- a/services/f3/driver/user.go +++ b/services/f3/driver/user.go @@ -6,11 +6,13 @@ import ( "context" "fmt" + auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/util" user_service "code.gitea.io/gitea/services/user" + "lab.forgefriends.org/friendlyforgeformat/gof3/forges/common" "lab.forgefriends.org/friendlyforgeformat/gof3/format" f3_util "lab.forgefriends.org/friendlyforgeformat/gof3/util" ) @@ -46,7 +48,7 @@ func (o *User) IsNil() bool { } func (o *User) Equals(other *User) bool { - return (o.Name == other.Name) + return (o.ID == other.ID) } func (o *User) ToFormatInterface() format.Interface { @@ -79,6 +81,36 @@ type UserProvider struct { BaseProvider } +func getLocalMatchingRemote(ctx context.Context, authenticationSource int64, id string) *user_model.User { + u := &user_model.User{ + LoginName: id, + LoginSource: authenticationSource, + LoginType: auth_model.OAuth2, + Type: user_model.UserTypeIndividual, + } + has, err := db.GetEngine(ctx).Get(u) + if err != nil { + panic(err) + } else if !has { + return nil + } + return u +} + +func (o *UserProvider) GetLocalMatchingRemote(ctx context.Context, format format.Interface, parents ...common.ContainerObjectInterface) (string, bool) { + authenticationSource := o.g.GetAuthenticationSource() + if authenticationSource == 0 { + return "", false + } + user := getLocalMatchingRemote(ctx, authenticationSource, format.GetIDString()) + if user != nil { + o.g.GetLogger().Debug("found existing user %d with a matching authentication source for %s", user.ID, format.GetIDString()) + return fmt.Sprintf("%d", user.ID), true + } + o.g.GetLogger().Debug("no pre-existing local user for %s", format.GetIDString()) + return "", false +} + func (o *UserProvider) ToFormat(ctx context.Context, user *User) *format.User { return user.ToFormat() } @@ -105,19 +137,21 @@ func (o *UserProvider) ProcessObject(ctx context.Context, user *User) { } func (o *UserProvider) Get(ctx context.Context, exemplar *User) *User { + o.g.GetLogger().Debug("%+v", exemplar) var user *user_model.User var err error if exemplar.GetID() > 0 { user, err = user_model.GetUserByID(ctx, exemplar.GetID()) + o.g.GetLogger().Debug("GetUserByID: %+v %v", user, err) } else if exemplar.Name != "" { user, err = user_model.GetUserByName(ctx, exemplar.Name) } else { panic("GetID() == 0 and UserName == \"\"") } - if user_model.IsErrUserNotExist(err) { - return &User{} - } if err != nil { + if user_model.IsErrUserNotExist(err) { + return &User{} + } panic(fmt.Errorf("user %v %w", exemplar, err)) } return UserConverter(user) @@ -127,7 +161,12 @@ func (o *UserProvider) Put(ctx context.Context, user *User) *User { overwriteDefault := &user_model.CreateUserOverwriteOptions{ IsActive: util.OptionalBoolTrue, } - u := user.User + u := user_model.User{ + Name: user.Name, + FullName: user.FullName, + Email: user.Email, + Passwd: user.Passwd, + } err := user_model.CreateUser(&u, overwriteDefault) if err != nil { panic(err) diff --git a/services/f3/util/util.go b/services/f3/util/util.go index b6f3ae3e4a..fd07611615 100644 --- a/services/f3/util/util.go +++ b/services/f3/util/util.go @@ -15,7 +15,9 @@ import ( func ToF3Logger(messenger base.Messenger) gof3.Logger { if messenger == nil { - messenger = func(string, ...interface{}) {} + messenger = func(message string, args ...interface{}) { + log.Info("Message: "+message, args...) + } } return gof3.Logger{ Message: messenger, @@ -29,13 +31,14 @@ func ToF3Logger(messenger base.Messenger) gof3.Logger { } } -func ForgejoForgeRoot(features gof3.Features, doer *user_model.User) *f3_forges.ForgeRoot { +func ForgejoForgeRoot(features gof3.Features, doer *user_model.User, authenticationSource int64) *f3_forges.ForgeRoot { forgeRoot := f3_forges.NewForgeRootFromDriver(&driver.Forgejo{}, &driver.Options{ Options: gof3.Options{ Features: features, Logger: ToF3Logger(nil), }, - Doer: doer, + Doer: doer, + AuthenticationSource: authenticationSource, }) return forgeRoot } diff --git a/tests/integration/f3_test.go b/tests/integration/f3_test.go index 2a179d9d9f..1c3ab2b02b 100644 --- a/tests/integration/f3_test.go +++ b/tests/integration/f3_test.go @@ -12,6 +12,7 @@ import ( auth_model "code.gitea.io/gitea/models/auth" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/services/f3/util" @@ -69,7 +70,7 @@ func TestF3(t *testing.T) { fixture.NewAsset() fixture.NewIssueComment(nil) fixture.NewPullRequestComment() - fixture.NewReview() + // fixture.NewReview() fixture.NewIssueReaction() fixture.NewCommentReaction() @@ -78,7 +79,7 @@ func TestF3(t *testing.T) { // doer, err := user_model.GetAdminUser() assert.NoError(t, err) - forgejoLocal := util.ForgejoForgeRoot(gof3.AllFeatures, doer) + forgejoLocal := util.ForgejoForgeRoot(gof3.AllFeatures, doer, 0) options := f3_common.NewMirrorOptionsRecurse() forgejoLocal.Forge.Mirror(context.Background(), fixture.Forge, options) @@ -117,7 +118,7 @@ func TestF3(t *testing.T) { assert.Contains(t, files, "/release/") assert.Contains(t, files, "/asset/") assert.Contains(t, files, "/comment/") - assert.Contains(t, files, "/review/") + // assert.Contains(t, files, "/review/") assert.Contains(t, files, "/reaction/") // f3_util.Command(context.Background(), "cp", "-a", f3.GetDirectory(), "abc") }) @@ -179,3 +180,91 @@ func TestMaybePromoteF3User(t *testing.T) { assert.Equal(t, userBeforeSignIn.Email, "") assert.Equal(t, userAfterSignIn.Email, gitlabEmail) } + +func TestF3UserMapping(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + AllowLocalNetworks := setting.Migrations.AllowLocalNetworks + setting.F3.Enabled = true + setting.Migrations.AllowLocalNetworks = true + AppVer := setting.AppVer + // Gitea SDK (go-sdk) need to parse the AppVer from server response, so we must set it to a valid version string. + setting.AppVer = "1.16.0" + defer func() { + setting.Migrations.AllowLocalNetworks = AllowLocalNetworks + setting.AppVer = AppVer + }() + + log.Debug("Step 1: create a fixture") + fixtureNewF3Forge := func(t f3_tests.TestingT, user *format.User, tmpDir string) *f3_forges.ForgeRoot { + root := f3_forges.NewForgeRoot(&f3_f3.Options{ + Options: gof3.Options{ + Configuration: gof3.Configuration{ + Directory: tmpDir, + }, + Features: gof3.AllFeatures, + Logger: util.ToF3Logger(nil), + }, + Remap: true, + }) + return root + } + fixture := f3_forges.NewFixture(t, f3_forges.FixtureForgeFactory{Fun: fixtureNewF3Forge, AdminRequired: false}) + userID := int64(5432) + fixture.NewUser(userID) + // fixture.NewProject() + + log.Debug("Step 2: mirror the fixture into Forgejo") + // + // OAuth2 authentication source GitLab + // + gitlabName := "gitlab" + gitlab := addAuthSource(t, authSourcePayloadGitLabCustom(gitlabName)) + // + // Create a user as if it had been previously been created by the F3 + // authentication source. + // + gitlabUserID := fmt.Sprintf("%d", userID) + gitlabUser := &user_model.User{ + Name: "gitlabuser", + Email: "gitlabuser@example.com", + LoginType: auth_model.OAuth2, + LoginSource: gitlab.ID, + LoginName: gitlabUserID, + } + defer createUser(context.Background(), t, gitlabUser)() + + doer, err := user_model.GetAdminUser() + assert.NoError(t, err) + forgejoLocal := util.ForgejoForgeRoot(gof3.AllFeatures, doer, gitlab.ID) + options := f3_common.NewMirrorOptionsRecurse() + forgejoLocal.Forge.Mirror(context.Background(), fixture.Forge, options) + + log.Debug("Step 3: mirror Forgejo into F3") + adminUsername := "user1" + forgejoAPI := f3_forges.NewForgeRootFromDriver(&f3_forgejo.Forgejo{}, &f3_forgejo.Options{ + Options: gof3.Options{ + Configuration: gof3.Configuration{ + URL: setting.AppURL, + Directory: t.TempDir(), + }, + Features: gof3.AllFeatures, + Logger: util.ToF3Logger(nil), + }, + AuthToken: getUserToken(t, adminUsername, auth_model.AccessTokenScopeWriteAdmin, auth_model.AccessTokenScopeAll), + }) + + f3 := f3_forges.FixtureNewF3Forge(t, nil, t.TempDir()) + apiForge := forgejoAPI.Forge + apiUser := apiForge.Users.GetFromFormat(context.Background(), &format.User{UserName: gitlabUser.Name}) + // apiProject := apiUser.Projects.GetFromFormat(context.Background(), &format.Project{Name: fixture.ProjectFormat.Name}) + // options = f3_common.NewMirrorOptionsRecurse(apiUser, apiProject) + options = f3_common.NewMirrorOptionsRecurse(apiUser) + f3.Forge.Mirror(context.Background(), apiForge, options) + + // + // Step 4: verify the fixture and F3 are equivalent + // + files := f3_util.Command(context.Background(), "find", f3.GetDirectory()) + assert.Contains(t, files, fmt.Sprintf("/user/%d", gitlabUser.ID)) + }) +}