thepaperpilot/forgejo
1
0
Fork 0
Code Issues Pull requests Releases Activity Actions

[F3] replace f3 with forgejo-cli f3

Browse source
This commit is contained in:
Loïc Dachary 2023-07-29 13:37:52 +02:00
parent 9835fbefa2
commit 7ba7ceef1b
No known key found for this signature in database
GPG key ID: 992D23B392F9E4F2
9 changed files with 171 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
cmd/forgejo/actions.go
View file

@ -35,7 +35,8 @@ func SubcmdActionsGenerateRunnerToken(ctx context.Context) *cli.Command {
return &cli.Command{
Name: "generate-runner-token",
Usage: "Generate a new token for a runner to use to register with the server",
Action: prepareWorkPathAndCustomConf(ctx, func(cliCtx *cli.Context) error { return RunGenerateActionsRunnerToken(ctx, cliCtx) }),
Before: prepareWorkPathAndCustomConf(ctx),
Action: func(cliCtx *cli.Context) error { return RunGenerateActionsRunnerToken(ctx, cliCtx) },
Flags: []cli.Flag{
&cli.StringFlag{
Name: "scope",
@ -59,7 +60,8 @@ func SubcmdActionsRegister(ctx context.Context) *cli.Command {
return &cli.Command{
Name: "register",
Usage: "Idempotent registration of a runner using a shared secret",
Action: prepareWorkPathAndCustomConf(ctx, func(cliCtx *cli.Context) error { return RunRegister(ctx, cliCtx) }),
Before: prepareWorkPathAndCustomConf(ctx),
Action: func(cliCtx *cli.Context) error { return RunRegister(ctx, cliCtx) },
Flags: []cli.Flag{
&cli.StringFlag{
Name: "secret",
@ -219,25 +221,3 @@ func RunGenerateActionsRunnerToken(ctx context.Context, cliCtx *cli.Context) err
}
return nil
}
func prepareWorkPathAndCustomConf(ctx context.Context, action cli.ActionFunc) func(cliCtx *cli.Context) error {
return func(cliCtx *cli.Context) error {
if !ContextGetNoInit(ctx) {
var args setting.ArgWorkPathAndCustomConf
// from children to parent, check the global flags
for _, curCtx := range cliCtx.Lineage() {
if curCtx.IsSet("work-path") && args.WorkPath == "" {
args.WorkPath = curCtx.String("work-path")
}
if curCtx.IsSet("custom-path") && args.CustomPath == "" {
args.CustomPath = curCtx.String("custom-path")
}
if curCtx.IsSet("config") && args.CustomConf == "" {
args.CustomConf = curCtx.String("config")
}
}
setting.InitWorkPathAndCommonConfig(os.Getenv, args)
}
return action(cliCtx)
}
}

115
cmd/forgejo/f3.go
View file

@ -4,74 +4,32 @@ package forgejo
import (
"context"
"fmt"
auth_model "code.gitea.io/gitea/models/auth"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/services/f3/util"
"github.com/urfave/cli/v2"
f3_types "lab.forgefriends.org/friendlyforgeformat/gof3/config/types"
f3_common "lab.forgefriends.org/friendlyforgeformat/gof3/forges/common"
f3_format "lab.forgefriends.org/friendlyforgeformat/gof3/format"
f3_cmd "lab.forgefriends.org/friendlyforgeformat/gof3/cmd"
)
func CmdF3(ctx context.Context) *cli.Command {
return &cli.Command{
Name: "f3",
Usage: "Friendly Forge Format (F3) format export/import.",
Description: "Import or export a repository from or to the Friendly Forge Format (F3) format.",
Action: prepareWorkPathAndCustomConf(ctx, func(cliCtx *cli.Context) error { return RunF3(ctx, cliCtx) }),
Flags: []cli.Flag{
&cli.StringFlag{
Name: "directory",
Value: "./f3",
Usage: "Path of the directory where the F3 dump is stored",
},
&cli.StringFlag{
Name: "user",
Value: "",
Usage: "The name of the user who owns the repository",
},
&cli.StringFlag{
Name: "repository",
Value: "",
Usage: "The name of the repository",
},
&cli.StringFlag{
Name: "authentication-source",
Value: "",
Usage: "The name of the authentication source matching the forge of origin",
},
&cli.BoolFlag{
Name: "no-pull-request",
Usage: "Do not dump pull requests",
},
&cli.BoolFlag{
Name: "import",
Usage: "Import from the directory",
},
&cli.BoolFlag{
Name: "export",
Usage: "Export to the directory",
},
Name: "f3",
Usage: "F3",
Subcommands: []*cli.Command{
SubcmdF3Mirror(ctx),
},
}
}
func getAuthenticationSource(ctx context.Context, authenticationSource string) (*auth_model.Source, error) {
source, err := auth_model.GetSourceByName(ctx, authenticationSource)
if err != nil {
if auth_model.IsErrSourceNotExist(err) {
return nil, nil
}
return nil, err
}
return source, nil
func SubcmdF3Mirror(ctx context.Context) *cli.Command {
mirrorCmd := f3_cmd.CreateCmdMirror(ctx)
mirrorCmd.Before = prepareWorkPathAndCustomConf(ctx)
f3Action := mirrorCmd.Action
mirrorCmd.Action = func(c *cli.Context) error { return runMirror(ctx, c, f3Action) }
return mirrorCmd
}
func RunF3(ctx context.Context, cliCtx *cli.Context) error {
func runMirror(ctx context.Context, c *cli.Context, action cli.ActionFunc) error {
var cancel context.CancelFunc
if !ContextGetNoInit(ctx) {
ctx, cancel = installSignals(ctx)
@ -86,52 +44,5 @@ func RunF3(ctx context.Context, cliCtx *cli.Context) error {
}
}
doer, err := user_model.GetAdminUser(ctx)
if err != nil {
return err
}
features := f3_types.AllFeatures
if cliCtx.Bool("no-pull-request") {
features.PullRequests = false
}
var sourceID int64
sourceName := cliCtx.String("authentication-source")
source, err := getAuthenticationSource(ctx, sourceName)
if err != nil {
return fmt.Errorf("error retrieving the authentication-source %s %v", sourceName, err)
}
if source != nil {
sourceID = source.ID
}
forgejo := util.ForgejoForgeRoot(features, doer, sourceID)
f3 := util.F3ForgeRoot(features, cliCtx.String("directory"))
if cliCtx.Bool("export") {
forgejo.Forge.Users.List(ctx)
user := forgejo.Forge.Users.GetFromFormat(ctx, &f3_format.User{UserName: cliCtx.String("user")})
if user.IsNil() {
return fmt.Errorf("%s is not a known user", cliCtx.String("user"))
}
user.Projects.List(ctx)
project := user.Projects.GetFromFormat(ctx, &f3_format.Project{Name: cliCtx.String("repository")})
if project.IsNil() {
return fmt.Errorf("%s/%s is not a known repository", cliCtx.String("user"), cliCtx.String("repository"))
}
options := f3_common.NewMirrorOptionsRecurse(user, project)
f3.Forge.Mirror(ctx, forgejo.Forge, options)
fmt.Fprintln(ContextGetStdout(ctx), "exported")
} else if cliCtx.Bool("import") {
options := f3_common.NewMirrorOptionsRecurse()
forgejo.Forge.Mirror(ctx, f3.Forge, options)
fmt.Fprintln(ContextGetStdout(ctx), "imported")
} else {
return fmt.Errorf("either --import or --export must be specified")
}
return nil
return action(c)
}

22
cmd/forgejo/forgejo.go
View file

@ -146,3 +146,25 @@ func handleCliResponseExtra(ctx context.Context, extra private.ResponseExtra) er
}
return cli.Exit(extra.Error, 1)
}
func prepareWorkPathAndCustomConf(ctx context.Context) func(c *cli.Context) error {
return func(c *cli.Context) error {
if !ContextGetNoInit(ctx) {
var args setting.ArgWorkPathAndCustomConf
// from children to parent, check the global flags
for _, curCtx := range c.Lineage() {
if curCtx.IsSet("work-path") && args.WorkPath == "" {
args.WorkPath = curCtx.String("work-path")
}
if curCtx.IsSet("custom-path") && args.CustomPath == "" {
args.CustomPath = curCtx.String("custom-path")
}
if curCtx.IsSet("config") && args.CustomConf == "" {
args.CustomConf = curCtx.String("config")
}
}
setting.InitWorkPathAndCommonConfig(os.Getenv, args)
}
return nil
}
}

2
go.mod
View file

@ -117,7 +117,7 @@ require (
gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v3 v3.0.1
lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230727095825-ce0c3d1c91ae
lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230731200648-8a343728fa3d
mvdan.cc/xurls/v2 v2.5.0
strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251
xorm.io/builder v0.3.13

4
go.sum
View file

@ -1573,8 +1573,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230727095825-ce0c3d1c91ae h1:O5colcyXVMuo4iVaXmBux8wa+HROJfhFOnA/TLKXIew=
lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230727095825-ce0c3d1c91ae/go.mod h1:TcKaEsgVihjAjw290iDvvirCT0P+DZNpzc0ZgNgy3E4=
lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230731200648-8a343728fa3d h1:xI6pGGV0z7Q6/YWUwYl/PEAcRwFN6nEzk7yD8MPnF9A=
lab.forgefriends.org/friendlyforgeformat/gof3 v0.0.0-20230731200648-8a343728fa3d/go.mod h1:TcKaEsgVihjAjw290iDvvirCT0P+DZNpzc0ZgNgy3E4=
lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=

84
services/f3/driver/driver.go
View file

@ -3,18 +3,34 @@
package driver
import (
"context"
"fmt"
auth_model "code.gitea.io/gitea/models/auth"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/services/migrations"
"github.com/urfave/cli/v2"
config_factory "lab.forgefriends.org/friendlyforgeformat/gof3/config/factory"
f3_types "lab.forgefriends.org/friendlyforgeformat/gof3/config/types"
"lab.forgefriends.org/friendlyforgeformat/gof3/forges/common"
"lab.forgefriends.org/friendlyforgeformat/gof3/forges/driver"
"lab.forgefriends.org/friendlyforgeformat/gof3/format"
)
var Name = "InternalForgejo"
func init() {
config_factory.RegisterFactory(Name, f3_types.OptionsFactory{
Name: Name,
New: func() f3_types.OptionsInterface { return &Options{} },
Flags: GetFlags,
}, func() common.DriverInterface { return &Forgejo{} })
}
type Options struct {
f3_types.Options
@ -22,11 +38,79 @@ type Options struct {
Doer *user_model.User
}
func getAuthenticationSource(ctx context.Context, authenticationSource string) (*auth_model.Source, error) {
source, err := auth_model.GetSourceByName(ctx, authenticationSource)
if err != nil {
if auth_model.IsErrSourceNotExist(err) {
return nil, nil
}
return nil, err
}
return source, nil
}
func ToF3Logger(messenger base.Messenger) *f3_types.Logger {
if messenger == nil {
messenger = func(message string, args ...interface{}) {
log.Info("Message: "+message, args...)
}
}
return &f3_types.Logger{
Message: f3_types.LoggerFun(messenger),
Trace: log.Trace,
Debug: log.Debug,
Info: log.Info,
Warn: log.Warn,
Error: log.Error,
Critical: log.Critical,
Fatal: log.Fatal,
}
}
func (o *Options) FromFlags(ctx context.Context, c *cli.Context, prefix string) f3_types.OptionsInterface {
o.Options.FromFlags(ctx, c, prefix)
o.Options.Logger = ToF3Logger(nil)
sourceName := c.String("authentication-source")
if sourceName != "" {
source, err := getAuthenticationSource(ctx, sourceName)
if err != nil {
panic(fmt.Errorf("error retrieving the authentication-source %s %v", sourceName, err))
}
if source != nil {
o.AuthenticationSource = source.ID
}
}
doer, err := user_model.GetAdminUser(ctx)
if err != nil {
panic(fmt.Errorf("GetAdminUser %v", err))
}
o.Doer = doer
return o
}
func GetFlags(prefix, category string) []cli.Flag {
flags := make([]cli.Flag, 0, 10)
flags = append(flags, &cli.StringFlag{
Name: "authentication-source",
Value: "",
Usage: "The name of the authentication source matching the forge of origin",
})
return flags
}
type Forgejo struct {
perPage int
options *Options
}
func (o *Forgejo) GetName() string {
return Name
}
func (o *Forgejo) GetPerPage() int {
return o.perPage
}

8
services/f3/util/util.go
View file

@ -13,13 +13,13 @@ import (
"lab.forgefriends.org/friendlyforgeformat/gof3/forges/f3"
)
func ToF3Logger(messenger base.Messenger) f3_types.Logger {
func ToF3Logger(messenger base.Messenger) *f3_types.Logger {
if messenger == nil {
messenger = func(message string, args ...interface{}) {
log.Info("Message: "+message, args...)
}
}
return f3_types.Logger{
return &f3_types.Logger{
Message: f3_types.LoggerFun(messenger),
Trace: log.Trace,
Debug: log.Debug,
@ -32,7 +32,7 @@ func ToF3Logger(messenger base.Messenger) f3_types.Logger {
}
func ForgejoForgeRoot(features f3_types.Features, doer *user_model.User, authenticationSource int64) *f3_forges.ForgeRoot {
forgeRoot := f3_forges.NewForgeRootFromDriver(&driver.Forgejo{}, &driver.Options{
forgeRoot := f3_forges.NewForgeRoot(&driver.Forgejo{}, &driver.Options{
Options: f3_types.Options{
Features: features,
Logger: ToF3Logger(nil),
@ -44,7 +44,7 @@ func ForgejoForgeRoot(features f3_types.Features, doer *user_model.User, authent
}
func F3ForgeRoot(features f3_types.Features, directory string) *f3_forges.ForgeRoot {
forgeRoot := f3_forges.NewForgeRoot(&f3.Options{
forgeRoot := f3_forges.NewForgeRoot(&f3.F3{}, &f3.Options{
Options: f3_types.Options{
Configuration: f3_types.Configuration{
Directory: directory,

67
tests/integration/cmd_forgejo_f3_test.go
View file

@ -7,8 +7,10 @@ import (
"net/url"
"testing"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/test"
"code.gitea.io/gitea/services/f3/driver"
"code.gitea.io/gitea/services/migrations"
"github.com/stretchr/testify/assert"
@ -16,7 +18,7 @@ import (
f3_util "lab.forgefriends.org/friendlyforgeformat/gof3/util"
)
func Test_CmdF3(t *testing.T) {
func TestF3_CmdMirror_LocalForgejo(t *testing.T) {
onGiteaRun(t, func(*testing.T, *url.URL) {
defer test.MockVariable(&setting.F3.Enabled, true)()
defer test.MockVariable(&setting.Migrations.AllowLocalNetworks, true)()
@ -26,50 +28,45 @@ func Test_CmdF3(t *testing.T) {
// a http call fails with "...migration can only call allowed HTTP servers..."
migrations.Init()
ctx := context.Background()
var userID int64 = 700
//
// Step 1: create a fixture
// Step 1: create a fixture as an F3 archive
//
userID++
fixture := f3_forges.NewFixture(t, f3_forges.FixtureF3Factory)
fixture.NewUser(1234)
fixture.NewMilestone()
fixture.NewLabel()
fixture.NewUser(userID)
fixture.NewIssue()
fixture.NewTopic()
fixture.NewRepository()
fixture.NewRelease()
fixture.NewAsset()
fixture.NewIssueComment(nil)
fixture.NewIssueReaction()
//
// Step 2: import the fixture into Gitea
// Step 3: mirror the F3 archive to the forge
//
{
output, err := cmdForgejoCaptureOutput(t, []string{"forgejo", "forgejo-cli", "f3", "--import", "--directory", fixture.ForgeRoot.GetDirectory()})
assert.NoError(t, err)
assert.EqualValues(t, "imported\n", output)
}
_, err := cmdForgejoCaptureOutput(t, []string{
"forgejo", "forgejo-cli", "f3", "mirror",
"--from-type=f3", "--from", fixture.ForgeRoot.GetDirectory(),
"--to-type", driver.Name,
})
assert.NoError(t, err)
user, err := user_model.GetUserByName(ctx, fixture.UserFormat.UserName)
assert.NoError(t, err)
//
// Step 4: mirror the forge to an F3 archive
//
dumpDir := t.TempDir()
_, err = cmdForgejoCaptureOutput(t, []string{
"forgejo", "forgejo-cli", "f3", "mirror",
"--user", user.Name, "--repository", fixture.ProjectFormat.Name,
"--from-type", driver.Name,
"--to-type=f3", "--to", dumpDir,
})
assert.NoError(t, err)
//
// Step 3: export Gitea into F3
// Step 5: verify the F3 archive content
//
directory := t.TempDir()
{
output, err := cmdForgejoCaptureOutput(t, []string{"forgejo", "forgejo-cli", "f3", "--export", "--no-pull-request", "--user", fixture.UserFormat.UserName, "--repository", fixture.ProjectFormat.Name, "--directory", directory})
assert.NoError(t, err)
assert.EqualValues(t, "exported\n", output)
}
//
// Step 4: verify the export and import are equivalent
//
files := f3_util.Command(context.Background(), "find", directory)
assert.Contains(t, files, "/label/")
assert.Contains(t, files, "/issue/")
assert.Contains(t, files, "/milestone/")
assert.Contains(t, files, "/topic/")
assert.Contains(t, files, "/release/")
assert.Contains(t, files, "/asset/")
assert.Contains(t, files, "/reaction/")
files := f3_util.Command(context.Background(), "find", dumpDir)
assert.Contains(t, files, "/user/")
assert.Contains(t, files, "/project/")
})
}

18
tests/integration/f3_test.go
View file

@ -30,7 +30,7 @@ import (
f3_util "lab.forgefriends.org/friendlyforgeformat/gof3/util"
)
func TestF3Mirror(t *testing.T) {
func TestF3_Mirror(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
AllowLocalNetworks := setting.Migrations.AllowLocalNetworks
setting.F3.Enabled = true
@ -47,7 +47,7 @@ func TestF3Mirror(t *testing.T) {
// Step 1: create a fixture
//
fixtureNewF3Forge := func(t f3_tests.TestingT, user *format.User, tmpDir string) *f3_forges.ForgeRoot {
root := f3_forges.NewForgeRoot(&f3_f3.Options{
root := f3_forges.NewForgeRoot(&f3_f3.F3{}, &f3_f3.Options{
Options: f3_types.Options{
Configuration: f3_types.Configuration{
Directory: tmpDir,
@ -88,7 +88,7 @@ func TestF3Mirror(t *testing.T) {
// Step 3: mirror Forgejo into F3
//
adminUsername := "user1"
forgejoAPI := f3_forges.NewForgeRootFromDriver(&f3_forgejo.Forgejo{}, &f3_forgejo.Options{
forgejoAPI := f3_forges.NewForgeRoot(&f3_forgejo.Forgejo{}, &f3_forgejo.Options{
Options: f3_types.Options{
Configuration: f3_types.Configuration{
URL: setting.AppURL,
@ -125,7 +125,7 @@ func TestF3Mirror(t *testing.T) {
})
}
func TestMaybePromoteF3User(t *testing.T) {
func TestF3_MaybePromoteUser(t *testing.T) {
defer tests.PrepareTestEnv(t)()
//
@ -182,7 +182,7 @@ func TestMaybePromoteF3User(t *testing.T) {
assert.Equal(t, userAfterSignIn.Email, gitlabEmail)
}
func TestF3UserMappingExisting(t *testing.T) {
func TestF3_UserMappingExisting(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
AllowLocalNetworks := setting.Migrations.AllowLocalNetworks
setting.F3.Enabled = true
@ -197,7 +197,7 @@ func TestF3UserMappingExisting(t *testing.T) {
log.Debug("Step 1: create a fixture in F3")
fixtureNewF3Forge := func(t f3_tests.TestingT, user *format.User, tmpDir string) *f3_forges.ForgeRoot {
root := f3_forges.NewForgeRoot(&f3_f3.Options{
root := f3_forges.NewForgeRoot(&f3_f3.F3{}, &f3_f3.Options{
Options: f3_types.Options{
Configuration: f3_types.Configuration{
Directory: tmpDir,
@ -242,7 +242,7 @@ func TestF3UserMappingExisting(t *testing.T) {
log.Debug("Step 3: mirror Forgejo into F3")
adminUsername := "user1"
forgejoAPI := f3_forges.NewForgeRootFromDriver(&f3_forgejo.Forgejo{}, &f3_forgejo.Options{
forgejoAPI := f3_forges.NewForgeRoot(&f3_forgejo.Forgejo{}, &f3_forgejo.Options{
Options: f3_types.Options{
Configuration: f3_types.Configuration{
URL: setting.AppURL,
@ -270,7 +270,7 @@ func TestF3UserMappingExisting(t *testing.T) {
})
}
func TestF3UserMappingNew(t *testing.T) {
func TestF3_UserMappingNew(t *testing.T) {
onGiteaRun(t, func(t *testing.T, u *url.URL) {
AllowLocalNetworks := setting.Migrations.AllowLocalNetworks
setting.F3.Enabled = true
@ -285,7 +285,7 @@ func TestF3UserMappingNew(t *testing.T) {
log.Debug("Step 1: create a fixture in F3")
fixtureNewF3Forge := func(t f3_tests.TestingT, user *format.User, tmpDir string) *f3_forges.ForgeRoot {
root := f3_forges.NewForgeRoot(&f3_f3.Options{
root := f3_forges.NewForgeRoot(&f3_f3.F3{}, &f3_f3.Options{
Options: f3_types.Options{
Configuration: f3_types.Configuration{
Directory: tmpDir,