[DB] Forgejo database migrations

- Implements https://codeberg.org/forgejo/discussions/issues/32#issuecomment-918737
- Allows to add Forgejo-specific migrations that don't interfere with Gitea's migration logic. Please do note that we cannot liberally add migrations for Gitea tables, as they might do their own migrations in a future version on that table, and that could undo our migrations. Luckily,  we don't have a scenario where that's needed and thus not taken into account.

Co-authored-by: Gusted <postmaster@gusted.xyz>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/795
(cherry picked from commit 8ee32978c0)
(cherry picked from commit c240b34f59)
(cherry picked from commit 03936c6492)
(cherry picked from commit a20ed852f8)
(cherry picked from commit 1dfa82676f)
(cherry picked from commit c39ae0bf8a)
(cherry picked from commit cfaff08996)
(cherry picked from commit 94a458835a)
(cherry picked from commit 61a3cf77df)
(cherry picked from commit abb350fde8)
(cherry picked from commit 5194829d6b)
(cherry picked from commit 89239a60f2)
(cherry picked from commit 683cfd86ef)
(cherry picked from commit f4546cfed9)
(cherry picked from commit 86614d5826)
(cherry picked from commit e4b9c32187)
(cherry picked from commit 8c253719af)
(cherry picked from commit 857365d6c1)
(cherry picked from commit a488b3952f)
(cherry picked from commit 98313c4910)
(cherry picked from commit 430d95e824)
(cherry picked from commit 08bf9d918f)
(cherry picked from commit f8a170e2d0)
(cherry picked from commit d20e325378)
(cherry picked from commit 6c0aa7dd4f)
(cherry picked from commit 46c08c26c7)
(cherry picked from commit 9ee22153c4)

[DB] Ensure forgejo migration up to date (squash)

- Hook Forgejo's `EnsureUpToDate` to Gitea's `EnsureUpToDate`, such that
the Forgejo migrations are also being checked to be up to date.
- I'm not sure how I missed this and if this has caused any problems,
but due to the lack of any open issue about it it seems to not be a big
problem.

(cherry picked from commit 6c65b6dcf6)
(cherry picked from commit 6d45c37d84)

[DB] Add test for TestEnsureUpToDate (squash)

- Add a test for the behavior of `EnsureUpToDate`, to ensure it will
error when needed and succeed when the forgejo version is up to date.
- Add forgejo_migrations package to GO_TEST_PACKAGES, to avoid running
it with `test-unit` and instead test it with `test-*-migration`.

(cherry picked from commit b172a50691)
(cherry picked from commit d8af308820)
(cherry picked from commit e69e64a32c)
(cherry picked from commit 4e8363fad4)
(cherry picked from commit fc9ecd6c53)
(cherry picked from commit e5c446e3dc)

Makefile

@@ -100,7 +100,7 @@ LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(G
 LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64
 
 GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
-GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
+GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) $(shell $(GO) list code.gitea.io/gitea/models/forgejo_migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/))
 
 FOMANTIC_WORK_DIR := web_src/fomantic
 
@@ -712,7 +712,7 @@ migrations.sqlite.test: $(GO_SOURCES) generate-ini-sqlite
 
 .PHONY: migrations.individual.mysql.test
 migrations.individual.mysql.test: $(GO_SOURCES)
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
+	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/... code.gitea.io/gitea/models/forgejo_migrations/...); do \
 		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/mysql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg; \
 	done
 
@@ -722,7 +722,7 @@ migrations.individual.sqlite.test\#%: $(GO_SOURCES) generate-ini-sqlite
 
 .PHONY: migrations.individual.pgsql.test
 migrations.individual.pgsql.test: $(GO_SOURCES)
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
+	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/... code.gitea.io/gitea/models/forgejo_migrations/...); do \
 		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/pgsql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg; \
 	done
 
@@ -733,7 +733,7 @@ migrations.individual.pgsql.test\#%: $(GO_SOURCES) generate-ini-pgsql
 
 .PHONY: migrations.individual.mssql.test
 migrations.individual.mssql.test: $(GO_SOURCES) generate-ini-mssql
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
+	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/... code.gitea.io/gitea/models/forgejo_migrations/...); do \
 		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/mssql.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg -test.failfast; \
 	done
 
@@ -743,7 +743,7 @@ migrations.individual.mssql.test\#%: $(GO_SOURCES) generate-ini-mssql
 
 .PHONY: migrations.individual.sqlite.test
 migrations.individual.sqlite.test: $(GO_SOURCES) generate-ini-sqlite
-	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/...); do \
+	for pkg in $(shell $(GO) list code.gitea.io/gitea/models/migrations/... code.gitea.io/gitea/models/forgejo_migrations/...); do \
 		GITEA_ROOT="$(CURDIR)" GITEA_CONF=tests/sqlite.ini $(GO) test $(GOTESTFLAGS) -tags '$(TEST_TAGS)' $$pkg; \
 	done
 

models/forgejo_migrations/main_test.go

@@ -0,0 +1,14 @@
+// Copyright 2023 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package forgejo_migrations //nolint:revive
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/migrations/base"
+)
+
+func TestMain(m *testing.M) {
+	base.MainTest(m)
+}

models/forgejo_migrations/migrate.go

@@ -0,0 +1,139 @@
+// Copyright 2023 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package forgejo_migrations //nolint:revive
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+	"xorm.io/xorm/names"
+)
+
+// ForgejoVersion describes the Forgejo version table. Should have only one row with id = 1.
+type ForgejoVersion struct {
+	ID      int64 `xorm:"pk autoincr"`
+	Version int64
+}
+
+type Migration struct {
+	description string
+	migrate     func(*xorm.Engine) error
+}
+
+// NewMigration creates a new migration.
+func NewMigration(desc string, fn func(*xorm.Engine) error) *Migration {
+	return &Migration{desc, fn}
+}
+
+// This is a sequence of additional Forgejo migrations.
+// Add new migrations to the bottom of the list.
+var migrations = []*Migration{}
+
+// GetCurrentDBVersion returns the current Forgejo database version.
+func GetCurrentDBVersion(x *xorm.Engine) (int64, error) {
+	if err := x.Sync(new(ForgejoVersion)); err != nil {
+		return -1, fmt.Errorf("sync: %w", err)
+	}
+
+	currentVersion := &ForgejoVersion{ID: 1}
+	has, err := x.Get(currentVersion)
+	if err != nil {
+		return -1, fmt.Errorf("get: %w", err)
+	}
+	if !has {
+		return -1, nil
+	}
+	return currentVersion.Version, nil
+}
+
+// ExpectedVersion returns the expected Forgejo database version.
+func ExpectedVersion() int64 {
+	return int64(len(migrations))
+}
+
+// EnsureUpToDate will check if the Forgejo database is at the correct version.
+func EnsureUpToDate(x *xorm.Engine) error {
+	currentDB, err := GetCurrentDBVersion(x)
+	if err != nil {
+		return err
+	}
+
+	if currentDB < 0 {
+		return fmt.Errorf("database has not been initialized")
+	}
+
+	expected := ExpectedVersion()
+
+	if currentDB != expected {
+		return fmt.Errorf(`current Forgejo database version %d is not equal to the expected version %d. Please run "forgejo [--config /path/to/app.ini] migrate" to update the database version`, currentDB, expected)
+	}
+
+	return nil
+}
+
+// Migrate Forgejo database to current version.
+func Migrate(x *xorm.Engine) error {
+	// Set a new clean the default mapper to GonicMapper as that is the default for .
+	x.SetMapper(names.GonicMapper{})
+	if err := x.Sync(new(ForgejoVersion)); err != nil {
+		return fmt.Errorf("sync: %w", err)
+	}
+
+	currentVersion := &ForgejoVersion{ID: 1}
+	has, err := x.Get(currentVersion)
+	if err != nil {
+		return fmt.Errorf("get: %w", err)
+	} else if !has {
+		// If the version record does not exist we think
+		// it is a fresh installation and we can skip all migrations.
+		currentVersion.ID = 0
+		currentVersion.Version = ExpectedVersion()
+
+		if _, err = x.InsertOne(currentVersion); err != nil {
+			return fmt.Errorf("insert: %w", err)
+		}
+	}
+
+	v := currentVersion.Version
+
+	// Downgrading Forgejo's database version not supported
+	if v > ExpectedVersion() {
+		msg := fmt.Sprintf("Your Forgejo database (migration version: %d) is for a newer version of Forgejo, you cannot use the newer database for this old Forgejo release (%d).", v, ExpectedVersion())
+		msg += "\nForgejo will exit to keep your database safe and unchanged. Please use the correct Forgejo release, do not change the migration version manually (incorrect manual operation may cause data loss)."
+		if !setting.IsProd {
+			msg += fmt.Sprintf("\nIf you are in development and really know what you're doing, you can force changing the migration version by executing: UPDATE forgejo_version SET version=%d WHERE id=1;", ExpectedVersion())
+		}
+		_, _ = fmt.Fprintln(os.Stderr, msg)
+		log.Fatal(msg)
+		return nil
+	}
+
+	// Some migration tasks depend on the git command
+	if git.DefaultContext == nil {
+		if err = git.InitSimple(context.Background()); err != nil {
+			return err
+		}
+	}
+
+	// Migrate
+	for i, m := range migrations[v:] {
+		log.Info("Migration[%d]: %s", v+int64(i), m.description)
+		// Reset the mapper between each migration - migrations are not supposed to depend on each other
+		x.SetMapper(names.GonicMapper{})
+		if err = m.migrate(x); err != nil {
+			return fmt.Errorf("migration[%d]: %s failed: %w", v+int64(i), m.description, err)
+		}
+		currentVersion.Version = v + int64(i) + 1
+		if _, err = x.ID(1).Update(currentVersion); err != nil {
+			return err
+		}
+	}
+	return nil
+}

models/forgejo_migrations/migrate_test.go

@@ -0,0 +1,39 @@
+// Copyright 2023 The Forgejo Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package forgejo_migrations //nolint:revive
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/migrations/base"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestEnsureUpToDate tests the behavior of EnsureUpToDate.
+func TestEnsureUpToDate(t *testing.T) {
+	x, deferable := base.PrepareTestEnv(t, 0, new(ForgejoVersion))
+	defer deferable()
+	if x == nil || t.Failed() {
+		return
+	}
+
+	// Ensure error if there's no row in Forgejo Version.
+	err := EnsureUpToDate(x)
+	assert.Error(t, err)
+
+	// Insert 'good' Forgejo Version row.
+	_, err = x.InsertOne(&ForgejoVersion{ID: 1, Version: ExpectedVersion()})
+	assert.NoError(t, err)
+
+	err = EnsureUpToDate(x)
+	assert.NoError(t, err)
+
+	// Modify forgejo version to have a lower version.
+	_, err = x.Exec("UPDATE `forgejo_version` SET version = ? WHERE id = 1", ExpectedVersion()-1)
+	assert.NoError(t, err)
+
+	err = EnsureUpToDate(x)
+	assert.Error(t, err)
+}

models/migrations/migrations.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 
+	"code.gitea.io/gitea/models/forgejo_migrations"
 	"code.gitea.io/gitea/models/migrations/v1_10"
 	"code.gitea.io/gitea/models/migrations/v1_11"
 	"code.gitea.io/gitea/models/migrations/v1_12"
@@ -597,7 +598,7 @@ func EnsureUpToDate(x *xorm.Engine) error {
 		return fmt.Errorf(`Current database version %d is not equal to the expected version %d. Please run "gitea [--config /path/to/app.ini] migrate" to update the database version`, currentDB, expected)
 	}
 
-	return nil
+	return forgejo_migrations.EnsureUpToDate(x)
 }
 
 // Migrate database to current version
@@ -661,5 +662,7 @@ Please try upgrading to a lower version first (suggested v1.6.4), then upgrade t
 			return err
 		}
 	}
-	return nil
+
+	// Execute Forgejo specific migrations.
+	return forgejo_migrations.Migrate(x)
 }