From 206d0b3886b2d56b585bf552e53d952b35f07284 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Wed, 14 Jun 2023 13:32:20 +0200 Subject: [PATCH] [CI] Forgejo Actions based release process Refs: https://codeberg.org/forgejo/website/pulls/230 (cherry picked from commit 87d56bf6c73d726dae8aafbc7e147969f1899931) [CI] Forgejo Actions based release process (squash) base64 -w0 to avoid wrapping when the doer name is long as it creates a broken config.json (cherry picked from commit 9efdc27e49bdfb3e62401baf27b224385f9f3e5e) [CI] Forgejo Actions based release process (squash) generate .xz files and sources Generate .xz files Check .sha256 Generate the source tarbal (cherry picked from commit 7afec520c4b1032d7e67a05a41e4e2913bcd9312) [CI] Forgejo Actions based release process (squash) release notes (cherry picked from commit d8f4f4807b28297b318d2f555a76d0efef762cf7) [CI] Forgejo Actions based release process (squash) publish and sign release (cherry picked from commit a52778c74785fe57cdee3b64b4c6c8a326471532) (cherry picked from commit cf2ec6274094ac7aebda71d54c64581f528df00a) [CI] Forgejo Actions based release process (squash) version use Actions environment variables in Makefile (#25319) (#25318) uses Actions variable to determine the version. But Forgejo builds happen in a container where they are not available. Do not use them. Also verify the version of the binary is as expected for sanity check. (cherry picked from commit 6decf111a132a869f9e5c6f4d20e368b8f74309f) --- .forgejo/actions/build-release/action.yml | 154 ++++++++++++++++++ .forgejo/actions/publish-release/action.yml | 93 +++++++++++ .../workflows/build-release-integration.yml | 104 ++++++++++++ .forgejo/workflows/build-release.yml | 134 +++++++++++++++ .forgejo/workflows/publish-release.yml | 25 +++ Dockerfile | 35 ++-- Dockerfile.rootless | 35 ++-- Makefile | 31 +--- 8 files changed, 565 insertions(+), 46 deletions(-) create mode 100644 .forgejo/actions/build-release/action.yml create mode 100644 .forgejo/actions/publish-release/action.yml create mode 100644 .forgejo/workflows/build-release-integration.yml create mode 100644 .forgejo/workflows/build-release.yml create mode 100644 .forgejo/workflows/publish-release.yml diff --git a/.forgejo/actions/build-release/action.yml b/.forgejo/actions/build-release/action.yml new file mode 100644 index 0000000000..ad9818e9f0 --- /dev/null +++ b/.forgejo/actions/build-release/action.yml @@ -0,0 +1,154 @@ +name: 'Build release' +author: 'Forgejo authors' +description: | + Build release + +inputs: + forgejo: + description: 'URL of the Forgejo instance where the release is uploaded' + required: true + owner: + description: 'User or organization where the release is uploaded, relative to the Forgejo instance' + required: true + repository: + description: 'Repository where the release is uploaded, relative to the owner' + required: true + doer: + description: 'Name of the user authoring the release' + required: true + tag-version: + description: 'Version of the release derived from the tag withint the leading v' + required: true + suffix: + description: 'Suffix to add to the image tag' + token: + description: 'token' + required: true + dockerfile: + description: 'path to the dockerfile' + default: 'Dockerfile' + platforms: + description: 'Coma separated list of platforms' + default: 'linux/amd64,linux/arm64' + release-notes: + description: 'Full text of the release notes' + default: 'Release notes placeholder' + binary-name: + description: 'Name of the binary' + binary-path: + description: 'Path of the binary within the container to extract into binary-name' + verbose: + description: 'Increase the verbosity level' + default: 'false' + +runs: + using: "composite" + steps: + - run: echo "${{ github.action_path }}" >> $GITHUB_PATH + shell: bash + + - name: Install dependencies + run: | + apt-get install -y -qq xz-utils + + - name: set -x if verbose is required + id: verbose + run: | + if ${{ inputs.verbose }} ; then + echo "shell=set -x" >> "$GITHUB_OUTPUT" + fi + + - name: Create the insecure and buildx-config variables for the container registry + id: registry + run: | + ${{ steps.verbose.outputs.shell }} + url="${{ inputs.forgejo }}" + hostport=${url##http*://} + hostport=${hostport%%/} + echo "host-port=${hostport}" >> "$GITHUB_OUTPUT" + if ! [[ $url =~ ^http:// ]] ; then + exit 0 + fi + cat >> "$GITHUB_OUTPUT" < /etc/docker/daemon.json < ~/.docker/config.json + env: + CI_REGISTRY: "${{ steps.registry.outputs.host-port }}" + + - name: Build the container image for each architecture + uses: https://github.com/docker/build-push-action@v4 + # workaround until https://github.com/docker/build-push-action/commit/d8823bfaed2a82c6f5d4799a2f8e86173c461aba is in @v4 or @v5 is released + env: + ACTIONS_RUNTIME_TOKEN: '' + with: + context: . + push: true + file: ${{ inputs.dockerfile }} + platforms: ${{ inputs.platforms }} + tags: ${{ steps.registry.outputs.host-port }}/${{ inputs.owner }}/${{ inputs.repository }}:${{ inputs.tag-version }}${{ inputs.suffix }} + + - name: Extract the binary from the container images into the release directory + if: inputs.binary-name != '' + run: | + ${{ steps.verbose.outputs.shell }} + mkdir -p release + cd release + for platform in $(echo ${{ inputs.platforms }} | tr ',' ' '); do + arch=$(echo $platform | sed -e 's|linux/||g' -e 's|arm/v6|arm-6|g') + docker create --platform $platform --name forgejo-$arch ${{ steps.registry.outputs.host-port }}/${{ inputs.owner }}/${{ inputs.repository }}:${{ inputs.tag-version }}${{ inputs.suffix }} + binary="${{ inputs.binary-name }}-${{ inputs.tag-version }}-linux" + docker cp forgejo-$arch:${{ inputs.binary-path }} $binary-$arch + chmod +x $binary-$arch + # the displayed version is converted with - sometime changed into +, deal with it + pattern=$(echo "${{ inputs.tag-version }}" | tr - .) + if ! ./$binary-$arch --version | grep "$pattern" ; then + echo "ERROR: expected version pattern $pattern not found in the output of $binary-$arch --version" + ./$binary-$arch --version + exit 1 + fi + xz --keep -9 $binary-$arch + shasum -a 256 $binary-$arch > $binary-$arch.sha256 + shasum -a 256 $binary-$arch.xz > $binary-$arch.xz.sha256 + docker rm forgejo-$arch + done + + - name: publish release + if: inputs.binary-name != '' + uses: https://code.forgejo.org/actions/forgejo-release@v1 + with: + direction: upload + release-dir: release + release-notes: "${{ inputs.release-notes }}" + token: ${{ inputs.token }} + verbose: ${{ steps.verbose.outputs.value }} diff --git a/.forgejo/actions/publish-release/action.yml b/.forgejo/actions/publish-release/action.yml new file mode 100644 index 0000000000..bafa473000 --- /dev/null +++ b/.forgejo/actions/publish-release/action.yml @@ -0,0 +1,93 @@ +name: 'Publish release' +author: 'Forgejo authors' +description: | + Publish release + +inputs: + forgejo: + description: 'URL of the Forgejo instance where the release is uploaded' + required: true + from-owner: + description: 'the owner from which a release is to be copied' + required: true + to-owner: + description: 'the owner to which a release is to be copied' + required: true + repo: + description: 'the repository from which a release is to be copied relative to from-owner and to-owner' + default: 'forgejo' + ref-name: + description: 'ref_name of the tag of the release to be copied' + required: true + doer: + description: 'Name of the user authoring the release' + required: true + token: + description: 'application token on FORGEJO with permission to the repository and the packages' + required: true + gpg-private-key: + description: 'GPG Private Key to sign the release artifacts' + gpg-passphrase: + description: 'Passphrase of the GPG Private Key' + +runs: + using: "composite" + steps: + - id: hostport + run: | + url="${{ inputs.forgejo }}" + hostport=${url##http*://} + hostport=${hostport%%/} + echo "value=$hostport" >> "$GITHUB_OUTPUT" + + - id: tag-version + run: | + version="${{ inputs.ref-name }}" + version=${version##*v} + echo "value=$version" >> "$GITHUB_OUTPUT" + + - name: apt-get install docker.io + run: | + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -qq -y docker.io + + - name: download release + uses: https://code.forgejo.org/actions/forgejo-release@v1 + with: + url: ${{ inputs.forgejo }} + repo: ${{ inputs.from-owner }}/${{ inputs.repo }} + direction: download + release-dir: release + download-retry: 60 + token: ${{ inputs.token }} + + - name: upload release + uses: https://code.forgejo.org/actions/forgejo-release@v1 + with: + url: ${{ inputs.forgejo }} + repo: ${{ inputs.to-owner }}/${{ inputs.repo }} + direction: upload + release-dir: release + release-notes: "See https://codeberg.org/forgejo/forgejo/src/branch/forgejo/RELEASE-NOTES.md#${{ steps.tag-version.outputs.value }}" + token: ${{ inputs.token }} + gpg-private-key: ${{ inputs.gpg-private-key }} + gpg-passphrase: ${{ inputs.gpg-passphrase }} + + - name: login to the registry + uses: https://github.com/docker/login-action@v2 + with: + registry: ${{ steps.hostport.outputs.value }} + username: ${{ inputs.doer }} + password: ${{ inputs.token }} + + - uses: https://code.forgejo.org/forgejo/forgejo-container-image@v1 + env: + VERIFY: 'false' + with: + url: https://${{ steps.hostport.outputs.value }} + destination-owner: ${{ inputs.to-owner }} + owner: ${{ inputs.from-owner }} + suffixes: '-rootless' + project: ${{ inputs.repo }} + tag: ${{ steps.tag-version.outputs.value }} + doer: ${{ inputs.doer }} + token: ${{ inputs.token }} diff --git a/.forgejo/workflows/build-release-integration.yml b/.forgejo/workflows/build-release-integration.yml new file mode 100644 index 0000000000..476ce93282 --- /dev/null +++ b/.forgejo/workflows/build-release-integration.yml @@ -0,0 +1,104 @@ +name: Integration tests for the release process + +on: + push: + paths: + - Makefile + - Dockerfile + - Dockerfile.rootless + - docker/** + - .forgejo/actions/build-release/action.yml + - .forgejo/workflows/build-release.yml + - .forgejo/workflows/build-release-integration.yml + +jobs: + release-simulation: + runs-on: self-hosted + if: secrets.ROLE != 'forgejo-integration' && secrets.ROLE != 'forgejo-experimental' && secrets.ROLE != 'forgejo-release' + steps: + - uses: actions/checkout@v3 + + - id: forgejo + uses: https://code.forgejo.org/actions/setup-forgejo@v1 + with: + user: root + password: admin1234 + image-version: 1.19 + lxc-ip-prefix: 10.0.9 + + - name: publish the forgejo release + run: | + set -x + + version=1.2.3 + cat > /etc/docker/daemon.json < $dir/Dockerfile < /app/gitea/gitea ; chmod +x /app/gitea/gitea + EOF + cp $dir/Dockerfile $dir/Dockerfile.rootless + sources=forgejo-src-$version.tar.gz + ( echo 'sources-tarbal:' ; echo -e '\tmkdir -p dist/release ; cd dist/release ; sources=forgejo-src-$(VERSION).tar.gz ; echo sources > $$sources ; shasum -a 256 $$sources > $$sources.sha256' ) > $dir/Makefile + + forgejo-test-helper.sh push $dir $url root forgejo |& tee $dir/pushed + eval $(grep '^sha=' < $dir/pushed) + + # + # Push a tag to trigger the release workflow and wait for it to complete + # + forgejo-test-helper.sh api POST $url repos/root/forgejo/tags ${{ steps.forgejo.outputs.token }} --data-raw '{"tag_name": "v'$version'", "target": "'$sha'"}' + LOOPS=180 forgejo-test-helper.sh wait_success "$url" root/forgejo $sha + + # + # uncomment to see the logs even when everything is reported to be working ok + # + #cat $FORGEJO_RUNNER_LOGS + + # + # Minimal sanity checks. e2e test is for the setup-forgejo + # action and the infrastructure playbook. Since the binary + # is a script shell it does not test the sanity of the cross + # build, only the sanity of the naming of the binaries. + # + for arch in amd64 arm64 arm-6 ; do + binary=forgejo-$version-linux-$arch + for suffix in '' '.xz' ; do + curl --fail -L -sS $url/root/forgejo/releases/download/v$version/$binary$suffix > $binary$suffix + if test "$suffix" = .xz ; then + unxz --keep $binary$suffix + fi + chmod +x $binary + ./$binary --version | grep $version + curl --fail -L -sS $url/root/forgejo/releases/download/v$version/$binary$suffix.sha256 > $binary$suffix.sha256 + shasum -a 256 --check $binary$suffix.sha256 + rm $binary$suffix + done + done + + curl --fail -L -sS $url/root/forgejo/releases/download/v$version/$sources > $sources + curl --fail -L -sS $url/root/forgejo/releases/download/v$version/$sources.sha256 > $sources.sha256 + shasum -a 256 --check $sources.sha256 + + docker pull ${{ steps.forgejo.outputs.host-port }}/root/forgejo:$version + docker pull ${{ steps.forgejo.outputs.host-port }}/root/forgejo:$version-rootless diff --git a/.forgejo/workflows/build-release.yml b/.forgejo/workflows/build-release.yml new file mode 100644 index 0000000000..ffae76f23c --- /dev/null +++ b/.forgejo/workflows/build-release.yml @@ -0,0 +1,134 @@ +name: Build release + +on: + push: + tags: 'v*' + +jobs: + release: + runs-on: self-hosted + # root is used for testing, allow it + if: secrets.ROLE == 'forgejo-integration' || github.repository_owner == 'root' + steps: + - uses: actions/checkout@v3 + + - name: Increase the verbosity when there are no secrets + id: verbose + run: | + if test -z "${{ secrets.TOKEN }}"; then + value=true + else + value=false + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + + - name: Sanitize the name of the repository + id: repository + run: | + set -x # comment out + repository="${{ github.repository }}" + echo "value=${repository##*/}" >> "$GITHUB_OUTPUT" + + - name: When in a test environment, create a token + id: token + if: ${{ secrets.TOKEN == '' }} + run: | + apt-get -qq install -y jq + url="${{ env.GITHUB_SERVER_URL }}" + hostport=${url##http*://} + hostport=${hostport%%/} + doer=root + api=http://$doer:admin1234@$hostport/api/v1/users/$doer/tokens + curl -sS -X DELETE $api/release + token=$(curl -sS -X POST -H 'Content-Type: application/json' --data-raw '{"name": "release", "scopes": ["all"]}' $api | jq --raw-output .sha1) + echo "value=${token}" >> "$GITHUB_OUTPUT" + + - uses: https://code.forgejo.org/actions/setup-go@v4 + with: + go-version: ">=1.20" + check-latest: true + + - name: Create the version from ref_name + id: tag-version + run: | + version="${{ github.ref_name }}" + version=${version##*v} + echo "value=$version" >> "$GITHUB_OUTPUT" + + - name: Create the release notes + id: release-notes + run: | + cat >> "$GITHUB_OUTPUT" </dev/null) - ifneq ($(STORED_VERSION),) - GITEA_VERSION ?= $(STORED_VERSION) - else - GITEA_VERSION ?= $(shell git describe --tags --always | sed 's/-/+/' | sed 's/^v//') - endif -endif - -# if version = "main" then update version to "nightly" -ifeq ($(VERSION),main) - VERSION := main-nightly -endif +GITEA_VERSION ?= $(shell git describe --tags --always | sed 's/-/+/' | sed 's/^v//') +VERSION = ${GITEA_VERSION} LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(GITEA_VERSION)" -X "main.Tags=$(TAGS)" @@ -825,11 +802,13 @@ security-check: go run $(GOVULNCHECK_PACKAGE) ./... $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ) - CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@ + CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -o $@ .PHONY: release release: frontend generate release-linux release-copy release-compress vendor release-sources release-check +sources-tarbal: vendor release-sources release-check + $(DIST_DIRS): mkdir -p $(DIST_DIRS)