From 87d56bf6c73d726dae8aafbc7e147969f1899931 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Wed, 14 Jun 2023 13:32:20 +0200 Subject: [PATCH] [CI] Forgejo Actions based release process Refs: https://codeberg.org/forgejo/website/pulls/230 --- .forgejo/actions/build-release/action.yml | 144 ++++++++++++++++++ .../workflows/build-release-integration.yml | 86 +++++++++++ .forgejo/workflows/build-release.yml | 104 +++++++++++++ Dockerfile | 35 +++-- Dockerfile.rootless | 35 +++-- Makefile | 2 +- 6 files changed, 385 insertions(+), 21 deletions(-) create mode 100644 .forgejo/actions/build-release/action.yml create mode 100644 .forgejo/workflows/build-release-integration.yml create mode 100644 .forgejo/workflows/build-release.yml diff --git a/.forgejo/actions/build-release/action.yml b/.forgejo/actions/build-release/action.yml new file mode 100644 index 0000000000..2037c76773 --- /dev/null +++ b/.forgejo/actions/build-release/action.yml @@ -0,0 +1,144 @@ +name: 'Build release' +author: 'Forgejo authors' +description: | + Build release + +inputs: + forgejo: + description: 'URL of the Forgejo instance where the release is uploaded' + required: true + owner: + description: 'User or organization where the release is uploaded, relative to the Forgejo instance' + required: true + repository: + description: 'Repository where the release is uploaded, relative to the owner' + required: true + doer: + description: 'Name of the user authoring the release' + required: true + ref_name: + description: 'Git reference of the tag for which the release must be built' + required: true + suffix: + description: 'Suffix to add to the image tag' + token: + description: 'token' + required: true + dockerfile: + description: 'path to the dockerfile' + default: 'Dockerfile' + platforms: + description: 'Coma separated list of platforms' + default: 'linux/amd64,linux/arm64' + binary-name: + description: 'Name of the binary' + binary-path: + description: 'Path of the binary within the container to extract into binary-name' + verbose: + description: 'Increase the verbosity level' + default: 'false' + +runs: + using: "composite" + steps: + - run: echo "${{ github.action_path }}" >> $GITHUB_PATH + shell: bash + + - name: set -x if verbose is required + id: verbose + run: | + if ${{ inputs.verbose }} ; then + echo "shell=set -x" >> "$GITHUB_OUTPUT" + fi + + - name: Create the insecure and buildx-config variables for the container registry + id: registry + run: | + ${{ steps.verbose.outputs.shell }} + url="${{ inputs.forgejo }}" + hostport=${url##http*://} + hostport=${hostport%%/} + echo "host-port=${hostport}" >> "$GITHUB_OUTPUT" + if ! [[ $url =~ ^http:// ]] ; then + exit 0 + fi + cat >> "$GITHUB_OUTPUT" < /etc/docker/daemon.json < ~/.docker/config.json + env: + CI_REGISTRY: "${{ steps.registry.outputs.host-port }}" + + - name: Create the tag and image variable + id: build + run: | + ${{ steps.verbose.outputs.shell }} + tag="${{ inputs.ref_name }}" + tag=${tag##*v} + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: Build the container image for each architecture + uses: https://github.com/docker/build-push-action@v4 + # workaround until https://github.com/docker/build-push-action/commit/d8823bfaed2a82c6f5d4799a2f8e86173c461aba is in @v4 or @v5 is released + env: + ACTIONS_RUNTIME_TOKEN: '' + with: + context: . + push: true + file: ${{ inputs.dockerfile }} + platforms: ${{ inputs.platforms }} + tags: ${{ steps.registry.outputs.host-port }}/${{ inputs.owner }}/${{ inputs.repository }}:${{ steps.build.outputs.tag }}${{ inputs.suffix }} + + - name: Extract the binary from the container images into the release directory + if: inputs.binary-name != '' + run: | + ${{ steps.verbose.outputs.shell }} + mkdir -p release + for platform in $(echo ${{ inputs.platforms }} | tr ',' ' '); do + arch=$(echo $platform | sed -e 's|linux/||g' -e 's|arm/v6|arm-6|g') + docker create --platform $platform --name forgejo-$arch ${{ steps.registry.outputs.host-port }}/${{ inputs.owner }}/${{ inputs.repository }}:${{ steps.build.outputs.tag }}${{ inputs.suffix }} + binary="${{ inputs.binary-name }}-${{ steps.build.outputs.tag }}" + docker cp forgejo-$arch:${{ inputs.binary-path }} release/$binary-$arch + shasum -a 256 < release/$binary-$arch | cut -f1 -d ' ' > release/$binary-$arch.sha256 + docker rm forgejo-$arch + done + + - name: publish release + if: inputs.binary-name != '' + uses: https://code.forgejo.org/actions/forgejo-release@v1 + with: + direction: upload + release-dir: release + release-notes: "RELEASE-NOTES#${{ steps.build.outputs.tag }}" + token: ${{ inputs.token }} + verbose: ${{ steps.verbose.outputs.value }} diff --git a/.forgejo/workflows/build-release-integration.yml b/.forgejo/workflows/build-release-integration.yml new file mode 100644 index 0000000000..8ca995cc29 --- /dev/null +++ b/.forgejo/workflows/build-release-integration.yml @@ -0,0 +1,86 @@ +name: Integration tests for the release process + +on: + push: + paths: + - Makefile + - docker/** + - .forgejo/actions/build-release/action.yml + - .forgejo/workflows/build-release.yml + - .forgejo/workflows/build-release-integration.yml + +jobs: + release-simulation: + runs-on: self-hosted + if: secrets.ROLE != 'forgejo-integration' && secrets.ROLE != 'forgejo-experimental' && secrets.ROLE != 'forgejo-release' + steps: + - uses: actions/checkout@v3 + + - id: forgejo + uses: https://code.forgejo.org/actions/setup-forgejo@v1 + with: + user: root + password: admin1234 + image-version: 1.19 + lxc-ip-prefix: 10.0.9 + + - name: publish the forgejo release + run: | + set -x + + cat > /etc/docker/daemon.json < $dir/Dockerfile < /app/gitea/gitea ; chmod +x /app/gitea/gitea + EOF + cp $dir/Dockerfile $dir/Dockerfile.rootless + + forgejo-test-helper.sh push $dir $url root forgejo |& tee $dir/pushed + eval $(grep '^sha=' < $dir/pushed) + + # + # Push a tag to trigger the release workflow and wait for it to complete + # + forgejo-test-helper.sh api POST $url repos/root/forgejo/tags ${{ steps.forgejo.outputs.token }} --data-raw '{"tag_name": "v1.2.3", "target": "'$sha'"}' + LOOPS=180 forgejo-test-helper.sh wait_success "$url" root/forgejo $sha + + # + # uncomment to see the logs even when everything is reported to be working ok + # + cat $FORGEJO_RUNNER_LOGS + + # + # Minimal sanity checks. e2e test is for the setup-forgejo + # action and the infrastructure playbook. Since the binary + # is a script shell it does not test the sanity of the cross + # build, only the sanity of the naming of the binaries. + # + for arch in amd64 arm64 arm-6 ; do + curl -L -sS $url/root/forgejo/releases/download/v1.2.3/forgejo-1.2.3-$arch > forgejo + chmod +x forgejo + ./forgejo --version | grep 1.2.3 + curl -L -sS $url/root/forgejo/releases/download/v1.2.3/forgejo-1.2.3-$arch.sha256 > forgejo.one + shasum -a 256 < forgejo | cut -f1 -d ' ' > forgejo.two + diff forgejo.one forgejo.two + done + docker pull ${{ steps.forgejo.outputs.host-port }}/root/forgejo:1.2.3 + docker pull ${{ steps.forgejo.outputs.host-port }}/root/forgejo:1.2.3-rootless diff --git a/.forgejo/workflows/build-release.yml b/.forgejo/workflows/build-release.yml new file mode 100644 index 0000000000..908f52e5c0 --- /dev/null +++ b/.forgejo/workflows/build-release.yml @@ -0,0 +1,104 @@ +name: Build release + +on: + push: + tags: 'v*' + +jobs: + release: + runs-on: self-hosted + # root is used for testing, allow it + if: secrets.ROLE == 'forgejo-integration' || github.repository_owner == 'root' + steps: + - uses: actions/checkout@v3 + + - id: verbose + run: | + # if there are no secrets, be verbose + if test -z "${{ secrets.TOKEN }}"; then + value=true + else + value=false + fi + echo "value=$value" >> "$GITHUB_OUTPUT" + + - id: repository + run: | + set -x # comment out + repository="${{ github.repository }}" + echo "value=${repository##*/}" >> "$GITHUB_OUTPUT" + + - name: when in a test environment, create a token + id: token + if: ${{ secrets.TOKEN == '' }} + run: | + set -x # comment out + apt-get -qq install -y jq + url="${{ env.GITHUB_SERVER_URL }}" + hostport=${url##http*://} + hostport=${hostport%%/} + doer=root + api=http://$doer:admin1234@$hostport/api/v1/users/$doer/tokens + curl -sS -X DELETE $api/release + token=$(curl -sS -X POST -H 'Content-Type: application/json' --data-raw '{"name": "release", "scopes": ["all"]}' $api | jq --raw-output .sha1) + echo "value=${token}" >> "$GITHUB_OUTPUT" + + - name: build container & release (when TOKEN secret is not set) + if: ${{ secrets.TOKEN == '' }} + uses: ./.forgejo/actions/build-release + with: + forgejo: "${{ env.GITHUB_SERVER_URL }}" + owner: "${{ env.GITHUB_REPOSITORY_OWNER }}" + repository: "${{ steps.repository.outputs.value }}" + doer: root + ref_name: "${{ github.ref_name }}" + token: ${{ steps.token.outputs.value }} + platforms: linux/amd64,linux/arm64,linux/arm/v6 + binary-name: forgejo + binary-path: /app/gitea/gitea + verbose: ${{ steps.verbose.outputs.value }} + + - name: build rootless container (when TOKEN secret is not set) + if: ${{ secrets.TOKEN == '' }} + uses: ./.forgejo/actions/build-release + with: + forgejo: "${{ env.GITHUB_SERVER_URL }}" + owner: "${{ env.GITHUB_REPOSITORY_OWNER }}" + repository: "${{ steps.repository.outputs.value }}" + doer: root + ref_name: "${{ github.ref_name }}" + token: ${{ steps.token.outputs.value }} + platforms: linux/amd64,linux/arm64,linux/arm/v6 + suffix: -rootless + dockerfile: Dockerfile.rootless + verbose: ${{ steps.verbose.outputs.value }} + + - name: build container & release (when TOKEN secret is set) + if: ${{ secrets.TOKEN != '' }} + uses: ./.forgejo/actions/build-release + with: + forgejo: "${{ env.GITHUB_SERVER_URL }}" + owner: "${{ env.GITHUB_REPOSITORY_OWNER }}" + repository: "${{ steps.repository.outputs.value }}" + doer: "${{ secrets.DOER }}" + ref_name: "${{ github.ref_name }}" + token: "${{ secrets.TOKEN }}" + platforms: linux/amd64,linux/arm64,linux/arm/v6 + binary-name: forgejo + binary-path: /app/gitea/gitea + verbose: ${{ steps.verbose.outputs.value }} + + - name: build rootless container (when TOKEN secret is set) + if: ${{ secrets.TOKEN != '' }} + uses: ./.forgejo/actions/build-release + with: + forgejo: "${{ env.GITHUB_SERVER_URL }}" + owner: "${{ env.GITHUB_REPOSITORY_OWNER }}" + repository: "${{ steps.repository.outputs.value }}" + doer: "${{ secrets.DOER }}" + ref_name: "${{ github.ref_name }}" + token: "${{ secrets.TOKEN }}" + platforms: linux/amd64,linux/arm64,linux/arm/v6 + suffix: -rootless + dockerfile: Dockerfile.rootless + verbose: ${{ steps.verbose.outputs.value }} diff --git a/Dockerfile b/Dockerfile index 730413ebb8..5a87a6de92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ -#Build stage -FROM docker.io/library/golang:1.20-alpine3.18 AS build-env +FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx + +FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.20-alpine3.18 as build-env ARG GOPROXY ENV GOPROXY ${GOPROXY:-direct} @@ -9,19 +10,33 @@ ARG TAGS="sqlite sqlite_unlock_notify" ENV TAGS "bindata timetzdata $TAGS" ARG CGO_EXTRA_CFLAGS -#Build deps +# +# Transparently cross compile for the target platform +# +COPY --from=xx / / +ARG TARGETPLATFORM +RUN apk --no-cache add clang lld +RUN xx-apk --no-cache add gcc musl-dev +ENV CGO_ENABLED=1 +RUN xx-go --wrap +# +# for go generate and binfmt to find +# without it the generate phase will fail with +# #19 25.04 modules/public/public_bindata.go:8: running "go": exit status 1 +# #19 25.39 aarch64-binfmt-P: Could not open '/lib/ld-musl-aarch64.so.1': No such file or directory +# why exactly is it needed? where is binfmt involved? +# +RUN cp /*-alpine-linux-musl*/lib/ld-musl-*.so.1 /lib || true + RUN apk --no-cache add build-base git nodejs npm -#Setup repo COPY . ${GOPATH}/src/code.gitea.io/gitea WORKDIR ${GOPATH}/src/code.gitea.io/gitea -#Checkout version if set -RUN if [ -n "${GITEA_VERSION}" ]; then git checkout "${GITEA_VERSION}"; fi \ - && make clean-all build - -# Begin env-to-ini build -RUN go build contrib/environment-to-ini/environment-to-ini.go +RUN make clean-all +RUN make frontend +RUN go build contrib/environment-to-ini/environment-to-ini.go && xx-verify environment-to-ini +RUN make backend && xx-verify gitea FROM docker.io/library/alpine:3.18 LABEL maintainer="contact@forgejo.org" diff --git a/Dockerfile.rootless b/Dockerfile.rootless index 39c13fbb6d..9bcac34411 100644 --- a/Dockerfile.rootless +++ b/Dockerfile.rootless @@ -1,5 +1,6 @@ -#Build stage -FROM docker.io/library/golang:1.20-alpine3.18 AS build-env +FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx + +FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.20-alpine3.18 as build-env ARG GOPROXY ENV GOPROXY ${GOPROXY:-direct} @@ -9,19 +10,33 @@ ARG TAGS="sqlite sqlite_unlock_notify" ENV TAGS "bindata timetzdata $TAGS" ARG CGO_EXTRA_CFLAGS -#Build deps +# +# Transparently cross compile for the target platform +# +COPY --from=xx / / +ARG TARGETPLATFORM +RUN apk --no-cache add clang lld +RUN xx-apk --no-cache add gcc musl-dev +ENV CGO_ENABLED=1 +RUN xx-go --wrap +# +# for go generate and binfmt to find +# without it the generate phase will fail with +# #19 25.04 modules/public/public_bindata.go:8: running "go": exit status 1 +# #19 25.39 aarch64-binfmt-P: Could not open '/lib/ld-musl-aarch64.so.1': No such file or directory +# why exactly is it needed? where is binfmt involved? +# +RUN cp /*-alpine-linux-musl*/lib/ld-musl-*.so.1 /lib || true + RUN apk --no-cache add build-base git nodejs npm -#Setup repo COPY . ${GOPATH}/src/code.gitea.io/gitea WORKDIR ${GOPATH}/src/code.gitea.io/gitea -#Checkout version if set -RUN if [ -n "${GITEA_VERSION}" ]; then git checkout "${GITEA_VERSION}"; fi \ - && make clean-all build - -# Begin env-to-ini build -RUN go build contrib/environment-to-ini/environment-to-ini.go +RUN make clean-all +RUN make frontend +RUN go build contrib/environment-to-ini/environment-to-ini.go && xx-verify environment-to-ini +RUN make backend && xx-verify gitea FROM docker.io/library/alpine:3.18 LABEL maintainer="contact@forgejo.org" diff --git a/Makefile b/Makefile index 5400deea80..fa664b5c57 100644 --- a/Makefile +++ b/Makefile @@ -816,7 +816,7 @@ security-check: go run $(GOVULNCHECK_PACKAGE) ./... $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ) - CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags '$(TAGS)' -ldflags '-s -w $(LDFLAGS)' -o $@ + CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GOFLAGS) $(EXTRA_GOFLAGS) -tags 'netgo osusergo $(TAGS)' -ldflags '-s -w -linkmode external -extldflags "-static" $(LDFLAGS)' -o $@ .PHONY: release release: frontend generate release-linux release-copy release-compress vendor release-sources release-check