Actions Artifacts v4 backend (#28965)
Fixes #28853 Needs both https://gitea.com/gitea/act_runner/pulls/473 and https://gitea.com/gitea/act_runner/pulls/471 on the runner side and patched `actions/upload-artifact@v4` / `actions/download-artifact@v4`, like `christopherhx/gitea-upload-artifact@v4` and `christopherhx/gitea-download-artifact@v4`, to not return errors due to GHES not beeing supported yet. (cherry picked from commit a53d268aca87a281aadc2246541f8749eddcebed)
This commit is contained in:
ChristopherHX
Earl Warren
parent
fcd055d74b
commit
66632c4958
11 changed files with 2034 additions and 29 deletions
|
|
@ -17,3 +17,22 @@
|
||||||
updated: 1683636626
|
updated: 1683636626
|
||||||
need_approval: 0
|
need_approval: 0
|
||||||
approved_by: 0
|
approved_by: 0
|
||||||
|
-
|
||||||
|
id: 792
|
||||||
|
title: "update actions"
|
||||||
|
repo_id: 4
|
||||||
|
owner_id: 1
|
||||||
|
workflow_id: "artifact.yaml"
|
||||||
|
index: 188
|
||||||
|
trigger_user_id: 1
|
||||||
|
ref: "refs/heads/master"
|
||||||
|
commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0"
|
||||||
|
event: "push"
|
||||||
|
is_fork_pull_request: 0
|
||||||
|
status: 1
|
||||||
|
started: 1683636528
|
||||||
|
stopped: 1683636626
|
||||||
|
created: 1683636108
|
||||||
|
updated: 1683636626
|
||||||
|
need_approval: 0
|
||||||
|
approved_by: 0
|
||||||
|
|
|
||||||
|
|
@ -12,3 +12,17 @@
|
||||||
status: 1
|
status: 1
|
||||||
started: 1683636528
|
started: 1683636528
|
||||||
stopped: 1683636626
|
stopped: 1683636626
|
||||||
|
-
|
||||||
|
id: 193
|
||||||
|
run_id: 792
|
||||||
|
repo_id: 4
|
||||||
|
owner_id: 1
|
||||||
|
commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
|
||||||
|
is_fork_pull_request: 0
|
||||||
|
name: job_2
|
||||||
|
attempt: 1
|
||||||
|
job_id: job_2
|
||||||
|
task_id: 48
|
||||||
|
status: 1
|
||||||
|
started: 1683636528
|
||||||
|
stopped: 1683636626
|
||||||
|
|
|
||||||
|
|
@ -18,3 +18,23 @@
|
||||||
log_length: 707
|
log_length: 707
|
||||||
log_size: 90179
|
log_size: 90179
|
||||||
log_expired: 0
|
log_expired: 0
|
||||||
|
-
|
||||||
|
id: 48
|
||||||
|
job_id: 193
|
||||||
|
attempt: 1
|
||||||
|
runner_id: 1
|
||||||
|
status: 6 # 6 is the status code for "running", running task can upload artifacts
|
||||||
|
started: 1683636528
|
||||||
|
stopped: 1683636626
|
||||||
|
repo_id: 4
|
||||||
|
owner_id: 1
|
||||||
|
commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
|
||||||
|
is_fork_pull_request: 0
|
||||||
|
token_hash: ffffcfffffffbffffffffffffffffefffffffafffffffffffffffffffffffffffffdffffffffffffffffffffffffffffffff
|
||||||
|
token_salt: ffffffffff
|
||||||
|
token_last_eight: ffffffff
|
||||||
|
log_filename: artifact-test2/2f/47.log
|
||||||
|
log_in_storage: 1
|
||||||
|
log_length: 707
|
||||||
|
log_size: 90179
|
||||||
|
log_expired: 0
|
||||||
|
|
|
||||||
1058
routers/api/actions/artifact.pb.go
Normal file
1058
routers/api/actions/artifact.pb.go
Normal file
File diff suppressed because it is too large
Load diff
73
routers/api/actions/artifact.proto
Normal file
73
routers/api/actions/artifact.proto
Normal file
|
|
@ -0,0 +1,73 @@
|
||||||
|
syntax = "proto3";
|
||||||
|
|
||||||
|
import "google/protobuf/timestamp.proto";
|
||||||
|
import "google/protobuf/wrappers.proto";
|
||||||
|
|
||||||
|
package github.actions.results.api.v1;
|
||||||
|
|
||||||
|
message CreateArtifactRequest {
|
||||||
|
string workflow_run_backend_id = 1;
|
||||||
|
string workflow_job_run_backend_id = 2;
|
||||||
|
string name = 3;
|
||||||
|
google.protobuf.Timestamp expires_at = 4;
|
||||||
|
int32 version = 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
message CreateArtifactResponse {
|
||||||
|
bool ok = 1;
|
||||||
|
string signed_upload_url = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message FinalizeArtifactRequest {
|
||||||
|
string workflow_run_backend_id = 1;
|
||||||
|
string workflow_job_run_backend_id = 2;
|
||||||
|
string name = 3;
|
||||||
|
int64 size = 4;
|
||||||
|
google.protobuf.StringValue hash = 5;
|
||||||
|
}
|
||||||
|
|
||||||
|
message FinalizeArtifactResponse {
|
||||||
|
bool ok = 1;
|
||||||
|
int64 artifact_id = 2;
|
||||||
|
}
|
||||||
|
|
||||||
|
message ListArtifactsRequest {
|
||||||
|
string workflow_run_backend_id = 1;
|
||||||
|
string workflow_job_run_backend_id = 2;
|
||||||
|
google.protobuf.StringValue name_filter = 3;
|
||||||
|
google.protobuf.Int64Value id_filter = 4;
|
||||||
|
}
|
||||||
|
|
||||||
|
message ListArtifactsResponse {
|
||||||
|
repeated ListArtifactsResponse_MonolithArtifact artifacts = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message ListArtifactsResponse_MonolithArtifact {
|
||||||
|
string workflow_run_backend_id = 1;
|
||||||
|
string workflow_job_run_backend_id = 2;
|
||||||
|
int64 database_id = 3;
|
||||||
|
string name = 4;
|
||||||
|
int64 size = 5;
|
||||||
|
google.protobuf.Timestamp created_at = 6;
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetSignedArtifactURLRequest {
|
||||||
|
string workflow_run_backend_id = 1;
|
||||||
|
string workflow_job_run_backend_id = 2;
|
||||||
|
string name = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetSignedArtifactURLResponse {
|
||||||
|
string signed_url = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DeleteArtifactRequest {
|
||||||
|
string workflow_run_backend_id = 1;
|
||||||
|
string workflow_job_run_backend_id = 2;
|
||||||
|
string name = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
message DeleteArtifactResponse {
|
||||||
|
bool ok = 1;
|
||||||
|
int64 artifact_id = 2;
|
||||||
|
}
|
||||||
|
|
@ -5,11 +5,16 @@ package actions
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/md5"
|
"crypto/md5"
|
||||||
|
"crypto/sha256"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"hash"
|
||||||
"io"
|
"io"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"code.gitea.io/gitea/models/actions"
|
"code.gitea.io/gitea/models/actions"
|
||||||
|
|
@ -18,6 +23,52 @@ import (
|
||||||
"code.gitea.io/gitea/modules/storage"
|
"code.gitea.io/gitea/modules/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func saveUploadChunkBase(st storage.ObjectStorage, ctx *ArtifactContext,
|
||||||
|
artifact *actions.ActionArtifact,
|
||||||
|
contentSize, runID, start, end, length int64, checkMd5 bool,
|
||||||
|
) (int64, error) {
|
||||||
|
// build chunk store path
|
||||||
|
storagePath := fmt.Sprintf("tmp%d/%d-%d-%d-%d.chunk", runID, runID, artifact.ID, start, end)
|
||||||
|
var r io.Reader = ctx.Req.Body
|
||||||
|
var hasher hash.Hash
|
||||||
|
if checkMd5 {
|
||||||
|
// use io.TeeReader to avoid reading all body to md5 sum.
|
||||||
|
// it writes data to hasher after reading end
|
||||||
|
// if hash is not matched, delete the read-end result
|
||||||
|
hasher = md5.New()
|
||||||
|
r = io.TeeReader(r, hasher)
|
||||||
|
}
|
||||||
|
// save chunk to storage
|
||||||
|
writtenSize, err := st.Save(storagePath, r, -1)
|
||||||
|
if err != nil {
|
||||||
|
return -1, fmt.Errorf("save chunk to storage error: %v", err)
|
||||||
|
}
|
||||||
|
var checkErr error
|
||||||
|
if checkMd5 {
|
||||||
|
// check md5
|
||||||
|
reqMd5String := ctx.Req.Header.Get(artifactXActionsResultsMD5Header)
|
||||||
|
chunkMd5String := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
|
||||||
|
log.Info("[artifact] check chunk md5, sum: %s, header: %s", chunkMd5String, reqMd5String)
|
||||||
|
// if md5 not match, delete the chunk
|
||||||
|
if reqMd5String != chunkMd5String {
|
||||||
|
checkErr = fmt.Errorf("md5 not match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if writtenSize != contentSize {
|
||||||
|
checkErr = errors.Join(checkErr, fmt.Errorf("contentSize not match body size"))
|
||||||
|
}
|
||||||
|
if checkErr != nil {
|
||||||
|
if err := st.Delete(storagePath); err != nil {
|
||||||
|
log.Error("Error deleting chunk: %s, %v", storagePath, err)
|
||||||
|
}
|
||||||
|
return -1, checkErr
|
||||||
|
}
|
||||||
|
log.Info("[artifact] save chunk %s, size: %d, artifact id: %d, start: %d, end: %d",
|
||||||
|
storagePath, contentSize, artifact.ID, start, end)
|
||||||
|
// return chunk total size
|
||||||
|
return length, nil
|
||||||
|
}
|
||||||
|
|
||||||
func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
|
func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
|
||||||
artifact *actions.ActionArtifact,
|
artifact *actions.ActionArtifact,
|
||||||
contentSize, runID int64,
|
contentSize, runID int64,
|
||||||
|
|
@ -29,33 +80,15 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
|
||||||
log.Warn("parse content range error: %v, content-range: %s", err, contentRange)
|
log.Warn("parse content range error: %v, content-range: %s", err, contentRange)
|
||||||
return -1, fmt.Errorf("parse content range error: %v", err)
|
return -1, fmt.Errorf("parse content range error: %v", err)
|
||||||
}
|
}
|
||||||
// build chunk store path
|
return saveUploadChunkBase(st, ctx, artifact, contentSize, runID, start, end, length, true)
|
||||||
storagePath := fmt.Sprintf("tmp%d/%d-%d-%d-%d.chunk", runID, runID, artifact.ID, start, end)
|
|
||||||
// use io.TeeReader to avoid reading all body to md5 sum.
|
|
||||||
// it writes data to hasher after reading end
|
|
||||||
// if hash is not matched, delete the read-end result
|
|
||||||
hasher := md5.New()
|
|
||||||
r := io.TeeReader(ctx.Req.Body, hasher)
|
|
||||||
// save chunk to storage
|
|
||||||
writtenSize, err := st.Save(storagePath, r, -1)
|
|
||||||
if err != nil {
|
|
||||||
return -1, fmt.Errorf("save chunk to storage error: %v", err)
|
|
||||||
}
|
}
|
||||||
// check md5
|
|
||||||
reqMd5String := ctx.Req.Header.Get(artifactXActionsResultsMD5Header)
|
func appendUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
|
||||||
chunkMd5String := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
|
artifact *actions.ActionArtifact,
|
||||||
log.Info("[artifact] check chunk md5, sum: %s, header: %s", chunkMd5String, reqMd5String)
|
start, contentSize, runID int64,
|
||||||
// if md5 not match, delete the chunk
|
) (int64, error) {
|
||||||
if reqMd5String != chunkMd5String || writtenSize != contentSize {
|
end := start + contentSize - 1
|
||||||
if err := st.Delete(storagePath); err != nil {
|
return saveUploadChunkBase(st, ctx, artifact, contentSize, runID, start, end, contentSize, false)
|
||||||
log.Error("Error deleting chunk: %s, %v", storagePath, err)
|
|
||||||
}
|
|
||||||
return -1, fmt.Errorf("md5 not match")
|
|
||||||
}
|
|
||||||
log.Info("[artifact] save chunk %s, size: %d, artifact id: %d, start: %d, end: %d",
|
|
||||||
storagePath, contentSize, artifact.ID, start, end)
|
|
||||||
// return chunk total size
|
|
||||||
return length, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type chunkFileItem struct {
|
type chunkFileItem struct {
|
||||||
|
|
@ -111,14 +144,14 @@ func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID int
|
||||||
log.Debug("artifact %d chunks not found", art.ID)
|
log.Debug("artifact %d chunks not found", art.ID)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if err := mergeChunksForArtifact(ctx, chunks, st, art); err != nil {
|
if err := mergeChunksForArtifact(ctx, chunks, st, art, ""); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st storage.ObjectStorage, artifact *actions.ActionArtifact) error {
|
func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st storage.ObjectStorage, artifact *actions.ActionArtifact, checksum string) error {
|
||||||
sort.Slice(chunks, func(i, j int) bool {
|
sort.Slice(chunks, func(i, j int) bool {
|
||||||
return chunks[i].Start < chunks[j].Start
|
return chunks[i].Start < chunks[j].Start
|
||||||
})
|
})
|
||||||
|
|
@ -157,6 +190,14 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
|
||||||
readers = append(readers, readCloser)
|
readers = append(readers, readCloser)
|
||||||
}
|
}
|
||||||
mergedReader := io.MultiReader(readers...)
|
mergedReader := io.MultiReader(readers...)
|
||||||
|
shaPrefix := "sha256:"
|
||||||
|
var hash hash.Hash
|
||||||
|
if strings.HasPrefix(checksum, shaPrefix) {
|
||||||
|
hash = sha256.New()
|
||||||
|
}
|
||||||
|
if hash != nil {
|
||||||
|
mergedReader = io.TeeReader(mergedReader, hash)
|
||||||
|
}
|
||||||
|
|
||||||
// if chunk is gzip, use gz as extension
|
// if chunk is gzip, use gz as extension
|
||||||
// download-artifact action will use content-encoding header to decide if it should decompress the file
|
// download-artifact action will use content-encoding header to decide if it should decompress the file
|
||||||
|
|
@ -185,6 +226,14 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
if hash != nil {
|
||||||
|
rawChecksum := hash.Sum(nil)
|
||||||
|
actualChecksum := hex.EncodeToString(rawChecksum)
|
||||||
|
if !strings.HasSuffix(checksum, actualChecksum) {
|
||||||
|
return fmt.Errorf("update artifact error checksum is invalid")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// save storage path to artifact
|
// save storage path to artifact
|
||||||
log.Debug("[artifact] merge chunks to artifact: %d, %s, old:%s", artifact.ID, storagePath, artifact.StoragePath)
|
log.Debug("[artifact] merge chunks to artifact: %d, %s, old:%s", artifact.ID, storagePath, artifact.StoragePath)
|
||||||
// if artifact is already uploaded, delete the old file
|
// if artifact is already uploaded, delete the old file
|
||||||
|
|
|
||||||
|
|
@ -43,6 +43,17 @@ func validateRunID(ctx *ArtifactContext) (*actions.ActionTask, int64, bool) {
|
||||||
return task, runID, true
|
return task, runID, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateRunIDV4(ctx *ArtifactContext, rawRunID string) (*actions.ActionTask, int64, bool) {
|
||||||
|
task := ctx.ActionTask
|
||||||
|
runID, err := strconv.ParseInt(rawRunID, 10, 64)
|
||||||
|
if err != nil || task.Job.RunID != runID {
|
||||||
|
log.Error("Error runID not match")
|
||||||
|
ctx.Error(http.StatusBadRequest, "run-id does not match")
|
||||||
|
return nil, 0, false
|
||||||
|
}
|
||||||
|
return task, runID, true
|
||||||
|
}
|
||||||
|
|
||||||
func validateArtifactHash(ctx *ArtifactContext, artifactName string) bool {
|
func validateArtifactHash(ctx *ArtifactContext, artifactName string) bool {
|
||||||
paramHash := ctx.Params("artifact_hash")
|
paramHash := ctx.Params("artifact_hash")
|
||||||
// use artifact name to create upload url
|
// use artifact name to create upload url
|
||||||
|
|
|
||||||
512
routers/api/actions/artifactsv4.go
Normal file
512
routers/api/actions/artifactsv4.go
Normal file
|
|
@ -0,0 +1,512 @@
|
||||||
|
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package actions
|
||||||
|
|
||||||
|
// GitHub Actions Artifacts V4 API Simple Description
|
||||||
|
//
|
||||||
|
// 1. Upload artifact
|
||||||
|
// 1.1. CreateArtifact
|
||||||
|
// Post: /twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact
|
||||||
|
// Request:
|
||||||
|
// {
|
||||||
|
// "workflow_run_backend_id": "21",
|
||||||
|
// "workflow_job_run_backend_id": "49",
|
||||||
|
// "name": "test",
|
||||||
|
// "version": 4
|
||||||
|
// }
|
||||||
|
// Response:
|
||||||
|
// {
|
||||||
|
// "ok": true,
|
||||||
|
// "signedUploadUrl": "http://localhost:3000/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?sig=mO7y35r4GyjN7fwg0DTv3-Fv1NDXD84KLEgLpoPOtDI=&expires=2024-01-23+21%3A48%3A37.20833956+%2B0100+CET&artifactName=test&taskID=75"
|
||||||
|
// }
|
||||||
|
// 1.2. Upload Zip Content to Blobstorage (unauthenticated request)
|
||||||
|
// PUT: http://localhost:3000/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?sig=mO7y35r4GyjN7fwg0DTv3-Fv1NDXD84KLEgLpoPOtDI=&expires=2024-01-23+21%3A48%3A37.20833956+%2B0100+CET&artifactName=test&taskID=75&comp=block
|
||||||
|
// 1.3. Continue Upload Zip Content to Blobstorage (unauthenticated request), repeat until everything is uploaded
|
||||||
|
// PUT: http://localhost:3000/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?sig=mO7y35r4GyjN7fwg0DTv3-Fv1NDXD84KLEgLpoPOtDI=&expires=2024-01-23+21%3A48%3A37.20833956+%2B0100+CET&artifactName=test&taskID=75&comp=appendBlock
|
||||||
|
// 1.4. Unknown xml payload to Blobstorage (unauthenticated request), ignored for now
|
||||||
|
// PUT: http://localhost:3000/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?sig=mO7y35r4GyjN7fwg0DTv3-Fv1NDXD84KLEgLpoPOtDI=&expires=2024-01-23+21%3A48%3A37.20833956+%2B0100+CET&artifactName=test&taskID=75&comp=blockList
|
||||||
|
// 1.5. FinalizeArtifact
|
||||||
|
// Post: /twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact
|
||||||
|
// Request
|
||||||
|
// {
|
||||||
|
// "workflow_run_backend_id": "21",
|
||||||
|
// "workflow_job_run_backend_id": "49",
|
||||||
|
// "name": "test",
|
||||||
|
// "size": "2097",
|
||||||
|
// "hash": "sha256:b6325614d5649338b87215d9536b3c0477729b8638994c74cdefacb020a2cad4"
|
||||||
|
// }
|
||||||
|
// Response
|
||||||
|
// {
|
||||||
|
// "ok": true,
|
||||||
|
// "artifactId": "4"
|
||||||
|
// }
|
||||||
|
// 2. Download artifact
|
||||||
|
// 2.1. ListArtifacts and optionally filter by artifact exact name or id
|
||||||
|
// Post: /twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts
|
||||||
|
// Request
|
||||||
|
// {
|
||||||
|
// "workflow_run_backend_id": "21",
|
||||||
|
// "workflow_job_run_backend_id": "49",
|
||||||
|
// "name_filter": "test"
|
||||||
|
// }
|
||||||
|
// Response
|
||||||
|
// {
|
||||||
|
// "artifacts": [
|
||||||
|
// {
|
||||||
|
// "workflowRunBackendId": "21",
|
||||||
|
// "workflowJobRunBackendId": "49",
|
||||||
|
// "databaseId": "4",
|
||||||
|
// "name": "test",
|
||||||
|
// "size": "2093",
|
||||||
|
// "createdAt": "2024-01-23T00:13:28Z"
|
||||||
|
// }
|
||||||
|
// ]
|
||||||
|
// }
|
||||||
|
// 2.2. GetSignedArtifactURL get the URL to download the artifact zip file of a specific artifact
|
||||||
|
// Post: /twirp/github.actions.results.api.v1.ArtifactService/GetSignedArtifactURL
|
||||||
|
// Request
|
||||||
|
// {
|
||||||
|
// "workflow_run_backend_id": "21",
|
||||||
|
// "workflow_job_run_backend_id": "49",
|
||||||
|
// "name": "test"
|
||||||
|
// }
|
||||||
|
// Response
|
||||||
|
// {
|
||||||
|
// "signedUrl": "http://localhost:3000/twirp/github.actions.results.api.v1.ArtifactService/DownloadArtifact?sig=wHzFOwpF-6220-5CA0CIRmAX9VbiTC2Mji89UOqo1E8=&expires=2024-01-23+21%3A51%3A56.872846295+%2B0100+CET&artifactName=test&taskID=76"
|
||||||
|
// }
|
||||||
|
// 2.3. Download Zip from Blobstorage (unauthenticated request)
|
||||||
|
// GET: http://localhost:3000/twirp/github.actions.results.api.v1.ArtifactService/DownloadArtifact?sig=wHzFOwpF-6220-5CA0CIRmAX9VbiTC2Mji89UOqo1E8=&expires=2024-01-23+21%3A51%3A56.872846295+%2B0100+CET&artifactName=test&taskID=76
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/models/actions"
|
||||||
|
"code.gitea.io/gitea/models/db"
|
||||||
|
"code.gitea.io/gitea/modules/log"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
|
"code.gitea.io/gitea/modules/storage"
|
||||||
|
"code.gitea.io/gitea/modules/util"
|
||||||
|
"code.gitea.io/gitea/modules/web"
|
||||||
|
"code.gitea.io/gitea/services/context"
|
||||||
|
|
||||||
|
"google.golang.org/protobuf/encoding/protojson"
|
||||||
|
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
||||||
|
"google.golang.org/protobuf/types/known/timestamppb"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ArtifactV4RouteBase = "/twirp/github.actions.results.api.v1.ArtifactService"
|
||||||
|
ArtifactV4ContentEncoding = "application/zip"
|
||||||
|
)
|
||||||
|
|
||||||
|
type artifactV4Routes struct {
|
||||||
|
prefix string
|
||||||
|
fs storage.ObjectStorage
|
||||||
|
}
|
||||||
|
|
||||||
|
func ArtifactV4Contexter() func(next http.Handler) http.Handler {
|
||||||
|
return func(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
|
||||||
|
base, baseCleanUp := context.NewBaseContext(resp, req)
|
||||||
|
defer baseCleanUp()
|
||||||
|
|
||||||
|
ctx := &ArtifactContext{Base: base}
|
||||||
|
ctx.AppendContextValue(artifactContextKey, ctx)
|
||||||
|
|
||||||
|
next.ServeHTTP(ctx.Resp, ctx.Req)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ArtifactsV4Routes(prefix string) *web.Route {
|
||||||
|
m := web.NewRoute()
|
||||||
|
|
||||||
|
r := artifactV4Routes{
|
||||||
|
prefix: prefix,
|
||||||
|
fs: storage.ActionsArtifacts,
|
||||||
|
}
|
||||||
|
|
||||||
|
m.Group("", func() {
|
||||||
|
m.Post("CreateArtifact", r.createArtifact)
|
||||||
|
m.Post("FinalizeArtifact", r.finalizeArtifact)
|
||||||
|
m.Post("ListArtifacts", r.listArtifacts)
|
||||||
|
m.Post("GetSignedArtifactURL", r.getSignedArtifactURL)
|
||||||
|
m.Post("DeleteArtifact", r.deleteArtifact)
|
||||||
|
}, ArtifactContexter())
|
||||||
|
m.Group("", func() {
|
||||||
|
m.Put("UploadArtifact", r.uploadArtifact)
|
||||||
|
m.Get("DownloadArtifact", r.downloadArtifact)
|
||||||
|
}, ArtifactV4Contexter())
|
||||||
|
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r artifactV4Routes) buildSignature(endp, expires, artifactName string, taskID int64) []byte {
|
||||||
|
mac := hmac.New(sha256.New, setting.GetGeneralTokenSigningSecret())
|
||||||
|
mac.Write([]byte(endp))
|
||||||
|
mac.Write([]byte(expires))
|
||||||
|
mac.Write([]byte(artifactName))
|
||||||
|
mac.Write([]byte(fmt.Sprint(taskID)))
|
||||||
|
return mac.Sum(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r artifactV4Routes) buildArtifactURL(endp, artifactName string, taskID int64) string {
|
||||||
|
expires := time.Now().Add(60 * time.Minute).Format("2006-01-02 15:04:05.999999999 -0700 MST")
|
||||||
|
uploadURL := strings.TrimSuffix(setting.AppURL, "/") + strings.TrimSuffix(r.prefix, "/") +
|
||||||
|
"/" + endp + "?sig=" + base64.URLEncoding.EncodeToString(r.buildSignature(endp, expires, artifactName, taskID)) + "&expires=" + url.QueryEscape(expires) + "&artifactName=" + url.QueryEscape(artifactName) + "&taskID=" + fmt.Sprint(taskID)
|
||||||
|
return uploadURL
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r artifactV4Routes) verifySignature(ctx *ArtifactContext, endp string) (*actions.ActionTask, string, bool) {
|
||||||
|
rawTaskID := ctx.Req.URL.Query().Get("taskID")
|
||||||
|
sig := ctx.Req.URL.Query().Get("sig")
|
||||||
|
expires := ctx.Req.URL.Query().Get("expires")
|
||||||
|
artifactName := ctx.Req.URL.Query().Get("artifactName")
|
||||||
|
dsig, _ := base64.URLEncoding.DecodeString(sig)
|
||||||
|
taskID, _ := strconv.ParseInt(rawTaskID, 10, 64)
|
||||||
|
|
||||||
|
expecedsig := r.buildSignature(endp, expires, artifactName, taskID)
|
||||||
|
if !hmac.Equal(dsig, expecedsig) {
|
||||||
|
log.Error("Error unauthorized")
|
||||||
|
ctx.Error(http.StatusUnauthorized, "Error unauthorized")
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
t, err := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", expires)
|
||||||
|
if err != nil || t.Before(time.Now()) {
|
||||||
|
log.Error("Error link expired")
|
||||||
|
ctx.Error(http.StatusUnauthorized, "Error link expired")
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
task, err := actions.GetTaskByID(ctx, taskID)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error runner api getting task by ID: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error runner api getting task by ID")
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
if task.Status != actions.StatusRunning {
|
||||||
|
log.Error("Error runner api getting task: task is not running")
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error runner api getting task: task is not running")
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
if err := task.LoadJob(ctx); err != nil {
|
||||||
|
log.Error("Error runner api getting job: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error runner api getting job")
|
||||||
|
return nil, "", false
|
||||||
|
}
|
||||||
|
return task, artifactName, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) getArtifactByName(ctx *ArtifactContext, runID int64, name string) (*actions.ActionArtifact, error) {
|
||||||
|
var art actions.ActionArtifact
|
||||||
|
has, err := db.GetEngine(ctx).Where("run_id = ? AND artifact_name = ? AND artifact_path = ? AND content_encoding = ?", runID, name, name+".zip", ArtifactV4ContentEncoding).Get(&art)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else if !has {
|
||||||
|
return nil, util.ErrNotExist
|
||||||
|
}
|
||||||
|
return &art, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) parseProtbufBody(ctx *ArtifactContext, req protoreflect.ProtoMessage) bool {
|
||||||
|
body, err := io.ReadAll(ctx.Req.Body)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error decode request body: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error decode request body")
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
err = protojson.Unmarshal(body, req)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error decode request body: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error decode request body")
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) sendProtbufBody(ctx *ArtifactContext, req protoreflect.ProtoMessage) {
|
||||||
|
resp, err := protojson.Marshal(req)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error encode response body: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error encode response body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.Resp.Header().Set("Content-Type", "application/json;charset=utf-8")
|
||||||
|
ctx.Resp.WriteHeader(http.StatusOK)
|
||||||
|
_, _ = ctx.Resp.Write(resp)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) createArtifact(ctx *ArtifactContext) {
|
||||||
|
var req CreateArtifactRequest
|
||||||
|
|
||||||
|
if ok := r.parseProtbufBody(ctx, &req); !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
artifactName := req.Name
|
||||||
|
|
||||||
|
rententionDays := setting.Actions.ArtifactRetentionDays
|
||||||
|
if req.ExpiresAt != nil {
|
||||||
|
rententionDays = int64(time.Until(req.ExpiresAt.AsTime()).Hours() / 24)
|
||||||
|
}
|
||||||
|
// create or get artifact with name and path
|
||||||
|
artifact, err := actions.CreateArtifact(ctx, ctx.ActionTask, artifactName, artifactName+".zip", rententionDays)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error create or get artifact: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error create or get artifact")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
artifact.ContentEncoding = ArtifactV4ContentEncoding
|
||||||
|
if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
|
||||||
|
log.Error("Error UpdateArtifactByID: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error UpdateArtifactByID")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
respData := CreateArtifactResponse{
|
||||||
|
Ok: true,
|
||||||
|
SignedUploadUrl: r.buildArtifactURL("UploadArtifact", artifactName, ctx.ActionTask.ID),
|
||||||
|
}
|
||||||
|
r.sendProtbufBody(ctx, &respData)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) uploadArtifact(ctx *ArtifactContext) {
|
||||||
|
task, artifactName, ok := r.verifySignature(ctx, "UploadArtifact")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
comp := ctx.Req.URL.Query().Get("comp")
|
||||||
|
switch comp {
|
||||||
|
case "block", "appendBlock":
|
||||||
|
// get artifact by name
|
||||||
|
artifact, err := r.getArtifactByName(ctx, task.Job.RunID, artifactName)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error artifact not found: %v", err)
|
||||||
|
ctx.Error(http.StatusNotFound, "Error artifact not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if comp == "block" {
|
||||||
|
artifact.FileSize = 0
|
||||||
|
artifact.FileCompressedSize = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = appendUploadChunk(r.fs, ctx, artifact, artifact.FileSize, ctx.Req.ContentLength, artifact.RunID)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error runner api getting task: task is not running")
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error runner api getting task: task is not running")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
artifact.FileCompressedSize += ctx.Req.ContentLength
|
||||||
|
artifact.FileSize += ctx.Req.ContentLength
|
||||||
|
if err := actions.UpdateArtifactByID(ctx, artifact.ID, artifact); err != nil {
|
||||||
|
log.Error("Error UpdateArtifactByID: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error UpdateArtifactByID")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx.JSON(http.StatusCreated, "appended")
|
||||||
|
case "blocklist":
|
||||||
|
ctx.JSON(http.StatusCreated, "created")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) finalizeArtifact(ctx *ArtifactContext) {
|
||||||
|
var req FinalizeArtifactRequest
|
||||||
|
|
||||||
|
if ok := r.parseProtbufBody(ctx, &req); !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// get artifact by name
|
||||||
|
artifact, err := r.getArtifactByName(ctx, runID, req.Name)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error artifact not found: %v", err)
|
||||||
|
ctx.Error(http.StatusNotFound, "Error artifact not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
chunkMap, err := listChunksByRunID(r.fs, runID)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error merge chunks: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error merge chunks")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
chunks, ok := chunkMap[artifact.ID]
|
||||||
|
if !ok {
|
||||||
|
log.Error("Error merge chunks")
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error merge chunks")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
checksum := ""
|
||||||
|
if req.Hash != nil {
|
||||||
|
checksum = req.Hash.Value
|
||||||
|
}
|
||||||
|
if err := mergeChunksForArtifact(ctx, chunks, r.fs, artifact, checksum); err != nil {
|
||||||
|
log.Error("Error merge chunks: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, "Error merge chunks")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
respData := FinalizeArtifactResponse{
|
||||||
|
Ok: true,
|
||||||
|
ArtifactId: artifact.ID,
|
||||||
|
}
|
||||||
|
r.sendProtbufBody(ctx, &respData)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) listArtifacts(ctx *ArtifactContext) {
|
||||||
|
var req ListArtifactsRequest
|
||||||
|
|
||||||
|
if ok := r.parseProtbufBody(ctx, &req); !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
artifacts, err := db.Find[actions.ActionArtifact](ctx, actions.FindArtifactsOptions{RunID: runID})
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error getting artifacts: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(artifacts) == 0 {
|
||||||
|
log.Debug("[artifact] handleListArtifacts, no artifacts")
|
||||||
|
ctx.Error(http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
list := []*ListArtifactsResponse_MonolithArtifact{}
|
||||||
|
|
||||||
|
table := map[string]*ListArtifactsResponse_MonolithArtifact{}
|
||||||
|
for _, artifact := range artifacts {
|
||||||
|
if _, ok := table[artifact.ArtifactName]; ok || req.IdFilter != nil && artifact.ID != req.IdFilter.Value || req.NameFilter != nil && artifact.ArtifactName != req.NameFilter.Value || artifact.ArtifactName+".zip" != artifact.ArtifactPath || artifact.ContentEncoding != ArtifactV4ContentEncoding {
|
||||||
|
table[artifact.ArtifactName] = nil
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
table[artifact.ArtifactName] = &ListArtifactsResponse_MonolithArtifact{
|
||||||
|
Name: artifact.ArtifactName,
|
||||||
|
CreatedAt: timestamppb.New(artifact.CreatedUnix.AsTime()),
|
||||||
|
DatabaseId: artifact.ID,
|
||||||
|
WorkflowRunBackendId: req.WorkflowRunBackendId,
|
||||||
|
WorkflowJobRunBackendId: req.WorkflowJobRunBackendId,
|
||||||
|
Size: artifact.FileSize,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, artifact := range table {
|
||||||
|
if artifact != nil {
|
||||||
|
list = append(list, artifact)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
respData := ListArtifactsResponse{
|
||||||
|
Artifacts: list,
|
||||||
|
}
|
||||||
|
r.sendProtbufBody(ctx, &respData)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) getSignedArtifactURL(ctx *ArtifactContext) {
|
||||||
|
var req GetSignedArtifactURLRequest
|
||||||
|
|
||||||
|
if ok := r.parseProtbufBody(ctx, &req); !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
artifactName := req.Name
|
||||||
|
|
||||||
|
// get artifact by name
|
||||||
|
artifact, err := r.getArtifactByName(ctx, runID, artifactName)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error artifact not found: %v", err)
|
||||||
|
ctx.Error(http.StatusNotFound, "Error artifact not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
respData := GetSignedArtifactURLResponse{}
|
||||||
|
|
||||||
|
if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
|
||||||
|
u, err := storage.ActionsArtifacts.URL(artifact.StoragePath, artifact.ArtifactPath)
|
||||||
|
if u != nil && err == nil {
|
||||||
|
respData.SignedUrl = u.String()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if respData.SignedUrl == "" {
|
||||||
|
respData.SignedUrl = r.buildArtifactURL("DownloadArtifact", artifactName, ctx.ActionTask.ID)
|
||||||
|
}
|
||||||
|
r.sendProtbufBody(ctx, &respData)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) downloadArtifact(ctx *ArtifactContext) {
|
||||||
|
task, artifactName, ok := r.verifySignature(ctx, "DownloadArtifact")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// get artifact by name
|
||||||
|
artifact, err := r.getArtifactByName(ctx, task.Job.RunID, artifactName)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error artifact not found: %v", err)
|
||||||
|
ctx.Error(http.StatusNotFound, "Error artifact not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
file, _ := r.fs.Open(artifact.StoragePath)
|
||||||
|
|
||||||
|
_, _ = io.Copy(ctx.Resp, file)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *artifactV4Routes) deleteArtifact(ctx *ArtifactContext) {
|
||||||
|
var req DeleteArtifactRequest
|
||||||
|
|
||||||
|
if ok := r.parseProtbufBody(ctx, &req); !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, runID, ok := validateRunIDV4(ctx, req.WorkflowRunBackendId)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// get artifact by name
|
||||||
|
artifact, err := r.getArtifactByName(ctx, runID, req.Name)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error artifact not found: %v", err)
|
||||||
|
ctx.Error(http.StatusNotFound, "Error artifact not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
err = actions.SetArtifactNeedDelete(ctx, runID, req.Name)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Error deleting artifacts: %v", err)
|
||||||
|
ctx.Error(http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
respData := DeleteArtifactResponse{
|
||||||
|
Ok: true,
|
||||||
|
ArtifactId: artifact.ID,
|
||||||
|
}
|
||||||
|
r.sendProtbufBody(ctx, &respData)
|
||||||
|
}
|
||||||
|
|
@ -200,6 +200,8 @@ func NormalRoutes() *web.Route {
|
||||||
// TODO: this prefix should be generated with a token string with runner ?
|
// TODO: this prefix should be generated with a token string with runner ?
|
||||||
prefix = "/api/actions_pipeline"
|
prefix = "/api/actions_pipeline"
|
||||||
r.Mount(prefix, actions_router.ArtifactsRoutes(prefix))
|
r.Mount(prefix, actions_router.ArtifactsRoutes(prefix))
|
||||||
|
prefix = actions_router.ArtifactV4RouteBase
|
||||||
|
r.Mount(prefix, actions_router.ArtifactsV4Routes(prefix))
|
||||||
}
|
}
|
||||||
|
|
||||||
return r
|
return r
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ import (
|
||||||
"code.gitea.io/gitea/models/unit"
|
"code.gitea.io/gitea/models/unit"
|
||||||
"code.gitea.io/gitea/modules/actions"
|
"code.gitea.io/gitea/modules/actions"
|
||||||
"code.gitea.io/gitea/modules/base"
|
"code.gitea.io/gitea/modules/base"
|
||||||
|
"code.gitea.io/gitea/modules/setting"
|
||||||
"code.gitea.io/gitea/modules/storage"
|
"code.gitea.io/gitea/modules/storage"
|
||||||
"code.gitea.io/gitea/modules/timeutil"
|
"code.gitea.io/gitea/modules/timeutil"
|
||||||
"code.gitea.io/gitea/modules/util"
|
"code.gitea.io/gitea/modules/util"
|
||||||
|
|
@ -647,6 +648,28 @@ func ArtifactsDownloadView(ctx *context_module.Context) {
|
||||||
|
|
||||||
ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s.zip; filename*=UTF-8''%s.zip", url.PathEscape(artifactName), artifactName))
|
ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s.zip; filename*=UTF-8''%s.zip", url.PathEscape(artifactName), artifactName))
|
||||||
|
|
||||||
|
// Artifacts using the v4 backend are stored as a single combined zip file per artifact on the backend
|
||||||
|
// The v4 backend enshures ContentEncoding is set to "application/zip", which is not the case for the old backend
|
||||||
|
if len(artifacts) == 1 && artifacts[0].ArtifactName+".zip" == artifacts[0].ArtifactPath && artifacts[0].ContentEncoding == "application/zip" {
|
||||||
|
art := artifacts[0]
|
||||||
|
if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
|
||||||
|
u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath)
|
||||||
|
if u != nil && err == nil {
|
||||||
|
ctx.Redirect(u.String())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f, err := storage.ActionsArtifacts.Open(art.StoragePath)
|
||||||
|
if err != nil {
|
||||||
|
ctx.Error(http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = io.Copy(ctx.Resp, f)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Artifacts using the v1-v3 backend are stored as multiple individual files per artifact on the backend
|
||||||
|
// Those need to be zipped for download
|
||||||
writer := zip.NewWriter(ctx.Resp)
|
writer := zip.NewWriter(ctx.Resp)
|
||||||
defer writer.Close()
|
defer writer.Close()
|
||||||
for _, art := range artifacts {
|
for _, art := range artifacts {
|
||||||
|
|
|
||||||
224
tests/integration/api_actions_artifact_v4_test.go
Normal file
224
tests/integration/api_actions_artifact_v4_test.go
Normal file
|
|
@ -0,0 +1,224 @@
|
||||||
|
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||||
|
// SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
|
package integration
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"code.gitea.io/gitea/routers/api/actions"
|
||||||
|
actions_service "code.gitea.io/gitea/services/actions"
|
||||||
|
"code.gitea.io/gitea/tests"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"google.golang.org/protobuf/encoding/protojson"
|
||||||
|
"google.golang.org/protobuf/reflect/protoreflect"
|
||||||
|
"google.golang.org/protobuf/types/known/timestamppb"
|
||||||
|
"google.golang.org/protobuf/types/known/wrapperspb"
|
||||||
|
)
|
||||||
|
|
||||||
|
func toProtoJSON(m protoreflect.ProtoMessage) io.Reader {
|
||||||
|
resp, _ := protojson.Marshal(m)
|
||||||
|
buf := bytes.Buffer{}
|
||||||
|
buf.Write(resp)
|
||||||
|
return &buf
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActionsArtifactV4UploadSingleFile(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
|
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// acquire artifact upload url
|
||||||
|
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
|
||||||
|
Version: 4,
|
||||||
|
Name: "artifact",
|
||||||
|
WorkflowRunBackendId: "792",
|
||||||
|
WorkflowJobRunBackendId: "193",
|
||||||
|
})).AddTokenAuth(token)
|
||||||
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
|
var uploadResp actions.CreateArtifactResponse
|
||||||
|
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
|
||||||
|
assert.True(t, uploadResp.Ok)
|
||||||
|
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
|
||||||
|
|
||||||
|
// get upload url
|
||||||
|
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
|
||||||
|
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block"
|
||||||
|
|
||||||
|
// upload artifact chunk
|
||||||
|
body := strings.Repeat("A", 1024)
|
||||||
|
req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
|
||||||
|
MakeRequest(t, req, http.StatusCreated)
|
||||||
|
|
||||||
|
t.Logf("Create artifact confirm")
|
||||||
|
|
||||||
|
sha := sha256.Sum256([]byte(body))
|
||||||
|
|
||||||
|
// confirm artifact upload
|
||||||
|
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
|
||||||
|
Name: "artifact",
|
||||||
|
Size: 1024,
|
||||||
|
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha[:])),
|
||||||
|
WorkflowRunBackendId: "792",
|
||||||
|
WorkflowJobRunBackendId: "193",
|
||||||
|
})).
|
||||||
|
AddTokenAuth(token)
|
||||||
|
resp = MakeRequest(t, req, http.StatusOK)
|
||||||
|
var finalizeResp actions.FinalizeArtifactResponse
|
||||||
|
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
|
||||||
|
assert.True(t, finalizeResp.Ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActionsArtifactV4UploadSingleFileWrongChecksum(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
|
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// acquire artifact upload url
|
||||||
|
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
|
||||||
|
Version: 4,
|
||||||
|
Name: "artifact-invalid-checksum",
|
||||||
|
WorkflowRunBackendId: "792",
|
||||||
|
WorkflowJobRunBackendId: "193",
|
||||||
|
})).AddTokenAuth(token)
|
||||||
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
|
var uploadResp actions.CreateArtifactResponse
|
||||||
|
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
|
||||||
|
assert.True(t, uploadResp.Ok)
|
||||||
|
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
|
||||||
|
|
||||||
|
// get upload url
|
||||||
|
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
|
||||||
|
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block"
|
||||||
|
|
||||||
|
// upload artifact chunk
|
||||||
|
body := strings.Repeat("B", 1024)
|
||||||
|
req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
|
||||||
|
MakeRequest(t, req, http.StatusCreated)
|
||||||
|
|
||||||
|
t.Logf("Create artifact confirm")
|
||||||
|
|
||||||
|
sha := sha256.Sum256([]byte(strings.Repeat("A", 1024)))
|
||||||
|
|
||||||
|
// confirm artifact upload
|
||||||
|
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
|
||||||
|
Name: "artifact-invalid-checksum",
|
||||||
|
Size: 1024,
|
||||||
|
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha[:])),
|
||||||
|
WorkflowRunBackendId: "792",
|
||||||
|
WorkflowJobRunBackendId: "193",
|
||||||
|
})).
|
||||||
|
AddTokenAuth(token)
|
||||||
|
MakeRequest(t, req, http.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActionsArtifactV4UploadSingleFileWithRetentionDays(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
|
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// acquire artifact upload url
|
||||||
|
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
|
||||||
|
Version: 4,
|
||||||
|
ExpiresAt: timestamppb.New(time.Now().Add(5 * 24 * time.Hour)),
|
||||||
|
Name: "artifactWithRetentionDays",
|
||||||
|
WorkflowRunBackendId: "792",
|
||||||
|
WorkflowJobRunBackendId: "193",
|
||||||
|
})).AddTokenAuth(token)
|
||||||
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
|
var uploadResp actions.CreateArtifactResponse
|
||||||
|
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
|
||||||
|
assert.True(t, uploadResp.Ok)
|
||||||
|
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
|
||||||
|
|
||||||
|
// get upload url
|
||||||
|
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
|
||||||
|
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block"
|
||||||
|
|
||||||
|
// upload artifact chunk
|
||||||
|
body := strings.Repeat("A", 1024)
|
||||||
|
req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
|
||||||
|
MakeRequest(t, req, http.StatusCreated)
|
||||||
|
|
||||||
|
t.Logf("Create artifact confirm")
|
||||||
|
|
||||||
|
sha := sha256.Sum256([]byte(body))
|
||||||
|
|
||||||
|
// confirm artifact upload
|
||||||
|
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
|
||||||
|
Name: "artifactWithRetentionDays",
|
||||||
|
Size: 1024,
|
||||||
|
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha[:])),
|
||||||
|
WorkflowRunBackendId: "792",
|
||||||
|
WorkflowJobRunBackendId: "193",
|
||||||
|
})).
|
||||||
|
AddTokenAuth(token)
|
||||||
|
resp = MakeRequest(t, req, http.StatusOK)
|
||||||
|
var finalizeResp actions.FinalizeArtifactResponse
|
||||||
|
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
|
||||||
|
assert.True(t, finalizeResp.Ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActionsArtifactV4DownloadSingle(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
|
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// acquire artifact upload url
|
||||||
|
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(&actions.ListArtifactsRequest{
|
||||||
|
NameFilter: wrapperspb.String("artifact"),
|
||||||
|
WorkflowRunBackendId: "792",
|
||||||
|
WorkflowJobRunBackendId: "193",
|
||||||
|
})).AddTokenAuth(token)
|
||||||
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
|
var listResp actions.ListArtifactsResponse
|
||||||
|
protojson.Unmarshal(resp.Body.Bytes(), &listResp)
|
||||||
|
assert.Len(t, listResp.Artifacts, 1)
|
||||||
|
|
||||||
|
// confirm artifact upload
|
||||||
|
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/GetSignedArtifactURL", toProtoJSON(&actions.GetSignedArtifactURLRequest{
|
||||||
|
Name: "artifact",
|
||||||
|
WorkflowRunBackendId: "792",
|
||||||
|
WorkflowJobRunBackendId: "193",
|
||||||
|
})).
|
||||||
|
AddTokenAuth(token)
|
||||||
|
resp = MakeRequest(t, req, http.StatusOK)
|
||||||
|
var finalizeResp actions.GetSignedArtifactURLResponse
|
||||||
|
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
|
||||||
|
assert.NotEmpty(t, finalizeResp.SignedUrl)
|
||||||
|
|
||||||
|
req = NewRequest(t, "GET", finalizeResp.SignedUrl)
|
||||||
|
resp = MakeRequest(t, req, http.StatusOK)
|
||||||
|
body := strings.Repeat("A", 1024)
|
||||||
|
assert.Equal(t, resp.Body.String(), body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestActionsArtifactV4Delete(t *testing.T) {
|
||||||
|
defer tests.PrepareTestEnv(t)()
|
||||||
|
|
||||||
|
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// delete artifact by name
|
||||||
|
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/DeleteArtifact", toProtoJSON(&actions.DeleteArtifactRequest{
|
||||||
|
Name: "artifact",
|
||||||
|
WorkflowRunBackendId: "792",
|
||||||
|
WorkflowJobRunBackendId: "193",
|
||||||
|
})).AddTokenAuth(token)
|
||||||
|
resp := MakeRequest(t, req, http.StatusOK)
|
||||||
|
var deleteResp actions.DeleteArtifactResponse
|
||||||
|
protojson.Unmarshal(resp.Body.Bytes(), &deleteResp)
|
||||||
|
assert.True(t, deleteResp.Ok)
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue