From e897d2a78c0f77a93ce1732e195f8b1cbd4e584b Mon Sep 17 00:00:00 2001 From: Boming Zhang Date: Wed, 22 Jul 2026 22:07:17 -0700 Subject: [PATCH] fix(executor/sandbox): stream all files --- .gitea/workflows/build.yaml | 24 +-------- .gitea/workflows/submodule.yaml | 11 +--- internal/executor/sandbox/executor.go | 73 +++++++++++++++++++++++---- 3 files changed, 67 insertions(+), 41 deletions(-) diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 8f9b673..5e686e1 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -12,23 +12,12 @@ jobs: run: go version - name: Prepare run: | - echo ">>> set GO111MODULE" go env -w GO111MODULE=on - echo ">>> set GOPROXY" go env -w GOPROXY=https://goproxy.cn,direct - echo ">>> mkdir ~/.ssh" mkdir -p ~/.ssh - echo ">>> write private key" echo "${{ secrets.DEPLOY_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 - echo ">>> write public key" echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_ed25519.pub - echo ">>> chmod private key" - chmod 600 ~/.ssh/id_ed25519 - echo ">>> chmod public key" - chmod 600 ~/.ssh/id_ed25519.pub - echo ">>> ssh-keyscan" - timeout 10 ssh-keyscan -p 2222 focs.gc.sjtu.edu.cn >> ~/.ssh/known_hosts || echo "ssh-keyscan failed, using StrictHostKeyChecking=accept-new" - echo ">>> ssh test" + chmod 600 ~/.ssh/id_ed25519 ~/.ssh/id_ed25519.pub ssh -o StrictHostKeyChecking=accept-new -T git@focs.gc.sjtu.edu.cn -p 2222 # - name: Lint # run: make lint @@ -47,19 +36,10 @@ jobs: steps: - name: Set up SSH run: | - echo ">>> mkdir ~/.ssh" mkdir -p ~/.ssh - echo ">>> write private key" echo "${{ secrets.DEPLOY_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 - echo ">>> write public key" echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_ed25519.pub - echo ">>> chmod private key" - chmod 600 ~/.ssh/id_ed25519 - echo ">>> chmod public key" - chmod 600 ~/.ssh/id_ed25519.pub - echo ">>> ssh-keyscan" - timeout 10 ssh-keyscan -p 2222 focs.gc.sjtu.edu.cn >> ~/.ssh/known_hosts || echo "ssh-keyscan failed, using StrictHostKeyChecking=accept-new" - echo ">>> ssh test" + chmod 600 ~/.ssh/id_ed25519 ~/.ssh/id_ed25519.pub ssh -o StrictHostKeyChecking=accept-new -T git@focs.gc.sjtu.edu.cn -p 2222 - name: Set up Git run: | diff --git a/.gitea/workflows/submodule.yaml b/.gitea/workflows/submodule.yaml index 28da2bb..959a90b 100644 --- a/.gitea/workflows/submodule.yaml +++ b/.gitea/workflows/submodule.yaml @@ -13,19 +13,10 @@ jobs: fetch-depth: 0 - name: Set up SSH run: | - echo ">>> mkdir ~/.ssh" mkdir -p ~/.ssh - echo ">>> write private key" echo "${{ secrets.DEPLOY_PRIVATE_KEY }}" > ~/.ssh/id_ed25519 - echo ">>> write public key" echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_ed25519.pub - echo ">>> chmod private key" - chmod 600 ~/.ssh/id_ed25519 - echo ">>> chmod public key" - chmod 600 ~/.ssh/id_ed25519.pub - echo ">>> ssh-keyscan" - timeout 10 ssh-keyscan -p 2222 focs.gc.sjtu.edu.cn >> ~/.ssh/known_hosts || echo "ssh-keyscan failed, using StrictHostKeyChecking=accept-new" - echo ">>> ssh test" + chmod 600 ~/.ssh/id_ed25519 ~/.ssh/id_ed25519.pub ssh -o StrictHostKeyChecking=accept-new -T git@focs.gc.sjtu.edu.cn -p 2222 - name: Set up Git run: | diff --git a/internal/executor/sandbox/executor.go b/internal/executor/sandbox/executor.go index 127aeff..ed8ede3 100644 --- a/internal/executor/sandbox/executor.go +++ b/internal/executor/sandbox/executor.go @@ -55,13 +55,12 @@ func prepareTarStream(cmds []stage.Cmd) (bool, []byte) { for i := range cmds { if cmds[i].CopyInDir != "" && estimateCopyInSize(&cmds[i]) >= tarStreamThreshold { - tarData, err := createCopyInTar(&cmds[i]) - if err != nil { - slog.Error("create copyIn tar", "error", err) - return false, nil - } + tarData, keysInTar := createCopyInTar(&cmds[i]) for j := range cmds { cmds[j].CopyInDir = "" + for _, k := range keysInTar { + delete(cmds[j].CopyIn, k) + } } return true, tarData } @@ -222,9 +221,10 @@ func estimateCopyInSize(cmd *stage.Cmd) int { return total } -func createCopyInTar(cmd *stage.Cmd) ([]byte, error) { +func createCopyInTar(cmd *stage.Cmd) ([]byte, []string) { var buf bytes.Buffer tw := tar.NewWriter(&buf) + tarKeys := make([]string, 0, len(cmd.CopyIn)) if cmd.CopyInDir != "" { err := filepath.Walk(cmd.CopyInDir, @@ -280,14 +280,69 @@ func createCopyInTar(cmd *stage.Cmd) ([]byte, error) { return err }) if err != nil { - return nil, fmt.Errorf("walk copyInDir: %w", err) + slog.Error("create copyIn tar walk", "error", err) + return nil, nil + } + } + + for k, f := range cmd.CopyIn { + if f.FileID != nil || f.Symlink != nil || f.StreamIn || f.StreamOut || f.Pipe { + continue + } + if f.Content != nil { + hdr := &tar.Header{ + Name: k, + Mode: 0o644, + Size: int64(len(*f.Content)), + } + if err := tw.WriteHeader(hdr); err != nil { + slog.Error("create copyIn tar write header", "key", k, "error", err) + continue + } + if _, err := tw.Write([]byte(*f.Content)); err != nil { + slog.Error("create copyIn tar write content", "key", k, "error", err) + continue + } + tarKeys = append(tarKeys, k) + } else if f.Src != nil { + fi, err := os.Stat(*f.Src) + if err != nil { + slog.Error("create copyIn tar stat", "key", k, "src", *f.Src, "error", err) + continue + } + if fi.IsDir() { + continue + } + hdr, err := tar.FileInfoHeader(fi, "") + if err != nil { + slog.Error("create copyIn tar file info header", "key", k, "error", err) + continue + } + hdr.Name = k + if err := tw.WriteHeader(hdr); err != nil { + slog.Error("create copyIn tar write header", "key", k, "error", err) + continue + } + srcFile, err := os.Open(*f.Src) + if err != nil { + slog.Error("create copyIn tar open src", "key", k, "src", *f.Src, "error", err) + continue + } + _, err = io.Copy(tw, srcFile) + srcFile.Close() + if err != nil { + slog.Error("create copyIn tar copy", "key", k, "error", err) + continue + } + tarKeys = append(tarKeys, k) } } if err := tw.Close(); err != nil { - return nil, fmt.Errorf("close tar: %w", err) + slog.Error("create copyIn tar close", "error", err) + return nil, nil } - return buf.Bytes(), nil + return buf.Bytes(), tarKeys } func (e *Sandbox) Cleanup() error {