fix(executor/sandbox): stream all files

.gitea/workflows/build.yaml

@@ -12,23 +12,12 @@ jobs:
         run: go version
       - name: Prepare
         run: |
-          echo ">>> set GO111MODULE"
           go env -w GO111MODULE=on
-          echo ">>> set GOPROXY"
           go env -w GOPROXY=https://goproxy.cn,direct
-          echo ">>> mkdir ~/.ssh"
           mkdir -p ~/.ssh
-          echo ">>> write private key"
           echo "${{ secrets.DEPLOY_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
-          echo ">>> write public key"
           echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_ed25519.pub
-          echo ">>> chmod private key"
+          chmod 600 ~/.ssh/id_ed25519 ~/.ssh/id_ed25519.pub
-          chmod 600 ~/.ssh/id_ed25519
-          echo ">>> chmod public key"
-          chmod 600 ~/.ssh/id_ed25519.pub
-          echo ">>> ssh-keyscan"
-          timeout 10 ssh-keyscan -p 2222 focs.gc.sjtu.edu.cn >> ~/.ssh/known_hosts || echo "ssh-keyscan failed, using StrictHostKeyChecking=accept-new"
-          echo ">>> ssh test"
           ssh -o StrictHostKeyChecking=accept-new -T git@focs.gc.sjtu.edu.cn -p 2222
         # - name: Lint
         #   run: make lint
@@ -47,19 +36,10 @@ jobs:
     steps:
       - name: Set up SSH
         run: |
-          echo ">>> mkdir ~/.ssh"
           mkdir -p ~/.ssh
-          echo ">>> write private key"
           echo "${{ secrets.DEPLOY_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
-          echo ">>> write public key"
           echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_ed25519.pub
-          echo ">>> chmod private key"
+          chmod 600 ~/.ssh/id_ed25519 ~/.ssh/id_ed25519.pub
-          chmod 600 ~/.ssh/id_ed25519
-          echo ">>> chmod public key"
-          chmod 600 ~/.ssh/id_ed25519.pub
-          echo ">>> ssh-keyscan"
-          timeout 10 ssh-keyscan -p 2222 focs.gc.sjtu.edu.cn >> ~/.ssh/known_hosts || echo "ssh-keyscan failed, using StrictHostKeyChecking=accept-new"
-          echo ">>> ssh test"
           ssh -o StrictHostKeyChecking=accept-new -T git@focs.gc.sjtu.edu.cn -p 2222
       - name: Set up Git
         run: |

.gitea/workflows/submodule.yaml

@@ -13,19 +13,10 @@ jobs:
           fetch-depth: 0
       - name: Set up SSH
         run: |
-          echo ">>> mkdir ~/.ssh"
           mkdir -p ~/.ssh
-          echo ">>> write private key"
           echo "${{ secrets.DEPLOY_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
-          echo ">>> write public key"
           echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_ed25519.pub
-          echo ">>> chmod private key"
+          chmod 600 ~/.ssh/id_ed25519 ~/.ssh/id_ed25519.pub
-          chmod 600 ~/.ssh/id_ed25519
-          echo ">>> chmod public key"
-          chmod 600 ~/.ssh/id_ed25519.pub
-          echo ">>> ssh-keyscan"
-          timeout 10 ssh-keyscan -p 2222 focs.gc.sjtu.edu.cn >> ~/.ssh/known_hosts || echo "ssh-keyscan failed, using StrictHostKeyChecking=accept-new"
-          echo ">>> ssh test"
           ssh -o StrictHostKeyChecking=accept-new -T git@focs.gc.sjtu.edu.cn -p 2222
       - name: Set up Git
         run: |

internal/executor/sandbox/executor.go

@@ -55,13 +55,12 @@ func prepareTarStream(cmds []stage.Cmd) (bool, []byte) {
 	for i := range cmds {
 		if cmds[i].CopyInDir != "" &&
 			estimateCopyInSize(&cmds[i]) >= tarStreamThreshold {
-			tarData, err := createCopyInTar(&cmds[i])
+			tarData, keysInTar := createCopyInTar(&cmds[i])
-			if err != nil {
-				slog.Error("create copyIn tar", "error", err)
-				return false, nil
-			}
 			for j := range cmds {
 				cmds[j].CopyInDir = ""
+				for _, k := range keysInTar {
+					delete(cmds[j].CopyIn, k)
+				}
 			}
 			return true, tarData
 		}
@@ -222,9 +221,10 @@ func estimateCopyInSize(cmd *stage.Cmd) int {
 	return total
 }
 
-func createCopyInTar(cmd *stage.Cmd) ([]byte, error) {
+func createCopyInTar(cmd *stage.Cmd) ([]byte, []string) {
 	var buf bytes.Buffer
 	tw := tar.NewWriter(&buf)
+	tarKeys := make([]string, 0, len(cmd.CopyIn))
 
 	if cmd.CopyInDir != "" {
 		err := filepath.Walk(cmd.CopyInDir,
@@ -280,14 +280,69 @@ func createCopyInTar(cmd *stage.Cmd) ([]byte, error) {
 				return err
 			})
 		if err != nil {
-			return nil, fmt.Errorf("walk copyInDir: %w", err)
+			slog.Error("create copyIn tar walk", "error", err)
+			return nil, nil
+		}
+	}
+
+	for k, f := range cmd.CopyIn {
+		if f.FileID != nil || f.Symlink != nil || f.StreamIn || f.StreamOut || f.Pipe {
+			continue
+		}
+		if f.Content != nil {
+			hdr := &tar.Header{
+				Name: k,
+				Mode: 0o644,
+				Size: int64(len(*f.Content)),
+			}
+			if err := tw.WriteHeader(hdr); err != nil {
+				slog.Error("create copyIn tar write header", "key", k, "error", err)
+				continue
+			}
+			if _, err := tw.Write([]byte(*f.Content)); err != nil {
+				slog.Error("create copyIn tar write content", "key", k, "error", err)
+				continue
+			}
+			tarKeys = append(tarKeys, k)
+		} else if f.Src != nil {
+			fi, err := os.Stat(*f.Src)
+			if err != nil {
+				slog.Error("create copyIn tar stat", "key", k, "src", *f.Src, "error", err)
+				continue
+			}
+			if fi.IsDir() {
+				continue
+			}
+			hdr, err := tar.FileInfoHeader(fi, "")
+			if err != nil {
+				slog.Error("create copyIn tar file info header", "key", k, "error", err)
+				continue
+			}
+			hdr.Name = k
+			if err := tw.WriteHeader(hdr); err != nil {
+				slog.Error("create copyIn tar write header", "key", k, "error", err)
+				continue
+			}
+			srcFile, err := os.Open(*f.Src)
+			if err != nil {
+				slog.Error("create copyIn tar open src", "key", k, "src", *f.Src, "error", err)
+				continue
+			}
+			_, err = io.Copy(tw, srcFile)
+			srcFile.Close()
+			if err != nil {
+				slog.Error("create copyIn tar copy", "key", k, "error", err)
+				continue
+			}
+			tarKeys = append(tarKeys, k)
 		}
 	}
 
 	if err := tw.Close(); err != nil {
-		return nil, fmt.Errorf("close tar: %w", err)
+		slog.Error("create copyIn tar close", "error", err)
+		return nil, nil
 	}
-	return buf.Bytes(), nil
+	return buf.Bytes(), tarKeys
 }
 
 func (e *Sandbox) Cleanup() error {