JOJ/JOJ3
26
7
Fork 1
Code Issues 6 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(executor/sandbox): stream all files
Some checks failed
submodules sync / sync (push) Successful in 28s
Details
build / build (push) Failing after 1m11s
Details
build / trigger-build-image (push) Has been skipped
Details

Browse Source
This commit is contained in:
张泊明518370910136 2026-07-22 22:07:17 -07:00
parent 90602b358f
commit e897d2a78c
Signed by: 张泊明518370910136
GPG Key ID: D47306D7062CDA9D
3 changed files with 67 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.gitea/workflows/build.yaml
View File

@ -12,23 +12,12 @@ jobs:
run: go version
- name: Prepare
run: |
echo ">>> set GO111MODULE"
go env -w GO111MODULE=on
echo ">>> set GOPROXY"
go env -w GOPROXY=https://goproxy.cn,direct
echo ">>> mkdir ~/.ssh"
mkdir -p ~/.ssh
echo ">>> write private key"
echo "${{ secrets.DEPLOY_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
echo ">>> write public key"
echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_ed25519.pub
echo ">>> chmod private key"
chmod 600 ~/.ssh/id_ed25519
echo ">>> chmod public key"
chmod 600 ~/.ssh/id_ed25519.pub
echo ">>> ssh-keyscan"
timeout 10 ssh-keyscan -p 2222 focs.gc.sjtu.edu.cn >> ~/.ssh/known_hosts || echo "ssh-keyscan failed, using StrictHostKeyChecking=accept-new"
echo ">>> ssh test"
chmod 600 ~/.ssh/id_ed25519 ~/.ssh/id_ed25519.pub
ssh -o StrictHostKeyChecking=accept-new -T git@focs.gc.sjtu.edu.cn -p 2222
# - name: Lint
# run: make lint
@ -47,19 +36,10 @@ jobs:
steps:
- name: Set up SSH
run: |
echo ">>> mkdir ~/.ssh"
mkdir -p ~/.ssh
echo ">>> write private key"
echo "${{ secrets.DEPLOY_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
echo ">>> write public key"
echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_ed25519.pub
echo ">>> chmod private key"
chmod 600 ~/.ssh/id_ed25519
echo ">>> chmod public key"
chmod 600 ~/.ssh/id_ed25519.pub
echo ">>> ssh-keyscan"
timeout 10 ssh-keyscan -p 2222 focs.gc.sjtu.edu.cn >> ~/.ssh/known_hosts || echo "ssh-keyscan failed, using StrictHostKeyChecking=accept-new"
echo ">>> ssh test"
chmod 600 ~/.ssh/id_ed25519 ~/.ssh/id_ed25519.pub
ssh -o StrictHostKeyChecking=accept-new -T git@focs.gc.sjtu.edu.cn -p 2222
- name: Set up Git
run: |

11
.gitea/workflows/submodule.yaml
View File

@ -13,19 +13,10 @@ jobs:
fetch-depth: 0
- name: Set up SSH
run: |
echo ">>> mkdir ~/.ssh"
mkdir -p ~/.ssh
echo ">>> write private key"
echo "${{ secrets.DEPLOY_PRIVATE_KEY }}" > ~/.ssh/id_ed25519
echo ">>> write public key"
echo "${{ secrets.DEPLOY_KEY }}" > ~/.ssh/id_ed25519.pub
echo ">>> chmod private key"
chmod 600 ~/.ssh/id_ed25519
echo ">>> chmod public key"
chmod 600 ~/.ssh/id_ed25519.pub
echo ">>> ssh-keyscan"
timeout 10 ssh-keyscan -p 2222 focs.gc.sjtu.edu.cn >> ~/.ssh/known_hosts || echo "ssh-keyscan failed, using StrictHostKeyChecking=accept-new"
echo ">>> ssh test"
chmod 600 ~/.ssh/id_ed25519 ~/.ssh/id_ed25519.pub
ssh -o StrictHostKeyChecking=accept-new -T git@focs.gc.sjtu.edu.cn -p 2222
- name: Set up Git
run: |

73
internal/executor/sandbox/executor.go
View File

@ -55,13 +55,12 @@ func prepareTarStream(cmds []stage.Cmd) (bool, []byte) {
for i := range cmds {
if cmds[i].CopyInDir != "" &&
estimateCopyInSize(&cmds[i]) >= tarStreamThreshold {
tarData, err := createCopyInTar(&cmds[i])
if err != nil {
slog.Error("create copyIn tar", "error", err)
return false, nil
}
tarData, keysInTar := createCopyInTar(&cmds[i])
for j := range cmds {
cmds[j].CopyInDir = ""
for _, k := range keysInTar {
delete(cmds[j].CopyIn, k)
}
}
return true, tarData
}
@ -222,9 +221,10 @@ func estimateCopyInSize(cmd *stage.Cmd) int {
return total
}
func createCopyInTar(cmd *stage.Cmd) ([]byte, error) {
func createCopyInTar(cmd *stage.Cmd) ([]byte, []string) {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
tarKeys := make([]string, 0, len(cmd.CopyIn))
if cmd.CopyInDir != "" {
err := filepath.Walk(cmd.CopyInDir,
@ -280,14 +280,69 @@ func createCopyInTar(cmd *stage.Cmd) ([]byte, error) {
return err
})
if err != nil {
return nil, fmt.Errorf("walk copyInDir: %w", err)
slog.Error("create copyIn tar walk", "error", err)
return nil, nil
}
}
for k, f := range cmd.CopyIn {
if f.FileID != nil || f.Symlink != nil || f.StreamIn || f.StreamOut || f.Pipe {
continue
}
if f.Content != nil {
hdr := &tar.Header{
Name: k,
Mode: 0o644,
Size: int64(len(*f.Content)),
}
if err := tw.WriteHeader(hdr); err != nil {
slog.Error("create copyIn tar write header", "key", k, "error", err)
continue
}
if _, err := tw.Write([]byte(*f.Content)); err != nil {
slog.Error("create copyIn tar write content", "key", k, "error", err)
continue
}
tarKeys = append(tarKeys, k)
} else if f.Src != nil {
fi, err := os.Stat(*f.Src)
if err != nil {
slog.Error("create copyIn tar stat", "key", k, "src", *f.Src, "error", err)
continue
}
if fi.IsDir() {
continue
}
hdr, err := tar.FileInfoHeader(fi, "")
if err != nil {
slog.Error("create copyIn tar file info header", "key", k, "error", err)
continue
}
hdr.Name = k
if err := tw.WriteHeader(hdr); err != nil {
slog.Error("create copyIn tar write header", "key", k, "error", err)
continue
}
srcFile, err := os.Open(*f.Src)
if err != nil {
slog.Error("create copyIn tar open src", "key", k, "src", *f.Src, "error", err)
continue
}
_, err = io.Copy(tw, srcFile)
srcFile.Close()
if err != nil {
slog.Error("create copyIn tar copy", "key", k, "error", err)
continue
}
tarKeys = append(tarKeys, k)
}
}
if err := tw.Close(); err != nil {
return nil, fmt.Errorf("close tar: %w", err)
slog.Error("create copyIn tar close", "error", err)
return nil, nil
}
return buf.Bytes(), nil
return buf.Bytes(), tarKeys
}
func (e *Sandbox) Cleanup() error {