From 8176ea8c5ea578738406fe33f8fa26439ee16f61 Mon Sep 17 00:00:00 2001 From: Mack Wang Date: Wed, 22 Apr 2026 17:06:54 +0800 Subject: [PATCH] feat: add whitelist character support in repo toml (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit introduces a new key, `health-check.whitelisted-chars` for repo.toml. It allows TAs to configure repo-wide allowed non ASCII chars for the repo-health-checker binary. It results in a new command line switch, `-whitelisted-chars=X,Y,Z`, in the generated task.json. Co-Authored-By: GitHub Copilot
Copilot Prompt
This repository contains a Python app that does conversion from TOML config files to a complex, multistage JSON config file for an online judge system. For example, under `tests/convert/full`, input are the two TOML files `repo.toml` and `task.toml`, output is one JSON file `task,json`. Now, I want the repo-specific config (**repo.toml**) to accept an extra dotted key, "health-check.whitelisted-chars". This key shall accept an array of UTF-8 non-ASCII characters. To do so, I want you to - Model. Under `joj3_config_generator/models/repo.py`, add an extra field *whitelisted_chars* to class *HealthCheck*, identified by both "whitelisted-chars" and "whitelisted_chars"; - Transforming. Under `joj3_config_generator/transforers/repo.py`, translate the field to an additional command line switch `-whitelisted-chars`, comma-separated. - Other files you deem necessary, based on your understanding of this repo. IMPORTANT. Before you start, explore this repo to under the file structure and file-function relations. This repo uses the PDM package manager. After you finish your work, test your work appropriately. You should create new testcases under `tests/`. - Details: Read the output `task.json` after running the test, to verify whether the command line switch was added to the *Health Check* `stage` or not.
Reviewed-on: https://focs.ji.sjtu.edu.cn/git/JOJ/JOJ3-config-generator/pulls/29 Reviewed-by: 李衍志523370910113 Reviewed-by: 张泊明518370910136 Co-authored-by: Mack Wang Co-committed-by: Mack Wang --- joj3_config_generator/models/repo.py | 13 ++ joj3_config_generator/transformers/repo.py | 15 +- tests/convert/full/repo.toml | 1 + tests/convert/full/task.json | 1 + tests/convert/test_convert_cases.py | 4 + tests/convert/whitelisted-chars/repo.toml | 1 + tests/convert/whitelisted-chars/task.json | 188 +++++++++++++++++++++ tests/convert/whitelisted-chars/task.toml | 2 + 8 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 tests/convert/whitelisted-chars/repo.toml create mode 100644 tests/convert/whitelisted-chars/task.json create mode 100644 tests/convert/whitelisted-chars/task.toml diff --git a/joj3_config_generator/models/repo.py b/joj3_config_generator/models/repo.py index d97da28..18512dd 100644 --- a/joj3_config_generator/models/repo.py +++ b/joj3_config_generator/models/repo.py @@ -50,6 +50,9 @@ class HealthCheck(StrictBaseModel): required_files: List[str] = Field( [], validation_alias=AliasChoices("required-files", "required_files") ) + whitelisted_chars: str = Field( + "", validation_alias=AliasChoices("whitelisted-chars", "whitelisted_chars") + ) @field_validator("max_size", mode="before") @classmethod @@ -58,6 +61,16 @@ class HealthCheck(StrictBaseModel): return Memory(v) raise ValueError(f'Must be a string, e.g., "256m" or "1g", but got {v}') + @field_validator("whitelisted_chars") + @classmethod + def ensure_non_ascii_chars(cls, chars: str) -> str: + for c in chars: + if c.isascii(): + raise ValueError( + "Each whitelisted character must be a non-ASCII character" + ) + return chars + class Config(StrictBaseModel): root: Path = Field(Path("."), exclude=True) diff --git a/joj3_config_generator/transformers/repo.py b/joj3_config_generator/transformers/repo.py index 7e50439..82f0f93 100644 --- a/joj3_config_generator/transformers/repo.py +++ b/joj3_config_generator/transformers/repo.py @@ -123,14 +123,23 @@ def get_check_lists(repo_conf: repo.Config) -> Tuple[List[str], List[str]]: def get_health_check_args(repo_conf: repo.Config) -> List[str]: file_sums, file_paths = get_check_lists(repo_conf) - return [ + args = [ "/usr/local/bin/repo-health-checker", "-root=.", f"-repoSize={str(repo_conf.health_check.max_size / 1024 / 1024)}", # B -> MB *[f"-meta={meta}" for meta in repo_conf.health_check.required_files], - f"-checkFileSumList={','.join(file_sums)}", - f"-checkFileNameList={','.join(file_paths)}", ] + if repo_conf.health_check.whitelisted_chars: + args.append( + f"-whitelistedChars={','.join(list(repo_conf.health_check.whitelisted_chars))}" + ) + args.extend( + [ + f"-checkFileSumList={','.join(file_sums)}", + f"-checkFileNameList={','.join(file_paths)}", + ] + ) + return args def get_teapot_check_args(repo_conf: repo.Config, task_conf: task.Config) -> List[str]: diff --git a/tests/convert/full/repo.toml b/tests/convert/full/repo.toml index 4c3b171..ab4edcc 100644 --- a/tests/convert/full/repo.toml +++ b/tests/convert/full/repo.toml @@ -2,6 +2,7 @@ sandbox-token = "" # sandbox token health-check.score = 0 # score for health check stage health-check.max-size = "10m" # max size of the repository +health-check.whitelisted-chars = "あいうえお" # allowed non-ASCII characters in healthcheck stage health-check.immutable-path = "immutable" # path for immutable files, relative to the path of repo.toml health-check.required-files = ["README.md", "Changelog.md"] # required files name, case insensitive diff --git a/tests/convert/full/task.json b/tests/convert/full/task.json index d1c3c17..3eeea9e 100644 --- a/tests/convert/full/task.json +++ b/tests/convert/full/task.json @@ -61,6 +61,7 @@ "-repoSize=10.0", "-meta=README.md", "-meta=Changelog.md", + "-whitelistedChars=あ,い,う,え,お", "-checkFileSumList=b1bbad25b830db0a77b15a033f9ca1b7ab44c1d2d05056412bd3e4421645f0bf,2ba059f3977e2e3dee6cacbfbf0ba2578baa1b8e04b4977aec400868b6e49856,3db23f7fb2ca9814617e767ddc41b77073180b3b0b73e87b5f2a6d3129f88f3a,a5b63323a692d3d8b952442969649b4f823d58dae26429494f613df160710dfc", "-checkFileNameList=.gitattributes,.gitea/workflows/push.yaml,.gitea/workflows/release.yaml,.gitignore" ] diff --git a/tests/convert/test_convert_cases.py b/tests/convert/test_convert_cases.py index 284043a..2beea68 100644 --- a/tests/convert/test_convert_cases.py +++ b/tests/convert/test_convert_cases.py @@ -51,3 +51,7 @@ def test_result_detail() -> None: def test_unnecessary() -> None: load_case("unnecessary") + + +def test_whitelisted_chars() -> None: + load_case("whitelisted-chars") diff --git a/tests/convert/whitelisted-chars/repo.toml b/tests/convert/whitelisted-chars/repo.toml new file mode 100644 index 0000000..bd66ab0 --- /dev/null +++ b/tests/convert/whitelisted-chars/repo.toml @@ -0,0 +1 @@ +health-check.whitelisted-chars = "你好！" diff --git a/tests/convert/whitelisted-chars/task.json b/tests/convert/whitelisted-chars/task.json new file mode 100644 index 0000000..97ad5a0 --- /dev/null +++ b/tests/convert/whitelisted-chars/task.json @@ -0,0 +1,188 @@ +{ + "name": "health check", + "logPath": "/home/tt/.cache/joj3/health/joj3.log", + "actorCsvPath": "/home/tt/.config/joj/students.csv", + "sandboxExecServer": "172.17.0.1:5051", + "sandboxToken": "", + "outputPath": "/tmp/joj3_result.json", + "preStages": [], + "stages": [ + { + "name": "Health Check", + "groups": [], + "executor": { + "name": "local", + "with": { + "default": { + "args": [], + "env": [ + "PATH=/usr/bin:/bin:/usr/local/bin" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 10000000000, + "clockLimit": 20000000000, + "memoryLimit": 268435456, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [ + { + "args": [ + "/usr/local/bin/repo-health-checker", + "-root=.", + "-repoSize=10.0", + "-whitelistedChars=你,好,！", + "-checkFileSumList=", + "-checkFileNameList=" + ] + }, + { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-check-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "JOJ3-config-generator", + "--scoreboard-filename", + "scoreboard.csv" + ], + "env": [ + "REPOS_DIR=/home/tt/.cache", + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ] + } + ] + } + }, + "parsers": [ + { + "name": "healthcheck", + "with": { + "score": 0 + } + }, + { + "name": "debug", + "with": {} + } + ] + } + ], + "postStages": [ + { + "name": "teapot", + "groups": [], + "executor": { + "name": "local", + "with": { + "default": { + "args": [ + "/usr/local/bin/joint-teapot", + "joj3-all-env", + "/home/tt/.config/teapot/teapot.env", + "--grading-repo-name", + "JOJ3-config-generator", + "--max-total-score", + "0", + "--issue-label-name", + "Kind/Testing", + "--issue-label-color", + "#795548", + "--scoreboard-filename", + "scoreboard.csv" + ], + "env": [ + "REPOS_DIR=/home/tt/.cache", + "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log" + ], + "stdin": { + "content": "" + }, + "stdout": { + "name": "stdout", + "max": 33554432, + "pipe": true + }, + "stderr": { + "name": "stderr", + "max": 33554432, + "pipe": true + }, + "cpuLimit": 30000000000, + "clockLimit": 60000000000, + "memoryLimit": 268435456, + "stackLimit": 0, + "procLimit": 50, + "cpuRateLimit": 0, + "cpuSetLimit": "", + "copyIn": {}, + "copyInCached": {}, + "copyInDir": ".", + "copyOut": [ + "stdout", + "stderr" + ], + "copyOutCached": [], + "copyOutMax": 0, + "copyOutDir": "", + "tty": false, + "strictMemoryLimit": false, + "dataSegmentLimit": false, + "addressSpaceLimit": false + }, + "cases": [] + } + }, + "parsers": [ + { + "name": "log", + "with": { + "filename": "stdout", + "msg": "joj3 summary", + "level": -4 + } + }, + { + "name": "log", + "with": { + "filename": "stderr", + "msg": "joint-teapot stderr", + "level": 0 + } + }, + { + "name": "debug", + "with": {} + } + ] + } + ] +} diff --git a/tests/convert/whitelisted-chars/task.toml b/tests/convert/whitelisted-chars/task.toml new file mode 100644 index 0000000..e42a6ce --- /dev/null +++ b/tests/convert/whitelisted-chars/task.toml @@ -0,0 +1,2 @@ +name = "health check" +max-total-score = 0