feat: add whitelist character support in repo toml (#29)

This commit introduces a new key, `health-check.whitelisted-chars` for repo.toml. It allows TAs to configure repo-wide allowed non ASCII chars for the repo-health-checker binary. It results in a new command line switch, `-whitelisted-chars=X,Y,Z`, in the generated task.json.

Co-Authored-By: GitHub Copilot <noreply@microsoft.com>
<details>
<summary>Copilot Prompt</summary>
<br>
This repository contains a Python app that does conversion from TOML config files to a complex, multistage JSON config file for an online judge system. For example, under `tests/convert/full`, input are the two TOML files `repo.toml` and `task.toml`, output is one JSON file `task,json`.

Now, I want the repo-specific config (**repo.toml**) to accept an extra dotted key, "health-check.whitelisted-chars". This key shall accept an array of UTF-8 non-ASCII characters. To do so, I want you to
- Model. Under `joj3_config_generator/models/repo.py`, add an extra field *whitelisted_chars* to class *HealthCheck*, identified by both "whitelisted-chars" and "whitelisted_chars";
- Transforming. Under `joj3_config_generator/transforers/repo.py`, translate the field to an additional command line switch `-whitelisted-chars`, comma-separated.
- Other files you deem necessary, based on your understanding of this repo.

IMPORTANT. Before you start, explore this repo to under the file structure and file-function relations.

This repo uses the PDM package manager. After you finish your work, test your work appropriately. You should create new testcases under `tests/`.
- Details: Read the output `task.json` after running the test, to verify whether the command line switch was added to the *Health Check* `stage` or not.
</details>

Reviewed-on: https://focs.ji.sjtu.edu.cn/git/JOJ/JOJ3-config-generator/pulls/29
Reviewed-by: 李衍志523370910113 <jon-lee@sjtu.edu.cn>
Reviewed-by: 张泊明518370910136 <bomingzh@sjtu.edu.cn>
Co-authored-by: Mack Wang <mac-wang@outlook.com>
Co-committed-by: Mack Wang <mac-wang@outlook.com>

joj3_config_generator/models/repo.py

@@ -50,6 +50,9 @@ class HealthCheck(StrictBaseModel):
     required_files: List[str] = Field(
         [], validation_alias=AliasChoices("required-files", "required_files")
     )
+    whitelisted_chars: str = Field(
+        "", validation_alias=AliasChoices("whitelisted-chars", "whitelisted_chars")
+    )
 
     @field_validator("max_size", mode="before")
     @classmethod
@@ -58,6 +61,16 @@ class HealthCheck(StrictBaseModel):
             return Memory(v)
         raise ValueError(f'Must be a string, e.g., "256m" or "1g", but got {v}')
 
+    @field_validator("whitelisted_chars")
+    @classmethod
+    def ensure_non_ascii_chars(cls, chars: str) -> str:
+        for c in chars:
+            if c.isascii():
+                raise ValueError(
+                    "Each whitelisted character must be a non-ASCII character"
+                )
+        return chars
+
 
 class Config(StrictBaseModel):
     root: Path = Field(Path("."), exclude=True)

joj3_config_generator/transformers/repo.py

@@ -123,14 +123,23 @@ def get_check_lists(repo_conf: repo.Config) -> Tuple[List[str], List[str]]:
 
 def get_health_check_args(repo_conf: repo.Config) -> List[str]:
     file_sums, file_paths = get_check_lists(repo_conf)
-    return [
+    args = [
         "/usr/local/bin/repo-health-checker",
         "-root=.",
         f"-repoSize={str(repo_conf.health_check.max_size / 1024 / 1024)}",  # B -> MB
         *[f"-meta={meta}" for meta in repo_conf.health_check.required_files],
-        f"-checkFileSumList={','.join(file_sums)}",
-        f"-checkFileNameList={','.join(file_paths)}",
     ]
+    if repo_conf.health_check.whitelisted_chars:
+        args.append(
+            f"-whitelistedChars={','.join(list(repo_conf.health_check.whitelisted_chars))}"
+        )
+    args.extend(
+        [
+            f"-checkFileSumList={','.join(file_sums)}",
+            f"-checkFileNameList={','.join(file_paths)}",
+        ]
+    )
+    return args
 
 
 def get_teapot_check_args(repo_conf: repo.Config, task_conf: task.Config) -> List[str]:

tests/convert/full/repo.toml

@@ -2,6 +2,7 @@ sandbox-token = "" # sandbox token
 
 health-check.score = 0 # score for health check stage
 health-check.max-size = "10m" # max size of the repository
+health-check.whitelisted-chars = "あいうえお" # allowed non-ASCII characters in healthcheck stage
 health-check.immutable-path = "immutable" # path for immutable files, relative to the path of repo.toml
 health-check.required-files = ["README.md", "Changelog.md"] # required files name, case insensitive
 

tests/convert/full/task.json

@@ -61,6 +61,7 @@
                                 "-repoSize=10.0",
                                 "-meta=README.md",
                                 "-meta=Changelog.md",
+                                "-whitelistedChars=あ,い,う,え,お",
                                 "-checkFileSumList=b1bbad25b830db0a77b15a033f9ca1b7ab44c1d2d05056412bd3e4421645f0bf,2ba059f3977e2e3dee6cacbfbf0ba2578baa1b8e04b4977aec400868b6e49856,3db23f7fb2ca9814617e767ddc41b77073180b3b0b73e87b5f2a6d3129f88f3a,a5b63323a692d3d8b952442969649b4f823d58dae26429494f613df160710dfc",
                                 "-checkFileNameList=.gitattributes,.gitea/workflows/push.yaml,.gitea/workflows/release.yaml,.gitignore"
                             ]

tests/convert/test_convert_cases.py

@@ -51,3 +51,7 @@ def test_result_detail() -> None:
 
 def test_unnecessary() -> None:
     load_case("unnecessary")
+
+
+def test_whitelisted_chars() -> None:
+    load_case("whitelisted-chars")

tests/convert/whitelisted-chars/repo.toml

@@ -0,0 +1 @@
+health-check.whitelisted-chars = "你好！"

tests/convert/whitelisted-chars/task.json

@@ -0,0 +1,188 @@
+{
+    "name": "health check",
+    "logPath": "/home/tt/.cache/joj3/health/joj3.log",
+    "actorCsvPath": "/home/tt/.config/joj/students.csv",
+    "sandboxExecServer": "172.17.0.1:5051",
+    "sandboxToken": "",
+    "outputPath": "/tmp/joj3_result.json",
+    "preStages": [],
+    "stages": [
+        {
+            "name": "Health Check",
+            "groups": [],
+            "executor": {
+                "name": "local",
+                "with": {
+                    "default": {
+                        "args": [],
+                        "env": [
+                            "PATH=/usr/bin:/bin:/usr/local/bin"
+                        ],
+                        "stdin": {
+                            "content": ""
+                        },
+                        "stdout": {
+                            "name": "stdout",
+                            "max": 33554432,
+                            "pipe": true
+                        },
+                        "stderr": {
+                            "name": "stderr",
+                            "max": 33554432,
+                            "pipe": true
+                        },
+                        "cpuLimit": 10000000000,
+                        "clockLimit": 20000000000,
+                        "memoryLimit": 268435456,
+                        "stackLimit": 0,
+                        "procLimit": 50,
+                        "cpuRateLimit": 0,
+                        "cpuSetLimit": "",
+                        "copyIn": {},
+                        "copyInCached": {},
+                        "copyInDir": ".",
+                        "copyOut": [
+                            "stdout",
+                            "stderr"
+                        ],
+                        "copyOutCached": [],
+                        "copyOutMax": 0,
+                        "copyOutDir": "",
+                        "tty": false,
+                        "strictMemoryLimit": false,
+                        "dataSegmentLimit": false,
+                        "addressSpaceLimit": false
+                    },
+                    "cases": [
+                        {
+                            "args": [
+                                "/usr/local/bin/repo-health-checker",
+                                "-root=.",
+                                "-repoSize=10.0",
+                                "-whitelistedChars=你,好,！",
+                                "-checkFileSumList=",
+                                "-checkFileNameList="
+                            ]
+                        },
+                        {
+                            "args": [
+                                "/usr/local/bin/joint-teapot",
+                                "joj3-check-env",
+                                "/home/tt/.config/teapot/teapot.env",
+                                "--grading-repo-name",
+                                "JOJ3-config-generator",
+                                "--scoreboard-filename",
+                                "scoreboard.csv"
+                            ],
+                            "env": [
+                                "REPOS_DIR=/home/tt/.cache",
+                                "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log"
+                            ]
+                        }
+                    ]
+                }
+            },
+            "parsers": [
+                {
+                    "name": "healthcheck",
+                    "with": {
+                        "score": 0
+                    }
+                },
+                {
+                    "name": "debug",
+                    "with": {}
+                }
+            ]
+        }
+    ],
+    "postStages": [
+        {
+            "name": "teapot",
+            "groups": [],
+            "executor": {
+                "name": "local",
+                "with": {
+                    "default": {
+                        "args": [
+                            "/usr/local/bin/joint-teapot",
+                            "joj3-all-env",
+                            "/home/tt/.config/teapot/teapot.env",
+                            "--grading-repo-name",
+                            "JOJ3-config-generator",
+                            "--max-total-score",
+                            "0",
+                            "--issue-label-name",
+                            "Kind/Testing",
+                            "--issue-label-color",
+                            "#795548",
+                            "--scoreboard-filename",
+                            "scoreboard.csv"
+                        ],
+                        "env": [
+                            "REPOS_DIR=/home/tt/.cache",
+                            "LOG_FILE_PATH=/home/tt/.cache/joint-teapot-debug.log"
+                        ],
+                        "stdin": {
+                            "content": ""
+                        },
+                        "stdout": {
+                            "name": "stdout",
+                            "max": 33554432,
+                            "pipe": true
+                        },
+                        "stderr": {
+                            "name": "stderr",
+                            "max": 33554432,
+                            "pipe": true
+                        },
+                        "cpuLimit": 30000000000,
+                        "clockLimit": 60000000000,
+                        "memoryLimit": 268435456,
+                        "stackLimit": 0,
+                        "procLimit": 50,
+                        "cpuRateLimit": 0,
+                        "cpuSetLimit": "",
+                        "copyIn": {},
+                        "copyInCached": {},
+                        "copyInDir": ".",
+                        "copyOut": [
+                            "stdout",
+                            "stderr"
+                        ],
+                        "copyOutCached": [],
+                        "copyOutMax": 0,
+                        "copyOutDir": "",
+                        "tty": false,
+                        "strictMemoryLimit": false,
+                        "dataSegmentLimit": false,
+                        "addressSpaceLimit": false
+                    },
+                    "cases": []
+                }
+            },
+            "parsers": [
+                {
+                    "name": "log",
+                    "with": {
+                        "filename": "stdout",
+                        "msg": "joj3 summary",
+                        "level": -4
+                    }
+                },
+                {
+                    "name": "log",
+                    "with": {
+                        "filename": "stderr",
+                        "msg": "joint-teapot stderr",
+                        "level": 0
+                    }
+                },
+                {
+                    "name": "debug",
+                    "with": {}
+                }
+            ]
+        }
+    ]
+}

tests/convert/whitelisted-chars/task.toml

@@ -0,0 +1,2 @@
+name = "health check"
+max-total-score = 0