| from typing import List | |
| COLUMNS_PRETTY = { | |
| "bleu": "BLEU", | |
| "chrf": "ChrF", | |
| "rouge1": "ROUGE-1", | |
| "rouge2": "ROUGE-2", | |
| "rougeL": "ROUGE-L", | |
| "bertscore": "BERTScore", | |
| "bertscore_normalized": "BERTScore (Normalized)", | |
| "model_name": "Model Name", | |
| "model_availability": "Availability", | |
| "urls": "Resources", | |
| "context_size": "Context Size", | |
| "submitted_by": "Submitted By", | |
| } | |
| METRICS_PER_TASK = { | |
| "commit_message_generation": [ | |
| "BLEU", | |
| "ChrF", | |
| "ROUGE-1", | |
| "ROUGE-2", | |
| "ROUGE-L", | |
| "BERTScore", | |
| "BERTScore (Normalized)", | |
| ], | |
| "project_code_completion": [ | |
| "EM infile", | |
| "EM inproject", | |
| "EM common", | |
| "EM commited", | |
| "EM non-informative", | |
| "EM random", | |
| "EM all", | |
| ] | |
| } | |
| SORT_COLUMN_PER_TASK = { | |
| "commit_message_generation": "ROUGE-1", | |
| "project_code_completion": "EM infile" | |
| } | |
| def get_columns_per_task(task_id: str) -> List[str]: | |
| metrics_per_task = METRICS_PER_TASK[task_id] | |
| return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"] | |
| def get_types_per_task(task_id: str) -> List[str]: | |
| metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0)) | |
| return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"] | |