Add dataset field to code completion task
Browse files
src/leaderboard_formatting.py
CHANGED
@@ -20,6 +20,7 @@ COLUMNS_PRETTY = {
|
|
20 |
"EM non_informative": "EM non-informative",
|
21 |
"EM random": "EM random",
|
22 |
"EM all": "EM all",
|
|
|
23 |
}
|
24 |
|
25 |
|
@@ -47,16 +48,19 @@ METRICS_PER_TASK = {
|
|
47 |
|
48 |
SORT_COLUMN_PER_TASK = {
|
49 |
"commit_message_generation": "ROUGE-1",
|
50 |
-
"project_code_completion": "
|
51 |
}
|
52 |
|
53 |
|
54 |
def get_columns_per_task(task_id: str) -> List[str]:
|
55 |
metrics_per_task = METRICS_PER_TASK[task_id]
|
56 |
-
|
|
|
57 |
return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"]
|
58 |
|
59 |
|
60 |
def get_types_per_task(task_id: str) -> List[str]:
|
61 |
metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0))
|
|
|
|
|
62 |
return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]
|
|
|
20 |
"EM non_informative": "EM non-informative",
|
21 |
"EM random": "EM random",
|
22 |
"EM all": "EM all",
|
23 |
+
"dataset": "Dataset",
|
24 |
}
|
25 |
|
26 |
|
|
|
48 |
|
49 |
SORT_COLUMN_PER_TASK = {
|
50 |
"commit_message_generation": "ROUGE-1",
|
51 |
+
"project_code_completion": "Dataset"
|
52 |
}
|
53 |
|
54 |
|
55 |
def get_columns_per_task(task_id: str) -> List[str]:
|
56 |
metrics_per_task = METRICS_PER_TASK[task_id]
|
57 |
+
if task_id == 'project_code_completion':
|
58 |
+
return ["Model Name", "Availability", "Context Size", "Dataset"] + metrics_per_task + ["Submitted By", "Resources"]
|
59 |
return ["Model Name", "Availability", "Context Size"] + metrics_per_task + ["Submitted By", "Resources"]
|
60 |
|
61 |
|
62 |
def get_types_per_task(task_id: str) -> List[str]:
|
63 |
metrics_per_task = METRICS_PER_TASK.get(task_id, (0, 0, 0, 0, 0))
|
64 |
+
if task_id == 'project_code_completion':
|
65 |
+
return ["html", "markdown", "markdown", "html"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]
|
66 |
return ["html", "markdown", "markdown"] + ["number" for _ in metrics_per_task] + ["markdown", "html"]
|