saridormi commited on
Commit
2d1a3df
β€’
1 Parent(s): 1a033b9

Fix returns and file names

Browse files
Files changed (1) hide show
  1. src/submission_uploader.py +55 -57
src/submission_uploader.py CHANGED
@@ -1,10 +1,8 @@
1
  import json
2
- import logging
3
  import os
4
  from typing import List, Optional
5
 
6
- from huggingface_hub import (CommitInfo, CommitOperationAdd, Discussion, HfApi,
7
- HfFileSystem)
8
 
9
  from .tasks import TASKS_PRETTY_REVERSE
10
 
@@ -56,7 +54,7 @@ class SubmissionUploader:
56
  # add predictions files
57
  commit_operations = [
58
  CommitOperationAdd(
59
- path_in_repo=f"{task_id}/{model_folder}/predictions/{filename}",
60
  path_or_fileobj=filename,
61
  )
62
  for filename in filenames
@@ -92,58 +90,58 @@ class SubmissionUploader:
92
  submitted_by: str,
93
  filenames: Optional[List[str]],
94
  force: bool = False,
95
- ) -> Optional[CommitInfo]:
96
- pr_title = f"πŸš€ New submission to {task_pretty} task: {model_name_pretty} with {context_size} context size from {submitted_by}"
97
-
98
- task_id = TASKS_PRETTY_REVERSE[task_pretty]
99
-
100
- if not force:
101
- if model_name_pretty in self._fs.ls(
102
- f"datasets/{self._dataset_id}/{task_id}/predictions"
103
- ) and all(
104
- filename
105
- in self._fs.ls(
106
- f"datasets/{self._dataset_id}/{task_id}/predictions/{model_name_pretty}"
107
- )
108
- for filename in filenames + ["metadata.json"]
109
- ):
110
- raise AlreadyExists(
111
- f"{model_name_pretty} is already present in {self._dataset_id}."
112
- )
113
-
114
- prev_pr = self._get_previous_pr(pr_title)
115
- if prev_pr is not None:
116
- url = f"https://huggingface.co/datasets/{self._dataset_id}/discussions/{prev_pr.num}"
117
- raise AlreadyExists(
118
- f"{self._dataset_id} already has an open PR for this submission: {url}."
119
- )
120
-
121
- commit_operations = self._upload_files(
122
- task_id=task_id,
123
- model_folder=model_folder,
124
- model_name_pretty=model_name_pretty,
125
- model_availability=model_availability,
126
- urls=urls,
127
- context_size=context_size,
128
- submitted_by=submitted_by,
129
- filenames=filenames,
130
- )
131
 
132
- new_pr = self._api.create_commit(
133
- repo_id=self._dataset_id,
134
- operations=commit_operations,
135
- commit_message=pr_title,
136
- commit_description=f"""New submission to {task_pretty} task in 🏟️Long Code Arena benchmark!
137
-
138
- * Model name: {model_name_pretty}
139
- * Model availability: {model_availability}
140
- * Context Size: {context_size}
141
- * Relevant URLs: {urls}
142
- * Submitted By: {submitted_by}
143
- """,
144
- create_pr=True,
145
- repo_type="dataset",
146
- )
147
- logging.info(f"PR created at {new_pr.pr_url}")
148
 
149
- return new_pr
 
 
1
  import json
 
2
  import os
3
  from typing import List, Optional
4
 
5
+ from huggingface_hub import CommitOperationAdd, Discussion, HfApi, HfFileSystem
 
6
 
7
  from .tasks import TASKS_PRETTY_REVERSE
8
 
 
54
  # add predictions files
55
  commit_operations = [
56
  CommitOperationAdd(
57
+ path_in_repo=f"{task_id}/{model_folder}/predictions/{os.path.basename(filename)}",
58
  path_or_fileobj=filename,
59
  )
60
  for filename in filenames
 
90
  submitted_by: str,
91
  filenames: Optional[List[str]],
92
  force: bool = False,
93
+ ) -> str:
94
+ try:
95
+ pr_title = f"πŸš€ New submission to {task_pretty} task: {model_name_pretty} with {context_size} context size from {submitted_by}"
96
+
97
+ task_id = TASKS_PRETTY_REVERSE[task_pretty]
98
+
99
+ if not force:
100
+ if model_name_pretty in self._fs.ls(
101
+ f"datasets/{self._dataset_id}/{task_id}/predictions"
102
+ ) and all(
103
+ filename
104
+ in self._fs.ls(
105
+ f"datasets/{self._dataset_id}/{task_id}/predictions/{model_name_pretty}"
106
+ )
107
+ for filename in filenames + ["metadata.json"]
108
+ ):
109
+ return (
110
+ f"{model_name_pretty} is already present in {self._dataset_id}."
111
+ )
112
+
113
+ prev_pr = self._get_previous_pr(pr_title)
114
+ if prev_pr is not None:
115
+ url = f"https://huggingface.co/datasets/{self._dataset_id}/discussions/{prev_pr.num}"
116
+ return f"{self._dataset_id} already has an open PR for this submission: {url}."
117
+
118
+ commit_operations = self._upload_files(
119
+ task_id=task_id,
120
+ model_folder=model_folder,
121
+ model_name_pretty=model_name_pretty,
122
+ model_availability=model_availability,
123
+ urls=urls,
124
+ context_size=context_size,
125
+ submitted_by=submitted_by,
126
+ filenames=filenames,
127
+ )
 
128
 
129
+ new_pr = self._api.create_commit(
130
+ repo_id=self._dataset_id,
131
+ operations=commit_operations,
132
+ commit_message=pr_title,
133
+ commit_description=f"""New submission to {task_pretty} task in 🏟️ Long Code Arena benchmark!
134
+
135
+ * Model name: {model_name_pretty}
136
+ * Model availability: {model_availability}
137
+ * Context Size: {context_size}
138
+ * Relevant URLs: {urls}
139
+ * Submitted By: {submitted_by}
140
+ """,
141
+ create_pr=True,
142
+ repo_type="dataset",
143
+ )
144
+ return f"πŸŽ‰ PR created at {new_pr.pr_url}."
145
 
146
+ except Exception:
147
+ return "An exception occured."