Commit
•
45f6546
1
Parent(s):
0bb343d
tokenizer improved
Browse files- tokenizer.json +0 -6
tokenizer.json
CHANGED
|
@@ -1128,12 +1128,6 @@
|
|
| 1128 |
"prepend_scheme": "never",
|
| 1129 |
"split": true
|
| 1130 |
},
|
| 1131 |
-
{
|
| 1132 |
-
"type": "Strip",
|
| 1133 |
-
"content": " ",
|
| 1134 |
-
"start": 1,
|
| 1135 |
-
"stop": 0
|
| 1136 |
-
},
|
| 1137 |
{
|
| 1138 |
"type": "Replace",
|
| 1139 |
"pattern": {
|
|
|
|
| 1128 |
"prepend_scheme": "never",
|
| 1129 |
"split": true
|
| 1130 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1131 |
{
|
| 1132 |
"type": "Replace",
|
| 1133 |
"pattern": {
|