dvilasuero HF staff commited on
Commit
5731897
1 Parent(s): add5a21

Upload tokenizer

Browse files
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "<|im_end|>": 32001,
3
+ "<|im_start|>": 32000
4
+ }
special_tokens_map.json CHANGED
@@ -1,30 +1,19 @@
1
  {
2
- "additional_special_tokens": [
3
- "<unk>",
4
- "<s>",
5
- "</s>"
6
- ],
7
  "bos_token": {
8
- "content": "<s>",
9
  "lstrip": false,
10
  "normalized": false,
11
  "rstrip": false,
12
  "single_word": false
13
  },
14
  "eos_token": {
15
- "content": "</s>",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false
20
- },
21
- "pad_token": {
22
- "content": "</s>",
23
  "lstrip": false,
24
  "normalized": false,
25
  "rstrip": false,
26
  "single_word": false
27
  },
 
28
  "unk_token": {
29
  "content": "<unk>",
30
  "lstrip": false,
 
1
  {
 
 
 
 
 
2
  "bos_token": {
3
+ "content": "<|im_start|>",
4
  "lstrip": false,
5
  "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "<|im_end|>",
 
 
 
 
 
 
 
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
+ "pad_token": "<|im_end|>",
17
  "unk_token": {
18
  "content": "<unk>",
19
  "lstrip": false,
tokenizer.json CHANGED
@@ -29,6 +29,24 @@
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  }
33
  ],
34
  "normalizer": {
@@ -53,7 +71,7 @@
53
  "single": [
54
  {
55
  "SpecialToken": {
56
- "id": "<s>",
57
  "type_id": 0
58
  }
59
  },
@@ -67,7 +85,7 @@
67
  "pair": [
68
  {
69
  "SpecialToken": {
70
- "id": "<s>",
71
  "type_id": 0
72
  }
73
  },
@@ -79,7 +97,7 @@
79
  },
80
  {
81
  "SpecialToken": {
82
- "id": "<s>",
83
  "type_id": 1
84
  }
85
  },
@@ -91,13 +109,13 @@
91
  }
92
  ],
93
  "special_tokens": {
94
- "<s>": {
95
- "id": "<s>",
96
  "ids": [
97
- 1
98
  ],
99
  "tokens": [
100
- "<s>"
101
  ]
102
  }
103
  }
 
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
32
+ },
33
+ {
34
+ "id": 32000,
35
+ "content": "<|im_start|>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ },
42
+ {
43
+ "id": 32001,
44
+ "content": "<|im_end|>",
45
+ "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
+ "normalized": false,
49
+ "special": true
50
  }
51
  ],
52
  "normalizer": {
 
71
  "single": [
72
  {
73
  "SpecialToken": {
74
+ "id": "<|im_start|>",
75
  "type_id": 0
76
  }
77
  },
 
85
  "pair": [
86
  {
87
  "SpecialToken": {
88
+ "id": "<|im_start|>",
89
  "type_id": 0
90
  }
91
  },
 
97
  },
98
  {
99
  "SpecialToken": {
100
+ "id": "<|im_start|>",
101
  "type_id": 1
102
  }
103
  },
 
109
  }
110
  ],
111
  "special_tokens": {
112
+ "<|im_start|>": {
113
+ "id": "<|im_start|>",
114
  "ids": [
115
+ 32000
116
  ],
117
  "tokens": [
118
+ "<|im_start|>"
119
  ]
120
  }
121
  }
tokenizer_config.json CHANGED
@@ -25,24 +25,37 @@
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  }
29
  },
30
- "additional_special_tokens": [
31
- "<unk>",
32
- "<s>",
33
- "</s>"
34
- ],
35
- "bos_token": "<s>",
36
  "clean_up_tokenization_spaces": false,
37
- "eos_token": "</s>",
38
  "legacy": true,
39
  "model_max_length": 1000000000000000019884624838656,
40
- "pad_token": "</s>",
41
- "padding_side": "left",
42
  "sp_model_kwargs": {},
43
  "spaces_between_special_tokens": false,
44
- "split_special_tokens": false,
45
  "tokenizer_class": "LlamaTokenizer",
 
46
  "unk_token": "<unk>",
47
- "use_default_system_prompt": true
 
48
  }
 
25
  "rstrip": false,
26
  "single_word": false,
27
  "special": true
28
+ },
29
+ "32000": {
30
+ "content": "<|im_start|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "32001": {
38
+ "content": "<|im_end|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
  }
45
  },
46
+ "additional_special_tokens": [],
47
+ "bos_token": "<|im_start|>",
48
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
 
 
 
49
  "clean_up_tokenization_spaces": false,
50
+ "eos_token": "<|im_end|>",
51
  "legacy": true,
52
  "model_max_length": 1000000000000000019884624838656,
53
+ "pad_token": "<|im_end|>",
 
54
  "sp_model_kwargs": {},
55
  "spaces_between_special_tokens": false,
 
56
  "tokenizer_class": "LlamaTokenizer",
57
+ "trust_remote_code": false,
58
  "unk_token": "<unk>",
59
+ "use_default_system_prompt": true,
60
+ "use_fast": true
61
  }