Austism commited on
Commit
b5a5141
1 Parent(s): 475b072

Delete tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +0 -32
tokenizer_config.json DELETED
@@ -1,32 +0,0 @@
1
- {
2
- "bos_token": {
3
- "__type": "AddedToken",
4
- "content": "<s>",
5
- "lstrip": false,
6
- "normalized": true,
7
- "rstrip": false,
8
- "single_word": false
9
- },
10
- "clean_up_tokenization_spaces": false,
11
- "eos_token": {
12
- "__type": "AddedToken",
13
- "content": "</s>",
14
- "lstrip": false,
15
- "normalized": true,
16
- "rstrip": false,
17
- "single_word": false
18
- },
19
- "legacy": false,
20
- "model_max_length": 1000000000000000019884624838656,
21
- "pad_token": null,
22
- "sp_model_kwargs": {},
23
- "tokenizer_class": "LlamaTokenizer",
24
- "unk_token": {
25
- "__type": "AddedToken",
26
- "content": "<unk>",
27
- "lstrip": false,
28
- "normalized": true,
29
- "rstrip": false,
30
- "single_word": false
31
- }
32
- }