File size: 2,561 Bytes
8cabccb 3d7ea69 e125c93 8cabccb e125c93 3d7ea69 2c37af2 3d7ea69 2c37af2 b2d8d13 e125c93 3d7ea69 b2d8d13 3d7ea69 b2d8d13 3d7ea69 e125c93 3d7ea69 e125c93 3d7ea69 e125c93 3d7ea69 e125c93 3d7ea69 e125c93 3d7ea69 8cabccb e125c93 8cabccb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
{
"add_bos_token": false,
"add_eos_token": false,
"added_tokens_decoder": {
"0": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32000": {
"content": "<image>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"32001": {
"content": "</image>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"32002": {
"content": "<audio>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"32003": {
"content": "</audio>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
}
},
"bos_token": "<s>",
"chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{% if message['content'] is not none and message['content'].strip() != '' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% endif %}{% elif message['role'] == 'assistant' %}{% if messages[loop.index0 - 1]['content'] is not none and messages[loop.index0 - 1]['content'].strip() != '' and message['content'] is not none and message['content'].strip() != '' %}{{ message['content'] + eos_token }}{% endif %}{% else %}{{ print('Unexpected role encountered:', message['role']) }}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
"clean_up_tokenization_spaces": false,
"eos_token": "</s>",
"legacy": false,
"max_length": 8192,
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<unk>",
"padding_side": "right",
"sp_model_kwargs": {},
"stride": 0,
"tokenizer_class": "LlamaTokenizer",
"truncation_side": "right",
"truncation_strategy": "longest_first",
"unk_token": "<unk>",
"use_default_system_prompt": false
}
|