ruslanmv commited on
Commit
5ff4013
1 Parent(s): 2c17890

Upload tokenizer.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer.json +1 -0
tokenizer.json CHANGED
@@ -2348,6 +2348,7 @@
2348
  "end_of_word_suffix": null,
2349
  "fuse_unk": false,
2350
  "byte_fallback": false,
 
2351
  "vocab": {
2352
  "!": 0,
2353
  "\"": 1,
 
2348
  "end_of_word_suffix": null,
2349
  "fuse_unk": false,
2350
  "byte_fallback": false,
2351
+ "ignore_merges": false,
2352
  "vocab": {
2353
  "!": 0,
2354
  "\"": 1,