| { |
| "bos_token": { |
| "content": "<[begin▁of▁sentence]>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "eos_token": { |
| "content": "<[end▁of▁sentence]>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "pad_token": { |
| "content": "<[PAD▁TOKEN]>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "sep_token": { |
| "content": "<[SEP▁TOKEN]>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "svector_token": { |
| "content": "<[SVECTOR]>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| } |
| } |
