| { |
| "additional_special_tokens": [ |
| "<|endoftext|>", |
| "<|fim_prefix|>", |
| "<|fim_middle|>", |
| "<|fim_suffix|>", |
| "<|endofprompt|>", |
| "<|_unuse_missing_100256|>", |
| "<|_unuse_missing_100261|>", |
| "<|_unuse_missing_100262|>", |
| "<|_unuse_missing_100263|>", |
| "<|_unuse_missing_100264|>", |
| "<|_unuse_missing_100265|>", |
| "<|_unuse_missing_100266|>", |
| "<|_unuse_missing_100267|>", |
| "<|_unuse_missing_100268|>", |
| "<|_unuse_missing_100269|>", |
| "<|_unuse_missing_100270|>", |
| "<|dummy3|>", |
| "<|im_start|>", |
| "<|im_end|>", |
| "<|stop|>", |
| "<|endofturn|>", |
| "<repo_name>", |
| "<file_sep>", |
| "<issue_start>", |
| "<issue_comment>", |
| "<issue_closed>", |
| "<jupyter_start>", |
| "<jupyter_text>", |
| "<jupyter_code>", |
| "<jupyter_output>", |
| "<jupyter_script>", |
| "<empty_output>", |
| "<code_to_intermediate>", |
| "<intermediate_to_code>", |
| "<pr>", |
| "<pr_status>", |
| "<pr_is_merged>", |
| "<pr_base>", |
| "<pr_file>", |
| "<pr_base_code>", |
| "<pr_diff>", |
| "<pr_diff_hunk>", |
| "<pr_comment>", |
| "<pr_event_id>", |
| "<pr_review>", |
| "<pr_review_state>", |
| "<pr_review_comment>", |
| "<pr_in_reply_to_review_id>", |
| "<pr_in_reply_to_comment_id>", |
| "<pr_diff_hunk_comment_line>", |
| "<NAME>", |
| "<EMAIL>", |
| "<KEY>", |
| "<PASSWORD>" |
| ], |
| "bos_token": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "eos_token": { |
| "content": "<|endofturn|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "pad_token": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| }, |
| "unk_token": { |
| "content": "<|endoftext|>", |
| "lstrip": false, |
| "normalized": false, |
| "rstrip": false, |
| "single_word": false |
| } |
| } |
|
|
