Upload tokenizer

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer.json

@@ -0,0 +1,1279 @@
+{
+  "version": "1.0",
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "BertNormalizer",
+    "clean_text": true,
+    "handle_chinese_chars": true,
+    "strip_accents": null,
+    "lowercase": true
+  },
+  "pre_tokenizer": {
+    "type": "BertPreTokenizer"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "[CLS]": {
+        "id": "[CLS]",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "[CLS]"
+        ]
+      },
+      "[SEP]": {
+        "id": "[SEP]",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "[SEP]"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "WordPiece",
+    "prefix": "##",
+    "cleanup": true
+  },
+  "model": {
+    "type": "WordPiece",
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": "##",
+    "max_input_chars_per_word": 100,
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "[CLS]": 2,
+      "[SEP]": 3,
+      "[MASK]": 4,
+      "!": 5,
+      "\"": 6,
+      "#": 7,
+      "$": 8,
+      "%": 9,
+      "&": 10,
+      "'": 11,
+      "(": 12,
+      ")": 13,
+      "*": 14,
+      "+": 15,
+      ",": 16,
+      "-": 17,
+      ".": 18,
+      "/": 19,
+      "0": 20,
+      "1": 21,
+      "2": 22,
+      "3": 23,
+      "4": 24,
+      "5": 25,
+      "6": 26,
+      "7": 27,
+      "8": 28,
+      "9": 29,
+      ":": 30,
+      ";": 31,
+      "<": 32,
+      "=": 33,
+      ">": 34,
+      "?": 35,
+      "@": 36,
+      "[": 37,
+      "\\": 38,
+      "]": 39,
+      "^": 40,
+      "_": 41,
+      "`": 42,
+      "a": 43,
+      "b": 44,
+      "c": 45,
+      "d": 46,
+      "e": 47,
+      "f": 48,
+      "g": 49,
+      "h": 50,
+      "i": 51,
+      "j": 52,
+      "k": 53,
+      "l": 54,
+      "m": 55,
+      "n": 56,
+      "o": 57,
+      "p": 58,
+      "q": 59,
+      "r": 60,
+      "s": 61,
+      "t": 62,
+      "u": 63,
+      "v": 64,
+      "w": 65,
+      "x": 66,
+      "y": 67,
+      "z": 68,
+      "|": 69,
+      "}": 70,
+      "~": 71,
+      "¡": 72,
+      "¢": 73,
+      "£": 74,
+      "¥": 75,
+      "§": 76,
+      "°": 77,
+      "±": 78,
+      "²": 79,
+      "³": 80,
+      "´": 81,
+      "µ": 82,
+      "·": 83,
+      "º": 84,
+      "½": 85,
+      "¿": 86,
+      "×": 87,
+      "ß": 88,
+      "æ": 89,
+      "ð": 90,
+      "ø": 91,
+      "þ": 92,
+      "đ": 93,
+      "ħ": 94,
+      "ı": 95,
+      "ł": 96,
+      "œ": 97,
+      "ɐ": 98,
+      "ɑ": 99,
+      "ɒ": 100,
+      "ɔ": 101,
+      "ə": 102,
+      "ɛ": 103,
+      "ɜ": 104,
+      "ɡ": 105,
+      "ɢ": 106,
+      "ɪ": 107,
+      "ɫ": 108,
+      "ɳ": 109,
+      "ɽ": 110,
+      "ɾ": 111,
+      "ʁ": 112,
+      "ʃ": 113,
+      "ʊ": 114,
+      "ʋ": 115,
+      "ʒ": 116,
+      "ʔ": 117,
+      "ʕ": 118,
+      "ʲ": 119,
+      "ʻ": 120,
+      "ʼ": 121,
+      "ʾ": 122,
+      "ʿ": 123,
+      "ˈ": 124,
+      "ˌ": 125,
+      "ː": 126,
+      "α": 127,
+      "β": 128,
+      "γ": 129,
+      "δ": 130,
+      "ε": 131,
+      "η": 132,
+      "θ": 133,
+      "ι": 134,
+      "κ": 135,
+      "λ": 136,
+      "μ": 137,
+      "ν": 138,
+      "ξ": 139,
+      "ο": 140,
+      "π": 141,
+      "ρ": 142,
+      "ς": 143,
+      "σ": 144,
+      "τ": 145,
+      "υ": 146,
+      "φ": 147,
+      "χ": 148,
+      "ψ": 149,
+      "ω": 150,
+      "а": 151,
+      "б": 152,
+      "в": 153,
+      "г": 154,
+      "д": 155,
+      "е": 156,
+      "ж": 157,
+      "з": 158,
+      "и": 159,
+      "к": 160,
+      "л": 161,
+      "м": 162,
+      "н": 163,
+      "о": 164,
+      "п": 165,
+      "р": 166,
+      "с": 167,
+      "т": 168,
+      "у": 169,
+      "х": 170,
+      "ц": 171,
+      "ш": 172,
+      "ъ": 173,
+      "ы": 174,
+      "ь": 175,
+      "ю": 176,
+      "я": 177,
+      "є": 178,
+      "א": 179,
+      "ב": 180,
+      "ג": 181,
+      "ה": 182,
+      "ו": 183,
+      "ז": 184,
+      "ח": 185,
+      "י": 186,
+      "ל": 187,
+      "ם": 188,
+      "מ": 189,
+      "ן": 190,
+      "נ": 191,
+      "ס": 192,
+      "ף": 193,
+      "פ": 194,
+      "צ": 195,
+      "ר": 196,
+      "ש": 197,
+      "ת": 198,
+      "ء": 199,
+      "ا": 200,
+      "ب": 201,
+      "ة": 202,
+      "ت": 203,
+      "ث": 204,
+      "ج": 205,
+      "ح": 206,
+      "خ": 207,
+      "د": 208,
+      "ذ": 209,
+      "ر": 210,
+      "س": 211,
+      "ش": 212,
+      "ص": 213,
+      "ع": 214,
+      "ف": 215,
+      "ق": 216,
+      "ك": 217,
+      "ل": 218,
+      "م": 219,
+      "ن": 220,
+      "ه": 221,
+      "و": 222,
+      "ي": 223,
+      "ܐ": 224,
+      "ܕ": 225,
+      "ܗ": 226,
+      "ܝ": 227,
+      "ܠ": 228,
+      "ܢ": 229,
+      "ܬ": 230,
+      "अ": 231,
+      "ई": 232,
+      "क": 233,
+      "ग": 234,
+      "ण": 235,
+      "त": 236,
+      "द": 237,
+      "न": 238,
+      "प": 239,
+      "ब": 240,
+      "म": 241,
+      "य": 242,
+      "र": 243,
+      "ल": 244,
+      "व": 245,
+      "स": 246,
+      "ह": 247,
+      "ा": 248,
+      "ि": 249,
+      "আ": 250,
+      "ল": 251,
+      "হ": 252,
+      "া": 253,
+      "ਅ": 254,
+      "ਲ": 255,
+      "ਹ": 256,
+      "ਾ": 257,
+      "അ": 258,
+      "ള": 259,
+      "ഹ": 260,
+      "ാ": 261,
+      "ก": 262,
+      "ค": 263,
+      "ง": 264,
+      "ช": 265,
+      "ซ": 266,
+      "ญ": 267,
+      "ฐ": 268,
+      "ณ": 269,
+      "ด": 270,
+      "ต": 271,
+      "น": 272,
+      "บ": 273,
+      "ป": 274,
+      "พ": 275,
+      "ภ": 276,
+      "ม": 277,
+      "ย": 278,
+      "ร": 279,
+      "ล": 280,
+      "ว": 281,
+      "ศ": 282,
+      "ษ": 283,
+      "ส": 284,
+      "ห": 285,
+      "อ": 286,
+      "ฮ": 287,
+      "ะ": 288,
+      "า": 289,
+      "เ": 290,
+      "แ": 291,
+      "ไ": 292,
+      "ა": 293,
+      "ბ": 294,
+      "გ": 295,
+      "დ": 296,
+      "ე": 297,
+      "ვ": 298,
+      "ზ": 299,
+      "თ": 300,
+      "ი": 301,
+      "კ": 302,
+      "ლ": 303,
+      "მ": 304,
+      "ნ": 305,
+      "ო": 306,
+      "პ": 307,
+      "ჟ": 308,
+      "რ": 309,
+      "ს": 310,
+      "ტ": 311,
+      "უ": 312,
+      "ფ": 313,
+      "ქ": 314,
+      "ღ": 315,
+      "ყ": 316,
+      "შ": 317,
+      "ჩ": 318,
+      "ც": 319,
+      "ძ": 320,
+      "წ": 321,
+      "ჭ": 322,
+      "ხ": 323,
+      "ჯ": 324,
+      "ჰ": 325,
+      "ჱ": 326,
+      "ჲ": 327,
+      "ჳ": 328,
+      "ჴ": 329,
+      "ჵ": 330,
+      "ჶ": 331,
+      "ჷ": 332,
+      "ჸ": 333,
+      "ჹ": 334,
+      "ჺ": 335,
+      "჻": 336,
+      "ᄃ": 337,
+      "ᄅ": 338,
+      "ᄇ": 339,
+      "ᄋ": 340,
+      "ᄌ": 341,
+      "ᅡ": 342,
+      "ᅢ": 343,
+      "ᅦ": 344,
+      "ᅧ": 345,
+      "ᅩ": 346,
+      "ᅮ": 347,
+      "ᅵ": 348,
+      "ᆨ": 349,
+      "ᆫ": 350,
+      "ᆯ": 351,
+      "ᆸ": 352,
+      "ᆼ": 353,
+      "ᵻ": 354,
+      "‐": 355,
+      "‑": 356,
+      "–": 357,
+      "—": 358,
+      "―": 359,
+      "‘": 360,
+      "’": 361,
+      "“": 362,
+      "”": 363,
+      "„": 364,
+      "†": 365,
+      "‡": 366,
+      "•": 367,
+      "…": 368,
+      "′": 369,
+      "″": 370,
+      "⁄": 371,
+      "₣": 372,
+      "₤": 373,
+      "€": 374,
+      "₹": 375,
+      "⅓": 376,
+      "⅔": 377,
+      "→": 378,
+      "−": 379,
+      "≡": 380,
+      "≤": 381,
+      "①": 382,
+      "☉": 383,
+      "☫": 384,
+      "♀": 385,
+      "♭": 386,
+      "♯": 387,
+      "⚳": 388,
+      "ⴀ": 389,
+      "ⴂ": 390,
+      "ⴃ": 391,
+      "ⴈ": 392,
+      "ⴌ": 393,
+      "ⴕ": 394,
+      "ⴟ": 395,
+      "〈": 396,
+      "〉": 397,
+      "〜": 398,
+      "あ": 399,
+      "い": 400,
+      "う": 401,
+      "お": 402,
+      "か": 403,
+      "き": 404,
+      "く": 405,
+      "け": 406,
+      "こ": 407,
+      "さ": 408,
+      "し": 409,
+      "す": 410,
+      "せ": 411,
+      "た": 412,
+      "ち": 413,
+      "っ": 414,
+      "つ": 415,
+      "と": 416,
+      "な": 417,
+      "に": 418,
+      "の": 419,
+      "は": 420,
+      "ひ": 421,
+      "ふ": 422,
+      "ほ": 423,
+      "ま": 424,
+      "み": 425,
+      "め": 426,
+      "も": 427,
+      "ゃ": 428,
+      "ゆ": 429,
+      "ょ": 430,
+      "ら": 431,
+      "り": 432,
+      "る": 433,
+      "れ": 434,
+      "わ": 435,
+      "を": 436,
+      "ん": 437,
+      "ァ": 438,
+      "ア": 439,
+      "ィ": 440,
+      "イ": 441,
+      "ゥ": 442,
+      "ウ": 443,
+      "ェ": 444,
+      "エ": 445,
+      "ォ": 446,
+      "オ": 447,
+      "カ": 448,
+      "キ": 449,
+      "ク": 450,
+      "ケ": 451,
+      "コ": 452,
+      "サ": 453,
+      "シ": 454,
+      "ス": 455,
+      "セ": 456,
+      "タ": 457,
+      "チ": 458,
+      "ッ": 459,
+      "ツ": 460,
+      "テ": 461,
+      "ト": 462,
+      "ナ": 463,
+      "ニ": 464,
+      "ネ": 465,
+      "ノ": 466,
+      "ハ": 467,
+      "フ": 468,
+      "ヘ": 469,
+      "マ": 470,
+      "ミ": 471,
+      "ム": 472,
+      "モ": 473,
+      "ャ": 474,
+      "ュ": 475,
+      "ョ": 476,
+      "ラ": 477,
+      "リ": 478,
+      "ル": 479,
+      "レ": 480,
+      "ロ": 481,
+      "ン": 482,
+      "・": 483,
+      "ー": 484,
+      "一": 485,
+      "七": 486,
+      "下": 487,
+      "世": 488,
+      "丙": 489,
+      "中": 490,
+      "主": 491,
+      "乃": 492,
+      "之": 493,
+      "乙": 494,
+      "九": 495,
+      "二": 496,
+      "云": 497,
+      "人": 498,
+      "今": 499,
+      "付": 500,
+      "作": 501,
+      "侗": 502,
+      "依": 503,
+      "信": 504,
+      "傳": 505,
+      "儚": 506,
+      "充": 507,
+      "光": 508,
+      "全": 509,
+      "兵": 510,
+      "其": 511,
+      "具": 512,
+      "円": 513,
+      "再": 514,
+      "出": 515,
+      "判": 516,
+      "前": 517,
+      "剛": 518,
+      "劇": 519,
+      "劉": 520,
+      "動": 521,
+      "化": 522,
+      "北": 523,
+      "华": 524,
+      "厂": 525,
+      "去": 526,
+      "古": 527,
+      "可": 528,
+      "台": 529,
+      "史": 530,
+      "同": 531,
+      "名": 532,
+      "君": 533,
+      "吳": 534,
+      "周": 535,
+      "命": 536,
+      "和": 537,
+      "咲": 538,
+      "善": 539,
+      "四": 540,
+      "國": 541,
+      "園": 542,
+      "圣": 543,
+      "在": 544,
+      "坂": 545,
+      "堤": 546,
+      "場": 547,
+      "塘": 548,
+      "夕": 549,
+      "大": 550,
+      "天": 551,
+      "夫": 552,
+      "女": 553,
+      "妙": 554,
+      "姚": 555,
+      "子": 556,
+      "孟": 557,
+      "守": 558,
+      "安": 559,
+      "宋": 560,
+      "完": 561,
+      "宗": 562,
+      "宝": 563,
+      "宫": 564,
+      "寝": 565,
+      "寺": 566,
+      "小": 567,
+      "少": 568,
+      "尾": 569,
+      "山": 570,
+      "岳": 571,
+      "川": 572,
+      "州": 573,
+      "巳": 574,
+      "市": 575,
+      "師": 576,
+      "平": 577,
+      "广": 578,
+      "庆": 579,
+      "府": 580,
+      "座": 581,
+      "廬": 582,
+      "建": 583,
+      "式": 584,
+      "張": 585,
+      "彌": 586,
+      "彩": 587,
+      "彼": 588,
+      "後": 589,
+      "御": 590,
+      "德": 591,
+      "思": 592,
+      "愛": 593,
+      "憑": 594,
+      "憶": 595,
+      "應": 596,
+      "懷": 597,
+      "战": 598,
+      "戦": 599,
+      "扈": 600,
+      "技": 601,
+      "拉": 602,
+      "拳": 603,
+      "挑": 604,
+      "揺": 605,
+      "攻": 606,
+      "放": 607,
+      "政": 608,
+      "散": 609,
+      "斯": 610,
+      "方": 611,
+      "日": 612,
+      "旦": 613,
+      "旭": 614,
+      "昌": 615,
+      "明": 616,
+      "星": 617,
+      "春": 618,
+      "晋": 619,
+      "景": 620,
+      "曦": 621,
+      "月": 622,
+      "望": 623,
+      "未": 624,
+      "本": 625,
+      "李": 626,
+      "村": 627,
+      "杜": 628,
+      "束": 629,
+      "来": 630,
+      "林": 631,
+      "桜": 632,
+      "梶": 633,
+      "棘": 634,
+      "椎": 635,
+      "楊": 636,
+      "楚": 637,
+      "榮": 638,
+      "橘": 639,
+      "機": 640,
+      "正": 641,
+      "殻": 642,
+      "殿": 643,
+      "母": 644,
+      "水": 645,
+      "汉": 646,
+      "沂": 647,
+      "沙": 648,
+      "河": 649,
+      "泗": 650,
+      "波": 651,
+      "泣": 652,
+      "洪": 653,
+      "淹": 654,
+      "清": 655,
+      "湯": 656,
+      "漢": 657,
+      "澄": 658,
+      "澤": 659,
+      "火": 660,
+      "灯": 661,
+      "灵": 662,
+      "灼": 663,
+      "焼": 664,
+      "熱": 665,
+      "物": 666,
+      "狐": 667,
+      "狸": 668,
+      "玄": 669,
+      "王": 670,
+      "玩": 671,
+      "珂": 672,
+      "珙": 673,
+      "球": 674,
+      "理": 675,
+      "琦": 676,
+      "琪": 677,
+      "瓊": 678,
+      "生": 679,
+      "田": 680,
+      "畢": 681,
+      "番": 682,
+      "瘡": 683,
+      "白": 684,
+      "皮": 685,
+      "真": 686,
+      "砲": 687,
+      "礮": 688,
+      "祈": 689,
+      "神": 690,
+      "祠": 691,
+      "秋": 692,
+      "空": 693,
+      "立": 694,
+      "精": 695,
+      "約": 696,
+      "絵": 697,
+      "織": 698,
+      "義": 699,
+      "翠": 700,
+      "者": 701,
+      "耕": 702,
+      "肖": 703,
+      "胡": 704,
+      "膀": 705,
+      "臂": 706,
+      "興": 707,
+      "良": 708,
+      "花": 709,
+      "芳": 710,
+      "芽": 711,
+      "若": 712,
+      "英": 713,
+      "藕": 714,
+      "藥": 715,
+      "蘄": 716,
+      "蘇": 717,
+      "行": 718,
+      "裁": 719,
+      "規": 720,
+      "覺": 721,
+      "观": 722,
+      "解": 723,
+      "記": 724,
+      "誓": 725,
+      "誡": 726,
+      "誰": 727,
+      "謎": 728,
+      "许": 729,
+      "谭": 730,
+      "豪": 731,
+      "豫": 732,
+      "費": 733,
+      "贵": 734,
+      "赤": 735,
+      "趙": 736,
+      "足": 737,
+      "跡": 738,
+      "転": 739,
+      "辛": 740,
+      "逆": 741,
+      "遇": 742,
+      "運": 743,
+      "過": 744,
+      "遠": 745,
+      "選": 746,
+      "邦": 747,
+      "邱": 748,
+      "部": 749,
+      "郭": 750,
+      "都": 751,
+      "酈": 752,
+      "里": 753,
+      "野": 754,
+      "金": 755,
+      "銃": 756,
+      "鋼": 757,
+      "錄": 758,
+      "錡": 759,
+      "鍵": 760,
+      "鐵": 761,
+      "钱": 762,
+      "铁": 763,
+      "關": 764,
+      "防": 765,
+      "阿": 766,
+      "陈": 767,
+      "陳": 768,
+      "陽": 769,
+      "隊": 770,
+      "階": 771,
+      "集": 772,
+      "雪": 773,
+      "雲": 774,
+      "霖": 775,
+      "霹": 776,
+      "靂": 777,
+      "韓": 778,
+      "願": 779,
+      "顯": 780,
+      "颜": 781,
+      "马": 782,
+      "高": 783,
+      "龍": 784,
+      "ﷲ": 785,
+      "ﻋ": 786,
+      "／": 787,
+      "３": 788,
+      "～": 789,
+      "##o": 790,
+      "##l": 791,
+      "##e": 792,
+      "##y": 793,
+      "##i": 794,
+      "##n": 795,
+      "##g": 796,
+      "##r": 797,
+      "##c": 798,
+      "##u": 799,
+      "##d": 800,
+      "##m": 801,
+      "##b": 802,
+      "##a": 803,
+      "##p": 804,
+      "##h": 805,
+      "##s": 806,
+      "##t": 807,
+      "##v": 808,
+      "##w": 809,
+      "##k": 810,
+      "##ı": 811,
+      "##z": 812,
+      "##f": 813,
+      "##x": 814,
+      "##η": 815,
+      "##μ": 816,
+      "##τ": 817,
+      "##ρ": 818,
+      "##α": 819,
+      "##2": 820,
+      "##5": 821,
+      "##7": 822,
+      "##8": 823,
+      "##j": 824,
+      "##q": 825,
+      "##0": 826,
+      "##4": 827,
+      "##6": 828,
+      "##ł": 829,
+      "##3": 830,
+      "##1": 831,
+      "##ل": 832,
+      "##ع": 833,
+      "##ة": 834,
+      "##ɛ": 835,
+      "##ß": 836,
+      "##ə": 837,
+      "##ˈ": 838,
+      "##9": 839,
+      "##ɪ": 840,
+      "##ป": 841,
+      "##ย": 842,
+      "##о": 843,
+      "##ю": 844,
+      "##з": 845,
+      "##æ": 846,
+      "##の": 847,
+      "##な": 848,
+      "##ら": 849,
+      "##რ": 850,
+      "##ი": 851,
+      "##ს": 852,
+      "##ტ": 853,
+      "##ე": 854,
+      "##ʻ": 855,
+      "##ᄇ": 856,
+      "##ᅮ": 857,
+      "##ᄃ": 858,
+      "##ᅢ": 859,
+      "##お": 860,
+      "##ო": 861,
+      "##მ": 862,
+      "##თ": 863,
+      "##ა": 864,
+      "##ვ": 865,
+      "##უ": 866,
+      "##ლ": 867,
+      "##ث": 868,
+      "##م": 869,
+      "##د": 870,
+      "##ウ": 871,
+      "##ァ": 872,
+      "##ル": 873,
+      "##キ": 874,
+      "##ュ": 875,
+      "##リ": 876,
+      "##ア": 877,
+      "##а": 878,
+      "##т": 879,
+      "##р": 880,
+      "##и": 881,
+      "##ш": 882,
+      "##е": 883,
+      "##с": 884,
+      "##к": 885,
+      "##ʊ": 886,
+      "##ˌ": 887,
+      "##ː": 888,
+      "##つ": 889,
+      "##せ": 890,
+      "##ん": 891,
+      "##ッ": 892,
+      "##ク": 893,
+      "##マ": 894,
+      "##ン": 895,
+      "##ɫ": 896,
+      "##ق": 897,
+      "##ي": 898,
+      "##ش": 899,
+      "##و": 900,
+      "##ب": 901,
+      "##ø": 902,
+      "##ノ": 903,
+      "##ー": 904,
+      "##ล": 905,
+      "##อ": 906,
+      "##ฮ": 907,
+      "##ラ": 908,
+      "##ω": 909,
+      "##ε": 910,
+      "##ι": 911,
+      "##ο": 912,
+      "##ς": 913,
+      "##พ": 914,
+      "##ร": 915,
+      "##ל": 916,
+      "##ה": 917,
+      "##ह": 918,
+      "##ण": 919,
+      "##đ": 920,
+      "##ゆ": 921,
+      "##り": 922,
+      "##ल": 923,
+      "##ν": 924,
+      "##κ": 925,
+      "##ص": 926,
+      "##ن": 927,
+      "##イ": 928,
+      "##ス": 929,
+      "##ा": 930,
+      "##द": 931,
+      "##き": 932,
+      "##こ": 933,
+      "##と": 934,
+      "##あ": 935,
+      "##ɑ": 936,
+      "##セ": 937,
+      "##オ": 938,
+      "##ʿ": 939,
+      "##ᅡ": 940,
+      "##ᆯ": 941,
+      "##ᄅ": 942,
+      "##テ": 943,
+      "##ィ": 944,
+      "##ニ": 945,
+      "##ল": 946,
+      "##º": 947,
+      "##ก": 948,
+      "##ว": 949,
+      "##ɳ": 950,
+      "##β": 951,
+      "##ت": 952,
+      "##פ": 953,
+      "##ხ": 954,
+      "##ษ": 955,
+      "##ฐ": 956,
+      "##ן": 957,
+      "##ɜ": 958,
+      "##ᵻ": 959,
+      "##व": 960,
+      "##ि": 961,
+      "##क": 962,
+      "##υ": 963,
+      "##н": 964,
+      "##わ": 965,
+      "##い": 966,
+      "##ɡ": 967,
+      "##は": 968,
+      "##く": 969,
+      "##ც": 970,
+      "##χ": 971,
+      "##た": 972,
+      "##ð": 973,
+      "##δ": 974,
+      "##ʁ": 975,
+      "##σ": 976,
+      "##ر": 977,
+      "##ナ": 978,
+      "##ミ": 979,
+      "##ב": 980,
+      "##ר": 981,
+      "##ם": 982,
+      "##წ": 983,
+      "##ფ": 984,
+      "##う": 985,
+      "##カ": 986,
+      "##დ": 987,
+      "##ま": 988,
+      "##る": 989,
+      "##λ": 990,
+      "##π": 991,
+      "##ъ": 992,
+      "##в": 993,
+      "##ാ": 994,
+      "##ഹ": 995,
+      "##ا": 996,
+      "##３": 997,
+      "##ハ": 998,
+      "##ト": 999,
+      "##न": 1000,
+      "##ई": 1001,
+      "##י": 1002,
+      "##ש": 1003,
+      "##מ": 1004,
+      "##צ": 1005,
+      "##ะ": 1006,
+      "##ช": 1007,
+      "##น": 1008,
+      "##ロ": 1009,
+      "##ჲ": 1010,
+      "##œ": 1011,
+      "##ɾ": 1012,
+      "##เ": 1013,
+      "##д": 1014,
+      "##ж": 1015,
+      "##л": 1016,
+      "##я": 1017,
+      "##θ": 1018,
+      "##ง": 1019,
+      "##ไ": 1020,
+      "##า": 1021,
+      "##っ": 1022,
+      "##ɒ": 1023,
+      "##か": 1024,
+      "##レ": 1025,
+      "##チ": 1026,
+      "##ы": 1027,
+      "##ォ": 1028,
+      "##ゥ": 1029,
+      "##ツ": 1030,
+      "##ネ": 1031,
+      "##ц": 1032,
+      "##ь": 1033,
+      "##め": 1034,
+      "##ち": 1035,
+      "##ゃ": 1036,
+      "##є": 1037,
+      "##ء": 1038,
+      "##れ": 1039,
+      "##み": 1040,
+      "##п": 1041,
+      "##г": 1042,
+      "##ف": 1043,
+      "##س": 1044,
+      "##サ": 1045,
+      "##ו": 1046,
+      "##נ": 1047,
+      "##у": 1048,
+      "##シ": 1049,
+      "##ョ": 1050,
+      "##ه": 1051,
+      "##ェ": 1052,
+      "##х": 1053,
+      "##し": 1054,
+      "##す": 1055,
+      "##ひ": 1056,
+      "##フ": 1057,
+      "##ャ": 1058,
+      "##モ": 1059,
+      "##ภ": 1060,
+      "##ด": 1061,
+      "##र": 1062,
+      "##ح": 1063,
+      "##ס": 1064,
+      "##א": 1065,
+      "##ɔ": 1066,
+      "##ʃ": 1067,
+      "##ਲ": 1068,
+      "##ਾ": 1069,
+      "##ਹ": 1070,
+      "##ჱ": 1071,
+      "##ʔ": 1072,
+      "##ܕ": 1073,
+      "##ܝ": 1074,
+      "##ܢ": 1075,
+      "##ܬ": 1076,
+      "##エ": 1077,
+      "##タ": 1078,
+      "##ك": 1079,
+      "##خ": 1080,
+      "##ξ": 1081,
+      "##ذ": 1082,
+      "##ქ": 1083,
+      "##ნ": 1084,
+      "##ܠ": 1085,
+      "##ܗ": 1086,
+      "##ܐ": 1087,
+      "##ム": 1088,
+      "##м": 1089,
+      "##γ": 1090,
+      "##ψ": 1091,
+      "##ള": 1092,
+      "##ჳ": 1093,
+      "##แ": 1094,
+      "##स": 1095,
+      "##ת": 1096,
+      "##ค": 1097,
+      "##ต": 1098,
+      "##გ": 1099,
+      "##ʒ": 1100,
+      "##ซ": 1101,
+      "##ʲ": 1102,
+      "##も": 1103,
+      "##ょ": 1104,
+      "##に": 1105,
+      "##া": 1106,
+      "##হ": 1107,
+      "##ɽ": 1108,
+      "##ᅦ": 1109,
+      "##ᅩ": 1110,
+      "##ᆨ": 1111,
+      "##ᅵ": 1112,
+      "##ᆸ": 1113,
+      "##ᅧ": 1114,
+      "##ᆼ": 1115,
+      "##ᄋ": 1116,
+      "##ᆫ": 1117,
+      "##ग": 1118,
+      "##ɐ": 1119,
+      "##を": 1120,
+      "##ף": 1121,
+      "##ณ": 1122,
+      "##ม": 1123
+    }
+  }
+}

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "name_or_path": "hf-internal-testing/tiny-random-distilbert",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "DistilBertTokenizer",
+  "unk_token": "[UNK]"
+}

vocab.txt

@@ -0,0 +1,1124 @@
+[PAD]
+[UNK]
+[CLS]
+[SEP]
+[MASK]
+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+:
+;
+<
+=
+>
+?
+@
+[
+\
+]
+^
+_
+`
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+|
+}
+~
+¡
+¢
+£
+¥
+§
+°
+±
+²
+³
+´
+µ
+·
+º
+½
+¿
+×
+ß
+æ
+ð
+ø
+þ
+đ
+ħ
+ı
+ł
+œ
+ɐ
+ɑ
+ɒ
+ɔ
+ə
+ɛ
+ɜ
+ɡ
+ɢ
+ɪ
+ɫ
+ɳ
+ɽ
+ɾ
+ʁ
+ʃ
+ʊ
+ʋ
+ʒ
+ʔ
+ʕ
+ʲ
+ʻ
+ʼ
+ʾ
+ʿ
+ˈ
+ˌ
+ː
+α
+β
+γ
+δ
+ε
+η
+θ
+ι
+κ
+λ
+μ
+ν
+ξ
+ο
+π
+ρ
+ς
+σ
+τ
+υ
+φ
+χ
+ψ
+ω
+а
+б
+в
+г
+д
+е
+ж
+з
+и
+к
+л
+м
+н
+о
+п
+р
+с
+т
+у
+х
+ц
+ш
+ъ
+ы
+ь
+ю
+я
+є
+א
+ב
+ג
+ה
+ו
+ז
+ח
+י
+ל
+ם
+מ
+ן
+נ
+ס
+ף
+פ
+צ
+ר
+ש
+ת
+ء
+ا
+ب
+ة
+ت
+ث
+ج
+ح
+خ
+د
+ذ
+ر
+س
+ش
+ص
+ع
+ف
+ق
+ك
+ل
+م
+ن
+ه
+و
+ي
+ܐ
+ܕ
+ܗ
+ܝ
+ܠ
+ܢ
+ܬ
+अ
+ई
+क
+ग
+ण
+त
+द
+न
+प
+ब
+म
+य
+र
+ल
+व
+स
+ह
+ा
+ि
+আ
+ল
+হ
+া
+ਅ
+ਲ
+ਹ
+ਾ
+അ
+ള
+ഹ
+ാ
+ก
+ค
+ง
+ช
+ซ
+ญ
+ฐ
+ณ
+ด
+ต
+น
+บ
+ป
+พ
+ภ
+ม
+ย
+ร
+ล
+ว
+ศ
+ษ
+ส
+ห
+อ
+ฮ
+ะ
+า
+เ
+แ
+ไ
+ა
+ბ
+გ
+დ
+ე
+ვ
+ზ
+თ
+ი
+კ
+ლ
+მ
+ნ
+ო
+პ
+ჟ
+რ
+ს
+ტ
+უ
+ფ
+ქ
+ღ
+ყ
+შ
+ჩ
+ც
+ძ
+წ
+ჭ
+ხ
+ჯ
+ჰ
+ჱ
+ჲ
+ჳ
+ჴ
+ჵ
+ჶ
+ჷ
+ჸ
+ჹ
+ჺ
+჻
+ᄃ
+ᄅ
+ᄇ
+ᄋ
+ᄌ
+ᅡ
+ᅢ
+ᅦ
+ᅧ
+ᅩ
+ᅮ
+ᅵ
+ᆨ
+ᆫ
+ᆯ
+ᆸ
+ᆼ
+ᵻ
+‐
+‑
+–
+—
+―
+‘
+’
+“
+”
+„
+†
+‡
+•
+…
+′
+″
+⁄
+₣
+₤
+€
+₹
+⅓
+⅔
+→
+−
+≡
+≤
+①
+☉
+☫
+♀
+♭
+♯
+⚳
+ⴀ
+ⴂ
+ⴃ
+ⴈ
+ⴌ
+ⴕ
+ⴟ
+〈
+〉
+〜
+あ
+い
+う
+お
+か
+き
+く
+け
+こ
+さ
+し
+す
+せ
+た
+ち
+っ
+つ
+と
+な
+に
+の
+は
+ひ
+ふ
+ほ
+ま
+み
+め
+も
+ゃ
+ゆ
+ょ
+ら
+り
+る
+れ
+わ
+を
+ん
+ァ
+ア
+ィ
+イ
+ゥ
+ウ
+ェ
+エ
+ォ
+オ
+カ
+キ
+ク
+ケ
+コ
+サ
+シ
+ス
+セ
+タ
+チ
+ッ
+ツ
+テ
+ト
+ナ
+ニ
+ネ
+ノ
+ハ
+フ
+ヘ
+マ
+ミ
+ム
+モ
+ャ
+ュ
+ョ
+ラ
+リ
+ル
+レ
+ロ
+ン
+・
+ー
+一
+七
+下
+世
+丙
+中
+主
+乃
+之
+乙
+九
+二
+云
+人
+今
+付
+作
+侗
+依
+信
+傳
+儚
+充
+光
+全
+兵
+其
+具
+円
+再
+出
+判
+前
+剛
+劇
+劉
+動
+化
+北
+华
+厂
+去
+古
+可
+台
+史
+同
+名
+君
+吳
+周
+命
+和
+咲
+善
+四
+國
+園
+圣
+在
+坂
+堤
+場
+塘
+夕
+大
+天
+夫
+女
+妙
+姚
+子
+孟
+守
+安
+宋
+完
+宗
+宝
+宫
+寝
+寺
+小
+少
+尾
+山
+岳
+川
+州
+巳
+市
+師
+平
+广
+庆
+府
+座
+廬
+建
+式
+張
+彌
+彩
+彼
+後
+御
+德
+思
+愛
+憑
+憶
+應
+懷
+战
+戦
+扈
+技
+拉
+拳
+挑
+揺
+攻
+放
+政
+散
+斯
+方
+日
+旦
+旭
+昌
+明
+星
+春
+晋
+景
+曦
+月
+望
+未
+本
+李
+村
+杜
+束
+来
+林
+桜
+梶
+棘
+椎
+楊
+楚
+榮
+橘
+機
+正
+殻
+殿
+母
+水
+汉
+沂
+沙
+河
+泗
+波
+泣
+洪
+淹
+清
+湯
+漢
+澄
+澤
+火
+灯
+灵
+灼
+焼
+熱
+物
+狐
+狸
+玄
+王
+玩
+珂
+珙
+球
+理
+琦
+琪
+瓊
+生
+田
+畢
+番
+瘡
+白
+皮
+真
+砲
+礮
+祈
+神
+祠
+秋
+空
+立
+精
+約
+絵
+織
+義
+翠
+者
+耕
+肖
+胡
+膀
+臂
+興
+良
+花
+芳
+芽
+若
+英
+藕
+藥
+蘄
+蘇
+行
+裁
+規
+覺
+观
+解
+記
+誓
+誡
+誰
+謎
+许
+谭
+豪
+豫
+費
+贵
+赤
+趙
+足
+跡
+転
+辛
+逆
+遇
+運
+過
+遠
+選
+邦
+邱
+部
+郭
+都
+酈
+里
+野
+金
+銃
+鋼
+錄
+錡
+鍵
+鐵
+钱
+铁
+關
+防
+阿
+陈
+陳
+陽
+隊
+階
+集
+雪
+雲
+霖
+霹
+靂
+韓
+願
+顯
+颜
+马
+高
+龍
+ﷲ
+ﻋ
+／
+３
+～
+##o
+##l
+##e
+##y
+##i
+##n
+##g
+##r
+##c
+##u
+##d
+##m
+##b
+##a
+##p
+##h
+##s
+##t
+##v
+##w
+##k
+##ı
+##z
+##f
+##x
+##η
+##μ
+##τ
+##ρ
+##α
+##2
+##5
+##7
+##8
+##j
+##q
+##0
+##4
+##6
+##ł
+##3
+##1
+##ل
+##ع
+##ة
+##ɛ
+##ß
+##ə
+##ˈ
+##9
+##ɪ
+##ป
+##ย
+##о
+##ю
+##з
+##æ
+##の
+##な
+##ら
+##რ
+##ი
+##ს
+##ტ
+##ე
+##ʻ
+##ᄇ
+##ᅮ
+##ᄃ
+##ᅢ
+##お
+##ო
+##მ
+##თ
+##ა
+##ვ
+##უ
+##ლ
+##ث
+##م
+##د
+##ウ
+##ァ
+##ル
+##キ
+##ュ
+##リ
+##ア
+##а
+##т
+##р
+##и
+##ш
+##е
+##с
+##к
+##ʊ
+##ˌ
+##ː
+##つ
+##せ
+##ん
+##ッ
+##ク
+##マ
+##ン
+##ɫ
+##ق
+##ي
+##ش
+##و
+##ب
+##ø
+##ノ
+##ー
+##ล
+##อ
+##ฮ
+##ラ
+##ω
+##ε
+##ι
+##ο
+##ς
+##พ
+##ร
+##ל
+##ה
+##ह
+##ण
+##đ
+##ゆ
+##り
+##ल
+##ν
+##κ
+##ص
+##ن
+##イ
+##ス
+##ा
+##द
+##き
+##こ
+##と
+##あ
+##ɑ
+##セ
+##オ
+##ʿ
+##ᅡ
+##ᆯ
+##ᄅ
+##テ
+##ィ
+##ニ
+##ল
+##º
+##ก
+##ว
+##ɳ
+##β
+##ت
+##פ
+##ხ
+##ษ
+##ฐ
+##ן
+##ɜ
+##ᵻ
+##व
+##ि
+##क
+##υ
+##н
+##わ
+##い
+##ɡ
+##は
+##く
+##ც
+##χ
+##た
+##ð
+##δ
+##ʁ
+##σ
+##ر
+##ナ
+##ミ
+##ב
+##ר
+##ם
+##წ
+##ფ
+##う
+##カ
+##დ
+##ま
+##る
+##λ
+##π
+##ъ
+##в
+##ാ
+##ഹ
+##ا
+##３
+##ハ
+##ト
+##न
+##ई
+##י
+##ש
+##מ
+##צ
+##ะ
+##ช
+##น
+##ロ
+##ჲ
+##œ
+##ɾ
+##เ
+##д
+##ж
+##л
+##��
+##θ
+##ง
+##ไ
+##า
+##っ
+##ɒ
+##か
+##レ
+##チ
+##ы
+##ォ
+##ゥ
+##ツ
+##ネ
+##ц
+##ь
+##め
+##ち
+##ゃ
+##є
+##ء
+##れ
+##み
+##п
+##г
+##ف
+##س
+##サ
+##ו
+##נ
+##у
+##シ
+##ョ
+##ه
+##ェ
+##х
+##し
+##す
+##ひ
+##フ
+##ャ
+##モ
+##ภ
+##ด
+##र
+##ح
+##ס
+##א
+##ɔ
+##ʃ
+##ਲ
+##ਾ
+##ਹ
+##ჱ
+##ʔ
+##ܕ
+##ܝ
+##ܢ
+##ܬ
+##エ
+##タ
+##ك
+##خ
+##ξ
+##ذ
+##ქ
+##ნ
+##ܠ
+##ܗ
+##ܐ
+##ム
+##м
+##γ
+##ψ
+##ള
+##ჳ
+##แ
+##स
+##ת
+##ค
+##ต
+##გ
+##ʒ
+##ซ
+##ʲ
+##も
+##ょ
+##に
+##া
+##হ
+##ɽ
+##ᅦ
+##ᅩ
+##ᆨ
+##ᅵ
+##ᆸ
+##ᅧ
+##ᆼ
+##ᄋ
+##ᆫ
+##ग
+##ɐ
+##を
+##ף
+##ณ
+##ม