```python
##################################################################################
# 分为三个部分
# 1. tokenizer部分
# 2. transformer部分
# 3. pooling部分

# from multiprocessing.pool import Pool
import numpy as np
import onnxruntime
import psutil
from sympy import im
from transformers import (AutoConfig, AutoModel, AutoTokenizer)
import os
import json
from sentence_transformers.models import Pooling

from sentence_transformers import SentenceTransformer as sbert

from tqdm import tqdm
import torch as t

```

```python

##################################################################################
# 处理transformer和 tokenizer部分

big_model_path = "../models/paraphrase-multilingual-MiniLM-L12-v2"

modules_json_path = os.path.join(big_model_path, 'modules.json')
with open(modules_json_path) as fIn:
    modules_config = json.load(fIn)

tf_from_s_path = os.path.join(big_model_path, modules_config[0].get('path'))

# 基本参数

max_seq_length = 128
doc_stride = 128
max_query_length = 64
# Enable overwrite to export onnx model and download latest script each time when running this notebook.
enable_overwrite = True
# Total samples to inference. It shall be large enough to get stable latency measurement.
total_samples = 1000

# # Load pretrained model and tokenizer
# Load pretrained model and tokenizer
config_class, model_class, tokenizer_class = (
    AutoConfig, AutoModel, AutoTokenizer)

cache_dir = os.path.join(".", "cache_models")
config = config_class.from_pretrained(tf_from_s_path, cache_dir=cache_dir)
tokenizer = tokenizer_class.from_pretrained(
    tf_from_s_path, do_lower_case=True, cache_dir=cache_dir)
model_transformer = model_class.from_pretrained(
    tf_from_s_path, from_tf=False, config=config, cache_dir=cache_dir)

```

```python

##################################################################################
# 使用onnx 和cuda推理部分

output_dir = os.path.join("..", "onnx_models")
export_model_path = os.path.join(output_dir, 'Multilingual_MiniLM_L12.onnx')
device_name = 'gpu'
sess_options = onnxruntime.SessionOptions()
sess_options.optimized_model_filepath = os.path.join(
    output_dir, "optimized_model_{}.onnx".format(device_name))
# Please change the value according to best setting in Performance Test Tool result.
sess_options.intra_op_num_threads = psutil.cpu_count(logical=True)
session = onnxruntime.InferenceSession(
    export_model_path, sess_options, providers=['CUDAExecutionProvider'])

```

    2022-02-25 14:01:58.432835797 [W:onnxruntime:, inference_session.cc:1407 Initialize] Serializing optimized model with Graph Optimization level greater than ORT_ENABLE_EXTENDED and the NchwcTransformer enabled. The generated model may contain hardware specific optimizations, and should only be used in the same environment the model was optimized in.

```python
##################################################################################
# 处理pooling部分
pooling_model_path = os.path.join(big_model_path, modules_config[1].get('path'))
pooling_model = Pooling.load(pooling_model_path)
# pooling_model_path
```

```python

##################################################################################
# 推理函数

def inferpart(session, sentences, pooling_model):
    if isinstance(sentences, str) or not hasattr(sentences, '__len__'):
        sentences = [sentences]

    inputs = tokenizer(
        sentences,
        padding=True,
        truncation=True,
        max_length=512,
        return_tensors="pt"
    )
    ort_inputs = {k:v.cpu().numpy() for k, v in inputs.items()}
    ort_outputs_gpu = session.run(None, ort_inputs)
    ort_result = pooling_model.forward(features={'token_embeddings':t.Tensor(ort_outputs_gpu[0]),
    'attention_mask':inputs.get('attention_mask')})
    result = ort_result.get('sentence_embedding')
    return result

_ = [inferpart(session=session, sentences = ['您好'], pooling_model=pooling_model) for i in tqdm(range(2000))]
```

    100%|██████████| 2000/2000 [00:05<00:00, 340.29it/s]

```python
# 使用原生的sentence transformer代码
model_sbert_raw = sbert(big_model_path, device='cuda')

_ = [model_sbert_raw.encode(['您好'],device='cuda') for i in tqdm(range(2000))]
```

    100%|██████████| 2000/2000 [00:25<00:00, 77.36it/s]

```python

```

```python

```


#1 导入NeMo NLP工具类及相关工具库

```python
!pip install Cython
!pip install nemo_toolkit['all']
```

    Requirement already satisfied: Cython in /usr/local/lib/python3.7/dist-packages (0.29.24)
    Collecting nemo_toolkit[all]
      Downloading nemo_toolkit-1.2.0-py3-none-any.whl (1.2 MB)
         |████████████████████████████████| 1.2 MB 7.1 MB/s
    Collecting hydra-core>=1.1.0
      Downloading hydra_core-1.1.1-py3-none-any.whl (145 kB)
         |████████████████████████████████| 145 kB 59.4 MB/s
    Collecting transformers<=4.8.1,>=4.0.1
      Downloading transformers-4.8.1-py3-none-any.whl (2.5 MB)
         |████████████████████████████████| 2.5 MB 29.8 MB/s
    Collecting ruamel.yaml
      Downloading ruamel.yaml-0.17.13-py3-none-any.whl (108 kB)
         |████████████████████████████████| 108 kB 74.7 MB/s
    Requirement already satisfied: python-dateutil in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (2.8.2)
    Requirement already satisfied: tqdm>=4.41.0 in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (4.62.0)
    Collecting onnx>=1.7.0
      Downloading onnx-1.10.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (12.3 MB)
         |████████████████████████████████| 12.3 MB 178 kB/s
    Collecting opencc
      Downloading OpenCC-1.1.2-cp37-cp37m-manylinux1_x86_64.whl (765 kB)
         |████████████████████████████████| 765 kB 59.1 MB/s
    Collecting webdataset<=0.1.62,>=0.1.48
      Downloading webdataset-0.1.62-py3-none-any.whl (32 kB)
    Requirement already satisfied: wrapt in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (1.12.1)
    Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (1.9.0+cu102)
    Collecting pangu
      Downloading pangu-4.0.6.1-py3-none-any.whl (6.4 kB)
    Requirement already satisfied: numpy>=1.18.2 in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (1.19.5)
    Requirement already satisfied: numba in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (0.51.2)
    Collecting sentencepiece<1.0.0
      Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)
         |████████████████████████████████| 1.2 MB 62.3 MB/s
    Collecting pytorch-lightning<1.4.0,>=1.3.0
      Downloading pytorch_lightning-1.3.8-py3-none-any.whl (813 kB)
         |████████████████████████████████| 813 kB 64.6 MB/s
    Collecting wget
      Downloading wget-3.2.zip (10 kB)
    Collecting omegaconf>=2.1.0
      Downloading omegaconf-2.1.1-py3-none-any.whl (74 kB)
         |████████████████████████████████| 74 kB 4.8 MB/s
    Collecting torchmetrics>=0.4.1rc0
      Downloading torchmetrics-0.5.0-py3-none-any.whl (272 kB)
         |████████████████████████████████| 272 kB 83.7 MB/s
    Requirement already satisfied: jieba in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (0.42.1)
    Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (0.22.2.post1)
    Requirement already satisfied: sphinx in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (1.8.5)
    Collecting sacremoses>=0.0.43
      Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)
         |████████████████████████████████| 895 kB 41.3 MB/s
    Collecting g2p-en
      Downloading g2p_en-2.1.0-py3-none-any.whl (3.1 MB)
         |████████████████████████████████| 3.1 MB 76.0 MB/s
    Collecting sox
      Downloading sox-1.4.1-py2.py3-none-any.whl (39 kB)
    Collecting black==19.10b0
      Downloading black-19.10b0-py36-none-any.whl (97 kB)
         |████████████████████████████████| 97 kB 10.3 MB/s
    Requirement already satisfied: librosa in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (0.8.1)
    Collecting marshmallow
      Downloading marshmallow-3.13.0-py2.py3-none-any.whl (47 kB)
         |████████████████████████████████| 47 kB 6.0 MB/s
    Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (3.2.2)
    Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (7.1.2)
    Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (2019.12.20)
    Collecting pytest-runner
      Downloading pytest_runner-5.3.1-py3-none-any.whl (7.1 kB)
    Collecting wordninja==2.0.0
      Downloading wordninja-2.0.0.tar.gz (541 kB)
         |████████████████████████████████| 541 kB 68.9 MB/s
    Collecting rapidfuzz
      Downloading rapidfuzz-1.5.0-cp37-cp37m-manylinux2010_x86_64.whl (1.4 MB)
         |████████████████████████████████| 1.4 MB 58.2 MB/s
    Collecting nltk==3.6.2
      Downloading nltk-3.6.2-py3-none-any.whl (1.5 MB)
         |████████████████████████████████| 1.5 MB 20.2 MB/s
    Collecting pyannote.metrics
      Downloading pyannote.metrics-3.0.1-py3-none-any.whl (49 kB)
         |████████████████████████████████| 49 kB 7.6 MB/s
    Collecting matplotlib>=3.3.2
      Downloading matplotlib-3.4.3-cp37-cp37m-manylinux1_x86_64.whl (10.3 MB)
         |████████████████████████████████| 10.3 MB 20.5 MB/s
    Collecting youtokentome>=1.0.5
      Downloading youtokentome-1.0.6-cp37-cp37m-manylinux2010_x86_64.whl (1.7 MB)
         |████████████████████████████████| 1.7 MB 78.0 MB/s
    Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (0.10.0+cu102)
    Requirement already satisfied: gdown in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (3.6.4)
    Collecting pystoi
      Downloading pystoi-0.3.3.tar.gz (7.0 kB)
    Requirement already satisfied: inflect in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (2.1.0)
    Collecting sphinxcontrib-bibtex
      Downloading sphinxcontrib_bibtex-2.3.0-py3-none-any.whl (35 kB)
    Collecting isort[requirements]<5
      Downloading isort-4.3.21-py2.py3-none-any.whl (42 kB)
         |████████████████████████████████| 42 kB 1.2 MB/s
    Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (3.2.5)
    Collecting attrdict
      Downloading attrdict-2.0.1-py2.py3-none-any.whl (9.9 kB)
    Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (1.4.1)
    Collecting kaldi-io
      Downloading kaldi_io-0.9.4-py3-none-any.whl (14 kB)
    Collecting wandb
      Downloading wandb-0.12.0-py2.py3-none-any.whl (1.6 MB)
         |████████████████████████████████| 1.6 MB 68.5 MB/s
    Collecting pydub
      Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)
    Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (21.0)
    Collecting kaldi-python-io
      Downloading kaldi-python-io-1.2.2.tar.gz (8.8 kB)
    Requirement already satisfied: ipywidgets in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (7.6.3)
    Collecting parameterized
      Downloading parameterized-0.8.1-py2.py3-none-any.whl (26 kB)
    Collecting pyannote.core
      Downloading pyannote.core-4.1-py3-none-any.whl (56 kB)
         |████████████████████████████████| 56 kB 6.4 MB/s
    Requirement already satisfied: editdistance in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (0.5.3)
    Requirement already satisfied: soundfile in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (0.10.3.post1)
    Collecting sacrebleu[ja]
      Downloading sacrebleu-2.0.0-py3-none-any.whl (90 kB)
         |████████████████████████████████| 90 kB 13.6 MB/s
    Collecting torch-stft
      Downloading torch_stft-0.1.4-py3-none-any.whl (6.2 kB)
    Collecting megatron-lm==2.2.0
      Downloading megatron_lm-2.2.0-py3-none-any.whl (171 kB)
         |████████████████████████████████| 171 kB 77.3 MB/s
    Collecting frozendict
      Downloading frozendict-2.0.6-py3-none-any.whl (8.3 kB)
    Collecting kaldiio
      Downloading kaldiio-2.17.2.tar.gz (24 kB)
    Collecting unidecode
      Downloading Unidecode-1.2.0-py2.py3-none-any.whl (241 kB)
         |████████████████████████████████| 241 kB 67.8 MB/s
    Collecting pypinyin
      Downloading pypinyin-0.42.0-py2.py3-none-any.whl (1.3 MB)
         |████████████████████████████████| 1.3 MB 67.6 MB/s
    Collecting boto3
      Downloading boto3-1.18.26-py3-none-any.whl (131 kB)
         |████████████████████████████████| 131 kB 67.4 MB/s
    Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (3.1.0)
    Requirement already satisfied: pytest in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (3.6.4)
    Collecting braceexpand
      Downloading braceexpand-0.1.7-py2.py3-none-any.whl (5.9 kB)
    Collecting pesq
      Downloading pesq-0.0.3.tar.gz (35 kB)
    Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from nemo_toolkit[all]) (1.1.5)
    Requirement already satisfied: click>=6.5 in /usr/local/lib/python3.7/dist-packages (from black==19.10b0->nemo_toolkit[all]) (7.1.2)
    Collecting typed-ast>=1.4.0
      Downloading typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl (743 kB)
         |████████████████████████████████| 743 kB 64.0 MB/s
    Requirement already satisfied: attrs>=18.1.0 in /usr/local/lib/python3.7/dist-packages (from black==19.10b0->nemo_toolkit[all]) (21.2.0)
    Collecting pathspec<1,>=0.6
      Downloading pathspec-0.9.0-py2.py3-none-any.whl (31 kB)
    Requirement already satisfied: appdirs in /usr/local/lib/python3.7/dist-packages (from black==19.10b0->nemo_toolkit[all]) (1.4.4)
    Requirement already satisfied: toml>=0.9.4 in /usr/local/lib/python3.7/dist-packages (from black==19.10b0->nemo_toolkit[all]) (0.10.2)
    Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from megatron-lm==2.2.0->nemo_toolkit[all]) (1.15.0)
    Collecting pybind11
      Downloading pybind11-2.7.1-py2.py3-none-any.whl (200 kB)
         |████████████████████████████████| 200 kB 72.7 MB/s
    Collecting antlr4-python3-runtime==4.8
      Downloading antlr4-python3-runtime-4.8.tar.gz (112 kB)
         |████████████████████████████████| 112 kB 55.7 MB/s
    Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from hydra-core>=1.1.0->nemo_toolkit[all]) (5.2.2)
    Collecting PyYAML>=5.1.0
      Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)
         |████████████████████████████████| 636 kB 60.1 MB/s
    Collecting pip-api
      Downloading pip_api-0.0.20-py3-none-any.whl (112 kB)
         |████████████████████████████████| 112 kB 70.3 MB/s
    Collecting pipreqs
      Downloading pipreqs-0.4.10-py2.py3-none-any.whl (25 kB)
    Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (1.3.1)
    Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (0.10.0)
    Requirement already satisfied: pyparsing>=2.2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.3.2->nemo_toolkit[all]) (2.4.7)
    Requirement already satisfied: typing-extensions>=3.6.2.1 in /usr/local/lib/python3.7/dist-packages (from onnx>=1.7.0->nemo_toolkit[all]) (3.7.4.3)
    Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from onnx>=1.7.0->nemo_toolkit[all]) (3.17.3)
    Collecting future>=0.17.1
      Downloading future-0.18.2.tar.gz (829 kB)
         |████████████████████████████████| 829 kB 67.3 MB/s
    Collecting pyDeprecate==0.3.0
      Downloading pyDeprecate-0.3.0-py3-none-any.whl (10 kB)
    Collecting fsspec[http]!=2021.06.0,>=2021.05.0
      Downloading fsspec-2021.7.0-py3-none-any.whl (118 kB)
         |████████████████████████████████| 118 kB 64.5 MB/s
    Requirement already satisfied: tensorboard!=2.5.0,>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (2.6.0)
    Collecting aiohttp
      Downloading aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)
         |████████████████████████████████| 1.3 MB 58.5 MB/s
    Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (2.23.0)
    Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses>=0.0.43->nemo_toolkit[all]) (1.0.1)
    Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (0.12.0)
    Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (0.4.5)
    Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (1.34.0)
    Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (1.8.0)
    Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (1.39.0)
    Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (1.0.1)
    Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (57.4.0)
    Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (0.6.1)
    Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (0.37.0)
    Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (3.3.4)
    Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (4.2.2)
    Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (4.7.2)
    Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (0.2.8)
    Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (1.3.0)
    Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (4.6.4)
    Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (0.4.8)
    Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (3.0.4)
    Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (1.24.3)
    Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (2021.5.30)
    Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (2.10)
    Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (3.1.1)
    Collecting huggingface-hub==0.0.12
      Downloading huggingface_hub-0.0.12-py3-none-any.whl (37 kB)
    Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<=4.8.1,>=4.0.1->nemo_toolkit[all]) (3.0.12)
    Collecting tokenizers<0.11,>=0.10.1
      Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)
         |████████████████████████████████| 3.3 MB 69.3 MB/s
    Collecting multidict<7.0,>=4.5
      Downloading multidict-5.1.0-cp37-cp37m-manylinux2014_x86_64.whl (142 kB)
         |████████████████████████████████| 142 kB 71.7 MB/s
    Collecting yarl<2.0,>=1.0
      Downloading yarl-1.6.3-cp37-cp37m-manylinux2014_x86_64.whl (294 kB)
         |████████████████████████████████| 294 kB 77.1 MB/s
    Collecting async-timeout<4.0,>=3.0
      Downloading async_timeout-3.0.1-py3-none-any.whl (8.2 kB)
    Collecting s3transfer<0.6.0,>=0.5.0
      Downloading s3transfer-0.5.0-py3-none-any.whl (79 kB)
         |████████████████████████████████| 79 kB 11.6 MB/s
    Collecting botocore<1.22.0,>=1.21.26
      Downloading botocore-1.21.26-py3-none-any.whl (7.8 MB)
         |████████████████████████████████| 7.8 MB 68.7 MB/s
    Collecting jmespath<1.0.0,>=0.7.1
      Downloading jmespath-0.10.0-py2.py3-none-any.whl (24 kB)
    Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1
      Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)
         |████████████████████████████████| 127 kB 72.9 MB/s
    Collecting distance>=0.1.3
      Downloading Distance-0.1.3.tar.gz (180 kB)
         |████████████████████████████████| 180 kB 74.5 MB/s
    Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py->nemo_toolkit[all]) (1.5.2)
    Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard!=2.5.0,>=2.2.0->pytorch-lightning<1.4.0,>=1.3.0->nemo_toolkit[all]) (3.5.0)
    Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->nemo_toolkit[all]) (5.5.0)
    Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->nemo_toolkit[all]) (3.5.1)
    Requirement already satisfied: ipykernel>=4.5.1 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->nemo_toolkit[all]) (4.10.1)
    Requirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->nemo_toolkit[all]) (5.0.5)
    Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->nemo_toolkit[all]) (5.1.3)
    Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->nemo_toolkit[all]) (1.0.0)
    Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/dist-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (5.3.5)
    Requirement already satisfied: tornado>=4.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (5.1.1)
    Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.8.1)
    Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (4.4.2)
    Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (4.8.0)
    Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.7.5)
    Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (2.6.1)
    Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (1.0.18)
    Requirement already satisfied: jupyter-core in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (4.7.1)
    Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (2.6.0)
    Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (0.2.0)
    Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.2.5)
    Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.7/dist-packages (from widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (5.3.1)
    Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (1.8.0)
    Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.11.0)
    Requirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (5.6.1)
    Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (2.11.3)
    Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (22.2.1)
    Requirement already satisfied: ptyprocess in /usr/local/lib/python3.7/dist-packages (from terminado>=0.8.1->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.7.0)
    Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (2.0.1)
    Requirement already satisfied: resampy>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from librosa->nemo_toolkit[all]) (0.2.2)
    Requirement already satisfied: pooch>=1.0 in /usr/local/lib/python3.7/dist-packages (from librosa->nemo_toolkit[all]) (1.4.0)
    Requirement already satisfied: audioread>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from librosa->nemo_toolkit[all]) (2.1.9)
    Requirement already satisfied: llvmlite<0.35,>=0.34.0.dev0 in /usr/local/lib/python3.7/dist-packages (from numba->nemo_toolkit[all]) (0.34.0)
    Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.7/dist-packages (from soundfile->nemo_toolkit[all]) (1.14.6)
    Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.0->soundfile->nemo_toolkit[all]) (2.20)
    Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (4.0.0)
    Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.5.0)
    Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.3)
    Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.8.4)
    Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.7.1)
    Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (1.4.3)
    Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.5.1)
    Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->nemo_toolkit[all]) (2018.9)
    Requirement already satisfied: pip in /usr/local/lib/python3.7/dist-packages (from pip-api->isort[requirements]<5->nemo_toolkit[all]) (21.1.3)
    Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from pipreqs->isort[requirements]<5->nemo_toolkit[all]) (0.6.2)
    Collecting yarg
      Downloading yarg-0.1.9-py2.py3-none-any.whl (19 kB)
    Collecting simplejson>=3.8.1
      Downloading simplejson-3.17.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (129 kB)
         |████████████████████████████████| 129 kB 55.8 MB/s
    Requirement already satisfied: sortedcontainers>=2.0.4 in /usr/local/lib/python3.7/dist-packages (from pyannote.core->nemo_toolkit[all]) (2.4.0)
    Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.7/dist-packages (from pyannote.metrics->nemo_toolkit[all]) (0.8.9)
    Requirement already satisfied: sympy>=1.1 in /usr/local/lib/python3.7/dist-packages (from pyannote.metrics->nemo_toolkit[all]) (1.7.1)
    Collecting pyannote.database>=4.0.1
      Downloading pyannote.database-4.1.1-py3-none-any.whl (41 kB)
         |████████████████████████████████| 41 kB 358 kB/s
    Collecting typer[all]>=0.2.1
      Downloading typer-0.3.2-py3-none-any.whl (21 kB)
    Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.7/dist-packages (from sympy>=1.1->pyannote.metrics->nemo_toolkit[all]) (1.2.1)
    Collecting colorama<0.5.0,>=0.4.3
      Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)
    Collecting shellingham<2.0.0,>=1.3.0
      Downloading shellingham-1.4.0-py2.py3-none-any.whl (9.4 kB)
    Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.7/dist-packages (from pytest->nemo_toolkit[all]) (0.7.1)
    Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.7/dist-packages (from pytest->nemo_toolkit[all]) (1.4.0)
    Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from pytest->nemo_toolkit[all]) (1.10.0)
    Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytest->nemo_toolkit[all]) (8.8.0)
    Collecting ruamel.yaml.clib>=0.1.2
      Downloading ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl (546 kB)
         |████████████████████████████████| 546 kB 73.2 MB/s
    Collecting portalocker
      Downloading portalocker-2.3.1-py2.py3-none-any.whl (15 kB)
    Collecting ipadic<2.0,>=1.0
      Downloading ipadic-1.0.0.tar.gz (13.4 MB)
         |████████████████████████████████| 13.4 MB 210 kB/s
    Collecting mecab-python3==1.0.3
      Downloading mecab_python3-1.0.3-cp37-cp37m-manylinux1_x86_64.whl (487 kB)
         |████████████████████████████████| 487 kB 65.2 MB/s
    Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx->nemo_toolkit[all]) (2.9.1)
    Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.7/dist-packages (from sphinx->nemo_toolkit[all]) (1.2.4)
    Requirement already satisfied: docutils>=0.11 in /usr/local/lib/python3.7/dist-packages (from sphinx->nemo_toolkit[all]) (0.17.1)
    Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx->nemo_toolkit[all]) (1.2.0)
    Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx->nemo_toolkit[all]) (0.7.12)
    Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx->nemo_toolkit[all]) (2.1.0)
    Collecting sphinx
      Downloading Sphinx-4.1.2-py3-none-any.whl (2.9 MB)
         |████████████████████████████████| 2.9 MB 57.8 MB/s
    Collecting pybtex>=0.20
      Downloading pybtex-0.24.0-py2.py3-none-any.whl (561 kB)
         |████████████████████████████████| 561 kB 61.2 MB/s
    Collecting pybtex-docutils>=1.0.0
      Downloading pybtex_docutils-1.0.1-py3-none-any.whl (4.8 kB)
    Collecting latexcodec>=1.0.4
      Downloading latexcodec-2.0.1-py2.py3-none-any.whl (18 kB)
    Collecting sphinxcontrib-jsmath
      Downloading sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl (5.1 kB)
    Requirement already satisfied: sphinxcontrib-serializinghtml>=1.1.5 in /usr/local/lib/python3.7/dist-packages (from sphinx->nemo_toolkit[all]) (1.1.5)
    Collecting sphinxcontrib-htmlhelp>=2.0.0
      Downloading sphinxcontrib_htmlhelp-2.0.0-py2.py3-none-any.whl (100 kB)
         |████████████████████████████████| 100 kB 14.8 MB/s
    Collecting sphinxcontrib-applehelp
      Downloading sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl (121 kB)
         |████████████████████████████████| 121 kB 73.0 MB/s
    Collecting sphinxcontrib-qthelp
      Downloading sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl (90 kB)
         |████████████████████████████████| 90 kB 14.7 MB/s
    Collecting sphinxcontrib-devhelp
      Downloading sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl (84 kB)
         |████████████████████████████████| 84 kB 5.3 MB/s
    Collecting GitPython>=1.0.0
      Downloading GitPython-3.1.18-py3-none-any.whl (170 kB)
         |████████████████████████████████| 170 kB 77.5 MB/s
    Collecting docker-pycreds>=0.4.0
      Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)
    Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb->nemo_toolkit[all]) (5.4.8)
    Collecting configparser>=3.8.1
      Downloading configparser-5.0.2-py3-none-any.whl (19 kB)
    Collecting shortuuid>=0.5.0
      Downloading shortuuid-1.0.1-py3-none-any.whl (7.5 kB)
    Collecting sentry-sdk>=1.0.0
      Downloading sentry_sdk-1.3.1-py2.py3-none-any.whl (133 kB)
         |████████████████████████████████| 133 kB 78.4 MB/s
    Collecting subprocess32>=3.5.3
      Downloading subprocess32-3.5.4.tar.gz (97 kB)
         |████████████████████████████████| 97 kB 10.3 MB/s
    Collecting pathtools
      Downloading pathtools-0.1.2.tar.gz (11 kB)
    Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb->nemo_toolkit[all]) (2.3)
    Collecting gitdb<5,>=4.0.1
      Downloading gitdb-4.0.7-py3-none-any.whl (63 kB)
         |████████████████████████████████| 63 kB 2.5 MB/s
    Collecting smmap<5,>=3.0.1
      Downloading smmap-4.0.0-py2.py3-none-any.whl (24 kB)
    Building wheels for collected packages: wordninja, antlr4-python3-runtime, future, distance, kaldi-python-io, kaldiio, pesq, pystoi, ipadic, subprocess32, pathtools, wget
      Building wheel for wordninja (setup.py) ... done
      Created wheel for wordninja: filename=wordninja-2.0.0-py3-none-any.whl size=541551 sha256=8c42269fbea84faf2b9b649c89973429d4044b23a9f221f24560849fb4b7a497
      Stored in directory: /root/.cache/pip/wheels/dd/3f/eb/a2692e3d2b9deb1487b09ba4967dd6920bd5032bfd9ff7acfc
      Building wheel for antlr4-python3-runtime (setup.py) ... done
      Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-py3-none-any.whl size=141230 sha256=a2794b0128ad124969b11f910f591c2bcc361d0d131b5a149846a40524c45484
      Stored in directory: /root/.cache/pip/wheels/ca/33/b7/336836125fc9bb4ceaa4376d8abca10ca8bc84ddc824baea6c
      Building wheel for future (setup.py) ... done
      Created wheel for future: filename=future-0.18.2-py3-none-any.whl size=491070 sha256=ffdc73454f71505e101dca629191ddccc8d592f457e72f557ad71c3096d9d3e4
      Stored in directory: /root/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0
      Building wheel for distance (setup.py) ... done
      Created wheel for distance: filename=Distance-0.1.3-py3-none-any.whl size=16275 sha256=b6b56fad8f27c26c371a25e15e78d272003e0108ab75496f8855bf3fd9f2fb9b
      Stored in directory: /root/.cache/pip/wheels/b2/10/1b/96fca621a1be378e2fe104cfb0d160bb6cdf3d04a3d35266cc
      Building wheel for kaldi-python-io (setup.py) ... done
      Created wheel for kaldi-python-io: filename=kaldi_python_io-1.2.2-py3-none-any.whl size=8968 sha256=2ca439f96a389e7abd1b13fcbd83b5ca3493dff3659a2e49b01cafdcbcbff7f7
      Stored in directory: /root/.cache/pip/wheels/a9/26/38/7678d1ff6cd1bbcbfc0d80b0a29d94d917dfa9ad790b4a85a9
      Building wheel for kaldiio (setup.py) ... done
      Created wheel for kaldiio: filename=kaldiio-2.17.2-py3-none-any.whl size=24472 sha256=108e433cf9c7acc191fc98f3868ac6c3d1bd6b7525e3dcc2fbc8ad3aff3fcd53
      Stored in directory: /root/.cache/pip/wheels/04/07/e8/45641287c59bf6ce41e22259f8680b521c31e6306cb88392ac
      Building wheel for pesq (setup.py) ... done
      Created wheel for pesq: filename=pesq-0.0.3-cp37-cp37m-linux_x86_64.whl size=209964 sha256=b8706fd82c4909091af59a6582fe9392e058015900358aab1d2bf4da6491e998
      Stored in directory: /root/.cache/pip/wheels/4f/67/5b/aa7cf31fe0c7199e35c604bb7bc91c629a13726bf221fedba0
      Building wheel for pystoi (setup.py) ... done
      Created wheel for pystoi: filename=pystoi-0.3.3-py2.py3-none-any.whl size=7793 sha256=f8e282fd6375967c4edad0cb4ea1dd68768f5c773c70f6199cf6c1964fe1a15e
      Stored in directory: /root/.cache/pip/wheels/46/4a/ad/3ab460193ed0535430b4b1575f255aa6bae69df17453628e86
      Building wheel for ipadic (setup.py) ... done
      Created wheel for ipadic: filename=ipadic-1.0.0-py3-none-any.whl size=13556723 sha256=dead0d48768f013a31661b02bd7688f8c4f8bf846e9326ffb4b62f31090c113c
      Stored in directory: /root/.cache/pip/wheels/33/8b/99/cf0d27191876637cd3639a560f93aa982d7855ce826c94348b
      Building wheel for subprocess32 (setup.py) ... done
      Created wheel for subprocess32: filename=subprocess32-3.5.4-py3-none-any.whl size=6502 sha256=a92331d36a1377c6d92a11c5e3c0c0eae192006db6aa932347becd109606d9e0
      Stored in directory: /root/.cache/pip/wheels/50/ca/fa/8fca8d246e64f19488d07567547ddec8eb084e8c0d7a59226a
      Building wheel for pathtools (setup.py) ... done
      Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8807 sha256=6576947218f7fcb666dbb7af4fba4042d4438b174850490cbb852532784ed5c7
      Stored in directory: /root/.cache/pip/wheels/3e/31/09/fa59cef12cdcfecc627b3d24273699f390e71828921b2cbba2
      Building wheel for wget (setup.py) ... done
      Created wheel for wget: filename=wget-3.2-py3-none-any.whl size=9672 sha256=339c981f99d8e9bfc538c16bd5933f8901811a7baef470dd569c50a175238e5e
      Stored in directory: /root/.cache/pip/wheels/a1/b6/7c/0e63e34eb06634181c63adacca38b79ff8f35c37e3c13e3c02
    Successfully built wordninja antlr4-python3-runtime future distance kaldi-python-io kaldiio pesq pystoi ipadic subprocess32 pathtools wget
    Installing collected packages: urllib3, multidict, yarl, async-timeout, typer, smmap, simplejson, shellingham, PyYAML, matplotlib, latexcodec, jmespath, fsspec, colorama, antlr4-python3-runtime, aiohttp, yarg, torchmetrics, tokenizers, sphinxcontrib-qthelp, sphinxcontrib-jsmath, sphinxcontrib-htmlhelp, sphinxcontrib-devhelp, sphinxcontrib-applehelp, sacremoses, ruamel.yaml.clib, pyDeprecate, pybtex, pyannote.core, portalocker, omegaconf, huggingface-hub, gitdb, future, braceexpand, botocore, wget, webdataset, typed-ast, transformers, subprocess32, sphinx, shortuuid, sentry-sdk, sentencepiece, sacrebleu, s3transfer, ruamel.yaml, pytorch-lightning, pybtex-docutils, pybind11, pyannote.database, pipreqs, pip-api, pathtools, pathspec, pangu, opencc, onnx, mecab-python3, isort, ipadic, hydra-core, GitPython, docker-pycreds, distance, configparser, youtokentome, wordninja, wandb, unidecode, torch-stft, sphinxcontrib-bibtex, sox, rapidfuzz, pytest-runner, pystoi, pypinyin, pydub, pyannote.metrics, pesq, parameterized, nemo-toolkit, megatron-lm, marshmallow, kaldiio, kaldi-python-io, kaldi-io, g2p-en, frozendict, boto3, black, attrdict
      Attempting uninstall: urllib3
        Found existing installation: urllib3 1.24.3
        Uninstalling urllib3-1.24.3:
          Successfully uninstalled urllib3-1.24.3
      Attempting uninstall: PyYAML
        Found existing installation: PyYAML 3.13
        Uninstalling PyYAML-3.13:
          Successfully uninstalled PyYAML-3.13
      Attempting uninstall: matplotlib
        Found existing installation: matplotlib 3.2.2
        Uninstalling matplotlib-3.2.2:
          Successfully uninstalled matplotlib-3.2.2
      Attempting uninstall: future
        Found existing installation: future 0.16.0
        Uninstalling future-0.16.0:
          Successfully uninstalled future-0.16.0
      Attempting uninstall: sphinx
        Found existing installation: Sphinx 1.8.5
        Uninstalling Sphinx-1.8.5:
          Successfully uninstalled Sphinx-1.8.5
    ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
    datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.
    albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.
    Successfully installed GitPython-3.1.18 PyYAML-5.4.1 aiohttp-3.7.4.post0 antlr4-python3-runtime-4.8 async-timeout-3.0.1 attrdict-2.0.1 black-19.10b0 boto3-1.18.26 botocore-1.21.26 braceexpand-0.1.7 colorama-0.4.4 configparser-5.0.2 distance-0.1.3 docker-pycreds-0.4.0 frozendict-2.0.6 fsspec-2021.7.0 future-0.18.2 g2p-en-2.1.0 gitdb-4.0.7 huggingface-hub-0.0.12 hydra-core-1.1.1 ipadic-1.0.0 isort-4.3.21 jmespath-0.10.0 kaldi-io-0.9.4 kaldi-python-io-1.2.2 kaldiio-2.17.2 latexcodec-2.0.1 marshmallow-3.13.0 matplotlib-3.4.3 mecab-python3-1.0.3 megatron-lm-2.2.0 multidict-5.1.0 nemo-toolkit-1.2.0 omegaconf-2.1.1 onnx-1.10.1 opencc-1.1.2 pangu-4.0.6.1 parameterized-0.8.1 pathspec-0.9.0 pathtools-0.1.2 pesq-0.0.3 pip-api-0.0.20 pipreqs-0.4.10 portalocker-2.3.1 pyDeprecate-0.3.0 pyannote.core-4.1 pyannote.database-4.1.1 pyannote.metrics-3.0.1 pybind11-2.7.1 pybtex-0.24.0 pybtex-docutils-1.0.1 pydub-0.25.1 pypinyin-0.42.0 pystoi-0.3.3 pytest-runner-5.3.1 pytorch-lightning-1.3.8 rapidfuzz-1.5.0 ruamel.yaml-0.17.13 ruamel.yaml.clib-0.2.6 s3transfer-0.5.0 sacrebleu-2.0.0 sacremoses-0.0.45 sentencepiece-0.1.96 sentry-sdk-1.3.1 shellingham-1.4.0 shortuuid-1.0.1 simplejson-3.17.4 smmap-4.0.0 sox-1.4.1 sphinx-4.1.2 sphinxcontrib-applehelp-1.0.2 sphinxcontrib-bibtex-2.3.0 sphinxcontrib-devhelp-1.0.2 sphinxcontrib-htmlhelp-2.0.0 sphinxcontrib-jsmath-1.0.1 sphinxcontrib-qthelp-1.0.3 subprocess32-3.5.4 tokenizers-0.10.3 torch-stft-0.1.4 torchmetrics-0.5.0 transformers-4.8.1 typed-ast-1.4.3 typer-0.3.2 unidecode-1.2.0 urllib3-1.25.11 wandb-0.12.0 webdataset-0.1.62 wget-3.2 wordninja-2.0.0 yarg-0.1.9 yarl-1.6.3 youtokentome-1.0.6

```python
from nemo.collections import nlp as nemo_nlp
from nemo.utils.exp_manager import exp_manager #模型训练管理工具

import os
import wget
import torch
import pytorch_lightning as pl
from omegaconf import OmegaConf #配置文件工具库
```

    [NeMo W 2021-08-22 14:02:52 optimizers:47] Apex was not found. Using the lamb optimizer will error out.
    [NeMo W 2021-08-22 14:02:56 experimental:28] Module <class 'nemo.collections.nlp.data.text_normalization.decoder_dataset.TextNormalizationDecoderDataset'> is experimental, not ready for production and is not fully supported. Use at your own risk.
    [NeMo W 2021-08-22 14:02:56 experimental:28] Module <class 'nemo.collections.nlp.data.text_normalization.tagger_dataset.TextNormalizationTaggerDataset'> is experimental, not ready for production and is not fully supported. Use at your own risk.
    [NeMo W 2021-08-22 14:02:56 experimental:28] Module <class 'nemo.collections.nlp.data.text_normalization.test_dataset.TextNormalizationTestDataset'> is experimental, not ready for production and is not fully supported. Use at your own risk.

    [nltk_data] Downloading package punkt to /root/nltk_data...
    [nltk_data]   Unzipping tokenizers/punkt.zip.

    [NeMo W 2021-08-22 14:02:57 experimental:28] Module <class 'nemo.collections.nlp.models.duplex_text_normalization.duplex_decoder.DuplexDecoderModel'> is experimental, not ready for production and is not fully supported. Use at your own risk.
    [NeMo W 2021-08-22 14:02:57 experimental:28] Module <class 'nemo.collections.nlp.models.duplex_text_normalization.duplex_tagger.DuplexTaggerModel'> is experimental, not ready for production and is not fully supported. Use at your own risk.

    [nltk_data] Downloading package punkt to /root/nltk_data...
    [nltk_data]   Package punkt is already up-to-date!

    [NeMo W 2021-08-22 14:02:57 experimental:28] Module <class 'nemo.collections.nlp.models.duplex_text_normalization.duplex_tn.DuplexTextNormalizationModel'> is experimental, not ready for production and is not fully supported. Use at your own risk.

```python
!pwd
!wget -c https://dl.fbaipublicfiles.com/glue/data/SST-2.zip
```

    /content
    --2021-08-22 14:06:06--  https://dl.fbaipublicfiles.com/glue/data/SST-2.zip
    Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 172.67.9.4, 104.22.74.142, 104.22.75.142, ...
    Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|172.67.9.4|:443... connected.
    HTTP request sent, awaiting response... 200 OK
    Length: 7439277 (7.1M) [application/zip]
    Saving to: ‘SST-2.zip’

    SST-2.zip           100%[===================>]   7.09M  28.8MB/s    in 0.2s

    2021-08-22 14:06:07 (28.8 MB/s) - ‘SST-2.zip’ saved [7439277/7439277]

#2 创建用来存放数据集和配置文件的目录

```python
#调用系统模块定义并创建用来存放数据集和配置文件的目录
DATA_DIR = "/content"
WORK_DIR = "/content"
os.environ['DATA_DIR'] = DATA_DIR
#! unzip -o SST-2.zip -d {DATA_DIR}
os.makedirs(WORK_DIR, exist_ok=True)
os.makedirs(DATA_DIR, exist_ok=True)

#打印数据集中的前n行进行简单查看
print('Contents (first 5 lines) of train.tsv:')
! head -n 20 {DATA_DIR}/SST-2/train.tsv

print('\nContents (first 5 lines) of test.tsv:')
! head -n 10 {DATA_DIR}/SST-2/test.tsv
```

    Contents (first 5 lines) of train.tsv:
    sentence	label
    hide new secretions from the parental units 	0
    contains no wit , only labored gags 	0
    that loves its characters and communicates something rather beautiful about human nature 	1
    remains utterly satisfied to remain the same throughout 	0
    on the worst revenge-of-the-nerds clichés the filmmakers could dredge up 	0
    that 's far too tragic to merit such superficial treatment 	0
    demonstrates that the director of such hollywood blockbusters as patriot games can still turn out a small , personal film with an emotional wallop . 	1
    of saucy 	1
    a depressed fifteen-year-old 's suicidal poetry 	0
    are more deeply thought through than in most ` right-thinking ' films 	1
    goes to absurd lengths 	0
    for those moviegoers who complain that ` they do n't make movies like they used to anymore 	0
    the part where nothing 's happening , 	0
    saw how bad this movie was 	0
    lend some dignity to a dumb story 	0
    the greatest musicians 	1
    cold movie 	0
    with his usual intelligence and subtlety 	1
    redundant concept 	0

    Contents (first 5 lines) of test.tsv:
    index	sentence
    0	uneasy mishmash of styles and genres .
    1	this film 's relationship to actual tension is the same as what christmas-tree flocking in a spray can is to actual snow : a poor -- if durable -- imitation .
    2	by the end of no such thing the audience , like beatrice , has a watchful affection for the monster .
    3	director rob marshall went out gunning to make a great one .
    4	lathan and diggs have considerable personal charm , and their screen rapport makes the old story seem new .
    5	a well-made and often lovely depiction of the mysteries of friendship .
    6	none of this violates the letter of behan 's book , but missing is its spirit , its ribald , full-throated humor .
    7	although it bangs a very cliched drum at times , this crowd-pleaser 's fresh dialogue , energetic music , and good-natured spunk are often infectious .
    8	it is not a mass-market entertainment but an uncompromising attempt by one artist to think about another .

#3 加载模型的配置文件并打印

```python
#定义模型配置文件的路径
MODEL_CONFIG = "text_classification_config.yaml"
CONFIG_DIR = WORK_DIR + '/configs/'
os.makedirs(CONFIG_DIR, exist_ok=True)
#判断配置文件是否已经存在
if not os.path.exists(CONFIG_DIR + MODEL_CONFIG):
    print('Downloading config file...')
    wget.download('https://raw.githubusercontent.com/NVIDIA/NeMo/main/examples/nlp/text_classification/conf/' + MODEL_CONFIG, CONFIG_DIR)
    print('Config file downloaded!')
else:
    print ('config file already exists')
config_path = f'{WORK_DIR}/configs/{MODEL_CONFIG}'
print(config_path)
# 通过OmegaConf工具库加载模型配置文件，定义config对象
config = OmegaConf.load(config_path)
print(OmegaConf.to_yaml(config))
```

    Downloading config file...
    Config file downloaded!
    /content/configs/text_classification_config.yaml
    trainer:
      gpus: 1
      num_nodes: 1
      max_epochs: 100
      max_steps: null
      accumulate_grad_batches: 1
      gradient_clip_val: 0.0
      amp_level: O0
      precision: 32
      accelerator: ddp
      log_every_n_steps: 1
      val_check_interval: 1.0
      resume_from_checkpoint: null
      num_sanity_val_steps: 0
      checkpoint_callback: false
      logger: false
    model:
      nemo_path: text_classification_model.nemo
      tokenizer:
        tokenizer_name: ${model.language_model.pretrained_model_name}
        vocab_file: null
        tokenizer_model: null
        special_tokens: null
      language_model:
        pretrained_model_name: bert-base-uncased
        lm_checkpoint: null
        config_file: null
        config: null
      classifier_head:
        num_output_layers: 2
        fc_dropout: 0.1
      class_labels:
        class_labels_file: null
      dataset:
        num_classes: ???
        do_lower_case: false
        max_seq_length: 256
        class_balancing: null
        use_cache: false
      train_ds:
        file_path: null
        batch_size: 64
        shuffle: true
        num_samples: -1
        num_workers: 3
        drop_last: false
        pin_memory: false
      validation_ds:
        file_path: null
        batch_size: 64
        shuffle: false
        num_samples: -1
        num_workers: 3
        drop_last: false
        pin_memory: false
      test_ds:
        file_path: null
        batch_size: 64
        shuffle: false
        num_samples: -1
        num_workers: 3
        drop_last: false
        pin_memory: false
      optim:
        name: adam
        lr: 2.0e-05
        betas:
        - 0.9
        - 0.999
        weight_decay: 0.01
        sched:
          name: WarmupAnnealing
          warmup_steps: null
          warmup_ratio: 0.1
          last_epoch: -1
          monitor: val_loss
          reduce_on_plateau: false
      infer_samples:
      - by the end of no such thing the audience , like beatrice , has a watchful affection
        for the monster .
      - director rob marshall went out gunning to make a great one .
      - uneasy mishmash of styles and genres .
    exp_manager:
      exp_dir: null
      name: TextClassification
      create_tensorboard_logger: true
      create_checkpoint_callback: true

#4 设置配置文件参数，指定数据集

```python
config.model.dataset.num_classes=2 #通过config来定义数据集分类的类别：2分类任务
config.model.train_ds.file_path = os.path.join(DATA_DIR, 'SST-2/train.tsv')#指定训练集路径
config.model.validation_ds.file_path = os.path.join(DATA_DIR, 'SST-2/dev.tsv')#指定验证集路径
print("Train dataloader's config: \n")
print(OmegaConf.to_yaml(config.model.train_ds))
```

    Train dataloader's config:

    file_path: /content/SST-2/train.tsv
    batch_size: 64
    shuffle: true
    num_samples: -1
    num_workers: 3
    drop_last: false
    pin_memory: false

#5 设置参数构建训练器

```python
config.trainer.gpus = 1 if torch.cuda.is_available() else 0 # 通过判断cuda是否用来设置gpu的数量
config.trainer.precision = 16 # 若使用混合精度计算可以设置为16bit精度
config.trainer.amp_level = "O1"# 若使用混合精度计算这里参数需设为“01”
config.trainer.accelerator = None #使用分布式训练 in colab set "dpp" to None
config.trainer.max_epochs = 10 # 设置训练的epoch
trainer = pl.Trainer(**config.trainer) #调用pytorchlighting的Trainer将设置好参数的模型实例化
```

    GPU available: True, used: True
    TPU available: False, using: 0 TPU cores
    Using native 16bit precision.

#6 设置 NeMo的日志模型管理器¶

```python
print(OmegaConf.to_yaml(config.exp_manager))
exp_dir = exp_manager(trainer, config.exp_manager)
print(exp_dir)
```

    exp_dir: null
    name: TextClassification
    create_tensorboard_logger: true
    create_checkpoint_callback: true

    [NeMo I 2021-08-22 14:12:35 exp_manager:219] Experiments will be logged at /content/nemo_experiments/TextClassification/2021-08-22_14-12-35
    [NeMo I 2021-08-22 14:12:35 exp_manager:568] TensorboardLogger has been set up
    /content/nemo_experiments/TextClassification/2021-08-22_14-12-35

#7 打印NeMo中可用的语言模型

```python
print(nemo_nlp.modules.get_pretrained_lm_models_list())
```

    ['megatron-bert-345m-uncased', 'megatron-bert-345m-cased', 'megatron-bert-uncased', 'megatron-bert-cased', 'biomegatron-bert-345m-uncased', 'biomegatron-bert-345m-cased', 'bert-base-uncased', 'bert-large-uncased', 'bert-base-cased', 'bert-large-cased', 'bert-base-multilingual-uncased', 'bert-base-multilingual-cased', 'bert-base-chinese', 'bert-base-german-cased', 'bert-large-uncased-whole-word-masking', 'bert-large-cased-whole-word-masking', 'bert-large-uncased-whole-word-masking-finetuned-squad', 'bert-large-cased-whole-word-masking-finetuned-squad', 'bert-base-cased-finetuned-mrpc', 'bert-base-german-dbmdz-cased', 'bert-base-german-dbmdz-uncased', 'cl-tohoku/bert-base-japanese', 'cl-tohoku/bert-base-japanese-whole-word-masking', 'cl-tohoku/bert-base-japanese-char', 'cl-tohoku/bert-base-japanese-char-whole-word-masking', 'TurkuNLP/bert-base-finnish-cased-v1', 'TurkuNLP/bert-base-finnish-uncased-v1', 'wietsedv/bert-base-dutch-cased', 'distilbert-base-uncased', 'distilbert-base-uncased-distilled-squad', 'distilbert-base-cased', 'distilbert-base-cased-distilled-squad', 'distilbert-base-german-cased', 'distilbert-base-multilingual-cased', 'distilbert-base-uncased-finetuned-sst-2-english', 'roberta-base', 'roberta-large', 'roberta-large-mnli', 'distilroberta-base', 'roberta-base-openai-detector', 'roberta-large-openai-detector', 'albert-base-v1', 'albert-large-v1', 'albert-xlarge-v1', 'albert-xxlarge-v1', 'albert-base-v2', 'albert-large-v2', 'albert-xlarge-v2', 'albert-xxlarge-v2']

```python
# HF_DATASETS_OFFLINE=1
TRANSFORMERS_OFFLINE=1 #可以使用transformer中加载过语言模型的缓存，避免每次都重新远程加载。
```

# 8 模型的初始化，将中文Bert设置为预训练模型

```python
config.model.language_model.pretrained_model_name = "bert-base-chinese"
model = nemo_nlp.models.TextClassificationModel(cfg=config.model, trainer=trainer)
```

    [NeMo I 2021-08-22 14:12:56 tokenizer_utils:101] Getting HuggingFace AutoTokenizer with pretrained_model_name: bert-base-chinese, vocab_file: None, special_tokens_dict: {}, and use_fast: False

    Downloading:   0%|          | 0.00/29.0 [00:00<?, ?B/s]

    Downloading:   0%|          | 0.00/624 [00:00<?, ?B/s]

    Downloading:   0%|          | 0.00/110k [00:00<?, ?B/s]

    Downloading:   0%|          | 0.00/269k [00:00<?, ?B/s]

    Using bos_token, but it is not set yet.
    Using eos_token, but it is not set yet.

    [NeMo I 2021-08-22 14:12:59 text_classification_dataset:120] Read 67350 examples from /content/SST-2/train.tsv.
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:238] *** Example ***
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:239] example 0: ['a', 'moving', 'tale']
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:240] subtokens: [CLS] a mo ##ving ta ##le [SEP]
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:241] input_ids: 101 143 11303 10369 8346 8268 102
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:242] segment_ids: 0 0 0 0 0 0 0
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:243] input_mask: 1 1 1 1 1 1 1
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:244] label: 1
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:238] *** Example ***
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:239] example 1: ['intriguing', ',', 'observant']
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:240] subtokens: [CLS] int ##ri ##g ##ui ##ng , ob ##ser ##van ##t [SEP]
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:241] input_ids: 101 10673 8641 8181 9273 8291 117 12639 9520 13045 8165 102
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:242] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:243] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1
    [NeMo I 2021-08-22 14:13:00 text_classification_dataset:244] label: 1
    [NeMo I 2021-08-22 14:13:34 data_preprocessing:358] Some stats of the lengths of the sequences:
    [NeMo I 2021-08-22 14:13:34 data_preprocessing:364] Min: 3 |                  Max: 102 |                  Mean: 19.74785074759833 |                  Median: 15.0
    [NeMo I 2021-08-22 14:13:34 data_preprocessing:366] 75 percentile: 26.00
    [NeMo I 2021-08-22 14:13:34 data_preprocessing:367] 99 percentile: 67.00

    [NeMo W 2021-08-22 14:13:35 nemo_logging:349] /usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 3 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
          cpuset_checked))

    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:120] Read 873 examples from /content/SST-2/dev.tsv.
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:238] *** Example ***
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:239] example 0: ['it', "'s", 'a', 'charming', 'and', 'often', 'affecting', 'journey', '.']
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:240] subtokens: [CLS] it ' s a ch ##ar ##ming and of ##ten af ##fe ##ct ##ing jo ##urn ##ey . [SEP]
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:241] input_ids: 101 8233 112 161 143 9537 8458 10693 8256 8205 11598 11782 9568 8722 8221 13269 12760 8603 119 102
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:242] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:243] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:244] label: 1
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:238] *** Example ***
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:239] example 1: ['unflinchingly', 'bleak', 'and', 'desperate']
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:240] subtokens: [CLS] u ##n ##fl ##in ##chi ##ng ##ly bl ##ea ##k and des ##per ##ate [SEP]
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:241] input_ids: 101 163 8171 11690 8277 10525 8291 8436 10353 10073 8197 8256 11081 9063 9690 102
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:242] segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:243] input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
    [NeMo I 2021-08-22 14:13:35 text_classification_dataset:244] label: 0
    [NeMo I 2021-08-22 14:13:36 data_preprocessing:358] Some stats of the lengths of the sequences:
    [NeMo I 2021-08-22 14:13:36 data_preprocessing:364] Min: 4 |                  Max: 86 |                  Mean: 37.30733944954128 |                  Median: 36.0
    [NeMo I 2021-08-22 14:13:36 data_preprocessing:366] 75 percentile: 48.00
    [NeMo I 2021-08-22 14:13:36 data_preprocessing:367] 99 percentile: 76.00
    [NeMo I 2021-08-22 14:13:36 text_classification_model:217] Dataloader config or file_path for the test is missing, so no data loader for test is created!

    [NeMo W 2021-08-22 14:13:36 modelPT:200] You tried to register an artifact under config key=tokenizer.vocab_file but an artifact for it has already been registered.

    Downloading:   0%|          | 0.00/412M [00:00<?, ?B/s]

    Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']
    - This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
    - This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
    Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertEncoder: ['cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']
    - This IS expected if you are initializing BertEncoder from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
    - This IS NOT expected if you are initializing BertEncoder from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).

#9 调用trainer中的fit函数结合数据集开始模型训练

```python
trainer.fit(model)
```

    LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

    [NeMo I 2021-08-22 14:14:00 modelPT:750] Optimizer config = Adam (
        Parameter Group 0
            amsgrad: False
            betas: [0.9, 0.999]
            eps: 1e-08
            lr: 2e-05
            weight_decay: 0.01
        )
    [NeMo I 2021-08-22 14:14:00 lr_scheduler:625] Scheduler "<nemo.core.optim.lr_scheduler.WarmupAnnealing object at 0x7f0c7b569450>"
        will be used during training (effective maximum steps = 10530) -
        Parameters :
        (warmup_steps: null
        warmup_ratio: 0.1
        last_epoch: -1
        max_steps: 10530
        )

      | Name                  | Type                 | Params
    ---------------------------------------------------------------
    0 | loss                  | CrossEntropyLoss     | 0
    1 | bert_model            | BertEncoder          | 102 M
    2 | classifier            | SequenceClassifier   | 592 K
    3 | classification_report | ClassificationReport | 0
    ---------------------------------------------------------------
    102 M     Trainable params
    0         Non-trainable params
    102 M     Total params
    411.439   Total estimated model params size (MB)

    Training: 0it [00:00, ?it/s]

    [NeMo W 2021-08-22 14:14:00 nemo_logging:349] /usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 3 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
          cpuset_checked))

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 14:18:51 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             77.17      85.28      81.02        428
        label_id: 1                                             84.21      75.68      79.72        444
        -------------------
        micro avg                                               80.39      80.39      80.39        872
        macro avg                                               80.69      80.48      80.37        872
        weighted avg                                            80.75      80.39      80.36        872

    Epoch 0, global step 1052: val_loss reached 0.41953 (best 0.41953), saving model to "/content/nemo_experiments/TextClassification/2021-08-22_14-12-35/checkpoints/TextClassification--val_loss=0.42-epoch=0.ckpt" as top 3

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 14:24:28 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             81.37      85.75      83.50        428
        label_id: 1                                             85.51      81.08      83.24        444
        -------------------
        micro avg                                               83.37      83.37      83.37        872
        macro avg                                               83.44      83.41      83.37        872
        weighted avg                                            83.48      83.37      83.37        872

    Epoch 1, global step 2105: val_loss reached 0.38472 (best 0.38472), saving model to "/content/nemo_experiments/TextClassification/2021-08-22_14-12-35/checkpoints/TextClassification--val_loss=0.38-epoch=1.ckpt" as top 3

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 14:30:05 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             89.02      71.96      79.59        428
        label_id: 1                                             77.19      91.44      83.71        444
        -------------------
        micro avg                                               81.88      81.88      81.88        872
        macro avg                                               83.10      81.70      81.65        872
        weighted avg                                            82.99      81.88      81.69        872

    Epoch 2, global step 3158: val_loss reached 0.43869 (best 0.38472), saving model to "/content/nemo_experiments/TextClassification/2021-08-22_14-12-35/checkpoints/TextClassification--val_loss=0.44-epoch=2.ckpt" as top 3

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 14:35:39 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             76.64      92.76      83.93        428
        label_id: 1                                             91.24      72.75      80.95        444
        -------------------
        micro avg                                               82.57      82.57      82.57        872
        macro avg                                               83.94      82.75      82.44        872
        weighted avg                                            84.08      82.57      82.42        872

    Epoch 3, global step 4211: val_loss reached 0.42369 (best 0.38472), saving model to "/content/nemo_experiments/TextClassification/2021-08-22_14-12-35/checkpoints/TextClassification--val_loss=0.42-epoch=3.ckpt" as top 3

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 14:41:14 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             86.45      74.53      80.05        428
        label_id: 1                                             78.33      88.74      83.21        444
        -------------------
        micro avg                                               81.77      81.77      81.77        872
        macro avg                                               82.39      81.64      81.63        872
        weighted avg                                            82.32      81.77      81.66        872

    Epoch 4, global step 5264: val_loss was not in top 3

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 14:46:24 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             87.82      55.61      68.10        428
        label_id: 1                                             68.39      92.57      78.66        444
        -------------------
        micro avg                                               74.43      74.43      74.43        872
        macro avg                                               78.10      74.09      73.38        872
        weighted avg                                            77.93      74.43      73.48        872

    Epoch 5, global step 6317: val_loss was not in top 3

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 14:51:35 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             86.34      73.83      79.60        428
        label_id: 1                                             77.87      88.74      82.95        444
        -------------------
        micro avg                                               81.42      81.42      81.42        872
        macro avg                                               82.10      81.29      81.27        872
        weighted avg                                            82.02      81.42      81.30        872

    Epoch 6, global step 7370: val_loss was not in top 3

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 14:56:46 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             84.34      78.04      81.07        428
        label_id: 1                                             80.25      86.04      83.04        444
        -------------------
        micro avg                                               82.11      82.11      82.11        872
        macro avg                                               82.30      82.04      82.06        872
        weighted avg                                            82.26      82.11      82.07        872

    Epoch 7, global step 8423: val_loss was not in top 3

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 15:01:55 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             86.55      69.16      76.88        428
        label_id: 1                                             75.09      89.64      81.72        444
        -------------------
        micro avg                                               79.59      79.59      79.59        872
        macro avg                                               80.82      79.40      79.30        872
        weighted avg                                            80.72      79.59      79.35        872

    Epoch 8, global step 9476: val_loss was not in top 3

    Validating: 0it [00:00, ?it/s]

    [NeMo I 2021-08-22 15:07:06 text_classification_model:165] val_report:
        label                                                precision    recall       f1           support
        label_id: 0                                             86.71      70.09      77.52        428
        label_id: 1                                             75.67      89.64      82.06        444
        -------------------
        micro avg                                               80.05      80.05      80.05        872
        macro avg                                               81.19      79.87      79.79        872
        weighted avg                                            81.08      80.05      79.83        872

    Epoch 9, global step 10529: val_loss was not in top 3
    Saving latest checkpoint...

#10 模型的验证Evaluation

```python
queries = ['"很好,很实惠"',
            '配送员很好',
            '不错，就是餐盒贵',
            '火烧夹肉好咸啊！没法吃。其他还行',
            "味道不错,配送速度比预计快",
            '没什么味道，送来的晚凉了',
            '完全不值得信任，出尔反而',
            '没发票，乱收费，订单没到就被完成了',
            '鸡蛋都坏了，凉菜也洒了',
            '很 快 ， 好 吃 ， 味 道 足 ， 量 大 ',
           '很棒！太快了',
           '和店里点的水平差很多',
           '挺快的，但是味道不太好',
           '我点的凉菜，给我送一盒热的烂白菜',
           '好服务，就是习惯百度外卖的快速',
           '挺好吃的，就是有点难吃',
           '我家的狗吃了连夜给我做了四菜一汤',
           '不错，只不过这味道对于人类来讲超前了一些，狗觉得挺好吃的']

results = model.classifytext(queries=queries, batch_size=3, max_seq_length=512)

print('The prediction results of some sample queries with the trained model:')
for query, result in zip(queries, results):
    print(f'Query : {query}')
    print(f'Predicted label: {result}')
```

    [NeMo W 2021-08-22 15:07:51 nemo_logging:349] /usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:481: UserWarning: This DataLoader will create 3 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
          cpuset_checked))

    The prediction results of some sample queries with the trained model:
    Query : "很好,很实惠"
    Predicted label: 1
    Query : 配送员很好
    Predicted label: 1
    Query : 不错，就是餐盒贵
    Predicted label: 0
    Query : 火烧夹肉好咸啊！没法吃。其他还行
    Predicted label: 0
    Query : 味道不错,配送速度比预计快
    Predicted label: 0
    Query : 没什么味道，送来的晚凉了
    Predicted label: 0
    Query : 完全不值得信任，出尔反而
    Predicted label: 0
    Query : 没发票，乱收费，订单没到就被完成了
    Predicted label: 0
    Query : 鸡蛋都坏了，凉菜也洒了
    Predicted label: 0
    Query : 很 快 ， 好 吃 ， 味 道 足 ， 量 大
    Predicted label: 0
    Query : 很棒！太快了
    Predicted label: 1
    Query : 和店里点的水平差很多
    Predicted label: 0
    Query : 挺快的，但是味道不太好
    Predicted label: 0
    Query : 我点的凉菜，给我送一盒热的烂白菜
    Predicted label: 0
    Query : 好服务，就是习惯百度外卖的快速
    Predicted label: 0
    Query : 挺好吃的，就是有点难吃
    Predicted label: 0
    Query : 我家的狗吃了连夜给我做了四菜一汤
    Predicted label: 0
    Query : 不错，只不过这味道对于人类来讲超前了一些，狗觉得挺好吃的
    Predicted label: 0

```python
model.save_to("sentiment_classify_model.nemo")
```