最强 NLP 预训练模型库 PyTorch-Transformers 正式开源：支持 6 个预训练框架，27 个预训练模型

pip install pytorch-transformers

pip install [--editable] .

python -m pytest -sv ./pytorch_transformers/tests/python -m pytest -sv ./examples/

import torchfrom pytorch_transformers import *
# PyTorch-Transformers has a unified API# for 6 transformer architectures and 27 pretrained weights.#          Model          | Tokenizer          | Pretrained weights shortcutMODELS = [(BertModel,       BertTokenizer,      'bert-base-uncased'),          (OpenAIGPTModel,  OpenAIGPTTokenizer, 'openai-gpt'),          (GPT2Model,       GPT2Tokenizer,      'gpt2'),          (TransfoXLModel,  TransfoXLTokenizer, 'transfo-xl-wt103'),          (XLNetModel,      XLNetTokenizer,     'xlnet-base-cased'),          (XLMModel,        XLMTokenizer,       'xlm-mlm-enfr-1024')]
# Let's encode some text in a sequence of hidden-states using each model:for model_class, tokenizer_class, pretrained_weights in MODELS:    # Load pretrained model/tokenizer    tokenizer = tokenizer_class.from_pretrained(pretrained_weights)    model = model_class.from_pretrained(pretrained_weights)
    # Encode text    input_ids = torch.tensor([tokenizer.encode("Here is some text to encode")])    last_hidden_states = model(input_ids)[0]  # Models outputs are now tuples
# Each architecture is provided with several class for fine-tuning on down-stream tasks, e.g.BERT_MODEL_CLASSES = [BertModel, BertForPreTraining, BertForMaskedLM, BertForNextSentencePrediction,                      BertForSequenceClassification, BertForMultipleChoice, BertForTokenClassification,                      BertForQuestionAnswering]
# All the classes for an architecture can be initiated from pretrained weights for this architecture# Note that additional weights added for fine-tuning are only initialized# and need to be trained on the down-stream tasktokenizer = BertTokenizer.from_pretrained(from_pretrained('bert-base-uncased')for model_class in BERT_MODEL_CLASSES:    # Load pretrained model/tokenizer    model = model_class.from_pretrained('bert-base-uncased')
# Models can return full list of hidden-states & attentions weights at each layermodel = model_class.from_pretrained(pretrained_weights,                                    output_hidden_states=True,                                    output_attentions=True)input_ids = torch.tensor([tokenizer.encode("Let's see all hidden-states and attentions on this text")])all_hidden_states, all_attentions = model(input_ids)[-2:]
# Models are compatible with Torchscriptmodel = model_class.from_pretrained(pretrained_weights, torchscript=True)traced_model = torch.jit.trace(model, (input_ids,))
# Simple serialization for models and tokenizersmodel.save_pretrained('./directory/to/save/')  # savemodel = model_class.from_pretrained('./directory/to/save/')  # re-loadtokenizer.save_pretrained('./directory/to/save/')  # savetokenizer = tokenizer_class.from_pretrained(pretrained_weights)
# SOTA examples for GLUE, SQUAD, text generation...

pip install -r ./examples/requirements.txt

export GLUE_DIR=/path/to/glueexport TASK_NAME=MRPC
python ./examples/run_glue.py \    --model_type bert \    --model_name_or_path bert-base-uncased \    --task_name $TASK_NAME \    --do_train \    --do_eval \    --do_lower_case \    --data_dir $GLUE_DIR/$TASK_NAME \    --max_seq_length 128 \    --per_gpu_eval_batch_size=8   \    --per_gpu_train_batch_size=8   \    --learning_rate 2e-5 \    --num_train_epochs 3.0 \    --output_dir /tmp/$TASK_NAME/

export GLUE_DIR=/path/to/glue
python ./examples/run_glue.py \    --model_type xlnet \    --model_name_or_path xlnet-large-cased \    --do_train  \    --do_eval   \    --task_name=sts-b     \    --data_dir=${GLUE_DIR}/STS-B  \    --output_dir=./proc_data/sts-b-110   \    --max_seq_length=128   \    --per_gpu_eval_batch_size=8   \    --per_gpu_train_batch_size=8   \    --gradient_accumulation_steps=1 \    --max_steps=1200  \    --model_name=xlnet-large-cased   \    --overwrite_output_dir   \    --overwrite_cache \    --warmup_steps=120

python -m torch.distributed.launch --nproc_per_node 8 ./examples/run_glue.py   \    --model_type bert \    --model_name_or_path bert-large-uncased-whole-word-masking \    --task_name MRPC \    --do_train   \    --do_eval   \    --do_lower_case   \    --data_dir $GLUE_DIR/MRPC/   \    --max_seq_length 128   \    --per_gpu_eval_batch_size=8   \    --per_gpu_train_batch_size=8   \    --learning_rate 2e-5   \    --num_train_epochs 3.0  \    --output_dir /tmp/mrpc_output/ \    --overwrite_output_dir   \    --overwrite_cache \

  acc = 0.8823529411764706  acc_and_f1 = 0.901702786377709  eval_loss = 0.3418912578906332  f1 = 0.9210526315789473  global_step = 174  loss = 0.07231863956341798

python -m torch.distributed.launch --nproc_per_node=8 ./examples/run_squad.py \    --model_type bert \    --model_name_or_path bert-large-uncased-whole-word-masking \    --do_train \    --do_eval \    --do_lower_case \    --train_file $SQUAD_DIR/train-v1.1.json \    --predict_file $SQUAD_DIR/dev-v1.1.json \    --learning_rate 3e-5 \    --num_train_epochs 2 \    --max_seq_length 384 \    --doc_stride 128 \    --output_dir ../models/wwm_uncased_finetuned_squad/ \    --per_gpu_eval_batch_size=3   \    --per_gpu_train_batch_size=3   \

python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json ../models/wwm_uncased_finetuned_squad/predictions.json{"exact_match": 86.91579943235573, "f1": 93.1532499015869}

python ./examples/run_generation.py \    --model_type=gpt2 \    --length=20 \    --model_name_or_path=gpt2 \

# Let's load our modelmodel = BertForSequenceClassification.from_pretrained('bert-base-uncased')
# If you used to have this line in pytorch-pretrained-bert:loss = model(input_ids, labels=labels)
# Now just use this line in pytorch-transformers to extract the loss from the output tuple:outputs = model(input_ids, labels=labels)loss = outputs[0]
# In pytorch-transformers you can also have access to the logits:loss, logits = outputs[:2]
# And even the attention weigths if you configure the model to output them (and other outputs too, see the docstrings and documentation)model = BertForSequenceClassification.from_pretrained('bert-base-uncased', output_attentions=True)outputs = model(input_ids, labels=labels)loss, logits, attentions = outputs

### Let's load a model and tokenizermodel = BertForSequenceClassification.from_pretrained('bert-base-uncased')tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
### Do some stuff to our model and tokenizer# Ex: add new tokens to the vocabulary and embeddings of our modeltokenizer.add_tokens(['[SPECIAL_TOKEN_1]', '[SPECIAL_TOKEN_2]'])model.resize_token_embeddings(len(tokenizer))# Train our modeltrain(model)
### Now let's save our model and tokenizer to a directorymodel.save_pretrained('./my_saved_model_directory/')tokenizer.save_pretrained('./my_saved_model_directory/')
### Reload the model and the tokenizermodel = BertForSequenceClassification.from_pretrained('./my_saved_model_directory/')tokenizer = BertTokenizer.from_pretrained('./my_saved_model_directory/')

# Parameters:lr = 1e-3num_total_steps = 1000num_warmup_steps = 100warmup_proportion = float(num_warmup_steps) / float(num_total_steps)  # 0.1
### Previously BertAdam optimizer was instantiated like this:optimizer = BertAdam(model.parameters(), lr=lr, schedule='warmup_linear', warmup=warmup_proportion, t_total=num_total_steps)### and used like this:for batch in train_data:    loss = model(batch)    loss.backward()    optimizer.step()
### In PyTorch-Transformers, optimizer and schedules are splitted and instantiated like this:optimizer = AdamW(model.parameters(), lr=lr, correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=Falsescheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)  # PyTorch scheduler### and used like this:for batch in train_data:    loss = model(batch)    loss.backward()    scheduler.step()    optimizer.step()