Allennlp_tutorial.py

# -*- coding: utf-8 -*-
"""Untitled15.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1v5O1keH-ubftBgBvlfH_bYz4HBpLJ6zA
"""

!pip install allennlp
!pip install fairseq

from google.colab import drive
drive.mount('/content/drive')
!ls "/content/drive/My Drive/"

label_cols={'toxic','severe_toxic','obscene','threat','insult','identity_hate'}

import torch
import torch.nn as nn
import numpy as np
import pandas as pd
import torch.optim as optim
from typing import *
from overrides import overrides

from allennlp.data import Instance
from allennlp.data.dataset_readers import DatasetReader
from allennlp.data.fields import *
from allennlp.data.token_indexers import SingleIdTokenIndexer
from allennlp.data.tokenizers import Token
from allennlp.data.token_indexers import TokenIndexer

class Config(dict):
    def __init__(self,**kwargs):
        super().__init__(**kwargs)
        for k,v in kwargs.items():
            setattr(self,k,v)
    def set(self,key,val):
        self[key] = val
        setattr(self,key,value)

config = Config(testing=True,
               seed=1,
               batch_size = 64,
               lr=3e-4,
               epochs=1000,
               hidden_size = 64,
               max_seq_len = 100,
               max_vocab_size=10000,)

DATA_PATH = "/content/drive/My Drive/jigsaw/"
USE_CUDA = torch.cuda.is_available()

torch.manual_seed(config.seed)

class JigsawDataReader(DatasetReader):
    def __init__(self,tokenizer,token_indexers,max_seq_len=config.max_seq_len):
        super().__init__(lazy=False)
        self.tokenizer = tokenizer
        self.token_indexers = token_indexers or {"tokens":SingleIdTokenIndexer()}
        self.max_seq_len = max_seq_len
    
    @overrides
    def text_to_instance(self,token,id,labels):
        sentence_field = TextField(token,self.token_indexers)
        fields = {'tokens':sentence_field}
        
        id_field = MetadataField(id)
        fields['id'] = id_field
        
        if labels is None:
            labels = np.zeros(len(label))
        label_field = ArrayField(array=labels)
        fields['label'] = label_field
        return Instance(fields)
            
        
    @overrides
    def _read(self,path=DATA_PATH):
        df = pd.read_csv(path)
        if config.testing == True:df = df.head(1000)
        for i,row in df.iterrows():
            yield self.text_to_instance([Token(x) for x in self.tokenizer(row["comment_text"])],
                                       row["id"],row[label_cols].values)

from allennlp.data.tokenizers.word_splitter import SpacyWordSplitter

token_indexers = SingleIdTokenIndexer()

def tokenizer(x):
    return [w.text for w in SpacyWordSplitter(language='en_core_web_sm',pos_tags=False).split_words(x)[:config.max_seq_len]]

reader = JigsawDataReader(tokenizer=tokenizer,token_indexers={'tokens':token_indexers})

train_ds,test_ds = (reader.read(DATA_PATH+w) for w in ["train.csv", "test_proced.csv"])
val_ds = None

vars(train_ds[0].fields['tokens'])

from allennlp.data.vocabulary import Vocabulary
vocab = Vocabulary.from_instances(train_ds,max_vocab_size = config.max_vocab_size)

from allennlp.data.iterators import BucketIterator
iterator = BucketIterator(batch_size=config.batch_size,sorting_keys=[('tokens','num_tokens')],)

iterator.index_with(vocab)

batch = next(iter(iterator(train_ds)))

from allennlp.modules.seq2vec_encoders import Seq2VecEncoder,PytorchSeq2VecWrapper
from allennlp.modules.text_field_embedders import TextFieldEmbedder
from allennlp.nn.util import get_text_field_mask
from allennlp.models import Model

class BaslinModel(Model):
    def __init__(self,word_embeddings,encoder,out_sz=len(label_cols)):
        super().__init__(vocab)
        self.word_embeddings = word_embeddings
        self.encoder = encoder
        self.projection = nn.Linear(self.encoder.get_output_dim(),out_sz)
        self.loss = nn.BCEWithLogitsLoss()
    
    def forward(self,tokens,id,label):
        mask = get_text_field_mask(tokens)
        embeddings = self.word_embeddings(tokens)
        state = self.encoder(embeddings,mask)
        class_logits = self.projection(state)       
        out = {'class_logits':class_logits}
        out['loss'] = self.loss(class_logits,label)
        return out

from allennlp.modules.token_embedders import Embedding
from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder

token_embedding = Embedding(num_embeddings=config.max_vocab_size+2,embedding_dim=300,padding_index=0)
word_embeddings = BasicTextFieldEmbedder({'tokens':token_embedding})
encoder = PytorchSeq2VecWrapper(nn.LSTM(word_embeddings.get_output_dim(),config.batch_size,bidirectional=True,batch_first = True))

model = BaslinModel(word_embeddings,encoder)

if USE_CUDA:model.cuda()
else:
    model()

import allennlp.nn.util as nn_util

batch = nn_util.move_to_device(batch,0 if USE_CUDA else -1)

tokens = batch['tokens']
label = batch['label']

mask = get_text_field_mask(tokens)
embeddings = model.word_embeddings(tokens)
state = model.encoder(embeddings, mask)
class_logits = model.projection(state)
class_logits

loss = model(**batch)["loss"]
loss.backward()
optimizer = optim.Adam(model.parameters(),config.lr)

from allennlp.training.trainer import Trainer
trainer = Trainer(
            model=model,
            optimizer = optimizer,
            iterator=iterator,
            train_dataset=train_ds,
            cuda_device=0 if USE_CUDA else -1,
            num_epochs=config.epochs,)

metrics = trainer.train()

from allennlp.data.iterators import DataIterator
from tqdm import tqdm
from scipy.special import expit

def tonp(tsr):return tsr.detach().cpu().numpy()

class Predictor:
    def __init__(self,model,iterator,device=-1):
        self.model = model
        self.iterator = iterator
        self.device = device
    
    def _extract_data(self,batch):
        out_dict = model(**batch)
        return expit(tonp(out_dict['class_logits']))
        
        
    def predict(self,ds):
        pred_generator = self.iterator(ds,num_epochs=1,shuffle=False)
        self.model.eval()
        pred_generator_tqdm = tqdm(pred_generator,
                                   total=self.iterator.get_num_batches(ds))
        preds = []
        with torch.no_grad():
            for batch in pred_generator_tqdm:
                batch = nn_util.move_to_device(batch, self.device)
                preds.append(self._extract_data(batch))
        return np.concatenate(preds, axis=0)

from allennlp.data.iterators import BasicIterator
seq_iterator = BasicIterator(batch_size=64)
seq_iterator.index_with(vocab)
predictor = Predictor(model, seq_iterator, device=0 if USE_CUDA else -1)
train_preds = predictor.predict(train_ds) 
test_preds = predictor.predict(test_ds)

# from allennlp.predictors.sentence_tagger import SentenceTaggerPredictor
# tagger = SentenceTaggerPredictor(model, reader)
# tagger.predict("this tutorial was great!")
# tagger.predict("this tutorial was horrible!")
train_preds

from typing import Iterator, List, Dict
import torch
import torch.optim as optim
import numpy as np
from allennlp.data import Instance
from allennlp.data.fields import TextField, SequenceLabelField
from allennlp.data.dataset_readers import DatasetReader
from allennlp.common.file_utils import cached_path
from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer
from allennlp.data.tokenizers import Token
from allennlp.data.vocabulary import Vocabulary
from allennlp.models import Model
from allennlp.modules.text_field_embedders import TextFieldEmbedder, BasicTextFieldEmbedder
from allennlp.modules.token_embedders import Embedding
from allennlp.modules.seq2seq_encoders import Seq2SeqEncoder, PytorchSeq2SeqWrapper
from allennlp.nn.util import get_text_field_mask, sequence_cross_entropy_with_logits
from allennlp.training.metrics import CategoricalAccuracy
from allennlp.data.iterators import BucketIterator
from allennlp.training.trainer import Trainer
from allennlp.predictors import SentenceTaggerPredictor

torch.manual_seed(1)
class PosDatasetReader(DatasetReader):
    """
    DatasetReader for PoS tagging data, one sentence per line, like

        The###DET dog###NN ate###V the###DET apple###NN
    """
    def __init__(self, token_indexers: Dict[str, TokenIndexer] = None) -> None:
        super().__init__(lazy=False)
        self.token_indexers = token_indexers or {"tokens": SingleIdTokenIndexer()}
        
    def text_to_instance(self, tokens: List[Token], tags: List[str] = None) -> Instance:
        sentence_field = TextField(tokens, self.token_indexers)
        fields = {"sentence": sentence_field}

        if tags:
            label_field = SequenceLabelField(labels=tags, sequence_field=sentence_field)
            fields["labels"] = label_field

        return Instance(fields)
      
    def _read(self, file_path: str) -> Iterator[Instance]:
        with open(file_path) as f:
            for line in f:
                pairs = line.strip().split()
                sentence, tags = zip(*(pair.split("###") for pair in pairs))
                yield self.text_to_instance([Token(word) for word in sentence], tags)
                
class LstmTagger(Model):
    def __init__(self,
                 word_embeddings: TextFieldEmbedder,
                 encoder: Seq2SeqEncoder,
                 vocab: Vocabulary) -> None:
        super().__init__(vocab)
        self.word_embeddings = word_embeddings
        self.encoder = encoder
        self.hidden2tag = torch.nn.Linear(in_features=encoder.get_output_dim(),
                                          out_features=vocab.get_vocab_size('labels'))
        self.accuracy = CategoricalAccuracy()
    def forward(self,
                sentence: Dict[str, torch.Tensor],
                labels: torch.Tensor = None) -> Dict[str, torch.Tensor]:
        mask = get_text_field_mask(sentence)
        embeddings = self.word_embeddings(sentence)
        encoder_out = self.encoder(embeddings, mask)
        tag_logits = self.hidden2tag(encoder_out)
        output = {"tag_logits": tag_logits}
        if labels is not None:
            self.accuracy(tag_logits, labels, mask)
            output["loss"] = sequence_cross_entropy_with_logits(tag_logits, labels, mask)

        return output
    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        return {"accuracy": self.accuracy.get_metric(reset)}
    
reader = PosDatasetReader()
train_dataset = reader.read(cached_path(
    'https://raw.githubusercontent.com/allenai/allennlp'
    '/master/tutorials/tagger/training.txt'))
validation_dataset = reader.read(cached_path(
    'https://raw.githubusercontent.com/allenai/allennlp'
    '/master/tutorials/tagger/validation.txt'))
vocab = Vocabulary.from_instances(train_dataset + validation_dataset)
EMBEDDING_DIM = 6
HIDDEN_DIM = 6
token_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
                            embedding_dim=EMBEDDING_DIM)
word_embeddings = BasicTextFieldEmbedder({"tokens": token_embedding})
lstm = PytorchSeq2SeqWrapper(torch.nn.LSTM(EMBEDDING_DIM, HIDDEN_DIM, batch_first=True))
model = LstmTagger(word_embeddings, lstm, vocab)
if torch.cuda.is_available():
    cuda_device = 0
    model = model.cuda(cuda_device)
else:
    cuda_device = -1
optimizer = optim.SGD(model.parameters(), lr=0.1)
iterator = BucketIterator(batch_size=2, sorting_keys=[("sentence", "num_tokens")])
iterator.index_with(vocab)
trainer = Trainer(model=model,
                  optimizer=optimizer,
                  iterator=iterator,
                  train_dataset=train_dataset,
                  validation_dataset=validation_dataset,
                  patience=10,
                  num_epochs=1000,
                  cuda_device=cuda_device)
trainer.train()
predictor = SentenceTaggerPredictor(model, dataset_reader=reader)
tag_logits = predictor.predict("The dog ate the apple")['tag_logits']
tag_ids = np.argmax(tag_logits, axis=-1)
print([model.vocab.get_token_from_index(i, 'labels') for i in tag_ids])
# Here's how to save the model.
with open("/tmp/model.th", 'wb') as f:
    torch.save(model.state_dict(), f)
vocab.save_to_files("/tmp/vocabulary")
# And here's how to reload the model.
vocab2 = Vocabulary.from_files("/tmp/vocabulary")
model2 = LstmTagger(word_embeddings, lstm, vocab2)
with open("/tmp/model.th", 'rb') as f:
    model2.load_state_dict(torch.load(f))
if cuda_device > -1:
    model2.cuda(cuda_device)
predictor2 = SentenceTaggerPredictor(model2, dataset_reader=reader)
tag_logits2 = predictor2.predict("The dog ate the apple")['tag_logits']
np.testing.assert_array_almost_equal(tag_logits2, tag_logits)





# 导入必要的库
import torch
from pytorch_transformers import GPT2Tokenizer, GPT2LMHeadModel
# 加载预训练模型tokenizer (vocabulary)
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
# 对文本输入进行编码
text = "What is the fastest car in the"
indexed_tokens = tokenizer.encode(text)
# 在PyTorch张量中转换indexed_tokens
tokens_tensor = torch.tensor([indexed_tokens])
# 加载预训练模型 (weights)
model = GPT2LMHeadModel.from_pretrained('gpt2')
#将模型设置为evaluation模式，关闭DropOut模块
model.eval()
# 如果你有GPU，把所有东西都放在cuda上
tokens_tensor = tokens_tensor.to('cuda')
model.to('cuda')
# 预测所有的tokens
with torch.no_grad(): 
  outputs = model(tokens_tensor) 
  predictions = outputs[0]
# 得到预测的单词
  predicted_index = torch.argmax(predictions[0, -1, :]).item()
  predicted_text = tokenizer.decode(indexed_tokens + [predicted_index])
  # 打印预测单词
  print(predicted_text)
  #代码很直观，我们将文本标记为数字序列并将其索引，然后将其传递给GPT2LMHeadModel。