masking机制、bert head和BertForMaskedLM

import torch
from torch import nn
from transformers.model.bert import BertModel,BertTokenizer,BertForMaskedLM

model load and data preprocessing

model_type = 'bert-base-uncased'

tokenizer = BertTokenizer.from_pretained(model_type)
bert = BertModel.from_pretained(model_type)
mlm = BertForMaskedLM.from_pretained(model_type,output_hidden_states=True)

text = ("After Abraham Lincoln won the November 1860 presidential "
        "election on an anti-slavery platform, an initial seven "
        "slave states declared their secession from the country "
        "to form the Confederacy. War broke out in April 1861 "
        "when secessionist forces attacked Fort Sumter in South "
        "Carolina, just over a month after Lincoln's "
        "inauguration.")

text

bert

mlm

inputs = tokenizer(text,return_tensors='pt')

inputs

inputs['input_ids'].shape

' '.join(tokenizer.convert_ids_to_tokens(inputs['input_ids'][0]))

masking

inputs['labels']=inputs['input_ids'].detach().clone()

inputs['labels']

mask = torch.rand(inputs['input_ids'].shape)<0.15

mask

sum(mask[0])

mask_arr = (torch.rand(inputs['input_ids'].shape)<0.15)\
		*(inputs['input_ids']!=101)\
		*(inputs['input_ids']!=102)

mask_arr

sum(mask_arr[0])

selection = torch.flatten(mask_arr[0].nonzero().tolist())

selection

tokenizer.specical_tokens_map

tokenizer.vocab['[MASK]']

inputs['input_ids'][0,selection]=103

inputs

' '.join(tokenizer.convert_ids_to_tokens(inputs['input_ids'][0]))

' '.join(tokenizer.convert_ids_to_tokens(inputs['labels'][0]))

forward and calculate loss

mlm.eval()
with torch.no_grad():
	output = mlm(**inputs)

output.keys()

output.logits

output.loss

type(output['hidden_states'])

output['hidden_states'][-1]

from scratch

mlm.cls(output['hidden_states'][-1])

output.logits

last_hidden_state = output['hidden_states'][-1]

last_hidden_state.shape

mlm.eval()
with torch.no_grad():
	transformed = mlm.cls.predictions.transform(last_hidden_state)
	print(transformed.shape)
	logits = mlm.cls.prediction.decoder(transformed)
	print(logits.shape)
logits

loss and translate

ce = nn.CrossEntropyLoss()

logits.shape

inputs['label'].shape

inputs['label'][0].view(-1).shape

ce(logits[0],inputs['labels'][0].view(-1))

torch.argmax(logits[0],dim=1)

' '.join(tokenizer.convert_ids_to_tokens(torch.argmax[logits[0],dim=1]))

' '.join(toeknizer.convet_ids_to_tokens(inputs['labels'][0]))