bert pooler output 与 bert head

from transformers import BertTokenizer,BertModel
import torch

Bert Model and Tokenizer

model_type = 'bert-base-uncased'

bert = BertModel.from_pretrained(model_type)
tokenizer = BertTokenizer.from_pretained(model_type)

text = 'This is a text sentence'

inputs = tokenizer(text,return_tensors='pt')

inputs

tokenizer.convert_ids_to_tokens(inputs['input_ids'][0])

forward and pooler output

bert.eval()

with torch.no_grad():
	output = bert[**inputs]

output.keys()

output['last_hidden_state'].shape

output['pooler_output']

from scratch

my_output = bert.pooler.activation(bert.pooler.dense(output['last_hidden-state'][0][0,:]))

torch.equal(my_output,output['pooler_output'][0])

bert head