bert embedding,word embedding,pooler embedding

bert embedding

summary

• bert input embedding:一种查表结构(LUT)
  • LUT
    • token embeddings:
    • segment embeddings
    • position embeddings
  • 后处理:
    • layer norm
    • dropout

from transformers import BertTokenizer,BertModel
import torch
from torch import nn

model_name = 'bert-base-uncased'
tokenizer = BertTokenizer.from_pretained(model_name)
model = BertModel.from_pretained(model_nmae)

test_sent = 'this is a test sentence'

input = tokenizer(test_sent,return_tensors='pt')

input_ids = input['input_ids']
token_type_ids = input['token_type_ids']

pos_ids = torch.arange(input_ids.shape[1])

pos_ids

token_embed = model.embeddings.word_embeddings(input_ids)

model.embeddings(input_ids,token_type_ids)

model.embeddings(input_ids,token_type_ids)

token embedding

token_embed = model.embeddings.word_embeddings(input_ids)
token_embed.shape

segment type embedding

seg_embed=model.embeddings.token_type_embeddings(token_type_ids)
seg_embed.shape

pos embedding

pos_embed = model.embeddings.position_embeddings(pos_ids)

pos_embed.shape

input embedding

input_embed = token_embed+seg_embed+pos_embed.unsqueeze(0)

后处理

model=model.embeddings.LayerNorm(input_embed)