浙江龙网

  1. 首页 > 生活百科 > >

BERT模型源码解析( 二 )

生活百科


self.num_attention_heads = num_attention_heads
self.hidden_act = hidden_act
self.intermediate_size = intermediate_size
self.hidden_dropout_prob = hidden_dropout_prob
self.attention_probs_dropout_prob = attention_probs_dropout_prob
self.max_position_embeddings = max_position_embeddings
self.type_vocab_size = type_vocab_size
self.initializer_range = initializer_range
@classmethod 类方法
def from_dict(cls, json_object):
"""Constructs a `BertConfig` from a Python dictionary of parameters."""
从一个参数字典构造配置参数
config = BertConfig(vocab_size=None)
for (key, value) in six.iteritems(json_object):
config.__dict__[key] = value
return config
@classmethod
def from_json_file(cls, json_file): 从JSON文件构造BertConfig对象
"""Constructs a `BertConfig` from a json file of parameters."""
从一个JSON文件构造配置参数
with tf.gfile.GFile(json_file, "r") as reader:
text = reader.read()
return cls.from_dict(json.loads(text))
def to_dict(self): 将BertConfig对象转换为字典
"""Serializes this instance to a Python dictionary."""
output = copy.deepcopy(self.__dict__)
return output
def to_json_string(self):  将BertConfig对象转换为JSON格式字符串
"""Serializes this instance to a JSON string."""
return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n"
BERT模型类
BertModel
class BertModel(object):
  """BERT model ("Bidirectional Encoder Representations from Transformers").
Transformers模型的双向编码表示
Example usage:示例
```python
已经转换为 词片 id形式
# Already been converted into WordPiece token ids
 tf.constant用于创建常量张量
input_ids = tf.constant([[31, 51, 99], [15, 5, 0]])
input_mask = tf.constant([[1, 1, 1], [1, 1, 0]])
token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]])
创建配置参数对象config
config = modeling.BertConfig(vocab_size=32000, hidden_size=512,
num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024)
创建模型对象model
model = modeling.BertModel(config=config, is_training=True,
input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids)
嵌入层标签、池化输出
label_embeddings = tf.get_variable(...)
pooled_output = model.get_pooled_output()
tf.matmul=matrix multiply矩阵相乘
logits = tf.matmul(pooled_output, label_embeddings)
...
```
"""
def __init__(self,
config,
is_training,
input_ids,
input_mask=None,
token_type_ids=None,
use_one_hot_embeddings=False,
scope=None):
"""Constructor for BertModel.
Args:
config: `BertConfig` instance.配置参数对象
is_training: bool. true for training model, false for eval model. Controls
whether dropout will be applied.是否进行训练
input_ids: int32 Tensor of shape [batch_size, seq_length].输入维度
input_mask: (optional) int32 Tensor of shape [batch_size, seq_length].输入掩码
token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length].令牌类型
use_one_hot_embeddings: (optional) bool. Whether to use one-hot word
embeddings or tf.embedding_lookup() for the word embeddings.
嵌入层：是否使用one-hot词嵌入
scope: (optional) variable scope. Defaults to "bert".
可选变量，默认值"bert".
Raises:
ValueError: The config is invalid or one of the input tensor shapes
is invalid.
异常：值错误，配置参数无效或者张量形状无效
"""
config = copy.deepcopy(config) 配置参数对象深度克隆
if not is_training: 如果不训练
config.hidden_dropout_prob = 0.0  放弃比例设置为0，表示不放弃参数
config.attention_probs_dropout_prob = 0.0 放弃比例设置为0，表示不放弃参数
获取输入形状
input_shape = get_shape_list(input_ids, expected_rank=2)
batch_size = input_shape[0]  批处理量，每一批处理的数据条数
seq_length = input_shape[1]  序列长度
if input_mask is None:  如果没有输出掩码，则将掩码全部设置为1
input_mask = tf.ones(shape=[batch_size, seq_length], dtype=tf.int32)
if token_type_ids is None:  如果没有令牌，则将令牌全部设置为0
token_type_ids = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32)
variable_scope 变量共享、上下文管理器，作用域；
在这个管理器下做的事情 ， 会被这个管理器管着
with tf.variable_scope(scope, default_name="bert"):
with tf.variable_scope("embeddings"):
# Perform embedding lookup on the word ids.对单词id执行嵌入查找 。

推荐阅读


上一篇：【lwip】11-UDP协议&源码分析

下一篇：Go实现栈与队列基本操作