!1398 Update the bert scripts according to rules of modelzoo
Merge pull request !1398 from chenhaozhe/update_bert_scriptpull/1398/MERGE
mindspore-ci-bot
5 years ago
committed by
Gitee
commit
b46ad9a1bb
@ -0,0 +1,121 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""fused layernorm"""
|
||||
from mindspore.ops import operations as P
|
||||
from mindspore.ops import functional as F
|
||||
from mindspore.common.parameter import Parameter
|
||||
from mindspore.common.initializer import initializer
|
||||
from mindspore.ops.primitive import constexpr
|
||||
import mindspore.common.dtype as mstype
|
||||
from mindspore.nn.cell import Cell
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
__all__ = ['FusedLayerNorm']
|
||||
|
||||
@constexpr
|
||||
def get_shape_for_norm(x_shape, begin_norm_axis):
|
||||
print("input_shape: ", x_shape)
|
||||
norm_shape = x_shape[begin_norm_axis:]
|
||||
output_shape = (1, -1, 1, int(np.prod(norm_shape)))
|
||||
print("output_shape: ", output_shape)
|
||||
return output_shape
|
||||
|
||||
class FusedLayerNorm(Cell):
|
||||
r"""
|
||||
Applies Layer Normalization over a mini-batch of inputs.
|
||||
|
||||
Layer normalization is widely used in recurrent neural networks. It applies
|
||||
normalization over a mini-batch of inputs for each single training case as described
|
||||
in the paper `Layer Normalization <https://arxiv.org/pdf/1607.06450.pdf>`_. Unlike batch
|
||||
normalization, layer normalization performs exactly the same computation at training and
|
||||
testing times. It can be described using the following formula. It is applied across all channels
|
||||
and pixel but only one batch size.
|
||||
|
||||
.. math::
|
||||
y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta
|
||||
|
||||
Args:
|
||||
normalized_shape (Union(tuple[int], list[int]): The normalization is performed over axis
|
||||
`begin_norm_axis ... R - 1`.
|
||||
begin_norm_axis (int): It first normalization dimension: normalization will be performed along dimensions
|
||||
`begin_norm_axis: rank(inputs)`, the value should be in [-1, rank(input)). Default: -1.
|
||||
begin_params_axis (int): The first parameter(beta, gamma)dimension: scale and centering parameters
|
||||
will have dimensions `begin_params_axis: rank(inputs)` and will be broadcast with
|
||||
the normalized inputs accordingly, the value should be in [-1, rank(input)). Default: -1.
|
||||
gamma_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the gamma weight.
|
||||
The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform',
|
||||
'he_uniform', etc. Default: 'ones'.
|
||||
beta_init (Union[Tensor, str, Initializer, numbers.Number]): Initializer for the beta weight.
|
||||
The values of str refer to the function `initializer` including 'zeros', 'ones', 'xavier_uniform',
|
||||
'he_uniform', etc. Default: 'zeros'.
|
||||
use_batch_nrom (bool): Whether use batchnorm to preocess.
|
||||
|
||||
Inputs:
|
||||
- **input_x** (Tensor) - The shape of 'input_x' is :math:`(x_1, x_2, ..., x_R)`,
|
||||
and `input_shape[begin_norm_axis:]` is equal to `normalized_shape`.
|
||||
|
||||
Outputs:
|
||||
Tensor, the normalized and scaled offset tensor, has the same shape and data type as the `input_x`.
|
||||
|
||||
Examples:
|
||||
>>> x = Tensor(np.ones([20, 5, 10, 10]), mindspore.float32)
|
||||
>>> shape1 = x.shape()[1:]
|
||||
>>> m = nn.LayerNorm(shape1, begin_norm_axis=1, begin_params_axis=1)
|
||||
>>> m(x)
|
||||
"""
|
||||
def __init__(self,
|
||||
normalized_shape,
|
||||
begin_norm_axis=-1,
|
||||
begin_params_axis=-1,
|
||||
gamma_init='ones',
|
||||
beta_init='zeros',
|
||||
use_batch_norm=False):
|
||||
super(FusedLayerNorm, self).__init__()
|
||||
if not isinstance(normalized_shape, (tuple, list)):
|
||||
raise TypeError("The type of 'normalized_shape' should be tuple[int] or list[int], but '{}' type is {}."
|
||||
.format(normalized_shape, type(normalized_shape)))
|
||||
self.normalized_shape = normalized_shape
|
||||
self.begin_norm_axis = begin_norm_axis
|
||||
self.begin_params_axis = begin_params_axis
|
||||
self.gamma = Parameter(initializer(
|
||||
gamma_init, normalized_shape), name="gamma")
|
||||
self.beta = Parameter(initializer(
|
||||
beta_init, normalized_shape), name="beta")
|
||||
self.layer_norm = P.LayerNorm(begin_norm_axis=self.begin_norm_axis, begin_params_axis=self.begin_params_axis)
|
||||
|
||||
self.batch_norm = P.BatchNorm(is_training=True, epsilon=1e-5)
|
||||
self.use_batch_norm = use_batch_norm
|
||||
|
||||
def construct(self, input_x):
|
||||
if self.use_batch_norm and self.training:
|
||||
ones = P.Fill()(mstype.float32, F.shape(input_x)[:self.begin_norm_axis], 1.0)
|
||||
zeros = P.Fill()(mstype.float32, F.shape(input_x)[:self.begin_norm_axis], 0.0)
|
||||
shape_x = F.shape(input_x)
|
||||
norm_shape = get_shape_for_norm(shape_x, self.begin_norm_axis)
|
||||
input_x = F.reshape(input_x, norm_shape)
|
||||
output, _, _, _, _, _ = self.batch_norm(input_x, ones, zeros, None, None)
|
||||
output = F.reshape(output, shape_x)
|
||||
y = output * self.gamma + self.beta
|
||||
else:
|
||||
y, _, _ = self.layer_norm(input_x, self.gamma, self.beta)
|
||||
return y
|
||||
|
||||
def extend_repr(self):
|
||||
"""Display instance object as string."""
|
||||
s = 'normalized_shape={}, begin_norm_axis={}, begin_params_axis={}, gamma{}, beta={}'.format(
|
||||
self.normalized_shape, self.begin_norm_axis, self.begin_params_axis, self.gamma, self.beta)
|
||||
return s
|
||||
@ -0,0 +1,177 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
|
||||
'''
|
||||
CRF script.
|
||||
'''
|
||||
|
||||
import numpy as np
|
||||
import mindspore.nn as nn
|
||||
from mindspore.ops import operations as P
|
||||
from mindspore.common.tensor import Tensor
|
||||
from mindspore.common.parameter import Parameter
|
||||
import mindspore.common.dtype as mstype
|
||||
|
||||
class CRF(nn.Cell):
|
||||
'''
|
||||
Conditional Random Field
|
||||
Args:
|
||||
tag_to_index: The dict for tag to index mapping with extra "<START>" and "<STOP>"sign.
|
||||
batch_size: Batch size, i.e., the length of the first dimension.
|
||||
seq_length: Sequence length, i.e., the length of the second dimention.
|
||||
is_training: Specifies whether to use training mode.
|
||||
Returns:
|
||||
Training mode: Tensor, total loss.
|
||||
Evaluation mode: Tuple, the index for each step with the highest score; Tuple, the index for the last
|
||||
step with the highest score.
|
||||
'''
|
||||
def __init__(self, tag_to_index, batch_size=1, seq_length=128, is_training=True):
|
||||
|
||||
super(CRF, self).__init__()
|
||||
self.target_size = len(tag_to_index)
|
||||
self.is_training = is_training
|
||||
self.tag_to_index = tag_to_index
|
||||
self.batch_size = batch_size
|
||||
self.seq_length = seq_length
|
||||
self.START_TAG = "<START>"
|
||||
self.STOP_TAG = "<STOP>"
|
||||
self.START_VALUE = Tensor(self.target_size-2, dtype=mstype.int32)
|
||||
self.STOP_VALUE = Tensor(self.target_size-1, dtype=mstype.int32)
|
||||
transitions = np.random.normal(size=(self.target_size, self.target_size)).astype(np.float32)
|
||||
transitions[tag_to_index[self.START_TAG], :] = -10000
|
||||
transitions[:, tag_to_index[self.STOP_TAG]] = -10000
|
||||
self.transitions = Parameter(Tensor(transitions), name="transition_matrix")
|
||||
self.cat = P.Concat(axis=-1)
|
||||
self.argmax = P.ArgMaxWithValue(axis=-1)
|
||||
self.log = P.Log()
|
||||
self.exp = P.Exp()
|
||||
self.sum = P.ReduceSum()
|
||||
self.tile = P.Tile()
|
||||
self.reduce_sum = P.ReduceSum(keep_dims=True)
|
||||
self.reshape = P.Reshape()
|
||||
self.expand = P.ExpandDims()
|
||||
self.mean = P.ReduceMean()
|
||||
init_alphas = np.ones(shape=(self.batch_size, self.target_size)) * -10000.0
|
||||
init_alphas[:, self.tag_to_index[self.START_TAG]] = 0.
|
||||
self.init_alphas = Tensor(init_alphas, dtype=mstype.float32)
|
||||
self.cast = P.Cast()
|
||||
self.reduce_max = P.ReduceMax(keep_dims=True)
|
||||
self.on_value = Tensor(1.0, dtype=mstype.float32)
|
||||
self.off_value = Tensor(0.0, dtype=mstype.float32)
|
||||
self.onehot = P.OneHot()
|
||||
|
||||
def log_sum_exp(self, logits):
|
||||
'''
|
||||
Compute the log_sum_exp score for normalization factor.
|
||||
'''
|
||||
max_score = self.reduce_max(logits, -1) #16 5 5
|
||||
score = self.log(self.reduce_sum(self.exp(logits - max_score), -1))
|
||||
score = max_score + score
|
||||
return score
|
||||
|
||||
def _realpath_score(self, features, label):
|
||||
'''
|
||||
Compute the emission and transition score for the real path.
|
||||
'''
|
||||
label = label * 1
|
||||
concat_A = self.tile(self.reshape(self.START_VALUE, (1,)), (self.batch_size,))
|
||||
concat_A = self.reshape(concat_A, (self.batch_size, 1))
|
||||
labels = self.cat((concat_A, label))
|
||||
onehot_label = self.onehot(label, self.target_size, self.on_value, self.off_value)
|
||||
emits = features * onehot_label
|
||||
labels = self.onehot(labels, self.target_size, self.on_value, self.off_value)
|
||||
label1 = labels[:, 1:, :]
|
||||
label2 = labels[:, :self.seq_length, :]
|
||||
label1 = self.expand(label1, 3)
|
||||
label2 = self.expand(label2, 2)
|
||||
label_trans = label1 * label2
|
||||
transitions = self.expand(self.expand(self.transitions, 0), 0)
|
||||
trans = transitions * label_trans
|
||||
score = self.sum(emits, (1, 2)) + self.sum(trans, (1, 2, 3))
|
||||
stop_value_index = labels[:, (self.seq_length-1):self.seq_length, :]
|
||||
stop_value = self.transitions[(self.target_size-1):self.target_size, :]
|
||||
stop_score = stop_value * self.reshape(stop_value_index, (self.batch_size, self.target_size))
|
||||
score = score + self.sum(stop_score, 1)
|
||||
score = self.reshape(score, (self.batch_size, -1))
|
||||
return score
|
||||
|
||||
def _normalization_factor(self, features):
|
||||
'''
|
||||
Compute the total score for all the paths.
|
||||
'''
|
||||
forward_var = self.init_alphas
|
||||
forward_var = self.expand(forward_var, 1)
|
||||
for idx in range(self.seq_length):
|
||||
feat = features[:, idx:(idx+1), :]
|
||||
emit_score = self.reshape(feat, (self.batch_size, self.target_size, 1))
|
||||
next_tag_var = emit_score + self.transitions + forward_var
|
||||
forward_var = self.log_sum_exp(next_tag_var)
|
||||
forward_var = self.reshape(forward_var, (self.batch_size, 1, self.target_size))
|
||||
terminal_var = forward_var + self.reshape(self.transitions[(self.target_size-1):self.target_size, :], (1, -1))
|
||||
alpha = self.log_sum_exp(terminal_var)
|
||||
alpha = self.reshape(alpha, (self.batch_size, -1))
|
||||
return alpha
|
||||
|
||||
def _decoder(self, features):
|
||||
'''
|
||||
Viterbi decode for evaluation.
|
||||
'''
|
||||
backpointers = ()
|
||||
forward_var = self.init_alphas
|
||||
for idx in range(self.seq_length):
|
||||
feat = features[:, idx:(idx+1), :]
|
||||
feat = self.reshape(feat, (self.batch_size, self.target_size))
|
||||
bptrs_t = ()
|
||||
|
||||
next_tag_var = self.expand(forward_var, 1) + self.transitions
|
||||
best_tag_id, best_tag_value = self.argmax(next_tag_var)
|
||||
bptrs_t += (best_tag_id,)
|
||||
forward_var = best_tag_value + feat
|
||||
|
||||
backpointers += (bptrs_t,)
|
||||
terminal_var = forward_var + self.reshape(self.transitions[(self.target_size-1):self.target_size, :], (1, -1))
|
||||
best_tag_id, _ = self.argmax(terminal_var)
|
||||
return backpointers, best_tag_id
|
||||
|
||||
def construct(self, features, label):
|
||||
if self.is_training:
|
||||
forward_score = self._normalization_factor(features)
|
||||
gold_score = self._realpath_score(features, label)
|
||||
return_value = self.mean(forward_score - gold_score)
|
||||
else:
|
||||
path_list, tag = self._decoder(features)
|
||||
return_value = path_list, tag
|
||||
return return_value
|
||||
|
||||
def postprocess(backpointers, best_tag_id):
|
||||
'''
|
||||
Do postprocess
|
||||
'''
|
||||
best_tag_id = best_tag_id.asnumpy()
|
||||
batch_size = len(best_tag_id)
|
||||
best_path = []
|
||||
for i in range(batch_size):
|
||||
best_path.append([])
|
||||
best_local_id = best_tag_id[i]
|
||||
best_path[-1].append(best_local_id)
|
||||
for bptrs_t in reversed(backpointers):
|
||||
bptrs_t = bptrs_t[0].asnumpy()
|
||||
local_idx = bptrs_t[i]
|
||||
best_local_id = local_idx[best_local_id]
|
||||
best_path[-1].append(best_local_id)
|
||||
# Pop off the start tag (we dont want to return that to the caller)
|
||||
best_path[-1].pop()
|
||||
best_path[-1].reverse()
|
||||
return best_path
|
||||
@ -0,0 +1,31 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""Bert Init."""
|
||||
from .bert_for_pre_training import BertNetworkWithLoss, BertPreTraining, \
|
||||
BertPretrainingLoss, GetMaskedLMOutput, GetNextSentenceOutput, \
|
||||
BertTrainOneStepCell, BertTrainOneStepWithLossScaleCell
|
||||
from .bert_model import BertAttention, BertConfig, BertEncoderCell, BertModel, \
|
||||
BertOutput, BertSelfAttention, BertTransformer, EmbeddingLookup, \
|
||||
EmbeddingPostprocessor, RelaPosEmbeddingsGenerator, RelaPosMatrixGenerator, \
|
||||
SaturateCast, CreateAttentionMaskFromInputMask
|
||||
|
||||
__all__ = [
|
||||
"BertNetworkWithLoss", "BertPreTraining", "BertPretrainingLoss",
|
||||
"GetMaskedLMOutput", "GetNextSentenceOutput", "BertTrainOneStepCell", "BertTrainOneStepWithLossScaleCell",
|
||||
"BertAttention", "BertConfig", "BertEncoderCell", "BertModel", "BertOutput",
|
||||
"BertSelfAttention", "BertTransformer", "EmbeddingLookup",
|
||||
"EmbeddingPostprocessor", "RelaPosEmbeddingsGenerator",
|
||||
"RelaPosMatrixGenerator", "SaturateCast", "CreateAttentionMaskFromInputMask"
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,73 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
|
||||
'''bert clue evaluation'''
|
||||
|
||||
import json
|
||||
import numpy as np
|
||||
import mindspore.common.dtype as mstype
|
||||
from mindspore.common.tensor import Tensor
|
||||
import tokenization
|
||||
from sample_process import label_generation, process_one_example_p
|
||||
from .evaluation_config import cfg
|
||||
from .CRF import postprocess
|
||||
|
||||
vocab_file = "./vocab.txt"
|
||||
tokenizer_ = tokenization.FullTokenizer(vocab_file=vocab_file)
|
||||
|
||||
def process(model, text, sequence_length):
|
||||
"""
|
||||
process text.
|
||||
"""
|
||||
data = [text]
|
||||
features = []
|
||||
res = []
|
||||
ids = []
|
||||
for i in data:
|
||||
feature = process_one_example_p(tokenizer_, i, max_seq_len=sequence_length)
|
||||
features.append(feature)
|
||||
input_ids, input_mask, token_type_id = feature
|
||||
input_ids = Tensor(np.array(input_ids), mstype.int32)
|
||||
input_mask = Tensor(np.array(input_mask), mstype.int32)
|
||||
token_type_id = Tensor(np.array(token_type_id), mstype.int32)
|
||||
if cfg.use_crf:
|
||||
backpointers, best_tag_id = model.predict(input_ids, input_mask, token_type_id, Tensor(1))
|
||||
best_path = postprocess(backpointers, best_tag_id)
|
||||
logits = []
|
||||
for ele in best_path:
|
||||
logits.extend(ele)
|
||||
ids = logits
|
||||
else:
|
||||
logits = model.predict(input_ids, input_mask, token_type_id, Tensor(1))
|
||||
ids = logits.asnumpy()
|
||||
ids = np.argmax(ids, axis=-1)
|
||||
ids = list(ids)
|
||||
res = label_generation(text, ids)
|
||||
return res
|
||||
|
||||
def submit(model, path, sequence_length):
|
||||
"""
|
||||
submit task
|
||||
"""
|
||||
data = []
|
||||
for line in open(path):
|
||||
if not line.strip():
|
||||
continue
|
||||
oneline = json.loads(line.strip())
|
||||
res = process(model, oneline["text"], sequence_length)
|
||||
print("text", oneline["text"])
|
||||
print("res:", res)
|
||||
data.append(json.dumps({"label": res}, ensure_ascii=False))
|
||||
open("ner_predict.json", "w").write("\n".join(data))
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue