From 641888f6031f536120063524e2e9be73d8e2d68b Mon Sep 17 00:00:00 2001 From: alouhahaha Date: Wed, 18 Nov 2020 11:33:22 +0800 Subject: [PATCH] add definition of scale for GPT --- model_zoo/official/nlp/gpt/src/gpt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model_zoo/official/nlp/gpt/src/gpt.py b/model_zoo/official/nlp/gpt/src/gpt.py index fb00d35aa6..06280ef654 100644 --- a/model_zoo/official/nlp/gpt/src/gpt.py +++ b/model_zoo/official/nlp/gpt/src/gpt.py @@ -361,6 +361,7 @@ class Block(nn.Cell): """ def __init__(self, config, layer_idx): super(Block, self).__init__() + scale = 1.0 self.layernorm1 = LayerNorm((config.embedding_size,)).to_float(config.compute_dtype) self.attention = Attention(config, scale, layer_idx) self.layernorm2 = LayerNorm((config.embedding_size,)).to_float(config.compute_dtype)