You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
91 lines
3.4 KiB
91 lines
3.4 KiB
import paddle.v2.activation as activation
|
|
import paddle.v2.attr as attr
|
|
import paddle.v2.data_type as data_type
|
|
import paddle.v2.layer as layer
|
|
import paddle.v2.networks as networks
|
|
|
|
|
|
def seqToseq_net_v2(source_dict_dim, target_dict_dim):
|
|
### Network Architecture
|
|
word_vector_dim = 512 # dimension of word vector
|
|
decoder_size = 512 # dimension of hidden unit in GRU Decoder network
|
|
encoder_size = 512 # dimension of hidden unit in GRU Encoder network
|
|
|
|
#### Encoder
|
|
src_word_id = layer.data(
|
|
name='source_language_word',
|
|
type=data_type.dense_vector(source_dict_dim))
|
|
src_embedding = layer.embedding(
|
|
input=src_word_id,
|
|
size=word_vector_dim,
|
|
param_attr=attr.ParamAttr(name='_source_language_embedding'))
|
|
src_forward = networks.simple_gru(input=src_embedding, size=encoder_size)
|
|
src_backward = networks.simple_gru(
|
|
input=src_embedding, size=encoder_size, reverse=True)
|
|
encoded_vector = layer.concat(input=[src_forward, src_backward])
|
|
|
|
#### Decoder
|
|
with layer.mixed(size=decoder_size) as encoded_proj:
|
|
encoded_proj += layer.full_matrix_projection(input=encoded_vector)
|
|
|
|
backward_first = layer.first_seq(input=src_backward)
|
|
|
|
with layer.mixed(size=decoder_size, act=activation.Tanh()) as decoder_boot:
|
|
decoder_boot += layer.full_matrix_projection(input=backward_first)
|
|
|
|
def gru_decoder_with_attention(enc_vec, enc_proj, current_word):
|
|
|
|
decoder_mem = layer.memory(
|
|
name='gru_decoder', size=decoder_size, boot_layer=decoder_boot)
|
|
|
|
context = networks.simple_attention(
|
|
encoded_sequence=enc_vec,
|
|
encoded_proj=enc_proj,
|
|
decoder_state=decoder_mem)
|
|
|
|
with layer.mixed(size=decoder_size * 3) as decoder_inputs:
|
|
decoder_inputs += layer.full_matrix_projection(input=context)
|
|
decoder_inputs += layer.full_matrix_projection(input=current_word)
|
|
|
|
gru_step = layer.gru_step(
|
|
name='gru_decoder',
|
|
input=decoder_inputs,
|
|
output_mem=decoder_mem,
|
|
size=decoder_size)
|
|
|
|
with layer.mixed(
|
|
size=target_dict_dim, bias_attr=True,
|
|
act=activation.Softmax()) as out:
|
|
out += layer.full_matrix_projection(input=gru_step)
|
|
return out
|
|
|
|
decoder_group_name = "decoder_group"
|
|
group_input1 = layer.StaticInputV2(input=encoded_vector, is_seq=True)
|
|
group_input2 = layer.StaticInputV2(input=encoded_proj, is_seq=True)
|
|
group_inputs = [group_input1, group_input2]
|
|
|
|
trg_embedding = layer.embedding(
|
|
input=layer.data(
|
|
name='target_language_word',
|
|
type=data_type.dense_vector(target_dict_dim)),
|
|
size=word_vector_dim,
|
|
param_attr=attr.ParamAttr(name='_target_language_embedding'))
|
|
group_inputs.append(trg_embedding)
|
|
|
|
# For decoder equipped with attention mechanism, in training,
|
|
# target embeding (the groudtruth) is the data input,
|
|
# while encoded source sequence is accessed to as an unbounded memory.
|
|
# Here, the StaticInput defines a read-only memory
|
|
# for the recurrent_group.
|
|
decoder = layer.recurrent_group(
|
|
name=decoder_group_name,
|
|
step=gru_decoder_with_attention,
|
|
input=group_inputs)
|
|
|
|
lbl = layer.data(
|
|
name='target_language_next_word',
|
|
type=data_type.dense_vector(target_dict_dim))
|
|
cost = layer.classification_cost(input=decoder, label=lbl)
|
|
|
|
return cost
|