|
|
|
@ -1,13 +1,17 @@
|
|
|
|
import sys
|
|
|
|
import sys
|
|
|
|
|
|
|
|
|
|
|
|
import paddle.v2 as paddle
|
|
|
|
import paddle.v2 as paddle
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def seqToseq_net(source_dict_dim, target_dict_dim):
|
|
|
|
def seqToseq_net(source_dict_dim, target_dict_dim, is_generating=False):
|
|
|
|
### Network Architecture
|
|
|
|
### Network Architecture
|
|
|
|
word_vector_dim = 512 # dimension of word vector
|
|
|
|
word_vector_dim = 512 # dimension of word vector
|
|
|
|
decoder_size = 512 # dimension of hidden unit in GRU Decoder network
|
|
|
|
decoder_size = 512 # dimension of hidden unit in GRU Decoder network
|
|
|
|
encoder_size = 512 # dimension of hidden unit in GRU Encoder network
|
|
|
|
encoder_size = 512 # dimension of hidden unit in GRU Encoder network
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
beam_size = 3
|
|
|
|
|
|
|
|
max_length = 250
|
|
|
|
|
|
|
|
|
|
|
|
#### Encoder
|
|
|
|
#### Encoder
|
|
|
|
src_word_id = paddle.layer.data(
|
|
|
|
src_word_id = paddle.layer.data(
|
|
|
|
name='source_language_word',
|
|
|
|
name='source_language_word',
|
|
|
|
@ -67,79 +71,143 @@ def seqToseq_net(source_dict_dim, target_dict_dim):
|
|
|
|
group_input2 = paddle.layer.StaticInputV2(input=encoded_proj, is_seq=True)
|
|
|
|
group_input2 = paddle.layer.StaticInputV2(input=encoded_proj, is_seq=True)
|
|
|
|
group_inputs = [group_input1, group_input2]
|
|
|
|
group_inputs = [group_input1, group_input2]
|
|
|
|
|
|
|
|
|
|
|
|
trg_embedding = paddle.layer.embedding(
|
|
|
|
if not is_generating:
|
|
|
|
input=paddle.layer.data(
|
|
|
|
trg_embedding = paddle.layer.embedding(
|
|
|
|
name='target_language_word',
|
|
|
|
input=paddle.layer.data(
|
|
|
|
type=paddle.data_type.integer_value_sequence(target_dict_dim)),
|
|
|
|
name='target_language_word',
|
|
|
|
size=word_vector_dim,
|
|
|
|
type=paddle.data_type.integer_value_sequence(target_dict_dim)),
|
|
|
|
param_attr=paddle.attr.ParamAttr(name='_target_language_embedding'))
|
|
|
|
size=word_vector_dim,
|
|
|
|
group_inputs.append(trg_embedding)
|
|
|
|
param_attr=paddle.attr.ParamAttr(name='_target_language_embedding'))
|
|
|
|
|
|
|
|
group_inputs.append(trg_embedding)
|
|
|
|
# For decoder equipped with attention mechanism, in training,
|
|
|
|
|
|
|
|
# target embeding (the groudtruth) is the data input,
|
|
|
|
# For decoder equipped with attention mechanism, in training,
|
|
|
|
# while encoded source sequence is accessed to as an unbounded memory.
|
|
|
|
# target embeding (the groudtruth) is the data input,
|
|
|
|
# Here, the StaticInput defines a read-only memory
|
|
|
|
# while encoded source sequence is accessed to as an unbounded memory.
|
|
|
|
# for the recurrent_group.
|
|
|
|
# Here, the StaticInput defines a read-only memory
|
|
|
|
decoder = paddle.layer.recurrent_group(
|
|
|
|
# for the recurrent_group.
|
|
|
|
name=decoder_group_name,
|
|
|
|
decoder = paddle.layer.recurrent_group(
|
|
|
|
step=gru_decoder_with_attention,
|
|
|
|
name=decoder_group_name,
|
|
|
|
input=group_inputs)
|
|
|
|
step=gru_decoder_with_attention,
|
|
|
|
|
|
|
|
input=group_inputs)
|
|
|
|
lbl = paddle.layer.data(
|
|
|
|
|
|
|
|
name='target_language_next_word',
|
|
|
|
lbl = paddle.layer.data(
|
|
|
|
type=paddle.data_type.integer_value_sequence(target_dict_dim))
|
|
|
|
name='target_language_next_word',
|
|
|
|
cost = paddle.layer.classification_cost(input=decoder, label=lbl)
|
|
|
|
type=paddle.data_type.integer_value_sequence(target_dict_dim))
|
|
|
|
|
|
|
|
cost = paddle.layer.classification_cost(input=decoder, label=lbl)
|
|
|
|
return cost
|
|
|
|
|
|
|
|
|
|
|
|
return cost
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
# In generation, the decoder predicts a next target word based on
|
|
|
|
|
|
|
|
# the encoded source sequence and the last generated target word.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# The encoded source sequence (encoder's output) must be specified by
|
|
|
|
|
|
|
|
# StaticInput, which is a read-only memory.
|
|
|
|
|
|
|
|
# Embedding of the last generated word is automatically gotten by
|
|
|
|
|
|
|
|
# GeneratedInputs, which is initialized by a start mark, such as <s>,
|
|
|
|
|
|
|
|
# and must be included in generation.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
trg_embedding = paddle.layer.GeneratedInputV2(
|
|
|
|
|
|
|
|
size=target_dict_dim,
|
|
|
|
|
|
|
|
embedding_name='_target_language_embedding',
|
|
|
|
|
|
|
|
embedding_size=word_vector_dim)
|
|
|
|
|
|
|
|
group_inputs.append(trg_embedding)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
beam_gen = paddle.layer.beam_search(
|
|
|
|
|
|
|
|
name=decoder_group_name,
|
|
|
|
|
|
|
|
step=gru_decoder_with_attention,
|
|
|
|
|
|
|
|
input=group_inputs,
|
|
|
|
|
|
|
|
bos_id=0,
|
|
|
|
|
|
|
|
eos_id=1,
|
|
|
|
|
|
|
|
beam_size=beam_size,
|
|
|
|
|
|
|
|
max_length=max_length)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return beam_gen
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main():
|
|
|
|
def main():
|
|
|
|
paddle.init(use_gpu=False, trainer_count=1)
|
|
|
|
paddle.init(use_gpu=False, trainer_count=1)
|
|
|
|
|
|
|
|
is_generating = False
|
|
|
|
|
|
|
|
|
|
|
|
# source and target dict dim.
|
|
|
|
# source and target dict dim.
|
|
|
|
dict_size = 30000
|
|
|
|
dict_size = 30000
|
|
|
|
source_dict_dim = target_dict_dim = dict_size
|
|
|
|
source_dict_dim = target_dict_dim = dict_size
|
|
|
|
|
|
|
|
|
|
|
|
# define network topology
|
|
|
|
# train the network
|
|
|
|
cost = seqToseq_net(source_dict_dim, target_dict_dim)
|
|
|
|
if not is_generating:
|
|
|
|
parameters = paddle.parameters.create(cost)
|
|
|
|
cost = seqToseq_net(source_dict_dim, target_dict_dim)
|
|
|
|
|
|
|
|
parameters = paddle.parameters.create(cost)
|
|
|
|
# define optimize method and trainer
|
|
|
|
|
|
|
|
optimizer = paddle.optimizer.Adam(
|
|
|
|
# define optimize method and trainer
|
|
|
|
learning_rate=5e-5,
|
|
|
|
optimizer = paddle.optimizer.Adam(
|
|
|
|
regularization=paddle.optimizer.L2Regularization(rate=1e-3))
|
|
|
|
learning_rate=5e-5,
|
|
|
|
trainer = paddle.trainer.SGD(cost=cost,
|
|
|
|
regularization=paddle.optimizer.L2Regularization(rate=8e-4))
|
|
|
|
parameters=parameters,
|
|
|
|
trainer = paddle.trainer.SGD(cost=cost,
|
|
|
|
update_equation=optimizer)
|
|
|
|
parameters=parameters,
|
|
|
|
|
|
|
|
update_equation=optimizer)
|
|
|
|
# define data reader
|
|
|
|
# define data reader
|
|
|
|
feeding = {
|
|
|
|
wmt14_reader = paddle.batch(
|
|
|
|
'source_language_word': 0,
|
|
|
|
paddle.reader.shuffle(
|
|
|
|
'target_language_word': 1,
|
|
|
|
paddle.dataset.wmt14.train(dict_size), buf_size=8192),
|
|
|
|
'target_language_next_word': 2
|
|
|
|
batch_size=5)
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
# define event_handler callback
|
|
|
|
wmt14_reader = paddle.batch(
|
|
|
|
def event_handler(event):
|
|
|
|
paddle.reader.shuffle(
|
|
|
|
if isinstance(event, paddle.event.EndIteration):
|
|
|
|
paddle.dataset.wmt14.train(dict_size=dict_size), buf_size=8192),
|
|
|
|
if event.batch_id % 10 == 0:
|
|
|
|
batch_size=5)
|
|
|
|
print "\nPass %d, Batch %d, Cost %f, %s" % (
|
|
|
|
|
|
|
|
event.pass_id, event.batch_id, event.cost,
|
|
|
|
# define event_handler callback
|
|
|
|
event.metrics)
|
|
|
|
def event_handler(event):
|
|
|
|
else:
|
|
|
|
if isinstance(event, paddle.event.EndIteration):
|
|
|
|
sys.stdout.write('.')
|
|
|
|
if event.batch_id % 10 == 0:
|
|
|
|
sys.stdout.flush()
|
|
|
|
print "\nPass %d, Batch %d, Cost %f, %s" % (
|
|
|
|
|
|
|
|
event.pass_id, event.batch_id, event.cost, event.metrics)
|
|
|
|
# start to train
|
|
|
|
|
|
|
|
trainer.train(
|
|
|
|
|
|
|
|
reader=wmt14_reader, event_handler=event_handler, num_passes=2)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# generate a english sequence to french
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
# use the first 3 samples for generation
|
|
|
|
|
|
|
|
gen_creator = paddle.dataset.wmt14.gen(dict_size)
|
|
|
|
|
|
|
|
gen_data = []
|
|
|
|
|
|
|
|
gen_num = 3
|
|
|
|
|
|
|
|
for item in gen_creator():
|
|
|
|
|
|
|
|
gen_data.append((item[0], ))
|
|
|
|
|
|
|
|
if len(gen_data) == gen_num:
|
|
|
|
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
beam_gen = seqToseq_net(source_dict_dim, target_dict_dim, is_generating)
|
|
|
|
|
|
|
|
# get the pretrained model, whose bleu = 26.92
|
|
|
|
|
|
|
|
parameters = paddle.dataset.wmt14.model()
|
|
|
|
|
|
|
|
# prob is the prediction probabilities, and id is the prediction word.
|
|
|
|
|
|
|
|
beam_result = paddle.infer(
|
|
|
|
|
|
|
|
output_layer=beam_gen,
|
|
|
|
|
|
|
|
parameters=parameters,
|
|
|
|
|
|
|
|
input=gen_data,
|
|
|
|
|
|
|
|
field=['prob', 'id'])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# get the dictionary
|
|
|
|
|
|
|
|
src_dict, trg_dict = paddle.dataset.wmt14.get_dict(dict_size)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# the delimited element of generated sequences is -1,
|
|
|
|
|
|
|
|
# the first element of each generated sequence is the sequence length
|
|
|
|
|
|
|
|
seq_list = []
|
|
|
|
|
|
|
|
seq = []
|
|
|
|
|
|
|
|
for w in beam_result[1]:
|
|
|
|
|
|
|
|
if w != -1:
|
|
|
|
|
|
|
|
seq.append(w)
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
sys.stdout.write('.')
|
|
|
|
seq_list.append(' '.join([trg_dict.get(w) for w in seq[1:]]))
|
|
|
|
sys.stdout.flush()
|
|
|
|
seq = []
|
|
|
|
|
|
|
|
|
|
|
|
# start to train
|
|
|
|
prob = beam_result[0]
|
|
|
|
trainer.train(
|
|
|
|
beam_size = 3
|
|
|
|
reader=wmt14_reader,
|
|
|
|
for i in xrange(gen_num):
|
|
|
|
event_handler=event_handler,
|
|
|
|
print "\n*******************************************************\n"
|
|
|
|
num_passes=10000,
|
|
|
|
print "src:", ' '.join(
|
|
|
|
feeding=feeding)
|
|
|
|
[src_dict.get(w) for w in gen_data[i][0]]), "\n"
|
|
|
|
|
|
|
|
for j in xrange(beam_size):
|
|
|
|
|
|
|
|
print "prob = %f:" % (prob[i][j]), seq_list[i * beam_size + j]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
typhoonzero
8 years ago