Yao Cheng
7 years ago
commit
8cbb49ce63
@ -0,0 +1,21 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Update to point to the source file.
|
||||||
|
VGG_SRC="vgg16_fluid.py"
|
||||||
|
|
||||||
|
export TRAINING_ROLE=PSERVER
|
||||||
|
export TRAINERS=2
|
||||||
|
export POD_IP=127.0.0.1
|
||||||
|
export PADDLE_INIT_PORT=6174
|
||||||
|
MKL_NUM_THREADS=1 python -u ${VGG_SRC} --local 0 --ps_host=127.0.0.1:6174 --trainer_hosts=127.0.0.1:6174 &
|
||||||
|
|
||||||
|
# Need to wait for the ps to start first.
|
||||||
|
sleep 10
|
||||||
|
echo "done start ps"
|
||||||
|
|
||||||
|
export TRAINING_ROLE=TRAINER
|
||||||
|
export TRAINERS=2
|
||||||
|
export POD_IP=127.0.0.1
|
||||||
|
export PADDLE_INIT_PORT=6174
|
||||||
|
CUDA_VISIBLE_DEVICES=4 MKL_NUM_THREADS=1 python -u ${VGG_SRC} --local 0 --ps_host=127.0.0.1:6174 --trainer_hosts=127.0.0.1:6174 --device=GPU --task_index=0 &
|
||||||
|
CUDA_VISIBLE_DEVICES=5 MKL_NUM_THREADS=1 python -u ${VGG_SRC} --local 0 --ps_host=127.0.0.1:6174 --trainer_hosts=127.0.0.1:6174 --device=GPU --task_index=1 &
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,224 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from __future__ import absolute_import
|
||||||
|
from __future__ import division
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import argparse
|
||||||
|
import time
|
||||||
|
|
||||||
|
import paddle
|
||||||
|
import paddle.fluid as fluid
|
||||||
|
import paddle.fluid.profiler as profiler
|
||||||
|
|
||||||
|
SEED = 1
|
||||||
|
DTYPE = "float32"
|
||||||
|
|
||||||
|
# random seed must set before configuring the network.
|
||||||
|
# fluid.default_startup_program().random_seed = SEED
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser("mnist model benchmark.")
|
||||||
|
parser.add_argument(
|
||||||
|
'--batch_size', type=int, default=128, help='The minibatch size.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--skip_batch_num',
|
||||||
|
type=int,
|
||||||
|
default=5,
|
||||||
|
help='The first num of minibatch num to skip, for better performance test'
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--iterations', type=int, default=35, help='The number of minibatches.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--pass_num', type=int, default=5, help='The number of passes.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--device',
|
||||||
|
type=str,
|
||||||
|
default='GPU',
|
||||||
|
choices=['CPU', 'GPU'],
|
||||||
|
help='The device type.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--infer_only', action='store_true', help='If set, run forward only.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--use_cprof', action='store_true', help='If set, use cProfile.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--use_nvprof',
|
||||||
|
action='store_true',
|
||||||
|
help='If set, use nvprof for CUDA.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--with_test',
|
||||||
|
action='store_true',
|
||||||
|
help='If set, test the testset during training.')
|
||||||
|
args = parser.parse_args()
|
||||||
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
def cnn_model(data):
|
||||||
|
conv_pool_1 = fluid.nets.simple_img_conv_pool(
|
||||||
|
input=data,
|
||||||
|
filter_size=5,
|
||||||
|
num_filters=20,
|
||||||
|
pool_size=2,
|
||||||
|
pool_stride=2,
|
||||||
|
act="relu")
|
||||||
|
conv_pool_2 = fluid.nets.simple_img_conv_pool(
|
||||||
|
input=conv_pool_1,
|
||||||
|
filter_size=5,
|
||||||
|
num_filters=50,
|
||||||
|
pool_size=2,
|
||||||
|
pool_stride=2,
|
||||||
|
act="relu")
|
||||||
|
|
||||||
|
# TODO(dzhwinter) : refine the initializer and random seed settting
|
||||||
|
SIZE = 10
|
||||||
|
input_shape = conv_pool_2.shape
|
||||||
|
param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE]
|
||||||
|
scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5
|
||||||
|
|
||||||
|
predict = fluid.layers.fc(
|
||||||
|
input=conv_pool_2,
|
||||||
|
size=SIZE,
|
||||||
|
act="softmax",
|
||||||
|
param_attr=fluid.param_attr.ParamAttr(
|
||||||
|
initializer=fluid.initializer.NormalInitializer(
|
||||||
|
loc=0.0, scale=scale)))
|
||||||
|
return predict
|
||||||
|
|
||||||
|
|
||||||
|
def eval_test(exe, batch_acc, batch_size_tensor, inference_program):
|
||||||
|
test_reader = paddle.batch(
|
||||||
|
paddle.dataset.mnist.test(), batch_size=args.batch_size)
|
||||||
|
test_pass_acc = fluid.average.WeightedAverage()
|
||||||
|
for batch_id, data in enumerate(test_reader()):
|
||||||
|
img_data = np.array(map(lambda x: x[0].reshape([1, 28, 28]),
|
||||||
|
data)).astype(DTYPE)
|
||||||
|
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
|
||||||
|
y_data = y_data.reshape([len(y_data), 1])
|
||||||
|
|
||||||
|
acc, weight = exe.run(inference_program,
|
||||||
|
feed={"pixel": img_data,
|
||||||
|
"label": y_data},
|
||||||
|
fetch_list=[batch_acc, batch_size_tensor])
|
||||||
|
test_pass_acc.add(value=acc, weight=weight)
|
||||||
|
pass_acc = test_pass_acc.eval()
|
||||||
|
return pass_acc
|
||||||
|
|
||||||
|
|
||||||
|
def run_benchmark(model, args):
|
||||||
|
if args.use_cprof:
|
||||||
|
pr = cProfile.Profile()
|
||||||
|
pr.enable()
|
||||||
|
start_time = time.time()
|
||||||
|
# Input data
|
||||||
|
images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE)
|
||||||
|
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
|
||||||
|
|
||||||
|
# Train program
|
||||||
|
predict = model(images)
|
||||||
|
cost = fluid.layers.cross_entropy(input=predict, label=label)
|
||||||
|
avg_cost = fluid.layers.mean(x=cost)
|
||||||
|
|
||||||
|
# Evaluator
|
||||||
|
batch_size_tensor = fluid.layers.create_tensor(dtype='int64')
|
||||||
|
batch_acc = fluid.layers.accuracy(
|
||||||
|
input=predict, label=label, total=batch_size_tensor)
|
||||||
|
|
||||||
|
# inference program
|
||||||
|
inference_program = fluid.default_main_program().clone()
|
||||||
|
|
||||||
|
# Optimization
|
||||||
|
opt = fluid.optimizer.AdamOptimizer(
|
||||||
|
learning_rate=0.001, beta1=0.9, beta2=0.999)
|
||||||
|
opt.minimize(avg_cost)
|
||||||
|
|
||||||
|
fluid.memory_optimize(fluid.default_main_program())
|
||||||
|
|
||||||
|
# Initialize executor
|
||||||
|
place = fluid.CPUPlace() if args.device == 'CPU' else fluid.CUDAPlace(0)
|
||||||
|
exe = fluid.Executor(place)
|
||||||
|
|
||||||
|
# Parameter initialization
|
||||||
|
exe.run(fluid.default_startup_program())
|
||||||
|
|
||||||
|
# Reader
|
||||||
|
train_reader = paddle.batch(
|
||||||
|
paddle.dataset.mnist.train(), batch_size=args.batch_size)
|
||||||
|
|
||||||
|
accuracy = fluid.metrics.Accuracy()
|
||||||
|
iters, num_samples, start_time = 0, 0, time.time()
|
||||||
|
for pass_id in range(args.pass_num):
|
||||||
|
accuracy.reset()
|
||||||
|
train_accs = []
|
||||||
|
train_losses = []
|
||||||
|
for batch_id, data in enumerate(train_reader()):
|
||||||
|
if iters == args.skip_batch_num:
|
||||||
|
start_time = time.time()
|
||||||
|
num_samples = 0
|
||||||
|
if iters == args.iterations:
|
||||||
|
break
|
||||||
|
img_data = np.array(
|
||||||
|
map(lambda x: x[0].reshape([1, 28, 28]), data)).astype(DTYPE)
|
||||||
|
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
|
||||||
|
y_data = y_data.reshape([len(y_data), 1])
|
||||||
|
|
||||||
|
outs = exe.run(
|
||||||
|
fluid.default_main_program(),
|
||||||
|
feed={"pixel": img_data,
|
||||||
|
"label": y_data},
|
||||||
|
fetch_list=[avg_cost, batch_acc, batch_size_tensor]
|
||||||
|
) # The accuracy is the accumulation of batches, but not the current batch.
|
||||||
|
accuracy.update(value=outs[1], weight=outs[2])
|
||||||
|
iters += 1
|
||||||
|
num_samples += len(y_data)
|
||||||
|
loss = np.array(outs[0])
|
||||||
|
acc = np.array(outs[1])
|
||||||
|
train_losses.append(loss)
|
||||||
|
train_accs.append(acc)
|
||||||
|
print("Pass: %d, Iter: %d, Loss: %f, Accuracy: %f" %
|
||||||
|
(pass_id, iters, loss, acc))
|
||||||
|
|
||||||
|
print("Pass: %d, Loss: %f, Train Accuray: %f\n" %
|
||||||
|
(pass_id, np.mean(train_losses), np.mean(train_accs)))
|
||||||
|
train_elapsed = time.time() - start_time
|
||||||
|
examples_per_sec = num_samples / train_elapsed
|
||||||
|
|
||||||
|
print('\nTotal examples: %d, total time: %.5f, %.5f examples/sed\n' %
|
||||||
|
(num_samples, train_elapsed, examples_per_sec))
|
||||||
|
# evaluation
|
||||||
|
if args.with_test:
|
||||||
|
test_avg_acc = eval_test(exe, batch_acc, batch_size_tensor,
|
||||||
|
inference_program)
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def print_arguments(args):
|
||||||
|
vars(args)['use_nvprof'] = (vars(args)['use_nvprof'] and
|
||||||
|
vars(args)['device'] == 'GPU')
|
||||||
|
print('----------- mnist Configuration Arguments -----------')
|
||||||
|
for arg, value in sorted(vars(args).iteritems()):
|
||||||
|
print('%s: %s' % (arg, value))
|
||||||
|
print('------------------------------------------------')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
args = parse_args()
|
||||||
|
print_arguments(args)
|
||||||
|
if args.use_nvprof and args.device == 'GPU':
|
||||||
|
with profiler.cuda_profiler("cuda_profiler.txt", 'csv') as nvprof:
|
||||||
|
run_benchmark(cnn_model, args)
|
||||||
|
else:
|
||||||
|
run_benchmark(cnn_model, args)
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,105 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# This script benchmarking the PaddlePaddle Fluid on
|
||||||
|
# single thread single GPU.
|
||||||
|
|
||||||
|
#export FLAGS_fraction_of_gpu_memory_to_use=0.0
|
||||||
|
export CUDNN_PATH=/paddle/cudnn_v5
|
||||||
|
|
||||||
|
# disable openmp and mkl parallel
|
||||||
|
#https://github.com/PaddlePaddle/Paddle/issues/7199
|
||||||
|
export MKL_NUM_THREADS=1
|
||||||
|
export OMP_NUM_THREADS=1
|
||||||
|
ht=`lscpu |grep "per core"|awk -F':' '{print $2}'|xargs`
|
||||||
|
if [ $ht -eq 1 ]; then # HT is OFF
|
||||||
|
if [ -z "$KMP_AFFINITY" ]; then
|
||||||
|
export KMP_AFFINITY="granularity=fine,compact,0,0"
|
||||||
|
fi
|
||||||
|
if [ -z "$OMP_DYNAMIC" ]; then
|
||||||
|
export OMP_DYNAMIC="FALSE"
|
||||||
|
fi
|
||||||
|
else # HT is ON
|
||||||
|
if [ -z "$KMP_AFFINITY" ]; then
|
||||||
|
export KMP_AFFINITY="granularity=fine,compact,1,0"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
# disable multi-gpu if have more than one
|
||||||
|
export CUDA_VISIBLE_DEVICES=0
|
||||||
|
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
|
||||||
|
export LD_LIBRARY_PATH=$CUDNN_PATH:$LD_LIBRARY_PATH
|
||||||
|
|
||||||
|
# only query the gpu used
|
||||||
|
nohup stdbuf -oL nvidia-smi \
|
||||||
|
--id=${CUDA_VISIBLE_DEVICES} \
|
||||||
|
--query-gpu=timestamp \
|
||||||
|
--query-compute-apps=pid,process_name,used_memory \
|
||||||
|
--format=csv \
|
||||||
|
--filename=mem.log \
|
||||||
|
-l 1 &
|
||||||
|
# mnist
|
||||||
|
# mnist gpu mnist 128
|
||||||
|
FLAGS_benchmark=true stdbuf -oL python fluid/mnist.py \
|
||||||
|
--device=GPU \
|
||||||
|
--batch_size=128 \
|
||||||
|
--skip_batch_num=5 \
|
||||||
|
--iterations=500 \
|
||||||
|
2>&1 | tee -a mnist_gpu_128.log
|
||||||
|
|
||||||
|
# vgg16
|
||||||
|
# gpu cifar10 128
|
||||||
|
FLAGS_benchmark=true stdbuf -oL python fluid/vgg16.py \
|
||||||
|
--device=GPU \
|
||||||
|
--batch_size=128 \
|
||||||
|
--skip_batch_num=5 \
|
||||||
|
--iterations=30 \
|
||||||
|
2>&1 | tee -a vgg16_gpu_128.log
|
||||||
|
|
||||||
|
# flowers gpu 128
|
||||||
|
FLAGS_benchmark=true stdbuf -oL python fluid/vgg16.py \
|
||||||
|
--device=GPU \
|
||||||
|
--batch_size=32 \
|
||||||
|
--data_set=flowers \
|
||||||
|
--skip_batch_num=5 \
|
||||||
|
--iterations=30 \
|
||||||
|
2>&1 | tee -a vgg16_gpu_flowers_32.log
|
||||||
|
|
||||||
|
# resnet50
|
||||||
|
# resnet50 gpu cifar10 128
|
||||||
|
FLAGS_benchmark=true stdbuf -oL python fluid/resnet50.py \
|
||||||
|
--device=GPU \
|
||||||
|
--batch_size=128 \
|
||||||
|
--data_set=cifar10 \
|
||||||
|
--model=resnet_cifar10 \
|
||||||
|
--skip_batch_num=5 \
|
||||||
|
--iterations=30 \
|
||||||
|
2>&1 | tee -a resnet50_gpu_128.log
|
||||||
|
|
||||||
|
# resnet50 gpu flowers 64
|
||||||
|
FLAGS_benchmark=true stdbuf -oL python fluid/resnet50.py \
|
||||||
|
--device=GPU \
|
||||||
|
--batch_size=64 \
|
||||||
|
--data_set=flowers \
|
||||||
|
--model=resnet_imagenet \
|
||||||
|
--skip_batch_num=5 \
|
||||||
|
--iterations=30 \
|
||||||
|
2>&1 | tee -a resnet50_gpu_flowers_64.log
|
||||||
|
|
||||||
|
# lstm
|
||||||
|
# lstm gpu imdb 32 # tensorflow only support batch=32
|
||||||
|
FLAGS_benchmark=true stdbuf -oL python fluid/stacked_dynamic_lstm.py \
|
||||||
|
--device=GPU \
|
||||||
|
--batch_size=32 \
|
||||||
|
--skip_batch_num=5 \
|
||||||
|
--iterations=30 \
|
||||||
|
--hidden_dim=512 \
|
||||||
|
--emb_dim=512 \
|
||||||
|
--crop_size=1500 \
|
||||||
|
2>&1 | tee -a lstm_gpu_32.log
|
||||||
|
|
||||||
|
# seq2seq
|
||||||
|
# seq2seq gpu wmb 128
|
||||||
|
FLAGS_benchmark=true stdbuf -oL python fluid/machine_translation.py \
|
||||||
|
--device=GPU \
|
||||||
|
--batch_size=128 \
|
||||||
|
--skip_batch_num=5 \
|
||||||
|
--iterations=30 \
|
||||||
|
2>&1 | tee -a lstm_gpu_128.log
|
||||||
@ -0,0 +1,236 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from __future__ import absolute_import
|
||||||
|
from __future__ import division
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import cPickle
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import time
|
||||||
|
|
||||||
|
import numpy
|
||||||
|
import paddle
|
||||||
|
import paddle.dataset.imdb as imdb
|
||||||
|
import paddle.fluid as fluid
|
||||||
|
import paddle.batch as batch
|
||||||
|
import paddle.fluid.profiler as profiler
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser("Understand Sentiment by Dynamic RNN.")
|
||||||
|
parser.add_argument(
|
||||||
|
'--batch_size',
|
||||||
|
type=int,
|
||||||
|
default=32,
|
||||||
|
help='The sequence number of a batch data. (default: %(default)d)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--skip_batch_num',
|
||||||
|
type=int,
|
||||||
|
default=5,
|
||||||
|
help='The first num of minibatch num to skip, for better performance test'
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--iterations', type=int, default=80, help='The number of minibatches.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--emb_dim',
|
||||||
|
type=int,
|
||||||
|
default=512,
|
||||||
|
help='Dimension of embedding table. (default: %(default)d)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--hidden_dim',
|
||||||
|
type=int,
|
||||||
|
default=512,
|
||||||
|
help='Hidden size of lstm unit. (default: %(default)d)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--pass_num',
|
||||||
|
type=int,
|
||||||
|
default=100,
|
||||||
|
help='Epoch number to train. (default: %(default)d)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--device',
|
||||||
|
type=str,
|
||||||
|
default='CPU',
|
||||||
|
choices=['CPU', 'GPU'],
|
||||||
|
help='The device type.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--crop_size',
|
||||||
|
type=int,
|
||||||
|
default=int(os.environ.get('CROP_SIZE', '1500')),
|
||||||
|
help='The max sentence length of input. Since this model use plain RNN,'
|
||||||
|
' Gradient could be explored if sentence is too long')
|
||||||
|
parser.add_argument(
|
||||||
|
'--with_test',
|
||||||
|
action='store_true',
|
||||||
|
help='If set, test the testset during training.')
|
||||||
|
args = parser.parse_args()
|
||||||
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
word_dict = imdb.word_dict()
|
||||||
|
|
||||||
|
|
||||||
|
def crop_sentence(reader, crop_size):
|
||||||
|
unk_value = word_dict['<unk>']
|
||||||
|
|
||||||
|
def __impl__():
|
||||||
|
for item in reader():
|
||||||
|
if len([x for x in item[0] if x != unk_value]) < crop_size:
|
||||||
|
yield item
|
||||||
|
|
||||||
|
return __impl__
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = parse_args()
|
||||||
|
lstm_size = args.hidden_dim
|
||||||
|
|
||||||
|
data = fluid.layers.data(
|
||||||
|
name="words", shape=[1], lod_level=1, dtype='int64')
|
||||||
|
sentence = fluid.layers.embedding(
|
||||||
|
input=data, size=[len(word_dict), args.emb_dim])
|
||||||
|
|
||||||
|
sentence = fluid.layers.fc(input=sentence, size=lstm_size, act='tanh')
|
||||||
|
|
||||||
|
rnn = fluid.layers.DynamicRNN()
|
||||||
|
with rnn.block():
|
||||||
|
word = rnn.step_input(sentence)
|
||||||
|
prev_hidden = rnn.memory(value=0.0, shape=[lstm_size])
|
||||||
|
prev_cell = rnn.memory(value=0.0, shape=[lstm_size])
|
||||||
|
|
||||||
|
def gate_common(
|
||||||
|
ipt,
|
||||||
|
hidden,
|
||||||
|
size, ):
|
||||||
|
gate0 = fluid.layers.fc(input=ipt, size=size, bias_attr=True)
|
||||||
|
gate1 = fluid.layers.fc(input=hidden, size=size, bias_attr=False)
|
||||||
|
gate = fluid.layers.sums(input=[gate0, gate1])
|
||||||
|
return gate
|
||||||
|
|
||||||
|
forget_gate = fluid.layers.sigmoid(
|
||||||
|
x=gate_common(word, prev_hidden, lstm_size))
|
||||||
|
input_gate = fluid.layers.sigmoid(
|
||||||
|
x=gate_common(word, prev_hidden, lstm_size))
|
||||||
|
output_gate = fluid.layers.sigmoid(
|
||||||
|
x=gate_common(word, prev_hidden, lstm_size))
|
||||||
|
cell_gate = fluid.layers.tanh(
|
||||||
|
x=gate_common(word, prev_hidden, lstm_size))
|
||||||
|
|
||||||
|
cell = fluid.layers.sums(input=[
|
||||||
|
fluid.layers.elementwise_mul(
|
||||||
|
x=forget_gate, y=prev_cell), fluid.layers.elementwise_mul(
|
||||||
|
x=input_gate, y=cell_gate)
|
||||||
|
])
|
||||||
|
|
||||||
|
hidden = fluid.layers.elementwise_mul(
|
||||||
|
x=output_gate, y=fluid.layers.tanh(x=cell))
|
||||||
|
|
||||||
|
rnn.update_memory(prev_cell, cell)
|
||||||
|
rnn.update_memory(prev_hidden, hidden)
|
||||||
|
rnn.output(hidden)
|
||||||
|
|
||||||
|
last = fluid.layers.sequence_pool(rnn(), 'last')
|
||||||
|
logit = fluid.layers.fc(input=last, size=2, act='softmax')
|
||||||
|
loss = fluid.layers.cross_entropy(
|
||||||
|
input=logit,
|
||||||
|
label=fluid.layers.data(
|
||||||
|
name='label', shape=[1], dtype='int64'))
|
||||||
|
loss = fluid.layers.mean(x=loss)
|
||||||
|
|
||||||
|
# add acc
|
||||||
|
batch_size_tensor = fluid.layers.create_tensor(dtype='int64')
|
||||||
|
batch_acc = fluid.layers.accuracy(input=logit, label=fluid.layers.data(name='label', \
|
||||||
|
shape=[1], dtype='int64'), total=batch_size_tensor)
|
||||||
|
|
||||||
|
inference_program = fluid.default_main_program().clone()
|
||||||
|
with fluid.program_guard(inference_program):
|
||||||
|
inference_program = fluid.io.get_inference_program(
|
||||||
|
target_vars=[batch_acc, batch_size_tensor])
|
||||||
|
|
||||||
|
adam = fluid.optimizer.Adam()
|
||||||
|
adam.minimize(loss)
|
||||||
|
|
||||||
|
fluid.memory_optimize(fluid.default_main_program())
|
||||||
|
|
||||||
|
place = fluid.CPUPlace() if args.device == 'CPU' else fluid.CUDAPlace(0)
|
||||||
|
exe = fluid.Executor(place)
|
||||||
|
exe.run(fluid.default_startup_program())
|
||||||
|
|
||||||
|
train_reader = batch(
|
||||||
|
paddle.reader.shuffle(
|
||||||
|
crop_sentence(imdb.train(word_dict), args.crop_size),
|
||||||
|
buf_size=25000),
|
||||||
|
batch_size=args.batch_size)
|
||||||
|
|
||||||
|
iters, num_samples, start_time = 0, 0, time.time()
|
||||||
|
for pass_id in range(args.pass_num):
|
||||||
|
train_accs = []
|
||||||
|
train_losses = []
|
||||||
|
for batch_id, data in enumerate(train_reader()):
|
||||||
|
if iters == args.skip_batch_num:
|
||||||
|
start_time = time.time()
|
||||||
|
num_samples = 0
|
||||||
|
if iters == args.iterations:
|
||||||
|
break
|
||||||
|
tensor_words = to_lodtensor([x[0] for x in data], place)
|
||||||
|
label = numpy.array([x[1] for x in data]).astype("int64")
|
||||||
|
label = label.reshape((-1, 1))
|
||||||
|
loss_np, acc, weight = exe.run(
|
||||||
|
fluid.default_main_program(),
|
||||||
|
feed={"words": tensor_words,
|
||||||
|
"label": label},
|
||||||
|
fetch_list=[loss, batch_acc, batch_size_tensor])
|
||||||
|
iters += 1
|
||||||
|
for x in data:
|
||||||
|
num_samples += len(x[0])
|
||||||
|
print(
|
||||||
|
"Pass = %d, Iter = %d, Loss = %f, Accuracy = %f" %
|
||||||
|
(pass_id, iters, loss_np, acc)
|
||||||
|
) # The accuracy is the accumulation of batches, but not the current batch.
|
||||||
|
|
||||||
|
train_elapsed = time.time() - start_time
|
||||||
|
examples_per_sec = num_samples / train_elapsed
|
||||||
|
print('\nTotal examples: %d, total time: %.5f, %.5f examples/sed\n' %
|
||||||
|
(num_samples, train_elapsed, examples_per_sec))
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def to_lodtensor(data, place):
|
||||||
|
seq_lens = [len(seq) for seq in data]
|
||||||
|
cur_len = 0
|
||||||
|
lod = [cur_len]
|
||||||
|
for l in seq_lens:
|
||||||
|
cur_len += l
|
||||||
|
lod.append(cur_len)
|
||||||
|
flattened_data = numpy.concatenate(data, axis=0).astype("int64")
|
||||||
|
flattened_data = flattened_data.reshape([len(flattened_data), 1])
|
||||||
|
res = fluid.LoDTensor()
|
||||||
|
res.set(flattened_data, place)
|
||||||
|
res.set_lod([lod])
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
def print_arguments(args):
|
||||||
|
print('----------- lstm Configuration Arguments -----------')
|
||||||
|
for arg, value in sorted(vars(args).iteritems()):
|
||||||
|
print('%s: %s' % (arg, value))
|
||||||
|
print('------------------------------------------------')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
args = parse_args()
|
||||||
|
print_arguments(args)
|
||||||
|
main()
|
||||||
@ -0,0 +1,224 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
"""VGG16 benchmark in Fluid"""
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import numpy as np
|
||||||
|
import paddle
|
||||||
|
import paddle.fluid as fluid
|
||||||
|
import paddle.fluid.core as core
|
||||||
|
import argparse
|
||||||
|
import functools
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument(
|
||||||
|
'--batch_size', type=int, default=128, help="Batch size for training.")
|
||||||
|
parser.add_argument(
|
||||||
|
'--skip_batch_num',
|
||||||
|
type=int,
|
||||||
|
default=5,
|
||||||
|
help='The first num of minibatch num to skip, for better performance test')
|
||||||
|
parser.add_argument(
|
||||||
|
'--iterations', type=int, default=80, help='The number of minibatches.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--learning_rate',
|
||||||
|
type=float,
|
||||||
|
default=1e-3,
|
||||||
|
help="Learning rate for training.")
|
||||||
|
parser.add_argument('--pass_num', type=int, default=50, help="No. of passes.")
|
||||||
|
parser.add_argument(
|
||||||
|
'--device',
|
||||||
|
type=str,
|
||||||
|
default='GPU',
|
||||||
|
choices=['CPU', 'GPU'],
|
||||||
|
help="The device type.")
|
||||||
|
parser.add_argument(
|
||||||
|
'--data_format',
|
||||||
|
type=str,
|
||||||
|
default='NCHW',
|
||||||
|
choices=['NCHW', 'NHWC'],
|
||||||
|
help='The data order, now only support NCHW.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--data_set',
|
||||||
|
type=str,
|
||||||
|
default='cifar10',
|
||||||
|
choices=['cifar10', 'flowers'],
|
||||||
|
help='Optional dataset for benchmark.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--with_test',
|
||||||
|
action='store_true',
|
||||||
|
help='If set, test the testset during training.')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def vgg16_bn_drop(input):
|
||||||
|
def conv_block(input, num_filter, groups, dropouts):
|
||||||
|
return fluid.nets.img_conv_group(
|
||||||
|
input=input,
|
||||||
|
pool_size=2,
|
||||||
|
pool_stride=2,
|
||||||
|
conv_num_filter=[num_filter] * groups,
|
||||||
|
conv_filter_size=3,
|
||||||
|
conv_act='relu',
|
||||||
|
conv_with_batchnorm=True,
|
||||||
|
conv_batchnorm_drop_rate=dropouts,
|
||||||
|
pool_type='max')
|
||||||
|
|
||||||
|
conv1 = conv_block(input, 64, 2, [0.3, 0])
|
||||||
|
conv2 = conv_block(conv1, 128, 2, [0.4, 0])
|
||||||
|
conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0])
|
||||||
|
conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0])
|
||||||
|
conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0])
|
||||||
|
|
||||||
|
drop = fluid.layers.dropout(x=conv5, dropout_prob=0.5)
|
||||||
|
fc1 = fluid.layers.fc(input=drop, size=512, act=None)
|
||||||
|
bn = fluid.layers.batch_norm(input=fc1, act='relu')
|
||||||
|
drop2 = fluid.layers.dropout(x=bn, dropout_prob=0.5)
|
||||||
|
fc2 = fluid.layers.fc(input=drop2, size=512, act=None)
|
||||||
|
return fc2
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if args.data_set == "cifar10":
|
||||||
|
classdim = 10
|
||||||
|
if args.data_format == 'NCHW':
|
||||||
|
data_shape = [3, 32, 32]
|
||||||
|
else:
|
||||||
|
data_shape = [32, 32, 3]
|
||||||
|
else:
|
||||||
|
classdim = 102
|
||||||
|
if args.data_format == 'NCHW':
|
||||||
|
data_shape = [3, 224, 224]
|
||||||
|
else:
|
||||||
|
data_shape = [224, 224, 3]
|
||||||
|
|
||||||
|
# Input data
|
||||||
|
images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32')
|
||||||
|
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
|
||||||
|
|
||||||
|
# Train program
|
||||||
|
net = vgg16_bn_drop(images)
|
||||||
|
predict = fluid.layers.fc(input=net, size=classdim, act='softmax')
|
||||||
|
cost = fluid.layers.cross_entropy(input=predict, label=label)
|
||||||
|
avg_cost = fluid.layers.mean(x=cost)
|
||||||
|
|
||||||
|
# Evaluator
|
||||||
|
batch_size_tensor = fluid.layers.create_tensor(dtype='int64')
|
||||||
|
batch_acc = fluid.layers.accuracy(
|
||||||
|
input=predict, label=label, total=batch_size_tensor)
|
||||||
|
|
||||||
|
# inference program
|
||||||
|
inference_program = fluid.default_main_program().clone()
|
||||||
|
with fluid.program_guard(inference_program):
|
||||||
|
inference_program = fluid.io.get_inference_program(
|
||||||
|
target_vars=[batch_acc, batch_size_tensor])
|
||||||
|
|
||||||
|
# Optimization
|
||||||
|
optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate)
|
||||||
|
opts = optimizer.minimize(avg_cost)
|
||||||
|
|
||||||
|
fluid.memory_optimize(fluid.default_main_program())
|
||||||
|
|
||||||
|
# Initialize executor
|
||||||
|
place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(0)
|
||||||
|
exe = fluid.Executor(place)
|
||||||
|
|
||||||
|
# Parameter initialization
|
||||||
|
exe.run(fluid.default_startup_program())
|
||||||
|
|
||||||
|
# data reader
|
||||||
|
train_reader = paddle.batch(
|
||||||
|
paddle.reader.shuffle(
|
||||||
|
paddle.dataset.cifar.train10()
|
||||||
|
if args.data_set == 'cifar10' else paddle.dataset.flowers.train(),
|
||||||
|
buf_size=5120),
|
||||||
|
batch_size=args.batch_size)
|
||||||
|
test_reader = paddle.batch(
|
||||||
|
paddle.dataset.cifar.test10()
|
||||||
|
if args.data_set == 'cifar10' else paddle.dataset.flowers.test(),
|
||||||
|
batch_size=args.batch_size)
|
||||||
|
|
||||||
|
# test
|
||||||
|
def test(exe):
|
||||||
|
test_accuracy = fluid.average.WeightedAverage()
|
||||||
|
for batch_id, data in enumerate(test_reader()):
|
||||||
|
img_data = np.array(map(lambda x: x[0].reshape(data_shape),
|
||||||
|
data)).astype("float32")
|
||||||
|
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
|
||||||
|
y_data = y_data.reshape([-1, 1])
|
||||||
|
|
||||||
|
acc, weight = exe.run(inference_program,
|
||||||
|
feed={"pixel": img_data,
|
||||||
|
"label": y_data},
|
||||||
|
fetch_list=[batch_acc, batch_size_tensor])
|
||||||
|
test_accuracy.add(value=acc, weight=weight)
|
||||||
|
return test_accuracy.eval()
|
||||||
|
|
||||||
|
iters, num_samples, start_time = 0, 0, time.time()
|
||||||
|
accuracy = fluid.average.WeightedAverage()
|
||||||
|
for pass_id in range(args.pass_num):
|
||||||
|
accuracy.reset()
|
||||||
|
train_accs = []
|
||||||
|
train_losses = []
|
||||||
|
for batch_id, data in enumerate(train_reader()):
|
||||||
|
if iters == args.skip_batch_num:
|
||||||
|
start_time = time.time()
|
||||||
|
num_samples = 0
|
||||||
|
if iters == args.iterations:
|
||||||
|
break
|
||||||
|
img_data = np.array(map(lambda x: x[0].reshape(data_shape),
|
||||||
|
data)).astype("float32")
|
||||||
|
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
|
||||||
|
y_data = y_data.reshape([-1, 1])
|
||||||
|
|
||||||
|
loss, acc, weight = exe.run(
|
||||||
|
fluid.default_main_program(),
|
||||||
|
feed={"pixel": img_data,
|
||||||
|
"label": y_data},
|
||||||
|
fetch_list=[avg_cost, batch_acc, batch_size_tensor])
|
||||||
|
accuracy.add(value=acc, weight=weight)
|
||||||
|
iters += 1
|
||||||
|
num_samples += len(y_data)
|
||||||
|
print(
|
||||||
|
"Pass = %d, Iter = %d, Loss = %f, Accuracy = %f" %
|
||||||
|
(pass_id, iters, loss, acc)
|
||||||
|
) # The accuracy is the accumulation of batches, but not the current batch.
|
||||||
|
|
||||||
|
# pass_train_acc = accuracy.eval()
|
||||||
|
train_losses.append(loss)
|
||||||
|
train_accs.append(acc)
|
||||||
|
print("Pass: %d, Loss: %f, Train Accuray: %f\n" %
|
||||||
|
(pass_id, np.mean(train_losses), np.mean(train_accs)))
|
||||||
|
train_elapsed = time.time() - start_time
|
||||||
|
examples_per_sec = num_samples / train_elapsed
|
||||||
|
print('\nTotal examples: %d, total time: %.5f, %.5f examples/sed\n' %
|
||||||
|
(num_samples, train_elapsed, examples_per_sec))
|
||||||
|
# evaluation
|
||||||
|
if args.with_test:
|
||||||
|
pass_test_acc = test(exe)
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def print_arguments():
|
||||||
|
print('----------- vgg Configuration Arguments -----------')
|
||||||
|
for arg, value in sorted(vars(args).iteritems()):
|
||||||
|
print('%s: %s' % (arg, value))
|
||||||
|
print('------------------------------------------------')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
print_arguments()
|
||||||
|
main()
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,180 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from __future__ import absolute_import
|
||||||
|
from __future__ import division
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import time
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
import tensorflow as tf
|
||||||
|
import paddle.v2 as paddle
|
||||||
|
|
||||||
|
DTYPE = tf.float32
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser("mnist model benchmark.")
|
||||||
|
parser.add_argument(
|
||||||
|
'--batch_size', type=int, default=128, help='The minibatch size.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--iterations', type=int, default=35, help='The number of minibatches.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--pass_num', type=int, default=5, help='The number of passes.')
|
||||||
|
parser.add_argument(
|
||||||
|
'--device',
|
||||||
|
type=str,
|
||||||
|
default='GPU',
|
||||||
|
choices=['CPU', 'GPU'],
|
||||||
|
help='The device type.')
|
||||||
|
args = parser.parse_args()
|
||||||
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
def run_benchmark(args):
|
||||||
|
def weight_variable(dtype, shape):
|
||||||
|
initial = tf.truncated_normal(shape, stddev=0.1, dtype=dtype)
|
||||||
|
return tf.Variable(initial)
|
||||||
|
|
||||||
|
def bias_variable(dtype, shape):
|
||||||
|
initial = tf.constant(0.1, shape=shape, dtype=dtype)
|
||||||
|
return tf.Variable(initial)
|
||||||
|
|
||||||
|
device = '/cpu:0' if args.device == 'CPU' else '/device:GPU:0'
|
||||||
|
with tf.device(device):
|
||||||
|
images = tf.placeholder(DTYPE, shape=(None, 28, 28, 1))
|
||||||
|
labels = tf.placeholder(tf.int64, shape=(None, ))
|
||||||
|
|
||||||
|
# conv1, relu, pool1
|
||||||
|
conv1_weights = weight_variable(DTYPE, [5, 5, 1, 20])
|
||||||
|
conv1_bias = bias_variable(DTYPE, [20])
|
||||||
|
conv1 = tf.nn.conv2d(
|
||||||
|
images, conv1_weights, strides=[1, 1, 1, 1], padding="VALID")
|
||||||
|
relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_bias))
|
||||||
|
pool1 = tf.nn.max_pool(
|
||||||
|
relu1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")
|
||||||
|
|
||||||
|
# conv2, relu, pool2
|
||||||
|
conv2_weights = weight_variable(DTYPE, [5, 5, 20, 50])
|
||||||
|
conv2_bias = bias_variable(DTYPE, [50])
|
||||||
|
conv2 = tf.nn.conv2d(
|
||||||
|
pool1, conv2_weights, strides=[1, 1, 1, 1], padding="VALID")
|
||||||
|
relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_bias))
|
||||||
|
pool2 = tf.nn.max_pool(
|
||||||
|
relu2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID")
|
||||||
|
|
||||||
|
# FC
|
||||||
|
pool_shape = pool2.get_shape().as_list()
|
||||||
|
hidden_dim = reduce(lambda a, b: a * b, pool_shape[1:], 1)
|
||||||
|
reshape = tf.reshape(pool2, shape=(tf.shape(pool2)[0], hidden_dim))
|
||||||
|
fc_weights = weight_variable(DTYPE, [hidden_dim, 10])
|
||||||
|
fc_bias = bias_variable(DTYPE, [10])
|
||||||
|
logits = tf.matmul(reshape, fc_weights) + fc_bias
|
||||||
|
|
||||||
|
# Get prediction
|
||||||
|
prediction = tf.nn.softmax(logits)
|
||||||
|
|
||||||
|
# Loss
|
||||||
|
one_hot_labels = tf.one_hot(labels, depth=10)
|
||||||
|
cost = -tf.reduce_sum(tf.log(prediction) * one_hot_labels, [1])
|
||||||
|
avg_cost = tf.reduce_mean(cost)
|
||||||
|
|
||||||
|
# Get accuracy
|
||||||
|
correct = tf.equal(tf.argmax(prediction, 1), labels)
|
||||||
|
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
|
||||||
|
|
||||||
|
# metrics, g_accuracy
|
||||||
|
with tf.variable_scope("reset_metrics_accuracy_scope") as scope:
|
||||||
|
g_accuracy = tf.metrics.accuracy(
|
||||||
|
labels, tf.argmax(
|
||||||
|
prediction, axis=1))
|
||||||
|
vars = tf.contrib.framework.get_variables(
|
||||||
|
scope, collection=tf.GraphKeys.LOCAL_VARIABLES)
|
||||||
|
g_accuracy_reset_op = tf.variables_initializer(vars)
|
||||||
|
|
||||||
|
# Optimizer
|
||||||
|
opt = tf.train.AdamOptimizer(
|
||||||
|
learning_rate=0.001, beta1=0.9, beta2=0.999)
|
||||||
|
train_op = opt.minimize(avg_cost)
|
||||||
|
# train_op = tf.train.AdamOptimizer(1e-4).minimize(avg_cost)
|
||||||
|
|
||||||
|
train_reader = paddle.batch(
|
||||||
|
paddle.dataset.mnist.train(), batch_size=args.batch_size)
|
||||||
|
test_reader = paddle.batch(
|
||||||
|
paddle.dataset.mnist.test(), batch_size=args.batch_size)
|
||||||
|
|
||||||
|
def eval_test():
|
||||||
|
sess.run(g_accuracy_reset_op)
|
||||||
|
for batch_id, data in enumerate(test_reader()):
|
||||||
|
images_data = np.array(
|
||||||
|
map(lambda x: np.transpose(x[0].reshape([1, 28, 28]), axes=[1,2,0]), data)).astype("float32")
|
||||||
|
labels_data = np.array(map(lambda x: x[1], data)).astype("int64")
|
||||||
|
|
||||||
|
loss, acc, g_acc = sess.run(
|
||||||
|
[avg_cost, accuracy, g_accuracy],
|
||||||
|
feed_dict={images: images_data,
|
||||||
|
labels: labels_data})
|
||||||
|
return g_acc[1]
|
||||||
|
|
||||||
|
config = tf.ConfigProto(
|
||||||
|
intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
|
||||||
|
config.gpu_options.allow_growth = True
|
||||||
|
|
||||||
|
with tf.Session(config=config) as sess:
|
||||||
|
init_g = tf.global_variables_initializer()
|
||||||
|
init_l = tf.local_variables_initializer()
|
||||||
|
sess.run(init_g)
|
||||||
|
sess.run(init_l)
|
||||||
|
for pass_id in range(args.pass_num):
|
||||||
|
sess.run(g_accuracy_reset_op)
|
||||||
|
|
||||||
|
pass_start = time.time()
|
||||||
|
for batch_id, data in enumerate(train_reader()):
|
||||||
|
images_data = np.array(
|
||||||
|
map(lambda x: np.transpose(x[0].reshape([1, 28, 28]), axes=[1,2,0]), data)).astype("float32")
|
||||||
|
labels_data = np.array(map(lambda x: x[1], data)).astype(
|
||||||
|
"int64")
|
||||||
|
|
||||||
|
start = time.time()
|
||||||
|
_, loss, acc, g_acc = sess.run(
|
||||||
|
[train_op, avg_cost, accuracy, g_accuracy],
|
||||||
|
feed_dict={images: images_data,
|
||||||
|
labels: labels_data})
|
||||||
|
end = time.time()
|
||||||
|
|
||||||
|
print("pass=%d, batch=%d, loss=%f, error=%f, elapse=%f" %
|
||||||
|
(pass_id, batch_id, loss, 1 - acc, (end - start) / 1000))
|
||||||
|
|
||||||
|
pass_end = time.time()
|
||||||
|
test_avg_acc = eval_test()
|
||||||
|
|
||||||
|
print(
|
||||||
|
"pass=%d, training_avg_accuracy=%f, test_avg_acc=%f, elapse=%f"
|
||||||
|
% (pass_id, g_acc[1], test_avg_acc,
|
||||||
|
(pass_end - pass_start) / 1000))
|
||||||
|
|
||||||
|
|
||||||
|
def print_arguments(args):
|
||||||
|
print('----------- Configuration Arguments -----------')
|
||||||
|
for arg, value in sorted(vars(args).iteritems()):
|
||||||
|
print('%s: %s' % (arg, value))
|
||||||
|
print('------------------------------------------------')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
args = parse_args()
|
||||||
|
print_arguments(args)
|
||||||
|
run_benchmark(args)
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,220 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from __future__ import absolute_import
|
||||||
|
from __future__ import division
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import argparse
|
||||||
|
import time
|
||||||
|
import tensorflow as tf
|
||||||
|
|
||||||
|
import paddle.v2 as paddle
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args():
|
||||||
|
parser = argparse.ArgumentParser("LSTM model benchmark.")
|
||||||
|
parser.add_argument(
|
||||||
|
'--batch_size',
|
||||||
|
type=int,
|
||||||
|
default=32,
|
||||||
|
help='The sequence number of a batch data. (default: %(default)d)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--stacked_num',
|
||||||
|
type=int,
|
||||||
|
default=5,
|
||||||
|
help='Number of lstm layers to stack. (default: %(default)d)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--embedding_dim',
|
||||||
|
type=int,
|
||||||
|
default=512,
|
||||||
|
help='Dimension of embedding table. (default: %(default)d)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--hidden_dim',
|
||||||
|
type=int,
|
||||||
|
default=512,
|
||||||
|
help='Hidden size of lstm unit. (default: %(default)d)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--pass_num',
|
||||||
|
type=int,
|
||||||
|
default=10,
|
||||||
|
help='Epoch number to train. (default: %(default)d)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--learning_rate',
|
||||||
|
type=float,
|
||||||
|
default=0.0002,
|
||||||
|
help='Learning rate used to train. (default: %(default)f)')
|
||||||
|
parser.add_argument(
|
||||||
|
'--infer_only', action='store_true', help='If set, run forward only.')
|
||||||
|
args = parser.parse_args()
|
||||||
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
def print_arguments(args):
|
||||||
|
print('----------- Configuration Arguments -----------')
|
||||||
|
for arg, value in sorted(vars(args).iteritems()):
|
||||||
|
print('%s: %s' % (arg, value))
|
||||||
|
print('------------------------------------------------')
|
||||||
|
|
||||||
|
|
||||||
|
def dynamic_lstm_model(dict_size,
|
||||||
|
embedding_dim,
|
||||||
|
hidden_dim,
|
||||||
|
stacked_num,
|
||||||
|
class_num=2,
|
||||||
|
is_train=True):
|
||||||
|
word_idx = tf.placeholder(tf.int64, shape=[None, None])
|
||||||
|
sequence_length = tf.placeholder(tf.int64, shape=[None, ])
|
||||||
|
|
||||||
|
embedding_weights = tf.get_variable('word_embeddings',
|
||||||
|
[dict_size, embedding_dim])
|
||||||
|
embedding = tf.nn.embedding_lookup(embedding_weights, word_idx)
|
||||||
|
|
||||||
|
lstm_cell = tf.nn.rnn_cell.LSTMCell(
|
||||||
|
num_units=hidden_dim, use_peepholes=False)
|
||||||
|
stacked_cell = tf.nn.rnn_cell.MultiRNNCell([lstm_cell] * stacked_num)
|
||||||
|
|
||||||
|
# final_state [LSTMTuple(c, h), LSTMTuple(c, h) ...] total stacked_num LSTMTuples
|
||||||
|
_, final_state = tf.nn.dynamic_rnn(
|
||||||
|
cell=stacked_cell,
|
||||||
|
inputs=embedding,
|
||||||
|
dtype=tf.float32,
|
||||||
|
sequence_length=sequence_length)
|
||||||
|
|
||||||
|
w = tf.Variable(
|
||||||
|
tf.truncated_normal([hidden_dim, class_num]), dtype=tf.float32)
|
||||||
|
bias = tf.Variable(
|
||||||
|
tf.constant(
|
||||||
|
value=0.0, shape=[class_num], dtype=tf.float32))
|
||||||
|
prediction = tf.matmul(final_state[-1][1], w) + bias
|
||||||
|
|
||||||
|
if not is_train:
|
||||||
|
return (word_idx, sequence_length), tf.nn.softmax(prediction)
|
||||||
|
|
||||||
|
label = tf.placeholder(tf.int64, shape=[None, ])
|
||||||
|
loss = tf.nn.softmax_cross_entropy_with_logits(
|
||||||
|
labels=tf.one_hot(label, 2), logits=prediction)
|
||||||
|
avg_loss = tf.reduce_mean(loss)
|
||||||
|
|
||||||
|
correct_count = tf.equal(tf.argmax(prediction, 1), label)
|
||||||
|
acc = tf.reduce_mean(tf.cast(correct_count, tf.float32))
|
||||||
|
|
||||||
|
with tf.variable_scope("reset_metrics_accuracy_scope") as scope:
|
||||||
|
g_acc = tf.metrics.accuracy(label, tf.argmax(prediction, axis=1))
|
||||||
|
vars = tf.contrib.framework.get_variables(
|
||||||
|
scope, collection=tf.GraphKeys.LOCAL_VARIABLES)
|
||||||
|
reset_op = tf.variables_initializer(vars)
|
||||||
|
|
||||||
|
return (word_idx, sequence_length, label), avg_loss, acc, g_acc, reset_op
|
||||||
|
|
||||||
|
|
||||||
|
def padding_data(data, padding_size, value):
|
||||||
|
data = data + [value] * padding_size
|
||||||
|
return data[:padding_size]
|
||||||
|
|
||||||
|
|
||||||
|
def train(args):
|
||||||
|
word_dict = paddle.dataset.imdb.word_dict()
|
||||||
|
dict_size = len(word_dict)
|
||||||
|
|
||||||
|
feeding_list, avg_loss, acc, g_acc, reset_op = dynamic_lstm_model(
|
||||||
|
dict_size, args.embedding_dim, args.hidden_dim, args.stacked_num)
|
||||||
|
|
||||||
|
adam_optimizer = tf.train.AdamOptimizer(learning_rate=args.learning_rate)
|
||||||
|
train_op = adam_optimizer.minimize(avg_loss)
|
||||||
|
|
||||||
|
train_reader = paddle.batch(
|
||||||
|
paddle.reader.shuffle(
|
||||||
|
paddle.dataset.imdb.train(word_dict), buf_size=25000),
|
||||||
|
batch_size=args.batch_size)
|
||||||
|
|
||||||
|
test_reader = paddle.batch(
|
||||||
|
paddle.reader.shuffle(
|
||||||
|
paddle.dataset.imdb.test(word_dict), buf_size=25000),
|
||||||
|
batch_size=args.batch_size)
|
||||||
|
|
||||||
|
def do_validation(sess):
|
||||||
|
sess.run(reset_op)
|
||||||
|
for batch_id, data in enumerate(test_reader()):
|
||||||
|
word_idx = map(lambda x: x[0], data)
|
||||||
|
sequence_length = np.array(
|
||||||
|
[len(seq) for seq in word_idx]).astype('int64')
|
||||||
|
maxlen = np.max(sequence_length)
|
||||||
|
word_idx = [padding_data(seq, maxlen, 0) for seq in word_idx]
|
||||||
|
word_idx = np.array(word_idx).astype('int64')
|
||||||
|
label = np.array(map(lambda x: x[1], data)).astype('int64')
|
||||||
|
|
||||||
|
_, loss, fetch_acc, fetch_g_acc = sess.run(
|
||||||
|
[train_op, avg_loss, acc, g_acc],
|
||||||
|
feed_dict={
|
||||||
|
feeding_list[0]: word_idx,
|
||||||
|
feeding_list[1]: sequence_length,
|
||||||
|
feeding_list[2]: label
|
||||||
|
})
|
||||||
|
|
||||||
|
return fetch_g_acc[1]
|
||||||
|
|
||||||
|
config = tf.ConfigProto(
|
||||||
|
intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
|
||||||
|
config.gpu_options.allow_growth = True
|
||||||
|
with tf.Session(config=config) as sess:
|
||||||
|
init_g = tf.global_variables_initializer()
|
||||||
|
init_l = tf.local_variables_initializer()
|
||||||
|
sess.run(init_l)
|
||||||
|
sess.run(init_g)
|
||||||
|
|
||||||
|
for pass_id in xrange(args.pass_num):
|
||||||
|
# clear accuracy local variable
|
||||||
|
sess.run(reset_op)
|
||||||
|
pass_start_time = time.time()
|
||||||
|
words_seen = 0
|
||||||
|
|
||||||
|
for batch_id, data in enumerate(train_reader()):
|
||||||
|
word_idx = map(lambda x: x[0], data)
|
||||||
|
sequence_length = np.array(
|
||||||
|
[len(seq) for seq in word_idx]).astype('int64')
|
||||||
|
words_seen += np.sum(sequence_length)
|
||||||
|
maxlen = np.max(sequence_length)
|
||||||
|
word_idx = [padding_data(seq, maxlen, 0) for seq in word_idx]
|
||||||
|
word_idx = np.array(word_idx).astype('int64')
|
||||||
|
label = np.array(map(lambda x: x[1], data)).astype('int64')
|
||||||
|
|
||||||
|
_, loss, fetch_acc, fetch_g_acc = sess.run(
|
||||||
|
[train_op, avg_loss, acc, g_acc],
|
||||||
|
feed_dict={
|
||||||
|
feeding_list[0]: word_idx,
|
||||||
|
feeding_list[1]: sequence_length,
|
||||||
|
feeding_list[2]: label
|
||||||
|
})
|
||||||
|
|
||||||
|
print("pass_id=%d, batch_id=%d, loss: %f, acc: %f, avg_acc: %f"
|
||||||
|
% (pass_id, batch_id, loss, fetch_acc, fetch_g_acc[1]))
|
||||||
|
|
||||||
|
pass_end_time = time.time()
|
||||||
|
time_consumed = pass_end_time - pass_start_time
|
||||||
|
words_per_sec = words_seen / time_consumed
|
||||||
|
test_acc = do_validation(sess)
|
||||||
|
print("pass_id=%d, test_acc: %f, words/s: %f, sec/pass: %f" %
|
||||||
|
(pass_id, test_acc, words_per_sec, time_consumed))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
args = parse_args()
|
||||||
|
print_arguments(args)
|
||||||
|
|
||||||
|
if args.infer_only:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
train(args)
|
||||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue