Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into feature/add_CUDAPinnedPlace
chengduoZH
8 years ago
commit
e099b18045
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,205 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
import argparse
|
||||
import time
|
||||
|
||||
import paddle.v2 as paddle
|
||||
import paddle.fluid as fluid
|
||||
import paddle.fluid.profiler as profiler
|
||||
|
||||
SEED = 1
|
||||
DTYPE = "float32"
|
||||
|
||||
# random seed must set before configuring the network.
|
||||
# fluid.default_startup_program().random_seed = SEED
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser("mnist model benchmark.")
|
||||
parser.add_argument(
|
||||
'--batch_size', type=int, default=128, help='The minibatch size.')
|
||||
parser.add_argument(
|
||||
'--iterations', type=int, default=35, help='The number of minibatches.')
|
||||
parser.add_argument(
|
||||
'--pass_num', type=int, default=5, help='The number of passes.')
|
||||
parser.add_argument(
|
||||
'--device',
|
||||
type=str,
|
||||
default='GPU',
|
||||
choices=['CPU', 'GPU'],
|
||||
help='The device type.')
|
||||
parser.add_argument(
|
||||
'--infer_only', action='store_true', help='If set, run forward only.')
|
||||
parser.add_argument(
|
||||
'--use_cprof', action='store_true', help='If set, use cProfile.')
|
||||
parser.add_argument(
|
||||
'--use_nvprof',
|
||||
action='store_true',
|
||||
help='If set, use nvprof for CUDA.')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def print_arguments(args):
|
||||
vars(args)['use_nvprof'] = (vars(args)['use_nvprof'] and
|
||||
vars(args)['device'] == 'GPU')
|
||||
print('----------- Configuration Arguments -----------')
|
||||
for arg, value in sorted(vars(args).iteritems()):
|
||||
print('%s: %s' % (arg, value))
|
||||
print('------------------------------------------------')
|
||||
|
||||
|
||||
def cnn_model(data):
|
||||
conv_pool_1 = fluid.nets.simple_img_conv_pool(
|
||||
input=data,
|
||||
filter_size=5,
|
||||
num_filters=20,
|
||||
pool_size=2,
|
||||
pool_stride=2,
|
||||
act="relu")
|
||||
conv_pool_2 = fluid.nets.simple_img_conv_pool(
|
||||
input=conv_pool_1,
|
||||
filter_size=5,
|
||||
num_filters=50,
|
||||
pool_size=2,
|
||||
pool_stride=2,
|
||||
act="relu")
|
||||
|
||||
# TODO(dzhwinter) : refine the initializer and random seed settting
|
||||
SIZE = 10
|
||||
input_shape = conv_pool_2.shape
|
||||
param_shape = [reduce(lambda a, b: a * b, input_shape[1:], 1)] + [SIZE]
|
||||
scale = (2.0 / (param_shape[0]**2 * SIZE))**0.5
|
||||
|
||||
predict = fluid.layers.fc(
|
||||
input=conv_pool_2,
|
||||
size=SIZE,
|
||||
act="softmax",
|
||||
param_attr=fluid.param_attr.ParamAttr(
|
||||
initializer=fluid.initializer.NormalInitializer(
|
||||
loc=0.0, scale=scale)))
|
||||
return predict
|
||||
|
||||
|
||||
def eval_test(exe, batch_acc, batch_size_tensor, inference_program):
|
||||
test_reader = paddle.batch(
|
||||
paddle.dataset.mnist.test(), batch_size=args.batch_size)
|
||||
test_pass_acc = fluid.average.WeightedAverage()
|
||||
for batch_id, data in enumerate(test_reader()):
|
||||
img_data = np.array(map(lambda x: x[0].reshape([1, 28, 28]),
|
||||
data)).astype(DTYPE)
|
||||
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
|
||||
y_data = y_data.reshape([len(y_data), 1])
|
||||
|
||||
acc, weight = exe.run(inference_program,
|
||||
feed={"pixel": img_data,
|
||||
"label": y_data},
|
||||
fetch_list=[batch_acc, batch_size_tensor])
|
||||
test_pass_acc.add(value=acc, weight=weight)
|
||||
pass_acc = test_pass_acc.eval()
|
||||
return pass_acc
|
||||
|
||||
|
||||
def run_benchmark(model, args):
|
||||
if args.use_cprof:
|
||||
pr = cProfile.Profile()
|
||||
pr.enable()
|
||||
start_time = time.time()
|
||||
# Input data
|
||||
images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE)
|
||||
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
|
||||
|
||||
# Train program
|
||||
predict = model(images)
|
||||
cost = fluid.layers.cross_entropy(input=predict, label=label)
|
||||
avg_cost = fluid.layers.mean(x=cost)
|
||||
|
||||
# Evaluator
|
||||
batch_size_tensor = fluid.layers.create_tensor(dtype='int64')
|
||||
batch_acc = fluid.layers.accuracy(
|
||||
input=predict, label=label, total=batch_size_tensor)
|
||||
|
||||
# inference program
|
||||
inference_program = fluid.default_main_program().clone()
|
||||
with fluid.program_guard(inference_program):
|
||||
inference_program = fluid.io.get_inference_program(
|
||||
target_vars=[batch_acc, batch_size_tensor])
|
||||
|
||||
# Optimization
|
||||
opt = fluid.optimizer.AdamOptimizer(
|
||||
learning_rate=0.001, beta1=0.9, beta2=0.999)
|
||||
opt.minimize(avg_cost)
|
||||
|
||||
fluid.memory_optimize(fluid.default_main_program())
|
||||
|
||||
# Initialize executor
|
||||
place = fluid.CPUPlace() if args.device == 'CPU' else fluid.CUDAPlace(0)
|
||||
exe = fluid.Executor(place)
|
||||
|
||||
# Parameter initialization
|
||||
exe.run(fluid.default_startup_program())
|
||||
|
||||
# Reader
|
||||
train_reader = paddle.batch(
|
||||
paddle.dataset.mnist.train(), batch_size=args.batch_size)
|
||||
|
||||
accuracy = fluid.average.WeightedAverage()
|
||||
for pass_id in range(args.pass_num):
|
||||
accuracy.reset()
|
||||
pass_start = time.time()
|
||||
for batch_id, data in enumerate(train_reader()):
|
||||
img_data = np.array(
|
||||
map(lambda x: x[0].reshape([1, 28, 28]), data)).astype(DTYPE)
|
||||
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
|
||||
y_data = y_data.reshape([len(y_data), 1])
|
||||
|
||||
start = time.time()
|
||||
outs = exe.run(
|
||||
fluid.default_main_program(),
|
||||
feed={"pixel": img_data,
|
||||
"label": y_data},
|
||||
fetch_list=[avg_cost, batch_acc, batch_size_tensor]
|
||||
) # The accuracy is the accumulation of batches, but not the current batch.
|
||||
accuracy.add(value=outs[1], weight=outs[2])
|
||||
end = time.time()
|
||||
loss = np.array(outs[0])
|
||||
acc = np.array(outs[1])
|
||||
print("pass=%d, batch=%d, loss=%f, error=%f, elapse=%f" %
|
||||
(pass_id, batch_id, loss, 1 - acc, (end - start) / 1000))
|
||||
|
||||
pass_end = time.time()
|
||||
|
||||
train_avg_acc = accuracy.eval()
|
||||
test_avg_acc = eval_test(exe, batch_acc, batch_size_tensor,
|
||||
inference_program)
|
||||
|
||||
print("pass=%d, train_avg_acc=%f, test_avg_acc=%f, elapse=%f" %
|
||||
(pass_id, train_avg_acc, test_avg_acc,
|
||||
(pass_end - pass_start) / 1000))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = parse_args()
|
||||
print_arguments(args)
|
||||
if args.use_nvprof and args.device == 'GPU':
|
||||
with profiler.cuda_profiler("cuda_profiler.txt", 'csv') as nvprof:
|
||||
run_benchmark(cnn_model, args)
|
||||
else:
|
||||
run_benchmark(cnn_model, args)
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,49 @@
|
||||
#!/bin/bash
|
||||
# This script benchmarking the PaddlePaddle Fluid on
|
||||
# single thread single GPU.
|
||||
export CUDNN_PATH=/paddle/cudnn_v5/cuda/lib
|
||||
|
||||
# disable openmp and mkl parallel
|
||||
#https://github.com/PaddlePaddle/Paddle/issues/7199
|
||||
export MKL_NUM_THREADS=1
|
||||
export OMP_NUM_THREADS=1
|
||||
ht=`lscpu |grep "per core"|awk -F':' '{print $2}'|xargs`
|
||||
if [ $ht -eq 1 ]; then # HT is OFF
|
||||
if [ -z "$KMP_AFFINITY" ]; then
|
||||
export KMP_AFFINITY="granularity=fine,compact,0,0"
|
||||
fi
|
||||
if [ -z "$OMP_DYNAMIC" ]; then
|
||||
export OMP_DYNAMIC="FALSE"
|
||||
fi
|
||||
else # HT is ON
|
||||
if [ -z "$KMP_AFFINITY" ]; then
|
||||
export KMP_AFFINITY="granularity=fine,compact,1,0"
|
||||
fi
|
||||
fi
|
||||
# disable multi-gpu if have more than one
|
||||
export CUDA_VISIBLE_DEVICES=0
|
||||
export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
|
||||
export LD_LIBRARY_PATH=$CUDNN_PATH:$LD_LIBRARY_PATH
|
||||
|
||||
|
||||
# vgg16
|
||||
# cifar10 gpu cifar10 128
|
||||
FLAGS_benchmark=true python fluid/vgg.py \
|
||||
--device=GPU \
|
||||
--batch_size=128 \
|
||||
--skip_batch_num=5 \
|
||||
--iterations=30 \
|
||||
2>&1 > vgg16_gpu_128.log
|
||||
|
||||
# resnet50
|
||||
# resnet50 gpu cifar10 128
|
||||
FLAGS_benchmark=true python fluid/resnet.py \
|
||||
--device=GPU \
|
||||
--batch_size=128 \
|
||||
--data_set=cifar10 \
|
||||
--model=resnet_cifar10 \
|
||||
--skip_batch_num=5 \
|
||||
--iterations=30 \
|
||||
2>&1 > resnet50_gpu_128.log
|
||||
|
||||
# lstm
|
||||
@ -0,0 +1,209 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import cPickle
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
|
||||
import numpy
|
||||
import paddle.v2 as paddle
|
||||
import paddle.v2.dataset.imdb as imdb
|
||||
import paddle.fluid as fluid
|
||||
from paddle.v2 import batch
|
||||
import paddle.fluid.profiler as profiler
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser("Understand Sentiment by Dynamic RNN.")
|
||||
parser.add_argument(
|
||||
'--batch_size',
|
||||
type=int,
|
||||
default=32,
|
||||
help='The sequence number of a batch data. (default: %(default)d)')
|
||||
parser.add_argument(
|
||||
'--emb_dim',
|
||||
type=int,
|
||||
default=512,
|
||||
help='Dimension of embedding table. (default: %(default)d)')
|
||||
parser.add_argument(
|
||||
'--hidden_dim',
|
||||
type=int,
|
||||
default=512,
|
||||
help='Hidden size of lstm unit. (default: %(default)d)')
|
||||
parser.add_argument(
|
||||
'--pass_num',
|
||||
type=int,
|
||||
default=100,
|
||||
help='Epoch number to train. (default: %(default)d)')
|
||||
parser.add_argument(
|
||||
'--device',
|
||||
type=str,
|
||||
default='CPU',
|
||||
choices=['CPU', 'GPU'],
|
||||
help='The device type.')
|
||||
parser.add_argument(
|
||||
'--crop_size',
|
||||
type=int,
|
||||
default=int(os.environ.get('CROP_SIZE', '1500')),
|
||||
help='The max sentence length of input. Since this model use plain RNN,'
|
||||
' Gradient could be explored if sentence is too long')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
word_dict = imdb.word_dict()
|
||||
|
||||
|
||||
def crop_sentence(reader, crop_size):
|
||||
unk_value = word_dict['<unk>']
|
||||
|
||||
def __impl__():
|
||||
for item in reader():
|
||||
if len([x for x in item[0] if x != unk_value]) < crop_size:
|
||||
yield item
|
||||
|
||||
return __impl__
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
lstm_size = args.hidden_dim
|
||||
|
||||
data = fluid.layers.data(
|
||||
name="words", shape=[1], lod_level=1, dtype='int64')
|
||||
sentence = fluid.layers.embedding(
|
||||
input=data, size=[len(word_dict), args.emb_dim])
|
||||
|
||||
sentence = fluid.layers.fc(input=sentence, size=lstm_size, act='tanh')
|
||||
|
||||
rnn = fluid.layers.DynamicRNN()
|
||||
with rnn.block():
|
||||
word = rnn.step_input(sentence)
|
||||
prev_hidden = rnn.memory(value=0.0, shape=[lstm_size])
|
||||
prev_cell = rnn.memory(value=0.0, shape=[lstm_size])
|
||||
|
||||
def gate_common(
|
||||
ipt,
|
||||
hidden,
|
||||
size, ):
|
||||
gate0 = fluid.layers.fc(input=ipt, size=size, bias_attr=True)
|
||||
gate1 = fluid.layers.fc(input=hidden, size=size, bias_attr=False)
|
||||
gate = fluid.layers.sums(input=[gate0, gate1])
|
||||
return gate
|
||||
|
||||
forget_gate = fluid.layers.sigmoid(
|
||||
x=gate_common(word, prev_hidden, lstm_size))
|
||||
input_gate = fluid.layers.sigmoid(
|
||||
x=gate_common(word, prev_hidden, lstm_size))
|
||||
output_gate = fluid.layers.sigmoid(
|
||||
x=gate_common(word, prev_hidden, lstm_size))
|
||||
cell_gate = fluid.layers.tanh(
|
||||
x=gate_common(word, prev_hidden, lstm_size))
|
||||
|
||||
cell = fluid.layers.sums(input=[
|
||||
fluid.layers.elementwise_mul(
|
||||
x=forget_gate, y=prev_cell), fluid.layers.elementwise_mul(
|
||||
x=input_gate, y=cell_gate)
|
||||
])
|
||||
|
||||
hidden = fluid.layers.elementwise_mul(
|
||||
x=output_gate, y=fluid.layers.tanh(x=cell))
|
||||
|
||||
rnn.update_memory(prev_cell, cell)
|
||||
rnn.update_memory(prev_hidden, hidden)
|
||||
rnn.output(hidden)
|
||||
|
||||
last = fluid.layers.sequence_pool(rnn(), 'last')
|
||||
logit = fluid.layers.fc(input=last, size=2, act='softmax')
|
||||
loss = fluid.layers.cross_entropy(
|
||||
input=logit,
|
||||
label=fluid.layers.data(
|
||||
name='label', shape=[1], dtype='int64'))
|
||||
loss = fluid.layers.mean(x=loss)
|
||||
|
||||
# add acc
|
||||
batch_size_tensor = fluid.layers.create_tensor(dtype='int64')
|
||||
batch_acc = fluid.layers.accuracy(input=logit, label=fluid.layers.data(name='label', \
|
||||
shape=[1], dtype='int64'), total=batch_size_tensor)
|
||||
|
||||
inference_program = fluid.default_main_program().clone()
|
||||
with fluid.program_guard(inference_program):
|
||||
inference_program = fluid.io.get_inference_program(
|
||||
target_vars=[batch_acc, batch_size_tensor])
|
||||
|
||||
adam = fluid.optimizer.Adam()
|
||||
adam.minimize(loss)
|
||||
|
||||
fluid.memory_optimize(fluid.default_main_program())
|
||||
|
||||
place = fluid.CPUPlace() if args.device == 'CPU' else fluid.CUDAPlace(0)
|
||||
exe = fluid.Executor(place)
|
||||
exe.run(fluid.default_startup_program())
|
||||
|
||||
def train_loop(pass_num, crop_size):
|
||||
with profiler.profiler(args.device, 'total') as prof:
|
||||
for pass_id in range(pass_num):
|
||||
train_reader = batch(
|
||||
paddle.reader.shuffle(
|
||||
crop_sentence(imdb.train(word_dict), crop_size),
|
||||
buf_size=25000),
|
||||
batch_size=args.batch_size)
|
||||
word_nums = 0
|
||||
pass_start_time = time.time()
|
||||
for batch_id, data in enumerate(train_reader()):
|
||||
tensor_words = to_lodtensor([x[0] for x in data], place)
|
||||
for x in data:
|
||||
word_nums += len(x[0])
|
||||
label = numpy.array([x[1] for x in data]).astype("int64")
|
||||
label = label.reshape((-1, 1))
|
||||
loss_np, acc, weight = exe.run(
|
||||
fluid.default_main_program(),
|
||||
feed={"words": tensor_words,
|
||||
"label": label},
|
||||
fetch_list=[loss, batch_acc, batch_size_tensor])
|
||||
print("pass_id=%d, batch_id=%d, loss=%f, acc=%f" %
|
||||
(pass_id, batch_id, loss_np, acc))
|
||||
|
||||
pass_end_time = time.time()
|
||||
time_consumed = pass_end_time - pass_start_time
|
||||
words_per_sec = word_nums / time_consumed
|
||||
print("pass_id=%d, sec/pass: %f, words/s: %f" %
|
||||
(pass_id, time_consumed, words_per_sec))
|
||||
|
||||
train_loop(args.pass_num, args.crop_size)
|
||||
|
||||
|
||||
def to_lodtensor(data, place):
|
||||
seq_lens = [len(seq) for seq in data]
|
||||
cur_len = 0
|
||||
lod = [cur_len]
|
||||
for l in seq_lens:
|
||||
cur_len += l
|
||||
lod.append(cur_len)
|
||||
flattened_data = numpy.concatenate(data, axis=0).astype("int64")
|
||||
flattened_data = flattened_data.reshape([len(flattened_data), 1])
|
||||
res = fluid.LoDTensor()
|
||||
res.set(flattened_data, place)
|
||||
res.set_lod([lod])
|
||||
return res
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -0,0 +1,220 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""VGG16 benchmark in Fluid"""
|
||||
from __future__ import print_function
|
||||
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
import paddle.v2 as paddle
|
||||
import paddle.fluid as fluid
|
||||
import paddle.fluid.core as core
|
||||
import argparse
|
||||
import functools
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
'--batch_size', type=int, default=128, help="Batch size for training.")
|
||||
parser.add_argument(
|
||||
'--skip_batch_num',
|
||||
type=int,
|
||||
default=5,
|
||||
help='The first num of minibatch num to skip, for better performance test')
|
||||
parser.add_argument(
|
||||
'--iterations', type=int, default=80, help='The number of minibatches.')
|
||||
parser.add_argument(
|
||||
'--learning_rate',
|
||||
type=float,
|
||||
default=1e-3,
|
||||
help="Learning rate for training.")
|
||||
parser.add_argument('--pass_num', type=int, default=50, help="No. of passes.")
|
||||
parser.add_argument(
|
||||
'--device',
|
||||
type=str,
|
||||
default='GPU',
|
||||
choices=['CPU', 'GPU'],
|
||||
help="The device type.")
|
||||
parser.add_argument(
|
||||
'--data_format',
|
||||
type=str,
|
||||
default='NCHW',
|
||||
choices=['NCHW', 'NHWC'],
|
||||
help='The data order, now only support NCHW.')
|
||||
parser.add_argument(
|
||||
'--data_set',
|
||||
type=str,
|
||||
default='cifar10',
|
||||
choices=['cifar10', 'flowers'],
|
||||
help='Optional dataset for benchmark.')
|
||||
parser.add_argument(
|
||||
'--with_test',
|
||||
action='store_true',
|
||||
help='If set, test the testset during training.')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def vgg16_bn_drop(input):
|
||||
def conv_block(input, num_filter, groups, dropouts):
|
||||
return fluid.nets.img_conv_group(
|
||||
input=input,
|
||||
pool_size=2,
|
||||
pool_stride=2,
|
||||
conv_num_filter=[num_filter] * groups,
|
||||
conv_filter_size=3,
|
||||
conv_act='relu',
|
||||
conv_with_batchnorm=True,
|
||||
conv_batchnorm_drop_rate=dropouts,
|
||||
pool_type='max')
|
||||
|
||||
conv1 = conv_block(input, 64, 2, [0.3, 0])
|
||||
conv2 = conv_block(conv1, 128, 2, [0.4, 0])
|
||||
conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0])
|
||||
conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0])
|
||||
conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0])
|
||||
|
||||
drop = fluid.layers.dropout(x=conv5, dropout_prob=0.5)
|
||||
fc1 = fluid.layers.fc(input=drop, size=512, act=None)
|
||||
bn = fluid.layers.batch_norm(input=fc1, act='relu')
|
||||
drop2 = fluid.layers.dropout(x=bn, dropout_prob=0.5)
|
||||
fc2 = fluid.layers.fc(input=drop2, size=512, act=None)
|
||||
return fc2
|
||||
|
||||
|
||||
def main():
|
||||
if args.data_set == "cifar10":
|
||||
classdim = 10
|
||||
if args.data_format == 'NCHW':
|
||||
data_shape = [3, 32, 32]
|
||||
else:
|
||||
data_shape = [32, 32, 3]
|
||||
else:
|
||||
classdim = 102
|
||||
if args.data_format == 'NCHW':
|
||||
data_shape = [3, 224, 224]
|
||||
else:
|
||||
data_shape = [224, 224, 3]
|
||||
|
||||
# Input data
|
||||
images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32')
|
||||
label = fluid.layers.data(name='label', shape=[1], dtype='int64')
|
||||
|
||||
# Train program
|
||||
net = vgg16_bn_drop(images)
|
||||
predict = fluid.layers.fc(input=net, size=classdim, act='softmax')
|
||||
cost = fluid.layers.cross_entropy(input=predict, label=label)
|
||||
avg_cost = fluid.layers.mean(x=cost)
|
||||
|
||||
# Evaluator
|
||||
batch_size_tensor = fluid.layers.create_tensor(dtype='int64')
|
||||
batch_acc = fluid.layers.accuracy(
|
||||
input=predict, label=label, total=batch_size_tensor)
|
||||
|
||||
# inference program
|
||||
inference_program = fluid.default_main_program().clone()
|
||||
with fluid.program_guard(inference_program):
|
||||
inference_program = fluid.io.get_inference_program(
|
||||
target_vars=[batch_acc, batch_size_tensor])
|
||||
|
||||
# Optimization
|
||||
optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate)
|
||||
opts = optimizer.minimize(avg_cost)
|
||||
|
||||
fluid.memory_optimize(fluid.default_main_program())
|
||||
|
||||
# Initialize executor
|
||||
place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(0)
|
||||
exe = fluid.Executor(place)
|
||||
|
||||
# Parameter initialization
|
||||
exe.run(fluid.default_startup_program())
|
||||
|
||||
# data reader
|
||||
train_reader = paddle.batch(
|
||||
paddle.reader.shuffle(
|
||||
paddle.dataset.cifar.train10()
|
||||
if args.data_set == 'cifar10' else paddle.dataset.flowers.train(),
|
||||
buf_size=5120),
|
||||
batch_size=args.batch_size)
|
||||
test_reader = paddle.batch(
|
||||
paddle.dataset.cifar.test10()
|
||||
if args.data_set == 'cifar10' else paddle.dataset.flowers.test(),
|
||||
batch_size=args.batch_size)
|
||||
|
||||
# test
|
||||
def test(exe):
|
||||
test_accuracy = fluid.average.WeightedAverage()
|
||||
for batch_id, data in enumerate(test_reader()):
|
||||
img_data = np.array(map(lambda x: x[0].reshape(data_shape),
|
||||
data)).astype("float32")
|
||||
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
|
||||
y_data = y_data.reshape([-1, 1])
|
||||
|
||||
acc, weight = exe.run(inference_program,
|
||||
feed={"pixel": img_data,
|
||||
"label": y_data},
|
||||
fetch_list=[batch_acc, batch_size_tensor])
|
||||
test_accuracy.add(value=acc, weight=weight)
|
||||
return test_accuracy.eval()
|
||||
|
||||
iters, num_samples, start_time = 0, 0, time.time()
|
||||
accuracy = fluid.average.WeightedAverage()
|
||||
for pass_id in range(args.pass_num):
|
||||
accuracy.reset()
|
||||
train_accs = []
|
||||
train_losses = []
|
||||
for batch_id, data in enumerate(train_reader()):
|
||||
if iters == args.skip_batch_num:
|
||||
start_time = time.time()
|
||||
num_samples = 0
|
||||
if iters == args.iterations:
|
||||
break
|
||||
img_data = np.array(map(lambda x: x[0].reshape(data_shape),
|
||||
data)).astype("float32")
|
||||
y_data = np.array(map(lambda x: x[1], data)).astype("int64")
|
||||
y_data = y_data.reshape([-1, 1])
|
||||
|
||||
loss, acc, weight = exe.run(
|
||||
fluid.default_main_program(),
|
||||
feed={"pixel": img_data,
|
||||
"label": y_data},
|
||||
fetch_list=[avg_cost, batch_acc, batch_size_tensor])
|
||||
accuracy.add(value=acc, weight=weight)
|
||||
iters += 1
|
||||
num_samples += len(data)
|
||||
print(
|
||||
"Pass = %d, Iter = %d, Loss = %f, Accuracy = %f" %
|
||||
(pass_id, iters, loss, acc)
|
||||
) # The accuracy is the accumulation of batches, but not the current batch.
|
||||
|
||||
pass_train_acc = accuracy.eval()
|
||||
train_losses.append(loss)
|
||||
train_accs.append(acc)
|
||||
# evaluation
|
||||
if args.with_test:
|
||||
pass_test_acc = test(exe)
|
||||
train_elapsed = time.time() - start_time
|
||||
print("Pass: %d, Loss: %f, Train Accuray: %f\n" %
|
||||
(pass_id, np.mean(train_losses), np.mean(train_accs)))
|
||||
|
||||
|
||||
def print_arguments():
|
||||
print('----------- Configuration Arguments -----------')
|
||||
for arg, value in sorted(vars(args).iteritems()):
|
||||
print('%s: %s' % (arg, value))
|
||||
print('------------------------------------------------')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print_arguments()
|
||||
main()
|
||||
@ -0,0 +1,30 @@
|
||||
INCLUDE(ExternalProject)
|
||||
|
||||
SET(THREADPOOL_SOURCE_DIR ${THIRD_PARTY_PATH}/threadpool)
|
||||
SET(THREADPOOL_INCLUDE_DIR ${THREADPOOL_SOURCE_DIR}/src/extern_threadpool)
|
||||
INCLUDE_DIRECTORIES(${THREADPOOL_INCLUDE_DIR})
|
||||
|
||||
ExternalProject_Add(
|
||||
extern_threadpool
|
||||
${EXTERNAL_PROJECT_LOG_ARGS}
|
||||
GIT_REPOSITORY "https://github.com/progschj/ThreadPool.git"
|
||||
GIT_TAG 9a42ec1329f259a5f4881a291db1dcb8f2ad9040
|
||||
PREFIX ${THREADPOOL_SOURCE_DIR}
|
||||
UPDATE_COMMAND ""
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
|
||||
if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
|
||||
set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/threadpool_dummy.c)
|
||||
file(WRITE ${dummyfile} "const char *dummy_threadpool = \"${dummyfile}\";")
|
||||
add_library(simple_threadpool STATIC ${dummyfile})
|
||||
else()
|
||||
add_library(simple_threadpool INTERFACE)
|
||||
endif()
|
||||
|
||||
add_dependencies(simple_threadpool extern_threadpool)
|
||||
|
||||
LIST(APPEND external_project_dependencies simple_threadpool)
|
||||
@ -1,2 +1,9 @@
|
||||
add_custom_target(paddle_apis ALL
|
||||
DEPENDS paddle_v2_apis paddle_fluid_apis)
|
||||
|
||||
add_custom_target(paddle_docs ALL
|
||||
DEPENDS paddle_v2_docs paddle_v2_docs_cn
|
||||
paddle_fluid_docs paddle_fluid_docs_cn)
|
||||
|
||||
add_subdirectory(v2)
|
||||
add_subdirectory(fluid)
|
||||
|
||||
@ -0,0 +1,83 @@
|
||||
digraph G {
|
||||
subgraph cluster_init {
|
||||
label="Initialization"
|
||||
startup_program [label="startup", shape=box]
|
||||
node_w_g0 [label="W\nGPU0"]
|
||||
startup_program -> node_w_g0 [label="Initialize"]
|
||||
node_w_g1 [label="W\nGPU1"]
|
||||
node_w_g0 -> node_w_g1 [label="broadcast"]
|
||||
}
|
||||
|
||||
subgraph cluster_train {
|
||||
label="forward_backward"
|
||||
|
||||
subgraph cluster_gpu0 {
|
||||
label="GPU0"
|
||||
fc_0 [label="fc\nGPU0", shape=box]
|
||||
hidden_0 [label="hidden\nGPU0"]
|
||||
node_w_g0 -> fc_0
|
||||
fc_0 -> hidden_0
|
||||
loss0 [label="loss\nGPU0"]
|
||||
hidden_0 -> loss0 [label="many ops omitted"]
|
||||
scale_loss_0 [label="scale_loss_gradient\nGPU0", shape=box]
|
||||
loss_g0 [label="loss_grad\nGPU0"]
|
||||
scale_loss_0->loss_g0
|
||||
|
||||
fc_g_0 [label="w_grad\nGPU0", shape=box]
|
||||
loss0 -> fc_g_0
|
||||
loss_g0 -> fc_g_0
|
||||
hidden_0 -> fc_g_0
|
||||
}
|
||||
|
||||
subgraph cluster_gpu1 {
|
||||
label="GPU1"
|
||||
fc_1 [label="fc\nGPU1", shape=box]
|
||||
hidden_1 [label="hidden\nGPU1"]
|
||||
node_w_g1 -> fc_1
|
||||
fc_1 -> hidden_1
|
||||
loss1 [label="loss\nGPU1"]
|
||||
hidden_1 -> loss1 [label="many ops omitted"]
|
||||
scale_loss_1 [label="scale_loss_gradient\nGPU1", shape=box]
|
||||
loss_g1 [label="loss_grad\nGPU1"]
|
||||
scale_loss_1->loss_g1
|
||||
|
||||
fc_g_1 [label="w_grad\nGPU1", shape=box]
|
||||
loss1 -> fc_g_1
|
||||
loss_g1 -> fc_g_1
|
||||
hidden_1 -> fc_g_1
|
||||
}
|
||||
}
|
||||
|
||||
all_reduce_w [label="Merge Gradients(AllReduce)", shape=box]
|
||||
fc_g_0 -> all_reduce_w
|
||||
fc_g_1 -> all_reduce_w
|
||||
|
||||
fc_g_0_merged [label="w_grad\nMerged\nGPU0"]
|
||||
fc_g_1_merged [label="w_grad\nMerged\nGPU1"]
|
||||
all_reduce_w -> fc_g_0_merged
|
||||
all_reduce_w -> fc_g_1_merged
|
||||
|
||||
subgraph cluster_optimization {
|
||||
label="Optimization"
|
||||
subgraph cluster_opt_gpu0 {
|
||||
label="GPU0"
|
||||
sgd_0 [label="SGD Op\nGPU0", shape=box]
|
||||
|
||||
fc_g_0_merged -> sgd_0
|
||||
node_w_g0 -> sgd_0
|
||||
optimized_w_0 [label="Optimized W\nGPU0"]
|
||||
sgd_0 -> optimized_w_0
|
||||
}
|
||||
subgraph cluster_opt_gpu1 {
|
||||
label="GPU1"
|
||||
sgd_1 [label="SGD Op\nGPU1", shape=box]
|
||||
|
||||
fc_g_1_merged -> sgd_1
|
||||
node_w_g1 -> sgd_1
|
||||
optimized_w_1 [label="Optimized W\nGPU0"]
|
||||
sgd_1 -> optimized_w_1
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
After Width: | Height: | Size: 175 KiB |
@ -0,0 +1,104 @@
|
||||
# ParallelExecutor
|
||||
|
||||
## Background
|
||||
|
||||
Neural network models are defined as a `ProgramDesc` in Fluid. The `ProgramDesc` can be executed by an interpreter(i.e. the `executor` concept in Fluid). The instructions or operators in a `Program` will be executed, and the results will be fetched in Python side.
|
||||
|
||||
The executor is a very naive interpreter. It runs operators one by one. We can use `Parallel.Do` to support data parallelism, however, lacking device information in `ProgramDesc`; it is not possible to optimize the performance of `Parallel.Do`.
|
||||
|
||||
We want a `ProgramDesc` can be run on different nodes. It is better not to contain device information in `ProgramDesc`. However, we can write a high-performance interpreter, which can hold an alternative intermediate representation of `ProgramDesc`, to take full usage of Multi-GPUs.
|
||||
|
||||
ParallelExecutor is an interpreter of `ProgramDesc` which will [out-of-order execute](https://en.wikipedia.org/wiki/Out-of-order_execution) `Program` in data parallelism mode and maximise the utility of Multi-GPUs.
|
||||
|
||||
|
||||
## Overview of MultiGPUs logic
|
||||
|
||||
The ParallelExecutor takes the startup program and main program as inputs. The parameters will be initialised on `GPU0` by startup program and will broadcast to multi-GPUs. The main program will be duplicated into multi-GPUs. The gradient will be merged during each iteration, and each device will optimize parameters independently. Since the gradients on each device will be merged before parameter optimization, the parameters will be the same on each device and it does not need to be broadcast the parameters.
|
||||
|
||||
|
||||
|
||||
There are several optimizations for this logic.
|
||||
|
||||
1. We use an alternate representation in ParallelExecutor. It because the device information is critical for performance optimization.
|
||||
2. The execution is out-of-order, i.e., an operator will be executed whenever the inputs of the operator are ready.
|
||||
* GPU is a high-performance device; only one CPU thread cannot fulfil one GPU. So there is a thread pool to execute operators.
|
||||
* Out-of-order also helps transpilers to generate `ProgramDesc`. It is no need to concern about the best order of performance when implementing a transpiler.
|
||||
3. The streams of computation, merge gradients and fetch data are different.
|
||||
|
||||
The performance of `ResNeXt152` on `TitanX` which `batch_size=12` is shown below.
|
||||
|
||||
| Number of GPUs | 1 | 2 | 3 | 4|
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Image/Sec | 17.9906 | 25.771 | 36.911 | 48.8428 |
|
||||
| Speed Up | N/A | 1.43247029 | 2.05168255 | 2.71490667 |
|
||||
|
||||
|
||||
## Static single assignment Graph
|
||||
|
||||
[Static single assignment form](https://en.wikipedia.org/wiki/Static_single_assignment_form)(`SSA` for short) is a common form for compiler optimization. To implement concurrent execution, we uses an `SSA` graph as an intermedia representation of `ProgramDesc`.
|
||||
|
||||
The `Program` is a directed acyclic graph, since a variable can be assigned multiple times. We enforce a variable will be assigned once, by adding version number to varaibles. We parsing the `Program` into a `SSA` graph. Also, ProgramExecutor duplicate `Program` into multi-devices. We also add a device number to varaibles and insert `NCCLAllReduce` into Graph.
|
||||
|
||||
The data structure of `SSA` graph is:
|
||||
|
||||
```c++
|
||||
struct VarHandleBase {
|
||||
OpHandleBase* generated_op_;
|
||||
vector<OpHandleBase*> pending_ops_;
|
||||
|
||||
string name;
|
||||
Place place;
|
||||
size_t version;
|
||||
};
|
||||
|
||||
struct OpHandleBase {
|
||||
vector<OpHandleBase*> inputs_;
|
||||
vector<OpHnadleBase*> outputs_;
|
||||
};
|
||||
|
||||
struct SSAGraph {
|
||||
// vars on each devices.
|
||||
// * the vars in each map in vector is on different device.
|
||||
// * the map is mapping a variable name to variable handles
|
||||
// with different versions
|
||||
vector<std::unordered_map<string, vector<VarHandleBase>>> vars_;
|
||||
|
||||
// All ops
|
||||
vector<OpHandleBase> ops_;
|
||||
};
|
||||
```
|
||||
The variable handles are the wrapper of `Variables`. The operator handles are the wrapper of `OperatorBase`. Some `OpHandle` is not an `OperatorBase`, such as `NCCLAllReduceOpHandle`, because `AllReduceOpHandle` will use new device contexts.
|
||||
|
||||
When the `ProgramDesc` converted into an `SSA` Graph, the [data hazard](https://en.wikipedia.org/wiki/Hazard_(computer_architecture)) problem is also need to be taken care. The dummy variables, which represent the dependency between operators, will be manually inserted into SSA graph to resolve the [data hazard](https://en.wikipedia.org/wiki/Hazard_(computer_architecture)) problem.
|
||||
|
||||
## Execute SSA Graph
|
||||
|
||||
The SSA graph can be out-of-order executed by an approximate [topological sorting](https://en.wikipedia.org/wiki/Topological_sorting) algorithm. The algorithm is
|
||||
|
||||
1. Maintaining a map of an operator and its needed input number.
|
||||
2. If a variable is not generated by an operator, i.e., `var.generated_op == nullptr`, decrease the needed input number of its pending operators.
|
||||
3. If there is an operator which needed input number is decreased to zero, just run this operator.
|
||||
4. After run this operator, just mark the variables are generated and repeat step 2 until all variables are generated.
|
||||
|
||||
Running an operator can be asynchronized. There is a thread pool to execute an `SSA` graph.
|
||||
|
||||
## Synchronize GPU Kernels
|
||||
|
||||
The GPU is a non-blocking device. The different streams need be synchronized when switing streams. In current implementation, the synchronization based on the following algorithm:
|
||||
|
||||
1. `OpHandle` will record `DeviceContext` that it is used.
|
||||
2. In `OpHandle::Run`, if the `DeviceContext` of current operator is different from `DeviceContext` of any input variable, just wait the generate operator of this input variable.
|
||||
|
||||
The `wait` are implemented by two strategies:
|
||||
|
||||
1. Invoke `DeviceContext->Wait()`, It will wait all operators on this device contexts complete.
|
||||
2. Uses `cudaStreamWaitEvent` to sending a event to the stream. It is a non-blocking call. The wait operators will be executed in GPU.
|
||||
|
||||
Generally, the `cudaStreamWaitEvent` will have a better perforamnce. However, `DeviceContext->Wait()` strategy is easier to debug. The strategy can be changed in runtime.
|
||||
|
||||
## What's next?
|
||||
|
||||
* Merging gradient of dense parameters has been done. However, the merging of sparse parameters has not been done.
|
||||
* The CPU version of Parallel Executor has not been implemented. The out-of-order logic will make CPU compuatation faster, too.
|
||||
* A better strategy to merge gradients can be introduced. We can shrink the gradients from `float32` to `int8` or `int4` while merging. It will significantly speed up multi-GPUs training without much loss of precision.
|
||||
* Combine multi-Nodes implementation. By the benifit of out-of-order, sending and recving operator can be an blocking operator, and the transpiler does not need to concern about the best position of operator.
|
||||
@ -0,0 +1,22 @@
|
||||
# configured documentation tools and intermediate build results
|
||||
set(BINARY_BUILD_DIR_EN "${CMAKE_CURRENT_BINARY_DIR}/en/_build")
|
||||
|
||||
# Sphinx cache with pickled ReST documents
|
||||
set(SPHINX_CACHE_DIR_EN "${CMAKE_CURRENT_BINARY_DIR}/en/_doctrees")
|
||||
|
||||
# HTML output director
|
||||
set(SPHINX_HTML_DIR_EN "${CMAKE_CURRENT_BINARY_DIR}/en/html")
|
||||
|
||||
configure_file(
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/../../templates/conf.py.en.in"
|
||||
"${BINARY_BUILD_DIR_EN}/conf.py"
|
||||
@ONLY)
|
||||
|
||||
sphinx_add_target(paddle_fluid_apis
|
||||
html
|
||||
${BINARY_BUILD_DIR_EN}
|
||||
${SPHINX_CACHE_DIR_EN}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}
|
||||
${SPHINX_HTML_DIR_EN})
|
||||
|
||||
add_dependencies(paddle_fluid_apis gen_proto_py framework_py_proto copy_paddle_pybind)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue