|
|
|
@ -62,21 +62,15 @@ __all__ = [
|
|
|
|
|
cp.begin_parse()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def init(**kwargs):
|
|
|
|
|
import py_paddle.swig_paddle as api
|
|
|
|
|
args = []
|
|
|
|
|
args_dict = {}
|
|
|
|
|
# NOTE: append arguments if they are in ENV
|
|
|
|
|
for ek, ev in os.environ.iteritems():
|
|
|
|
|
if ek.startswith("PADDLE_INIT_"):
|
|
|
|
|
args_dict[ek.replace("PADDLE_INIT_", "").lower()] = str(ev)
|
|
|
|
|
def set_omp_mkl_env_vars(trainer_count):
|
|
|
|
|
'''Auto set CPU environment if have not set before.
|
|
|
|
|
export KMP_AFFINITY, OMP_DYNAMIC according to the Hyper Threading status.
|
|
|
|
|
export OMP_NUM_THREADS, MKL_NUM_THREADS according to trainer_count.
|
|
|
|
|
'''
|
|
|
|
|
import platform
|
|
|
|
|
if not platform.system() in ['Linux', 'Darwin']:
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
args_dict.update(kwargs)
|
|
|
|
|
# NOTE: overwrite arguments from ENV if it is in kwargs
|
|
|
|
|
for key in args_dict.keys():
|
|
|
|
|
args.append('--%s=%s' % (key, str(args_dict[key])))
|
|
|
|
|
|
|
|
|
|
# auto set cpu environment
|
|
|
|
|
def set_env(key, value):
|
|
|
|
|
'''If the key has not been set in the environment, set it with value.'''
|
|
|
|
|
assert isinstance(key, str)
|
|
|
|
@ -85,22 +79,59 @@ def init(**kwargs):
|
|
|
|
|
if envset is None:
|
|
|
|
|
os.environ[key] = value
|
|
|
|
|
|
|
|
|
|
ht = os.popen("lscpu |grep \"per core\"|awk -F':' '{print $2}'|xargs")
|
|
|
|
|
ht = int(ht.read())
|
|
|
|
|
if ht == 1: # ht is off
|
|
|
|
|
set_env("OMP_DYNAMIC", "false")
|
|
|
|
|
set_env("KMP_AFFINITY", "granularity=fine,compact,0,0")
|
|
|
|
|
else:
|
|
|
|
|
def num_physical_cores():
|
|
|
|
|
'''Get the number of physical cores'''
|
|
|
|
|
if platform.system() == "Linux":
|
|
|
|
|
num_sockets = int(
|
|
|
|
|
os.popen("lscpu |grep \"Socket\" |awk -F':' '{print $2}'|xargs")
|
|
|
|
|
.read())
|
|
|
|
|
num_cores_per_socket = int(
|
|
|
|
|
os.popen(
|
|
|
|
|
"lscpu |grep \"per socket\" |awk -F':' '{print $2}'|xargs")
|
|
|
|
|
.read())
|
|
|
|
|
return num_sockets * num_cores_per_socket
|
|
|
|
|
else:
|
|
|
|
|
cmds = {"Darwin": "sysctl hw.physicalcpu"}
|
|
|
|
|
return int(os.popen(cmds.get(platform.system(), "expr 1")).read())
|
|
|
|
|
|
|
|
|
|
def num_logical_processors():
|
|
|
|
|
'''Get the number of logical processors'''
|
|
|
|
|
cmds = {
|
|
|
|
|
"Linux": "grep \"processor\" /proc/cpuinfo|sort -u|wc -l",
|
|
|
|
|
"Darwin": "sysctl hw.logicalcpu"
|
|
|
|
|
}
|
|
|
|
|
return int(os.popen(cmds.get(platform.system(), "expr 1")).read())
|
|
|
|
|
|
|
|
|
|
num_cores = num_physical_cores()
|
|
|
|
|
num_processors = num_logical_processors()
|
|
|
|
|
if num_processors > num_cores: # Hyper Threading is enabled
|
|
|
|
|
set_env("OMP_DYNAMIC", "true")
|
|
|
|
|
set_env("KMP_AFFINITY", "granularity=fine,compact,1,0")
|
|
|
|
|
processors = os.popen("grep \"processor\" /proc/cpuinfo|sort -u|wc -l")
|
|
|
|
|
processors = int(processors.read())
|
|
|
|
|
trainers = kwargs.get('trainer_count', 1)
|
|
|
|
|
threads = processors / trainers
|
|
|
|
|
else:
|
|
|
|
|
set_env("OMP_DYNAMIC", "false")
|
|
|
|
|
set_env("KMP_AFFINITY", "granularity=fine,compact,0,0")
|
|
|
|
|
threads = num_processors / trainer_count
|
|
|
|
|
threads = '1' if threads < 1 else str(threads)
|
|
|
|
|
set_env("OMP_NUM_THREADS", threads)
|
|
|
|
|
set_env("MKL_NUM_THREADS", threads)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def init(**kwargs):
|
|
|
|
|
import py_paddle.swig_paddle as api
|
|
|
|
|
args = []
|
|
|
|
|
args_dict = {}
|
|
|
|
|
# NOTE: append arguments if they are in ENV
|
|
|
|
|
for ek, ev in os.environ.iteritems():
|
|
|
|
|
if ek.startswith("PADDLE_INIT_"):
|
|
|
|
|
args_dict[ek.replace("PADDLE_INIT_", "").lower()] = str(ev)
|
|
|
|
|
|
|
|
|
|
args_dict.update(kwargs)
|
|
|
|
|
# NOTE: overwrite arguments from ENV if it is in kwargs
|
|
|
|
|
for key in args_dict.keys():
|
|
|
|
|
args.append('--%s=%s' % (key, str(args_dict[key])))
|
|
|
|
|
|
|
|
|
|
set_omp_mkl_env_vars(kwargs.get('trainer_count', 1))
|
|
|
|
|
|
|
|
|
|
if 'use_gpu' in kwargs:
|
|
|
|
|
cp.g_command_config_args['use_gpu'] = kwargs['use_gpu']
|
|
|
|
|
if 'use_mkldnn' in kwargs:
|
|
|
|
|