From 5f17f05bb856697d4ce02190a99ae03e69201d3f Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 29 Jun 2018 11:29:26 +0800 Subject: [PATCH 01/15] Add get_version function to get version from git tags --- python/setup.py.in | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index 8257f1d5e2..83f98daa88 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,5 +1,6 @@ from setuptools import setup, Distribution, Extension import subprocess +import re class BinaryDistribution(Distribution): def has_ext_modules(foo): return True @@ -20,6 +21,40 @@ def git_commit(): git_commit = 'Unknown' return git_commit +def _get_version_detail(idx): + assert idx < 3 + + version_details = '${PADDLE_VERSION}'.split('.') + if len(version_details) == 3: + if re.match('[0-9]+', version_details[idx]): + return int(version_details[idx]) + + return None + +def get_minor(): + minor = _get_version_detail(0) + if minor is not None: + return minor + + return MINOR + +def get_major(): + major = _get_version_detail(1) + if major is not None: + return major + + return MAJOR + +def get_patch(): + patch = _get_version_detail(2) + if patch is not None: + return patch + + return PATCH + +def is_taged(): + return ISTAGED + def write_version_py(filename='paddle/version.py'): cnt = ''' # THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY @@ -49,13 +84,13 @@ def mkl(): commit = git_commit() with open(filename, 'w') as f: f.write(cnt % { - 'major': MAJOR, - 'minor': MINOR, - 'patch': PATCH, + 'major': get_major(), + 'minor': get_minor(), + 'patch': get_patch(), 'rc': RC, 'version': '${PADDLE_VERSION}', 'commit': commit, - 'istaged': ISTAGED, + 'istaged': is_taged(), 'with_mkl': '@WITH_MKL@'}) write_version_py(filename='@PADDLE_BINARY_DIR@/python/paddle/version.py') @@ -113,7 +148,7 @@ package_dir={ } if '${WITH_FLUID_ONLY}'== 'OFF': package_dir['py_paddle']='${PADDLE_BINARY_DIR}/python/py_paddle' - + paddle_rt_lib_dir = 'lib' paddle_rt_libs = ['${WARPCTC_LIBRARIES}'] From b28b885eb7b57faa0b8470ee7a70f71e7099b9ea Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 29 Jun 2018 12:48:30 +0800 Subject: [PATCH 02/15] Add is_tag function fo ISTAGED --- python/setup.py.in | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index 83f98daa88..38c79d8399 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -5,11 +5,7 @@ class BinaryDistribution(Distribution): def has_ext_modules(foo): return True -MAJOR = 0 -MINOR = 11 -PATCH = 0 RC = 0 -ISTAGED = False @@ -53,6 +49,14 @@ def get_patch(): return PATCH def is_taged(): + try: + # release/0.13.0 + # git describe --exact-match --tags + cmd = ['git', 'rev-parse', 'HEAD'] + git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE).communicate()[0].strip() + except: + git_commit = 'Unknown' + return git_commit return ISTAGED def write_version_py(filename='paddle/version.py'): From 712986e2fdeb8888ec3d96e9d14ae4f2d19f6260 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 29 Jun 2018 14:33:26 +0800 Subject: [PATCH 03/15] Remove the global version varibales like MAJOR and so on --- python/setup.py.in | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index 38c79d8399..a27eb5d582 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -20,7 +20,7 @@ def git_commit(): def _get_version_detail(idx): assert idx < 3 - version_details = '${PADDLE_VERSION}'.split('.') + version_details = '@PADDLE_VERSION@'.split('.') if len(version_details) == 3: if re.match('[0-9]+', version_details[idx]): return int(version_details[idx]) @@ -50,14 +50,15 @@ def get_patch(): def is_taged(): try: - # release/0.13.0 - # git describe --exact-match --tags - cmd = ['git', 'rev-parse', 'HEAD'] - git_commit = subprocess.Popen(cmd, stdout = subprocess.PIPE).communicate()[0].strip() + cmd = ['git', 'describe', '--exact-match', '--tags'] + git_tag = subprocess.Popen(cmd, stdout = subprocess.PIPE).communicate()[0].strip() except: - git_commit = 'Unknown' - return git_commit - return ISTAGED + return False + + if git_tag.replace('v', '') == '@PADDLE_VERSION@': + return True + else: + return False; def write_version_py(filename='paddle/version.py'): cnt = ''' From fd3940b6c8f450a2ce2953b8fa0e876027da13f9 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 29 Jun 2018 14:36:29 +0800 Subject: [PATCH 04/15] Change the sequence of major and minor --- python/setup.py.in | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index a27eb5d582..333c24f341 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -27,20 +27,20 @@ def _get_version_detail(idx): return None -def get_minor(): - minor = _get_version_detail(0) - if minor is not None: - return minor - - return MINOR - def get_major(): - major = _get_version_detail(1) + major = _get_version_detail(0) if major is not None: return major return MAJOR +def get_minor(): + minor = _get_version_detail(1) + if minor is not None: + return minor + + return MINOR + def get_patch(): patch = _get_version_detail(2) if patch is not None: From 037f9ae9333fed2b86a8e1a90afe6e8b91cc7cde Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 29 Jun 2018 14:39:11 +0800 Subject: [PATCH 05/15] Remove wrong end of line --- python/setup.py.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py.in b/python/setup.py.in index 333c24f341..081c12a5cf 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -58,7 +58,7 @@ def is_taged(): if git_tag.replace('v', '') == '@PADDLE_VERSION@': return True else: - return False; + return False def write_version_py(filename='paddle/version.py'): cnt = ''' From 078da5dee81af1258f84c6e4458e15a9c3429bcf Mon Sep 17 00:00:00 2001 From: minqiyang Date: Tue, 3 Jul 2018 14:20:05 +0800 Subject: [PATCH 06/15] Remove MAJOR, MINOR, PATCH and add comment for assert --- python/setup.py.in | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index 081c12a5cf..e6170a574a 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -18,7 +18,8 @@ def git_commit(): return git_commit def _get_version_detail(idx): - assert idx < 3 + assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \ + so detail index must less than 3" version_details = '@PADDLE_VERSION@'.split('.') if len(version_details) == 3: @@ -32,21 +33,21 @@ def get_major(): if major is not None: return major - return MAJOR + return 0 def get_minor(): minor = _get_version_detail(1) if minor is not None: return minor - return MINOR + return 0 def get_patch(): patch = _get_version_detail(2) if patch is not None: return patch - return PATCH + return 0 def is_taged(): try: From 9ea8b7ba2144e4c3d007a1a5a1ceecbe16b87bd7 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Tue, 3 Jul 2018 14:46:01 +0800 Subject: [PATCH 07/15] Change the default return value of version from 0 to UNKNOWN --- python/setup.py.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index e6170a574a..f10ff853ee 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -33,21 +33,21 @@ def get_major(): if major is not None: return major - return 0 + return 'UNKNOWN' def get_minor(): minor = _get_version_detail(1) if minor is not None: return minor - return 0 + return 'UNKNOWN' def get_patch(): patch = _get_version_detail(2) if patch is not None: return patch - return 0 + return 'UNKNOWN' def is_taged(): try: From add2e1b79ca3702b20a180bcd448468a2abd738e Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 6 Jul 2018 11:41:35 +0800 Subject: [PATCH 08/15] Merge variable name to detail --- python/setup.py.in | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/setup.py.in b/python/setup.py.in index 2b052574c0..db4c7f044d 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -25,8 +25,9 @@ def _get_version_detail(idx): version_details = '@PADDLE_VERSION@'.split('.') if len(version_details) == 3: - if re.match('[0-9]+', version_details[idx]): - return int(version_details[idx]) + detail = version_details[idx] + if re.match('[0-9]+', detail): + return int(detail) return None From 37a4322112520dc36748327bbf529ff51cf4b598 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 6 Jul 2018 13:57:16 +0800 Subject: [PATCH 09/15] Polish the code in setup.py.in Change the PADDLE_VERSION in develop branch to latest --- cmake/version.cmake | 9 +++++++-- python/setup.py.in | 30 +++++++++--------------------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/cmake/version.cmake b/cmake/version.cmake index cde650128a..96f2eff6cb 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -1,16 +1,21 @@ # Get the latest git tag. set(PADDLE_VERSION $ENV{PADDLE_VERSION}) set(tmp_version "HEAD") +set(TAG_VERSION_REGEX "[0-9]+\\.[0-9]+\\.[0-9]+(\\.(a|b|rc)\\.[0-9]+)?") +set(COMMIT_VERSION_REGEX "[0-9a-f]{5,40}") while ("${PADDLE_VERSION}" STREQUAL "") execute_process( - COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0 ${tmp_version} + COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0 --always ${tmp_version} WORKING_DIRECTORY ${PADDLE_SOURCE_DIR} OUTPUT_VARIABLE GIT_TAG_NAME RESULT_VARIABLE GIT_RESULT ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) if (NOT ${GIT_RESULT}) # Check the tag is a correct version - if (${GIT_TAG_NAME} MATCHES "v[0-9]+\\.[0-9]+\\.[0-9]+(\\.(a|b|rc)\\.[0-9]+)?") + if (${GIT_TAG_NAME} MATCHES "${COMMIT_VERSION_REGEX}") + # if no tag was found, set PADDLE_VERSION to latest + set(PADDLE_VERSION "latest") + elseif (${GIT_TAG_NAME} MATCHES "v${VERSION_REGEX}") string(REPLACE "v" "" PADDLE_VERSION ${GIT_TAG_NAME}) else() # otherwise, get the previous git tag name. set(tmp_version "${GIT_TAG_NAME}~1") diff --git a/python/setup.py.in b/python/setup.py.in index db4c7f044d..8ea828ce65 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -23,34 +23,22 @@ def _get_version_detail(idx): assert idx < 3, "vesion info consists of %(major)d.%(minor)d.%(patch)d, \ so detail index must less than 3" - version_details = '@PADDLE_VERSION@'.split('.') - if len(version_details) == 3: - detail = version_details[idx] - if re.match('[0-9]+', detail): - return int(detail) + if re.match('@TAG_VERSION_REGEX@', '@PADDLE_VERSION@'): + version_details = '@PADDLE_VERSION@'.split('.') - return None - -def get_major(): - major = _get_version_detail(0) - if major is not None: - return major + if len(version_details) == 3: + return version_details[idx] return 'UNKNOWN' -def get_minor(): - minor = _get_version_detail(1) - if minor is not None: - return minor +def get_major(): + return _get_version_detail(0) - return 'UNKNOWN' +def get_minor(): + return _get_version_detail(1) def get_patch(): - patch = _get_version_detail(2) - if patch is not None: - return patch - - return 'UNKNOWN' + return _get_version_detail(2) def is_taged(): try: From 1af7d5a2e80f1abe12fcdbbc81262ceb287e051b Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 6 Jul 2018 15:03:39 +0800 Subject: [PATCH 10/15] Change the incorrect version result from UNKNOWN to 0 Replace {} to + in cmake regex match --- cmake/version.cmake | 4 ++-- python/setup.py.in | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/version.cmake b/cmake/version.cmake index 96f2eff6cb..4a86ad2e09 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -2,7 +2,7 @@ set(PADDLE_VERSION $ENV{PADDLE_VERSION}) set(tmp_version "HEAD") set(TAG_VERSION_REGEX "[0-9]+\\.[0-9]+\\.[0-9]+(\\.(a|b|rc)\\.[0-9]+)?") -set(COMMIT_VERSION_REGEX "[0-9a-f]{5,40}") +set(COMMIT_VERSION_REGEX "[0-9a-f]+") while ("${PADDLE_VERSION}" STREQUAL "") execute_process( COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0 --always ${tmp_version} @@ -15,7 +15,7 @@ while ("${PADDLE_VERSION}" STREQUAL "") if (${GIT_TAG_NAME} MATCHES "${COMMIT_VERSION_REGEX}") # if no tag was found, set PADDLE_VERSION to latest set(PADDLE_VERSION "latest") - elseif (${GIT_TAG_NAME} MATCHES "v${VERSION_REGEX}") + elseif (${GIT_TAG_NAME} MATCHES "v${TAG_VERSION_REGEX}") string(REPLACE "v" "" PADDLE_VERSION ${GIT_TAG_NAME}) else() # otherwise, get the previous git tag name. set(tmp_version "${GIT_TAG_NAME}~1") diff --git a/python/setup.py.in b/python/setup.py.in index 8ea828ce65..6e21512b86 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -29,7 +29,7 @@ def _get_version_detail(idx): if len(version_details) == 3: return version_details[idx] - return 'UNKNOWN' + return 0 def get_major(): return _get_version_detail(0) From fe49e4690450b2615519a86e4a1f79a1dfe4a8d9 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 6 Jul 2018 15:59:29 +0800 Subject: [PATCH 11/15] Fix the problem that CMake do not support {} regex Change patch version to str --- cmake/version.cmake | 2 +- python/setup.py.in | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/version.cmake b/cmake/version.cmake index 4a86ad2e09..79b8e8ac49 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -2,7 +2,7 @@ set(PADDLE_VERSION $ENV{PADDLE_VERSION}) set(tmp_version "HEAD") set(TAG_VERSION_REGEX "[0-9]+\\.[0-9]+\\.[0-9]+(\\.(a|b|rc)\\.[0-9]+)?") -set(COMMIT_VERSION_REGEX "[0-9a-f]+") +set(COMMIT_VERSION_REGEX "[0-9a-f]+[0-9a-f]+[0-9a-f]+[0-9a-f]+[0-9a-f]+") while ("${PADDLE_VERSION}" STREQUAL "") execute_process( COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0 --always ${tmp_version} diff --git a/python/setup.py.in b/python/setup.py.in index 6e21512b86..0fd676a702 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -32,13 +32,13 @@ def _get_version_detail(idx): return 0 def get_major(): - return _get_version_detail(0) + return int(_get_version_detail(0)) def get_minor(): - return _get_version_detail(1) + return int(_get_version_detail(1)) def get_patch(): - return _get_version_detail(2) + return str(_get_version_detail(2)) def is_taged(): try: @@ -56,10 +56,10 @@ def write_version_py(filename='paddle/version.py'): cnt = ''' # THIS FILE IS GENERATED FROM PADDLEPADDLE SETUP.PY # -full_version = '%(major)d.%(minor)d.%(patch)d' +full_version = '%(major)d.%(minor)d.%(patch)s' major = '%(major)d' minor = '%(minor)d' -patch = '%(patch)d' +patch = '%(patch)s' rc = '%(rc)d' istaged = %(istaged)s commit = '%(commit)s' From 1d8cbc1738096fd54a3a7a6f229f6fe60248660f Mon Sep 17 00:00:00 2001 From: minqiyang Date: Mon, 9 Jul 2018 13:55:52 +0800 Subject: [PATCH 12/15] Change develop to latest and other branch will get the tag to decide which version it is staying on --- cmake/version.cmake | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmake/version.cmake b/cmake/version.cmake index 79b8e8ac49..f95cc40d19 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -11,6 +11,20 @@ while ("${PADDLE_VERSION}" STREQUAL "") RESULT_VARIABLE GIT_RESULT ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) if (NOT ${GIT_RESULT}) + # Get current branch name + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref ${tmp_version} + WORKING_DIRECTORY ${PADDLE_SOURCE_DIR} + OUTPUT_VARIABLE GIT_BRANCH_NAME + RESULT_VARIABLE GIT_BRANCH_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + # If current branch is develop, then version will be latest + if (NOT ${GIT_BRANCH_RESULT}) + if (${GIT_BRANCH_NAME} MATCHES "develop") + set(PADDLE_VERSION "latest") + continue() + endif() + endif() # Check the tag is a correct version if (${GIT_TAG_NAME} MATCHES "${COMMIT_VERSION_REGEX}") # if no tag was found, set PADDLE_VERSION to latest From b8ff38ae7a21fb861b7733bf9fd60505dd37cc5e Mon Sep 17 00:00:00 2001 From: minqiyang Date: Mon, 9 Jul 2018 14:22:30 +0800 Subject: [PATCH 13/15] Remove the new git branching model from this pr --- cmake/version.cmake | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/cmake/version.cmake b/cmake/version.cmake index f95cc40d19..79b8e8ac49 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -11,20 +11,6 @@ while ("${PADDLE_VERSION}" STREQUAL "") RESULT_VARIABLE GIT_RESULT ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) if (NOT ${GIT_RESULT}) - # Get current branch name - execute_process( - COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref ${tmp_version} - WORKING_DIRECTORY ${PADDLE_SOURCE_DIR} - OUTPUT_VARIABLE GIT_BRANCH_NAME - RESULT_VARIABLE GIT_BRANCH_RESULT - ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - # If current branch is develop, then version will be latest - if (NOT ${GIT_BRANCH_RESULT}) - if (${GIT_BRANCH_NAME} MATCHES "develop") - set(PADDLE_VERSION "latest") - continue() - endif() - endif() # Check the tag is a correct version if (${GIT_TAG_NAME} MATCHES "${COMMIT_VERSION_REGEX}") # if no tag was found, set PADDLE_VERSION to latest From 091ab6333127f1f5b8b75a53f1f0b94b0b41d479 Mon Sep 17 00:00:00 2001 From: gongweibao Date: Mon, 9 Jul 2018 06:57:35 -0500 Subject: [PATCH 14/15] Fix singleton. (#11835) --- paddle/fluid/framework/op_info.cc | 4 +- paddle/fluid/memory/detail/buddy_allocator.cc | 5 +- paddle/fluid/memory/detail/buddy_allocator.h | 7 +- paddle/fluid/memory/malloc.cc | 69 +++++++++++-------- 4 files changed, 51 insertions(+), 34 deletions(-) diff --git a/paddle/fluid/framework/op_info.cc b/paddle/fluid/framework/op_info.cc index f1261dee03..af75baa5c4 100644 --- a/paddle/fluid/framework/op_info.cc +++ b/paddle/fluid/framework/op_info.cc @@ -21,8 +21,8 @@ namespace framework { // a static local variable is already being initialized. // https://stackoverflow.com/questions/11711920/how-to-implement-multithread-safe-singleton-in-c11-without-using-mutex OpInfoMap& OpInfoMap::Instance() { - static OpInfoMap* g_op_info_map = new OpInfoMap(); - return *g_op_info_map; + static OpInfoMap g_op_info_map; + return g_op_info_map; } } // namespace framework } // namespace paddle diff --git a/paddle/fluid/memory/detail/buddy_allocator.cc b/paddle/fluid/memory/detail/buddy_allocator.cc index 4194ba1979..01a8501dd4 100644 --- a/paddle/fluid/memory/detail/buddy_allocator.cc +++ b/paddle/fluid/memory/detail/buddy_allocator.cc @@ -19,8 +19,9 @@ namespace paddle { namespace memory { namespace detail { -BuddyAllocator::BuddyAllocator(SystemAllocator* system_allocator, - size_t min_chunk_size, size_t max_chunk_size) +BuddyAllocator::BuddyAllocator( + std::unique_ptr system_allocator, size_t min_chunk_size, + size_t max_chunk_size) : min_chunk_size_(min_chunk_size), max_chunk_size_(max_chunk_size), cache_(system_allocator->UseGpu()), diff --git a/paddle/fluid/memory/detail/buddy_allocator.h b/paddle/fluid/memory/detail/buddy_allocator.h index 2f39d774d6..f0c83efc23 100644 --- a/paddle/fluid/memory/detail/buddy_allocator.h +++ b/paddle/fluid/memory/detail/buddy_allocator.h @@ -14,6 +14,7 @@ limitations under the License. */ #pragma once +#include #include // NOLINT #include #include @@ -32,8 +33,8 @@ namespace detail { class BuddyAllocator { public: - BuddyAllocator(SystemAllocator* system_allocator, size_t min_chunk_size, - size_t max_chunk_size); + BuddyAllocator(std::unique_ptr system_allocator, + size_t min_chunk_size, size_t max_chunk_size); ~BuddyAllocator(); @@ -103,7 +104,7 @@ class BuddyAllocator { private: /*! Allocate CPU/GPU memory from system */ - SystemAllocator* system_allocator_; + std::unique_ptr system_allocator_; std::mutex mutex_; }; diff --git a/paddle/fluid/memory/malloc.cc b/paddle/fluid/memory/malloc.cc index bd98ed8189..7c800b3c16 100644 --- a/paddle/fluid/memory/malloc.cc +++ b/paddle/fluid/memory/malloc.cc @@ -12,6 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include + #include "paddle/fluid/memory/malloc.h" #include "glog/logging.h" @@ -34,12 +36,15 @@ namespace memory { using BuddyAllocator = detail::BuddyAllocator; BuddyAllocator* GetCPUBuddyAllocator() { + static std::once_flag init_flag; static detail::BuddyAllocator* a = nullptr; - if (a == nullptr) { - a = new detail::BuddyAllocator(new detail::CPUAllocator, - platform::CpuMinChunkSize(), - platform::CpuMaxChunkSize()); - } + + std::call_once(init_flag, []() { + a = new detail::BuddyAllocator( + std::unique_ptr(new detail::CPUAllocator), + platform::CpuMinChunkSize(), platform::CpuMaxChunkSize()); + }); + return a; } @@ -68,27 +73,33 @@ size_t Used(platform::CPUPlace place) { #ifdef PADDLE_WITH_CUDA BuddyAllocator* GetGPUBuddyAllocator(int gpu_id) { - static BuddyAllocator** as = NULL; - if (as == NULL) { + static std::once_flag init_flag; + static detail::BuddyAllocator** a_arr = nullptr; + + std::call_once(init_flag, [gpu_id]() { int gpu_num = platform::GetCUDADeviceCount(); - as = new BuddyAllocator*[gpu_num]; - for (int gpu = 0; gpu < gpu_num; gpu++) { - as[gpu] = nullptr; + PADDLE_ENFORCE(gpu_id < gpu_num, "gpu_id:%d should < gpu_num:%d", gpu_id, + gpu_num); + + a_arr = new BuddyAllocator*[gpu_num]; + for (int i = 0; i < gpu_num; i++) { + a_arr[i] = nullptr; + platform::SetDeviceId(i); + a_arr[i] = new BuddyAllocator( + std::unique_ptr(new detail::GPUAllocator(i)), + platform::GpuMinChunkSize(), platform::GpuMaxChunkSize()); + + VLOG(10) << "\n\nNOTE: each GPU device use " + << FLAGS_fraction_of_gpu_memory_to_use * 100 + << "% of GPU memory.\n" + << "You can set GFlags environment variable '" + << "FLAGS_fraction_of_gpu_memory_to_use" + << "' to change the fraction of GPU usage.\n\n"; } - } + }); + platform::SetDeviceId(gpu_id); - if (!as[gpu_id]) { - as[gpu_id] = new BuddyAllocator(new detail::GPUAllocator(gpu_id), - platform::GpuMinChunkSize(), - platform::GpuMaxChunkSize()); - VLOG(10) << "\n\nNOTE: each GPU device use " - << FLAGS_fraction_of_gpu_memory_to_use * 100 - << "% of GPU memory.\n" - << "You can set GFlags environment variable '" - << "FLAGS_fraction_of_gpu_memory_to_use" - << "' to change the fraction of GPU usage.\n\n"; - } - return as[gpu_id]; + return a_arr[gpu_id]; } template <> @@ -125,12 +136,16 @@ void Free(platform::CUDAPlace place, void* p) { } BuddyAllocator* GetCUDAPinnedBuddyAllocator() { - static BuddyAllocator* ba = NULL; - if (ba == NULL) { - ba = new BuddyAllocator(new detail::CUDAPinnedAllocator, + static std::once_flag init_flag; + static BuddyAllocator* ba = nullptr; + + std::call_once(init_flag, []() { + ba = new BuddyAllocator(std::unique_ptr( + new detail::CUDAPinnedAllocator), platform::CUDAPinnedMinChunkSize(), platform::CUDAPinnedMaxChunkSize()); - } + }); + return ba; } From ef4895df3bc247c5449a9a0a524839561225bd56 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Tue, 10 Jul 2018 09:39:47 +0800 Subject: [PATCH 15/15] Make IfElse operator works and fix unit testing. (#11972) 1. Fix bug when only true or false branch works. 2. Fix bug in unit testing. --- .../fluid/operators/conditional_block_op.cc | 7 +- paddle/fluid/operators/merge_lod_tensor_op.cc | 30 ++++--- ...mnist_if_else_op.py => test_if_else_op.py} | 84 ++++++++++++++++--- 3 files changed, 96 insertions(+), 25 deletions(-) rename python/paddle/fluid/tests/{test_mnist_if_else_op.py => test_if_else_op.py} (66%) diff --git a/paddle/fluid/operators/conditional_block_op.cc b/paddle/fluid/operators/conditional_block_op.cc index 8cc1d94260..580fde7538 100644 --- a/paddle/fluid/operators/conditional_block_op.cc +++ b/paddle/fluid/operators/conditional_block_op.cc @@ -205,9 +205,10 @@ class ConditionalBlockGradInferShape : public framework::InferShapeBase { context->SetOutputsDim(framework::GradVarName("Params"), context->GetInputsDim("Params")); } - PADDLE_ENFORCE(context->HasOutputs(framework::GradVarName("X"))); - context->SetOutputsDim(framework::GradVarName("X"), - context->GetInputsDim("X")); + if (context->HasOutputs(framework::GradVarName("X"))) { + context->SetOutputsDim(framework::GradVarName("X"), + context->GetInputsDim("X")); + } } }; diff --git a/paddle/fluid/operators/merge_lod_tensor_op.cc b/paddle/fluid/operators/merge_lod_tensor_op.cc index a16861b3b7..2dc1467b0d 100644 --- a/paddle/fluid/operators/merge_lod_tensor_op.cc +++ b/paddle/fluid/operators/merge_lod_tensor_op.cc @@ -44,8 +44,10 @@ class MergeLoDTensorOp : public framework::OperatorBase { scope.FindVar(Output("Out"))->GetMutable(); auto level = static_cast(Attr("level")); - auto &mask_dim = mask.dims(); + PADDLE_ENFORCE(in_true.numel() || in_false.numel(), + "Input(InTrue) or Input(InFalse) should be initialized."); + auto &mask_dim = mask.dims(); std::unique_ptr cpu_mask{new framework::LoDTensor()}; if (platform::is_cpu_place(mask.place())) { cpu_mask->ShareDataWith(mask); @@ -59,19 +61,27 @@ class MergeLoDTensorOp : public framework::OperatorBase { } auto *mask_data = cpu_mask->data(); - int rank = in_true.dims().size(); - platform::Place place = in_true.place(); - std::type_index data_type = in_true.type(); - framework::DDim in_true_dims = - framework::slice_ddim(in_true.dims(), 1, rank); - + platform::Place place = dev_place; int64_t batch_size = in_true.dims()[0] + in_false.dims()[0]; - auto in_true_dim_vec = framework::vectorize(in_true_dims); - in_true_dim_vec.insert(in_true_dim_vec.begin(), batch_size); + std::type_index data_type = + in_true.IsInitialized() ? in_true.type() : in_false.type(); + int rank; + framework::DDim in_dims; + if (in_true.IsInitialized()) { + rank = in_true.dims().size(); + in_dims = framework::slice_ddim(in_true.dims(), 1, rank); + } else { + rank = in_false.dims().size(); + in_dims = framework::slice_ddim(in_false.dims(), 1, rank); + } + + auto in_dim_vec = framework::vectorize(in_dims); + in_dim_vec.insert(in_dim_vec.begin(), batch_size); - framework::DDim out_dims = framework::make_ddim(in_true_dim_vec); + framework::DDim out_dims = framework::make_ddim(in_dim_vec); out->Resize(out_dims); + out->mutable_data(place, data_type); auto *out_lod = out->mutable_lod(); diff --git a/python/paddle/fluid/tests/test_mnist_if_else_op.py b/python/paddle/fluid/tests/test_if_else_op.py similarity index 66% rename from python/paddle/fluid/tests/test_mnist_if_else_op.py rename to python/paddle/fluid/tests/test_if_else_op.py index d34f52db5f..1b58925599 100644 --- a/python/paddle/fluid/tests/test_mnist_if_else_op.py +++ b/python/paddle/fluid/tests/test_if_else_op.py @@ -14,10 +14,11 @@ import paddle import paddle.fluid.layers as layers -from paddle.fluid.framework import Program, program_guard, default_main_program, default_startup_program +from paddle.fluid.framework import Program, program_guard from paddle.fluid.executor import Executor from paddle.fluid.optimizer import MomentumOptimizer import paddle.fluid.core as core +import paddle.fluid as fluid import unittest import numpy as np @@ -31,14 +32,13 @@ class TestMNISTIfElseOp(unittest.TestCase): label = layers.data(name='y', shape=[1], dtype='int64') - limit = layers.fill_constant_batch_size_like( - input=label, dtype='int64', shape=[1], value=5.0) + limit = layers.fill_constant(shape=[1], dtype='int64', value=5) cond = layers.less_than(x=label, y=limit) true_image, false_image = layers.split_lod_tensor( input=image, mask=cond) true_out = layers.create_tensor(dtype='float32') - true_cond = layers.ConditionalBlock([true_image]) + true_cond = layers.ConditionalBlock([cond]) with true_cond.block(): hidden = layers.fc(input=true_image, size=100, act='tanh') @@ -46,7 +46,7 @@ class TestMNISTIfElseOp(unittest.TestCase): layers.assign(input=prob, output=true_out) false_out = layers.create_tensor(dtype='float32') - false_cond = layers.ConditionalBlock([false_image]) + false_cond = layers.ConditionalBlock([cond]) with false_cond.block(): hidden = layers.fc(input=false_image, size=200, act='tanh') @@ -64,7 +64,7 @@ class TestMNISTIfElseOp(unittest.TestCase): train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=8192), - batch_size=200) + batch_size=10) place = core.CPUPlace() exe = Executor(place) @@ -94,8 +94,7 @@ class TestMNISTIfElseOp(unittest.TestCase): label = layers.data(name='y', shape=[1], dtype='int64') - limit = layers.fill_constant_batch_size_like( - input=label, dtype='int64', shape=[1], value=5.0) + limit = layers.fill_constant(shape=[1], dtype='int64', value=5) cond = layers.less_than(x=label, y=limit) ie = layers.IfElse(cond) @@ -125,7 +124,7 @@ class TestMNISTIfElseOp(unittest.TestCase): place = core.CPUPlace() exe = Executor(place) - exe.run(kwargs['startup_program']) + exe.run(startup_prog) PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): @@ -133,7 +132,7 @@ class TestMNISTIfElseOp(unittest.TestCase): y_data = np.array(map(lambda x: x[1], data)).astype("int64") y_data = y_data.reshape((y_data.shape[0], 1)) - outs = exe.run(kwargs['main_program'], + outs = exe.run(prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss]) @@ -143,6 +142,67 @@ class TestMNISTIfElseOp(unittest.TestCase): self.assertFalse(True) +class TestIfElse(unittest.TestCase): + def set_test_case(self): + # condiction is: self.data < self.cond_value + self.cond_value = 0.5 + self.data = np.random.rand(25, 1).astype(np.float32) + + def compare_ifelse_op_and_numpy(self, place): + self.set_test_case() + + prog = Program() + startup_prog = Program() + with program_guard(prog, startup_prog): + src = layers.data(name='data', shape=[1], dtype='float32') + cond = layers.fill_constant( + [1], dtype='float32', value=self.cond_value) + ifcond = layers.less_than(x=src, y=cond) + ie = layers.IfElse(ifcond) + with ie.true_block(): + true_target = ie.input(src) + ie.output(true_target) + + with ie.false_block(): + false_target = ie.input(src) + ie.output(false_target) + if_out = ie() + out = layers.reduce_sum(if_out) + + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) + fetch_list = [out] + o1, = exe.run(fluid.default_main_program(), + feed={'data': self.data}, + fetch_list=[out]) + o2 = np.sum(self.data) + self.assertTrue( + np.allclose( + o1, o2, atol=1e-8), + "IfElse result : " + str(o1) + "\n Numpy result :" + str(o2)) + + def test_cpu(self): + self.compare_ifelse_op_and_numpy(fluid.CPUPlace()) + + def test_cuda(self): + if not core.is_compiled_with_cuda(): + return + self.compare_ifelse_op_and_numpy(fluid.CUDAPlace(0)) + + +class TestIfElseTrueBranch(TestIfElse): + def set_test_case(self): + # condiction is: self.data < self.cond_value + self.cond_value = 10. + self.data = np.random.rand(25, 1).astype(np.float32) + + +class TestIfElseFalseBranch(TestIfElse): + def set_test_case(self): + # condiction is: self.data < self.cond_value + self.cond_value = -10. + self.data = np.random.rand(25, 1).astype(np.float32) + + if __name__ == '__main__': - # temp disable if else unittest since it could be buggy. - exit(0) + unittest.main()