[Feature] one ps (3/4) (#29604)
* oneps (3/4) Co-authored-by: MrChengmo <cmchengmo@163.com> Co-authored-by: malin10 <malin10@baidu.com> Co-authored-by: chengmo <chengmo@baidu.com>revert-31562-mean
tangwei12
5 years ago
committed by
GitHub
parent
edc06c6a1b
commit
032414ca2a
@ -0,0 +1,71 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
include (ExternalProject)
|
||||
|
||||
# NOTE: snappy is needed when linking with recordio
|
||||
|
||||
set(SNAPPY_SOURCES_DIR ${THIRD_PARTY_PATH}/snappy)
|
||||
set(SNAPPY_INSTALL_DIR ${THIRD_PARTY_PATH}/install/snappy)
|
||||
set(SNAPPY_INCLUDE_DIR "${SNAPPY_INSTALL_DIR}/include" CACHE PATH "snappy include directory." FORCE)
|
||||
|
||||
if(WIN32)
|
||||
SET(SNAPPY_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4267")
|
||||
else()
|
||||
SET(SNAPPY_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
|
||||
endif()
|
||||
|
||||
ExternalProject_Add(
|
||||
extern_snappy
|
||||
GIT_REPOSITORY "https://github.com/google/snappy"
|
||||
GIT_TAG "1.1.7"
|
||||
PREFIX ${SNAPPY_SOURCES_DIR}
|
||||
UPDATE_COMMAND ""
|
||||
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
|
||||
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
|
||||
-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
|
||||
-DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
|
||||
-DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
|
||||
-DCMAKE_CXX_FLAGS=${SNAPPY_CMAKE_CXX_FLAGS}
|
||||
-DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
|
||||
-DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
|
||||
-DCMAKE_INSTALL_PREFIX=${SNAPPY_INSTALL_DIR}
|
||||
-DCMAKE_INSTALL_LIBDIR=${SNAPPY_INSTALL_DIR}/lib
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
|
||||
-DBUILD_TESTING=OFF
|
||||
-DSNAPPY_BUILD_TESTS:BOOL=OFF
|
||||
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
|
||||
${EXTERNAL_OPTIONAL_ARGS}
|
||||
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_INSTALL_DIR}
|
||||
-DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_INSTALL_DIR}/lib
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
|
||||
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
|
||||
)
|
||||
IF(WIN32)
|
||||
IF(NOT EXISTS "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib")
|
||||
add_custom_command(TARGET extern_snappy POST_BUILD
|
||||
COMMAND cmake -E copy ${SNAPPY_INSTALL_DIR}/lib/snappy.lib ${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib
|
||||
)
|
||||
ENDIF()
|
||||
set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib")
|
||||
else(WIN32)
|
||||
set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.a")
|
||||
endif (WIN32)
|
||||
|
||||
add_library(snappy STATIC IMPORTED GLOBAL)
|
||||
set_property(TARGET snappy PROPERTY IMPORTED_LOCATION ${SNAPPY_LIBRARIES})
|
||||
|
||||
include_directories(${SNAPPY_INCLUDE_DIR})
|
||||
add_dependencies(snappy extern_snappy)
|
||||
|
||||
@ -0,0 +1,71 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include <ThreadPool.h>
|
||||
|
||||
#include <unistd.h>
|
||||
#include <string>
|
||||
#include <thread> // NOLINT
|
||||
|
||||
#include "google/protobuf/text_format.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "paddle/fluid/distributed/ps.pb.h"
|
||||
#include "paddle/fluid/distributed/table/common_sparse_table.h"
|
||||
#include "paddle/fluid/distributed/table/depends/large_scale_kv.h"
|
||||
#include "paddle/fluid/distributed/table/table.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace distributed {
|
||||
|
||||
TEST(BENCHMARK, LargeScaleKV) {
|
||||
int emb_dim = 10;
|
||||
int trainers = 2;
|
||||
float beta1 = 0.9;
|
||||
float beta2 = 0.999;
|
||||
float epsilon = 1.0e-8;
|
||||
|
||||
TableParameter table_config;
|
||||
table_config.set_table_class("CommonSparseTable");
|
||||
FsClientParameter fs_config;
|
||||
Table *table = new CommonSparseTable();
|
||||
TableAccessorParameter *accessor_config = table_config.mutable_accessor();
|
||||
accessor_config->set_accessor_class("CommMergeAccessor");
|
||||
CommonAccessorParameter *common_config = table_config.mutable_common();
|
||||
common_config->set_name("adam");
|
||||
common_config->set_table_name("adam_test_table");
|
||||
common_config->set_trainer_num(trainers);
|
||||
common_config->add_params("Param");
|
||||
common_config->add_dims(emb_dim);
|
||||
common_config->add_initializers("uniform_random&0&-1.0&1.0");
|
||||
common_config->add_params("LearningRate");
|
||||
common_config->add_dims(1);
|
||||
common_config->add_initializers("fill_constant&1.0");
|
||||
common_config->add_params("Moment1");
|
||||
common_config->add_dims(emb_dim);
|
||||
common_config->add_initializers("fill_constant&0.0");
|
||||
common_config->add_params("Moment2");
|
||||
common_config->add_dims(emb_dim);
|
||||
common_config->add_initializers("fill_constant&0.0");
|
||||
common_config->add_params("Beta1Pow");
|
||||
common_config->add_dims(1);
|
||||
common_config->add_initializers("fill_constant&1.0");
|
||||
common_config->add_params("Beta2Pow");
|
||||
common_config->add_dims(1);
|
||||
common_config->add_initializers("fill_constant&1.0");
|
||||
auto ret = table->initialize(table_config, fs_config);
|
||||
ASSERT_EQ(ret, 0);
|
||||
}
|
||||
|
||||
} // namespace distributed
|
||||
} // namespace paddle
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue