qijun
8 years ago
commit
4cc42171db
@ -0,0 +1,68 @@
|
|||||||
|
package master_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/PaddlePaddle/Paddle/go/master"
|
||||||
|
"github.com/coreos/etcd/clientv3"
|
||||||
|
"github.com/coreos/etcd/embed"
|
||||||
|
"github.com/docker/docker/pkg/ioutils"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewServiceWithEtcd(t *testing.T) {
|
||||||
|
// setup an embed etcd server
|
||||||
|
etcdDir, err := ioutils.TempDir("", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg := embed.NewConfig()
|
||||||
|
cfg.Dir = etcdDir
|
||||||
|
e, err := embed.StartEtcd(cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
e.Close()
|
||||||
|
if err := os.RemoveAll(etcdDir); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-e.Server.ReadyNotify():
|
||||||
|
t.Log("Server is ready!")
|
||||||
|
case <-time.After(60 * time.Second):
|
||||||
|
e.Server.Stop() // trigger a shutdown
|
||||||
|
t.Fatal("Server took too long to start!")
|
||||||
|
}
|
||||||
|
|
||||||
|
ep := []string{"127.0.0.1:2379"}
|
||||||
|
masterAddr := "127.0.0.1:3306"
|
||||||
|
store, err := master.NewEtcdClient(ep, masterAddr, master.DefaultLockPath, master.DefaultAddrPath, master.DefaultStatePath, 30)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = master.NewService(store, 10, 10, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cli, err := clientv3.New(clientv3.Config{
|
||||||
|
Endpoints: ep,
|
||||||
|
DialTimeout: 3 * time.Second,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
v, err := master.GetKey(cli, master.DefaultAddrPath, 3*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := cli.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// test master process registry itself into etcd server.
|
||||||
|
assert.Equal(t, masterAddr, v, "master process should registry itself into etcd server.")
|
||||||
|
}
|
||||||
@ -0,0 +1,142 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "paddle/memory/memcpy.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace framework {
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
inline void Tensor::check_memory_size() const {
|
||||||
|
PADDLE_ENFORCE(holder_ != nullptr,
|
||||||
|
"Tenosr holds no memory. Call Tensor::mutable_data first.");
|
||||||
|
PADDLE_ENFORCE(holder_->size() >= product(dims_) * sizeof(T) + offset_,
|
||||||
|
"Tensor's dims_ is out of bound. Call Tensor::mutable_data "
|
||||||
|
"first to re-allocate memory.");
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
inline const T* Tensor::data() const {
|
||||||
|
check_memory_size<T>();
|
||||||
|
return reinterpret_cast<const T*>(
|
||||||
|
reinterpret_cast<uintptr_t>(holder_->ptr()) + offset_);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
inline T* Tensor::data() {
|
||||||
|
check_memory_size<T>();
|
||||||
|
return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(holder_->ptr()) +
|
||||||
|
offset_);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
inline T* Tensor::mutable_data(DDim dims, platform::Place place) {
|
||||||
|
static_assert(std::is_pod<T>::value, "T must be POD");
|
||||||
|
Resize(dims);
|
||||||
|
return mutable_data<T>(place);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
inline T* Tensor::mutable_data(platform::Place place) {
|
||||||
|
static_assert(std::is_pod<T>::value, "T must be POD");
|
||||||
|
PADDLE_ENFORCE(product(dims_) > 0,
|
||||||
|
"Tensor's numel must be larger than zero to call "
|
||||||
|
"Tensor::mutable_data. Call Tensor::set_dim first.");
|
||||||
|
/* some versions of boost::variant don't have operator!= */
|
||||||
|
size_t size = product(dims_) * sizeof(T);
|
||||||
|
if (holder_ == nullptr || !(holder_->place() == place) ||
|
||||||
|
holder_->size() < size + offset_) {
|
||||||
|
if (platform::is_cpu_place(place)) {
|
||||||
|
holder_.reset(new PlaceholderImpl<T, platform::CPUPlace>(
|
||||||
|
boost::get<platform::CPUPlace>(place), size));
|
||||||
|
}
|
||||||
|
#ifndef PADDLE_ONLY_CPU
|
||||||
|
else if (platform::is_gpu_place(place)) {
|
||||||
|
holder_.reset(new PlaceholderImpl<T, platform::GPUPlace>(
|
||||||
|
boost::get<platform::GPUPlace>(place), size));
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
offset_ = 0;
|
||||||
|
}
|
||||||
|
return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(holder_->ptr()) +
|
||||||
|
offset_);
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
inline void Tensor::ShareDataWith(const Tensor& src) {
|
||||||
|
src.check_memory_size<T>();
|
||||||
|
*this = src;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
inline void Tensor::CopyFrom(const Tensor& src,
|
||||||
|
const platform::Place& dst_place) {
|
||||||
|
src.check_memory_size<T>();
|
||||||
|
Resize(src.dims());
|
||||||
|
|
||||||
|
auto src_place = src.holder_->place();
|
||||||
|
auto src_ptr = static_cast<const void*>(src.data<T>());
|
||||||
|
|
||||||
|
auto dst_ptr = static_cast<void*>(mutable_data<T>(dst_place));
|
||||||
|
|
||||||
|
auto size = product(src.dims_) * sizeof(T);
|
||||||
|
|
||||||
|
if (platform::is_cpu_place(src_place) && platform::is_cpu_place(dst_place)) {
|
||||||
|
memory::Copy(boost::get<platform::CPUPlace>(dst_place), dst_ptr,
|
||||||
|
boost::get<platform::CPUPlace>(src_place), src_ptr, size);
|
||||||
|
}
|
||||||
|
#ifndef PADDLE_ONLY_CPU
|
||||||
|
else if (platform::is_gpu_place(src_place) &&
|
||||||
|
platform::is_cpu_place(dst_place)) {
|
||||||
|
memory::Copy(boost::get<platform::CPUPlace>(dst_place), dst_ptr,
|
||||||
|
boost::get<platform::GPUPlace>(src_place), src_ptr, size, 0);
|
||||||
|
} else if (platform::is_cpu_place(src_place) &&
|
||||||
|
platform::is_gpu_place(dst_place)) {
|
||||||
|
memory::Copy(boost::get<platform::GPUPlace>(dst_place), dst_ptr,
|
||||||
|
boost::get<platform::CPUPlace>(src_place), src_ptr, size, 0);
|
||||||
|
} else if (platform::is_gpu_place(src_place) &&
|
||||||
|
platform::is_gpu_place(dst_place)) {
|
||||||
|
memory::Copy(boost::get<platform::GPUPlace>(dst_place), dst_ptr,
|
||||||
|
boost::get<platform::GPUPlace>(src_place), src_ptr, size, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
inline Tensor Tensor::Slice(const int& begin_idx, const int& end_idx) const {
|
||||||
|
check_memory_size<T>();
|
||||||
|
PADDLE_ENFORCE(begin_idx >= 0, "Slice begin index is less than zero.");
|
||||||
|
PADDLE_ENFORCE(end_idx <= dims_[0], "Slice end index is out of bound.");
|
||||||
|
PADDLE_ENFORCE(begin_idx < end_idx,
|
||||||
|
"Begin index must be less than end index.");
|
||||||
|
PADDLE_ENFORCE(dims_[0] != 1, "Can not slice a tensor with dims_[0] = 1.");
|
||||||
|
int base = product(dims_) / dims_[0];
|
||||||
|
Tensor dst;
|
||||||
|
dst.holder_ = holder_;
|
||||||
|
DDim dst_dims = dims_;
|
||||||
|
dst_dims[0] = end_idx - begin_idx;
|
||||||
|
dst.Resize(dst_dims);
|
||||||
|
dst.offset_ = offset_ + begin_idx * base * sizeof(T);
|
||||||
|
return dst;
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void Tensor::Resize(const DDim& dims) { dims_ = dims; }
|
||||||
|
|
||||||
|
inline const DDim& Tensor::dims() const { return dims_; }
|
||||||
|
|
||||||
|
} // namespace framework
|
||||||
|
} // namespace paddle
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue