Yancey1989
7 years ago
parent
d67b9ced05
commit
e8d802159e
@ -0,0 +1,154 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "paddle/fluid/framework/data_type.h"
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/operators/math/math_function.h"
|
||||
#include "paddle/fluid/platform/device_context.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
constexpr int64_t kNoPadding = -1;
|
||||
|
||||
class LookupSparseTableInferShape : public framework::InferShapeBase {
|
||||
public:
|
||||
void operator()(framework::InferShapeContext *ctx) const override {
|
||||
PADDLE_ENFORCE(ctx->HasOutput("Out"),
|
||||
"Output(Out) of LookupSparseTableOp should not be null.");
|
||||
auto shape_w = ctx->GetInputDim("W");
|
||||
auto shape_ids = ctx->GetInputDim("Ids");
|
||||
shape_w[0] = shape_ids.size();
|
||||
ctx->SetOutputDim("Out", shape_w);
|
||||
}
|
||||
};
|
||||
|
||||
class LookupSparseTableOp : public framework::OperatorBase {
|
||||
public:
|
||||
using framework::OperatorBase::OperatorBase;
|
||||
|
||||
private:
|
||||
void RunImpl(const framework::Scope &scope,
|
||||
const platform::Place &dev_place) const override {
|
||||
auto out_var = scope.FindVar(Output("Out"));
|
||||
auto w_var = scope.FindVar(Input("W"));
|
||||
auto ids_var = scope.FindVar(Input("Ids"));
|
||||
unsigned int seed = static_cast<unsigned int>(Attr<int>("seed"));
|
||||
float min = Attr<float>("min");
|
||||
float max = Attr<float>("max");
|
||||
|
||||
PADDLE_ENFORCE(out_var->IsType<framework::LoDTensor>(),
|
||||
"The type of Out var should be LodTensor.");
|
||||
PADDLE_ENFORCE(w_var->IsType<framework::SelectedRows>(),
|
||||
"The type of W var should be SelectedRows.");
|
||||
PADDLE_ENFORCE(ids_var->IsType<framework::SelectedRows>(),
|
||||
"The type of Ids var should be SelectedRows.");
|
||||
auto &ids_t = ids_var->Get<framework::SelectedRows>();
|
||||
auto out_t = out_var->GetMutable<framework::LoDTensor>();
|
||||
auto w_t = w_var->GetMutable<framework::SelectedRows>();
|
||||
auto keys = ids_t.rows();
|
||||
|
||||
// TODO(Yancey1989): support CUDA Place for the sparse table
|
||||
platform::CPUPlace cpu;
|
||||
auto out_shape = w_t->value().dims();
|
||||
out_shape[0] = keys.size();
|
||||
out_t->Resize(out_shape);
|
||||
out_t->mutable_data(cpu, w_t->value().type());
|
||||
|
||||
PADDLE_ENFORCE_EQ(framework::ToDataType(w_t->value().type()),
|
||||
framework::proto::VarType::FP32,
|
||||
"The sparse table only support FP32");
|
||||
|
||||
auto non_keys_pair = w_t->Get(keys, out_t);
|
||||
auto value_shape = w_t->value().dims();
|
||||
value_shape[0] = 1;
|
||||
for (const auto &it : non_keys_pair) {
|
||||
const auto key = it.first;
|
||||
const auto index = it.second;
|
||||
framework::Tensor value;
|
||||
value.Resize(value_shape);
|
||||
auto data = value.mutable_data<float>(cpu);
|
||||
|
||||
std::minstd_rand engine;
|
||||
engine.seed(seed);
|
||||
std::uniform_real_distribution<float> dist(min, max);
|
||||
int64_t size = value.numel();
|
||||
for (int64_t i = 0; i < size; ++i) {
|
||||
data[i] = dist(engine);
|
||||
}
|
||||
w_t->Set(key, value);
|
||||
memory::Copy(cpu, out_t->mutable_data<float>(cpu) + index * value.numel(),
|
||||
cpu, value.data<float>(), value.numel() * sizeof(float));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class LookupSparseTableOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
LookupSparseTableOpMaker(OpProto *proto, OpAttrChecker *op_checker)
|
||||
: framework::OpProtoAndCheckerMaker(proto, op_checker) {
|
||||
AddInput("W",
|
||||
"(SelectedRows) The input represents embedding table, "
|
||||
"which is a learnable parameter.");
|
||||
AddInput("Ids",
|
||||
"(SelectedRows) Ids's type should be SelectedRows "
|
||||
"the rows of Ids contains the Ids to be looked up in W.");
|
||||
AddOutput("Out",
|
||||
"(SelectedRows) The lookup results, which have the "
|
||||
"same type as W.");
|
||||
AddAttr<int64_t>("padding_idx",
|
||||
"(int64, default -1) "
|
||||
"If the value is -1, it makes no effect to lookup. "
|
||||
"Otherwise the given value indicates padding the output "
|
||||
"with zeros whenever lookup encounters it in Ids.")
|
||||
.SetDefault(kNoPadding);
|
||||
AddAttr<float>("min",
|
||||
"(float, default -1.0) "
|
||||
"Minimum value of uniform random")
|
||||
.SetDefault(-1.0f);
|
||||
AddAttr<float>("max",
|
||||
"(float, default 1.0) "
|
||||
"Maximun value of uniform random")
|
||||
.SetDefault(1.0f);
|
||||
AddAttr<int>("seed",
|
||||
"(int, default 0) "
|
||||
"Random seed used for generating samples. "
|
||||
"0 means use a seed generated by the system."
|
||||
"Note that if seed is not 0, this operator will always "
|
||||
"generate the same random numbers every time.")
|
||||
.SetDefault(0);
|
||||
AddComment(R"DOC(
|
||||
Lookup Sprase Tablel Operator.
|
||||
|
||||
This operator is used to perform lookup on parameter W,
|
||||
then concatenated into a sparse tensor.
|
||||
|
||||
The type of Ids(Input) is SelectedRows, the rows of Ids contains
|
||||
the ids to be looked up in W;
|
||||
if the Id is not in the sparse table, this operator will return a
|
||||
random value and set the value into the table for the next looking up.
|
||||
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
REGISTER_OPERATOR(lookup_sparse_table, ops::LookupSparseTableOp,
|
||||
ops::LookupSparseTableInferShape,
|
||||
ops::LookupSparseTableOpMaker,
|
||||
paddle::framework::EmptyGradOpMaker);
|
||||
@ -0,0 +1,86 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
import numpy as np
|
||||
from op_test import OpTest
|
||||
import paddle.fluid.core as core
|
||||
from paddle.fluid.op import Operator
|
||||
|
||||
|
||||
def output_hist(out):
|
||||
hist, _ = np.histogram(out, range=(-5, 10))
|
||||
hist = hist.astype("float32")
|
||||
hist /= float(out.size)
|
||||
prob = 0.1 * np.ones((10))
|
||||
return hist, prob
|
||||
|
||||
|
||||
class TestLookupSpraseTable(OpTest):
|
||||
def check_with_place(self, place):
|
||||
scope = core.Scope()
|
||||
|
||||
# create and initialize Id Variable
|
||||
ids = scope.var("Ids").get_selected_rows()
|
||||
ids_array = [0, 2, 3, 5, 100]
|
||||
ids.set_rows(ids_array)
|
||||
|
||||
# create and initialize W Variable
|
||||
rows = [0, 1, 2, 3, 4, 5, 6]
|
||||
row_numel = 10000
|
||||
|
||||
w_selected_rows = scope.var('W').get_selected_rows()
|
||||
w_selected_rows.set_height(len(rows))
|
||||
w_selected_rows.set_rows(rows)
|
||||
w_array = np.ones((len(rows), row_numel)).astype("float32")
|
||||
for i in range(len(rows)):
|
||||
w_array[i] *= i
|
||||
w_tensor = w_selected_rows.get_tensor()
|
||||
w_tensor.set(w_array, place)
|
||||
|
||||
# create Out Variable
|
||||
out_tensor = scope.var('Out').get_tensor()
|
||||
|
||||
# create and run lookup_table operator
|
||||
lookup_table = Operator(
|
||||
"lookup_sparse_table",
|
||||
W='W',
|
||||
Ids='Ids',
|
||||
Out='Out',
|
||||
min=-5.0,
|
||||
max=10.0,
|
||||
seed=10)
|
||||
lookup_table.run(scope, place)
|
||||
|
||||
# get result from Out
|
||||
result_array = np.array(out_tensor)
|
||||
# all(): return True if all elements of the iterable are true (or if the iterable is empty)
|
||||
for idx, row in enumerate(ids_array[:-2]):
|
||||
assert (row == result_array[idx]).all()
|
||||
|
||||
# check the random value
|
||||
hist, prob = output_hist(result_array[-1])
|
||||
self.assertTrue(
|
||||
np.allclose(
|
||||
hist, prob, rtol=0, atol=0.01), "hist: " + str(hist))
|
||||
|
||||
def test_w_is_selected_rows(self):
|
||||
places = [core.CPUPlace()]
|
||||
# currently only support CPU
|
||||
for place in places:
|
||||
self.check_with_place(place)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue