chendongsheng
4 years ago
parent
4ca4a148eb
commit
40f2571f7e
@ -0,0 +1,226 @@
|
||||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "ps/core/http_client.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace ps {
|
||||
namespace core {
|
||||
HttpClient::~HttpClient() {
|
||||
if (event_base_ != nullptr) {
|
||||
event_base_free(event_base_);
|
||||
event_base_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
void HttpClient::Init() {
|
||||
event_base_ = event_base_new();
|
||||
MS_EXCEPTION_IF_NULL(event_base_);
|
||||
dns_base_ = evdns_base_new(event_base_, 1);
|
||||
MS_EXCEPTION_IF_NULL(dns_base_);
|
||||
}
|
||||
|
||||
Status HttpClient::Post(const std::string &url, const void *body, size_t len, std::shared_ptr<std::vector<char>> output,
|
||||
const std::map<std::string, std::string> &headers) {
|
||||
MS_EXCEPTION_IF_NULL(body);
|
||||
MS_EXCEPTION_IF_NULL(output);
|
||||
auto handler = std::make_shared<HttpMessageHandler>();
|
||||
output->clear();
|
||||
handler->set_body(output);
|
||||
|
||||
struct evhttp_request *request = evhttp_request_new(ReadCallback, reinterpret_cast<void *>(handler.get()));
|
||||
MS_EXCEPTION_IF_NULL(request);
|
||||
|
||||
InitRequest(handler, url, request);
|
||||
|
||||
struct evhttp_connection *connection =
|
||||
evhttp_connection_base_new(event_base_, dns_base_, handler->GetHostByUri(), handler->GetUriPort());
|
||||
if (!connection) {
|
||||
MS_LOG(ERROR) << "Create http connection failed!";
|
||||
return Status::BADREQUEST;
|
||||
}
|
||||
|
||||
struct evbuffer *buffer = evhttp_request_get_output_buffer(request);
|
||||
if (evbuffer_add(buffer, body, len) != 0) {
|
||||
MS_LOG(ERROR) << "Add buffer failed!";
|
||||
return Status::INTERNAL;
|
||||
}
|
||||
|
||||
AddHeaders(headers, request, handler);
|
||||
|
||||
return CreateRequest(handler, connection, request, HttpMethod::HM_POST);
|
||||
}
|
||||
|
||||
Status HttpClient::Get(const std::string &url, std::shared_ptr<std::vector<char>> output,
|
||||
const std::map<std::string, std::string> &headers) {
|
||||
MS_EXCEPTION_IF_NULL(output);
|
||||
auto handler = std::make_shared<HttpMessageHandler>();
|
||||
output->clear();
|
||||
handler->set_body(output);
|
||||
|
||||
struct evhttp_request *request = evhttp_request_new(ReadCallback, reinterpret_cast<void *>(handler.get()));
|
||||
MS_EXCEPTION_IF_NULL(request);
|
||||
|
||||
InitRequest(handler, url, request);
|
||||
|
||||
struct evhttp_connection *connection =
|
||||
evhttp_connection_base_new(event_base_, dns_base_, handler->GetHostByUri(), handler->GetUriPort());
|
||||
if (!connection) {
|
||||
MS_LOG(ERROR) << "Create http connection failed!";
|
||||
return Status::BADREQUEST;
|
||||
}
|
||||
|
||||
AddHeaders(headers, request, handler);
|
||||
|
||||
return CreateRequest(handler, connection, request, HttpMethod::HM_GET);
|
||||
}
|
||||
|
||||
void HttpClient::set_connection_timeout(const int &timeout) { connection_timout_ = timeout; }
|
||||
|
||||
void HttpClient::ReadCallback(struct evhttp_request *request, void *arg) {
|
||||
MS_EXCEPTION_IF_NULL(request);
|
||||
MS_EXCEPTION_IF_NULL(arg);
|
||||
auto handler = static_cast<HttpMessageHandler *>(arg);
|
||||
if (event_base_loopexit(handler->http_base(), nullptr) != 0) {
|
||||
MS_LOG(EXCEPTION) << "event base loop exit failed!";
|
||||
}
|
||||
}
|
||||
|
||||
int HttpClient::ReadHeaderDoneCallback(struct evhttp_request *request, void *arg) {
|
||||
MS_EXCEPTION_IF_NULL(request);
|
||||
MS_EXCEPTION_IF_NULL(arg);
|
||||
auto handler = static_cast<HttpMessageHandler *>(arg);
|
||||
handler->set_request(request);
|
||||
MS_LOG(DEBUG) << "The http response code is:" << evhttp_request_get_response_code(request)
|
||||
<< ", The request code line is:" << evhttp_request_get_response_code_line(request);
|
||||
struct evkeyvalq *headers = evhttp_request_get_input_headers(request);
|
||||
struct evkeyval *header;
|
||||
TAILQ_FOREACH(header, headers, next) {
|
||||
MS_LOG(DEBUG) << "The key:" << header->key << ",The value:" << header->value;
|
||||
std::string len = "Content-Length";
|
||||
if (!strcmp(header->key, len.c_str())) {
|
||||
handler->set_content_len(strtouq(header->value, nullptr, 10));
|
||||
handler->InitBodySize();
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void HttpClient::ReadChunkDataCallback(struct evhttp_request *request, void *arg) {
|
||||
MS_EXCEPTION_IF_NULL(request);
|
||||
MS_EXCEPTION_IF_NULL(arg);
|
||||
auto handler = static_cast<HttpMessageHandler *>(arg);
|
||||
char buf[kMessageChunkLength];
|
||||
struct evbuffer *evbuf = evhttp_request_get_input_buffer(request);
|
||||
MS_EXCEPTION_IF_NULL(evbuf);
|
||||
int n = 0;
|
||||
while ((n = evbuffer_remove(evbuf, &buf, sizeof(buf))) > 0) {
|
||||
handler->ReceiveMessage(buf, n);
|
||||
}
|
||||
}
|
||||
|
||||
void HttpClient::RequestErrorCallback(enum evhttp_request_error error, void *arg) {
|
||||
MS_EXCEPTION_IF_NULL(arg);
|
||||
auto handler = static_cast<HttpMessageHandler *>(arg);
|
||||
MS_LOG(ERROR) << "The request failed, the error is:" << error;
|
||||
if (event_base_loopexit(handler->http_base(), nullptr) != 0) {
|
||||
MS_LOG(EXCEPTION) << "event base loop exit failed!";
|
||||
}
|
||||
}
|
||||
|
||||
void HttpClient::ConnectionCloseCallback(struct evhttp_connection *connection, void *arg) {
|
||||
MS_EXCEPTION_IF_NULL(connection);
|
||||
MS_EXCEPTION_IF_NULL(arg);
|
||||
MS_LOG(ERROR) << "Remote connection closed!";
|
||||
if (event_base_loopexit((struct event_base *)arg, nullptr) != 0) {
|
||||
MS_LOG(EXCEPTION) << "event base loop exit failed!";
|
||||
}
|
||||
}
|
||||
|
||||
void HttpClient::AddHeaders(const std::map<std::string, std::string> &headers, struct evhttp_request *request,
|
||||
std::shared_ptr<HttpMessageHandler> handler) {
|
||||
MS_EXCEPTION_IF_NULL(request);
|
||||
if (evhttp_add_header(evhttp_request_get_output_headers(request), "Host", handler->GetHostByUri()) != 0) {
|
||||
MS_LOG(EXCEPTION) << "Add header failed!";
|
||||
}
|
||||
for (auto &header : headers) {
|
||||
if (evhttp_add_header(evhttp_request_get_output_headers(request), header.first.data(), header.second.data()) != 0) {
|
||||
MS_LOG(EXCEPTION) << "Add header failed!";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void HttpClient::InitRequest(std::shared_ptr<HttpMessageHandler> handler, const std::string &url,
|
||||
struct evhttp_request *request) {
|
||||
MS_EXCEPTION_IF_NULL(request);
|
||||
MS_EXCEPTION_IF_NULL(handler);
|
||||
handler->set_http_base(event_base_);
|
||||
handler->ParseUrl(url);
|
||||
evhttp_request_set_header_cb(request, ReadHeaderDoneCallback);
|
||||
evhttp_request_set_chunked_cb(request, ReadChunkDataCallback);
|
||||
evhttp_request_set_error_cb(request, RequestErrorCallback);
|
||||
|
||||
MS_LOG(DEBUG) << "The url is:" << url << ", The host is:" << handler->GetHostByUri()
|
||||
<< ", The port is:" << handler->GetUriPort() << ", The request_url is:" << handler->GetRequestPath();
|
||||
}
|
||||
|
||||
Status HttpClient::CreateRequest(std::shared_ptr<HttpMessageHandler> handler, struct evhttp_connection *connection,
|
||||
struct evhttp_request *request, HttpMethod method) {
|
||||
MS_EXCEPTION_IF_NULL(handler);
|
||||
MS_EXCEPTION_IF_NULL(connection);
|
||||
MS_EXCEPTION_IF_NULL(request);
|
||||
evhttp_connection_set_closecb(connection, ConnectionCloseCallback, event_base_);
|
||||
evhttp_connection_set_timeout(connection, connection_timout_);
|
||||
|
||||
if (evhttp_make_request(connection, request, evhttp_cmd_type(method), handler->GetRequestPath().c_str()) != 0) {
|
||||
MS_LOG(ERROR) << "Make request failed!";
|
||||
return Status::INTERNAL;
|
||||
}
|
||||
|
||||
if (!Start()) {
|
||||
MS_LOG(ERROR) << "Start http client failed!";
|
||||
return Status::INTERNAL;
|
||||
}
|
||||
|
||||
if (handler->request()) {
|
||||
MS_LOG(DEBUG) << "The http response code is:" << evhttp_request_get_response_code(handler->request())
|
||||
<< ", The request code line is:" << evhttp_request_get_response_code_line(handler->request());
|
||||
return Status(evhttp_request_get_response_code(handler->request()));
|
||||
}
|
||||
return Status::INTERNAL;
|
||||
}
|
||||
|
||||
bool HttpClient::Start() {
|
||||
MS_EXCEPTION_IF_NULL(event_base_);
|
||||
// int ret = event_base_dispatch(event_base_);
|
||||
int ret = event_base_loop(event_base_, 0);
|
||||
if (ret == 0) {
|
||||
MS_LOG(DEBUG) << "Event base dispatch success!";
|
||||
return true;
|
||||
} else if (ret == 1) {
|
||||
MS_LOG(ERROR) << "Event base dispatch failed with no events pending or active!";
|
||||
return false;
|
||||
} else if (ret == -1) {
|
||||
MS_LOG(ERROR) << "Event base dispatch failed with error occurred!";
|
||||
return false;
|
||||
} else {
|
||||
MS_LOG(EXCEPTION) << "Event base dispatch with unexpected error code!";
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace core
|
||||
} // namespace ps
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,97 @@
|
||||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_PS_CORE_HTTP_CLIENT_H_
|
||||
#define MINDSPORE_CCSRC_PS_CORE_HTTP_CLIENT_H_
|
||||
|
||||
#include <event2/buffer.h>
|
||||
#include <event2/event.h>
|
||||
#include <event2/http.h>
|
||||
#include <event2/keyvalq_struct.h>
|
||||
#include <event2/listener.h>
|
||||
#include <event2/util.h>
|
||||
#include <event2/http_struct.h>
|
||||
#include <event2/dns.h>
|
||||
#include <event2/thread.h>
|
||||
#include <sys/queue.h>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <atomic>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
|
||||
#include "ps/core/http_message_handler.h"
|
||||
#include "ps/core/comm_util.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace ps {
|
||||
namespace core {
|
||||
|
||||
enum class HttpMethod { HM_GET = 1 << 0, HM_POST = 1 << 1 };
|
||||
|
||||
enum class Status : int {
|
||||
OK = 200, // request completed ok
|
||||
BADREQUEST = 400, // invalid http request was made
|
||||
NOTFOUND = 404, // could not find content for uri
|
||||
INTERNAL = 500 // internal error
|
||||
};
|
||||
|
||||
class HttpClient {
|
||||
public:
|
||||
HttpClient() : event_base_(nullptr), dns_base_(nullptr), is_init_(false), connection_timout_(kConnectionTimeout) {
|
||||
Init();
|
||||
}
|
||||
|
||||
virtual ~HttpClient();
|
||||
|
||||
Status Post(const std::string &url, const void *body, size_t len, std::shared_ptr<std::vector<char>> output,
|
||||
const std::map<std::string, std::string> &headers = {});
|
||||
Status Get(const std::string &url, std::shared_ptr<std::vector<char>> output,
|
||||
const std::map<std::string, std::string> &headers = {});
|
||||
|
||||
void set_connection_timeout(const int &timeout);
|
||||
|
||||
private:
|
||||
static void ReadCallback(struct evhttp_request *remote_rsp, void *arg);
|
||||
static int ReadHeaderDoneCallback(struct evhttp_request *remote_rsp, void *arg);
|
||||
static void ReadChunkDataCallback(struct evhttp_request *remote_rsp, void *arg);
|
||||
static void RequestErrorCallback(enum evhttp_request_error error, void *arg);
|
||||
static void ConnectionCloseCallback(struct evhttp_connection *connection, void *arg);
|
||||
|
||||
void AddHeaders(const std::map<std::string, std::string> &headers, struct evhttp_request *request,
|
||||
std::shared_ptr<HttpMessageHandler> handler);
|
||||
void InitRequest(std::shared_ptr<HttpMessageHandler> handler, const std::string &url, struct evhttp_request *request);
|
||||
Status CreateRequest(std::shared_ptr<HttpMessageHandler> handler, struct evhttp_connection *connection,
|
||||
struct evhttp_request *request, HttpMethod method);
|
||||
|
||||
bool Start();
|
||||
void Init();
|
||||
|
||||
struct event_base *event_base_;
|
||||
struct evdns_base *dns_base_;
|
||||
bool is_init_;
|
||||
int connection_timout_;
|
||||
};
|
||||
} // namespace core
|
||||
} // namespace ps
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_PS_CORE_HTTP_CLIENT_H_
|
||||
@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <memory>
|
||||
|
||||
#include "common/common_test.h"
|
||||
#include "ps/core/http_server.h"
|
||||
#include "ps/core/http_client.h"
|
||||
|
||||
using namespace std;
|
||||
|
||||
namespace mindspore {
|
||||
namespace ps {
|
||||
namespace core {
|
||||
class TestHttpClient : public UT::Common {
|
||||
public:
|
||||
TestHttpClient() : server_(nullptr), http_server_thread_(nullptr) {}
|
||||
|
||||
virtual ~TestHttpClient() = default;
|
||||
|
||||
OnRequestReceive http_get_func = std::bind(
|
||||
[](std::shared_ptr<HttpMessageHandler> resp) {
|
||||
EXPECT_STREQ(resp->GetUriPath().c_str(), "/httpget");
|
||||
const unsigned char ret[] = "get request success!\n";
|
||||
resp->QuickResponse(200, ret, 22);
|
||||
},
|
||||
std::placeholders::_1);
|
||||
|
||||
OnRequestReceive http_handler_func = std::bind(
|
||||
[](std::shared_ptr<HttpMessageHandler> resp) {
|
||||
std::string host = resp->GetRequestHost();
|
||||
EXPECT_STREQ(host.c_str(), "127.0.0.1");
|
||||
|
||||
std::string path_param = resp->GetPathParam("key1");
|
||||
std::string header_param = resp->GetHeadParam("headerKey");
|
||||
unsigned char *data = nullptr;
|
||||
const uint64_t len = resp->GetPostMsg(&data);
|
||||
char post_message[len + 1];
|
||||
if (memset_s(post_message, len + 1, 0, len + 1) != 0) {
|
||||
MS_LOG(EXCEPTION) << "The memset_s error";
|
||||
}
|
||||
if (memcpy_s(post_message, len, data, len) != 0) {
|
||||
MS_LOG(EXCEPTION) << "The memset_s error";
|
||||
}
|
||||
EXPECT_STREQ(path_param.c_str(), "value1");
|
||||
EXPECT_STREQ(header_param.c_str(), "headerValue");
|
||||
EXPECT_STREQ(post_message, "postKey=postValue");
|
||||
|
||||
const std::string rKey("headKey");
|
||||
const std::string rVal("headValue");
|
||||
const std::string rBody("post request success!\n");
|
||||
resp->AddRespHeadParam(rKey, rVal);
|
||||
resp->AddRespString(rBody);
|
||||
|
||||
resp->SetRespCode(200);
|
||||
resp->SendResponse();
|
||||
},
|
||||
std::placeholders::_1);
|
||||
|
||||
void SetUp() override {
|
||||
server_ = std::make_unique<HttpServer>("0.0.0.0", 9999);
|
||||
|
||||
server_->RegisterRoute("/httpget", &http_get_func);
|
||||
server_->RegisterRoute("/handler", &http_handler_func);
|
||||
http_server_thread_ = std::make_unique<std::thread>([&]() { server_->Start(); });
|
||||
http_server_thread_->detach();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
|
||||
}
|
||||
|
||||
void TearDown() override {
|
||||
server_->Stop();
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2000));
|
||||
}
|
||||
|
||||
private:
|
||||
std::unique_ptr<HttpServer> server_;
|
||||
std::unique_ptr<std::thread> http_server_thread_;
|
||||
};
|
||||
|
||||
TEST_F(TestHttpClient, Get) {
|
||||
HttpClient client;
|
||||
std::map<std::string, std::string> headers = {{"headerKey", "headerValue"}};
|
||||
auto output = std::make_shared<std::vector<char>>();
|
||||
auto ret = client.Get("http://127.0.0.1:9999/httpget", output, headers);
|
||||
EXPECT_STREQ("get request success!\n", output->data());
|
||||
EXPECT_EQ(Status::OK, ret);
|
||||
}
|
||||
|
||||
TEST_F(TestHttpClient, Post) {
|
||||
HttpClient client;
|
||||
std::map<std::string, std::string> headers = {{"headerKey", "headerValue"}};
|
||||
auto output = std::make_shared<std::vector<char>>();
|
||||
std::string post_data = "postKey=postValue";
|
||||
auto ret =
|
||||
client.Post("http://127.0.0.1:9999/handler?key1=value1", post_data.c_str(), post_data.length(), output, headers);
|
||||
EXPECT_STREQ("post request success!\n", output->data());
|
||||
EXPECT_EQ(Status::OK, ret);
|
||||
}
|
||||
} // namespace core
|
||||
} // namespace ps
|
||||
} // namespace mindspore
|
||||
Loading…
Reference in new issue