!11510 [MS][LITE] weight quant support huffman code
From: @jianghui58 Reviewed-by: Signed-off-by:pull/11510/MERGE
mindspore-ci-bot
4 years ago
committed by
Gitee
commit
86dfebd837
@ -0,0 +1,168 @@
|
||||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "src/huffman_decode.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace lite {
|
||||
|
||||
STATUS huffman_decode::DoHuffmanDecode(const std::string &input_str, void *decoded_data) {
|
||||
if (decoded_data == nullptr) {
|
||||
MS_LOG(ERROR) << "decoded_data is nullptr.";
|
||||
return RET_ERROR;
|
||||
}
|
||||
|
||||
int status;
|
||||
std::string huffman_decoded_str = "";
|
||||
|
||||
auto key_pos = input_str.find_first_of('#');
|
||||
auto code_pos = input_str.find_first_of('#', key_pos + 1);
|
||||
auto key = input_str.substr(0, key_pos);
|
||||
auto code = input_str.substr(key_pos + 1, code_pos - key_pos - 1);
|
||||
auto encoded_data = input_str.substr(code_pos + 1);
|
||||
|
||||
auto root = new (std::nothrow) HuffmanNode();
|
||||
if (root == nullptr) {
|
||||
MS_LOG(ERROR) << "new HuffmanNode failed.";
|
||||
return RET_MEMORY_FAILED;
|
||||
}
|
||||
root->left = nullptr;
|
||||
root->right = nullptr;
|
||||
root->parent = nullptr;
|
||||
|
||||
status = RebuildHuffmanTree(key, code, root);
|
||||
if (status != RET_OK) {
|
||||
MS_LOG(ERROR) << "Rebuild huffman tree failed.";
|
||||
delete root;
|
||||
return status;
|
||||
}
|
||||
|
||||
status = DoHuffmanDecompress(root, encoded_data, &huffman_decoded_str);
|
||||
if (status != RET_OK) {
|
||||
MS_LOG(ERROR) << "DoHuffmanDecompress failed.";
|
||||
delete root;
|
||||
return status;
|
||||
}
|
||||
|
||||
size_t len = huffman_decoded_str.length();
|
||||
memcpy(decoded_data, huffman_decoded_str.c_str(), len);
|
||||
|
||||
delete root;
|
||||
return RET_OK;
|
||||
}
|
||||
|
||||
STATUS huffman_decode::RebuildHuffmanTree(std::string keys, std::string codes, const HuffmanNodePtr &root) {
|
||||
HuffmanNodePtr cur_node, tmp_node, new_node;
|
||||
|
||||
auto huffman_keys = Str2Vec(std::move(keys));
|
||||
auto huffman_codes = Str2Vec(std::move(codes));
|
||||
|
||||
for (size_t i = 0; i < huffman_codes.size(); ++i) {
|
||||
auto key = stoi(huffman_keys[i]);
|
||||
auto code = huffman_codes[i];
|
||||
auto code_len = code.length();
|
||||
cur_node = root;
|
||||
for (size_t j = 0; j < code_len; ++j) {
|
||||
if (code[j] == '0') {
|
||||
tmp_node = cur_node->left;
|
||||
} else if (code[j] == '1') {
|
||||
tmp_node = cur_node->right;
|
||||
} else {
|
||||
MS_LOG(ERROR) << "find huffman code is not 0 or 1";
|
||||
return RET_ERROR;
|
||||
}
|
||||
|
||||
if (tmp_node == nullptr) {
|
||||
new_node = new (std::nothrow) HuffmanNode();
|
||||
if (new_node == nullptr) {
|
||||
MS_LOG(ERROR) << "new HuffmanNode failed.";
|
||||
return RET_MEMORY_FAILED;
|
||||
}
|
||||
this->huffman_nodes_.push_back(new_node);
|
||||
new_node->left = nullptr;
|
||||
new_node->right = nullptr;
|
||||
new_node->parent = cur_node;
|
||||
|
||||
if (j == code_len - 1) {
|
||||
new_node->key = key;
|
||||
new_node->code = code;
|
||||
}
|
||||
|
||||
if (code[j] == '0') {
|
||||
cur_node->left = new_node;
|
||||
} else {
|
||||
cur_node->right = new_node;
|
||||
}
|
||||
|
||||
tmp_node = new_node;
|
||||
} else if (j == code_len - 1) {
|
||||
MS_LOG(ERROR) << "the huffman code is incomplete.";
|
||||
return RET_ERROR;
|
||||
} else if (tmp_node->left == nullptr && tmp_node->right == nullptr) {
|
||||
MS_LOG(ERROR) << "the huffman code is incomplete";
|
||||
return RET_ERROR;
|
||||
}
|
||||
cur_node = tmp_node;
|
||||
}
|
||||
}
|
||||
return RET_OK;
|
||||
}
|
||||
|
||||
STATUS huffman_decode::DoHuffmanDecompress(HuffmanNodePtr root, std::string encoded_data, std::string *decoded_str) {
|
||||
HuffmanNodePtr cur_node = root;
|
||||
bool pseudo_eof = false;
|
||||
size_t pos = 0;
|
||||
unsigned char flag;
|
||||
|
||||
decoded_str->clear();
|
||||
while (pos < encoded_data.length()) {
|
||||
auto u_char = static_cast<unsigned char>(encoded_data[pos]);
|
||||
flag = 0x80;
|
||||
for (size_t i = 0; i < 8; ++i) { // traverse the 8 bit num, to find the leaf node
|
||||
if (u_char & flag) {
|
||||
cur_node = cur_node->right;
|
||||
} else {
|
||||
cur_node = cur_node->left;
|
||||
}
|
||||
if (cur_node->left == nullptr && cur_node->right == nullptr) {
|
||||
auto key = cur_node->key;
|
||||
if (key == PSEUDO_EOF) {
|
||||
pseudo_eof = true;
|
||||
break;
|
||||
} else {
|
||||
*decoded_str += static_cast<char>(cur_node->key);
|
||||
cur_node = root;
|
||||
}
|
||||
}
|
||||
flag = flag >> 1;
|
||||
}
|
||||
pos++;
|
||||
if (pseudo_eof) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return RET_OK;
|
||||
}
|
||||
|
||||
huffman_decode::~huffman_decode() {
|
||||
for (auto &node : this->huffman_nodes_) {
|
||||
delete node;
|
||||
}
|
||||
this->huffman_nodes_.resize(0);
|
||||
}
|
||||
|
||||
} // namespace lite
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_LITE_MINDSPORE_LITE_SRC_HUFFMAN_DECODE_H_
|
||||
#define MINDSPORE_LITE_MINDSPORE_LITE_SRC_HUFFMAN_DECODE_H_
|
||||
|
||||
#include <cstring>
|
||||
#include <utility>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "include/errorcode.h"
|
||||
#include "src/common/log_adapter.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace lite {
|
||||
|
||||
const int PSEUDO_EOF = 128;
|
||||
|
||||
struct HuffmanNode {
|
||||
int key;
|
||||
unsigned int freq;
|
||||
std::string code;
|
||||
HuffmanNode *left, *right, *parent;
|
||||
};
|
||||
using HuffmanNodePtr = HuffmanNode *;
|
||||
|
||||
class huffman_decode {
|
||||
public:
|
||||
huffman_decode() = default;
|
||||
|
||||
~huffman_decode();
|
||||
|
||||
STATUS DoHuffmanDecode(const std::string &input_str, void *decoded_data);
|
||||
|
||||
private:
|
||||
std::vector<HuffmanNodePtr> huffman_nodes_;
|
||||
STATUS RebuildHuffmanTree(std::string key, std::string code, const HuffmanNodePtr &root);
|
||||
|
||||
STATUS DoHuffmanDecompress(HuffmanNodePtr root, std::string encoded_data, std::string *decoded_str);
|
||||
|
||||
std::vector<std::string> Str2Vec(std::string s) {
|
||||
size_t i = 0;
|
||||
std::vector<std::string> vec;
|
||||
while (i < s.length()) {
|
||||
size_t j = i;
|
||||
while (j < s.length() && s[j] != ' ') {
|
||||
j++;
|
||||
}
|
||||
if (j != i) {
|
||||
vec.push_back(s.substr(i, j - i));
|
||||
i = j + 1;
|
||||
} else {
|
||||
i = j;
|
||||
}
|
||||
}
|
||||
return vec;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace lite
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_LITE_MINDSPORE_LITE_SRC_HUFFMAN_DECODE_H_
|
||||
@ -1 +1,2 @@
|
||||
ml_face_openclose.tflite
|
||||
ml_face_openclose.tflite 0.5
|
||||
hiai_ghostnet.tflite 5
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_HUFFMANCODE_HUFFMAN_H
|
||||
#define MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_HUFFMANCODE_HUFFMAN_H
|
||||
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <queue>
|
||||
#include <map>
|
||||
#include <fstream>
|
||||
#include "src/common/log_adapter.h"
|
||||
#include "src/ops/primitive_c.h"
|
||||
#include "ir/func_graph.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace lite {
|
||||
|
||||
using STATUS = int;
|
||||
|
||||
const int PSEUDO_EOF = 128;
|
||||
|
||||
struct HuffmanNode {
|
||||
int key;
|
||||
unsigned int freq;
|
||||
std::string code;
|
||||
HuffmanNode *left, *right, *parent;
|
||||
};
|
||||
using HuffmanNodePtr = HuffmanNode *;
|
||||
|
||||
struct cmp {
|
||||
public:
|
||||
bool operator()(const HuffmanNodePtr &c1, const HuffmanNodePtr &c2) const { return c1->freq > c2->freq; }
|
||||
};
|
||||
using HuffmanPriorityQueue = std::priority_queue<HuffmanNodePtr, std::vector<HuffmanNodePtr>, cmp>;
|
||||
|
||||
class huffman_encode {
|
||||
public:
|
||||
huffman_encode() = default;
|
||||
|
||||
~huffman_encode();
|
||||
|
||||
STATUS DoHuffmanEncode(const FuncGraphPtr &func_graph);
|
||||
|
||||
private:
|
||||
std::map<int, std::string> huffman_table_;
|
||||
std::string huffman_encoded_str_ = "";
|
||||
std::vector<HuffmanNodePtr> huffman_nodes_;
|
||||
|
||||
STATUS GetHuffmanPriorityQueue(const int8_t *input_datas, size_t input_data_size, HuffmanPriorityQueue *pq);
|
||||
|
||||
void GenerateHuffmanTable(HuffmanNodePtr node, bool is_left_node);
|
||||
|
||||
STATUS BuildHuffmanTree(HuffmanPriorityQueue *pq);
|
||||
|
||||
STATUS DoHuffmanCompress(const int8_t *input_datas, size_t data_size);
|
||||
};
|
||||
|
||||
} // namespace lite
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_HUFFMANCODE_HUFFMAN_H
|
||||
Loading…
Reference in new issue