!5199 format trans fusion optimize

Merge pull request !5199 from zhengjun10/master

mindspore/lite/test/models_caffe.cfg

@@ -16,8 +16,8 @@ tracking
 mtk_isface
 mtk_landmark
 mtk_pose_tuku
-mtk_face_recognition_v1
+# mtk_face_recognition_v1
-mtk_2012_ATLANTA_10class_20190614_v41
+# mtk_2012_ATLANTA_10class_20190614_v41
 mtk_detect-deeper-halfdeeper-mbv1-lastearlySSD-shortcut-400-400_nopostprocess_simplified
 detect-deeper-halfdeeper-mbv1-shortcut-400-400_nopostprocess_simplified
 hiai_face_detect_rfb
@@ -37,7 +37,7 @@ ml_hardware_pose
 ml_bank_recog
 2012_ATLANTA_10class_20190131_v4.0
 mnet
-recognition
+# recognition
 ml_face_landmark
 model_hebing_3branch
 hiai_cv_focusShootOCRModel_07
@@ -48,9 +48,9 @@ hiai_cv_focusShootOCRModel_04
 hiai_cv_focusShootOCRModel_06
 hiai_cpu_face_hat
 hiai_video_seg
-hiai_semantic_seg
+# hiai_semantic_seg
 hiai_human_seg
-hiai_face_recognition_1
+# hiai_face_recognition_1
 hiai_cpu_face_detect
 hiai_cpu_face_attr
 hiai_face_attr1

mindspore/lite/tools/converter/graphdef_transform.cc

@@ -27,8 +27,8 @@
 #include "tools/converter/legacy_optimizer/fusion/format_trans_fusion_pass.h"
 #include "tools/converter/legacy_optimizer/fusion/format_trans_transpose_fusion_pass.h"
 #include "tools/converter/legacy_optimizer/fusion/quant_cast_fusion_pass.h"
-#include "tools/converter/legacy_optimizer/fusion/batchnorm_convert_scale_pass.h"
 #include "tools/converter/legacy_optimizer/fusion/mul_add_fusion_pass.h"
+#include "tools/converter/legacy_optimizer/graph/batchnorm_convert_scale_pass.h"
 #include "tools/converter/legacy_optimizer/graph/weight_format_hardcode_pass.h"
 #include "tools/converter/legacy_optimizer/graph/weight_format_transform_pass.h"
 #include "tools/converter/legacy_optimizer/graph/format_trans_pass.h"

mindspore/lite/tools/converter/legacy_optimizer/fusion/CMakeLists.txt

@@ -7,7 +7,6 @@ add_library(fusion_mid OBJECT
         ${CMAKE_CURRENT_SOURCE_DIR}/batchnorm_fold_fusion_pass.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/format_trans_fusion_pass.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/format_trans_transpose_fusion_pass.cc
-        ${CMAKE_CURRENT_SOURCE_DIR}/batchnorm_convert_scale_pass.cc
         )
 
 target_link_libraries(fusion_mid securec)

mindspore/lite/tools/converter/legacy_optimizer/fusion/batchnorm_convert_scale_pass.h

@@ -1,100 +0,0 @@
-/**
- * Copyright 2020 Huawei Technologies Co., Ltd
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef MINDSPORE_PREDICT_BATCHNORM_CONVERT_SCALE_PASS_H
-#define MINDSPORE_PREDICT_BATCHNORM_CONVERT_SCALE_PASS_H
-
-#include <unordered_map>
-#include <memory>
-#include <string>
-#include <utility>
-#include "tools/converter/legacy_optimizer/fusion/fusion_pass.h"
-#include "tools/common/graph_util.h"
-
-namespace mindspore {
-namespace lite {
-struct BNWeightTensors {
-  TensorT *meanTensor = nullptr;
-  TensorT *varianceTensor = nullptr;
-  TensorT *scaleTensor = nullptr;
-  TensorT *biasTensor = nullptr;
-};
-class BatchNormConvertScalePass : public FusionPass {
- public:
-  BatchNormConvertScalePass() = default;
-
-  ~BatchNormConvertScalePass() = default;
-
-  STATUS DefinePattern() override;
-
-  STATUS DoFusion(MetaGraphT *graph, const std::string &patternName,
-                  std::unordered_map<std::string, std::shared_ptr<Path>> &matchedPath) override;
-
-  STATUS Run(MetaGraphT *graph) override;
-
- protected:
-  STATUS GetTransParam(MetaGraphT *graph, const std::shared_ptr<Path> &bnPath);
-
-  // Get and check BNNode weight tensor
-  STATUS GetBnWeightTensors(MetaGraphT *graph, const std::shared_ptr<Path> &bnPath, BNWeightTensors* bnWeightTensors);
-
-  STATUS GetBnEpsilon(MetaGraphT *graph);
-
-  STATUS FindNodes(MetaGraphT *graph, const std::unordered_map<std::string, std::shared_ptr<Path>> &matchedPath);
-
-  STATUS GenNewScaleTensor(MetaGraphT *graph, const std::shared_ptr<Path> &bnPath);
-
-  STATUS ConvertBNToScale(MetaGraphT *graph, const std::shared_ptr<Path> &bnPath);
-
-  CNodeT *inputNode = nullptr;
-  CNodeT *bnNode = nullptr;
-
-  std::string inputOpName = "Input";
-  std::string bnOpName = "BatchNorm";
-  std::string bnPatternName = "BnToScaleFusion";
-  uint32_t bnChannel = 0;
-  float eps = 0;
-  TensorT *bnMeanTensor = nullptr;
-  float *transScale = nullptr;
-  float *transBias = nullptr;
-  std::unique_ptr<TensorT> newScaleWeightTensor = nullptr;
-  std::unique_ptr<TensorT> newScaleBiasTensor = nullptr;
-
-  OpDefCopyer ScaleOpCopyer = [](CNodeT *inOpDef) -> std::unique_ptr<CNodeT> {
-    std::unique_ptr<CNodeT> newOpDef(new(std::nothrow) CNodeT);
-    if (newOpDef == nullptr) {
-      MS_LOG(ERROR) << "new OpDefT failed";
-      return nullptr;
-    }
-    newOpDef->name = inOpDef->name;
-    newOpDef->quantType = inOpDef->quantType;
-    newOpDef->primitive = std::make_unique<schema::PrimitiveT>();
-    newOpDef->primitive->value.type = schema::PrimitiveType_Scale;
-    auto scaleParam = new(std::nothrow) ScaleT;
-    if (scaleParam == nullptr) {
-      MS_LOG(ERROR) << "new scaleParam failed";
-      return nullptr;
-    }
-    auto inParam = inOpDef->primitive->value.AsScale();
-    MS_ASSERT(inParam != nullptr);
-    scaleParam->axis = inParam->axis;
-    newOpDef->primitive->value.value = scaleParam;
-    return std::move(newOpDef);
-  };
-};
-}  // namespace lite
-}  // namespace mindspore
-#endif  // MINDSPORE_PREDICT_BATCHNORM_CONVERT_SCALE_PASS_H

mindspore/lite/tools/converter/legacy_optimizer/graph/CMakeLists.txt

@@ -8,4 +8,5 @@ add_library(graph_pass_mid OBJECT
         ${CMAKE_CURRENT_SOURCE_DIR}/weight_format_transform_pass.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/topological_sort_pass.cc
         ${CMAKE_CURRENT_SOURCE_DIR}/unused_node_remove_pass.cc
+        ${CMAKE_CURRENT_SOURCE_DIR}/batchnorm_convert_scale_pass.cc
         )

mindspore/lite/tools/converter/legacy_optimizer/graph/batchnorm_convert_scale_pass.h

@@ -0,0 +1,66 @@
+/**
+ * Copyright 2020 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_PREDICT_BATCHNORM_CONVERT_SCALE_PASS_H
+#define MINDSPORE_PREDICT_BATCHNORM_CONVERT_SCALE_PASS_H
+
+#include <unordered_map>
+#include <memory>
+#include <string>
+#include <utility>
+#include "tools/common/graph_util.h"
+#include "tools/converter/optimizer.h"
+
+using mindspore::schema::TensorT;
+namespace mindspore {
+namespace lite {
+struct BNWeightTensors {
+  schema::TensorT *meanTensor = nullptr;
+  TensorT *varianceTensor = nullptr;
+  TensorT *scaleTensor = nullptr;
+  TensorT *biasTensor = nullptr;
+};
+class BatchNormConvertScalePass : public GraphPass {
+ public:
+  BatchNormConvertScalePass() = default;
+
+  ~BatchNormConvertScalePass() = default;
+
+  STATUS Run(MetaGraphT *graph) override;
+
+ protected:
+  STATUS GetTransParam(MetaGraphT *graph, const std::unique_ptr<CNodeT> &bnNode);
+
+  // Get and check BNNode weight tensor
+  STATUS GetBnWeightTensors(MetaGraphT *graph, BNWeightTensors *bnWeightTensors, const std::unique_ptr<CNodeT> &bnNode);
+
+  STATUS GetBnEpsilon(const std::unique_ptr<CNodeT> &bnNode);
+
+  STATUS GenNewScaleTensor(MetaGraphT *graph, const std::unique_ptr<CNodeT> &bnNode);
+
+  STATUS ConvertBNToScale(MetaGraphT *graph, const std::unique_ptr<CNodeT> &bnNode);
+
+  uint32_t bnChannel = 0;
+  float eps = 0;
+  TensorT *bnMeanTensor = nullptr;
+  float *transScale = nullptr;
+  float *transBias = nullptr;
+  std::unique_ptr<TensorT> newScaleWeightTensor = nullptr;
+  std::unique_ptr<TensorT> newScaleBiasTensor = nullptr;
+};
+}  // namespace lite
+}  // namespace mindspore
+#endif  // MINDSPORE_PREDICT_BATCHNORM_CONVERT_SCALE_PASS_H

mindspore/lite/tools/converter/legacy_optimizer/graph/eltwise_format_trans_pass.cc

@@ -121,7 +121,8 @@ STATUS EltwiseFormatTransPass::Run(schema::MetaGraphT *graph) {
   MS_ASSERT(graph != nullptr);
   for (auto iter = graph->nodes.begin(); iter != graph->nodes.end(); iter++) {
     auto &node = *iter;
-    if (node->primitive->value.type != PrimitiveType_Eltwise) {
+    auto type = node->primitive->value.type;
+    if (type != PrimitiveType_Eltwise && type != PrimitiveType_Activation) {
       continue;
     }
     auto node_name = node->name;

mindspore/lite/tools/optimizer/common/gllo_utils.cc

@@ -295,6 +295,9 @@ ParameterPtr AddNewBiasNode(float *bias_data, const FuncGraphPtr &func_graph, in
   MS_ASSERT(param_value != nullptr);
   param_value->set_tensor_addr(bias_data);
   param_value->set_tensor_size(kernel_num * sizeof(float) / sizeof(uint8_t));
+  param_value->set_format(weight_tensor->format());
+  param_value->set_tensor_type(weight_tensor->tensor_type());
+  param_value->set_tensor_shape(shape);
   bias_parameter->set_default_param(param_value);
   return bias_parameter;
 }

mindspore/lite/tools/optimizer/fusion/constant_folding_fusion.cc

@@ -83,6 +83,7 @@ const ParameterPtr CreateNewParamter(const FuncGraphPtr &func_graph, Tensor *ten
   MS_ASSERT(param_value != nullptr);
   param_value->set_tensor_shape(shape);
   param_value->set_tensor_type(type_id);
+  param_value->set_format(tensor->GetFormat());
   if (tensor->Data() != nullptr) {
     auto size = tensor->ElementsNum();
     auto tensor_data = new (std::nothrow) float[size];

mindspore/lite/tools/optimizer/fusion/conv_activation_fusion.cc

@@ -51,13 +51,13 @@ const AnfNodePtr ConvActivationFusion::Process(const FuncGraphPtr &func_graph, c
   auto act_primitivec = utils::cast<std::shared_ptr<mindspore::lite::Activation>>(primitivec);
   MS_ASSERT(act_primitivec != nullptr);
   if (act_primitivec->GetType() != activation_type) {
-    return node;
+    return nullptr;
   }
   AnfNodePtr pre_node = act_node->input(1);
   CheckIfAnfNodeIsNull(pre_node);
   if (pre_node != nullptr && pre_node->isa<CNode>()) {
     if (IsMultiOutputTensors(func_graph, pre_node)) {
-      return node;
+      return nullptr;
     }
     auto conv_node = pre_node->cast<CNodePtr>();
     auto node_type = GetCNodeType(conv_node);
@@ -80,9 +80,9 @@ const AnfNodePtr ConvActivationFusion::Process(const FuncGraphPtr &func_graph, c
         return pre_node;
       }
     } else {
-      MS_LOG(EXCEPTION) << "conv activation pass match only conv2d or depthwise_conv2d ";
+      MS_LOG(ERROR) << "conv activation pass match only conv2d or depthwise_conv2d ";
     }
   }
-  return node;
+  return nullptr;
 }
 }  // namespace mindspore::opt

mindspore/lite/tools/optimizer/fusion/conv_biasadd_fusion.cc

@@ -179,7 +179,8 @@ const AnfNodePtr ConvBiasaddFusion::Process(const FuncGraphPtr &func_graph, cons
     MS_ASSERT(primc != nullptr);
     primc->SetHasBias(true);
   } else {
-    MS_LOG(EXCEPTION) << "Unsupported opType, " << type;
+    MS_LOG(ERROR) << "Unsupported opType, " << type;
+    return nullptr;
   }
   return conv_node;
 }

mindspore/lite/tools/optimizer/fusion/conv_transform_fusion.cc

@@ -85,12 +85,13 @@ const AnfNodePtr ConvTransformFusion::Process(const FuncGraphPtr &func_graph, co
   auto trans_scale = new (std::nothrow) float[kernel_nums];
   if (trans_scale == nullptr) {
     MS_LOG(ERROR) << "tensor_data is nullptr";
+    delete[] trans_scale;
     return nullptr;
   }
   auto trans_bias = new (std::nothrow) float[kernel_nums];
   if (trans_bias == nullptr) {
     MS_LOG(ERROR) << "tensor_data is nullptr";
-    delete trans_scale;
+    delete[] trans_bias;
     return nullptr;
   }
   GenTransParam(transform_node, kernel_nums, trans_scale, trans_bias);
@@ -111,7 +112,8 @@ const AnfNodePtr ConvTransformFusion::Process(const FuncGraphPtr &func_graph, co
     MS_ASSERT(primc != nullptr);
     primc->SetHasBias(true);
   } else {
-    MS_LOG(EXCEPTION) << "Unsupported opType, " << type;
+    MS_LOG(ERROR) << "Unsupported opType, " << type;
+    return nullptr;
   }
   pre_node->set_abstract(abstr);
   return pre_node;
@@ -179,6 +181,7 @@ const void ConvTransformFusion::GenNewConvTensor(const FuncGraphPtr &func_graph,
     bias_data = new (std::nothrow) float[kernel_num];
     if (bias_data == nullptr) {
       MS_LOG(ERROR) << "tensor_data is nullptr";
+      delete[] bias_data;
       return;
     }
   }