Fix a wide&deep memory increasing problem

mindspore/ccsrc/minddata/dataset/util/services.cc

@@ -35,7 +35,8 @@ namespace mindspore {
 namespace dataset {
 std::unique_ptr<Services> Services::instance_ = nullptr;
 std::once_flag Services::init_instance_flag_;
-std::set<std::string> Services::unique_id_list_ = {};
+std::map<std::string, uint64_t> Services::unique_id_list_ = {};
+uint64_t Services::unique_id_count_ = 0;
 std::mutex Services::unique_id_mutex_;
 
 #if !defined(_WIN32) && !defined(_WIN64) && !defined(__ANDROID__) && !defined(ANDROID)
@@ -73,7 +74,22 @@ std::string Services::GetUniqueID() {
       if (unique_id_list_.find(std::string(buffer, UNIQUEID_LEN)) != unique_id_list_.end()) {
         continue;
       }
-      unique_id_list_.insert(std::string(buffer, UNIQUEID_LEN));
+      unique_id_list_[std::string(buffer, UNIQUEID_LEN)] = unique_id_count_;
+      unique_id_count_++;
+      // Temporary solution to solve a long stability memory increasing problem that
+      // we limit the size of unique_id_list_ not to greater than UNIQUEID_LIST_LIMITS(1024).
+      if (unique_id_list_.size() >= UNIQUEID_LIST_LIMITS) {
+        for (auto iter = unique_id_list_.begin(); iter != unique_id_list_.end();) {
+          if (iter->second < UNIQUEID_HALF_INDEX) {
+            iter = unique_id_list_.erase(iter);
+            unique_id_count_--;
+          } else {
+            iter->second -= UNIQUEID_HALF_INDEX;
+            iter++;
+          }
+        }
+      }
+      MS_LOG(DEBUG) << "unique_id_list_ size is " << unique_id_list_.size() << ", count is " << unique_id_count_;
       break;
     }
   }

mindspore/ccsrc/minddata/dataset/util/services.h

@@ -17,9 +17,9 @@
 #define MINDSPORE_CCSRC_MINDDATA_DATASET_UTIL_SERVICES_H_
 
 #include <algorithm>
+#include <map>
 #include <memory>
 #include <mutex>
-#include <set>
 #include <string>
 #include <vector>
 #include "minddata/dataset/util/memory_pool.h"
@@ -27,6 +27,8 @@
 #include "minddata/dataset/util/service.h"
 
 #define UNIQUEID_LEN 36
+#define UNIQUEID_LIST_LIMITS 1024
+#define UNIQUEID_HALF_INDEX ((UNIQUEID_LIST_LIMITS) / 2)
 namespace mindspore {
 namespace dataset {
 class TaskManager;
@@ -98,7 +100,8 @@ class Services {
  private:
   static std::once_flag init_instance_flag_;
   static std::unique_ptr<Services> instance_;
-  static std::set<std::string> unique_id_list_;
+  static std::map<std::string, uint64_t> unique_id_list_;
+  static uint64_t unique_id_count_;
   static std::mutex unique_id_mutex_;
   // A small pool used for small objects that last until the
   // Services Manager shuts down. Used by all sub-services.