remove async executor python api to fix document (#17174)

* remove async executor python api
test=develop

* remove test_async_executor.py
add executor train_from_dataset demo
test=develop

* fix import bug
test=develop

paddle/fluid/API.spec

@@ -39,16 +39,6 @@ paddle.fluid.DataFeedDesc.desc (ArgSpec(args=['self'], varargs=None, keywords=No
 paddle.fluid.DataFeedDesc.set_batch_size (ArgSpec(args=['self', 'batch_size'], varargs=None, keywords=None, defaults=None), ('document', '8d9f44601e0a99dd431f14fd9250cd21'))
 paddle.fluid.DataFeedDesc.set_dense_slots (ArgSpec(args=['self', 'dense_slots_name'], varargs=None, keywords=None, defaults=None), ('document', 'eb894b464bbcd1b4bc8038398954f766'))
 paddle.fluid.DataFeedDesc.set_use_slots (ArgSpec(args=['self', 'use_slots_name'], varargs=None, keywords=None, defaults=None), ('document', '415c56600ce4e198c071cad01409a690'))
-paddle.fluid.AsyncExecutor.__init__ (ArgSpec(args=['self', 'place', 'run_mode'], varargs=None, keywords=None, defaults=(None, '')), ('document', '4e85874dddcd06c38f5717992d741589'))
-paddle.fluid.AsyncExecutor.config_distributed_nodes (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '762980fe0181eb41e3d1081b26ed76b1'))
-paddle.fluid.AsyncExecutor.download_data (ArgSpec(args=['self', 'afs_path', 'local_path', 'fs_default_name', 'ugi', 'file_cnt', 'hadoop_home', 'process_num'], varargs=None, keywords=None, defaults=('$HADOOP_HOME', 12)), ('document', '39e3ccddf8ea8db75ea85287c9147c3b'))
-paddle.fluid.AsyncExecutor.get_instance (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', 'f8688f76a2db1243c7097a60c507b182'))
-paddle.fluid.AsyncExecutor.init_model (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '504f39be2007404a17e5cabea1256c7d'))
-paddle.fluid.AsyncExecutor.init_server (ArgSpec(args=['self', 'dist_desc'], varargs=None, keywords=None, defaults=None), ('document', '384fa5fbb99912db1baf7ef7784bd312'))
-paddle.fluid.AsyncExecutor.init_worker (ArgSpec(args=['self', 'dist_desc', 'startup_program'], varargs=None, keywords=None, defaults=None), ('document', 'f0a36d7c8561039f60a6f6555c7fee0b'))
-paddle.fluid.AsyncExecutor.run (ArgSpec(args=['self', 'program', 'data_feed', 'filelist', 'thread_num', 'fetch', 'mode', 'debug'], varargs=None, keywords=None, defaults=('', False)), ('document', '848fc53484e8326f6325feea87fe955c'))
-paddle.fluid.AsyncExecutor.save_model (ArgSpec(args=['self', 'save_path'], varargs=None, keywords=None, defaults=None), ('document', '145b5c0da01bfff397142e51361f4b75'))
-paddle.fluid.AsyncExecutor.stop (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '5f23d043607bb5d55e466ec3f578e093'))
 paddle.fluid.CompiledProgram.__init__ (ArgSpec(args=['self', 'program_or_graph'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754'))
 paddle.fluid.CompiledProgram.with_data_parallel (ArgSpec(args=['self', 'loss_name', 'build_strategy', 'exec_strategy', 'share_vars_from', 'places'], varargs=None, keywords=None, defaults=(None, None, None, None, None)), ('document', 'a8c7793803cf976680d9478e378fa356'))
 paddle.fluid.CompiledProgram.with_inference_optimize (ArgSpec(args=['self', 'config'], varargs=None, keywords=None, defaults=None), ('document', '9e5b009d850191a010e859189c127fd8'))

python/paddle/fluid/__init__.py

@@ -27,9 +27,6 @@ from .data_feed_desc import *
 from . import dataset
 from .dataset import *
 
-from . import async_executor
-from .async_executor import *
-
 from . import trainer_desc
 from . import inferencer
 
@@ -74,7 +71,7 @@ Tensor = LoDTensor
 __all__ = framework.__all__ + executor.__all__ + \
     trainer_desc.__all__ + inferencer.__all__ + transpiler.__all__ + \
     parallel_executor.__all__ + lod_tensor.__all__ + \
-    data_feed_desc.__all__ + async_executor.__all__ + compiler.__all__ + [
+    data_feed_desc.__all__ + compiler.__all__ + [
         'io',
         'initializer',
         'layers',

python/paddle/fluid/distributed/helper.py

@@ -15,7 +15,7 @@
 
 class FileSystem(object):
     """
-    A file system that support async_executor hadoop client desc. 
+    A file system that support hadoop client desc. 
 
     Args:
         fs_type (string): fs_type, for example is "afs"

python/paddle/fluid/tests/demo/executor_train_dataset.py

@@ -58,9 +58,8 @@ def train():
         tarf.close()
 
     # Initialize dataset description
-    dataset = fluid.DataFeedDesc('train_data/data.prototxt')
+    dataset = fluid.DatasetFactory().create_dataset()
     dataset.set_batch_size(128)  # See API doc for how to change other fields
-    print dataset.desc()  # Debug purpose: see what we get
 
     # define network
     # input text data
@@ -68,7 +67,7 @@ def train():
         name="words", shape=[1], dtype="int64", lod_level=1)
     # label data
     label = fluid.layers.data(name="label", shape=[1], dtype="int64")
-
+    dataset.set_use_var([data, label])
     avg_cost, acc, prediction = bow_net(data, label)
     sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=0.002)
     opt_ops, weight_and_grad = sgd_optimizer.minimize(avg_cost)
@@ -79,18 +78,15 @@ def train():
     executor = fluid.Executor(place)
     executor.run(startup_program)
 
-    async_executor = fluid.AsyncExecutor(place)
     main_program = fluid.default_main_program()
     epochs = 10
     filelist = ["train_data/part-%d" % i for i in range(12)]
+    dataset.set_filelist(filelist)
     for i in range(epochs):
-        thread_num = 4
+        dataset.set_thread(4)
-        async_executor.run(
+        executor.train_from_dataset(
             main_program,  # This can be changed during iteration
             dataset,  # This can be changed during iteration
-            filelist,  # This can be changed during iteration
-            thread_num,  # This can be changed during iteration
-            [data, acc],  # Multiple fetch targets can be specified
             debug=False)
         fluid.io.save_inference_model('imdb/epoch%d.model' % i,
                                       [data.name, label.name], [acc], executor)

python/paddle/fluid/tests/unittests/test_async_executor.py

@@ -1,86 +0,0 @@
-#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import paddle.fluid as fluid
-import paddle
-import unittest
-import tarfile
-import os
-import shutil
-
-proto_str = ('name: "MultiSlotDataFeed"\n'
-             'batch_size: 2\n'
-             'multi_slot_desc {\n'
-             '   slots {\n'
-             '       name: "words"\n'
-             '       type: "uint64"\n'
-             '       is_dense: false\n'
-             '       is_used: true\n'
-             '   }\n'
-             '   slots {\n'
-             '       name: "label"\n'
-             '       type: "uint64"\n'
-             '       is_dense: false\n'
-             '       is_used: true\n'
-             '   }\n'
-             '}')
-
-URL = 'http://paddle-unittest-data.gz.bcebos.com/python_paddle_fluid_tests_demo_async-executor/train_data.tar.gz'
-MD5 = '2a405a31508969b3ab823f42c0f522ca'
-
-
-def bow_net(data,
-            label,
-            dict_dim=89528,
-            emb_dim=128,
-            hid_dim=128,
-            hid_dim2=96,
-            class_dim=2):
-    """
-    BOW net
-    This model is from https://github.com/PaddlePaddle/models:
-    models/fluid/PaddleNLP/text_classification/nets.py
-    """
-    # embedding
-    emb = fluid.layers.embedding(
-        input=data, size=[dict_dim, emb_dim], is_sparse=True)
-    bow = fluid.layers.sequence_pool(input=emb, pool_type='sum')
-    bowh = fluid.layers.tanh(bow)
-    # fc layer after conv
-    fc_1 = fluid.layers.fc(input=bowh, size=hid_dim, act="tanh")
-    fc_2 = fluid.layers.fc(input=fc_1, size=hid_dim2, act="tanh")
-    # probability of each class
-    prediction = fluid.layers.fc(input=[fc_2], size=class_dim, act="softmax")
-    # cross entropy loss
-    cost = fluid.layers.cross_entropy(input=prediction, label=label)
-    # mean loss
-    avg_cost = fluid.layers.mean(x=cost)
-    acc = fluid.layers.accuracy(input=prediction, label=label)
-    return avg_cost, acc, prediction
-
-
-class TestAsyncExecutor(unittest.TestCase):
-    def setUp(self):
-        with open('./data.prototxt', 'w+') as f:
-            f.write(proto_str)
-            f.close()
-
-        with tarfile.open(paddle.dataset.common.download(URL, "imdb",
-                                                         MD5)) as tarf:
-            tarf.extractall(path='./')
-            tarf.close()
-
-
-if __name__ == '__main__':
-    unittest.main()