refine code

demo/mnist/api_train_v2.py

@@ -41,10 +41,12 @@ def main():
 
     trainer.train(
         train_data_reader=train_reader,
-        topology=[cost],
+        topology=cost,
         parameters=parameters,
         event_handler=event_handler,
-        batch_size=32)  # batch size should be refactor in Data reader
+        batch_size=32,  # batch size should be refactor in Data reader
+        reader_dict={images.name: 0,
+                     label.name: 1})
 
 
 if __name__ == '__main__':

python/paddle/v2/data_feeder.py

@@ -23,7 +23,7 @@ class DataFeeder(DataProviderConverter):
     """
     DataFeeder converts the data returned by paddle.reader into a data structure
     of Arguments which is defined in the API. The paddle.reader usually returns
-    a list of mini-batch data entries. Each data entry in the list is one sampe.
+    a list of mini-batch data entries. Each data entry in the list is one sample.
     Each sample is a list or a tuple with one feature or multiple features.
     DataFeeder converts this mini-batch data entries into Arguments in order
     to feed it to C++ interface.

python/paddle/v2/data_type.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 from paddle.trainer.PyDataProvider2 import \
-    InputType, dense_vector, sparse_binary_vector,\
+    InputType, DataType, dense_vector, sparse_binary_vector,\
     sparse_vector, integer_value, integer_value_sequence
 
 __all__ = [
-    'InputType', 'dense_vector', 'sparse_binary_vector', 'sparse_vector',
-    'integer_value', 'integer_value_sequence'
+    'InputType', 'DataType', 'dense_vector', 'sparse_binary_vector',
+    'sparse_vector', 'integer_value', 'integer_value_sequence'
 ]

python/paddle/v2/layer.py

@@ -284,6 +284,7 @@ def mixed(size=0,
     return MixedLayerV2(size, input, name, act, bias_attr, layer_attr)
 
 
+LayerV2 = Layer
 data = DataLayerV2
 AggregateLevel = conf_helps.layers.AggregateLevel
 ExpandLevel = conf_helps.layers.ExpandLevel

python/paddle/v2/parameters.py

@@ -2,7 +2,7 @@ import numpy as np
 import py_paddle.swig_paddle as api
 from paddle.proto.ParameterConfig_pb2 import ParameterConfig
 
-import topology as v2_topology
+from topology import Topology
 
 __all__ = ['Parameters', 'create']
 
@@ -13,7 +13,7 @@ def create(layers):
     :param layers:
     :return:
     """
-    topology = v2_topology.Topology(layers)
+    topology = Topology(layers)
     pool = Parameters()
     for param in topology.proto().parameters:
         pool.__append_config__(param)

python/paddle/v2/tests/CMakeLists.txt

@@ -8,5 +8,5 @@ add_test(NAME test_v2_api
 
 add_test(NAME topology_test
         COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/
-        ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/topology_test.py
+        ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/v2/tests/test_topology.py
         WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle)

python/paddle/v2/tests/test_topology.py

@@ -30,14 +30,19 @@ class TestTopology(unittest.TestCase):
                              act=conf_helps.SoftmaxActivation())
         cost = layer.classification_cost(input=inference, label=label)
         topo = topology.Topology(cost)
-        type = topo.data_type()
-        self.assertEqual(len(type), 2)
-        self.assertEqual(type[0][0], "pixel")
-        self.assertEqual(type[0][1].type, data_type.DataType.Dense)
-        self.assertEqual(type[0][1].dim, 784)
-        self.assertEqual(type[1][0], "label")
-        self.assertEqual(type[1][1].type, data_type.DataType.Index)
-        self.assertEqual(type[1][1].dim, 10)
+        data_types = topo.data_type()
+        self.assertEqual(len(data_types), 2)
+        pixel_data_type = filter(lambda type: type[0] == "pixel", data_types)
+        self.assertEqual(len(pixel_data_type), 1)
+        pixel_data_type = pixel_data_type[0]
+        self.assertEqual(pixel_data_type[1].type, data_type.DataType.Dense)
+        self.assertEqual(pixel_data_type[1].dim, 784)
+
+        label_data_type = filter(lambda type: type[0] == "label", data_types)
+        self.assertEqual(len(label_data_type), 1)
+        label_data_type = label_data_type[0]
+        self.assertEqual(label_data_type[1].type, data_type.DataType.Index)
+        self.assertEqual(label_data_type[1].dim, 10)
 
     def test_get_layer(self):
         pixel = layer.data(name='pixel', type=data_type.dense_vector(784))

python/paddle/v2/topology.py

@@ -49,30 +49,30 @@ class Topology(object):
         result_layer = []
 
         def find_layer_by_name(layer, layer_name):
-            if layer.name == layer_name and len(result_layer) == 0:
+            if len(result_layer) == 1:
+                return
+            elif layer.name == layer_name:
                 result_layer.append(layer)
+            else:
                 for parent_layer in layer.__parent_layers__.values():
                     find_layer_by_name(parent_layer, layer_name)
 
         for layer in self.layers:
             find_layer_by_name(layer, name)
 
+        assert len(result_layer) == 1
         return result_layer[0]
 
-    def data_layer(self):
+    def data_layers(self):
         """
         get all data layer
         :return:
         """
-        data_layers = []
+        data_layers = set()
 
         def find_data_layer(layer):
-            assert isinstance(layer, layer.LayerV2)
             if isinstance(layer, v2_layer.DataLayerV2):
-                if len(
-                        filter(lambda data_layer: data_layer.name == layer.name,
-                               data_layers)) == 0:
-                    data_layers.append(layer)
+                data_layers.add(layer)
             for parent_layer in layer.__parent_layers__.values():
                 find_data_layer(parent_layer)
 
@@ -85,14 +85,9 @@ class Topology(object):
         """
         get data_type from proto, such as:
         [('image', dense_vector(768)), ('label', integer_value(10))]
-        the order is the same with __model_config__.input_layer_names
         """
-        data_types_lists = []
-        for layer_name in self.__model_config__.input_layer_names:
-            data_types_lists.append(
-                (layer_name, self.get_layer(layer_name).type))
-
-        return data_types_lists
+        return [(data_layer.name, data_layer.type)
+                for data_layer in self.data_layers()]
 
 
 def __check_layer_type__(layer):

python/paddle/v2/trainer.py

@@ -1,13 +1,12 @@
 import collections
 
 import py_paddle.swig_paddle as api
-from py_paddle import DataProviderConverter
 
 from data_feeder import DataFeeder
+from topology import Topology
 from . import event as v2_event
 from . import optimizer as v2_optimizer
 from . import parameters as v2_parameters
-from . import topology as v2_topology
 
 __all__ = ['ITrainer', 'SGD']
 
@@ -69,7 +68,6 @@ class SGD(ITrainer):
               test_data_reader=None,
               event_handler=None,
               batch_size=32,
-              data_types=None,
               reader_dict=None):
         """
         Training method. Will train num_passes of input data.
@@ -83,13 +81,12 @@ class SGD(ITrainer):
                               occurred.
         :type event_handler: (BaseEvent) => None
         :param batch_size: Not important, will be removed after data refactor.
-        :param data_types: Not important, will be removed after data refactor.
         :return:
         """
         if event_handler is None:
             event_handler = default_event_handler
 
-        topology = v2_topology.Topology(topology)
+        topology = Topology(topology)
 
         __check_train_args__(**locals())
 
@@ -109,10 +106,7 @@ class SGD(ITrainer):
         assert isinstance(pass_evaluator, api.Evaluator)
         out_args = api.Arguments.createArguments(0)
 
-        data_types_lists = [data_type[1] for data_type in topology.data_type()]
-        converter = DataProviderConverter(input_types=data_types_lists)
-
-        feeder = DataFeeder(data_types, reader_dict)
+        feeder = DataFeeder(topology.data_type(), reader_dict)
 
         for pass_id in xrange(num_passes):
             event_handler(v2_event.BeginPass(pass_id))
@@ -195,7 +189,7 @@ def __check_train_args__(train_data_reader, topology, parameters,
             raise ValueError('test_data_reader should be a function, which can '
                              'return a iterator')
 
-    if not isinstance(topology, v2_topology.Topology):
+    if not isinstance(topology, Topology):
         raise ValueError('topology should be a model config')
 
     if not isinstance(parameters, v2_parameters.Parameters):