[CustomOp] Split build directory for each setup.py (#31124)

* split build directory for each setup.py

* fix template string

python/paddle/utils/cpp_extension/cpp_extension.py

@@ -22,6 +22,7 @@ import re
 import setuptools
 from setuptools.command.easy_install import easy_install
 from setuptools.command.build_ext import build_ext
+from distutils.command.build import build
 
 from .extension_utils import find_cuda_home, normalize_extension_kwargs, add_compile_flag, bootstrap_context
 from .extension_utils import is_cuda_file, prepare_unix_cudaflags, prepare_win_cudaflags, add_std_without_repeat, get_build_directory
@@ -103,6 +104,13 @@ def setup(**attr):
     assert 'easy_install' not in cmdclass
     cmdclass['easy_install'] = EasyInstallCommand
 
+    # Note(Aurelius84): Add rename build_base directory hook in build command.
+    # To avoid using same build directory that will lead to remove the directory
+    # by mistake while parallelling execute setup.py, for example on CI.
+    assert 'build' not in cmdclass
+    build_base = os.path.join('build', attr['name'])
+    cmdclass['build'] = BuildCommand.with_options(build_base=build_base)
+
     # Always set zip_safe=False to make compatible in PY2 and PY3
     # See http://peak.telecommunity.com/DevCenter/setuptools#setting-the-zip-safe-flag
     attr['zip_safe'] = False
@@ -491,6 +499,43 @@ class EasyInstallCommand(easy_install, object):
                 assert os.path.exists(new_so_path)
 
 
+class BuildCommand(build, object):
+    """
+    Extend build Command to control the behavior of specifying `build_base` root directory.
+
+    NOTE(Aurelius84): This is a hook subclass inherited Command used to specify customized
+                      build_base directory.
+    """
+
+    @classmethod
+    def with_options(cls, **options):
+        """
+        Returns a BuildCommand subclass containing use-defined options.
+        """
+
+        class cls_with_options(cls):
+            def __init__(self, *args, **kwargs):
+                kwargs.update(options)
+                cls.__init__(self, *args, **kwargs)
+
+        return cls_with_options
+
+    def __init__(self, *args, **kwargs):
+        # Note: shall put before super()
+        self._specified_build_base = kwargs.get('build_base', None)
+
+        super(BuildCommand, self).__init__(*args, **kwargs)
+
+    def initialize_options(self):
+        """
+        build_base is root directory for all sub-command, such as
+        build_lib, build_temp. See `distutils.command.build` for details.
+        """
+        super(BuildCommand, self).initialize_options()
+        if self._specified_build_base is not None:
+            self.build_base = self._specified_build_base
+
+
 def load(name,
          sources,
          extra_cflags=None,
@@ -569,11 +614,13 @@ def load(name,
           verbose)
 
     # write setup.py file and compile it
-    _write_setup_file(name, sources, file_path, extra_include_paths,
+    build_base_dir = os.path.join(build_directory, name)
-                      compile_flags, extra_ldflags, verbose)
+    _write_setup_file(name, sources, file_path, build_base_dir,
+                      extra_include_paths, compile_flags, extra_ldflags,
+                      verbose)
     _jit_compile(file_path, interpreter, verbose)
 
     # import as callable python api
-    custom_op_api = _import_module_from_library(name, build_directory, verbose)
+    custom_op_api = _import_module_from_library(name, build_base_dir, verbose)
 
     return custom_op_api

python/paddle/utils/cpp_extension/extension_utils.py

@@ -580,6 +580,7 @@ def _get_api_inputs_str(op_name):
 def _write_setup_file(name,
                       sources,
                       file_path,
+                      build_dir,
                       include_dirs,
                       compile_flags,
                       link_args,
@@ -600,7 +601,7 @@ def _write_setup_file(name,
                 extra_compile_args={extra_compile_args},
                 extra_link_args={extra_link_args})],
         cmdclass={{"build_ext" : BuildExtension.with_options(
-            output_dir=get_build_directory(),
+            output_dir='{build_dir}',
             no_python_abi_suffix=True,
             use_new_method={use_new_method})
         }})""").lstrip()
@@ -617,6 +618,7 @@ def _write_setup_file(name,
         include_dirs=list2str(include_dirs),
         extra_compile_args=list2str(compile_flags),
         extra_link_args=list2str(link_args),
+        build_dir=build_dir,
         use_new_method=use_new_custom_op_load_method())
 
     log_v('write setup.py into {}'.format(file_path), verbose)