|
|
@ -661,22 +661,25 @@ class TestLoadSliceVar(TranspilerTest):
|
|
|
|
|
|
|
|
|
|
|
|
class TestNCCL2Transpile(TranspilerTest):
|
|
|
|
class TestNCCL2Transpile(TranspilerTest):
|
|
|
|
def test_nccl2_transpile(self):
|
|
|
|
def test_nccl2_transpile(self):
|
|
|
|
main = fluid.Program()
|
|
|
|
if fluid.core.is_compiled_with_cuda(): #test nccl2 only with cuda
|
|
|
|
startup = fluid.Program()
|
|
|
|
main = fluid.Program()
|
|
|
|
with fluid.program_guard(main, startup):
|
|
|
|
startup = fluid.Program()
|
|
|
|
self.net_conf()
|
|
|
|
with fluid.program_guard(main, startup):
|
|
|
|
|
|
|
|
self.net_conf()
|
|
|
|
config = fluid.DistributeTranspilerConfig()
|
|
|
|
|
|
|
|
config.mode = "nccl2"
|
|
|
|
config = fluid.DistributeTranspilerConfig()
|
|
|
|
t = fluid.DistributeTranspiler(config=config)
|
|
|
|
config.mode = "nccl2"
|
|
|
|
t.transpile(
|
|
|
|
t = fluid.DistributeTranspiler(config=config)
|
|
|
|
0,
|
|
|
|
t.transpile(
|
|
|
|
trainers="127.0.0.1:6174,127.0.0.1:6175",
|
|
|
|
0,
|
|
|
|
current_endpoint="127.0.0.1:6174",
|
|
|
|
trainers="127.0.0.1:6174,127.0.0.1:6175",
|
|
|
|
startup_program=startup)
|
|
|
|
current_endpoint="127.0.0.1:6174",
|
|
|
|
print([op.type for op in startup.global_block().ops])
|
|
|
|
startup_program=startup)
|
|
|
|
self.assertEqual(startup.global_block().ops[-1].type, "gen_nccl_id")
|
|
|
|
print([op.type for op in startup.global_block().ops])
|
|
|
|
self.assertIsNotNone(startup.global_block().vars.get("NCCLID"))
|
|
|
|
self.assertEqual(startup.global_block().ops[-1].type, "gen_nccl_id")
|
|
|
|
|
|
|
|
self.assertIsNotNone(startup.global_block().vars.get("NCCLID"))
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
if __name__ == "__main__":
|
|
|
|