[Auto Parallel] Improve the interface and the underlying mechanisms (#36617)

* default dist op * add dist_attr for dist op * add unitest * update inputname * update function name * add unitest * update CMakeLists.txt for CI * fix dis_matmul * fix compile error * update matmul to matmul_v2 * unify api * unify api * todo * update distop forward func * update distop forward func * auto parallel backward * update dist op * autoparallel backward * add backward for embedding * temp1 * temp2 * temp3 * temp4 * backward done1 * backward done2 * backward done3 * dist embedding remove mp mode * dist matmul remove mp mode * update dist embedding 『 * dist op init1 * dist op init 2 * update unitest * context remove parallel mode * partitioner remove parallel mode * update unitest * a more general method to support varying mesh in pipeline parallel * support varying mesh in pipeline parallel * embedding support varying mesh in pipeline parallel * matmul support varying mesh in pipeline parallel * default dist op support varying mesh in pipeline parallel * dist attribute for startup program * default dist op support varying mesh in pipeline parallel 2 * partitoner support varying mesh in pipeline parallel * revise logic for auto compeletion * revise framework.py * revise reshard unitest * revise unitest for parallelize * chmod * fixed bug for dist embedding name mapping * Improve the interface and the underlying mechanisms of auto parallel * revise completion for backward * revise completion for update * revise completion for update * update unitest * chmod * bugfix for grad_op output var's mesh * Modify codes for pr 36744 * Remove unnecessary comments in framework.py * Remove unnecessary comments in completion.py Co-authored-by: JZ-LIANG <jianzhongliang10@gmail.com> Co-authored-by: zhaoyingli <zhaoyingli@baidu.com> Co-authored-by: JZ-LIANG <38102074+JZ-LIANG@users.noreply.github.com>

Author: 3 people

python/paddle/distributed/__init__.py

@@ -43,10 +43,6 @@
 
 from .auto_parallel import shard_op # noqa: F401
 from .auto_parallel import shard_tensor # noqa: F401
-from .auto_parallel import set_shard_mask # noqa: F401
-from .auto_parallel import set_offload_device # noqa: F401
-from .auto_parallel import set_pipeline_stage # noqa: F401
-from .auto_parallel import ProcessMesh # noqa: F401
 
 from .fleet import BoxPSDataset # noqa: F401
 

python/paddle/distributed/auto_parallel/__init__.py

@@ -14,10 +14,11 @@
 
 from .interface import shard_tensor # noqa: F401
 from .interface import shard_op # noqa: F401
-from .interface import set_shard_mask # noqa: F401
-from .interface import set_offload_device # noqa: F401
-from .interface import set_pipeline_stage # noqa: F401
-from .interface import ProcessMesh # noqa: F401
+from .process_mesh import ProcessMesh
+# from .interface import set_shard_mask # noqa: F401
+# from .interface import set_offload_device # noqa: F401
+# from .interface import set_pipeline_stage # noqa: F401
+# from .interface import ProcessMesh # noqa: F401
 from .completion import complete_annotation # noqa: F401
 from .completion import complete_backward_annotation # noqa: F401
 from .reshard import reshard # noqa: F401