Merge branch 'develop' into mv_xx_axis_op

Author: m3ngyang

.gitignore

@@ -7,9 +7,11 @@ paddle/fluid/op_use_default_grad_maker_DEV.spec
 paddle/fluid/op_use_default_grad_maker_PR.spec
 paddle/phi/api/backward/backward_api.h
 paddle/phi/api/include/api.h
+paddle/phi/api/include/sparse_api.h
 paddle/phi/api/lib/api.cc
 paddle/phi/api/lib/dygraph_api.*
 paddle/phi/api/lib/backward_api.cc
+paddle/phi/api/lib/sparse_api.cc
 paddle/phi/extension.h
 paddle/phi/include/*
 paddle/phi/infermeta/generated.*
@@ -49,6 +51,9 @@ tools/__pycache__
 # This file is automatically generated.
 # TODO(zhiqiang) Move this file to build directory.
 paddle/infrt/dialect/pd_ops.td
+paddle/infrt/dialect/phi/ir/phi_cpu_kernels.td
+paddle/infrt/dialect/phi/ir/phi_gpu_kernels.td
+tools/infrt/kernels.json
 paddle/infrt/dialect/pd_ops_info.h
 .lit_test_times.txt
 paddle/infrt/tests/dialect/Output

CMakeLists.txt

@@ -238,7 +238,8 @@ option(WITH_MIPS "Compile PaddlePaddle with mips support" OFF)
 option(WITH_MUSL "Compile with musl libc instead of gblic" OFF)
 option(WITH_UNITY_BUILD "Compile with UnityBuild mode" OFF)
 option(WITH_STRIP "Strip so files of Whl packages" OFF)
-option(NEW_RELEASE_CUBIN "PaddlePaddle next-level release strategy for pypi cubin package" OFF)
+option(NEW_RELEASE_PYPI "PaddlePaddle next-level release strategy for pypi cubin package" OFF)
+option(NEW_RELEASE_ALL "PaddlePaddle next-level release strategy for all arches cubin package" OFF)
 option(NEW_RELEASE_JIT "PaddlePaddle next-level release strategy for backup jit package" OFF)
 option(WITH_ASCEND_INT64 "Compile with int64 kernel for ascend NPU" OFF)
 option(WITH_POCKETFFT "Compile with pocketfft support" ON)

README.md

@@ -15,7 +15,7 @@ English | [简体中文](./README_cn.md)
 Welcome to the PaddlePaddle GitHub.
 
 PaddlePaddle, as the only independent R&D deep learning platform in China, has been officially open-sourced to professional communities since 2016. It is an industrial platform with advanced technologies and rich features that cover core deep learning frameworks, basic model libraries, end-to-end development kits, tools & components as well as service platforms.
-PaddlePaddle is originated from industrial practices with dedication and commitments to industrialization. It has been widely adopted by a wide range of sectors including manufacturing, agriculture, enterprise service, and so on while serving more than 2.3 million developers. With such advantages, PaddlePaddle has helped an increasing number of partners commercialize AI.
+PaddlePaddle is originated from industrial practices with dedication and commitments to industrialization. It has been widely adopted by a wide range of sectors including manufacturing, agriculture, enterprise service, and so on while serving more than 4 million developers. With such advantages, PaddlePaddle has helped an increasing number of partners commercialize AI.
 
 
 

README_cn.md

@@ -15,7 +15,7 @@
 
 欢迎来到 PaddlePaddle GitHub
 
-飞桨(PaddlePaddle)以百度多年的深度学习技术研究和业务应用为基础，是中国首个自主研发、功能完备、 开源开放的产业级深度学习平台，集深度学习核心训练和推理框架、基础模型库、端到端开发套件和丰富的工具组件于一体。目前，飞桨累计开发者265万，服务企业10万家，基于飞桨开源深度学习平台产生了34万个模型。飞桨助力开发者快速实现AI想法，快速上线AI业务。帮助越来越多的行业完成AI赋能，实现产业智能化升级。
+飞桨(PaddlePaddle)以百度多年的深度学习技术研究和业务应用为基础，是中国首个自主研发、功能完备、 开源开放的产业级深度学习平台，集深度学习核心训练和推理框架、基础模型库、端到端开发套件和丰富的工具组件于一体。目前，飞桨累计开发者406万，服务企业15.7万家，基于飞桨开源深度学习平台产生了47.6万个模型。飞桨助力开发者快速实现AI想法，快速上线AI业务。帮助越来越多的行业完成AI赋能，实现产业智能化升级。
 
 ## 安装
 

cmake/cuda.cmake

@@ -6,16 +6,22 @@ if(WITH_NV_JETSON)
  add_definitions(-DWITH_NV_JETSON)
  set(paddle_known_gpu_archs "53 62 72")
  set(paddle_known_gpu_archs10 "53 62 72")
-elseif(NEW_RELEASE_CUBIN)
+elseif(NEW_RELEASE_ALL)
+ message("Using New Release Strategy - All Arches Packge")
+ add_definitions(-DNEW_RELEASE_ALL)
+ set(paddle_known_gpu_archs "35 50 52 60 61 70 75 80 86")
+ set(paddle_known_gpu_archs10 "35 50 52 60 61 70 75")
+ set(paddle_known_gpu_archs11 "35 50 52 60 61 70 75 80")
+elseif(NEW_RELEASE_PYPI)
  message("Using New Release Strategy - Cubin Packge")
- add_definitions(-DNEW_RELEASE_CUBIN)
- set(paddle_known_gpu_archs "35 37 50 52 60 61 70 75 80 86")
- set(paddle_known_gpu_archs10 "50 60 70 75")
- set(paddle_known_gpu_archs11 "60 70 75 80")
+ add_definitions(-DNEW_RELEASE_PYPI)
+ set(paddle_known_gpu_archs "35 50 52 60 61 70 75 80 86")
+ set(paddle_known_gpu_archs10 "")
+ set(paddle_known_gpu_archs11 "60 61 70 75 80")
 elseif(NEW_RELEASE_JIT)
  message("Using New Release Strategy - JIT Packge")
  add_definitions(-DNEW_RELEASE_JIT)
- set(paddle_known_gpu_archs "35 37 50 52 60 61 70 75 80 86")
+ set(paddle_known_gpu_archs "35 50 52 60 61 70 75 80 86")
  set(paddle_known_gpu_archs10 "35 50 60 70 75")
  set(paddle_known_gpu_archs11 "35 50 60 70 75 80")
 else()
@@ -148,7 +154,7 @@ function(select_nvcc_arch_flags out_variable)
 
  # remove dots and convert to lists
  string(REGEX REPLACE "\\." "" cuda_arch_bin "${cuda_arch_bin}")
- string(REGEX REPLACE "\\." "" cuda_arch_ptx "${CUDA_ARCH_PTX}")
+ string(REGEX REPLACE "\\." "" cuda_arch_ptx "${cuda_arch_ptx}")
  string(REGEX MATCHALL "[0-9()]+" cuda_arch_bin "${cuda_arch_bin}")
  string(REGEX MATCHALL "[0-9]+" cuda_arch_ptx "${cuda_arch_ptx}")
 

cmake/external/xpu.cmake

@@ -36,7 +36,7 @@ ENDIF()
 
 if(NOT DEFINED XPU_BASE_URL)
  SET(XPU_BASE_URL_WITHOUT_DATE "https://baidu-kunlun-product.cdn.bcebos.com/KL-SDK/klsdk-dev")
- SET(XPU_BASE_URL "${XPU_BASE_URL_WITHOUT_DATE}/20220219")
+ SET(XPU_BASE_URL "${XPU_BASE_URL_WITHOUT_DATE}/20220228")
 else()
  SET(XPU_BASE_URL "${XPU_BASE_URL}")
 endif()

cmake/generic.cmake

@@ -580,8 +580,8 @@ function(hip_library TARGET_NAME)
  cmake_parse_arguments(hip_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
  if(hip_library_SRCS)
  # FindHIP.cmake defined hip_add_library, HIP_SOURCE_PROPERTY_FORMAT is requried if no .cu files found
- if(NOT ${CMAKE_CURRENT_SOURCE_DIR} MATCHES ".*/operators")
-  set_source_files_properties(${hip_library_SRCS} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
+ if(NOT (${CMAKE_CURRENT_SOURCE_DIR} MATCHES ".*/operators" OR ${CMAKE_CURRENT_SOURCE_DIR} MATCHES ".*/phi/kernels"))
+ set_source_files_properties(${hip_library_SRCS} PROPERTIES HIP_SOURCE_PROPERTY_FORMAT 1)
  endif()
  if (hip_library_SHARED OR hip_library_shared) # build *.so
  hip_add_library(${TARGET_NAME} SHARED ${hip_library_SRCS})
@@ -651,6 +651,7 @@ function(hip_test TARGET_NAME)
  set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cpu_deterministic=true)
  set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true)
  set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cudnn_deterministic=true)
+ set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR}/python/paddle/libs:$LD_LIBRARY_PATH")
  endif()
 endfunction(hip_test)
 
@@ -667,6 +668,7 @@ function(xpu_library TARGET_NAME)
  else()
  xpu_add_library(${TARGET_NAME} STATIC ${xpu_library_SRCS} DEPENDS ${xpu_library_DEPS})
  find_fluid_modules(${TARGET_NAME})
+ find_phi_modules(${TARGET_NAME})
  endif()
  if (xpu_library_DEPS)
  add_dependencies(${TARGET_NAME} ${xpu_library_DEPS})

cmake/phi.cmake

@@ -83,6 +83,8 @@ function(kernel_declare TARGET_LIST)
  file(APPEND ${kernel_declare_file} "PD_DECLARE_KERNEL(${kernel_name}, XPU, ALL_LAYOUT);\n")
  elseif (${kernel_path} MATCHES "./gpudnn\/")
  file(APPEND ${kernel_declare_file} "PD_DECLARE_KERNEL(${kernel_name}, GPUDNN, ALL_LAYOUT);\n")
+ elseif (${kernel_path} MATCHES "./kps\/")
+ file(APPEND ${kernel_declare_file} "PD_DECLARE_KERNEL(${kernel_name}, KPS, ALL_LAYOUT);\n")
  else ()
  # deal with device independent kernel, now we use CPU temporaary
  file(APPEND ${kernel_declare_file} "PD_DECLARE_KERNEL(${kernel_name}, CPU, ALL_LAYOUT);\n")
@@ -97,6 +99,7 @@ function(kernel_library TARGET)
  set(gpu_srcs)
  set(xpu_srcs)
  set(gpudnn_srcs)
+ set(kps_srcs)
  set(selected_rows_srcs)
  # parse and save the deps kerenl targets
  set(all_srcs)
@@ -128,6 +131,9 @@ function(kernel_library TARGET)
  if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu.cc)
  list(APPEND gpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu.cc)
  endif()
+ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu)
+ list(APPEND gpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu)
+ endif()
  if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}_gpudnn.cu)
  list(APPEND gpudnn_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}_gpudnn.cu)
  endif()
@@ -137,6 +143,15 @@ function(kernel_library TARGET)
  list(APPEND xpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/xpu/${TARGET}.cc)
  endif()
  endif()
+ if (WITH_XPU_KP)
+ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu)
+ # Change XPU2 file suffix
+ # NOTE(chenweihang): If we can be sure that the *.kps suffix is no longer used, it can be copied directly to *.xpu
+ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/kps)
+ file(RENAME ${CMAKE_CURRENT_BINARY_DIR}/kps/${TARGET}.cu ${CMAKE_CURRENT_BINARY_DIR}/kps/${TARGET}.kps)
+ list(APPEND kps_srcs ${CMAKE_CURRENT_BINARY_DIR}/kps/${TARGET}.kps)
+ endif()
+ endif()
  else()
  # TODO(chenweihang): impl compile by source later
  endif()
@@ -150,6 +165,7 @@ function(kernel_library TARGET)
  list(APPEND all_srcs ${gpu_srcs})
  list(APPEND all_srcs ${xpu_srcs})
  list(APPEND all_srcs ${gpudnn_srcs})
+ list(APPEND all_srcs ${kps_srcs})
  foreach(src ${all_srcs})
  file(READ ${src} target_content)
  string(REGEX MATCHALL "#include \"paddle\/phi\/kernels\/[a-z0-9_]+_kernel.h\"" include_kernels ${target_content})
@@ -159,11 +175,11 @@ function(kernel_library TARGET)
  string(REGEX MATCHALL "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/[a-z0-9_]+_kernel.h\"" include_kernels ${target_content})
  endif()
  foreach(include_kernel ${include_kernels})
- if ("${kernel_library_SUB_DIR}" STREQUAL "")
- string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/" "" kernel_name ${include_kernel})
- else()
- string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/" "" kernel_name ${include_kernel})
- endif()
+  if ("${kernel_library_SUB_DIR}" STREQUAL "")
+  string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/" "" kernel_name ${include_kernel})
+  else()
+  string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/" "" kernel_name ${include_kernel})
+  endif()
  string(REGEX REPLACE ".h\"" "" kernel_name ${kernel_name})
  list(APPEND kernel_deps ${kernel_name})
  endforeach()
@@ -176,11 +192,20 @@ function(kernel_library TARGET)
  list(LENGTH gpu_srcs gpu_srcs_len)
  list(LENGTH xpu_srcs xpu_srcs_len)
  list(LENGTH gpudnn_srcs gpudnn_srcs_len)
+ list(LENGTH kps_srcs kps_srcs_len)
  list(LENGTH selected_rows_srcs selected_rows_srcs_len)
 
+ # kernel source file level
+ # level 1: base device kernel
+ # - cpu_srcs / gpu_srcs / xpu_srcs / kps_srcs
+ # level 2: device-independent kernel
+ # - common_srcs
+ # level 3: Kernel implemented by reusing device-independent kernel
+ # - selected_rows_srcs
+
  # Build Target according different src organization
  if((${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR
- ${xpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0) AND
+ ${xpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0) AND
  (${common_srcs_len} GREATER 0 OR ${selected_rows_srcs_len} GREATER 0))
  # If the common_srcs/selected_rows_srcs depends on specific device srcs, build target using this rule.
  if (WITH_GPU)
@@ -193,14 +218,19 @@ function(kernel_library TARGET)
  hip_library(${TARGET}_part SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  hip_library(${TARGET} SRCS ${common_srcs} ${selected_rows_srcs} DEPS ${TARGET}_part)
  endif()
+ elseif (WITH_XPU_KP)
+ if (${cpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0)
+ xpu_library(${TARGET}_part SRCS ${cpu_srcs} ${xpu_srcs} ${kps_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ xpu_library(${TARGET} SRCS ${common_srcs} ${selected_rows_srcs} DEPS ${TARGET}_part)
+ endif()
  else()
  if (${cpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0)
  cc_library(${TARGET}_part SRCS ${cpu_srcs} ${xpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  cc_library(${TARGET} SRCS ${common_srcs} ${selected_rows_srcs} DEPS ${TARGET}_part)
  endif()
  endif()
  # If there are only specific device srcs, build target using this rule.
- elseif (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
+ elseif (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0)
  if (WITH_GPU)
  if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
  nv_library(${TARGET} SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
@@ -209,6 +239,10 @@ function(kernel_library TARGET)
  if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
  hip_library(${TARGET} SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  endif()
+ elseif (WITH_XPU_KP)
+ if (${cpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0)
+ xpu_library(${TARGET} SRCS ${cpu_srcs} ${xpu_srcs} ${kps_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ endif()
  else()
  if (${cpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0)
  cc_library(${TARGET} SRCS ${cpu_srcs} ${xpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
@@ -222,6 +256,9 @@ function(kernel_library TARGET)
  elseif (WITH_ROCM)
  hip_library(${TARGET}_part SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  hip_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${TARGET}_part)
+ elseif (WITH_XPU_KP)
+ xpu_library(${TARGET}_part SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ xpu_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${TARGET}_part)
  else()
  cc_library(${TARGET}_part SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  cc_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${TARGET}_part)
@@ -232,6 +269,8 @@ function(kernel_library TARGET)
  nv_library(${TARGET} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  elseif (WITH_ROCM)
  hip_library(${TARGET} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ elseif (WITH_XPU_KP)
+ xpu_library(${TARGET} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  else()
  cc_library(${TARGET} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  endif()
@@ -240,6 +279,8 @@ function(kernel_library TARGET)
  nv_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  elseif (WITH_ROCM)
  hip_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ elseif (WITH_XPU_KP)
+ xpu_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  else()
  cc_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  endif()
@@ -249,7 +290,7 @@ function(kernel_library TARGET)
 
  if (${target_build_flag} EQUAL 1)
  if (${common_srcs_len} GREATER 0 OR ${cpu_srcs_len} GREATER 0 OR
- ${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR
+ ${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0 OR
  ${gpudnn_srcs_len} GREATER 0 OR ${selected_rows_srcs_len} GREATER 0)
  # append target into PHI_KERNELS property
  get_property(phi_kernels GLOBAL PROPERTY PHI_KERNELS)
@@ -275,6 +316,9 @@ function(kernel_library TARGET)
  if (${gpudnn_srcs_len} GREATER 0)
  kernel_declare(${gpudnn_srcs})
  endif()
+ if (${kps_srcs_len} GREATER 0)
+ kernel_declare(${kps_srcs})
+ endif()
  if (${selected_rows_srcs_len} GREATER 0)
  kernel_declare(${selected_rows_srcs})
  endif()

paddle/fluid/distributed/collective/CMakeLists.txt

@@ -1,4 +1,8 @@
 cc_library(processgroup SRCS ProcessGroup.cc DEPS phi phi_api eager_api)
+if (WITH_DISTRIBUTE)
+ cc_library(processgroup_gloo SRCS ProcessGroupGloo.cc DEPS phi phi_api eager_api gloo_wrapper)
+endif()
+cc_library(eager_reducer SRCS reducer.cc DEPS eager_api processgroup)
 
 if(WITH_NCCL)
  cc_library(processgroup_nccl SRCS ProcessGroupNCCL.cc DEPS place cuda_stream enforce collective_helper device_context phi phi_api eager_api)