Merge branch 'develop' into group_norm_opt

Author: Zjq9409

CMakeLists.txt

@@ -330,6 +330,7 @@ if(WITH_BRPC_RDMA)
  endif()
 endif()
 
+
 if(WITH_GPU)
  include(cuda)
  # lite subgraph compilation depends on CUDNN_ROOT,

cmake/external/llvm.cmake

@@ -99,7 +99,7 @@ endfunction()
 
 function(mlir_add_rewriter td_base)
  set(LLVM_TARGET_DEFINITIONS ${td_base}.td)
- mlir_tablegen(${td_base}.hpp.inc -gen-rewriters "-I${CMAKE_SOURCE_DIR}/infrt/dialect/pass")
+ mlir_tablegen(${td_base}.cpp.inc -gen-rewriters "-I${CMAKE_SOURCE_DIR}/infrt/dialect/pass")
  add_public_tablegen_target(${td_base}_IncGen)
  add_custom_target(${td_base}_inc DEPENDS ${td_base}_IncGen)
 endfunction()

cmake/generic.cmake

@@ -116,19 +116,19 @@ function(find_fluid_modules TARGET_NAME)
  endif()
 endfunction(find_fluid_modules)
 
-set_property(GLOBAL PROPERTY PTEN_MODULES "")
-# find all pten modules is used for paddle static library
+set_property(GLOBAL PROPERTY PHI_MODULES "")
+# find all phi modules is used for paddle static library
 # for building inference libs
-function(find_pten_modules TARGET_NAME)
+function(find_phi_modules TARGET_NAME)
  get_filename_component(__target_path ${TARGET_NAME} ABSOLUTE)
  string(REGEX REPLACE "^${PADDLE_SOURCE_DIR}/" "" __target_path ${__target_path})
  string(FIND "${__target_path}" "phi" pos)
  if(pos GREATER 1)
- get_property(pten_modules GLOBAL PROPERTY PTEN_MODULES)
- set(pten_modules ${pten_modules} ${TARGET_NAME})
- set_property(GLOBAL PROPERTY PTEN_MODULES "${pten_modules}")
+ get_property(phi_modules GLOBAL PROPERTY PHI_MODULES)
+ set(phi_modules ${phi_modules} ${TARGET_NAME})
+ set_property(GLOBAL PROPERTY PHI_MODULES "${phi_modules}")
  endif()
-endfunction(find_pten_modules)
+endfunction(find_phi_modules)
 
 function(common_link TARGET_NAME)
  if (WITH_PROFILER)
@@ -324,7 +324,7 @@ function(cc_library TARGET_NAME)
  else()
  add_library(${TARGET_NAME} STATIC ${cc_library_SRCS})
  find_fluid_modules(${TARGET_NAME})
- find_pten_modules(${TARGET_NAME})
+ find_phi_modules(${TARGET_NAME})
  endif()
  if(cc_library_DEPS)
  # Don't need link libwarpctc.so
@@ -497,7 +497,7 @@ function(nv_library TARGET_NAME)
  else()
  add_library(${TARGET_NAME} STATIC ${nv_library_SRCS})
  find_fluid_modules(${TARGET_NAME})
- find_pten_modules(${TARGET_NAME})
+ find_phi_modules(${TARGET_NAME})
  endif()
  if (nv_library_DEPS)
  add_dependencies(${TARGET_NAME} ${nv_library_DEPS})
@@ -588,7 +588,7 @@ function(hip_library TARGET_NAME)
  else()
  hip_add_library(${TARGET_NAME} STATIC ${hip_library_SRCS})
  find_fluid_modules(${TARGET_NAME})
- find_pten_modules(${TARGET_NAME})
+ find_phi_modules(${TARGET_NAME})
  endif()
  if (hip_library_DEPS)
  add_dependencies(${TARGET_NAME} ${hip_library_DEPS})

cmake/inference_lib.cmake

@@ -224,7 +224,7 @@ copy(inference_lib_dist
  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/crypto/)
 include_directories(${CMAKE_BINARY_DIR}/../paddle/fluid/framework/io)
 
-# copy api headers for pten & custom op
+# copy api headers for phi & custom op
 copy(inference_lib_dist
  SRCS ${PADDLE_SOURCE_DIR}/paddle/phi/api/ext/*.h
  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/phi/api/ext/)
@@ -244,11 +244,11 @@ copy(inference_lib_dist
  SRCS ${PADDLE_SOURCE_DIR}/paddle/extension.h
  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/)
 
-# the header file of pten is copied to the experimental directory,
-# the include path of pten needs to be changed to adapt to inference api path
+# the header file of phi is copied to the experimental directory,
+# the include path of phi needs to be changed to adapt to inference api path
 add_custom_command(TARGET inference_lib_dist POST_BUILD
- COMMAND ${CMAKE_COMMAND} -P "${PADDLE_SOURCE_DIR}/cmake/pten_header.cmake"
- COMMENT "Change pten header include path to adapt to inference api path")
+ COMMAND ${CMAKE_COMMAND} -P "${PADDLE_SOURCE_DIR}/cmake/phi_header.cmake"
+ COMMENT "Change phi header include path to adapt to inference api path")
 
 # CAPI inference library for only inference
 set(PADDLE_INFERENCE_C_INSTALL_DIR "${CMAKE_BINARY_DIR}/paddle_inference_c_install_dir" CACHE STRING

cmake/operators.cmake

@@ -73,6 +73,12 @@ function(op_library TARGET)
  if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu)
  list(APPEND cu_srcs ${TARGET}.cu)
  endif()
+ # rename in KP: .kps -> .cu
+ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.kps)
+ file(COPY ${TARGET}.kps DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+ file(RENAME ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.kps ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.cu)
+ list(APPEND cu_srcs ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.cu)
+ endif()
  if (WITH_NV_JETSON)
  list(REMOVE_ITEM cu_srcs "decode_jpeg_op.cu")
  endif()
@@ -96,6 +102,12 @@ function(op_library TARGET)
  if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu)
  list(APPEND hip_srcs ${TARGET}.cu)
  endif()
+ # rename in KP: .kps -> .cu
+ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.kps)
+ file(COPY ${TARGET}.kps DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+ file(RENAME ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.kps ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.cu)
+ list(APPEND hip_srcs ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.cu)
+ endif()
  if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
  set(PART_CUDA_KERNEL_FILES ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu
  ${PART_CUDA_KERNEL_FILES} PARENT_SCOPE)

cmake/pten.cmake renamed to cmake/phi.cmake

@@ -51,22 +51,28 @@ function(generate_unify_header DIR_NAME)
  endforeach()
  # append header into extension.h
  string(REPLACE "${PADDLE_SOURCE_DIR}\/" "" header_file "${header_file}")
- file(APPEND ${pten_extension_header_file} "#include \"${header_file}\"\n")
+ file(APPEND ${phi_extension_header_file} "#include \"${header_file}\"\n")
 endfunction()
 
 # call kernel_declare need to make sure whether the target of input exists
 function(kernel_declare TARGET_LIST)
  foreach(kernel_path ${TARGET_LIST})
  file(READ ${kernel_path} kernel_impl)
- # TODO(chenweihang): rename PD_REGISTER_KERNEL to PD_REGISTER_KERNEL
- # NOTE(chenweihang): now we don't recommend to use digit in kernel name
- string(REGEX MATCH "(PD_REGISTER_KERNEL|PD_REGISTER_GENERAL_KERNEL)\\([ \t\r\n]*[a-z0-9_]*," first_registry "${kernel_impl}")
+ string(REGEX MATCH "(PD_REGISTER_KERNEL|PD_REGISTER_GENERAL_KERNEL)\\([ \t\r\n]*[a-z0-9_]*,[ \t\r\n\/]*[a-z0-9_]*" first_registry "${kernel_impl}")
  if (NOT first_registry STREQUAL "")
+ # some gpu kernel only can run on cuda, not support rocm, so we add this branch
+ if (WITH_ROCM)
+ string(FIND "${first_registry}" "cuda_only" pos)
+ if(pos GREATER 1)
+ continue()
+ endif()
+ endif()
  # parse the first kernel name
  string(REPLACE "PD_REGISTER_KERNEL(" "" kernel_name "${first_registry}")
  string(REPLACE "PD_REGISTER_GENERAL_KERNEL(" "" kernel_name "${kernel_name}")
  string(REPLACE "," "" kernel_name "${kernel_name}")
  string(REGEX REPLACE "[ \t\r\n]+" "" kernel_name "${kernel_name}")
+ string(REGEX REPLACE "//cuda_only" "" kernel_name "${kernel_name}")
  # append kernel declare into declarations.h
  # TODO(chenweihang): default declare ALL_LAYOUT for each kernel
  if (${kernel_path} MATCHES "./cpu\/")
@@ -75,6 +81,8 @@ function(kernel_declare TARGET_LIST)
  file(APPEND ${kernel_declare_file} "PD_DECLARE_KERNEL(${kernel_name}, GPU, ALL_LAYOUT);\n")
  elseif (${kernel_path} MATCHES "./xpu\/")
  file(APPEND ${kernel_declare_file} "PD_DECLARE_KERNEL(${kernel_name}, XPU, ALL_LAYOUT);\n")
+ elseif (${kernel_path} MATCHES "./gpudnn\/")
+ file(APPEND ${kernel_declare_file} "PD_DECLARE_KERNEL(${kernel_name}, GPUDNN, ALL_LAYOUT);\n")
  else ()
  # deal with device independent kernel, now we use CPU temporaary
  file(APPEND ${kernel_declare_file} "PD_DECLARE_KERNEL(${kernel_name}, CPU, ALL_LAYOUT);\n")
@@ -88,13 +96,16 @@ function(kernel_library TARGET)
  set(cpu_srcs)
  set(gpu_srcs)
  set(xpu_srcs)
+ set(gpudnn_srcs)
  set(selected_rows_srcs)
  # parse and save the deps kerenl targets
  set(all_srcs)
  set(kernel_deps)
 
  set(oneValueArgs SUB_DIR)
  set(multiValueArgs SRCS DEPS)
+ set(target_build_flag 1)
+
  cmake_parse_arguments(kernel_library "${options}" "${oneValueArgs}"
  "${multiValueArgs}" ${ARGN})
 
@@ -117,6 +128,9 @@ function(kernel_library TARGET)
  if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu.cc)
  list(APPEND gpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu.cc)
  endif()
+ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}_gpudnn.cu)
+ list(APPEND gpudnn_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}_gpudnn.cu)
+ endif()
  endif()
  if (WITH_XPU)
  if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/xpu/${TARGET}.cc)
@@ -135,6 +149,7 @@ function(kernel_library TARGET)
  list(APPEND all_srcs ${cpu_srcs})
  list(APPEND all_srcs ${gpu_srcs})
  list(APPEND all_srcs ${xpu_srcs})
+ list(APPEND all_srcs ${gpudnn_srcs})
  foreach(src ${all_srcs})
  file(READ ${src} target_content)
  string(REGEX MATCHALL "#include \"paddle\/phi\/kernels\/[a-z0-9_]+_kernel.h\"" include_kernels ${target_content})
@@ -160,21 +175,22 @@ function(kernel_library TARGET)
  list(LENGTH cpu_srcs cpu_srcs_len)
  list(LENGTH gpu_srcs gpu_srcs_len)
  list(LENGTH xpu_srcs xpu_srcs_len)
+ list(LENGTH gpudnn_srcs gpudnn_srcs_len)
  list(LENGTH selected_rows_srcs selected_rows_srcs_len)
 
  # Build Target according different src organization
  if((${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR
- ${xpu_srcs_len} GREATER 0) AND (${common_srcs_len} GREATER 0 OR
- ${selected_rows_srcs_len} GREATER 0))
+ ${xpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0) AND
+ (${common_srcs_len} GREATER 0 OR ${selected_rows_srcs_len} GREATER 0))
  # If the common_srcs/selected_rows_srcs depends on specific device srcs, build target using this rule.
  if (WITH_GPU)
- if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0)
- nv_library(${TARGET}_part SRCS ${cpu_srcs} ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
+ nv_library(${TARGET}_part SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  nv_library(${TARGET} SRCS ${common_srcs} ${selected_rows_srcs} DEPS ${TARGET}_part)
  endif()
  elseif (WITH_ROCM)
- if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0)
- hip_library(${TARGET}_part SRCS ${cpu_srcs} ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
+ hip_library(${TARGET}_part SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  hip_library(${TARGET} SRCS ${common_srcs} ${selected_rows_srcs} DEPS ${TARGET}_part)
  endif()
  else()
@@ -184,14 +200,14 @@ function(kernel_library TARGET)
  endif()
  endif()
  # If there are only specific device srcs, build target using this rule.
- elseif (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0)
+ elseif (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
  if (WITH_GPU)
- if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0)
- nv_library(${TARGET} SRCS ${cpu_srcs} ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
+ nv_library(${TARGET} SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  endif()
  elseif (WITH_ROCM)
- if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0)
- hip_library(${TARGET} SRCS ${cpu_srcs} ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
+ hip_library(${TARGET} SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  endif()
  else()
  if (${cpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0)
@@ -228,35 +244,40 @@ function(kernel_library TARGET)
  cc_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  endif()
  else()
- message(FATAL_ERROR "Cannot find any implementation for ${TARGET}")
+ set(target_build_flag 0)
  endif()
 
- if (${common_srcs_len} GREATER 0 OR ${cpu_srcs_len} GREATER 0 OR
- ${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR
- ${selected_rows_srcs_len} GREATER 0)
- # append target into PTEN_KERNELS property
- get_property(pten_kernels GLOBAL PROPERTY PTEN_KERNELS)
- set(pten_kernels ${pten_kernels} ${TARGET})
- set_property(GLOBAL PROPERTY PTEN_KERNELS ${pten_kernels})
- endif()
+ if (${target_build_flag} EQUAL 1)
+ if (${common_srcs_len} GREATER 0 OR ${cpu_srcs_len} GREATER 0 OR
+ ${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR
+ ${gpudnn_srcs_len} GREATER 0 OR ${selected_rows_srcs_len} GREATER 0)
+ # append target into PHI_KERNELS property
+ get_property(phi_kernels GLOBAL PROPERTY PHI_KERNELS)
+ set(phi_kernels ${phi_kernels} ${TARGET})
+ set_property(GLOBAL PROPERTY PHI_KERNELS ${phi_kernels})
+ endif()
 
- # parse kernel name and auto generate kernel declaration
- # here, we don't need to check WITH_XXX, because if not WITH_XXX, the
- # xxx_srcs_len will be equal to 0
- if (${common_srcs_len} GREATER 0)
- kernel_declare(${common_srcs})
- endif()
- if (${cpu_srcs_len} GREATER 0)
- kernel_declare(${cpu_srcs})
- endif()
- if (${gpu_srcs_len} GREATER 0)
- kernel_declare(${gpu_srcs})
- endif()
- if (${xpu_srcs_len} GREATER 0)
- kernel_declare(${xpu_srcs})
- endif()
- if (${selected_rows_srcs_len} GREATER 0)
- kernel_declare(${selected_rows_srcs})
+ # parse kernel name and auto generate kernel declaration
+ # here, we don't need to check WITH_XXX, because if not WITH_XXX, the
+ # xxx_srcs_len will be equal to 0
+ if (${common_srcs_len} GREATER 0)
+ kernel_declare(${common_srcs})
+ endif()
+ if (${cpu_srcs_len} GREATER 0)
+ kernel_declare(${cpu_srcs})
+ endif()
+ if (${gpu_srcs_len} GREATER 0)
+ kernel_declare(${gpu_srcs})
+ endif()
+ if (${xpu_srcs_len} GREATER 0)
+ kernel_declare(${xpu_srcs})
+ endif()
+ if (${gpudnn_srcs_len} GREATER 0)
+ kernel_declare(${gpudnn_srcs})
+ endif()
+ if (${selected_rows_srcs_len} GREATER 0)
+ kernel_declare(${selected_rows_srcs})
+ endif()
  endif()
 endfunction()
 

cmake/pten_header.cmake renamed to cmake/phi_header.cmake

@@ -14,8 +14,8 @@
 
 set(PADDLE_INFERENCE_INSTALL_DIR "${CMAKE_BINARY_DIR}/paddle_inference_install_dir")
 
-function(pten_header_path_compat TARGET_PATH)
-message(STATUS "pten header path compat processing: ${TARGET_PATH}")
+function(phi_header_path_compat TARGET_PATH)
+message(STATUS "phi header path compat processing: ${TARGET_PATH}")
 string(FIND ${TARGET_PATH} "experimental" pos)
 if (pos GREATER 1)
  file(GLOB HEADERS "${TARGET_PATH}/*" "*.h")
@@ -25,17 +25,17 @@ if (pos GREATER 1)
  string(REPLACE "paddle/phi/" "paddle/include/experimental/phi/" HEADER_CONTENT "${HEADER_CONTENT}")
  string(REPLACE "paddle/utils/" "paddle/include/experimental/utils/" HEADER_CONTENT "${HEADER_CONTENT}")
  file(WRITE ${header} "${HEADER_CONTENT}")
- message(STATUS "pten header path compat processing complete: ${header}")
+ message(STATUS "phi header path compat processing complete: ${header}")
  endif()
  endforeach()
 endif()
 endfunction()
 
-pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental)
-pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/phi/api)
-pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/phi/api/ext)
-pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/phi/api/include)
-pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/phi/common)
+phi_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental)
+phi_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/phi/api)
+phi_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/phi/api/ext)
+phi_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/phi/api/include)
+phi_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/phi/common)
 
 # In order to be compatible with the original behavior, the header file name needs to be changed
 file(RENAME ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/extension.h

paddle/fluid/distributed/CMakeLists.txt

@@ -1,5 +1,5 @@
+add_subdirectory(collective)
 add_subdirectory(store)
-
 if(NOT WITH_PSCORE)
  add_subdirectory(fleet_executor)
  return()

paddle/fluid/distributed/collective/CMakeLists.txt

@@ -0,0 +1,5 @@
+cc_library(processgroup SRCS ProcessGroup.cc DEPS phi phi_api eager_api)
+
+if(WITH_NCCL)
+ cc_library(processgroup_nccl SRCS ProcessGroupNCCL.cc DEPS place cuda_stream enforce collective_helper device_context phi phi_api eager_api)
+endif()