Merge branch 'PaddlePaddle:develop' into move_viterbi

Author: linjieccc

.gitignore

@@ -6,12 +6,14 @@ paddle/fluid/eager/api/generated/*
 paddle/fluid/op_use_default_grad_maker_DEV.spec
 paddle/fluid/op_use_default_grad_maker_PR.spec
 paddle/phi/api/backward/backward_api.h
+paddle/phi/api/backward/sparse_bw_api.h
 paddle/phi/api/include/api.h
 paddle/phi/api/include/sparse_api.h
 paddle/phi/api/lib/api.cc
 paddle/phi/api/lib/dygraph_api.*
 paddle/phi/api/lib/backward_api.cc
 paddle/phi/api/lib/sparse_api.cc
+paddle/phi/api/lib/sparse_bw_api.cc
 paddle/phi/extension.h
 paddle/phi/include/*
 paddle/phi/infermeta/generated.*

cmake/external/llvm.cmake

@@ -100,8 +100,8 @@ endfunction()
 function(mlir_add_rewriter td_base)
  set(LLVM_TARGET_DEFINITIONS ${td_base}.td)
  mlir_tablegen(${td_base}.cpp.inc -gen-rewriters "-I${CMAKE_SOURCE_DIR}/infrt/dialect/pass")
- add_public_tablegen_target(${td_base}_IncGen)
- add_custom_target(${td_base}_inc DEPENDS ${td_base}_IncGen)
+ add_public_tablegen_target(MLIR${td_base}IncGen)
+ add_dependencies(mlir-headers MLIR${td_base}IncGen)
 endfunction()
 
 # Execute the mlir script with infrt-exec program.

cmake/operators.cmake

@@ -293,11 +293,11 @@ function(op_library TARGET)
  # Define operators that don't need pybind here.
  foreach(manual_pybind_op "compare_all_op" "compare_op" "logical_op" "bitwise_op" "nccl_op"
  "tensor_array_read_write_op" "tensorrt_engine_op" "conv_fusion_op")
- 
-  if ("${TARGET}" STREQUAL "${manual_pybind_op}")
-  set(pybind_flag 1)
-  endif()
-  endforeach()
+
+ if ("${TARGET}" STREQUAL "${manual_pybind_op}")
+ set(pybind_flag 1)
+ endif()
+ endforeach()
 
  # The registration of USE_OP, please refer to paddle/fluid/framework/op_registry.h.
  # Note that it's enough to just adding one operator to pybind in a *_op.cc file.
@@ -478,7 +478,7 @@ function(op_library TARGET)
  if (${pybind_flag} EQUAL 0)
  # NOTE(*): activation use macro to regist the kernels, set use_op manually.
  if(${TARGET} STREQUAL "activation")
- file(APPEND ${pybind_file} "USE_OP(relu);\n")
+ file(APPEND ${pybind_file} "USE_OP_ITSELF(relu);\n")
  elseif(${TARGET} STREQUAL "fake_dequantize")
  file(APPEND ${pybind_file} "USE_OP(fake_dequantize_max_abs);\n")
  elseif(${TARGET} STREQUAL "fake_quantize")

cmake/phi.cmake

@@ -134,8 +134,8 @@ function(kernel_library TARGET)
  if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu)
  list(APPEND gpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu)
  endif()
- if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}_gpudnn.cu)
- list(APPEND gpudnn_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}_gpudnn.cu)
+ if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}.cu)
+ list(APPEND gpudnn_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}.cu)
  endif()
  endif()
  if (WITH_XPU)
@@ -197,92 +197,88 @@ function(kernel_library TARGET)
 
  # kernel source file level
  # level 1: base device kernel
- # - cpu_srcs / gpu_srcs / xpu_srcs / kps_srcs
+ # - cpu_srcs / gpu_srcs / xpu_srcs / gpudnn_srcs / kps_srcs
  # level 2: device-independent kernel
  # - common_srcs
  # level 3: Kernel implemented by reusing device-independent kernel
  # - selected_rows_srcs
+ set(base_device_kernels)
+ set(device_independent_kernel)
+ set(high_level_kernels)
 
- # Build Target according different src organization
- if((${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR
- ${xpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0) AND
- (${common_srcs_len} GREATER 0 OR ${selected_rows_srcs_len} GREATER 0))
- # If the common_srcs/selected_rows_srcs depends on specific device srcs, build target using this rule.
+ # 1. Base device kernel compile
+ if (${cpu_srcs_len} GREATER 0)
+ cc_library(${TARGET}_cpu SRCS ${cpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ list(APPEND base_device_kernels ${TARGET}_cpu)
+ endif()
+ if (${gpu_srcs_len} GREATER 0)
  if (WITH_GPU)
- if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
- nv_library(${TARGET}_part SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- nv_library(${TARGET} SRCS ${common_srcs} ${selected_rows_srcs} DEPS ${TARGET}_part)
- endif()
+ nv_library(${TARGET}_gpu SRCS ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  elseif (WITH_ROCM)
- if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
- hip_library(${TARGET}_part SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- hip_library(${TARGET} SRCS ${common_srcs} ${selected_rows_srcs} DEPS ${TARGET}_part)
- endif()
- elseif (WITH_XPU_KP)
- if (${cpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0)
- xpu_library(${TARGET}_part SRCS ${cpu_srcs} ${xpu_srcs} ${kps_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- xpu_library(${TARGET} SRCS ${common_srcs} ${selected_rows_srcs} DEPS ${TARGET}_part)
- endif()
- else()
- if (${cpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0)
- cc_library(${TARGET}_part SRCS ${cpu_srcs} ${xpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- cc_library(${TARGET} SRCS ${common_srcs} ${selected_rows_srcs} DEPS ${TARGET}_part)
- endif()
+ hip_library(${TARGET}_gpu SRCS ${gpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  endif()
- # If there are only specific device srcs, build target using this rule.
- elseif (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0)
+ list(APPEND base_device_kernels ${TARGET}_gpu)
+ endif()
+ if (${xpu_srcs_len} GREATER 0)
+ cc_library(${TARGET}_xpu SRCS ${xpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ list(APPEND base_device_kernels ${TARGET}_xpu)
+ endif()
+ if (${gpudnn_srcs_len} GREATER 0)
  if (WITH_GPU)
- if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
- nv_library(${TARGET} SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- endif()
+ nv_library(${TARGET}_gpudnn SRCS ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  elseif (WITH_ROCM)
- if (${cpu_srcs_len} GREATER 0 OR ${gpu_srcs_len} GREATER 0 OR ${gpudnn_srcs_len} GREATER 0)
- hip_library(${TARGET} SRCS ${cpu_srcs} ${gpu_srcs} ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- endif()
- elseif (WITH_XPU_KP)
- if (${cpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0 OR ${kps_srcs_len} GREATER 0)
- xpu_library(${TARGET} SRCS ${cpu_srcs} ${xpu_srcs} ${kps_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- endif()
- else()
- if (${cpu_srcs_len} GREATER 0 OR ${xpu_srcs_len} GREATER 0)
- cc_library(${TARGET} SRCS ${cpu_srcs} ${xpu_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- endif()
+ hip_library(${TARGET}_gpudnn SRCS ${gpudnn_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
  endif()
- # If the selected_rows_srcs depends on common_srcs, build target using this rule.
- elseif (${common_srcs_len} GREATER 0 AND ${selected_rows_srcs_len} GREATER 0)
+ list(APPEND base_device_kernels ${TARGET}_gpudnn)
+ endif()
+ if (${kps_srcs_len} GREATER 0)
+ # only when WITH_XPU_KP, the kps_srcs_len can be > 0
+ xpu_library(${TARGET}_kps SRCS ${kps_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ list(APPEND base_device_kernels ${TARGET}_kps)
+ endif()
+
+ # 2. Device-independent kernel compile
+ if (${common_srcs_len} GREATER 0)
  if (WITH_GPU)
- nv_library(${TARGET}_part SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- nv_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${TARGET}_part)
+ nv_library(${TARGET}_common SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
  elseif (WITH_ROCM)
- hip_library(${TARGET}_part SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- hip_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${TARGET}_part)
+ hip_library(${TARGET}_common SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
  elseif (WITH_XPU_KP)
- xpu_library(${TARGET}_part SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- xpu_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${TARGET}_part)
+ xpu_library(${TARGET}_common SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
  else()
- cc_library(${TARGET}_part SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
- cc_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${TARGET}_part)
+ cc_library(${TARGET}_common SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels})
  endif()
- # If there are only common_srcs or selected_rows_srcs, build target using below rules.
- elseif (${common_srcs_len} GREATER 0)
+ list(APPEND device_independent_kernel ${TARGET}_common)
+ endif()
+
+ # 3. Reusing kernel compile
+ if (${selected_rows_srcs_len} GREATER 0)
  if (WITH_GPU)
- nv_library(${TARGET} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ nv_library(${TARGET}_sr SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
  elseif (WITH_ROCM)
- hip_library(${TARGET} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ hip_library(${TARGET}_sr SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
  elseif (WITH_XPU_KP)
- xpu_library(${TARGET} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ xpu_library(${TARGET}_sr SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
  else()
- cc_library(${TARGET} SRCS ${common_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ cc_library(${TARGET}_sr SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel})
  endif()
- elseif (${selected_rows_srcs_len} GREATER 0)
+ list(APPEND high_level_kernels ${TARGET}_sr)
+ endif()
+
+ # 4. Unify target compile
+ list(LENGTH base_device_kernels base_device_kernels_len)
+ list(LENGTH device_independent_kernel device_independent_kernel_len)
+ list(LENGTH high_level_kernels high_level_kernels_len)
+ if (${base_device_kernels_len} GREATER 0 OR ${device_independent_kernel_len} GREATER 0 OR
+ ${high_level_kernels_len} GREATER 0)
  if (WITH_GPU)
- nv_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ nv_library(${TARGET} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel} ${high_level_kernels})
  elseif (WITH_ROCM)
- hip_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ hip_library(${TARGET} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel} ${high_level_kernels})
  elseif (WITH_XPU_KP)
- xpu_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ xpu_library(${TARGET} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel} ${high_level_kernels})
  else()
- cc_library(${TARGET} SRCS ${selected_rows_srcs} DEPS ${kernel_library_DEPS} ${kernel_deps})
+ cc_library(${TARGET} DEPS ${kernel_library_DEPS} ${kernel_deps} ${base_device_kernels} ${device_independent_kernel} ${high_level_kernels})
  endif()
  else()
  set(target_build_flag 0)

paddle/fluid/distributed/collective/CMakeLists.txt

@@ -7,3 +7,6 @@ cc_library(eager_reducer SRCS reducer.cc DEPS eager_api processgroup)
 if(WITH_NCCL)
  cc_library(processgroup_nccl SRCS ProcessGroupNCCL.cc DEPS place cuda_stream enforce collective_helper device_context phi phi_api eager_api)
 endif()
+if(WITH_ASCEND_CL)
+ cc_library(processgroup_hccl SRCS ProcessGroupHCCL.cc DEPS place npu_stream enforce collective_helper device_context phi phi_api eager_api)
+endif()

paddle/fluid/distributed/collective/HCCLTools.h

@@ -0,0 +1,174 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <error.h>
+#include <string>
+
+#include "boost/variant.hpp"
+#include "paddle/fluid/framework/data_type.h"
+#include "paddle/fluid/framework/variable.h"
+#include "paddle/fluid/platform/collective_helper.h"
+#include "paddle/fluid/platform/device/npu/enforce_npu.h"
+#include "paddle/fluid/platform/device/npu/npu_info.h"
+#include "paddle/fluid/platform/device_context.h"
+#include "paddle/fluid/platform/enforce.h"
+
+namespace paddle {
+namespace distributed {
+
+class NPUEventManager {
+ public:
+ NPUEventManager() = default;
+
+ ~NPUEventManager() {
+ if (is_created_) {
+ platform::NPUDeviceGuard guard(device_index_);
+ platform::NPUEventDestroy(event_);
+ }
+ }
+
+ NPUEventManager(const NPUEventManager&) = delete;
+ NPUEventManager& operator=(const NPUEventManager&) = delete;
+
+ NPUEventManager(NPUEventManager&& other) {
+ std::swap(is_created_, other.is_created_);
+ std::swap(device_index_, other.device_index_);
+ std::swap(event_, other.event_);
+ }
+
+ NPUEventManager& operator=(NPUEventManager&& other) {
+ std::swap(is_created_, other.is_created_);
+ std::swap(device_index_, other.device_index_);
+ std::swap(event_, other.event_);
+ return *this;
+ }
+
+ bool IsCreated() const { return is_created_; }
+ bool DeviceId() const { return device_index_; }
+ aclrtEvent GetRawNPUEvent() const { return event_; }
+
+ void Record(const paddle::platform::NPUDeviceContext& ctx) {
+ auto device_index = ctx.GetPlace().device;
+ if (!is_created_) {
+ CreateEvent(device_index);
+ }
+ PADDLE_ENFORCE_EQ(device_index, device_index_,
+ platform::errors::PreconditionNotMet(
+ "NPUDeviceContext's device %d does not match"
+ "Event's device %d",
+ device_index, device_index_));
+
+ platform::NPUDeviceGuard guard(device_index_);
+ platform::NPUEventRecord(event_, ctx.stream());
+ }
+
+ bool Query() const {
+ aclrtEventStatus status = ACL_EVENT_STATUS_COMPLETE;
+ platform::NPUEventQuery(event_, &status);
+ if (status == ACL_EVENT_STATUS_COMPLETE) {
+ return true;
+ }
+ return false;
+ }
+
+ void Block(const paddle::platform::NPUDeviceContext& ctx) const {
+ if (is_created_) {
+ auto device_index = ctx.GetPlace().device;
+ PADDLE_ENFORCE_EQ(device_index, device_index_,
+ platform::errors::PreconditionNotMet(
+ "CUDADeviceContext's device %d does not match"
+ "Event's device %d",
+ device_index, device_index_));
+ platform::NPUDeviceGuard guard(device_index_);
+ platform::NPUStreamWaitEvent(ctx.stream(), event_);
+ }
+ }
+
+ private:
+ bool is_created_{false};
+ aclrtEvent event_{};
+ int8_t device_index_{0};
+
+ private:
+ void CreateEvent(int device_index) {
+ device_index_ = device_index;
+ platform::NPUDeviceGuard guard(device_index);
+ platform::NPUEventCreate(&event_);
+ is_created_ = true;
+ }
+};
+
+class HCCLCommManager {
+ public:
+ explicit HCCLCommManager(HcclComm hcclComm) : hccl_comm_(hcclComm) {}
+
+ HCCLCommManager() : HCCLCommManager(nullptr) {}
+
+ ~HCCLCommManager() noexcept {
+ std::unique_lock<std::mutex> lock(mutex_);
+ if (hccl_comm_) {
+ platform::dynload::HcclCommDestroy(hccl_comm_);
+ }
+ }
+
+ static std::shared_ptr<HCCLCommManager> Create(int num_ranks, int rank,
+ HcclRootInfo* comm_id,
+ HcclComm hccl_comm) {
+ auto hccl_manager = std::make_shared<HCCLCommManager>();
+ auto ret = platform::dynload::HcclCommInitRootInfo(num_ranks, comm_id, rank,
+ &hccl_comm);
+ using __NPU_STATUS_TYPE__ = decltype(ret);
+ constexpr auto __success_type__ =
+ platform::details::NPUStatusType<__NPU_STATUS_TYPE__>::kSuccess;
+ if (UNLIKELY(ret != __success_type__)) {
+ VLOG(0) << "Error: create hccl_id error.";
+ exit(-1);
+ }
+
+ hccl_manager->hccl_id_ = comm_id;
+ hccl_manager->rank_ = rank;
+ hccl_manager->hccl_comm_ = hccl_comm;
+ return hccl_manager;
+ }
+
+ HcclRootInfo* GetHcclId() const {
+ std::unique_lock<std::mutex> lock(mutex_);
+ return hccl_id_;
+ }
+
+ HcclComm GetHcclComm() const {
+ std::unique_lock<std::mutex> lock(mutex_);
+ return hccl_comm_;
+ }
+
+ HCCLCommManager(const HCCLCommManager&) = delete;
+ HCCLCommManager& operator=(const HCCLCommManager&) = delete;
+ HCCLCommManager& operator=(HCCLCommManager&& other) = delete;
+
+ HCCLCommManager(HCCLCommManager&& other) {
+ std::unique_lock<std::mutex> lock(other.mutex_);
+ std::swap(hccl_comm_, other.hccl_comm_);
+ }
+
+ protected:
+ HcclComm hccl_comm_;
+ HcclRootInfo* hccl_id_;
+ int rank_;
+ mutable std::mutex mutex_;
+};
+
+} // namespace distributed
+} // namespace paddle