Merge remote-tracking branch 'upstream/develop' into add_gather_double_grad

Author: HydrogenSulfate

.github/workflows/CI-Build.yml

@@ -20,14 +20,14 @@ jobs:
  name: Clone-linux
  uses: ./.github/workflows/_Clone-linux.yml
  with:
- bos_dir: Paddle-build
+ clone_dir: Paddle-build
 
  build-docker:
  name: build docker images
  needs: clone
  uses: ./.github/workflows/docker.yml
  with:
- bos_dir: Paddle-build
+ clone_dir: Paddle-build
  task: build
 
  build:

.github/workflows/Coverage.yml

@@ -34,14 +34,14 @@ jobs:
  uses: ./.github/workflows/_Clone-linux.yml
  with:
  workflow-name: 'coverage'
- bos_dir: Paddle-coverage
+ clone_dir: Paddle-coverage
 
  build-docker:
  name: Coverage build docker
  needs: clone
  uses: ./.github/workflows/docker.yml
  with:
- bos_dir: Paddle-coverage
+ clone_dir: Paddle-coverage
  task: coverage
 
  build:

.github/workflows/Slice-baseline.yml

@@ -0,0 +1,45 @@
+name: Slice-baseline
+
+on:
+ schedule:
+ - cron: '30 2 * * *'
+
+permissions: read-all
+
+defaults:
+ run:
+ shell: bash
+
+jobs:
+ clone:
+ name: Slice-base clone
+ uses: ./.github/workflows/_Clone-linux.yml
+ with:
+ clone_dir: Paddle-build
+ is_pr: 'false'
+
+ build-docker:
+ name: Slice build docker
+ needs: clone
+ uses: ./.github/workflows/docker.yml
+ with:
+ clone_dir: Paddle-build
+ task: build
+
+ build:
+ name: Slice build
+ needs: [clone, build-docker]
+ uses: ./.github/workflows/_Linux-build.yml
+ with:
+ docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+ CI_name: slice-baseline
+ is_pr: 'false'
+
+ slice-test:
+ name: Slice test
+ uses: ./.github/workflows/_Slice.yml
+ needs: [clone, build-docker, build]
+ with:
+ docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
+ slice-check: 'true'
+ SLICE_TEST_MODE: insert_baseline

.github/workflows/_Clone-linux.yml

@@ -6,10 +6,14 @@ on:
  workflow-name:
  type: string
  required: false
- bos_dir:
+ clone_dir:
  type: string
  required: false
  default: 'Paddle'
+ is_pr:
+ type: string
+ required: false
+ default: 'true'
  outputs:
  can-skip:
  value: ${{ jobs.clone.outputs.can-skip }}
@@ -23,19 +27,18 @@ defaults:
  shell: bash
 
 env:
- PR_ID: ${{ github.event.pull_request.number }}
- COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+ PR_ID: ${{ github.event.pull_request.number || '0' }}
+ COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
  ci_scripts: ${{ github.workspace }}/ci
+ BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
 
 jobs:
  clone:
- # Don't run on forked repos.
  name: Clone Paddle
+ if: ${{ github.repository_owner == 'PaddlePaddle' }}
  outputs:
  can-skip: ${{ steps.check-bypass.outputs.can-skip }}
  slice-check: ${{ steps.check-execution.outputs.slice-check }}
-
- if: github.repository_owner == 'PaddlePaddle'
  runs-on:
  group: HK-Clone
 
@@ -49,6 +52,7 @@ jobs:
 
  - name: Merge PR to test branch
  id: check-execution
+ if: ${{ inputs.is_pr == 'true' }}
  run: |
  git config --unset http.https://github.com/.extraheader
  git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
@@ -96,10 +100,10 @@ jobs:
  echo "::group::Install bce-python-sdk"
  python -m pip install bce-python-sdk==0.8.74
  echo "::endgroup::"
- python ${bos_file} Paddle.tar.gz paddle-github-action/PR/${{ inputs.bos_dir }}/${PR_ID}/${COMMIT_ID}
+ python ${bos_file} Paddle.tar.gz paddle-github-action/PR/${{ inputs.clone_dir }}/${PR_ID}/${COMMIT_ID}
  rm Paddle.tar.gz
  cd -
- git switch ${{ github.event.pull_request.base.ref }}
+ git switch ${BRANCH}
  git branch -D test
  git gc
 

.github/workflows/_Doc-Preview.yml

@@ -1,4 +1,4 @@
-name: Model-benchmark
+name: Doc-Preview
 
 on:
  workflow_call:
@@ -17,6 +17,8 @@ env:
  BRANCH: ${{ github.event.pull_request.base.ref }}
  TASK: paddle-CI-${{ github.event.pull_request.number }}-Doc-Preview
  CI_name: Doc-Preview
+ CFS_DIR: /home/data/cfs
+ work_dir: /paddle
  no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
 
 defaults:
@@ -34,11 +36,15 @@ jobs:
 
  build-doc:
  name: Build doc
+ needs: check-bypass
+ if: ${{ inputs.can-skip != 'true' && needs.check-bypass.outputs.can-skip != 'true' }}
  runs-on:
  group: BD_BJ-V100
  steps:
- - name: Check docker image  and run container
+ - name: Check docker image and run container
  env:
+ CACHE_DIR: /home/data/cfs/.cache
+ GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  AGILE_PULL_ID: ${{ github.event.pull_request.number }}
  AGILE_REVISION: ${{ github.event.pull_request.head.sha }}
  BUILD_DOC: "true"
@@ -48,6 +54,7 @@ jobs:
  AGILE_COMPILE_BRANCH: ${{ github.event.pull_request.base.ref }}
  BOS_CREDENTIAL_AK: "paddle"
  BOS_CREDENTIAL_SK: "paddle"
+ PADDLE_WHL: https://paddle-github-action.cdn.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
  run: |
  container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
  echo "container_name=${container_name}" >> ${{ github.env }}
@@ -59,6 +66,8 @@ jobs:
  -v "/dev/shm:/dev/shm" \
  -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
  -v ${{ github.workspace }}:/paddle \
+ -e CACHE_DIR \
+ -e GITHUB_API_TOKEN \
  -e AGILE_PULL_ID \
  -e AGILE_REVISION \
  -e BUILD_DOC \
@@ -69,16 +78,21 @@ jobs:
  -e AGILE_COMPILE_BRANCH \
  -e BOS_CREDENTIAL_AK \
  -e BOS_CREDENTIAL_SK \
+ -e PADDLE_WHL \
+ -e CFS_DIR \
+ -e work_dir \
  -e no_proxy \
  -w /paddle --network host ${docker_image}
 
  - name: Doc build
  run: |
  docker exec -t ${{ env.container_name }} /bin/bash -c '
  rm -rf * .[^.]*
- wget -q https://xly-devops.bj.bcebos.com/PR/build_whl/${AGILE_PULL_ID}/${AGILE_REVISION}/build.tar.gz
- tar --use-compress-program="pigz -1" -xpf build.tar.gz
- cd ./paddle
+ echo "Downloading build.tar.gz from cfs"
+ cp ${CFS_DIR}/build_bos/${AGILE_PULL_ID}/${AGILE_REVISION}/build.tar.gz .
+ echo "Extracting build.tar.gz"
+ git config --global --add safe.directory ${work_dir}
+ tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
  api_doc_spec_diff=$(python tools/diff_api.py paddle/fluid/API_DEV.spec.doc paddle/fluid/API_PR.spec.doc)
  if [ "$api_doc_spec_diff" == "" ]; then
  echo "API documents no change."
@@ -94,5 +108,6 @@ jobs:
  - name: Terminate and delete the container
  if: always()
  run: |
+ set +e
  docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
  docker rm -f ${{ env.container_name }}

.github/workflows/_Linux-build.yml

@@ -6,20 +6,28 @@ on:
  docker_build_image:
  type: string
  required: true
+ CI_name:
+ type: string
+ required: false
+ default: 'build'
+ is_pr:
+ type: string
+ required: false
+ default: 'true'
  outputs:
  can-skip:
  description: "Whether to skip the job"
  value: ${{ jobs.build.outputs.can-skip }}
 
 env:
- PR_ID: ${{ github.event.pull_request.number }}
- COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+ PR_ID: ${{ github.event.pull_request.number || '0' }}
+ COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
  work_dir: /paddle
  PADDLE_ROOT: /paddle
  TASK: paddle-CI-${{ github.event.pull_request.number }}-build
  ci_scripts: /paddle/ci
- BRANCH: ${{ github.event.pull_request.base.ref }}
- CI_name: build
+ BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
+ CI_name: ${{ inputs.CI_name }}
  CFS_DIR: /home/data/cfs
  no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
 
@@ -116,7 +124,7 @@ jobs:
  -e CI_name \
  -w /paddle --network host ${docker_image}
 
- - name: Download paddle.tar.gz and merge target branch
+ - name: Download paddle.tar.gz
  env:
  work_dir: ${{ github.workspace }}
  run: |
@@ -136,6 +144,14 @@ jobs:
  set +e
  git remote add upstream https://github.com/PaddlePaddle/Paddle.git
  set -e
+ '
+
+ - name: Merge target branch
+ if: ${{ inputs.is_pr == 'true' }}
+ env:
+ work_dir: ${{ github.workspace }}
+ run: |
+ docker exec -t ${{ env.container_name }} /bin/bash -c '
  source ${{ github.workspace }}/../../../proxy
  cp ci/git_pull.sh ci/git_pull_copy.sh
  git checkout $BRANCH
@@ -147,6 +163,7 @@ jobs:
 
  - name: Check bypass
  id: check-bypass
+ if: ${{ inputs.is_pr == 'true' }}
  uses: ./.github/actions/check-bypass
  with:
  github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/_Slice.yml

@@ -12,15 +12,19 @@ on:
  slice-check:
  type: string
  required: false
+ SLICE_TEST_MODE:
+ type: string
+ required: false
+ default: 'test_ci'
 
 env:
- PR_ID: ${{ github.event.pull_request.number }}
- COMMIT_ID: ${{ github.event.pull_request.head.sha }}
+ PR_ID: ${{ github.event.pull_request.number || '0' }}
+ COMMIT_ID: ${{ github.event.pull_request.head.sha || github.sha }}
  work_dir: /paddle
  PADDLE_ROOT: /paddle
  TASK: paddle-CI-${{ github.event.pull_request.number }}-slice
  ci_scripts: /paddle/ci
- BRANCH: ${{ github.event.pull_request.base.ref }}
+ BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
  CI_name: slice
  no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
 
@@ -35,16 +39,11 @@ jobs:
  runs-on:
  group: slice
  steps:
- # - name: Download test script
- # run: |
- # sudo rm -rf * .[^.]*
- # source ${{ github.workspace }}/../../../proxy
- # wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/refs/heads/develop/ci/slice/test_slice_float32.py
-
  - name: Check docker image and run container
  env:
- wheel_link: https://paddle-github-action.bj.bcebos.com/PR/build/${{ github.event.pull_request.number }}/${{ github.event.pull_request.head.sha }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
+ wheel_link: https://paddle-github-action.bj.bcebos.com/PR/build/${{ env.PR_ID }}/${{ env.COMMIT_ID }}/paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
  RUN_ID: ${{ github.run_id }}
+ SLICE_TEST_MODE: ${{ inputs.SLICE_TEST_MODE }}
  run: |
  container_name="api_benchmark_ci_${RUN_ID}"
  echo "container_name=${container_name}" >> ${{ github.env }}
@@ -64,41 +63,42 @@ jobs:
  -e ci_scripts \
  -e no_proxy \
  -e CI_name \
+ -e SLICE_TEST_MODE \
  -w /paddle --network host ${docker_image}
 
- - name: Download Paddle
+ - name: Download PaddleTest
  env:
  work_dir: ${{ github.workspace }}
  run: |
  docker exec -t ${{ env.container_name }} /bin/bash -c '
  rm -rf * .[^.]*
- echo "Downloading build.tar.gz"
- wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/build/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate
- echo "Extracting build.tar.gz"
+ echo "Downloading PaddleTest"
+ wget -q https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz --no-proxy
+ echo "Extracting PaddleTest"
  git config --global --add safe.directory ${work_dir}
- tar --use-compress-program="pzstd -1" -xpf build.tar.gz --strip-components=1
+ tar -xzf PaddleTest.tar.gz && cd PaddleTest
  git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
- git checkout test
- rm build.tar.gz
  '
 
  - name: Test
  env:
  work_dir: ${{ github.workspace }}
  run: |
  docker exec -t ${{ env.container_name }} /bin/bash -c '
- set -x
  mkdir -p ${{ github.workspace }}/../../../.cache/pip
  source ${{ github.workspace }}/../../../proxy
  python3.10 -m pip config set global.cache-dir ${{ github.workspace }}/../../../.cache/pip
  python3.10 -m pip install $wheel_link
- python3.10 -m pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu118
- export FLAGS_share_tensor_for_grad_tensor_holder=True
- python3.10 ci/slice/test_slice_perform.py
+ python3.10 -m pip install -r PaddleTest/framework/e2e/api_benchmark/requirement.txt
+ cd PaddleTest/framework/slice_benchmark
+ cp ${{ github.workspace }}/../../../apibm_config.yml .
+ nvidia-smi
+ bash run.sh
  '
 
  - name: Terminate and delete the container
  if: always()
  run: |
+ set +e
  docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
  docker rm -f ${{ env.container_name }}