#!/bin/bash # # This build script is used for TeamCity CI, we need to  # set the follwoing parameters in the project configuration: # PARAMETER | EXAMPLE | COMMENTS # CTEST_OUTPUT_ON_FAILURE 1 // whether output failure message # CTEST_PARALLEL_LEVEL 5 // parallel level for running ctest # PADDLE_VERSION 0.11.0 // the latest tag # WITH_AVX ON // wheter the instruction support AVX # WITH_GOLANG OFF // don't build the Golang code # WITH_GPU ON // running on GPU # WITH_UBUNTU_MIRROR ON // use unbuntu mirro to speed up dev Docker image building # WITH_CACHE ON // whether using the cached third-party pacakge on host set -xe PR_ID=$(echo %teamcity.build.vcs.branch.Paddle_PaddlePaddle% | sed 's/[^0-9]*//g') echo ${PR_ID} CHANGED_FILES=$(cat %system.teamcity.build.changedFiles.file% | awk -F : '{print $1}') if [ -z "$CHANGED_FILES" ] then echo "empty changed files" else echo "changed files: $CHANGED_FILES" COUNT=$(echo $CHANGED_FILES | awk -F . '{print $NF}'|grep -i -E 'md|rst|jpg|png' | wc -l) FILE_COUNT=$(echo $CHANGED_FILES | wc -l) if [ "$COUNT" -eq "$FILE_COUNT" ] then echo "all readme files or pictures, skip CI" exit 0 fi fi apt_mirror='s#http://archive.ubuntu.com/ubuntu#mirror://mirrors.ubuntu.com/mirrors.txt#g' export PADDLE_DEV_NAME=paddlepaddle/paddle:dev nvidia-docker run -i --rm -v $PWD:/paddle ${PADDLE_DEV_NAME} rm -rf /paddle/build if [[ "%WITH_CACHE%" == "ON" ]]; then # Use the cached thirdparty package if the MD5 is mathes. md5_content=$(cat \  ${PWD}/cmake/external/*.cmake \  |md5sum | awk '{print $1}') tp_cache_dir=/root/.cache/third_party tp_cache_file=$tp_cache_dir/%system.teamcity.projectName%_${md5_content}.tar.gz if [ ! -d ${tp_cache_dir} ];then mkdir -p ${tp_cache_dir} fi if [ -f ${tp_cache_file} ];then mkdir -p ${PWD}/build tar zxvf ${tp_cache_file}.tar.gz -C $PWD else # clear the older tar files if MD5 has chanaged. rm -rf $tp_cache_dir/*.tar.gz fi fi # Check the MD5 of the Dockerfile of dev Docker image, # if there is no current Docker iamges, delete all the older  # dev Docker images to free up disk space. md5_dev_dockerfile=$(cat ${PWD}/Dockerfile | md5sum | awk '{print $1}') export PADDLE_DEV_NAME=paddlepaddle/paddle-dev:${md5_dev_dockerfile} dev_image_id=$(docker images paddlepaddle/paddle-dev:${md5_dev_dockerfile} -q) if [[ ${dev_image_id} == "" ]]; then docker rmi $(docker images paddlepaddle/paddle-dev -q) || true if [[ "%WITH_UBUNTU_MIRROR%" == "ON" ]]; then docker build -t ${PADDLE_DEV_NAME} --build-arg UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt . else docker build -t ${PADDLE_DEV_NAME} . fi fi # make sure CPU only build with openblas passes. nvidia-docker run -i --rm -v $PWD:/paddle -v /root/.cache:/root/.cache\ -e "APT_MIRROR=${apt_mirror}"\ -e "WITH_GPU=OFF"\ -e "WITH_AVX=%WITH_AVX%"\ -e "WITH_GOLANG=%WITH_GOLANG%"\ -e "WITH_TESTING=OFF"\ -e "WITH_COVERAGE=OFF" \ -e "COVERALLS_UPLOAD=OFF" \ -e "GIT_PR_ID=${PR_ID}" \ -e "WITH_C_API=%WITH_C_API%" \ -e "CMAKE_BUILD_TYPE=Debug" \ -e "WITH_MKL=OFF" \ -e "JSON_REPO_TOKEN=%JSON_REPO_TOKEN%" \ -e "WITH_DEB=OFF" \ -e "PADDLE_VERSION=%PADDLE_VERSION%" \ -e "PADDLE_FRACTION_GPU_MEMORY_TO_USE=0.15" \ -e "WITH_DISTRIBUTE=ON" \ -e "RUN_TEST=OFF" ${PADDLE_DEV_NAME} # run build with GPU and intel MKL and test. nvidia-docker run -i --rm -v $PWD:/paddle -v /root/.cache:/root/.cache\ -e "FLAGS_fraction_of_gpu_memory_to_use=0.15"\ -e "CTEST_OUTPUT_ON_FAILURE=%CTEST_OUTPUT_ON_FAILURE%"\ -e "CTEST_PARALLEL_LEVEL=%CTEST_PARALLEL_LEVEL%"\ -e "APT_MIRROR=${apt_mirror}"\ -e "WITH_GPU=ON"\ -e "CUDA_ARCH_NAME=Auto"\ -e "WITH_AVX=%WITH_AVX%"\ -e "WITH_GOLANG=%WITH_GOLANG%"\ -e "WITH_TESTING=ON"\ -e "WITH_C_API=%WITH_C_API%" \ -e "WITH_COVERAGE=ON" \ -e "COVERALLS_UPLOAD=ON" \ -e "GIT_PR_ID=${PR_ID}" \ -e "JSON_REPO_TOKEN=%JSON_REPO_TOKEN%" \ -e "WITH_DEB=OFF" \ -e "PADDLE_VERSION=%PADDLE_VERSION%" \ -e "PADDLE_FRACTION_GPU_MEMORY_TO_USE=0.15" \ -e "CUDA_VISIBLE_DEVICES=0,1" \ -e "WITH_DISTRIBUTE=ON" \ -e "RUN_TEST=ON" ${PADDLE_DEV_NAME} if [[ "%WITH_CACHE" == "ON" && ! -f $tp_cache_file ]]; then tar czvf $tp_cache_file ./build/third_party fi