[ML] Upgrade to PyTorch 2.7.1 (#2863)

PyTorch 2.7.1 is the latest stable release of PyTorch and contains several bug and security fixes.

Author: edsavage

.buildkite/pipelines/build_linux.json.py

@@ -36,7 +36,7 @@
  "cpu": "6",
  "ephemeralStorage": "20G",
  "memory": "64G",
- "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:33")
+ "image": os.getenv("DOCKER_IMAGE", "docker.elastic.co/ml-dev/ml-linux-build:34")
  },
  "aarch64": {
  "provider": "aws",
@@ -100,7 +100,7 @@ def main(args):
  "cpu": "6",
  "ephemeralStorage": "20G",
  "memory": "64G",
- "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:16"
+ "image": "docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:17"
  },
  "commands": [
  ".buildkite/scripts/steps/build_and_test.sh"

.buildkite/pipelines/build_macos.json.py

@@ -33,7 +33,7 @@
 agents = {
  "aarch64": {
  "provider": "orka",
- "image": "ml-macos-13-arm-004.orkasi"
+ "image": "ml-macos-13-arm-005.orkasi"
  }
 }
 envs = {

.buildkite/pipelines/build_windows.json.py

@@ -45,7 +45,7 @@ def main(args):
  "provider": "gcp",
  "machineType": "c2-standard-16",
  "minCpuPlatform": "Intel Cascade Lake",
- "image": "family/ml-cpp-4-windows-2022",
+ "image": "family/ml-cpp-5-windows-2022",
  },
  "commands": [
  f'if ( "{args.action}" -eq "debug" ) {{\$Env:ML_DEBUG="1"}}',

.ci/orka/README.md

@@ -9,7 +9,7 @@
 ## Files
 
 - `install.sh` The script that does the software installs on the image
-- `orka-macos-12-arm.pkr.hcl` The packer definition for a MacOS 12 ARM builder image
+- `orka-macos-13-arm.pkr.hcl` The packer definition for a MacOS 13 ARM builder image
 
 
 ## Set Up Packer
@@ -26,7 +26,7 @@ Make sure you are connected to the Orka VM.
 
 Packer requires access to secrets in vault, where VAULT_ADDR=https://vault-ci-prod.elastic.dev and VAULT_TOKEN must be set appropriately in the environment.
 
-Run the following to create the image (MacOS 12 ARM in this example):
+Run the following to create the image (MacOS 13 ARM in this example):
 
 ```
 packer build orka-macos-13-arm.pkr.hcl
@@ -63,7 +63,7 @@ The ML packer scripts do the following:
  * Install JDK `11.0.25`
  * Install CMake `3.30.5`
  * Install Boost `1.86.0` from source
- * Install PyTorch `2.5.1` from source
+ * Install PyTorch `2.7.1` from source
 
 ## Caveats
 

.ci/orka/orka-macos-13-arm.pkr.hcl

@@ -18,7 +18,7 @@ locals {
 
 source "macstadium-orka" "image" {
  source_image = "generic-13-ventura-arm-002.orkasi"
- image_name = "ml-macos-13-arm-004.orkasi"
+ image_name = "ml-macos-13-arm-005.orkasi"
  orka_endpoint = local.orka_endpoint
  orka_user = local.orka_user
  orka_password = local.orka_password

.ci/orka/third_party_deps.sh

@@ -34,7 +34,7 @@ sudo rm -rf boost_1_86_0
 sudo pip3 install numpy ninja pyyaml setuptools cffi typing_extensions future six requests dataclasses
 
 # Build and install PyTorch
-git clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git && \
+git clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git && \
 cd pytorch && \
 git submodule sync && \
 git submodule update --init --recursive && \
@@ -59,7 +59,7 @@ export DNNL_TARGET_ARCH=AARCH64
 export USE_MKLDNN=ON
 export USE_QNNPACK=OFF
 export USE_PYTORCH_QNNPACK=OFF
-export PYTORCH_BUILD_VERSION=2.5.1
+export PYTORCH_BUILD_VERSION=2.7.1
 export PYTORCH_BUILD_NUMBER=1
 python3 setup.py install
 

3rd_party/licenses/pytorch-INFO.csv

@@ -1,2 +1,2 @@
 name,version,revision,url,license,copyright,sourceURL
-PyTorch,2.5.1,a8d6afb511a69687bbb2b7e88a3cf67917e1697e,https://pytorch.org,BSD-3-Clause,,
+PyTorch,2.7.1,e2d141dbde55c2a4370fac5165b0561b6af4798b,https://pytorch.org,BSD-3-Clause,,

build-setup/linux.md

@@ -333,7 +333,7 @@ Then copy the shared libraries to the system directory:
 (cd /opt/intel/oneapi/mkl/2024.0 && tar cf - lib) | (cd /usr/local/gcc133 && sudo tar xvf -)
 ```
 
-### PyTorch 2.5.1
+### PyTorch 2.7.1
 
 (This step requires a reasonable amount of memory. It failed on a machine with 8GB of RAM. It succeeded on a 16GB machine. You can specify the number of parallel jobs using environment variable MAX_JOBS. Lower number of jobs will reduce memory usage.)
 
@@ -352,7 +352,7 @@ sudo /usr/local/gcc133/bin/python3.12 -m pip install numpy pyyaml setuptools cff
 Then obtain the PyTorch code:
 
 ```
-git clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git
+git clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git
 cd pytorch
 git submodule sync
 git submodule update --init --recursive
@@ -386,7 +386,7 @@ export USE_MKLDNN=ON
 export USE_QNNPACK=OFF
 export USE_PYTORCH_QNNPACK=OFF
 [ $(uname -m) = x86_64 ] && export USE_XNNPACK=OFF
-export PYTORCH_BUILD_VERSION=2.5.1
+export PYTORCH_BUILD_VERSION=2.7.1
 export PYTORCH_BUILD_NUMBER=1
 /usr/local/gcc133/bin/python3.12 setup.py install
 ```

build-setup/macos.md

@@ -133,18 +133,18 @@ Download the graphical installer for Python 3.10.10 from <https://www.python.org
 
 Install using all the default options. When the installer completes a Finder window pops up. Double click the `Install Certificates.command` file in this folder to install the SSL certificates Python needs.
 
-### PyTorch 2.5.1
+### PyTorch 2.7.1
 
 PyTorch requires that certain Python modules are installed. To install them:
 
 ```
-sudo /Library/Frameworks/Python.framework/Versions/3.10/bin/pip3.10 install numpy ninja pyyaml setuptools cffi typing_extensions future six requests dataclasses
+sudo /Library/Frameworks/Python.framework/Versions/3.12/bin/pip3.12 install numpy ninja pyyaml setuptools cffi typing_extensions future six requests dataclasses
 ```
 
 Then obtain the PyTorch code:
 
 ```
-git clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git
+git clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git
 cd pytorch
 git submodule sync
 git submodule update --init --recursive
@@ -188,9 +188,9 @@ export DNNL_TARGET_ARCH=AARCH64
 export USE_MKLDNN=ON
 export USE_QNNPACK=OFF
 export USE_PYTORCH_QNNPACK=OFF
-export PYTORCH_BUILD_VERSION=2.5.1
+export PYTORCH_BUILD_VERSION=2.7.1
 export PYTORCH_BUILD_NUMBER=1
-/Library/Frameworks/Python.framework/Versions/3.10/bin/python3.10 setup.py install
+/Library/Frameworks/Python.framework/Versions/3.12/bin/python3.12 setup.py install
 ```
 
 Once built copy headers and libraries to system directories:

build-setup/windows.md

@@ -202,7 +202,7 @@ On the "Advanced Options" screen, check "Install for all users" and "Add Python
 
 For the time being, do not take advantage of the option on the final installer screen to reconfigure the machine to allow paths longer than 260 characters. We still support Windows versions that do not have this option.
 
-### PyTorch 2.5.1
+### PyTorch 2.7.1
 
 (This step requires a lot of memory. It failed on a machine with 12GB of RAM. It just about fitted on a 20GB machine. 32GB RAM is recommended.)
 
@@ -216,7 +216,7 @@ Next, in a Git bash shell run:
 
 ```
 cd /c/tools
-git clone --depth=1 --branch=v2.5.1 https://github.com/pytorch/pytorch.git
+git clone --depth=1 --branch=v2.7.1 https://github.com/pytorch/pytorch.git
 cd pytorch
 git submodule sync
 git submodule update --init --recursive
@@ -272,7 +272,7 @@ set USE_QNNPACK=OFF
 set USE_PYTORCH_QNNPACK=OFF
 set USE_XNNPACK=OFF
 set MSVC_Z7_OVERRIDE=OFF
-set PYTORCH_BUILD_VERSION=2.5.1
+set PYTORCH_BUILD_VERSION=2.7.1
 set PYTORCH_BUILD_NUMBER=1
 python setup.py install
 ```