Refactor Docker setup and consolidate requirements

- Update Dockerfile and docker-compose.yml - Consolidate requirements files and remove duplicates - Clean up training/requirements_freeze.txt

Author: rjsu26

applications/DeepSpeed-Chat/Dockerfile

@@ -41,13 +41,12 @@ RUN python -m pip install --upgrade pip setuptools wheel
 
 WORKDIR /workspace
 
-# Copy and install dependencies
-COPY requirements.txt requirements_freeze.txt ./
-RUN pip install -r requirements_freeze.txt
+# Install PyTorch first with CUDA 11.8 support
+RUN pip install torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0 --index-url https://download.pytorch.org/whl/cu118
 
-# Install additional packages that may be needed
-# H100/H200 specific versions
-RUN pip install --force-reinstall numpy==1.24.4 torch==2.5.0 pyarrow==18.0.0 tensorboard
+# Copy and install other dependencies
+COPY training/requirements_freeze.txt ./
+RUN pip install -r requirements_freeze.txt
 
 # Copy project code
 COPY . .

applications/DeepSpeed-Chat/docker-compose.yml

@@ -12,10 +12,10 @@ services:
  command: /bin/bash -c "/usr/sbin/sshd && tail -f /dev/null"
 
  environment:
- - PROJECT_PATH=${PROJECT_PATH}
- - HF_HOME=${PROJECT_PATH}/.cache/huggingface
- - TRANSFORMERS_CACHE=${PROJECT_PATH}/.cache/huggingface/transformers
- - HF_DATASETS_CACHE=${PROJECT_PATH}/.cache/huggingface/datasets
+ - PROJECT_PATH=${PROJECT_PATH:-/workspace}
+ - HF_HOME=${PROJECT_PATH:-/workspace}/.cache/huggingface
+ - TRANSFORMERS_CACHE=${PROJECT_PATH:-/workspace}/.cache/huggingface/transformers
+ - HF_DATASETS_CACHE=${PROJECT_PATH:-/workspace}/.cache/huggingface/datasets
  - NVIDIA_VISIBLE_DEVICES=all
  - NVIDIA_DRIVER_CAPABILITIES=compute,utility
 

applications/DeepSpeed-Chat/requirements_freeze.txt

@@ -30,7 +30,7 @@ mpmath==1.3.0
 msgpack==1.1.2
 multidict==6.7.0
 multiprocess==0.70.15
-networkx==3.5
+networkx==3.1
 ninja==1.13.0
 numpy==1.26.4
 nvidia-cublas-cu12==12.8.4.1
@@ -75,9 +75,6 @@ sniffio==1.3.1
 sympy==1.14.0
 termcolor==3.1.0
 tokenizers==0.20.1
-torch==2.9.0+cu128
-torchaudio==2.9.0+cu128
-torchvision==0.24.0+cu128
 tqdm==4.67.1
 transformers==4.46.3
 triton==3.5.0