Skip to content

Commit 157ac58

Browse files
ydshiehydshieh
committed
Build docker images for tokenizers main
1 parent c20b2c7 commit 157ac58

File tree

5 files changed

+45
-91
lines changed

5 files changed

+45
-91
lines changed

‎.github/workflows/build-docker-images.yml‎

Lines changed: 5 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ on:
44
push:
55
branches:
66
- docker-image*
7+
- test-tokenizers-*
78
repository_dispatch:
89
workflow_call:
910
inputs:
@@ -42,36 +43,7 @@ jobs:
4243
build-args: |
4344
REF=main
4445
push: true
45-
tags: huggingface/transformers-all-latest-gpu${{ inputs.image_postfix }}
46-
47-
latest-with-torch-nightly-docker:
48-
name: "Nightly PyTorch + Stable TensorFlow"
49-
# Push CI doesn't need this image
50-
if: inputs.image_postfix != '-push-ci'
51-
runs-on: ubuntu-latest
52-
steps:
53-
-
54-
name: Set up Docker Buildx
55-
uses: docker/setup-buildx-action@v1
56-
-
57-
name: Check out code
58-
uses: actions/checkout@v2
59-
-
60-
name: Login to DockerHub
61-
uses: docker/login-action@v1
62-
with:
63-
username: ${{ secrets.DOCKERHUB_USERNAME }}
64-
password: ${{ secrets.DOCKERHUB_PASSWORD }}
65-
-
66-
name: Build and push
67-
uses: docker/build-push-action@v2
68-
with:
69-
context: ./docker/transformers-all-latest-gpu
70-
build-args: |
71-
REF=main
72-
PYTORCH=pre
73-
push: true
74-
tags: huggingface/transformers-all-latest-torch-nightly-gpu
46+
tags: huggingface/internal-transformers-all-latest-gpu-tokenizers-main
7547

7648
latest-torch-deepspeed-docker:
7749
name: "Latest PyTorch + DeepSpeed"
@@ -97,61 +69,7 @@ jobs:
9769
build-args: |
9870
REF=main
9971
push: true
100-
tags: huggingface/transformers-pytorch-deepspeed-latest-gpu${{ inputs.image_postfix }}
101-
102-
nightly-torch-deepspeed-docker:
103-
name: "Nightly PyTorch + DeepSpeed"
104-
# Push CI doesn't need this image
105-
if: inputs.image_postfix != '-push-ci'
106-
runs-on: ubuntu-latest
107-
steps:
108-
-
109-
name: Set up Docker Buildx
110-
uses: docker/setup-buildx-action@v1
111-
-
112-
name: Check out code
113-
uses: actions/checkout@v2
114-
-
115-
name: Login to DockerHub
116-
uses: docker/login-action@v1
117-
with:
118-
username: ${{ secrets.DOCKERHUB_USERNAME }}
119-
password: ${{ secrets.DOCKERHUB_PASSWORD }}
120-
-
121-
name: Build and push
122-
uses: docker/build-push-action@v2
123-
with:
124-
context: ./docker/transformers-pytorch-deepspeed-nightly-gpu
125-
build-args: |
126-
REF=main
127-
push: true
128-
tags: huggingface/transformers-pytorch-deepspeed-nightly-gpu
129-
130-
doc-builder:
131-
name: "Doc builder"
132-
# Push CI doesn't need this image
133-
if: inputs.image_postfix != '-push-ci'
134-
runs-on: ubuntu-latest
135-
steps:
136-
-
137-
name: Set up Docker Buildx
138-
uses: docker/setup-buildx-action@v1
139-
-
140-
name: Check out code
141-
uses: actions/checkout@v2
142-
-
143-
name: Login to DockerHub
144-
uses: docker/login-action@v1
145-
with:
146-
username: ${{ secrets.DOCKERHUB_USERNAME }}
147-
password: ${{ secrets.DOCKERHUB_PASSWORD }}
148-
-
149-
name: Build and push
150-
uses: docker/build-push-action@v2
151-
with:
152-
context: ./docker/transformers-doc-builder
153-
push: true
154-
tags: huggingface/transformers-doc-builder
72+
tags: huggingface/internal-transformers-pytorch-deepspeed-latest-gpu-tokenizers-main
15573

15674
latest-pytorch:
15775
name: "Latest PyTorch [dev]"
@@ -179,7 +97,7 @@ jobs:
17997
build-args: |
18098
REF=main
18199
push: true
182-
tags: huggingface/transformers-pytorch-gpu
100+
tags: huggingface/internal-transformers-pytorch-gpu-tokenizers-main
183101

184102
latest-tensorflow:
185103
name: "Latest TensorFlow [dev]"
@@ -207,4 +125,4 @@ jobs:
207125
build-args: |
208126
REF=main
209127
push: true
210-
tags: huggingface/transformers-tensorflow-gpu
128+
tags: huggingface/internal-transformers-tensorflow-gpu-tokenizers-main

‎docker/transformers-all-latest-gpu/Dockerfile‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ ARG INTEL_TORCH_EXT='1.11.0'
1616
ARG CUDA='cu113'
1717

1818
RUN apt update
19-
RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs
19+
RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs curl libssl-dev
2020
RUN git lfs install
2121
RUN python3 -m pip install --no-cache-dir --upgrade pip
2222

@@ -53,3 +53,12 @@ RUN python3 -m pip install --no-cache-dir decord
5353
# When installing in editable mode, `transformers` is not recognized as a package.
5454
# this line must be added in order for python to be aware of transformers.
5555
RUN cd transformers && python3 setup.py develop
56+
57+
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
58+
ENV PATH="/root/.cargo/bin:${PATH}"
59+
60+
RUN cargo --help
61+
RUN python3 -m pip uninstall -y tokenizers
62+
RUN git clone https://github.com/huggingface/tokenizers
63+
RUN python3 -m pip install --no-cache-dir setuptools-rust
64+
RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python

‎docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ ARG PYTORCH='1.12.1'
88
ARG CUDA='cu113'
99

1010
RUN apt -y update
11-
RUN apt install -y libaio-dev
11+
RUN apt install -y libaio-dev curl libssl-dev
1212
RUN python3 -m pip install --no-cache-dir --upgrade pip
1313

1414
ARG REF=main
@@ -33,3 +33,12 @@ RUN DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 py
3333
RUN cd transformers && python3 setup.py develop
3434

3535
RUN python3 -c "from deepspeed.launcher.runner import main"
36+
37+
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
38+
ENV PATH="/root/.cargo/bin:${PATH}"
39+
40+
RUN cargo --help
41+
RUN python3 -m pip uninstall -y tokenizers
42+
RUN git clone https://github.com/huggingface/tokenizers
43+
RUN python3 -m pip install --no-cache-dir setuptools-rust
44+
RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python

‎docker/transformers-pytorch-gpu/Dockerfile‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ LABEL maintainer="Hugging Face"
44
ARG DEBIAN_FRONTEND=noninteractive
55

66
RUN apt update
7-
RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg
7+
RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg curl libssl-dev
88
RUN python3 -m pip install --no-cache-dir --upgrade pip
99

1010
ARG REF=main
@@ -29,3 +29,12 @@ RUN python3 -m pip install -U "itsdangerous<2.1.0"
2929
# When installing in editable mode, `transformers` is not recognized as a package.
3030
# this line must be added in order for python to be aware of transformers.
3131
RUN cd transformers && python3 setup.py develop
32+
33+
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
34+
ENV PATH="/root/.cargo/bin:${PATH}"
35+
36+
RUN cargo --help
37+
RUN python3 -m pip uninstall -y tokenizers
38+
RUN git clone https://github.com/huggingface/tokenizers
39+
RUN python3 -m pip install --no-cache-dir setuptools-rust
40+
RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python

‎docker/transformers-tensorflow-gpu/Dockerfile‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ LABEL maintainer="Hugging Face"
44
ARG DEBIAN_FRONTEND=noninteractive
55

66
RUN apt update
7-
RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg
7+
RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg curl libssl-dev
88
RUN python3 -m pip install --no-cache-dir --upgrade pip
99

1010
ARG REF=main
@@ -21,3 +21,12 @@ RUN python3 -m pip install -U "itsdangerous<2.1.0"
2121
# When installing in editable mode, `transformers` is not recognized as a package.
2222
# this line must be added in order for python to be aware of transformers.
2323
RUN cd transformers && python3 setup.py develop
24+
25+
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
26+
ENV PATH="/root/.cargo/bin:${PATH}"
27+
28+
RUN cargo --help
29+
RUN python3 -m pip uninstall -y tokenizers
30+
RUN git clone https://github.com/huggingface/tokenizers
31+
RUN python3 -m pip install --no-cache-dir setuptools-rust
32+
RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python

0 commit comments

Comments
 (0)