Improve API inter-process queue fairness (#1526)

Author: RobertLucian

cli/local/docker_spec.go

@@ -19,7 +19,6 @@ package local
 import (
 "context"
 "fmt"
-"math"
 "path/filepath"
 "strings"
 
@@ -92,9 +91,7 @@ func getAPIEnv(api *spec.API, awsClient *aws.Client) []string {
 "CORTEX_PROJECT_DIR="+_projectDir,
 "CORTEX_PROCESSES_PER_REPLICA="+s.Int32(api.Predictor.ProcessesPerReplica),
 "CORTEX_THREADS_PER_PROCESS="+s.Int32(api.Predictor.ThreadsPerProcess),
-// add 1 because it was required to achieve the target concurrency for 1 process, 1 thread
-"CORTEX_MAX_PROCESS_CONCURRENCY="+s.Int64(1+int64(math.Round(float64(consts.DefaultMaxReplicaConcurrency)/float64(api.Predictor.ProcessesPerReplica)))),
-"CORTEX_SO_MAX_CONN="+s.Int64(consts.DefaultMaxReplicaConcurrency+100), // add a buffer to be safe
+"CORTEX_MAX_REPLICA_CONCURRENCY="+s.Int32(api.Predictor.ProcessesPerReplica*api.Predictor.ThreadsPerProcess+1024), // allow a queue of 1024
 "AWS_REGION="+awsClient.Region,
 )
 

dev/versions.md

@@ -172,6 +172,18 @@ Note: it's ok if example training notebooks aren't upgraded, as long as the expo
  * be careful not to update any of the versions for Inferentia that are not latest in `images/python-predictor-inf/Dockerfile`
 1. Rerun all examples and check their logs
 
+## S6-overlay supervisor
+
+1. Locate the `s6-overlay` installation in `images/python-predictor-*/Dockerfile`, `images/tensorflow-predictor/Dockerfile` and `images/onnx-predictor-*/Dockerfile`
+1. Update the version in each serving image with the newer one in https://github.com/just-containers/s6-overlay.
+
+## Nginx
+
+1. Run a base image of ubuntu that matches the version tag used for the serving images. The running command is `docker run -it --rm <base-image>`
+1. Run `apt update && apt-cache policy nginx`. Notice the latest minor version of nginx (e.g. `1.14`)
+1. Locate the `nginx` package in `images/python-predictor-*/Dockerfile`, `images/tensorflow-predictor/Dockerfile` and `images/onnx-predictor-*/Dockerfile`
+1. Update the version for all `nginx` appearances using the minor version from step 2 and add an asterisk at the end to denote any version (e.g. `1.14.*`)
+
 ## Istio
 
 1. Find the latest [release](https://istio.io/latest/news/releases) and check the release notes (here are the [latest IstioOperator Options](https://istio.io/latest/docs/reference/config/istio.operator.v1alpha1/))

examples/sklearn/iris-classifier/cortex.yaml

@@ -12,4 +12,4 @@
  model_type: classification
  compute:
  cpu: 0.2
- mem: 100M
+ mem: 200M

examples/sklearn/iris-classifier/requirements.txt

@@ -1 +1,2 @@
 boto3
+scikit-learn==0.21.3

images/neuron-rtd/Dockerfile

@@ -13,7 +13,8 @@ RUN yum install -y \
  aws-neuron-runtime-1.0.9592.0 \
  procps-ng-3.3.10-26.amzn2.x86_64 \
  gzip \
- tar
+ tar \
+ curl
 
 ENV PATH="/opt/aws/neuron/bin:${PATH}"
 

images/onnx-predictor-cpu/Dockerfile

@@ -9,8 +9,14 @@ RUN apt-get update -qq && apt-get install -y -q \
  unzip \
  zlib1g-dev \
  locales \
+ nginx=1.14.* \
  && apt-get clean -qq && rm -rf /var/lib/apt/lists/*
 
+RUN cd /tmp/ && \
+ curl -L --output s6-overlay-amd64-installer "https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer" && \
+ cd - && \
+ chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer
+
 RUN locale-gen en_US.UTF-8
 ENV LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8
 
@@ -68,4 +74,6 @@ COPY pkg/workloads/cortex/consts.py /src/cortex
 COPY pkg/workloads/cortex/lib /src/cortex/lib
 COPY pkg/workloads/cortex/serve /src/cortex/serve
 
-ENTRYPOINT ["/src/cortex/serve/run.sh"]
+RUN mv /src/cortex/serve/init/bootloader.sh /etc/cont-init.d/bootloader.sh
+
+ENTRYPOINT ["/init"]

images/onnx-predictor-gpu/Dockerfile

@@ -9,8 +9,14 @@ RUN apt-get update -qq && apt-get install -y -q \
  unzip \
  zlib1g-dev \
  locales \
+ nginx=1.14.* \
  && apt-get clean -qq && rm -rf /var/lib/apt/lists/*
 
+RUN cd /tmp/ && \
+ curl -L --output s6-overlay-amd64-installer "https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer" && \
+ cd - && \
+ chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer
+
 RUN locale-gen en_US.UTF-8
 ENV LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8
 
@@ -68,4 +74,6 @@ COPY pkg/workloads/cortex/consts.py /src/cortex
 COPY pkg/workloads/cortex/lib /src/cortex/lib
 COPY pkg/workloads/cortex/serve /src/cortex/serve
 
-ENTRYPOINT ["/src/cortex/serve/run.sh"]
+RUN mv /src/cortex/serve/init/bootloader.sh /etc/cont-init.d/bootloader.sh
+
+ENTRYPOINT ["/init"]

images/python-predictor-cpu/Dockerfile

@@ -9,8 +9,14 @@ RUN apt-get update -qq && apt-get install -y -q \
  unzip \
  zlib1g-dev \
  locales \
+ nginx=1.14.* \
  && apt-get clean -qq && rm -rf /var/lib/apt/lists/*
 
+RUN cd /tmp/ && \
+ curl -L --output s6-overlay-amd64-installer "https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer" && \
+ cd - && \
+ chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer
+
 RUN locale-gen en_US.UTF-8
 ENV LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8
 
@@ -88,4 +94,6 @@ COPY pkg/workloads/cortex/consts.py /src/cortex
 COPY pkg/workloads/cortex/lib /src/cortex/lib
 COPY pkg/workloads/cortex/serve /src/cortex/serve
 
-ENTRYPOINT ["/src/cortex/serve/run.sh"]
+RUN mv /src/cortex/serve/init/bootloader.sh /etc/cont-init.d/bootloader.sh
+
+ENTRYPOINT ["/init"]

images/python-predictor-gpu/Dockerfile

@@ -11,8 +11,14 @@ RUN apt-get update -qq && apt-get install -y -q \
  unzip \
  zlib1g-dev \
  locales \
+ nginx=1.14.* \
  && apt-get clean -qq && rm -rf /var/lib/apt/lists/*
 
+RUN cd /tmp/ && \
+ curl -L --output s6-overlay-amd64-installer "https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer" && \
+ cd - && \
+ chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer
+
 RUN locale-gen en_US.UTF-8
 ENV LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8
 
@@ -93,4 +99,6 @@ COPY pkg/workloads/cortex/consts.py /src/cortex
 COPY pkg/workloads/cortex/lib /src/cortex/lib
 COPY pkg/workloads/cortex/serve /src/cortex/serve
 
-ENTRYPOINT ["/src/cortex/serve/run.sh"]
+RUN mv /src/cortex/serve/init/bootloader.sh /etc/cont-init.d/bootloader.sh
+
+ENTRYPOINT ["/init"]

images/python-predictor-inf/Dockerfile

@@ -10,6 +10,9 @@ RUN apt-get update -qq && apt-get install -y -q \
  aws-neuron-runtime=1.1.1402.0 && \
  apt-get clean -qq && rm -rf /var/lib/apt/lists/*
 
+RUN wget -P /tmp/ https://github.com/just-containers/s6-overlay/releases/download/v2.1.0.2/s6-overlay-amd64-installer && \
+ chmod +x /tmp/s6-overlay-amd64-installer && /tmp/s6-overlay-amd64-installer / && rm /tmp/s6-overlay-amd64-installer
+
 ENV PATH=/opt/aws/neuron/bin/:$PATH
 
 RUN apt-get update -qq && apt-get install -y -q \
@@ -21,6 +24,7 @@ RUN apt-get update -qq && apt-get install -y -q \
  unzip \
  zlib1g-dev \
  locales \
+ nginx=1.14.* \
  && apt-get clean -qq && rm -rf /var/lib/apt/lists/*
 
 RUN locale-gen en_US.UTF-8
@@ -100,4 +104,6 @@ COPY pkg/workloads/cortex/consts.py /src/cortex
 COPY pkg/workloads/cortex/lib /src/cortex/lib
 COPY pkg/workloads/cortex/serve /src/cortex/serve
 
-ENTRYPOINT ["/src/cortex/serve/run.sh"]
+RUN mv /src/cortex/serve/init/bootloader.sh /etc/cont-init.d/bootloader.sh
+
+ENTRYPOINT ["/init"]