Add support for private docker image registries

cli/local/validations.go

@@ -118,7 +118,7 @@ func ValidateLocalAPIs(apis []userconfig.API, projectFiles ProjectFiles, awsClie
 for i := range apis {
 api := &apis[i]
 
-if err := spec.ValidateAPI(api, projectFiles, types.LocalProviderType, awsClient); err != nil {
+if err := spec.ValidateAPI(api, projectFiles, types.LocalProviderType, awsClient, nil); err != nil {
 return errors.Wrap(err, api.Identify())
 }
 

docs/guides/private-docker.md

@@ -0,0 +1,59 @@
+# Private docker registry
+
+_WARNING: you are on the master branch, please refer to the docs on the branch that matches your `cortex version`_
+
+Until [#1459](https://github.com/cortexlabs/cortex/issues/1459) is addressed, you can use a private docker registry for your Predictor images by following this guide.
+
+## Local
+
+When running Cortex locally, you can use private Docker images by running `docker login` and then `docker pull <your_image>`. The Docker image will be present on your machine, and will be accessible by Cortex the next time you run `cortex deploy`.
+
+## Cluster
+
+### Step 1
+
+Install and configure kubectl ([instructions](kubectl-setup.md)).
+
+### Step 2
+
+Set the following environment variables, replacing the placeholders with your docker username and password:
+
+```bash
+DOCKER_USERNAME=***
+DOCKER_PASSWORD=***
+```
+
+Run the following commands:
+
+```bash
+kubectl create secret docker-registry registry-credentials \
+ --namespace default \
+ --docker-username=$DOCKER_USERNAME \
+ --docker-password=$DOCKER_PASSWORD
+
+kubectl patch serviceaccount default \
+ --namespace default \
+ -p "{\"imagePullSecrets\": [{\"name\": \"registry-credentials\"}]}"
+```
+
+### Updating your credentials
+
+To remove your docker credentials from the cluster, run this command:
+
+```bash
+kubectl delete secret --namespace default registry-credentials
+```
+
+Then repeat step 2 above with your updated credentials.
+
+### Removing your credentials
+
+To remove your docker credentials from the cluster, run the following commands:
+
+```bash
+kubectl delete secret --namespace default registry-credentials
+
+kubectl patch serviceaccount default \
+ --namespace default \
+ -p "{\"imagePullSecrets\": []}"
+```

docs/summary.md

@@ -71,6 +71,7 @@
 * [SSH into worker instance](guides/ssh-instance.md)
 * [Single node deployment](guides/single-node-deployment.md)
 * [Set up kubectl](guides/kubectl-setup.md)
+* [Private docker registry](guides/private-docker.md)
 
 ## Contributing
 

pkg/lib/docker/docker.go

@@ -37,6 +37,7 @@ import (
 "github.com/cortexlabs/cortex/pkg/lib/parallel"
 "github.com/cortexlabs/cortex/pkg/lib/print"
 "github.com/cortexlabs/cortex/pkg/lib/slices"
+"github.com/cortexlabs/cortex/pkg/types"
 dockertypes "github.com/docker/docker/api/types"
 dockerclient "github.com/docker/docker/client"
 "github.com/docker/docker/pkg/jsonmessage"
@@ -47,15 +48,15 @@ var NoAuth string
 
 var _cachedClient *Client
 
+func init() {
+NoAuth, _ = EncodeAuthConfig(dockertypes.AuthConfig{})
+}
+
 type Client struct {
 *dockerclient.Client
 Info dockertypes.Info
 }
 
-func init() {
-NoAuth, _ = EncodeAuthConfig(dockertypes.AuthConfig{})
-}
-
 func GetDockerClient() (*Client, error) {
 if _cachedClient != nil {
 return _cachedClient, nil
@@ -146,7 +147,7 @@ func PullImage(image string, encodedAuthConfig string, pullVerbosity PullVerbosi
 return false, err
 }
 
-if err := CheckLocalImageAccessible(dockerClient, image); err == nil {
+if err := CheckImageExistsLocally(dockerClient, image); err == nil {
 return false, nil
 }
 
@@ -314,14 +315,14 @@ func EncodeAuthConfig(authConfig dockertypes.AuthConfig) (string, error) {
 return registryAuth, nil
 }
 
-func CheckImageAccessible(dockerClient *Client, dockerImage, authConfig string) error {
+func CheckImageAccessible(dockerClient *Client, dockerImage, authConfig string, providerType types.ProviderType) error {
 if _, err := dockerClient.DistributionInspect(context.Background(), dockerImage, authConfig); err != nil {
-return ErrorImageInaccessible(dockerImage, err)
+return ErrorImageInaccessible(dockerImage, providerType, err)
 }
 return nil
 }
 
-func CheckLocalImageAccessible(dockerClient *Client, dockerImage string) error {
+func CheckImageExistsLocally(dockerClient *Client, dockerImage string) error {
 images, err := dockerClient.ImageList(context.Background(), dockertypes.ImageListOptions{})
 if err != nil {
 return WrapDockerError(err)
@@ -337,7 +338,8 @@ func CheckLocalImageAccessible(dockerClient *Client, dockerImage string) error {
 return nil
 }
 }
-return ErrorImageInaccessible(dockerImage, nil)
+
+return ErrorImageDoesntExistLocally(dockerImage)
 }
 
 func ExtractImageTag(dockerImage string) string {

pkg/lib/docker/errors.go

@@ -21,13 +21,16 @@ import (
 "runtime"
 "strings"
 
+"github.com/cortexlabs/cortex/pkg/consts"
 "github.com/cortexlabs/cortex/pkg/lib/errors"
+"github.com/cortexlabs/cortex/pkg/types"
 )
 
 const (
-ErrConnectToDockerDaemon = "docker.connect_to_docker_daemon"
-ErrDockerPermissions = "docker.docker_permissions"
-ErrImageInaccessible = "docker.image_inaccessible"
+ErrConnectToDockerDaemon = "docker.connect_to_docker_daemon"
+ErrDockerPermissions = "docker.docker_permissions"
+ErrImageDoesntExistLocally = "docker.image_doesnt_exist_locally"
+ErrImageInaccessible = "docker.image_inaccessible"
 )
 
 func ErrorConnectToDockerDaemon() error {
@@ -56,10 +59,29 @@ func ErrorDockerPermissions(err error) error {
 })
 }
 
-func ErrorImageInaccessible(image string, cause error) error {
+func ErrorImageDoesntExistLocally(image string) error {
+return errors.WithStack(&errors.Error{
+Kind: ErrImageDoesntExistLocally,
+Message: fmt.Sprintf("%s does not exist locally; download it with `docker pull %s` (if your registry is private, run `docker login` first)", image, image),
+})
+}
+
+func ErrorImageInaccessible(image string, providerType types.ProviderType, cause error) error {
 message := fmt.Sprintf("%s is not accessible", image)
+
 if cause != nil {
-message += "\n" + errors.Message(cause) // add \n because docker client errors are
+message += "\n" + errors.Message(cause) // add \n because docker client errors are verbose but useful
+}
+
+if providerType == types.LocalProviderType {
+message += fmt.Sprintf("\n\nyou can download your image with `docker pull %s` and try this command again", image)
+if strings.Contains(cause.Error(), "authorized") || strings.Contains(cause.Error(), "authentication") {
+message += " (if your registry is private, run `docker login` first)"
+}
+} else if providerType == types.AWSProviderType {
+if strings.Contains(cause.Error(), "authorized") || strings.Contains(cause.Error(), "authentication") {
+message += fmt.Sprintf("\n\nif you would like to use a private docker registry, see https://docs.cortex.dev/v/%s/guides/private-docker", consts.CortexVersionMinor)
+}
 }
 
 return errors.WithStack(&errors.Error{

pkg/lib/k8s/k8s.go

@@ -56,6 +56,7 @@ type Client struct {
 nodeClient kclientcore.NodeInterface
 serviceClient kclientcore.ServiceInterface
 configMapClient kclientcore.ConfigMapInterface
+secretClient kclientcore.SecretInterface
 deploymentClient kclientapps.DeploymentInterface
 jobClient kclientbatch.JobInterface
 ingressClient kclientextensions.IngressInterface
@@ -100,6 +101,7 @@ func New(namespace string, inCluster bool) (*Client, error) {
 client.nodeClient = client.clientset.CoreV1().Nodes()
 client.serviceClient = client.clientset.CoreV1().Services(namespace)
 client.configMapClient = client.clientset.CoreV1().ConfigMaps(namespace)
+client.secretClient = client.clientset.CoreV1().Secrets(namespace)
 client.deploymentClient = client.clientset.AppsV1().Deployments(namespace)
 client.jobClient = client.clientset.BatchV1().Jobs(namespace)
 client.ingressClient = client.clientset.ExtensionsV1beta1().Ingresses(namespace)

pkg/lib/k8s/secret.go

@@ -0,0 +1,155 @@
+/*
+Copyright 2020 Cortex Labs, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8s
+
+import (
+"context"
+
+"github.com/cortexlabs/cortex/pkg/lib/errors"
+kcore "k8s.io/api/core/v1"
+kerrors "k8s.io/apimachinery/pkg/api/errors"
+kmeta "k8s.io/apimachinery/pkg/apis/meta/v1"
+klabels "k8s.io/apimachinery/pkg/labels"
+)
+
+var _secretTypeMeta = kmeta.TypeMeta{
+APIVersion: "v1",
+Kind: "Secret",
+}
+
+type SecretSpec struct {
+Name string
+Data map[string][]byte
+Labels map[string]string
+Annotations map[string]string
+}
+
+func Secret(spec *SecretSpec) *kcore.Secret {
+secret := &kcore.Secret{
+TypeMeta: _secretTypeMeta,
+ObjectMeta: kmeta.ObjectMeta{
+Name: spec.Name,
+Labels: spec.Labels,
+Annotations: spec.Annotations,
+},
+Data: spec.Data,
+}
+return secret
+}
+
+func (c *Client) CreateSecret(secret *kcore.Secret) (*kcore.Secret, error) {
+secret.TypeMeta = _secretTypeMeta
+secret, err := c.secretClient.Create(context.Background(), secret, kmeta.CreateOptions{})
+if err != nil {
+return nil, errors.WithStack(err)
+}
+return secret, nil
+}
+
+func (c *Client) UpdateSecret(secret *kcore.Secret) (*kcore.Secret, error) {
+secret.TypeMeta = _secretTypeMeta
+secret, err := c.secretClient.Update(context.Background(), secret, kmeta.UpdateOptions{})
+if err != nil {
+return nil, errors.WithStack(err)
+}
+return secret, nil
+}
+
+func (c *Client) ApplySecret(secret *kcore.Secret) (*kcore.Secret, error) {
+existing, err := c.GetSecret(secret.Name)
+if err != nil {
+return nil, err
+}
+if existing == nil {
+return c.CreateSecret(secret)
+}
+return c.UpdateSecret(secret)
+}
+
+func (c *Client) GetSecret(name string) (*kcore.Secret, error) {
+secret, err := c.secretClient.Get(context.Background(), name, kmeta.GetOptions{})
+if kerrors.IsNotFound(err) {
+return nil, nil
+}
+if err != nil {
+return nil, errors.WithStack(err)
+}
+secret.TypeMeta = _secretTypeMeta
+return secret, nil
+}
+
+func (c *Client) GetSecretData(name string) (map[string][]byte, error) {
+secret, err := c.GetSecret(name)
+if err != nil {
+return nil, err
+}
+if secret == nil {
+return nil, nil
+}
+return secret.Data, nil
+}
+
+func (c *Client) DeleteSecret(name string) (bool, error) {
+err := c.secretClient.Delete(context.Background(), name, _deleteOpts)
+if kerrors.IsNotFound(err) {
+return false, nil
+}
+if err != nil {
+return false, errors.WithStack(err)
+}
+return true, nil
+}
+
+func (c *Client) ListSecrets(opts *kmeta.ListOptions) ([]kcore.Secret, error) {
+if opts == nil {
+opts = &kmeta.ListOptions{}
+}
+secretList, err := c.secretClient.List(context.Background(), *opts)
+if err != nil {
+return nil, errors.WithStack(err)
+}
+for i := range secretList.Items {
+secretList.Items[i].TypeMeta = _secretTypeMeta
+}
+return secretList.Items, nil
+}
+
+func (c *Client) ListSecretsByLabels(labels map[string]string) ([]kcore.Secret, error) {
+opts := &kmeta.ListOptions{
+LabelSelector: klabels.SelectorFromSet(labels).String(),
+}
+return c.ListSecrets(opts)
+}
+
+func (c *Client) ListSecretsByLabel(labelKey string, labelValue string) ([]kcore.Secret, error) {
+return c.ListSecretsByLabels(map[string]string{labelKey: labelValue})
+}
+
+func (c *Client) ListSecretsWithLabelKeys(labelKeys ...string) ([]kcore.Secret, error) {
+opts := &kmeta.ListOptions{
+LabelSelector: LabelExistsSelector(labelKeys...),
+}
+return c.ListSecrets(opts)
+}
+
+func SecretMap(secrets []kcore.Secret) map[string]kcore.Secret {
+secretMap := map[string]kcore.Secret{}
+for _, secret := range secrets {
+secretMap[secret.Name] = secret
+}
+return secretMap
+}

pkg/operator/resources/validations.go

@@ -102,7 +102,7 @@ func ValidateClusterAPIs(apis []userconfig.API, projectFiles spec.ProjectFiles)
 for i := range apis {
 api := &apis[i]
 if api.Kind == userconfig.RealtimeAPIKind || api.Kind == userconfig.BatchAPIKind {
-if err := spec.ValidateAPI(api, projectFiles, types.AWSProviderType, config.AWS); err != nil {
+if err := spec.ValidateAPI(api, projectFiles, types.AWSProviderType, config.AWS, config.K8s); err != nil {
 return errors.Wrap(err, api.Identify())
 }
 if err := validateK8s(api, virtualServices, maxMem); err != nil {