Skip to content

Commit 9dea925

Browse files
bart0shbart0sh
authored
Merge pull request intel#1088 from hj-johannes-lee/dlb-initcontainer
dlb: add initcontainer to plugin
2 parents 75bff62 + 11b0442 commit 9dea925

File tree

16 files changed

+286
-17
lines changed

16 files changed

+286
-17
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ jobs:
120120
- intel-iaa-plugin
121121
- intel-idxd-config-initcontainer
122122
- intel-dlb-plugin
123+
- intel-dlb-initcontainer
123124

124125
# Demo images
125126
- crypto-perf

‎.github/workflows/e2e-dlb.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313
- 'release-*'
1414

1515
env:
16-
IMAGES: 'intel-dlb-plugin dlb-libdlb-demo'
16+
IMAGES: 'intel-dlb-plugin intel-dlb-initcontainer dlb-libdlb-demo'
1717

1818
permissions:
1919
contents: read

‎build/docker/intel-dlb-initcontainer.Dockerfile‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
## This is a generated file, do not edit directly. Edit build/docker/templates/intel-dlb-initcontainer.Dockerfile.in instead.
2+
##
3+
## Copyright 2022 Intel Corporation. All Rights Reserved.
4+
##
5+
## Licensed under the Apache License, Version 2.0 (the "License");
6+
## you may not use this file except in compliance with the License.
7+
## You may obtain a copy of the License at
8+
##
9+
## http://www.apache.org/licenses/LICENSE-2.0
10+
##
11+
## Unless required by applicable law or agreed to in writing, software
12+
## distributed under the License is distributed on an "AS IS" BASIS,
13+
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
## See the License for the specific language governing permissions and
15+
## limitations under the License.
16+
###
17+
## FINAL_BASE can be used to configure the base image of the final image.
18+
##
19+
## This is used in two ways:
20+
## 1) make <image-name> BUILDER=<docker|buildah>
21+
## 2) docker build ... -f <image-name>.Dockerfile
22+
##
23+
## The project default is 1) which sets FINAL_BASE=gcr.io/distroless/static
24+
## (see build-image.sh).
25+
## 2) and the default FINAL_BASE is primarily used to build Redhat Certified Openshift Operator container images that must be UBI based.
26+
## The RedHat build tool does not allow additional image build parameters.
27+
ARG FINAL_BASE=registry.access.redhat.com/ubi8-micro
28+
###
29+
##
30+
## GOLANG_BASE can be used to make the build reproducible by choosing an
31+
## image by its hash:
32+
## GOLANG_BASE=golang@sha256:9d64369fd3c633df71d7465d67d43f63bb31192193e671742fa1c26ebc3a6210
33+
##
34+
## This is used on release branches before tagging a stable version.
35+
## The main branch defaults to using the latest Golang base image.
36+
ARG GOLANG_BASE=golang:1.19-bullseye
37+
###
38+
FROM ${GOLANG_BASE} as builder
39+
ARG DIR=/intel-device-plugins-for-kubernetes
40+
WORKDIR $DIR
41+
COPY . .
42+
ARG TOYBOX_VERSION="0.8.7"
43+
ARG TOYBOX_SHA256="b6f43d5738df54623ed21c32f430d1d5c5ac7ef465a6a883890f104b59d5d9e4"
44+
ARG ROOT=/install_root
45+
RUN apt update && apt -y install musl musl-tools musl-dev
46+
RUN curl -SL https://github.com/landley/toybox/archive/refs/tags/$TOYBOX_VERSION.tar.gz -o toybox.tar.gz \
47+
&& echo "$TOYBOX_SHA256 toybox.tar.gz" | sha256sum -c - \
48+
&& tar -xzf toybox.tar.gz \
49+
&& rm toybox.tar.gz \
50+
&& cd toybox-$TOYBOX_VERSION \
51+
&& KCONFIG_CONFIG=${DIR}/build/docker/toybox-config LDFLAGS="--static" CC=musl-gcc PREFIX=$ROOT V=2 make toybox install \
52+
&& install -D LICENSE $ROOT/licenses/toybox \
53+
&& cp -r /usr/share/doc/musl $ROOT/licenses/
54+
###
55+
FROM ${FINAL_BASE}
56+
LABEL vendor='Intel®'
57+
LABEL version='devel'
58+
LABEL release='1'
59+
COPY --from=builder /install_root /
60+
COPY demo/dlb-init.sh /usr/bin/
61+
ENTRYPOINT [ "/bin/bash", "dlb-init.sh"]

‎build/docker/templates/intel-dlb-initcontainer.Dockerfile.in‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#include "final_base.docker"
2+
#include "golang_base.docker"
3+
4+
FROM ${GOLANG_BASE} as builder
5+
6+
ARG DIR=/intel-device-plugins-for-kubernetes
7+
WORKDIR $DIR
8+
COPY . .
9+
10+
#include "toybox_build.docker"
11+
12+
FROM ${FINAL_BASE}
13+
14+
#include "default_labels.docker"
15+
16+
COPY --from=builder /install_root /
17+
18+
COPY demo/dlb-init.sh /usr/bin/
19+
ENTRYPOINT [ "/bin/bash", "dlb-init.sh"]

‎build/docker/toybox-config‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#
22
# Automatically generated make config: don't edit
33
# ToyBox version: KCONFIG_VERSION
4-
# Wed Jun 8 09:08:35 2022
4+
# Thu Sep 29 05:22:30 2022
55
#
66
CONFIG_TOYBOX_CONTAINER=y
77
CONFIG_TOYBOX_FIFREEZE=y
@@ -19,7 +19,7 @@ CONFIG_TOYBOX_COPYFILERANGE=y
1919
#
2020
# Posix commands
2121
#
22-
# CONFIG_BASENAME is not set
22+
CONFIG_BASENAME=y
2323
# CONFIG_CAL is not set
2424
CONFIG_CAT=y
2525
# CONFIG_CHGRP is not set
@@ -38,7 +38,7 @@ CONFIG_CP=y
3838
# CONFIG_DF is not set
3939
# CONFIG_DIRNAME is not set
4040
# CONFIG_DU is not set
41-
# CONFIG_ECHO is not set
41+
CONFIG_ECHO=y
4242
# CONFIG_ENV is not set
4343
# CONFIG_EXPAND is not set
4444
# CONFIG_FALSE is not set
@@ -48,7 +48,7 @@ CONFIG_CP=y
4848
CONFIG_GREP=y
4949
CONFIG_EGREP=y
5050
CONFIG_FGREP=y
51-
# CONFIG_HEAD is not set
51+
CONFIG_HEAD=y
5252
# CONFIG_ICONV is not set
5353
# CONFIG_ID is not set
5454
# CONFIG_ID_Z is not set
@@ -263,7 +263,7 @@ CONFIG_LSPCI=y
263263
# CONFIG_READAHEAD is not set
264264
# CONFIG_READELF is not set
265265
# CONFIG_READLINK is not set
266-
# CONFIG_REALPATH is not set
266+
CONFIG_REALPATH=y
267267
# CONFIG_REBOOT is not set
268268
# CONFIG_RESET is not set
269269
# CONFIG_REV is not set

‎demo/dlb-init.sh‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/bin/sh -eu
2+
3+
enable_and_configure_vfs() {
4+
devpath=$1
5+
6+
sriov_numvfs_path="$devpath/sriov_numvfs"
7+
if ! test -w "$sriov_numvfs_path"; then
8+
echo "error: $sriov_numvfs_path is not found or not writable. Check if dlb driver module is loaded"
9+
exit 1
10+
fi
11+
if [ "$(cat "$sriov_numvfs_path")" -ne 0 ]; then
12+
echo "$devpath already configured"
13+
exit 0
14+
fi
15+
16+
# enable sriov
17+
echo -n 1 > "$sriov_numvfs_path"
18+
19+
# configure vf
20+
# unbind vf
21+
vf_pciid=$(basename "$(realpath "$devpath/virtfn0")")
22+
dlb_pci_driver_path=/sys/bus/pci/drivers/dlb2
23+
echo -n "$vf_pciid" > $dlb_pci_driver_path/unbind
24+
25+
# assign resources to vf
26+
vf_resources_path="$devpath/vf0_resources"
27+
echo -n 2048 > "$vf_resources_path/num_atomic_inflights"
28+
echo -n 2048 > "$vf_resources_path/num_dir_credits"
29+
echo -n 8 > "$vf_resources_path/num_dir_ports"
30+
echo -n 2048 > "$vf_resources_path/num_hist_list_entries"
31+
echo -n 8192 > "$vf_resources_path/num_ldb_credits"
32+
echo -n 4 > "$vf_resources_path/num_ldb_ports"
33+
echo -n 32 > "$vf_resources_path/num_ldb_queues"
34+
echo -n 32 > "$vf_resources_path/num_sched_domains"
35+
echo -n 2 > "$vf_resources_path/num_sn0_slots"
36+
echo -n 2 > "$vf_resources_path/num_sn1_slots"
37+
# bind vf back to dlb2 driver
38+
echo -n "$vf_pciid" > $dlb_pci_driver_path/bind
39+
40+
echo "$devpath configured"
41+
# TODO: Due to unknown e2e-dlb (Simics) limitations, it is not possible to configure per VF resources based on values
42+
# reported in /sys/bus/pci/devices/<dev_name>/total_resources/<resource_name>. Therefore, only one VF with
43+
# values known to work is enabled. This will be improved in the future to make the scrip more meaningful for real-world
44+
# deployments (see #1145).
45+
}
46+
47+
# use first dlb device to configure a vf
48+
DEVPATH=$(realpath /sys/bus/pci/drivers/dlb2/????:??:??\.0 | head -1)
49+
enable_and_configure_vfs "$DEVPATH"

‎deployments/dlb_plugin/overlays/dlb_initcontainer/dlb_initcontainer.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: intel-dlb-plugin
5+
spec:
6+
template:
7+
spec:
8+
initContainers:
9+
- name: intel-dlb-initcontainer
10+
image: intel/intel-dlb-initcontainer:devel
11+
securityContext:
12+
readOnlyRootFilesystem: true
13+
privileged: true
14+
volumeMounts:
15+
- name: sysfs-driver-dlb2
16+
mountPath: /sys/bus/pci/drivers/dlb2
17+
- name: sysfs-devices
18+
mountPath: /sys/devices
19+
volumes:
20+
- name: sysfs-driver-dlb2
21+
hostPath:
22+
path: /sys/bus/pci/drivers/dlb2
23+
- name: sysfs-devices
24+
hostPath:
25+
path: /sys/devices

‎deployments/dlb_plugin/overlays/dlb_initcontainer/kustomization.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
bases:
2+
- ../../base
3+
patches:
4+
- path: dlb_initcontainer.yaml

‎deployments/operator/crd/bases/deviceplugin.intel.com_dlbdeviceplugins.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,10 @@ spec:
5353
image:
5454
description: Image is a container image with DLB device plugin executable.
5555
type: string
56+
initImage:
57+
description: InitImage is a container image with a script that initializes
58+
devices.
59+
type: string
5660
logLevel:
5761
description: LogLevel sets the plugin's log level.
5862
minimum: 0

‎deployments/operator/samples/deviceplugin_v1_dlbdeviceplugin.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ metadata:
1010
# container.apparmor.security.beta.kubernetes.io/intel-dlb-plugin: unconfined
1111
spec:
1212
image: intel/intel-dlb-plugin:0.24.0
13+
initImage: intel/intel-dlb-initcontainer:0.24.0
1314
logLevel: 4
1415
nodeSelector:
1516
intel.feature.node.kubernetes.io/dlb: 'true'

0 commit comments

Comments
 (0)