Skip to content

Instantly share code, notes, and snippets.

@jacob-delgado

jacob-delgado/kind-ovn-multus-istio.log

Forked from s1061123/kind-multus-test.log
Last active June 13, 2023 23:39
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save jacob-delgado/bb3a08d21e13130e6b685df75ec54a10 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save jacob-delgado/bb3a08d21e13130e6b685df75ec54a10 to your computer and use it in GitHub Desktop.
Download ZIP
Kind with Multus log
Raw
kind-multus-istio.log
## Create 4-node environment config
$ cat << EOF > config-4node.yml
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
- role: worker
- role: worker
- role: worker
EOF
## Create cluster
$ kind create cluster --config config-4node.yml --name multus-kind
## get nodes
$ kubectl get nodes
NAME STATUS ROLES AGE VERSION
multus-kind-control-plane Ready control-plane 33s v1.27.1
multus-kind-worker Ready <none> 8s v1.27.1
multus-kind-worker2 Ready <none> 7s v1.27.1
multus-kind-worker3 Ready <none> 11s v1.27.1
## install multus
$ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.0.2/deployments/multus-daemonset.yml
## get koko
$ curl -LO https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64
$ chmod +x koko_0.83_linux_amd64
## Create veth interface between multus-kind-woker and multus-kind-worker2
$ sudo ./koko_0.83_linux_amd64 -d multus-kind-worker,eth1 -d multus-kind-worker2,eth1
# install cni reference plugins (kindnet doesn't install ipvlan or macvlan)
$ cat << EOF > cni-install.yml
---
kind: ConfigMap
apiVersion: v1
metadata:
name: cni-install-sh
namespace: kube-system
data:
install_cni.sh: |
cd /tmp
wget https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz
cd /host/opt/cni/bin
tar xvfzp /tmp/cni-plugins-linux-amd64-v1.3.0.tgz
sleep infinite
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: install-cni-plugins
namespace: kube-system
labels:
name: cni-plugins
spec:
selector:
matchLabels:
name: cni-plugins
template:
metadata:
labels:
name: cni-plugins
spec:
hostNetwork: true
nodeSelector:
kubernetes.io/arch: amd64
tolerations:
- operator: Exists
effect: NoSchedule
containers:
- name: install-cni-plugins
image: alpine
command: ["/bin/sh", "/scripts/install_cni.sh"]
resources:
requests:
cpu: "100m"
memory: "50Mi"
limits:
cpu: "100m"
memory: "50Mi"
securityContext:
privileged: true
volumeMounts:
- name: cni-bin
mountPath: /host/opt/cni/bin
- name: scripts
mountPath: /scripts
volumes:
- name: cni-bin
hostPath:
path: /opt/cni/bin
- name: scripts
configMap:
name: cni-install-sh
items:
- key: install_cni.sh
path: install_cni.sh
EOF
$ kubectl apply -f cni-install.yml
## create macvlan
$ cat << EOF > macvlan.yml
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
name: macvlan-conf
spec:
config: '{
"cniVersion": "0.3.1",
"plugins": [
{
"type": "macvlan",
"capabilities": { "ips": true },
"master": "eth1",
"mode": "bridge",
"ipam": {
"type": "static"
}
}, {
"type": "tuning"
} ]
}'
EOF
$ kubectl apply -f macvlan.yml
# install istio
$ helm repo add istio https://istio-release.storage.googleapis.com/charts
$ helm repo update
$ kubectl create namespace istio-system
$ cat << EOF > overrides.yml
istio_cni:
enabled: true
chained: false
cni:
enabled: true
chained: false
cniBinDir: /opt/cni/bin
cniConfDir: /etc/cni/multus/net.d
cniConfFileName: istio-cni.conf
excludeNamespaces:
- istio-system
- kube-system
EOF
$ helm install istio-base istio/base -n istio-system --version 1.17.2
$ helm install istio-cni istio/cni --namespace kube-system --wait --values overrides.yml --version 1.17.2
$ helm install istiod istio/istiod -n istio-system --wait --values overrides.yml --version 1.17.2
# istio-cni network-attachment-definition
$ cat <<EOF | kubectl apply -f -
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
name: istio-cni
spec:
config: ''
EOF
$ kubectl label namespace default istio-injection=enabled --overwrite
# httpbin-multus
# apply httpbin w/nodeSelector = multus-kind-worker
$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ServiceAccount
metadata:
name: httpbin
---
apiVersion: v1
kind: Service
metadata:
name: httpbin
labels:
app: httpbin
service: httpbin
spec:
ports:
- name: http
port: 8000
targetPort: 80
selector:
app: httpbin
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: httpbin
spec:
replicas: 1
selector:
matchLabels:
app: httpbin
version: v1
template:
metadata:
annotations:
k8s.v1.cni.cncf.io/networks: '[
{ "name": "macvlan-conf",
"ips": ["10.1.1.11/24"] }
]'
labels:
app: httpbin
version: v1
spec:
nodeSelector:
kubernetes.io/hostname: multus-kind-worker
serviceAccountName: httpbin
containers:
- image: docker.io/kennethreitz/httpbin
imagePullPolicy: IfNotPresent
name: httpbin
ports:
- containerPort: 80
EOF
# sleep-multus
# apply sleep w/nodeSelector = multus-kind-worker2
$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ServiceAccount
metadata:
name: sleep-multus
---
apiVersion: v1
kind: Service
metadata:
name: sleep-multus
labels:
app: sleep-multus
service: sleep-multus
spec:
ports:
- port: 80
name: http
selector:
app: sleep-multus
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: sleep-multus
spec:
replicas: 1
selector:
matchLabels:
app: sleep-multus
template:
metadata:
labels:
app: sleep-multus
annotations:
k8s.v1.cni.cncf.io/networks: '[
{ "name": "macvlan-conf",
"ips": ["10.1.1.12/24"] }
]'
spec:
nodeSelector:
kubernetes.io/hostname: multus-kind-worker2
terminationGracePeriodSeconds: 0
serviceAccountName: sleep-multus
containers:
- name: sleep-multus
image: curlimages/curl
command: ["/bin/sleep", "infinity"]
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /etc/sleep/tls
name: secret-volume
volumes:
- name: secret-volume
secret:
secretName: sleep-multus-secret
optional: true
EOF
# sleep
# not a part of the multus macvlan-conf network
$ cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ServiceAccount
metadata:
name: sleep
---
apiVersion: v1
kind: Service
metadata:
name: sleep
labels:
app: sleep
service: sleep
spec:
ports:
- port: 80
name: http
selector:
app: sleep
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: sleep
spec:
replicas: 1
selector:
matchLabels:
app: sleep
template:
metadata:
labels:
app: sleep
spec:
terminationGracePeriodSeconds: 0
serviceAccountName: sleep
containers:
- name: sleep
image: curlimages/curl
command: ["/bin/sleep", "infinity"]
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /etc/sleep/tls
name: secret-volume
volumes:
- name: secret-volume
secret:
secretName: sleep-secret
optional: true
EOF
$ kubectl create ns nosidecar
# sleep in nosidecar namespace on macvlan-conf multus network
$ cat <<EOF | kubectl apply -n nosidecar -f -
apiVersion: v1
kind: ServiceAccount
metadata:
name: sleep
---
apiVersion: v1
kind: Service
metadata:
name: sleep
labels:
app: sleep
service: sleep
spec:
ports:
- port: 80
name: http
selector:
app: sleep
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: sleep
spec:
replicas: 1
selector:
matchLabels:
app: sleep
template:
metadata:
labels:
app: sleep
annotations:
k8s.v1.cni.cncf.io/networks: '[
{ "name": "macvlan-conf",
"namespace": "default",
"ips": ["10.1.1.13/24"] }
]'
spec:
nodeSelector:
kubernetes.io/hostname: multus-kind-worker2
terminationGracePeriodSeconds: 0
serviceAccountName: sleep
containers:
- name: sleep
image: curlimages/curl
command: ["/bin/sleep", "infinity"]
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /etc/sleep/tls
name: secret-volume
volumes:
- name: secret-volume
secret:
secretName: sleep-secret
optional: true
EOF
# httpbin-multus
# apply httpbin w/nodeSelector = multus-kind-worker to nodesidecar namespace
$ cat <<EOF | kubectl apply -n nosidecar -f -
apiVersion: v1
kind: ServiceAccount
metadata:
name: httpbin
---
apiVersion: v1
kind: Service
metadata:
name: httpbin
labels:
app: httpbin
service: httpbin
spec:
ports:
- name: http
port: 8000
targetPort: 80
selector:
app: httpbin
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: httpbin
spec:
replicas: 1
selector:
matchLabels:
app: httpbin
version: v1
template:
metadata:
annotations:
k8s.v1.cni.cncf.io/networks: '[
{ "name": "macvlan-conf",
"namespace": "default",
"ips": ["10.1.1.14/24"] }
]'
labels:
app: httpbin
version: v1
spec:
nodeSelector:
kubernetes.io/hostname: multus-kind-worker
serviceAccountName: httpbin
containers:
- image: docker.io/kennethreitz/httpbin
imagePullPolicy: IfNotPresent
name: httpbin
ports:
- containerPort: 80
EOF
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment