Skip to content

Instantly share code, notes, and snippets.

@jacob-delgado

jacob-delgado/kind-ovn-multus-istio.log

Forked from s1061123/kind-multus-test.log
Last active June 13, 2023 23:39
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save jacob-delgado/bb3a08d21e13130e6b685df75ec54a10 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save jacob-delgado/bb3a08d21e13130e6b685df75ec54a10 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. jacob-delgado revised this gist Jun 13, 2023. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions kind-ovn-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -464,7 +464,7 @@ EOF

    $ kubectl apply -f sleep-httpbin-nomultus-nosidecar.yaml -n nosidecar

    $ kubebctl get svc -A
    $ kubectl get svc -A

    NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
    default httpbin ClusterIP 10.96.92.191 <none> 8000/TCP 32m
    @@ -491,7 +491,7 @@ $ export SLEEP_DEFAULT_POD=$(kubectl get pod -l app=sleep -o jsonpath={.items..m
    $ export SLEEP_NOSIDECAR_POD=$(kubectl get pod -n nosidecar -l app=sleep -o jsonpath={.items..metadata.name})


    $ kubectl exec -it $SLEEP_MULTUS_DEFAAULT_POD -- curl http://10.1.1.11/get
    $ kubectl exec -it $SLEEP_MULTUS_DEFAULT_POD -- curl http://10.1.1.11/get
    {
    "args": {},
    "headers": {
  2. jacob-delgado revised this gist Jun 13, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion kind-ovn-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -9,7 +9,7 @@
    # - role: worker
    # EOF
    ## Create cluster
    # $ sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"
    # $ kind create cluster --config config-4node.yml

    # clone ovn
    $ git clone https://github.com/ovn-org/ovn-kubernetes
  3. jacob-delgado revised this gist Jun 6, 2023. 1 changed file with 64 additions and 0 deletions.
    64 changes: 64 additions & 0 deletions kind-ovn-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -463,3 +463,67 @@ spec:
    EOF

    $ kubectl apply -f sleep-httpbin-nomultus-nosidecar.yaml -n nosidecar

    $ kubebctl get svc -A

    NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
    default httpbin ClusterIP 10.96.92.191 <none> 8000/TCP 32m
    default kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 49m
    default sleep ClusterIP 10.96.164.183 <none> 80/TCP 29m
    default sleep-multus ClusterIP 10.96.63.157 <none> 80/TCP 30m
    istio-system istiod ClusterIP 10.96.136.7 <none> 15010/TCP,15012/TCP,443/TCP,15014/TCP 38m
    kube-system kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 49m
    nosidecar httpbin ClusterIP 10.96.71.12 <none> 8000/TCP 27m
    nosidecar sleep ClusterIP 10.96.168.196 <none> 80/TCP 27m
    ovn-kubernetes ovnkube-db ClusterIP None <none> 6641/TCP,6642/TCP 47m


    # multus network topology
    # default namespace | IP address | node
    # httpbin | 10.1.1.11 | ovn-worker
    # sleep-multus | 10.1.1.12 | ovn-worker2
    # sleep | 10.1.1.13 | ovn-worker2
    # nosidecar namespace
    # httpbin | 10.1.1.14 | ovn-worker

    $ export SLEEP_MULTUS_DEFAULT_POD=$(kubectl get pod -l app=sleep-multus -o jsonpath={.items..metadata.name})
    $ export SLEEP_DEFAULT_POD=$(kubectl get pod -l app=sleep -o jsonpath={.items..metadata.name})
    $ export SLEEP_NOSIDECAR_POD=$(kubectl get pod -n nosidecar -l app=sleep -o jsonpath={.items..metadata.name})


    $ kubectl exec -it $SLEEP_MULTUS_DEFAAULT_POD -- curl http://10.1.1.11/get
    {
    "args": {},
    "headers": {
    "Accept": "*/*",
    "Host": "10.1.1.11",
    "User-Agent": "curl/8.1.1-DEV",
    "X-B3-Parentspanid": "584b291281342c63",
    "X-B3-Sampled": "0",
    "X-B3-Spanid": "8919de5bdc0fc795",
    "X-B3-Traceid": "176d409680c7ba84584b291281342c63",
    "X-Envoy-Attempt-Count": "1"
    },
    "origin": "127.0.0.6",
    "url": "http://10.1.1.11/get"
    }

    $ kubectl exec -it $SLEEP_DEFAULT_POD -- curl http://10.1.1.11/get

    This should hang.


    kubectl exec -it $SLEEP_NOSIDECAR_POD -n nosidecar -- curl http://10.1.1.11/get 1 ↵
    {
    "args": {},
    "headers": {
    "Accept": "*/*",
    "Host": "10.1.1.11",
    "User-Agent": "curl/8.1.1-DEV",
    "X-B3-Sampled": "0",
    "X-B3-Spanid": "4d0eb020ac5c6ce4",
    "X-B3-Traceid": "a07130fe440fefd74d0eb020ac5c6ce4"
    },
    "origin": "127.0.0.6",
    "url": "http://10.1.1.11/get"
    }
  4. jacob-delgado renamed this gist Jun 5, 2023. 1 changed file with 47 additions and 35 deletions.
    82 changes: 47 additions & 35 deletions kind-multus-istio.log → kind-ovn-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -1,24 +1,30 @@
    ## Create 4-node environment config
    $ cat << EOF > config-4node.yml
    kind: Cluster
    apiVersion: kind.x-k8s.io/v1alpha4
    nodes:
    - role: control-plane
    - role: worker
    - role: worker
    - role: worker
    EOF

    #$ cat << EOF > config-4node.yml
    #kind: Cluster
    #apiVersion: kind.x-k8s.io/v1alpha4
    #nodes:
    # - role: control-plane
    # - role: worker
    # - role: worker
    # - role: worker
    # EOF
    ## Create cluster
    $ kind create cluster --config config-4node.yml --name multus-kind
    # $ sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"

    # clone ovn
    $ git clone https://github.com/ovn-org/ovn-kubernetes
    $ cd ovn-kubernetes
    $ cd contrib
    $ ./kind.sh -wk 3
    $ export KUBECONFIG=${HOME}/ovn.conf

    ## get nodes
    $ kubectl get nodes
    NAME STATUS ROLES AGE VERSION
    multus-kind-control-plane Ready control-plane 33s v1.27.1
    multus-kind-worker Ready <none> 8s v1.27.1
    multus-kind-worker2 Ready <none> 7s v1.27.1
    multus-kind-worker3 Ready <none> 11s v1.27.1
    NAME STATUS ROLES AGE VERSION
    ovn-control-plane Ready control-plane 6m41s v1.26.0
    ovn-worker Ready <none> 6m10s v1.26.0
    ovn-worker2 Ready <none> 6m10s v1.26.0
    ovn-worker3 Ready <none> 6m10s v1.26.0

    ## install multus
    $ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.0.2/deployments/multus-daemonset.yml
    @@ -27,8 +33,8 @@ $ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus
    $ curl -LO https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64
    $ chmod +x koko_0.83_linux_amd64

    ## Create veth interface between multus-kind-woker and multus-kind-worker2
    $ sudo ./koko_0.83_linux_amd64 -d multus-kind-worker,eth1 -d multus-kind-worker2,eth1
    ## Create veth interface between ovn-woker and ovn-worker2
    $ sudo ./koko_0.83_linux_amd64 -d ovn-worker,eth1 -d ovn-worker2,eth1

    # install cni reference plugins (kindnet doesn't install ipvlan or macvlan)
    $ cat << EOF > cni-install.yml
    @@ -154,7 +160,7 @@ $ helm install istio-cni istio/cni --namespace kube-system --wait --values overr
    $ helm install istiod istio/istiod -n istio-system --wait --values overrides.yml --version 1.17.2

    # istio-cni network-attachment-definition
    $ cat <<EOF | kubectl apply -f -
    $ cat <<EOF > istio-cni.yaml
    apiVersion: "k8s.cni.cncf.io/v1"
    kind: NetworkAttachmentDefinition
    metadata:
    @@ -163,12 +169,13 @@ spec:
    config: ''
    EOF

    $ kubectl label namespace default istio-injection=enabled --overwrite
    $ kubectl apply -f istio-cni.yaml

    $ kubectl label namespace default istio-injection=enabled --overwrite

    # httpbin-multus
    # apply httpbin w/nodeSelector = multus-kind-worker
    $ cat <<EOF | kubectl apply -f -
    # apply httpbin w/nodeSelector = ovn-worker
    $ cat <<EOF > httpbin-multus-default.yaml
    apiVersion: v1
    kind: ServiceAccount
    metadata:
    @@ -211,7 +218,7 @@ spec:
    version: v1
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker
    kubernetes.io/hostname: ovn-worker
    serviceAccountName: httpbin
    containers:
    - image: docker.io/kennethreitz/httpbin
    @@ -221,9 +228,12 @@ spec:
    - containerPort: 80
    EOF

    $ kubectl apply -f httpbin-multus-default.yaml


    # sleep-multus
    # apply sleep w/nodeSelector = multus-kind-worker2
    $ cat <<EOF | kubectl apply -f -
    # apply sleep w/nodeSelector = ovn-worker2
    $ cat <<EOF > sleep-multus-default.yaml
    apiVersion: v1
    kind: ServiceAccount
    metadata:
    @@ -263,7 +273,7 @@ spec:
    ]'
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker2
    kubernetes.io/hostname: ovn-worker2
    terminationGracePeriodSeconds: 0
    serviceAccountName: sleep-multus
    containers:
    @@ -281,9 +291,11 @@ spec:
    optional: true
    EOF

    $ kubectl apply -f sleep-multus-default.yaml

    # sleep
    # not a part of the multus macvlan-conf network
    $ cat <<EOF | kubectl apply -f -
    $ cat <<EOF > sleep-nomultus-default.yaml
    apiVersion: v1
    kind: ServiceAccount
    metadata:
    @@ -334,10 +346,12 @@ spec:
    optional: true
    EOF

    $ kubectl apply -f sleep-nomultus-default.yaml

    $ kubectl create ns nosidecar

    # sleep in nosidecar namespace on macvlan-conf multus network
    $ cat <<EOF | kubectl apply -n nosidecar -f -
    $ cat <<EOF > sleep-httpbin-nomultus-nosidecar.yaml
    apiVersion: v1
    kind: ServiceAccount
    metadata:
    @@ -378,7 +392,7 @@ spec:
    ]'
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker2
    kubernetes.io/hostname: ovn-worker2
    terminationGracePeriodSeconds: 0
    serviceAccountName: sleep
    containers:
    @@ -394,11 +408,7 @@ spec:
    secret:
    secretName: sleep-secret
    optional: true
    EOF

    # httpbin-multus
    # apply httpbin w/nodeSelector = multus-kind-worker to nodesidecar namespace
    $ cat <<EOF | kubectl apply -n nosidecar -f -
    ---
    apiVersion: v1
    kind: ServiceAccount
    metadata:
    @@ -442,7 +452,7 @@ spec:
    version: v1
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker
    kubernetes.io/hostname: ovn-worker
    serviceAccountName: httpbin
    containers:
    - image: docker.io/kennethreitz/httpbin
    @@ -451,3 +461,5 @@ spec:
    ports:
    - containerPort: 80
    EOF

    $ kubectl apply -f sleep-httpbin-nomultus-nosidecar.yaml -n nosidecar
  5. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 4 additions and 4 deletions.
    8 changes: 4 additions & 4 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -204,7 +204,7 @@ spec:
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ips": ["10.1.1.11/32"] }
    "ips": ["10.1.1.11/24"] }
    ]'
    labels:
    app: httpbin
    @@ -259,7 +259,7 @@ spec:
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ips": ["10.1.1.12/32"] }
    "ips": ["10.1.1.12/24"] }
    ]'
    spec:
    nodeSelector:
    @@ -374,7 +374,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": ["10.1.1.13/32"] }
    "ips": ["10.1.1.13/24"] }
    ]'
    spec:
    nodeSelector:
    @@ -435,7 +435,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": ["10.1.1.14/32"] }
    "ips": ["10.1.1.14/24"] }
    ]'
    labels:
    app: httpbin
  6. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -374,7 +374,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": ["10.1.1.14/32"] }
    "ips": ["10.1.1.13/32"] }
    ]'
    spec:
    nodeSelector:
  7. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -374,7 +374,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": ["10.1.1.13/32"] ] }
    "ips": ["10.1.1.14/32"] }
    ]'
    spec:
    nodeSelector:
  8. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -374,7 +374,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": ["10.1.1.13/32]" ] }
    "ips": ["10.1.1.13/32"] ] }
    ]'
    spec:
    nodeSelector:
    @@ -435,7 +435,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": ["10.1.1.14/32]" }
    "ips": ["10.1.1.14/32"] }
    ]'
    labels:
    app: httpbin
  9. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 4 additions and 4 deletions.
    8 changes: 4 additions & 4 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -204,7 +204,7 @@ spec:
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ip": "10.1.1.11" }
    "ips": ["10.1.1.11/32"] }
    ]'
    labels:
    app: httpbin
    @@ -259,7 +259,7 @@ spec:
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ip": "10.1.1.12" }
    "ips": ["10.1.1.12/32"] }
    ]'
    spec:
    nodeSelector:
    @@ -374,7 +374,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ip": "10.1.1.13" ] }
    "ips": ["10.1.1.13/32]" ] }
    ]'
    spec:
    nodeSelector:
    @@ -435,7 +435,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ip": "10.1.1.14" }
    "ips": ["10.1.1.14/32]" }
    ]'
    labels:
    app: httpbin
  10. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 4 additions and 4 deletions.
    8 changes: 4 additions & 4 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -204,7 +204,7 @@ spec:
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ips": [ "10.1.1.11/24" ] }
    "ip": "10.1.1.11" }
    ]'
    labels:
    app: httpbin
    @@ -259,7 +259,7 @@ spec:
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ips": [ "10.1.1.12/24" ] }
    "ip": "10.1.1.12" }
    ]'
    spec:
    nodeSelector:
    @@ -374,7 +374,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": [ "10.1.1.12/24" ] }
    "ip": "10.1.1.13" ] }
    ]'
    spec:
    nodeSelector:
    @@ -435,7 +435,7 @@ spec:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": [ "10.1.1.12/24" ] }
    "ip": "10.1.1.14" }
    ]'
    labels:
    app: httpbin
  11. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 4 additions and 2 deletions.
    6 changes: 4 additions & 2 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -372,7 +372,8 @@ spec:
    app: sleep
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "default/macvlan-conf",
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": [ "10.1.1.12/24" ] }
    ]'
    spec:
    @@ -432,7 +433,8 @@ spec:
    metadata:
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "default/macvlan-conf",
    { "name": "macvlan-conf",
    "namespace": "default",
    "ips": [ "10.1.1.12/24" ] }
    ]'
    labels:
  12. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -372,7 +372,7 @@ spec:
    app: sleep
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    { "name": "default/macvlan-conf",
    "ips": [ "10.1.1.12/24" ] }
    ]'
    spec:
    @@ -432,7 +432,7 @@ spec:
    metadata:
    annotations:
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    { "name": "default/macvlan-conf",
    "ips": [ "10.1.1.12/24" ] }
    ]'
    labels:
  13. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -117,7 +117,7 @@ spec:
    "master": "eth1",
    "mode": "bridge",
    "ipam": {
    "type": "static",
    "type": "static"
    }
    }, {
    "type": "tuning"
  14. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 17 additions and 12 deletions.
    29 changes: 17 additions & 12 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -117,14 +117,7 @@ spec:
    "master": "eth1",
    "mode": "bridge",
    "ipam": {
    "type": "host-local",
    "subnet": "172.17.9.0/24",
    "rangeStart": "172.17.9.240",
    "rangeEnd": "172.17.9.250",
    "routes": [
    { "dst": "0.0.0.0/0" }
    ],
    "gateway": "172.17.9.1"
    "type": "static",
    }
    }, {
    "type": "tuning"
    @@ -209,7 +202,10 @@ spec:
    template:
    metadata:
    annotations:
    k8s.v1.cni.cncf.io/networks: macvlan-conf
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ips": [ "10.1.1.11/24" ] }
    ]'
    labels:
    app: httpbin
    version: v1
    @@ -261,7 +257,10 @@ spec:
    labels:
    app: sleep-multus
    annotations:
    k8s.v1.cni.cncf.io/networks: macvlan-conf
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ips": [ "10.1.1.12/24" ] }
    ]'
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker2
    @@ -372,7 +371,10 @@ spec:
    labels:
    app: sleep
    annotations:
    k8s.v1.cni.cncf.io/networks: default/macvlan-conf
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ips": [ "10.1.1.12/24" ] }
    ]'
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker2
    @@ -429,7 +431,10 @@ spec:
    template:
    metadata:
    annotations:
    k8s.v1.cni.cncf.io/networks: macvlan-conf
    k8s.v1.cni.cncf.io/networks: '[
    { "name": "macvlan-conf",
    "ips": [ "10.1.1.12/24" ] }
    ]'
    labels:
    app: httpbin
    version: v1
  15. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 51 additions and 0 deletions.
    51 changes: 51 additions & 0 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -393,3 +393,54 @@ spec:
    optional: true
    EOF

    # httpbin-multus
    # apply httpbin w/nodeSelector = multus-kind-worker to nodesidecar namespace
    $ cat <<EOF | kubectl apply -n nosidecar -f -
    apiVersion: v1
    kind: ServiceAccount
    metadata:
    name: httpbin
    ---
    apiVersion: v1
    kind: Service
    metadata:
    name: httpbin
    labels:
    app: httpbin
    service: httpbin
    spec:
    ports:
    - name: http
    port: 8000
    targetPort: 80
    selector:
    app: httpbin
    ---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
    name: httpbin
    spec:
    replicas: 1
    selector:
    matchLabels:
    app: httpbin
    version: v1
    template:
    metadata:
    annotations:
    k8s.v1.cni.cncf.io/networks: macvlan-conf
    labels:
    app: httpbin
    version: v1
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker
    serviceAccountName: httpbin
    containers:
    - image: docker.io/kennethreitz/httpbin
    imagePullPolicy: IfNotPresent
    name: httpbin
    ports:
    - containerPort: 80
    EOF
  16. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -372,7 +372,7 @@ spec:
    labels:
    app: sleep
    annotations:
    k8s.v1.cni.cncf.io/networks: macvlan-conf
    k8s.v1.cni.cncf.io/networks: default/macvlan-conf
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker2
  17. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -338,7 +338,7 @@ EOF
    $ kubectl create ns nosidecar

    # sleep in nosidecar namespace on macvlan-conf multus network
    $ cat <<EOF | kubectl apply -f -
    $ cat <<EOF | kubectl apply -n nosidecar -f -
    apiVersion: v1
    kind: ServiceAccount
    metadata:
  18. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 59 additions and 0 deletions.
    59 changes: 59 additions & 0 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -334,3 +334,62 @@ spec:
    secretName: sleep-secret
    optional: true
    EOF

    $ kubectl create ns nosidecar

    # sleep in nosidecar namespace on macvlan-conf multus network
    $ cat <<EOF | kubectl apply -f -
    apiVersion: v1
    kind: ServiceAccount
    metadata:
    name: sleep
    ---
    apiVersion: v1
    kind: Service
    metadata:
    name: sleep
    labels:
    app: sleep
    service: sleep
    spec:
    ports:
    - port: 80
    name: http
    selector:
    app: sleep
    ---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
    name: sleep
    spec:
    replicas: 1
    selector:
    matchLabels:
    app: sleep
    template:
    metadata:
    labels:
    app: sleep
    annotations:
    k8s.v1.cni.cncf.io/networks: macvlan-conf
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker2
    terminationGracePeriodSeconds: 0
    serviceAccountName: sleep
    containers:
    - name: sleep
    image: curlimages/curl
    command: ["/bin/sleep", "infinity"]
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - mountPath: /etc/sleep/tls
    name: secret-volume
    volumes:
    - name: secret-volume
    secret:
    secretName: sleep-secret
    optional: true
    EOF

  19. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 8 additions and 2 deletions.
    10 changes: 8 additions & 2 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -103,7 +103,6 @@ $ kubectl apply -f cni-install.yml

    ## create macvlan
    $ cat << EOF > macvlan.yml
    ---
    apiVersion: "k8s.cni.cncf.io/v1"
    kind: NetworkAttachmentDefinition
    metadata:
    @@ -118,7 +117,14 @@ spec:
    "master": "eth1",
    "mode": "bridge",
    "ipam": {
    "type": "static"
    "type": "host-local",
    "subnet": "172.17.9.0/24",
    "rangeStart": "172.17.9.240",
    "rangeEnd": "172.17.9.250",
    "routes": [
    { "dst": "0.0.0.0/0" }
    ],
    "gateway": "172.17.9.1"
    }
    }, {
    "type": "tuning"
  20. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 67 additions and 10 deletions.
    77 changes: 67 additions & 10 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -10,14 +10,15 @@ nodes:
    EOF

    ## Create cluster
    $ kind create cluster --config config-3node.yml --name multus-kind
    $ kind create cluster --config config-4node.yml --name multus-kind

    ## get nodes
    $ kubectl get nodes
    NAME STATUS ROLES AGE VERSION
    multus-kind-control-plane Ready control-plane 27s v1.27.1
    multus-kind-worker NotReady <none> 2s v1.27.1
    multus-kind-worker2 NotReady <none> 3s v1.27.1
    NAME STATUS ROLES AGE VERSION
    multus-kind-control-plane Ready control-plane 33s v1.27.1
    multus-kind-worker Ready <none> 8s v1.27.1
    multus-kind-worker2 Ready <none> 7s v1.27.1
    multus-kind-worker3 Ready <none> 11s v1.27.1

    ## install multus
    $ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.0.2/deployments/multus-daemonset.yml
    @@ -138,6 +139,7 @@ istio_cni:
    chained: false

    cni:
    enabled: true
    chained: false
    cniBinDir: /opt/cni/bin
    cniConfDir: /etc/cni/multus/net.d
    @@ -166,7 +168,7 @@ $ kubectl label namespace default istio-injection=enabled --overwrite


    # httpbin-multus
    # apply httpbin w/nodeSelector = kind-worker
    # apply httpbin w/nodeSelector = multus-kind-worker
    $ cat <<EOF | kubectl apply -f -
    apiVersion: v1
    kind: ServiceAccount
    @@ -207,7 +209,7 @@ spec:
    version: v1
    spec:
    nodeSelector:
    kubernetes.io/hostname: kind-worker
    kubernetes.io/hostname: multus-kind-worker
    serviceAccountName: httpbin
    containers:
    - image: docker.io/kennethreitz/httpbin
    @@ -218,7 +220,64 @@ spec:
    EOF

    # sleep-multus
    # apply sleep w/nodeSelector = kind-worker2
    # apply sleep w/nodeSelector = multus-kind-worker2
    $ cat <<EOF | kubectl apply -f -
    apiVersion: v1
    kind: ServiceAccount
    metadata:
    name: sleep-multus
    ---
    apiVersion: v1
    kind: Service
    metadata:
    name: sleep-multus
    labels:
    app: sleep-multus
    service: sleep-multus
    spec:
    ports:
    - port: 80
    name: http
    selector:
    app: sleep-multus
    ---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
    name: sleep-multus
    spec:
    replicas: 1
    selector:
    matchLabels:
    app: sleep-multus
    template:
    metadata:
    labels:
    app: sleep-multus
    annotations:
    k8s.v1.cni.cncf.io/networks: macvlan-conf
    spec:
    nodeSelector:
    kubernetes.io/hostname: multus-kind-worker2
    terminationGracePeriodSeconds: 0
    serviceAccountName: sleep-multus
    containers:
    - name: sleep-multus
    image: curlimages/curl
    command: ["/bin/sleep", "infinity"]
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - mountPath: /etc/sleep/tls
    name: secret-volume
    volumes:
    - name: secret-volume
    secret:
    secretName: sleep-multus-secret
    optional: true
    EOF

    # sleep
    # not a part of the multus macvlan-conf network
    $ cat <<EOF | kubectl apply -f -
    apiVersion: v1
    kind: ServiceAccount
    @@ -253,8 +312,6 @@ spec:
    labels:
    app: sleep
    spec:
    nodeSelector:
    kubernetes.io/hostname: kind-worker2
    terminationGracePeriodSeconds: 0
    serviceAccountName: sleep
    containers:
  21. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 3 additions and 2 deletions.
    5 changes: 3 additions & 2 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -1,11 +1,12 @@
    ## Create 3-node environment config
    $ cat << EOF > config-3node.yml
    ## Create 4-node environment config
    $ cat << EOF > config-4node.yml
    kind: Cluster
    apiVersion: kind.x-k8s.io/v1alpha4
    nodes:
    - role: control-plane
    - role: worker
    - role: worker
    - role: worker
    EOF

    ## Create cluster
  22. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 0 additions and 7 deletions.
    7 changes: 0 additions & 7 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -145,13 +145,6 @@ cni:
    excludeNamespaces:
    - istio-system
    - kube-system

    chained: false
    privileged: true

    sidecarInjectorWebhook:
    injectedAnnotations:
    k8s.v1.cni.cncf.io/networks: istio-cni
    EOF

    $ helm install istio-base istio/base -n istio-system --version 1.17.2
  23. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 59 additions and 1 deletion.
    60 changes: 59 additions & 1 deletion kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -172,6 +172,7 @@ $ kubectl label namespace default istio-injection=enabled --overwrite


    # httpbin-multus
    # apply httpbin w/nodeSelector = kind-worker
    $ cat <<EOF | kubectl apply -f -
    apiVersion: v1
    kind: ServiceAccount
    @@ -211,11 +212,68 @@ spec:
    app: httpbin
    version: v1
    spec:
    nodeSelector:
    kubernetes.io/hostname: kind-worker
    serviceAccountName: httpbin
    containers:
    - image: docker.io/kennethreitz/httpbin
    imagePullPolicy: IfNotPresent
    name: httpbin
    ports:
    - containerPort: 80
    EOF
    EOF

    # sleep-multus
    # apply sleep w/nodeSelector = kind-worker2
    $ cat <<EOF | kubectl apply -f -
    apiVersion: v1
    kind: ServiceAccount
    metadata:
    name: sleep
    ---
    apiVersion: v1
    kind: Service
    metadata:
    name: sleep
    labels:
    app: sleep
    service: sleep
    spec:
    ports:
    - port: 80
    name: http
    selector:
    app: sleep
    ---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
    name: sleep
    spec:
    replicas: 1
    selector:
    matchLabels:
    app: sleep
    template:
    metadata:
    labels:
    app: sleep
    spec:
    nodeSelector:
    kubernetes.io/hostname: kind-worker2
    terminationGracePeriodSeconds: 0
    serviceAccountName: sleep
    containers:
    - name: sleep
    image: curlimages/curl
    command: ["/bin/sleep", "infinity"]
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - mountPath: /etc/sleep/tls
    name: secret-volume
    volumes:
    - name: secret-volume
    secret:
    secretName: sleep-secret
    optional: true
    EOF
  24. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 16 additions and 9 deletions.
    25 changes: 16 additions & 9 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -108,15 +108,20 @@ metadata:
    name: macvlan-conf
    spec:
    config: '{
    "cniVersion": "0.3.1",
    "name": "macvlan-main",
    "type": "macvlan",
    "mode": "bridge",
    "master": "eth1",
    "ipam": {
    "type": "static"
    }
    }'
    "cniVersion": "0.3.1",
    "plugins": [
    {
    "type": "macvlan",
    "capabilities": { "ips": true },
    "master": "eth1",
    "mode": "bridge",
    "ipam": {
    "type": "static"
    }
    }, {
    "type": "tuning"
    } ]
    }'
    EOF

    $ kubectl apply -f macvlan.yml
    @@ -129,8 +134,10 @@ $ kubectl create namespace istio-system
    $ cat << EOF > overrides.yml
    istio_cni:
    enabled: true
    chained: false

    cni:
    chained: false
    cniBinDir: /opt/cni/bin
    cniConfDir: /etc/cni/multus/net.d
    cniConfFileName: istio-cni.conf
  25. jacob-delgado revised this gist Jun 1, 2023. 1 changed file with 4 additions and 4 deletions.
    8 changes: 4 additions & 4 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -14,12 +14,12 @@ $ kind create cluster --config config-3node.yml --name multus-kind
    ## get nodes
    $ kubectl get nodes
    NAME STATUS ROLES AGE VERSION
    multus-kind-control-plane Ready control-plane 34s v1.26.3
    multus-kind-worker NotReady <none> 2s v1.26.3
    multus-kind-worker2 NotReady <none> 3s v1.26.3
    multus-kind-control-plane Ready control-plane 27s v1.27.1
    multus-kind-worker NotReady <none> 2s v1.27.1
    multus-kind-worker2 NotReady <none> 3s v1.27.1

    ## install multus
    $ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.0.1/deployments/multus-daemonset.yml
    $ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.0.2/deployments/multus-daemonset.yml

    ## get koko
    $ curl -LO https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64
  26. jacob-delgado revised this gist May 31, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -105,7 +105,7 @@ $ cat << EOF > macvlan.yml
    apiVersion: "k8s.cni.cncf.io/v1"
    kind: NetworkAttachmentDefinition
    metadata:
    name: macvlan
    name: macvlan-conf
    spec:
    config: '{
    "cniVersion": "0.3.1",
  27. jacob-delgado revised this gist May 11, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -131,7 +131,7 @@ istio_cni:
    enabled: true

    cni:
    cniBinDir: /var/lib/cni/bin
    cniBinDir: /opt/cni/bin
    cniConfDir: /etc/cni/multus/net.d
    cniConfFileName: istio-cni.conf

  28. jacob-delgado revised this gist May 11, 2023. 1 changed file with 3 additions and 0 deletions.
    3 changes: 3 additions & 0 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -127,6 +127,9 @@ $ helm repo update
    $ kubectl create namespace istio-system

    $ cat << EOF > overrides.yml
    istio_cni:
    enabled: true

    cni:
    cniBinDir: /var/lib/cni/bin
    cniConfDir: /etc/cni/multus/net.d
  29. jacob-delgado revised this gist May 11, 2023. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -19,7 +19,7 @@ multus-kind-worker NotReady <none> 2s v1.26.3
    multus-kind-worker2 NotReady <none> 3s v1.26.3

    ## install multus
    $ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.0.1/deployments/multus-daemonset-thick.yml
    $ kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/v4.0.1/deployments/multus-daemonset.yml

    ## get koko
    $ curl -LO https://github.com/redhat-nfvpe/koko/releases/download/v0.83/koko_0.83_linux_amd64
  30. jacob-delgado revised this gist May 11, 2023. 1 changed file with 71 additions and 0 deletions.
    71 changes: 71 additions & 0 deletions kind-multus-istio.log
    View file
    Original file line number Diff line number Diff line change
    @@ -28,6 +28,77 @@ $ chmod +x koko_0.83_linux_amd64
    ## Create veth interface between multus-kind-woker and multus-kind-worker2
    $ sudo ./koko_0.83_linux_amd64 -d multus-kind-worker,eth1 -d multus-kind-worker2,eth1

    # install cni reference plugins (kindnet doesn't install ipvlan or macvlan)
    $ cat << EOF > cni-install.yml
    ---
    kind: ConfigMap
    apiVersion: v1
    metadata:
    name: cni-install-sh
    namespace: kube-system
    data:
    install_cni.sh: |
    cd /tmp
    wget https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz
    cd /host/opt/cni/bin
    tar xvfzp /tmp/cni-plugins-linux-amd64-v1.3.0.tgz
    sleep infinite
    ---
    apiVersion: apps/v1
    kind: DaemonSet
    metadata:
    name: install-cni-plugins
    namespace: kube-system
    labels:
    name: cni-plugins
    spec:
    selector:
    matchLabels:
    name: cni-plugins
    template:
    metadata:
    labels:
    name: cni-plugins
    spec:
    hostNetwork: true
    nodeSelector:
    kubernetes.io/arch: amd64
    tolerations:
    - operator: Exists
    effect: NoSchedule
    containers:
    - name: install-cni-plugins
    image: alpine
    command: ["/bin/sh", "/scripts/install_cni.sh"]
    resources:
    requests:
    cpu: "100m"
    memory: "50Mi"
    limits:
    cpu: "100m"
    memory: "50Mi"
    securityContext:
    privileged: true
    volumeMounts:
    - name: cni-bin
    mountPath: /host/opt/cni/bin
    - name: scripts
    mountPath: /scripts
    volumes:
    - name: cni-bin
    hostPath:
    path: /opt/cni/bin
    - name: scripts
    configMap:
    name: cni-install-sh
    items:
    - key: install_cni.sh
    path: install_cni.sh
    EOF

    $ kubectl apply -f cni-install.yml


    ## create macvlan
    $ cat << EOF > macvlan.yml
    ---