Skip to content

Instantly share code, notes, and snippets.

@zhangguanzhang

zhangguanzhang/flannel-compile

Last active July 22, 2021 04:13
Show Gist options
  • Save zhangguanzhang/1c3382f965107d072d68be6e4341b275 to your computer and use it in GitHub Desktop.
Save zhangguanzhang/1c3382f965107d072d68be6e4341b275 to your computer and use it in GitHub Desktop.
Download ZIP
flannel-compile
Raw
flannel-compile
```
wget https://github.com/multiarch/qemu-user-static/releases/download/v3.0.0/qemu-x86_64-static.tar.gz
docker run --privileged -ti --rm --name flannel --net host -e GOARCH=amd64 \
--cap-add=NET_ADMIN --cap-add=SYS_ADMIN \
-u $( id -u):$( id -g) \
-v $PWD/dist/qemu-amd64-static:/usr/bin/qemu-amd64-static \
-v $PWD:/go/src/github.com/coreos/flannel:ro \
-v $PWD/dist:/go/src/github.com/coreos/flannel/dist \
-v /run:/run \
--workdir=/go/src/github.com/coreos/flannel \
golang:1.16.5 /bin/bash
go install github.com/go-delve/delve/cmd/dlv@latest
CGO_ENABLED=1 GO111MODULE=off go build -o dist/flanneld \
-gcflags="all=-trimpath=$PWD -N -l" \
-asmflags "all=-trimpath=$PWD" \
-ldflags '-X github.com/coreos/flannel/version.Version=v0.11.0-go1.16.3 -extldflags "-static"' main.go
mkdir -p /etc/cni/net.d/
cat <<EOF > /etc/cni/net.d/10-flannel.conflist
{
"name": "cbr0",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
EOF
mkdir -p /etc/kube-flannel
cat <<EOF > /etc/kube-flannel/net-conf.json
{
"Network": "10.187.0.0/16",
"Backend": {
"Type": "vxlan"
}
}
EOF
export NODE_NAME=10.13.4.78
dlv exec ./dist/flanneld -- --kube-subnet-mgr --kubeconfig-file=./dist/kubeconfig --ip-masq -v=4
```
@zhangguanzhang
Copy link
Author

zhangguanzhang commented Jul 22, 2021 


10.13.4.78

susesp3:~ # kubectl get node
NAME          STATUS   ROLES         AGE   VERSION
10.13.4.135   Ready    master,node   16h   v1.15.5
10.13.4.136   Ready    master,node   16h   v1.15.5
10.13.4.137   Ready    master,node   16h   v1.15.5
10.13.4.138   Ready    node          16h   v1.15.5
10.13.4.150   Ready    node          16h   v1.15.5
10.13.4.151   Ready    node          16h   v1.15.5
10.13.4.152   Ready    node          35m   v1.15.5
10.13.4.153   Ready    node          35m   v1.15.5
10.13.4.78    Ready    node          35m   v1.15.5
susesp3:~ # kubectl -n kube-system get pod -o wide -l k8s-app=kube-dns
NAME                                  READY   STATUS    RESTARTS   AGE   IP            NODE          NOMINATED NODE   READINESS GATES
coredns-8f9b4c9f5-jzlj4               1/1     Running   0          16h   10.187.1.2    10.13.4.137   <none>           <none>
coredns-8f9b4c9f5-l87sk               1/1     Running   0          16h   10.187.2.2    10.13.4.135   <none>           <none>
coredns-8f9b4c9f5-wzq6h               1/1     Running   0          16h   10.187.0.3    10.13.4.136   <none>           <none>



 curl -v 10.187.0.3:9153
* Rebuilt URL to: 10.187.0.3:9153/
* Hostname was NOT found in DNS cache
*   Trying 10.187.0.3...
^C


 route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.13.4.1       0.0.0.0         UG    0      0        0 eth0
10.13.4.0       0.0.0.0         255.255.254.0   U     0      0        0 eth0
10.185.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
10.187.1.0      10.187.1.0      255.255.255.0   UG    0      0        0 flannel.1
10.187.2.0      10.187.2.0      255.255.255.0   UG    0      0        0 flannel.1
10.187.3.0      10.187.3.0      255.255.255.0   UG    0      0        0 flannel.1
10.187.4.0      10.187.4.0      255.255.255.0   UG    0      0        0 flannel.1
10.187.5.0      10.187.5.0      255.255.255.0   UG    0      0        0 flannel.1
10.187.6.0      0.0.0.0         255.255.255.0   U     0      0        0 cni0
10.187.7.0      10.187.7.0      255.255.255.0   UG    0      0        0 flannel.1
10.187.8.0      10.187.8.0      255.255.255.0   UG    0      0        0 flannel.1



susesp3:~ # docker ps -a | grep flanneld
2088aac705b9        f0fad859c909                                       "/opt/bin/flanneld..."   39 minutes ago      Up 39 minutes                                     k8s_kube-flannel_kube-flannel-ds-t6w8p_kube-system_9d462c9a-8211-45b3-9d78-aaa222b7bb96_0
susesp3:~ # docker logs 2088
I0415 03:04:45.950103       1 main.go:475] Determining IP address of default interface
I0415 03:04:45.950580       1 main.go:488] Using interface with name eth0 and address 10.13.4.78
I0415 03:04:45.950603       1 main.go:505] Defaulting external address to interface address (10.13.4.78)
I0415 03:04:46.054193       1 kube.go:131] Waiting 10m0s for node controller to sync
I0415 03:04:46.054598       1 kube.go:294] Starting kube subnet manager
I0415 03:04:47.054753       1 kube.go:138] Node controller sync successful
I0415 03:04:47.054798       1 main.go:235] Created subnet manager: Kubernetes Subnet Manager - 10.13.4.78
I0415 03:04:47.054803       1 main.go:238] Installing signal handlers
I0415 03:04:47.054957       1 main.go:353] Found network config - Backend type: vxlan
I0415 03:04:47.055029       1 vxlan.go:120] VXLAN config: VNI=1 Port=0 GBP=false DirectRouting=false
I0415 03:04:47.086132       1 main.go:300] Wrote subnet file to /run/flannel/subnet.env
I0415 03:04:47.086158       1 main.go:304] Running backend.
I0415 03:04:47.086175       1 main.go:322] Waiting for all goroutines to exit
I0415 03:04:47.086211       1 vxlan_network.go:60] watching for new subnet leases
E0415 03:04:47.147056       1 vxlan_network.go:158] failed to add vxlanRoute (10.187.0.0/24 -> 10.187.0.0): invalid argument
I0415 03:04:47.247138       1 iptables.go:115] Some iptables rules are missing; deleting and recreating rules
I0415 03:04:47.247179       1 iptables.go:137] Deleting iptables rule: -s 10.187.0.0/16 -d 10.187.0.0/16 -j RETURN
I0415 03:04:47.250541       1 iptables.go:137] Deleting iptables rule: -s 10.187.0.0/16 ! -d 224.0.0.0/4 -j MASQUERADE
I0415 03:04:47.253570       1 iptables.go:137] Deleting iptables rule: ! -s 10.187.0.0/16 -d 10.187.6.0/24 -j RETURN
I0415 03:04:47.347979       1 iptables.go:137] Deleting iptables rule: ! -s 10.187.0.0/16 -d 10.187.0.0/16 -j MASQUERADE
I0415 03:04:47.350959       1 iptables.go:125] Adding iptables rule: -s 10.187.0.0/16 -d 10.187.0.0/16 -j RETURN
I0415 03:04:47.547123       1 iptables.go:125] Adding iptables rule: -s 10.187.0.0/16 ! -d 224.0.0.0/4 -j MASQUERADE
I0415 03:04:47.749643       1 iptables.go:125] Adding iptables rule: ! -s 10.187.0.0/16 -d 10.187.6.0/24 -j RETURN
I0415 03:04:47.948175       1 iptables.go:125] Adding iptables rule: ! -s 10.187.0.0/16 -d 10.187.0.0/16 -j MASQUERADE
I0415 03:04:48.246645       1 iptables.go:115] Some iptables rules are missing; deleting and recreating rules
I0415 03:04:48.246809       1 iptables.go:137] Deleting iptables rule: -s 10.187.0.0/16 -j ACCEPT
I0415 03:04:48.248048       1 iptables.go:137] Deleting iptables rule: -d 10.187.0.0/16 -j ACCEPT
I0415 03:04:48.249078       1 iptables.go:125] Adding iptables rule: -s 10.187.0.0/16 -j ACCEPT
I0415 03:04:48.251011       1 iptables.go:125] Adding iptables rule: -d 10.187.0.0/16 -j ACCEPT


https://github.com/flannel-io/flannel/issues/959
https://github.com/flannel-io/flannel/issues/844


      flannel.alpha.coreos.com/backend-data: '{"VtepMAC":"4e:bf:14:ff:7e:c0"}'
      flannel.alpha.coreos.com/backend-type: vxlan
      flannel.alpha.coreos.com/kube-subnet-manager: "true"
      flannel.alpha.coreos.com/public-ip: 10.13.4.135
--
      flannel.alpha.coreos.com/backend-data: '{"VtepMAC":"b6:c2:7b:ad:6f:4b"}'
      flannel.alpha.coreos.com/backend-type: vxlan
      flannel.alpha.coreos.com/kube-subnet-manager: "true"
      flannel.alpha.coreos.com/public-ip: 10.13.4.136
--
      flannel.alpha.coreos.com/backend-data: '{"VtepMAC":"1a:7e:f7:cc:97:e4"}'
      flannel.alpha.coreos.com/backend-type: vxlan
      flannel.alpha.coreos.com/kube-subnet-manager: "true"
      flannel.alpha.coreos.com/public-ip: 10.13.4.137
--
      flannel.alpha.coreos.com/backend-data: '{"VtepMAC":"ca:63:85:51:bc:00"}'
      flannel.alpha.coreos.com/backend-type: vxlan
      flannel.alpha.coreos.com/kube-subnet-manager: "true"
      flannel.alpha.coreos.com/public-ip: 10.13.4.138
--
      flannel.alpha.coreos.com/backend-data: '{"VtepMAC":"66:33:09:52:75:18"}'
      flannel.alpha.coreos.com/backend-type: vxlan
      flannel.alpha.coreos.com/kube-subnet-manager: "true"
      flannel.alpha.coreos.com/public-ip: 10.13.4.150
--
      flannel.alpha.coreos.com/backend-data: '{"VtepMAC":"8e:2f:33:84:69:16"}'
      flannel.alpha.coreos.com/backend-type: vxlan
      flannel.alpha.coreos.com/kube-subnet-manager: "true"
      flannel.alpha.coreos.com/public-ip: 10.13.4.151
--
      flannel.alpha.coreos.com/backend-data: '{"VtepMAC":"86:c7:8e:6b:cf:6d"}'
      flannel.alpha.coreos.com/backend-type: vxlan
      flannel.alpha.coreos.com/kube-subnet-manager: "true"
      flannel.alpha.coreos.com/public-ip: 10.13.4.152
--
      flannel.alpha.coreos.com/backend-data: '{"VtepMAC":"ba:5b:d4:b1:5b:68"}'
      flannel.alpha.coreos.com/backend-type: vxlan
      flannel.alpha.coreos.com/kube-subnet-manager: "true"
      flannel.alpha.coreos.com/public-ip: 10.13.4.153
--
      flannel.alpha.coreos.com/backend-data: '{"VtepMAC":"e2:e3:b3:37:1e:60"}'
      flannel.alpha.coreos.com/backend-type: vxlan
      flannel.alpha.coreos.com/kube-subnet-manager: "true"
      flannel.alpha.coreos.com/public-ip: 10.13.4.78



wget https://github.com/multiarch/qemu-user-static/releases/download/v3.0.0/qemu-x86_64-static.tar.gz

docker run --privileged -ti --rm --name flannel --net host -e GOARCH=amd64 \
    -u $( id -u):$( id -g) \
    -v $PWD/dist/qemu-amd64-static:/usr/bin/qemu-amd64-static \
    -v $PWD:/go/src/github.com/coreos/flannel:ro \
    -v $PWD/dist:/go/src/github.com/coreos/flannel/dist \
    -v /run:/run \
    --workdir=/go/src/github.com/coreos/flannel \
    golang:1.16.3 /bin/bash

go install github.com/go-delve/delve/cmd/dlv@latest



  CGO_ENABLED=1 GO111MODULE=off  go build  -o dist/flanneld \
    -gcflags="all=-trimpath=$PWD -N -l" \
    -asmflags "all=-trimpath=$PWD" \
     -ldflags '-X github.com/coreos/flannel/version.Version=v0.11.0-go1.16.3 -extldflags "-static"' main.go


mkdir -p /etc/cni/net.d/
cat <<EOF > /etc/cni/net.d/10-flannel.conflist
{
  "name": "cbr0",
  "cniVersion": "0.3.1",
  "plugins": [
    {
      "type": "flannel",
      "delegate": {
        "hairpinMode": true,
        "isDefaultGateway": true
      }
    },
    {
      "type": "portmap",
      "capabilities": {
        "portMappings": true
      }
    }
  ]
}
EOF

mkdir -p /etc/kube-flannel
cat <<EOF > /etc/kube-flannel/net-conf.json
{
      "Network": "10.187.0.0/16",
      "Backend": {
            "Type": "vxlan"
      }
}
EOF

export NODE_NAME=10.13.4.78
dlv exec ./dist/flanneld -- --kube-subnet-mgr --kubeconfig-file=./dist/kubeconfig --ip-masq

(dlv) b backend/vxlan/vxlan_network.go:128
(dlv) c
I0422 09:58:22.001125    4017 main.go:514] Determining IP address of default interface
I0422 09:58:22.001635    4017 main.go:527] Using interface with name eth0 and address 10.13.4.78
I0422 09:58:22.001679    4017 main.go:544] Defaulting external address to interface address (10.13.4.78)
I0422 09:58:22.012215    4017 kube.go:126] Waiting 10m0s for node controller to sync
I0422 09:58:22.012349    4017 kube.go:309] Starting kube subnet manager
I0422 09:58:23.012856    4017 kube.go:133] Node controller sync successful
I0422 09:58:23.013012    4017 main.go:244] Created subnet manager: Kubernetes Subnet Manager - 10.13.4.78
I0422 09:58:23.013056    4017 main.go:247] Installing signal handlers
I0422 09:58:23.013714    4017 main.go:386] Found network config - Backend type: vxlan
I0422 09:58:23.013894    4017 vxlan.go:120] VXLAN config: VNI=1 Port=0 GBP=false DirectRouting=false
I0422 09:58:23.014493    4017 main.go:351] Current network or subnet (10.187.0.0/16, 10.187.6.0/24) is not equal to previous one (0.0.0.0/0, 0.0.0.0/0), trying to recycle old iptables rules
I0422 09:58:23.026665    4017 iptables.go:167] Deleting iptables rule: -s 0.0.0.0/0 -d 0.0.0.0/0 -j RETURN
I0422 09:58:23.030341    4017 iptables.go:167] Deleting iptables rule: -s 0.0.0.0/0 ! -d 224.0.0.0/4 -j MASQUERADE --random-fully
I0422 09:58:23.033775    4017 iptables.go:167] Deleting iptables rule: ! -s 0.0.0.0/0 -d 0.0.0.0/0 -j RETURN
I0422 09:58:23.036514    4017 iptables.go:167] Deleting iptables rule: ! -s 0.0.0.0/0 -d 0.0.0.0/0 -j MASQUERADE --random-fully
I0422 09:58:23.040762    4017 main.go:317] Wrote subnet file to /run/flannel/subnet.env
I0422 09:58:23.040798    4017 main.go:321] Running backend.
I0422 09:58:23.040827    4017 main.go:339] Waiting for all goroutines to exit
I0422 09:58:23.040853    4017 vxlan_network.go:60] watching for new subnet leases
> github.com/coreos/flannel/backend/vxlan.(*network).handleSubnetEvents() backend/vxlan/vxlan_network.go:128 (hits goroutine(62):1 total:1) (PC: 0x166daa9)
   123:				} else {
   124:					directRoutingOK = dr
   125:				}
   126:			}
   127:	
=> 128:			switch event.Type {
   129:			case subnet.EventAdded:
   130:				if directRoutingOK {
   131:					log.V(2).Infof("Adding direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIP)
   132:	
   133:					if err := netlink.RouteReplace(&directRoute); err != nil {
(dlv) p vxlanRoute
github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.Route {
	LinkIndex: 11,
	ILinkIndex: 0,
	Scope: SCOPE_UNIVERSE (0),
	Dst: *net.IPNet {
		IP: net.IP len: 16, cap: 16, [0,0,0,0,0,0,0,0,0,0,255,255,10,187,1,0],
		Mask: net.IPMask len: 4, cap: 4, [255,255,255,0],},
	Src: net.IP len: 0, cap: 0, nil,
	Gw: net.IP len: 16, cap: 16, [0,0,0,0,0,0,0,0,0,0,255,255,10,187,1,0],
	MultiPath: []*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.NexthopInfo len: 0, cap: 0, nil,
	Protocol: 0,
	Priority: 0,
	Table: 0,
	Type: 0,
	Tos: 0,
	Flags: 4,
	MPLSDst: *int nil,
	NewDst: github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.Destination nil,
	Encap: github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.Encap nil,}


> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).RouteReplace() vendor/github.com/vishvananda/netlink/route_linux.go:157 (PC: 0x828ec0)
   152:	// RouteReplace will add a route to the system.
   153:	// Equivalent to: `ip route replace $route`
   154:	func (h *Handle) RouteReplace(route *Route) error {
   155:		flags := syscall.NLM_F_CREATE | syscall.NLM_F_REPLACE | syscall.NLM_F_ACK
   156:		req := h.newNetlinkRequest(syscall.RTM_NEWROUTE, flags)
=> 157:		return h.routeHandle(route, req, nl.NewRtMsg())
   158:	}
   159:	
   160:	// RouteDel will delete a route from the system.
   161:	// Equivalent to: `ip route del $route`
   162:	func RouteDel(route *Route) error {
(dlv) p flags
1284
(dlv) p req
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequest {
	NlMsghdr: syscall.NlMsghdr {Len: 16, Type: 24, Flags: 1285, Seq: 9, Pid: 0},
	Data: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, nil,
	Sockets: map[int]*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.SocketHandle nil,}
(dlv) 


 101 0000 0100    101 0000 0101

 > github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:174 (PC: 0x8290bd)
   169:		req := h.newNetlinkRequest(syscall.RTM_DELROUTE, syscall.NLM_F_ACK)
   170:		return h.routeHandle(route, req, nl.NewRtDelMsg())
   171:	}
   172:	
   173:	func (h *Handle) routeHandle(route *Route, req *nl.NetlinkRequest, msg *nl.RtMsg) error {
=> 174:		if (route.Dst == nil || route.Dst.IP == nil) && route.Src == nil && route.Gw == nil && route.MPLSDst == nil {
   175:			return fmt.Errorf("one of Dst.IP, Src, or Gw must not be nil")
   176:		}
   177:	
   178:		family := -1
   179:		var rtAttrs []*nl.RtAttr
(dlv) p route
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.Route {
	LinkIndex: 11,
	ILinkIndex: 0,
	Scope: SCOPE_UNIVERSE (0),
	Dst: *net.IPNet {
		IP: net.IP len: 16, cap: 16, [0,0,0,0,0,0,0,0,0,0,255,255,10,187,1,0],
		Mask: net.IPMask len: 4, cap: 4, [255,255,255,0],},
	Src: net.IP len: 0, cap: 0, nil,
	Gw: net.IP len: 16, cap: 16, [0,0,0,0,0,0,0,0,0,0,255,255,10,187,1,0],
	MultiPath: []*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.NexthopInfo len: 0, cap: 0, nil,
	Protocol: 0,
	Priority: 0,
	Table: 0,
	Type: 0,
	Tos: 0,
	Flags: 4,
	MPLSDst: *int nil,
	NewDst: github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.Destination nil,
	Encap: github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.Encap nil,}
(dlv) p msg
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtMsg {
	RtMsg: syscall.RtMsg {Family: 0, Dst_len: 0, Src_len: 0, Tos: 0, Table: 254, Protocol: 3, Scope: 0, Type: 1, Flags: 0},}
(dlv) 


> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:187 (PC: 0x82931c)
   182:			dstLen, _ := route.Dst.Mask.Size()
   183:			msg.Dst_len = uint8(dstLen)
   184:			dstFamily := nl.GetIPFamily(route.Dst.IP)
   185:			family = dstFamily
   186:			var dstData []byte
=> 187:			if dstFamily == FAMILY_V4 {
   188:				dstData = route.Dst.IP.To4()
   189:			} else {
   190:				dstData = route.Dst.IP.To16()
   191:			}
   192:			rtAttrs = append(rtAttrs, nl.NewRtAttr(syscall.RTA_DST, dstData))
(dlv) s
I0422 10:18:25.299803    4017 iptables.go:155] Adding iptables rule: -s 10.187.0.0/16 -d 10.187.0.0/16 -j RETURN
> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:188 (PC: 0x829329)
   183:			msg.Dst_len = uint8(dstLen)
   184:			dstFamily := nl.GetIPFamily(route.Dst.IP)
   185:			family = dstFamily
   186:			var dstData []byte
   187:			if dstFamily == FAMILY_V4 {
=> 188:				dstData = route.Dst.IP.To4()
   189:			} else {
   190:				dstData = route.Dst.IP.To16()
   191:			}
   192:			rtAttrs = append(rtAttrs, nl.NewRtAttr(syscall.RTA_DST, dstData))
   193:		} else if route.MPLSDst != nil {
(dlv) call route.Dst.IP.To4()
E0422 10:18:37.541354    4017 iptables.go:115] Failed to ensure iptables rules: Error setting up rules: failed to insert IPTables rule: running [/usr/sbin/iptables -t nat -A POSTROUTING -s 10.187.0.0/16 -d 10.187.0.0/16 -j RETURN --wait]: exit status 1: iptables: Invalid argument. Run `dmesg' for more information.
E0422 10:18:37.541408    4017 iptables.go:115] Failed to ensure iptables rules: Error setting up rules: failed to insert IPTables rule: running [/usr/sbin/iptables -t filter -A FORWARD -s 10.187.0.0/16 -j ACCEPT --wait]: exit status 1: iptables: Invalid argument. Run `dmesg' for more information.
> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:188 (PC: 0x829329)
Values returned:
	~r0: net.IP len: 4, cap: 4, [10,187,1,0]

   183:			msg.Dst_len = uint8(dstLen)
   184:			dstFamily := nl.GetIPFamily(route.Dst.IP)
   185:			family = dstFamily
   186:			var dstData []byte
   187:			if dstFamily == FAMILY_V4 {
=> 188:				dstData = route.Dst.IP.To4()
   189:			} else {
   190:				dstData = route.Dst.IP.To16()
   191:			}
   192:			rtAttrs = append(rtAttrs, nl.NewRtAttr(syscall.RTA_DST, dstData))
   193:		} else if route.MPLSDst != nil {
(dlv) 


(dlv) s
> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:200 (PC: 0x829453)
   195:			msg.Dst_len = uint8(20)
   196:			msg.Type = syscall.RTN_UNICAST
   197:			rtAttrs = append(rtAttrs, nl.NewRtAttr(syscall.RTA_DST, nl.EncodeMPLSStack(*route.MPLSDst)))
   198:		}
   199:	
=> 200:		if route.NewDst != nil {
   201:			if family != -1 && family != route.NewDst.Family() {
   202:				return fmt.Errorf("new destination and destination are not the same address family")
   203:			}
   204:			buf, err := route.NewDst.Encode()
   205:			if err != nil {
(dlv) p rtAttrs
[]*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtAttr len: 1, cap: 1, [
	*{
		RtAttr: (*syscall.RtAttr)(0xc0006ec080),
		Data: []uint8 len: 4, cap: 4, [10,187,1,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
]
(dlv) 


> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:253 (PC: 0x829ead)
   248:				gwData = route.Gw.To16()
   249:			}
   250:			rtAttrs = append(rtAttrs, nl.NewRtAttr(syscall.RTA_GATEWAY, gwData))
   251:		}
   252:	
=> 253:		if len(route.MultiPath) > 0 {
   254:			buf := []byte{}
   255:			for _, nh := range route.MultiPath {
   256:				rtnh := &nl.RtNexthop{
   257:					RtNexthop: syscall.RtNexthop{
   258:						Hops:    uint8(nh.Hops),
(dlv) p rtAttrs
[]*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtAttr len: 2, cap: 2, [
	*{
		RtAttr: (*syscall.RtAttr)(0xc0006ec080),
		Data: []uint8 len: 4, cap: 4, [10,187,1,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
	*{
		RtAttr: (*syscall.RtAttr)(0xc000840000),
		Data: []uint8 len: 4, cap: 4, [10,187,1,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
]
(dlv) 


> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:330 (PC: 0x82b1fd)
   325:		}
   326:	
   327:		msg.Flags = uint32(route.Flags)
   328:		msg.Scope = uint8(route.Scope)
   329:		msg.Family = uint8(family)
=> 330:		req.AddData(msg)
   331:		for _, attr := range rtAttrs {
   332:			req.AddData(attr)
   333:		}
   334:	
   335:		var (
(dlv) p msg
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtMsg {
	RtMsg: syscall.RtMsg {Family: 2, Dst_len: 24, Src_len: 0, Tos: 0, Table: 254, Protocol: 3, Scope: 0, Type: 1, Flags: 4},}
(dlv) 


> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:336 (PC: 0x82b2f1)
   331:		for _, attr := range rtAttrs {
   332:			req.AddData(attr)
   333:		}
   334:	
   335:		var (
=> 336:			b      = make([]byte, 4)
   337:			native = nl.NativeEndian()
   338:		)
   339:		native.PutUint32(b, uint32(route.LinkIndex))
   340:	
   341:		req.AddData(nl.NewRtAttr(syscall.RTA_OIF, b))
(dlv) p req
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequest {
	NlMsghdr: syscall.NlMsghdr {Len: 16, Type: 24, Flags: 1285, Seq: 9, Pid: 0},
	Data: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 3, cap: 4, [
		...,
		...,
		...,
	],
	Sockets: map[int]*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.SocketHandle nil,}
(dlv) p req.Data
[]github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 3, cap: 4, [
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtMsg {
		RtMsg: (*syscall.RtMsg)(0xc000684030),},
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtAttr {
		RtAttr: (*syscall.RtAttr)(0xc0006ec080),
		Data: []uint8 len: 4, cap: 4, [10,187,1,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtAttr {
		RtAttr: (*syscall.RtAttr)(0xc000840000),
		Data: []uint8 len: 4, cap: 4, [10,187,1,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
]
(dlv) p req.Data[0].RtMsg
syscall.RtMsg {Family: 2, Dst_len: 24, Src_len: 0, Tos: 0, Table: 254, Protocol: 3, Scope: 0, Type: 1, Flags: 4}


(dlv) so
> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:343 (PC: 0x82b405)
Values returned:

   338:		)
   339:		native.PutUint32(b, uint32(route.LinkIndex))
   340:	
   341:		req.AddData(nl.NewRtAttr(syscall.RTA_OIF, b))
   342:	
=> 343:		_, err := req.Execute(syscall.NETLINK_ROUTE, 0)
   344:		return err
   345:	}
   346:	
   347:	// RouteList gets a list of routes in the system.
   348:	// Equivalent to: `ip route show`.
(dlv) p req
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequest {
	NlMsghdr: syscall.NlMsghdr {Len: 16, Type: 24, Flags: 1285, Seq: 9, Pid: 0},
	Data: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 4, cap: 4, [
		...,
		...,
		...,
		...,
	],
	Sockets: map[int]*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.SocketHandle nil,}
(dlv) p req.Data
[]github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 4, cap: 4, [
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtMsg {
		RtMsg: (*syscall.RtMsg)(0xc000684030),},
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtAttr {
		RtAttr: (*syscall.RtAttr)(0xc0006ec080),
		Data: []uint8 len: 4, cap: 4, [10,187,1,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtAttr {
		RtAttr: (*syscall.RtAttr)(0xc000840000),
		Data: []uint8 len: 4, cap: 4, [10,187,1,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtAttr {
		RtAttr: (*syscall.RtAttr)(0xc000804a00),
		Data: []uint8 len: 4, cap: 4, [11,0,0,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
]
(dlv) p req.Data[0].RtMsg
syscall.RtMsg {Family: 2, Dst_len: 24, Src_len: 0, Tos: 0, Table: 254, Protocol: 3, Scope: 0, Type: 1, Flags: 4}
(dlv) p req.Data[1].RtAttr
syscall.RtAttr {Len: 0, Type: 1}
(dlv) p req.Data[2].RtAttr
syscall.RtAttr {Len: 0, Type: 5}
(dlv) p req.Data[3].RtAttr
syscall.RtAttr {Len: 0, Type: 4}
(dlv) 



(dlv) s
> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.(*NetlinkRequest).Execute() vendor/github.com/vishvananda/netlink/nl/nl_linux.go:374 (PC: 0x8105fc)
   369:		}
   370:		sharedSocket := s != nil
   371:	
   372:		if s == nil {
   373:			s, err = getNetlinkSocket(sockType)
=> 374:			if err != nil {
   375:				return nil, err
   376:			}
   377:			defer s.Close()
   378:		} else {
   379:			s.Lock()
(dlv) p s
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkSocket {
	fd: 6,
	lsa: syscall.SockaddrNetlink {
		Family: 16,
		Pad: 0,
		Pid: 0,
		Groups: 0,
		raw: (*syscall.RawSockaddrNetlink)(0xc000814014),},
	Mutex: sync.Mutex {state: 0, sema: 0},}
(dlv) p s.lsa.raw
syscall.RawSockaddrNetlink {Family: 16, Pad: 0, Pid: 0, Groups: 0}


> syscall.Sendto() /usr/local/go/src/syscall/syscall_unix.go:294 (PC: 0x4e49cc)
   289:		}
   290:		return
   291:	}
   292:	
   293:	func Sendto(fd int, p []byte, flags int, to Sockaddr) (err error) {
=> 294:		ptr, n, err := to.sockaddr()
   295:		if err != nil {
   296:			return err
   297:		}
   298:		return sendto(fd, p, flags, ptr, n)
   299:	}
(dlv) p fd
6
(dlv) p p
[]uint8 len: 52, cap: 52, [52,0,0,0,24,0,5,5,9,0,0,0,0,0,0,0,2,24,0,0,254,3,0,1,4,0,0,0,8,0,1,0,10,187,1,0,8,0,5,0,10,187,1,0,8,0,4,0,11,0,0,0]
(dlv) p flags
0
(dlv) p to
syscall.Sockaddr(*syscall.SockaddrNetlink) *{
	Family: 16,
	Pad: 0,
	Pid: 0,
	Groups: 0,
	raw: syscall.RawSockaddrNetlink {Family: 16, Pad: 0, Pid: 0, Groups: 0},}
(dlv) 






-----------------


> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:174 (PC: 0x8290bd)
   169:		req := h.newNetlinkRequest(syscall.RTM_DELROUTE, syscall.NLM_F_ACK)
   170:		return h.routeHandle(route, req, nl.NewRtDelMsg())
   171:	}
   172:	
   173:	func (h *Handle) routeHandle(route *Route, req *nl.NetlinkRequest, msg *nl.RtMsg) error {
=> 174:		if (route.Dst == nil || route.Dst.IP == nil) && route.Src == nil && route.Gw == nil && route.MPLSDst == nil {
   175:			return fmt.Errorf("one of Dst.IP, Src, or Gw must not be nil")
   176:		}
   177:	
   178:		family := -1
   179:		var rtAttrs []*nl.RtAttr
(dlv) p route
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.Route {
	LinkIndex: 11,
	ILinkIndex: 0,
	Scope: SCOPE_UNIVERSE (0),
	Dst: *net.IPNet {
		IP: net.IP len: 16, cap: 16, [0,0,0,0,0,0,0,0,0,0,255,255,10,187,0,0],
		Mask: net.IPMask len: 4, cap: 4, [255,255,255,0],},
	Src: net.IP len: 0, cap: 0, nil,
	Gw: net.IP len: 16, cap: 16, [0,0,0,0,0,0,0,0,0,0,255,255,10,187,0,0],
	MultiPath: []*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.NexthopInfo len: 0, cap: 0, nil,
	Protocol: 0,
	Priority: 0,
	Table: 0,
	Type: 0,
	Tos: 0,
	Flags: 4,
	MPLSDst: *int nil,
	NewDst: github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.Destination nil,
	Encap: github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.Encap nil,}
(dlv) p req
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequest {
	NlMsghdr: syscall.NlMsghdr {Len: 16, Type: 24, Flags: 1285, Seq: 24, Pid: 0},
	Data: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, nil,
	Sockets: map[int]*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.SocketHandle nil,}
(dlv) p msg
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtMsg {
	RtMsg: syscall.RtMsg {Family: 0, Dst_len: 0, Src_len: 0, Tos: 0, Table: 254, Protocol: 3, Scope: 0, Type: 1, Flags: 0},}
(dlv) 


> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink.(*Handle).routeHandle() vendor/github.com/vishvananda/netlink/route_linux.go:341 (PC: 0x82b3a2)
Values returned:

   336:			b      = make([]byte, 4)
   337:			native = nl.NativeEndian()
   338:		)
   339:		native.PutUint32(b, uint32(route.LinkIndex))
   340:	
=> 341:		req.AddData(nl.NewRtAttr(syscall.RTA_OIF, b))
   342:	
   343:		_, err := req.Execute(syscall.NETLINK_ROUTE, 0)
   344:		return err
   345:	}
   346:	
(dlv) p req
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequest {
	NlMsghdr: syscall.NlMsghdr {Len: 16, Type: 24, Flags: 1285, Seq: 24, Pid: 0},
	Data: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 3, cap: 4, [
		...,
		...,
		...,
	],
	Sockets: map[int]*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.SocketHandle nil,}
(dlv) p req.Data
[]github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 3, cap: 4, [
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtMsg {
		RtMsg: (*syscall.RtMsg)(0xc0004ec000),},
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtAttr {
		RtAttr: (*syscall.RtAttr)(0xc000402000),
		Data: []uint8 len: 4, cap: 4, [10,187,0,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
	*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.RtAttr {
		RtAttr: (*syscall.RtAttr)(0xc000402040),
		Data: []uint8 len: 4, cap: 4, [10,187,0,0],
		children: []github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkRequestData len: 0, cap: 0, [],},
]
(dlv) p req.Data[0].RtMsg
syscall.RtMsg {Family: 2, Dst_len: 24, Src_len: 0, Tos: 0, Table: 254, Protocol: 3, Scope: 0, Type: 1, Flags: 4}
(dlv) p req.Data[1].RtAttr
syscall.RtAttr {Len: 0, Type: 1}
(dlv) p req.Data[2].RtAttr
syscall.RtAttr {Len: 0, Type: 5}


> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.(*NetlinkRequest).Execute() vendor/github.com/vishvananda/netlink/nl/nl_linux.go:374 (PC: 0x8105fc)
   369:		}
   370:		sharedSocket := s != nil
   371:	
   372:		if s == nil {
   373:			s, err = getNetlinkSocket(sockType)
=> 374:			if err != nil {
   375:				return nil, err
   376:			}
   377:			defer s.Close()
   378:		} else {
   379:			s.Lock()
(dlv) p s
*github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.NetlinkSocket {
	fd: 3,
	lsa: syscall.SockaddrNetlink {
		Family: 16,
		Pad: 0,
		Pid: 0,
		Groups: 0,
		raw: (*syscall.RawSockaddrNetlink)(0xc0008fc014),},
	Mutex: sync.Mutex {state: 0, sema: 0},}
(dlv) p s.lsa.raw
syscall.RawSockaddrNetlink {Family: 16, Pad: 0, Pid: 0, Groups: 0}

   469:	func (sa *SockaddrNetlink) sockaddr() (unsafe.Pointer, _Socklen, error) {
=> 470:		sa.raw.Family = AF_NETLINK
   471:		sa.raw.Pad = sa.Pad
   472:		sa.raw.Pid = sa.Pid
   473:		sa.raw.Groups = sa.Groups
   474:		return unsafe.Pointer(&sa.raw), SizeofSockaddrNetlink, nil
   475:	}
(dlv) so
> syscall.Sendto() /usr/local/go/src/syscall/syscall_unix.go:294 (PC: 0x4e49f9)
Values returned:
	~r0: unsafe.Pointer(0xc0008fc014)
	~r1: 12
	~r2: error nil

   289:		}
   290:		return
   291:	}

> syscall.sendto() /usr/local/go/src/syscall/zsyscall_linux_amd64.go:1541 (PC: 0x4e7dfa)
  1536:	// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
  1537:	
  1538:	func sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen _Socklen) (err error) {
  1539:		var _p0 unsafe.Pointer
  1540:		if len(buf) > 0 {
=>1541:			_p0 = unsafe.Pointer(&buf[0])
  1542:		} else {
  1543:			_p0 = unsafe.Pointer(&_zero)
  1544:		}
  1545:		_, _, e1 := Syscall6(SYS_SENDTO, uintptr(s), uintptr(_p0), uintptr(len(buf)), uintptr(flags), uintptr(to), uintptr(addrlen))
  1546:		if e1 != 0 {
(dlv) p s
3
(dlv) p buf
[]uint8 len: 52, cap: 52, [52,0,0,0,24,0,5,5,24,0,0,0,0,0,0,0,2,24,0,0,254,3,0,1,4,0,0,0,8,0,1,0,10,187,0,0,8,0,5,0,10,187,0,0,8,0,4,0,11,0,0,0]
(dlv) p flags
0
(dlv) p to
unsafe.Pointer(0xc0008fc014)
(dlv) p addrlen
12
(dlv) 



> github.com/coreos/flannel/vendor/github.com/vishvananda/netlink/nl.(*NetlinkRequest).Execute() vendor/github.com/vishvananda/netlink/nl/nl_linux.go:401 (PC: 0x80fe7c)
   396:			msgs, err := s.Receive()
   397:			if err != nil {
   398:				return nil, err
   399:			}
   400:			for _, m := range msgs {
=> 401:				if m.Header.Seq != req.Seq {
   402:					if sharedSocket {
   403:						continue
   404:					}
   405:					return nil, fmt.Errorf("Wrong Seq nr %d, expected %d", m.Header.Seq, req.Seq)
   406:				}
(dlv) p msgs
[]syscall.NetlinkMessage len: 1, cap: 1, [
	{
		Header: (*syscall.NlMsghdr)(0xc000186300),
		Data: []uint8 len: 20, cap: 4080, [0,0,0,0,52,0,0,0,24,0,5,5,24,0,0,0,0,0,0,0],},
]
(dlv) p msgs[0].Header
syscall.NlMsghdr {Len: 36, Type: 2, Flags: 0, Seq: 24, Pid: 4017}
(dlv) s

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment