2021/04/09 11:52:11 INFO fsmsupport.go:252 - *** Entering state SET_KERNELCONFIG after getting event: salt_finished (Src: SALT_COMPUTE_RUNNING) fatal error: sync: RUnlock of unlocked RWMutex [signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0x11f6601] goroutine 1 [running]: runtime.throw(0x14ed190, 0x21) /jump/software/rhel7/golang-1.16/src/runtime/panic.go:1117 +0x72 fp=0xc00001b478 sp=0xc00001b448 pc=0x4447f2 sync.throw(0x14ed190, 0x21) /jump/software/rhel7/golang-1.16/src/runtime/panic.go:1103 +0x35 fp=0xc00001b498 sp=0xc00001b478 pc=0x4758b5 sync.(*RWMutex).rUnlockSlow(0xc0004fc5d8, 0xffffffff) /jump/software/rhel7/golang-1.16/src/sync/rwmutex.go:93 +0x3f fp=0xc00001b4c0 sp=0xc00001b498 pc=0x4922ff sync.(*RWMutex).RUnlock(0xc0004fc5d8) /jump/software/rhel7/golang-1.16/src/sync/rwmutex.go:83 +0x49 fp=0xc00001b4e0 sp=0xc00001b4c0 pc=0x4922a9 runtime.call16(0x0, 0x1569eb8, 0xc00001bb78, 0x800000008) /jump/software/rhel7/golang-1.16/src/runtime/asm_amd64.s:550 +0x3e fp=0xc00001b500 sp=0xc00001b4e0 pc=0x47891e panic(0x1384020, 0x1fc8430) /jump/software/rhel7/golang-1.16/src/runtime/panic.go:971 +0x499 fp=0xc00001b5c8 sp=0xc00001b500 pc=0x444319 runtime.panicmem() /jump/software/rhel7/golang-1.16/src/runtime/panic.go:212 +0x5b fp=0xc00001b5e8 sp=0xc00001b5c8 pc=0x44243b runtime.sigpanic() /jump/software/rhel7/golang-1.16/src/runtime/signal_unix.go:734 +0x173 fp=0xc00001b620 sp=0xc00001b5e8 pc=0x45aeb3 main.(*NodeMoverFSM).LeaseIsExpired(0xc000784000, 0xc000128847, 0x7, 0xc0000f6630) /data/jenkins/jobs/automaton.master/workspace/cmd/nodemoverfsm/main.go:880 +0x201 fp=0xc00001b8b8 sp=0xc00001b620 pc=0x11f6601 main.(*NodeMoverFSM).updateSkynetKernelConfig(0xc000784000) /data/jenkins/jobs/automaton.master/workspace/cmd/nodemoverfsm/main.go:634 +0x45 fp=0xc00001b9b8 sp=0xc00001b8b8 pc=0x11f2765 main.(*NodeMoverFSM).updateSkynetKernelConfig-fm() /data/jenkins/jobs/automaton.master/workspace/cmd/nodemoverfsm/main.go:632 +0x33 fp=0xc00001b9d0 sp=0xc00001b9b8 pc=0x11f7593 git.w2k.jumptrading.com/linux/automaton/pkg/fsmsupport.M.func1(0xc0004c4070) /data/jenkins/jobs/automaton.master/workspace/pkg/fsmsupport/fsmsupport.go:171 +0x1a5 fp=0xc00001ba50 sp=0xc00001b9d0 pc=0x11ea025 github.com/looplab/fsm.(*FSM).enterStateCallbacks(0xc0004fc5a0, 0xc0004c4070) /data/jenkins/go/pkg/mod/github.com/looplab/fsm@v0.0.0-20180515091235-f980bdb68a89/fsm.go:403 +0x102 fp=0xc00001baa0 sp=0xc00001ba50 pc=0x11e0702 github.com/looplab/fsm.(*FSM).Event.func1() /data/jenkins/go/pkg/mod/github.com/looplab/fsm@v0.0.0-20180515091235-f980bdb68a89/fsm.go:308 +0x9e fp=0xc00001bae8 sp=0xc00001baa0 pc=0x11e11be github.com/looplab/fsm.transitionerStruct.transition(...) /data/jenkins/go/pkg/mod/github.com/looplab/fsm@v0.0.0-20180515091235-f980bdb68a89/fsm.go:354 github.com/looplab/fsm.(*transitionerStruct).transition(0x2071af8, 0xc0004fc5a0, 0x0, 0x0) :1 +0x64 fp=0xc00001baf8 sp=0xc00001bae8 pc=0x11e1264 github.com/looplab/fsm.(*FSM).doTransition(...) /data/jenkins/go/pkg/mod/github.com/looplab/fsm@v0.0.0-20180515091235-f980bdb68a89/fsm.go:339 github.com/looplab/fsm.(*FSM).Event(0xc0004fc5a0, 0x14d717a, 0xd, 0x0, 0x0, 0x0, 0x0, 0x0) /data/jenkins/go/pkg/mod/github.com/looplab/fsm@v0.0.0-20180515091235-f980bdb68a89/fsm.go:321 +0x5c2 fp=0xc00001bcb8 sp=0xc00001baf8 pc=0x11dfc22 git.w2k.jumptrading.com/linux/automaton/pkg/fsmsupport.(*CommonFSM).Launch(0xc000784078, 0x14d016b, 0x4) /data/jenkins/jobs/automaton.master/workspace/pkg/fsmsupport/fsmsupport.go:291 +0x19f fp=0xc00001bd48 sp=0xc00001bcb8 pc=0x11e301f main.main() /data/jenkins/jobs/automaton.master/workspace/cmd/nodemoverfsm/main.go:945 +0x553 fp=0xc00001bf88 sp=0xc00001bd48 pc=0x11f6c53 runtime.main() /jump/software/rhel7/golang-1.16/src/runtime/proc.go:225 +0x256 fp=0xc00001bfe0 sp=0xc00001bf88 pc=0x447036 runtime.goexit() /jump/software/rhel7/golang-1.16/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc00001bfe8 sp=0xc00001bfe0 pc=0x47a381 goroutine 6 [chan receive]: k8s.io/klog.(*loggingT).flushDaemon(0x2041de0) /data/jenkins/go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:1010 +0x8b created by k8s.io/klog.init.0 /data/jenkins/go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:411 +0xd8 goroutine 37 [select, 5 minutes]: database/sql.(*DB).connectionOpener(0xc0004cbe10, 0x1676bd8, 0xc0007a6040) /jump/software/rhel7/golang-1.16/src/database/sql/sql.go:1133 +0xb5 created by database/sql.OpenDB /jump/software/rhel7/golang-1.16/src/database/sql/sql.go:740 +0x12a goroutine 59 [select, 5 minutes]: github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1(0xc00010e300, 0xc0004cf2c0, 0xc0001302a0) /data/jenkins/go/pkg/mod/github.com/go-sql-driver/mysql@v1.4.1/connection_go18.go:178 +0xa5 created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher /data/jenkins/go/pkg/mod/github.com/go-sql-driver/mysql@v1.4.1/connection_go18.go:175 +0xbe goroutine 70 [select, 5 minutes]: github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1(0xc00010e8a0, 0xc00014e000, 0xc000130780) /data/jenkins/go/pkg/mod/github.com/go-sql-driver/mysql@v1.4.1/connection_go18.go:178 +0xa5 created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher /data/jenkins/go/pkg/mod/github.com/go-sql-driver/mysql@v1.4.1/connection_go18.go:175 +0xbe goroutine 60 [select, 5 minutes]: database/sql.(*DB).connectionOpener(0xc000138680, 0x1676bd8, 0xc0001341c0) /jump/software/rhel7/golang-1.16/src/database/sql/sql.go:1133 +0xb5 created by database/sql.OpenDB /jump/software/rhel7/golang-1.16/src/database/sql/sql.go:740 +0x12a goroutine 243 [IO wait]: internal/poll.runtime_pollWait(0x7fffce054e28, 0x72, 0xffffffffffffffff) /jump/software/rhel7/golang-1.16/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc00012e298, 0x72, 0xb00, 0xbb9, 0xffffffffffffffff) /jump/software/rhel7/golang-1.16/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /jump/software/rhel7/golang-1.16/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc00012e280, 0xc0002a2c00, 0xbb9, 0xbb9, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/internal/poll/fd_unix.go:166 +0x1d5 net.(*netFD).Read(0xc00012e280, 0xc0002a2c00, 0xbb9, 0xbb9, 0xbb4, 0xc0002a2c00, 0x5) /jump/software/rhel7/golang-1.16/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc000114298, 0xc0002a2c00, 0xbb9, 0xbb9, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/net/net.go:183 +0x91 crypto/tls.(*atLeastReader).Read(0xc0006ee138, 0xc0002a2c00, 0xbb9, 0xbb9, 0xbb4, 0x2042660, 0x0) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:776 +0x63 bytes.(*Buffer).ReadFrom(0xc000784cf8, 0x164ea60, 0xc0006ee138, 0x419c85, 0x13962e0, 0x147ed80) /jump/software/rhel7/golang-1.16/src/bytes/buffer.go:204 +0xbe crypto/tls.(*Conn).readFromUntil(0xc000784a80, 0x1650080, 0xc000114298, 0x5, 0xc000114298, 0x1b8) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:798 +0xf3 crypto/tls.(*Conn).readRecordOrCCS(0xc000784a80, 0x0, 0x0, 0x8) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:605 +0x115 crypto/tls.(*Conn).readRecord(...) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:573 crypto/tls.(*Conn).Read(0xc000784a80, 0xc000261000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:1276 +0x165 bufio.(*Reader).Read(0xc0004fd740, 0xc00025e3b8, 0x9, 0x9, 0x77ceeb, 0xc0002c0c78, 0x415505) /jump/software/rhel7/golang-1.16/src/bufio/bufio.go:227 +0x222 io.ReadAtLeast(0x164e920, 0xc0004fd740, 0xc00025e3b8, 0x9, 0x9, 0x9, 0xc000264880, 0xe6145989b0c100, 0xc000264880) /jump/software/rhel7/golang-1.16/src/io/io.go:328 +0x87 io.ReadFull(...) /jump/software/rhel7/golang-1.16/src/io/io.go:347 net/http.http2readFrameHeader(0xc00025e3b8, 0x9, 0x9, 0x164e920, 0xc0004fd740, 0x0, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:1477 +0x89 net/http.(*http2Framer).ReadFrame(0xc00025e380, 0xc0004aec90, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:1735 +0xa5 net/http.(*http2clientConnReadLoop).run(0xc0002c0fa8, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:8322 +0xd8 net/http.(*http2ClientConn).readLoop(0xc000582900) /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:8244 +0x6f created by net/http.(*http2Transport).newClientConn /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:7208 +0x6c5 goroutine 179 [IO wait]: internal/poll.runtime_pollWait(0x7fffce054ff8, 0x72, 0xffffffffffffffff) /jump/software/rhel7/golang-1.16/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc000244698, 0x72, 0xb00, 0xbbf, 0xffffffffffffffff) /jump/software/rhel7/golang-1.16/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /jump/software/rhel7/golang-1.16/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc000244680, 0xc0002a3800, 0xbbf, 0xbbf, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/internal/poll/fd_unix.go:166 +0x1d5 net.(*netFD).Read(0xc000244680, 0xc0002a3800, 0xbbf, 0xbbf, 0xbba, 0xc0002a3800, 0x5) /jump/software/rhel7/golang-1.16/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc000010368, 0xc0002a3800, 0xbbf, 0xbbf, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/net/net.go:183 +0x91 crypto/tls.(*atLeastReader).Read(0xc000722ea0, 0xc0002a3800, 0xbbf, 0xbbf, 0xc000276000, 0xc000700000, 0x0) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:776 +0x63 bytes.(*Buffer).ReadFrom(0xc000784978, 0x164ea60, 0xc000722ea0, 0x419c85, 0x13962e0, 0x147ed80) /jump/software/rhel7/golang-1.16/src/bytes/buffer.go:204 +0xbe crypto/tls.(*Conn).readFromUntil(0xc000784700, 0x1650080, 0xc000010368, 0x5, 0xc000010368, 0x400) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:798 +0xf3 crypto/tls.(*Conn).readRecordOrCCS(0xc000784700, 0x0, 0x0, 0x1e) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:605 +0x115 crypto/tls.(*Conn).readRecord(...) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:573 crypto/tls.(*Conn).Read(0xc000784700, 0xc000149000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/crypto/tls/conn.go:1276 +0x165 bufio.(*Reader).Read(0xc00010fa40, 0xc00025e2d8, 0x9, 0x9, 0x77ceeb, 0xc0002c1c78, 0x415505) /jump/software/rhel7/golang-1.16/src/bufio/bufio.go:227 +0x222 io.ReadAtLeast(0x164e920, 0xc00010fa40, 0xc00025e2d8, 0x9, 0x9, 0x9, 0xc0005d3c80, 0xe6145989b0c100, 0xc0005d3c80) /jump/software/rhel7/golang-1.16/src/io/io.go:328 +0x87 io.ReadFull(...) /jump/software/rhel7/golang-1.16/src/io/io.go:347 net/http.http2readFrameHeader(0xc00025e2d8, 0x9, 0x9, 0x164e920, 0xc00010fa40, 0x0, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:1477 +0x89 net/http.(*http2Framer).ReadFrame(0xc00025e2a0, 0xc0001bf350, 0x0, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:1735 +0xa5 net/http.(*http2clientConnReadLoop).run(0xc0002c1fa8, 0x0, 0x0) /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:8322 +0xd8 net/http.(*http2ClientConn).readLoop(0xc000683680) /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:8244 +0x6f created by net/http.(*http2Transport).newClientConn /jump/software/rhel7/golang-1.16/src/net/http/h2_bundle.go:7208 +0x6c5