Test-infra: tide: segfault: concurrent map writes

Created on 7 Nov 2019 · 12Comments · Source: kubernetes/test-infra

fatal error: concurrent map writes

goroutine 78348 [running]:
runtime.throw(0x17f7e36, 0x15)
    GOROOT/src/runtime/panic.go:774 +0x72 fp=0xc002e02e48 sp=0xc002e02e18 pc=0x42f302
runtime.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc012682f00)
    GOROOT/src/runtime/map.go:680 +0x5aa fp=0xc002e02ed0 sp=0xc002e02e48 pc=0x40e83a
reflect.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc008ed0990)
    GOROOT/src/runtime/map.go:1329 +0x3f fp=0xc002e02f00 sp=0xc002e02ed0 pc=0x4103bf
github.com/modern-go/reflect2.(*UnsafeMapType).UnsafeSetIndex(...)
    external/com_github_modern_go_reflect2/unsafe_map.go:76
github.com/json-iterator/go.(*mapDecoder).Decode(0xc0000da410, 0xc007957550, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_map.go:180 +0x1a5 fp=0xc002e02f80 sp=0xc002e02f00 pc=0xa22ae5
github.com/json-iterator/go.(*placeholderDecoder).Decode(0xc008ff00e0, 0xc007957550, 0xc012682f00)
    external/com_github_json_iterator_go/reflect.go:319 +0x47 fp=0xc002e02fa8 sp=0xc002e02f80 pc=0xa1bd37
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc00240aca0, 0xc007957500, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03038 sp=0xc002e02fa8 pc=0xa31b86
github.com/json-iterator/go.(*sevenFieldsStructDecoder).Decode(0xc002c7c900, 0xc007957500, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:806 +0x325 fp=0xc002e030b0 sp=0xc002e03038 pc=0xa30ef5
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc00240ad20, 0xc0136d8518, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_optional.go:43 +0x93 fp=0xc002e030e0 sp=0xc002e030b0 pc=0xa29753
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc00240b3a0, 0xc0136d8500, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03170 sp=0xc002e030e0 pc=0xa31b86
github.com/json-iterator/go.(*nineFieldsStructDecoder).Decode(0xc008540280, 0xc0136d8500, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:926 +0x295 fp=0xc002e031e8 sp=0xc002e03170 pc=0xa31585
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc00240b560, 0xc0034b6ae0, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_optional.go:43 +0x93 fp=0xc002e03218 sp=0xc002e031e8 pc=0xa29753
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0021a0a00, 0xc0034b6a20, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e032a8 sp=0xc002e03218 pc=0xa31b86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0021a0b00, 0xc0034b6a20, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212 fp=0xc002e03320 sp=0xc002e032a8 pc=0xa2f942
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0021a0b00, 0xc0034b6a20, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f fp=0xc002e033a0 sp=0xc002e03320 pc=0xa2f50f
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0020e81e0, 0xc0034b6900, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03430 sp=0xc002e033a0 pc=0xa31b86
github.com/json-iterator/go.(*fiveFieldsStructDecoder).Decode(0xc00130d560, 0xc0034b6900, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:720 +0xb1 fp=0xc002e034a8 sp=0xc002e03430 pc=0xa30641
github.com/json-iterator/go.(*Iterator).ReadVal(0xc012682f00, 0x17b8ac0, 0xc0034b6900)
    external/com_github_json_iterator_go/reflect.go:78 +0xae fp=0xc002e03510 sp=0xc002e034a8 pc=0xa19cae
github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc000218000, 0xc00108e000, 0xcea, 0x1e00, 0x17b8ac0, 0xc0034b6900, 0x0, 0x0)
    external/com_github_json_iterator_go/config.go:348 +0xd2 fp=0xc002e03590 sp=0xc002e03510 pc=0xa10612
k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode(0xc00926cf80, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0x1, 0x0, 0x1a0e760, ...)
    external/io_k8s_apimachinery/pkg/runtime/serializer/json/json.go:255 +0x7d5 fp=0xc002e038d8 sp=0xc002e03590 pc=0xa7a665
k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode(0x1a0df60, 0xc00926cf80, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0xc00a99a200, 0xc00a99a100, ...)
    external/io_k8s_apimachinery/pkg/runtime/helper.go:252 +0x97 fp=0xc002e03970 sp=0xc002e038d8 pc=0x8eeb57
k8s.io/apimachinery/pkg/runtime.(*WithoutVersionDecoder).Decode(0xc007c5a300, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0xc002e03a38, 0x0, 0x0, ...)
    <autogenerated>:1 +0x9f fp=0xc002e039e8 sp=0xc002e03970 pc=0x8f9c1f
k8s.io/client-go/rest.Result.Into(0xc00108e000, 0xcea, 0x1e00, 0xc014c43a90, 0x10, 0x0, 0x0, 0xc9, 0x1a0fd20, 0xc007c5a300, ...)
    external/io_k8s_client_go/rest/request.go:1112 +0xba fp=0xc002e03ab0 sp=0xc002e039e8 pc=0xaa409a
sigs.k8s.io/controller-runtime/pkg/client.(*typedClient).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0x10, 0x154f600)
    external/io_k8s_sigs_controller_runtime/pkg/client/typed_client.go:48 +0x33e fp=0xc002e03bb0 sp=0xc002e03ab0 pc=0x109cd8e
sigs.k8s.io/controller-runtime/pkg/client.(*client).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0x32576709, 0x8ae6007b52fd)
    external/io_k8s_sigs_controller_runtime/pkg/client/client.go:123 +0x11b fp=0xc002e03c10 sp=0xc002e03bb0 pc=0x10999eb
sigs.k8s.io/controller-runtime/pkg/client.(*DelegatingClient).Create(0xc0053aa0c0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0xc007bdfa40, 0x38)
    <autogenerated>:1 +0x97 fp=0xc002e03c70 sp=0xc002e03c10 pc=0x10a2647
k8s.io/test-infra/prow/tide.(*Controller).trigger(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
    prow/tide/tide.go:1070 +0x6a0 fp=0xc002e04ef0 sp=0xc002e03c70 pc=0x1433540
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
    prow/tide/tide.go:1108 +0x2a7 fp=0xc002e05398 sp=0xc002e04ef0 pc=0x1433e97
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
    prow/tide/tide.go:1301 +0xfad fp=0xc002e059f0 sp=0xc002e05398 pc=0x143716d
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc01303b5f0)
    prow/tide/tide.go:377 +0x157 fp=0xc002e05f60 sp=0xc002e059f0 pc=0x143b597
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
    prow/tide/tide.go:430 +0x78 fp=0xc002e05fc8 sp=0xc002e05f60 pc=0x143b948
runtime.goexit()
    src/runtime/asm_amd64.s:1357 +0x1 fp=0xc002e05fd0 sp=0xc002e05fc8 pc=0x45eec1
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
    prow/tide/tide.go:427 +0x19b

goroutine 1 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x5)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
    GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0x1890718)
    prow/interrupts/interrupts.go:88 +0x80
k8s.io/test-infra/prow/interrupts.WaitForGracefulShutdown()
    prow/interrupts/interrupts.go:100 +0x3f
main.main()
    prow/cmd/tide/main.go:217 +0x138f

goroutine 6 [syscall, 259 minutes]:
os/signal.signal_recv(0x0)
    GOROOT/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
    GOROOT/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
    GOROOT/src/os/signal/signal_unix.go:29 +0x41

goroutine 7 [chan receive, 259 minutes]:
k8s.io/test-infra/prow/interrupts.handleInterrupt()
    prow/interrupts/interrupts.go:62 +0x84
created by k8s.io/test-infra/prow/interrupts.init.0
    prow/interrupts/interrupts.go:41 +0xb6

goroutine 18 [chan receive]:
k8s.io/klog.(*loggingT).flushDaemon(0x257c2c0)
    external/io_k8s_klog/klog.go:1010 +0x8b
created by k8s.io/klog.init.0
    external/io_k8s_klog/klog.go:411 +0xd6

goroutine 38 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc000192870)
    external/io_opencensus_go/stats/view/worker.go:154 +0x100
created by go.opencensus.io/stats/view.init.0
    external/io_opencensus_go/stats/view/worker.go:32 +0x57

goroutine 83 [IO wait, 259 minutes]:
internal/poll.runtime_pollWait(0x7f5da1a1eea8, 0x72, 0x0)
    GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0001be198, 0x72, 0x0, 0x0, 0x17e5a26)
    GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0001be180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    GOROOT/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc0001be180, 0xc0000c7d20, 0xc00020ee00, 0x7f5da3ccb008)
    GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00019e640, 0xc0000c7d50, 0x40ca48, 0x30)
    GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00019e640, 0x16d8ba0, 0xc0003dc360, 0x15bd140, 0x25601d0)
    GOROOT/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0003fa000, 0x1a46180, 0xc00019e640, 0x0, 0x0)
    GOROOT/src/net/http/server.go:2896 +0x286
net/http.(*Server).ListenAndServe(0xc0003fa000, 0xc000511798, 0x40576f)
    GOROOT/src/net/http/server.go:2825 +0xb7
k8s.io/test-infra/prow/interrupts.ListenAndServe.func1(0xc0003fa000)
    prow/interrupts/interrupts.go:156 +0x6e
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
    prow/interrupts/interrupts.go:154 +0x62

goroutine 84 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x0)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
    GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc0003ec180)
    prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
    prow/interrupts/interrupts.go:159 +0x92

goroutine 56 [chan receive]:
k8s.io/test-infra/prow/config.(*Agent).Start.func1(0x7ffd505f479e, 0x17, 0x7ffd505f47fe, 0xf, 0xc0003e6700)
    prow/config/agent.go:58 +0xb1
created by k8s.io/test-infra/prow/config.(*Agent).Start
    prow/config/agent.go:52 +0xd2

goroutine 57 [chan receive]:
k8s.io/test-infra/prow/config/secret.(*Agent).reloadSecret(0xc002255c00, 0x7ffd505f47da, 0x11)
    prow/config/secret/agent.go:68 +0x122
created by k8s.io/test-infra/prow/config/secret.(*Agent).Start
    prow/config/secret/agent.go:52 +0x103

goroutine 58 [chan receive]:
k8s.io/test-infra/prow/github.(*client).Throttle.func1(0xc00440a820, 0xc0044708a0)
    prow/github/client.go:405 +0x59
created by k8s.io/test-infra/prow/github.(*client).Throttle
    prow/github/client.go:403 +0x26b

goroutine 59 [chan receive]:
k8s.io/test-infra/prow/github.(*client).Throttle.func1(0xc00440a8c0, 0xc0044709c0)
    prow/github/client.go:405 +0x59
created by k8s.io/test-infra/prow/github.(*client).Throttle
    prow/github/client.go:403 +0x26b

goroutine 233 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1ed08, 0x72, 0xffffffffffffffff)
    GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc008dc8798, 0x72, 0xff00, 0xffd8, 0xffffffffffffffff)
    GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc008dc8780, 0xc001dd4000, 0xffd8, 0xffd8, 0x0, 0x0, 0x0)
    GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc008dc8780, 0xc001dd4000, 0xffd8, 0xffd8, 0x203002, 0x0, 0xffd3)
    GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc008dd2000, 0xc001dd4000, 0xffd8, 0xffd8, 0x0, 0x0, 0x0)
    GOROOT/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc009d05ac0, 0xc001dd4000, 0xffd8, 0xffd8, 0xd22, 0x1a0c4e0, 0xc00016e970)
    GOROOT/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc000130958, 0x1a0c2e0, 0xc009d05ac0, 0x40a1b5, 0x15f4ca0, 0x177b480)
    GOROOT/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc000130700, 0x1a0e6c0, 0xc008dd2000, 0x5, 0xc008dd2000, 0x203002)
    GOROOT/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc000130700, 0x0, 0x0, 0xc009d05aa0)
    GOROOT/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
    GOROOT/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc000130700, 0xc008e11000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    GOROOT/src/crypto/tls/conn.go:1255 +0x161
bufio.(*Reader).Read(0xc008dd6c60, 0xc008df21f8, 0x9, 0x9, 0x3d67, 0xd04, 0x0)
    GOROOT/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x1a0c100, 0xc008dd6c60, 0xc008df21f8, 0x9, 0x9, 0x9, 0x0, 0x0, 0xc00016ef78)
    GOROOT/src/io/io.go:310 +0x87
io.ReadFull(...)
    GOROOT/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc008df21f8, 0x9, 0x9, 0x1a0c100, 0xc008dd6c60, 0x0, 0x0, 0xc006ec1c80, 0x0)
    external/org_golang_x_net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc008df21c0, 0xc006ec1c80, 0x0, 0x0, 0x0)
    external/org_golang_x_net/http2/frame.go:492 +0xa1
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00016efb8, 0x0, 0x0)
    external/org_golang_x_net/http2/transport.go:1710 +0x8c
golang.org/x/net/http2.(*ClientConn).readLoop(0xc008dd4180)
    external/org_golang_x_net/http2/transport.go:1638 +0xa3
created by golang.org/x/net/http2.(*Transport).newClientConn
    external/org_golang_x_net/http2/transport.go:673 +0x62f

goroutine 219 [chan receive, 259 minutes]:
k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0xc008da6fc0)
    external/io_k8s_apimachinery/pkg/watch/mux.go:207 +0x66
created by k8s.io/apimachinery/pkg/watch.NewBroadcaster
    external/io_k8s_apimachinery/pkg/watch/mux.go:75 +0xcc

goroutine 398 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x2)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
    GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc008f72720)
    prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.OnInterrupt
    prow/interrupts/interrupts.go:235 +0x8e

goroutine 90 [chan receive, 19 minutes]:
k8s.io/test-infra/prow/tide.(*statusController).save(0xc00021a6c0, 0xc00440a0a0)
    prow/tide/status.go:385 +0x7c
created by k8s.io/test-infra/prow/tide.(*statusController).run
    prow/tide/status.go:417 +0xa7

goroutine 465 [chan receive, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start(0xc0092c60e0, 0xc008f8ede0)
    external/io_k8s_sigs_controller_runtime/pkg/cache/internal/informers_map.go:132 +0x4c
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start
    external/io_k8s_sigs_controller_runtime/pkg/cache/internal/deleg_map.go:61 +0x80

goroutine 470 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0087840a0, 0x1a1d800, 0xc002363e80, 0xc00908fc00, 0xc001d0e540, 0xc008f8ede0, 0x0, 0x0)
    external/io_k8s_client_go/tools/cache/reflector.go:329 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0087840a0, 0xc008f8ede0, 0x0, 0x0)
    external/io_k8s_client_go/tools/cache/reflector.go:300 +0x833
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
    external/io_k8s_client_go/tools/cache/reflector.go:124 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc008e666f0)
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00908fef0, 0x3b9aca00, 0x0, 0x1, 0xc008f8ede0)
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0087840a0, 0xc008f8ede0)
    external/io_k8s_client_go/tools/cache/reflector.go:123 +0x169
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00168c020, 0xc004bb2120)
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 477 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x4)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
    GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc00505e0a0)
    prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
    prow/interrupts/interrupts.go:159 +0x92

goroutine 156 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x6)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
    GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc0084b3240)
    prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.Tick
    prow/interrupts/interrupts.go:218 +0x10b

goroutine 394 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x1)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
    GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc00454de60)
    prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.Run
    prow/interrupts/interrupts.go:145 +0xd2

goroutine 476 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e9c8, 0x72, 0x0)
    GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc008f0a498, 0x72, 0x0, 0x0, 0x17e5a26)
    GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc008f0a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    GOROOT/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc008f0a480, 0xc0000c1d50, 0x6e27e4, 0xc008412180)
    GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00505e0c0, 0x5dc43c1a, 0xc0000c1d50, 0x486416)
    GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00505e0c0, 0xc0000c1da0, 0x18, 0xc008e79200, 0x6e1cfe)
    GOROOT/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0084120e0, 0x1a46180, 0xc00505e0c0, 0x0, 0x0)
    GOROOT/src/net/http/server.go:2896 +0x286
net/http.(*Server).ListenAndServe(0xc0084120e0, 0x0, 0x0)
    GOROOT/src/net/http/server.go:2825 +0xb7
k8s.io/test-infra/prow/interrupts.ListenAndServe.func1(0xc0084120e0)
    prow/interrupts/interrupts.go:156 +0x6e
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
    prow/interrupts/interrupts.go:154 +0x62

goroutine 482 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x3)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
    GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc0050ca500)
    prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
    prow/interrupts/interrupts.go:159 +0x92

goroutine 473 [select, 259 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0087840a0, 0xc008f8ede0, 0xc00f1bc480, 0xc001d0e540)
    external/io_k8s_client_go/tools/cache/reflector.go:235 +0x178
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
    external/io_k8s_client_go/tools/cache/reflector.go:229 +0x260

goroutine 368 [chan receive, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start(0xc0092c6000, 0xc008f8ede0)
    external/io_k8s_sigs_controller_runtime/pkg/cache/internal/informers_map.go:132 +0x4c
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start
    external/io_k8s_sigs_controller_runtime/pkg/cache/internal/deleg_map.go:60 +0x50

goroutine 481 [IO wait, 259 minutes]:
internal/poll.runtime_pollWait(0x7f5da1a1edd8, 0x72, 0x0)
    GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00842af18, 0x72, 0x0, 0x0, 0x17e5a26)
    GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc00842af00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    GOROOT/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc00842af00, 0xc008e75d50, 0x6e27e4, 0xc008412340)
    GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0053dbb80, 0x5dc3ff7e, 0xc008e75d50, 0x486416)
    GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc0053dbb80, 0xc008e75da0, 0x18, 0xc008e79380, 0x6e1cfe)
    GOROOT/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0084122a0, 0x1a46180, 0xc0053dbb80, 0x0, 0x0)
    GOROOT/src/net/http/server.go:2896 +0x286
net/http.(*Server).ListenAndServe(0xc0084122a0, 0xc0090a1798, 0x40576f)
    GOROOT/src/net/http/server.go:2825 +0xb7
k8s.io/test-infra/prow/interrupts.ListenAndServe.func1(0xc0084122a0)
    prow/interrupts/interrupts.go:156 +0x6e
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
    prow/interrupts/interrupts.go:154 +0x62

goroutine 155 [semacquire]:
sync.runtime_Semacquire(0xc01373a208)
    GOROOT/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc01373a200)
    GOROOT/src/sync/waitgroup.go:130 +0x64
k8s.io/test-infra/prow/tide.subpoolsInParallel(0xf, 0xc00701a090, 0xc007ae13e0)
    prow/tide/tide.go:434 +0x1be
k8s.io/test-infra/prow/tide.(*Controller).Sync(0xc008509a80, 0x0, 0x0)
    prow/tide/tide.go:373 +0xbcb
main.sync(0xc008509a80)
    prow/cmd/tide/main.go:220 +0x2f
main.main.func3()
    prow/cmd/tide/main.go:213 +0x2a
k8s.io/test-infra/prow/interrupts.Tick.func1(0xc0084b3230, 0xc0020b7b60, 0xc0084b3220, 0xc00faa41c0)
    prow/interrupts/interrupts.go:210 +0x80
created by k8s.io/test-infra/prow/interrupts.Tick
    prow/interrupts/interrupts.go:196 +0xbd

goroutine 421 [chan receive, 259 minutes]:
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x1a1d840, 0xc0053bc420, 0xc0053bc3f0)
    external/io_k8s_client_go/tools/record/event.go:268 +0xc5
created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher
    external/io_k8s_client_go/tools/record/event.go:266 +0x6e

goroutine 468 [chan receive, 259 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc003d327e0, 0xc004df4b40)
    external/io_k8s_client_go/tools/cache/shared_informer.go:478 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00168c000, 0xc004bb20e0)
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 422 [chan receive, 259 minutes]:
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x1a1d840, 0xc0053bc450, 0xc00924f280)
    external/io_k8s_client_go/tools/record/event.go:268 +0xc5
created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher
    external/io_k8s_client_go/tools/record/event.go:266 +0x6e

goroutine 469 [chan receive, 259 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc008f8ede0, 0xc008356000)
    external/io_k8s_client_go/tools/cache/controller.go:103 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
    external/io_k8s_client_go/tools/cache/controller.go:102 +0xae

goroutine 392 [select]:
net/http.(*persistConn).roundTrip(0xc00e57e6c0, 0xc013d1e660, 0x0, 0x0, 0x0)
    GOROOT/src/net/http/transport.go:2417 +0x770
net/http.(*Transport).roundTrip(0x256bba0, 0xc0055ce600, 0xc0021d6258, 0xc00446d8b0, 0xc013d1e630)
    GOROOT/src/net/http/transport.go:535 +0x8f8
net/http.(*Transport).RoundTrip(0x256bba0, 0xc0055ce600, 0xc0055ce600, 0x0, 0x69961b)
    GOROOT/src/net/http/roundtrip.go:17 +0x35
golang.org/x/oauth2.(*Transport).RoundTrip(0xc00446d890, 0xc0055ce500, 0x0, 0x0, 0x0)
    external/org_golang_x_oauth2/transport.go:56 +0x124
net/http.send(0xc0055ce400, 0x1a0d460, 0xc00446d890, 0xbf692d31b22381b2, 0xe6d7101ce2a, 0x257be20, 0xc000314090, 0xbf692d31b22381b2, 0x1, 0x0)
    GOROOT/src/net/http/client.go:250 +0x43a
net/http.(*Client).send(0xc00446d8c0, 0xc0055ce400, 0xbf692d31b22381b2, 0xe6d7101ce2a, 0x257be20, 0xc000314090, 0x0, 0x1, 0xc0112d8080)
    GOROOT/src/net/http/client.go:174 +0xfa
net/http.(*Client).do(0xc00446d8c0, 0xc0055ce400, 0x0, 0x0, 0x0)
    GOROOT/src/net/http/client.go:641 +0x3ce
net/http.(*Client).Do(...)
    GOROOT/src/net/http/client.go:509
github.com/shurcooL/go/ctxhttp.Do(0x1a4b9c0, 0xc000046020, 0xc00446d8c0, 0xc0055ce200, 0xc0046bc608, 0x16, 0x1a0c140)
    external/com_github_shurcool_go/ctxhttp/ctxhttp.go:34 +0xef
github.com/shurcooL/go/ctxhttp.Post(0x1a4b9c0, 0xc000046020, 0xc00446d8c0, 0x7ffd505f4872, 0x16, 0x17f111c, 0x10, 0x1a0c140, 0xc013d1e4b0, 0x140, ...)
    external/com_github_shurcool_go/ctxhttp/ctxhttp.go:72 +0x1af
github.com/shurcooL/graphql.(*Client).do(0xc002255cc0, 0x1a4b9c0, 0xc000046020, 0x0, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0x0, 0x0)
    external/com_github_shurcool_graphql/graphql.go:68 +0x1d3
github.com/shurcooL/graphql.(*Client).Query(...)
    external/com_github_shurcool_graphql/graphql.go:37
github.com/shurcooL/githubv4.(*Client).Query(0xc0001a1c00, 0x1a4b9c0, 0xc000046020, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0xc013d1e0f0, 0x0)
    external/com_github_shurcool_githubv4/githubv4.go:42 +0x68
k8s.io/test-infra/prow/github.(*throttler).Query(0xc0003fa230, 0x1a4b9c0, 0xc000046020, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0x0, 0x0)
    prow/github/client.go:378 +0xf6
k8s.io/test-infra/prow/github.(*client).Query(0xc003f6b600, 0x1a4b9c0, 0xc000046020, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0x0, 0x0)
    prow/github/client.go:2459 +0x70
k8s.io/test-infra/prow/tide.search(0xc0021d7528, 0xc012e7c1c0, 0xc001d54500, 0x22b5, 0x0, 0xed556329a, 0x0, 0xbf692ce6b21e153b, 0xe279797a9b6, 0x257be20, ...)
    prow/tide/search.go:64 +0x8c6
k8s.io/test-infra/prow/tide.(*statusController).search(0xc00021a6c0, 0xe2797920c70, 0x257be20, 0xc0114ae180)
    prow/tide/status.go:482 +0x544
k8s.io/test-infra/prow/tide.(*statusController).sync(0xc00021a6c0, 0xc007ae1380, 0xc01325ac90, 0xc01325acc0)
    prow/tide/status.go:461 +0x91
k8s.io/test-infra/prow/tide.(*statusController).waitSync(0xc00021a6c0)
    prow/tide/status.go:443 +0x1c1
k8s.io/test-infra/prow/tide.(*statusController).run(0xc00021a6c0)
    prow/tide/status.go:424 +0xb7
created by k8s.io/test-infra/prow/tide.NewController
    prow/tide/tide.go:232 +0x3e4

goroutine 466 [sync.Cond.Wait]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0004ab0a8, 0x2694)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0004ab098)
    GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0004ab080, 0xc004504060, 0x0, 0x0, 0x0, 0x0)
    external/io_k8s_client_go/tools/cache/delta_fifo.go:423 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc008356000)
    external/io_k8s_client_go/tools/cache/controller.go:150 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000173e00)
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0090dbe00, 0x3b9aca00, 0x0, 0xc001c26901, 0xc008f8ede0)
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
    external/io_k8s_apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc008356000, 0xc008f8ede0)
    external/io_k8s_client_go/tools/cache/controller.go:124 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00875e090, 0xc008f8ede0)
    external/io_k8s_client_go/tools/cache/shared_informer.go:280 +0x47f
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start.func1
    external/io_k8s_sigs_controller_runtime/pkg/cache/internal/informers_map.go:126 +0x122

goroutine 393 [select, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start(0xc0092eec80, 0xc004df4240, 0x0, 0x0)
    external/io_k8s_sigs_controller_runtime/pkg/manager/internal.go:287 +0x170
main.main.func1(0x1a4b980, 0xc007e76ac0)
    prow/cmd/tide/main.go:178 +0x5b
k8s.io/test-infra/prow/interrupts.Run.func1(0xc004b78260, 0x1a4b980, 0xc007e76ac0)
    prow/interrupts/interrupts.go:142 +0x76
created by k8s.io/test-infra/prow/interrupts.Run
    prow/interrupts/interrupts.go:140 +0xb0

goroutine 365 [chan receive, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start(0xc0091fcde0, 0xc008f8ede0, 0x0, 0x0)
    external/io_k8s_sigs_controller_runtime/pkg/cache/internal/deleg_map.go:62 +0x97
sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).waitForCache.func1(0xc0092eec80)
    external/io_k8s_sigs_controller_runtime/pkg/manager/internal.go:343 +0x39
created by sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).waitForCache
    external/io_k8s_sigs_controller_runtime/pkg/manager/internal.go:342 +0x5a

goroutine 91 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1ea98, 0x72, 0xffffffffffffffff)
    GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc006372018, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc006372000, 0xc00045d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc006372000, 0xc00045d000, 0x1000, 0x1000, 0xc0008c7800, 0x7ea0fb, 0xc0083b82a0)
    GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000314008, 0xc00045d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    GOROOT/src/net/net.go:184 +0x68
net/http.(*connReader).Read(0xc0045ba120, 0xc00045d000, 0x1000, 0x1000, 0xc0008c78b8, 0xc0008c78a8, 0x48e00203004)
    GOROOT/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc001e12000)
    GOROOT/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).ReadSlice(0xc001e12000, 0xa, 0x7f5da0bb5878, 0xc0008c79a8, 0x40c1d6, 0xc00a88e500, 0x100)
    GOROOT/src/bufio/bufio.go:359 +0x3d
bufio.(*Reader).ReadLine(0xc001e12000, 0xc0008c79b0, 0xc0096ae000, 0x7f5da3cccb28, 0x0, 0x40ca48, 0x30)
    GOROOT/src/bufio/bufio.go:388 +0x34
net/textproto.(*Reader).readLineSlice(0xc013616240, 0xc00a88e500, 0xc006372000, 0x0, 0x0, 0x43112c)
    GOROOT/src/net/textproto/reader.go:57 +0x6c
net/textproto.(*Reader).ReadLine(...)
    GOROOT/src/net/textproto/reader.go:38
net/http.readRequest(0xc001e12000, 0x0, 0xc00a88e500, 0x0, 0x0)
    GOROOT/src/net/http/request.go:1012 +0x92
net/http.(*conn).readRequest(0xc00883a000, 0x1a4b980, 0xc009172040, 0x0, 0x0, 0x0)
    GOROOT/src/net/http/server.go:965 +0x15f
net/http.(*conn).serve(0xc00883a000, 0x1a4b980, 0xc009172040)
    GOROOT/src/net/http/server.go:1817 +0x6d4
created by net/http.(*Server).Serve
    GOROOT/src/net/http/server.go:2927 +0x38e

goroutine 149 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e8f8, 0x72, 0xffffffffffffffff)
    GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0093a2518, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0093a2500, 0xc009702000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0093a2500, 0xc009702000, 0x1000, 0x1000, 0xc0090ef800, 0x7ea0fb, 0xc0084a5180)
    GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00418c1a0, 0xc009702000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    GOROOT/src/net/net.go:184 +0x68
net/http.(*connReader).Read(0xc0045bb5c0, 0xc009702000, 0x1000, 0x1000, 0xc0090ef8b8, 0xc0090ef8a8, 0x48e00203004)
    GOROOT/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc001e121e0)
    GOROOT/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).ReadSlice(0xc001e121e0, 0xa, 0x7f5da123ce30, 0xc0090ef9a8, 0x40c1d6, 0xc006895a00, 0x100)
    GOROOT/src/bufio/bufio.go:359 +0x3d
bufio.(*Reader).ReadLine(0xc001e121e0, 0xc0090ef9b0, 0xc0004b1880, 0x7f5da3ccb008, 0x0, 0x40ca48, 0x30)
    GOROOT/src/bufio/bufio.go:388 +0x34
net/textproto.(*Reader).readLineSlice(0xc012ccd1d0, 0xc006895a00, 0xc0093a2500, 0x0, 0x0, 0x43112c)
    GOROOT/src/net/textproto/reader.go:57 +0x6c
net/textproto.(*Reader).ReadLine(...)
    GOROOT/src/net/textproto/reader.go:38
net/http.readRequest(0xc001e121e0, 0x0, 0xc006895a00, 0x0, 0x0)
    GOROOT/src/net/http/request.go:1012 +0x92
net/http.(*conn).readRequest(0xc00883a280, 0x1a4b980, 0xc007bee200, 0x0, 0x0, 0x0)
    GOROOT/src/net/http/server.go:965 +0x15f
net/http.(*conn).serve(0xc00883a280, 0x1a4b980, 0xc007bee200)
    GOROOT/src/net/http/server.go:1817 +0x6d4
created by net/http.(*Server).Serve
    GOROOT/src/net/http/server.go:2927 +0x38e

goroutine 77963 [select]:
net/http.(*persistConn).writeLoop(0xc00e57e6c0)
    GOROOT/src/net/http/transport.go:2196 +0x123
created by net/http.(*Transport).dialConn
    GOROOT/src/net/http/transport.go:1567 +0xb23

goroutine 77474 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f5da1a1e278, 0x72, 0xffffffffffffffff)
    GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc014e46518, 0x72, 0x1300, 0x1350, 0xffffffffffffffff)
    GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc014e46500, 0xc0005e6a00, 0x1350, 0x1350, 0x0, 0x0, 0x0)
    GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc014e46500, 0xc0005e6a00, 0x1350, 0x1350, 0x203000, 0x437ef8, 0xc008362a80)
    GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc008dd22f8, 0xc0005e6a00, 0x1350, 0x1350, 0x0, 0x0, 0x0)
    GOROOT/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc002d9a400, 0xc0005e6a00, 0x1350, 0x1350, 0x27, 0x0, 0xc008e76970)
    GOROOT/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc000151058, 0x1a0c2e0, 0xc002d9a400, 0x40a1b5, 0x15f4ca0, 0x177b480)
    GOROOT/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc000150e00, 0x1a0e6c0, 0xc008dd22f8, 0x5, 0xc008dd22f8, 0x5)
    GOROOT/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc000150e00, 0x0, 0x0, 0x0)
    GOROOT/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
    GOROOT/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc000150e00, 0xc0001b4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    GOROOT/src/crypto/tls/conn.go:1255 +0x161
bufio.(*Reader).Read(0xc005bda480, 0xc0003fa578, 0x9, 0x9, 0x11, 0x0, 0x0)
    GOROOT/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x1a0c100, 0xc005bda480, 0xc0003fa578, 0x9, 0x9, 0x9, 0xc008e76d20, 0xc008e76df0, 0x6c752a)
    GOROOT/src/io/io.go:310 +0x87
io.ReadFull(...)
    GOROOT/src/io/io.go:329
net/http.http2readFrameHeader(0xc0003fa578, 0x9, 0x9, 0x1a0c100, 0xc005bda480, 0x0, 0x0, 0x0, 0x0)
    GOROOT/src/net/http/h2_bundle.go:1477 +0x87
net/http.(*http2Framer).ReadFrame(0xc0003fa540, 0xc005e50bc0, 0x0, 0x0, 0x0)
    GOROOT/src/net/http/h2_bundle.go:1735 +0xa1
net/http.(*http2clientConnReadLoop).run(0xc008e76fb8, 0x1000000000001, 0x0)
    GOROOT/src/net/http/h2_bundle.go:8175 +0x8e
net/http.(*http2ClientConn).readLoop(0xc00fe9c180)
    GOROOT/src/net/http/h2_bundle.go:8103 +0xa3
created by net/http.(*http2Transport).newClientConn
    GOROOT/src/net/http/h2_bundle.go:7162 +0x62f

goroutine 78350 [syscall]:
syscall.Syscall6(0xf7, 0x1, 0x8bb, 0xc0021da338, 0x1000004, 0x0, 0x0, 0x4d7b01, 0xc00bd2df80, 0xc0021da378)
    src/syscall/asm_linux_amd64.s:44 +0x5
os.(*Process).blockUntilWaitable(0xc006cbfdd0, 0x203004, 0x160, 0x1)
    GOROOT/src/os/wait_waitid.go:31 +0x98
os.(*Process).wait(0xc006cbfdd0, 0x1890c10, 0x1890c18, 0x1890c08)
    GOROOT/src/os/exec_unix.go:22 +0x39
os.(*Process).Wait(...)
    GOROOT/src/os/exec.go:125
os/exec.(*Cmd).Wait(0xc0046bcb00, 0x0, 0x0)
    GOROOT/src/os/exec/exec.go:501 +0x60
os/exec.(*Cmd).Run(0xc0046bcb00, 0xc013aa8a20, 0xc0046bcb00)
    GOROOT/src/os/exec/exec.go:341 +0x5c
os/exec.(*Cmd).CombinedOutput(0xc0046bcb00, 0xc, 0xc008703e10, 0x1, 0x1, 0xc0046bcb00)
    GOROOT/src/os/exec/exec.go:561 +0x91
k8s.io/test-infra/prow/git.retryCmd(0xc003d33b90, 0xc006cbfcb0, 0x26, 0xc002684bc0, 0xc, 0xc008703e10, 0x1, 0x1, 0xc0021da6c0, 0x40c1d6, ...)
    prow/git/git.go:387 +0x22c
k8s.io/test-infra/prow/git.(*Client).Clone(0xc0035e6000, 0xc008b5da80, 0x10, 0x0, 0x0, 0x0)
    prow/git/git.go:163 +0x3c2
k8s.io/test-infra/prow/tide.(*Controller).pickBatch(0xc008509a80, 0xc016018540, 0xc006654820, 0x9, 0xc0066546f8, 0x6, 0xc0066542d0, 0x6, 0xc0061e5440, 0x28, ...)
    prow/tide/tide.go:814 +0x4f3
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc016018540, 0xc006654820, 0x9, 0xc0066546f8, 0x6, 0xc0066542d0, 0x6, 0xc0061e5440, 0x28, ...)
    prow/tide/tide.go:1097 +0x403
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc016018540, 0xc006654820, 0x9, 0xc0066546f8, 0x6, 0xc0066542d0, 0x6, 0xc0061e5440, 0x28, ...)
    prow/tide/tide.go:1301 +0xfad
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc0059c8900)
    prow/tide/tide.go:377 +0x157
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
    prow/tide/tide.go:430 +0x78
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
    prow/tide/tide.go:427 +0x19b

goroutine 78361 [runnable]:
github.com/json-iterator/go.(*sliceDecoder).Decode(0xc004bdede0, 0xc0108c69f0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_slice.go:59 +0x17f
github.com/json-iterator/go.(*placeholderDecoder).Decode(0xc008e35a80, 0xc0108c69f0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect.go:319 +0x47
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0029e1a00, 0xc0108c69c0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0029aa1c0, 0xc0108c69c0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0029aa1c0, 0xc0108c69c0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc0029aa1e0, 0xc0034b6dc8, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_optional.go:43 +0x93
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0021a0940, 0xc0034b6d20, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0021a0b00, 0xc0034b6d20, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0021a0b00, 0xc0034b6d20, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0020e81e0, 0xc0034b6c00, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*fiveFieldsStructDecoder).Decode(0xc00130d560, 0xc0034b6c00, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:720 +0xb1
github.com/json-iterator/go.(*Iterator).ReadVal(0xc0134e1200, 0x17b8ac0, 0xc0034b6c00)
    external/com_github_json_iterator_go/reflect.go:78 +0xae
github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc000218000, 0xc00002c000, 0xafd, 0xe00, 0x17b8ac0, 0xc0034b6c00, 0x0, 0x0)
    external/com_github_json_iterator_go/config.go:348 +0xd2
k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode(0xc00926cf80, 0xc00002c000, 0xafd, 0xe00, 0x0, 0x1a1e500, 0xc0034b6c00, 0x1, 0x0, 0x1a0e760, ...)
    external/io_k8s_apimachinery/pkg/runtime/serializer/json/json.go:255 +0x7d5
k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode(0x1a0df60, 0xc00926cf80, 0xc00002c000, 0xafd, 0xe00, 0x0, 0x1a1e500, 0xc0034b6c00, 0xc00a99a600, 0xc00a99a500, ...)
    external/io_k8s_apimachinery/pkg/runtime/helper.go:252 +0x97
k8s.io/client-go/rest.Result.Into(0xc00002c000, 0xafd, 0xe00, 0xc014c43a90, 0x10, 0x0, 0x0, 0xc9, 0x1a0fd20, 0xc007c5a300, ...)
    external/io_k8s_client_go/rest/request.go:1112 +0xba
sigs.k8s.io/controller-runtime/pkg/client.(*typedClient).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6c00, 0x0, 0x0, 0x0, 0x10, 0x154f600)
    external/io_k8s_sigs_controller_runtime/pkg/client/typed_client.go:48 +0x33e
sigs.k8s.io/controller-runtime/pkg/client.(*client).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6c00, 0x0, 0x0, 0x0, 0x32c7deb4, 0x8ae600ebca7e)
    external/io_k8s_sigs_controller_runtime/pkg/client/client.go:123 +0x11b
k8s.io/test-infra/prow/tide.(*Controller).trigger(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
    prow/tide/tide.go:1070 +0x6a0
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
    prow/tide/tide.go:1108 +0x2a7
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
    prow/tide/tide.go:1301 +0xfad
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc0134725a0)
    prow/tide/tide.go:377 +0x157
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
    prow/tide/tide.go:430 +0x78
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
    prow/tide/tide.go:427 +0x19b

goroutine 77964 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e348, 0x72, 0xffffffffffffffff)
    GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00bd2ded8, 0x72, 0x201, 0x200, 0xffffffffffffffff)
    GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00bd2dec0, 0xc009636200, 0x200, 0x200, 0x0, 0x0, 0x0)
    GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
os.(*File).read(...)
    GOROOT/src/os/file_unix.go:259
os.(*File).Read(0xc000314148, 0xc009636200, 0x200, 0x200, 0x19f0350, 0xc0091e4eb0, 0xc0091e4ea0)
    GOROOT/src/os/file.go:116 +0x71
bytes.(*Buffer).ReadFrom(0xc013aa8a20, 0x1a0eb00, 0xc000314148, 0x7f5da0e6c550, 0xc013aa8a20, 0x19f0301)
    GOROOT/src/bytes/buffer.go:204 +0xb4
io.copyBuffer(0x1a0c160, 0xc013aa8a20, 0x1a0eb00, 0xc000314148, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0)
    GOROOT/src/io/io.go:388 +0x2ed
io.Copy(...)
    GOROOT/src/io/io.go:364
os/exec.(*Cmd).writerDescriptor.func1(0x0, 0x0)
    GOROOT/src/os/exec/exec.go:311 +0x63
os/exec.(*Cmd).Start.func1(0xc0046bcb00, 0xc00a80f5e0)
    GOROOT/src/os/exec/exec.go:435 +0x27
created by os/exec.(*Cmd).Start
    GOROOT/src/os/exec/exec.go:434 +0x608

goroutine 77957 [select]:
net/http.setRequestCancel.func3(0x0, 0xc013d1e600, 0xc003896190, 0xc013ba821c, 0xc014568240)
    GOROOT/src/net/http/client.go:321 +0xcf
created by net/http.setRequestCancel
    GOROOT/src/net/http/client.go:320 +0x24b

goroutine 75422 [runnable]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00a9b2680, 0xc00000012d)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00a9b2670)
    GOROOT/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc00a9b2668, 0xc00106c001, 0x3dff, 0x3dff, 0x0, 0x0, 0x0)
    external/org_golang_x_net/http2/pipe.go:64 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc00a9b2640, 0xc00106c001, 0x3dff, 0x3dff, 0x0, 0x0, 0x0)
    external/org_golang_x_net/http2/transport.go:1983 +0xac
encoding/json.(*Decoder).refill(0xc00bd2a2c0, 0xc004a5db0a, 0x9)
    GOROOT/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc00bd2a2c0, 0x0, 0x0, 0x15c0b00)
    GOROOT/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc00bd2a2c0, 0x15dd6c0, 0xc004a5dbe0, 0x0, 0x0)
    GOROOT/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0137e8c90, 0xc002306000, 0x4000, 0x4a80, 0x1a1e500, 0x0, 0x38)
    external/io_k8s_apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc008324e60, 0x0, 0x1a1d340, 0xc00232f580, 0x0, 0x0, 0x0, 0xc00291f1f8, 0x458e30)
    external/io_k8s_apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00d270100, 0xc004497fa8, 0x8, 0x1a1e500, 0xc00cef9200, 0x0, 0x0)
    external/io_k8s_client_go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc002363e80)
    external/io_k8s_apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
    external/io_k8s_apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 77962 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e758, 0x72, 0xffffffffffffffff)
    GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00137c618, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00137c600, 0xc000350000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00137c600, 0xc000350000, 0x1000, 0x1000, 0x1a0c4e0, 0xc00008e050, 0x259ae01)
    GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0003140c8, 0xc000350000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    GOROOT/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00e57e6c0, 0xc000350000, 0x1000, 0x1000, 0x404c4d, 0x60, 0x0)
    GOROOT/src/net/http/transport.go:1744 +0x75
bufio.(*Reader).fill(0xc00bd2c840)
    GOROOT/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00bd2c840, 0x1, 0xc014568900, 0xc0091e4c98, 0x5bbe8d, 0x1652aa0, 0x7)
    GOROOT/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00e57e6c0)
    GOROOT/src/net/http/transport.go:1897 +0x1d6
created by net/http.(*Transport).dialConn
    GOROOT/src/net/http/transport.go:1566 +0xafe

/area prow/tide
/cc @cjwagner @alvaroaleman @fejta
/kind oncall-hostlist

areprotide kinbug

Source

stevekuznetsov

Most helpful comment

https://github.com/kubernetes/test-infra/blob/7bf2a7ae54cc133daa75a5269fcb6510f8b9a694/prow/apis/prowjobs/v1/types.go#L141-L156

specFromJobBase() is not thread safe:
https://github.com/kubernetes/test-infra/blob/7bf2a7ae54cc133daa75a5269fcb6510f8b9a694/prow/pjutil/pjutil.go#L154-L181

So we're likely decoding into similar pointers:
https://github.com/kubernetes/test-infra/blob/1c44920cd06dc70f96c6a51f7d46151bffe17f65/prow/tide/tide.go#L1055-L1070

https://github.com/kubernetes-sigs/controller-runtime/blob/47744b5a819ddb704a85e6b11408cdf45443861b/pkg/client/typed_client.go#L48

func (c *typedClient) Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error {
    o, err := c.cache.getObjMeta(obj)
    if err != nil {
        return err
    }

    createOpts := &CreateOptions{}
    createOpts.ApplyOptions(opts)
    return o.Post().
        NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()).
        Resource(o.resource()).
        Body(obj).
        VersionedParams(createOpts.AsCreateOptions(), c.paramCodec).
        Context(ctx).
        Do().
        Into(obj)
}

https://github.com/kubernetes/client-go/blob/f8f007fd456c8d3a9ea48a1f69b246acf86c0d15/rest/request.go#L1154

https://github.com/kubernetes/apimachinery/blob/c18f71bf29474a53323469937882e0ebb873cd8c/pkg/runtime/serializer/json/json.go#L259

fejta on 8 Nov 2019

🚀2

All 12 comments

@stevekuznetsov: The label(s) kind/oncall-hostlist cannot be applied. These labels are supported: api-review, community/discussion, community/maintenance, community/question, cuj/build-train-deploy, cuj/multi-user, platform/aws, platform/azure, platform/gcp, platform/minikube, platform/other

In response to this:

fatal error: concurrent map writes

goroutine 78348 [running]:
runtime.throw(0x17f7e36, 0x15)
  GOROOT/src/runtime/panic.go:774 +0x72 fp=0xc002e02e48 sp=0xc002e02e18 pc=0x42f302
runtime.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc012682f00)
  GOROOT/src/runtime/map.go:680 +0x5aa fp=0xc002e02ed0 sp=0xc002e02e48 pc=0x40e83a
reflect.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc008ed0990)
  GOROOT/src/runtime/map.go:1329 +0x3f fp=0xc002e02f00 sp=0xc002e02ed0 pc=0x4103bf
github.com/modern-go/reflect2.(*UnsafeMapType).UnsafeSetIndex(...)
  external/com_github_modern_go_reflect2/unsafe_map.go:76
github.com/json-iterator/go.(*mapDecoder).Decode(0xc0000da410, 0xc007957550, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_map.go:180 +0x1a5 fp=0xc002e02f80 sp=0xc002e02f00 pc=0xa22ae5
github.com/json-iterator/go.(*placeholderDecoder).Decode(0xc008ff00e0, 0xc007957550, 0xc012682f00)
  external/com_github_json_iterator_go/reflect.go:319 +0x47 fp=0xc002e02fa8 sp=0xc002e02f80 pc=0xa1bd37
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc00240aca0, 0xc007957500, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03038 sp=0xc002e02fa8 pc=0xa31b86
github.com/json-iterator/go.(*sevenFieldsStructDecoder).Decode(0xc002c7c900, 0xc007957500, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:806 +0x325 fp=0xc002e030b0 sp=0xc002e03038 pc=0xa30ef5
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc00240ad20, 0xc0136d8518, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_optional.go:43 +0x93 fp=0xc002e030e0 sp=0xc002e030b0 pc=0xa29753
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc00240b3a0, 0xc0136d8500, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03170 sp=0xc002e030e0 pc=0xa31b86
github.com/json-iterator/go.(*nineFieldsStructDecoder).Decode(0xc008540280, 0xc0136d8500, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:926 +0x295 fp=0xc002e031e8 sp=0xc002e03170 pc=0xa31585
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc00240b560, 0xc0034b6ae0, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_optional.go:43 +0x93 fp=0xc002e03218 sp=0xc002e031e8 pc=0xa29753
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0021a0a00, 0xc0034b6a20, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e032a8 sp=0xc002e03218 pc=0xa31b86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0021a0b00, 0xc0034b6a20, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212 fp=0xc002e03320 sp=0xc002e032a8 pc=0xa2f942
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0021a0b00, 0xc0034b6a20, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f fp=0xc002e033a0 sp=0xc002e03320 pc=0xa2f50f
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0020e81e0, 0xc0034b6900, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03430 sp=0xc002e033a0 pc=0xa31b86
github.com/json-iterator/go.(*fiveFieldsStructDecoder).Decode(0xc00130d560, 0xc0034b6900, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:720 +0xb1 fp=0xc002e034a8 sp=0xc002e03430 pc=0xa30641
github.com/json-iterator/go.(*Iterator).ReadVal(0xc012682f00, 0x17b8ac0, 0xc0034b6900)
  external/com_github_json_iterator_go/reflect.go:78 +0xae fp=0xc002e03510 sp=0xc002e034a8 pc=0xa19cae
github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc000218000, 0xc00108e000, 0xcea, 0x1e00, 0x17b8ac0, 0xc0034b6900, 0x0, 0x0)
  external/com_github_json_iterator_go/config.go:348 +0xd2 fp=0xc002e03590 sp=0xc002e03510 pc=0xa10612
k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode(0xc00926cf80, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0x1, 0x0, 0x1a0e760, ...)
  external/io_k8s_apimachinery/pkg/runtime/serializer/json/json.go:255 +0x7d5 fp=0xc002e038d8 sp=0xc002e03590 pc=0xa7a665
k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode(0x1a0df60, 0xc00926cf80, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0xc00a99a200, 0xc00a99a100, ...)
  external/io_k8s_apimachinery/pkg/runtime/helper.go:252 +0x97 fp=0xc002e03970 sp=0xc002e038d8 pc=0x8eeb57
k8s.io/apimachinery/pkg/runtime.(*WithoutVersionDecoder).Decode(0xc007c5a300, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0xc002e03a38, 0x0, 0x0, ...)
  <autogenerated>:1 +0x9f fp=0xc002e039e8 sp=0xc002e03970 pc=0x8f9c1f
k8s.io/client-go/rest.Result.Into(0xc00108e000, 0xcea, 0x1e00, 0xc014c43a90, 0x10, 0x0, 0x0, 0xc9, 0x1a0fd20, 0xc007c5a300, ...)
  external/io_k8s_client_go/rest/request.go:1112 +0xba fp=0xc002e03ab0 sp=0xc002e039e8 pc=0xaa409a
sigs.k8s.io/controller-runtime/pkg/client.(*typedClient).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0x10, 0x154f600)
  external/io_k8s_sigs_controller_runtime/pkg/client/typed_client.go:48 +0x33e fp=0xc002e03bb0 sp=0xc002e03ab0 pc=0x109cd8e
sigs.k8s.io/controller-runtime/pkg/client.(*client).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0x32576709, 0x8ae6007b52fd)
  external/io_k8s_sigs_controller_runtime/pkg/client/client.go:123 +0x11b fp=0xc002e03c10 sp=0xc002e03bb0 pc=0x10999eb
sigs.k8s.io/controller-runtime/pkg/client.(*DelegatingClient).Create(0xc0053aa0c0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0xc007bdfa40, 0x38)
  <autogenerated>:1 +0x97 fp=0xc002e03c70 sp=0xc002e03c10 pc=0x10a2647
k8s.io/test-infra/prow/tide.(*Controller).trigger(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
  prow/tide/tide.go:1070 +0x6a0 fp=0xc002e04ef0 sp=0xc002e03c70 pc=0x1433540
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
  prow/tide/tide.go:1108 +0x2a7 fp=0xc002e05398 sp=0xc002e04ef0 pc=0x1433e97
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
  prow/tide/tide.go:1301 +0xfad fp=0xc002e059f0 sp=0xc002e05398 pc=0x143716d
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc01303b5f0)
  prow/tide/tide.go:377 +0x157 fp=0xc002e05f60 sp=0xc002e059f0 pc=0x143b597
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
  prow/tide/tide.go:430 +0x78 fp=0xc002e05fc8 sp=0xc002e05f60 pc=0x143b948
runtime.goexit()
  src/runtime/asm_amd64.s:1357 +0x1 fp=0xc002e05fd0 sp=0xc002e05fc8 pc=0x45eec1
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
  prow/tide/tide.go:427 +0x19b

goroutine 1 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x5)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0x1890718)
  prow/interrupts/interrupts.go:88 +0x80
k8s.io/test-infra/prow/interrupts.WaitForGracefulShutdown()
  prow/interrupts/interrupts.go:100 +0x3f
main.main()
  prow/cmd/tide/main.go:217 +0x138f

goroutine 6 [syscall, 259 minutes]:
os/signal.signal_recv(0x0)
  GOROOT/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
  GOROOT/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
  GOROOT/src/os/signal/signal_unix.go:29 +0x41

goroutine 7 [chan receive, 259 minutes]:
k8s.io/test-infra/prow/interrupts.handleInterrupt()
  prow/interrupts/interrupts.go:62 +0x84
created by k8s.io/test-infra/prow/interrupts.init.0
  prow/interrupts/interrupts.go:41 +0xb6

goroutine 18 [chan receive]:
k8s.io/klog.(*loggingT).flushDaemon(0x257c2c0)
  external/io_k8s_klog/klog.go:1010 +0x8b
created by k8s.io/klog.init.0
  external/io_k8s_klog/klog.go:411 +0xd6

goroutine 38 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc000192870)
  external/io_opencensus_go/stats/view/worker.go:154 +0x100
created by go.opencensus.io/stats/view.init.0
  external/io_opencensus_go/stats/view/worker.go:32 +0x57

goroutine 83 [IO wait, 259 minutes]:
internal/poll.runtime_pollWait(0x7f5da1a1eea8, 0x72, 0x0)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0001be198, 0x72, 0x0, 0x0, 0x17e5a26)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0001be180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc0001be180, 0xc0000c7d20, 0xc00020ee00, 0x7f5da3ccb008)
  GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00019e640, 0xc0000c7d50, 0x40ca48, 0x30)
  GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00019e640, 0x16d8ba0, 0xc0003dc360, 0x15bd140, 0x25601d0)
  GOROOT/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0003fa000, 0x1a46180, 0xc00019e640, 0x0, 0x0)
  GOROOT/src/net/http/server.go:2896 +0x286
net/http.(*Server).ListenAndServe(0xc0003fa000, 0xc000511798, 0x40576f)
  GOROOT/src/net/http/server.go:2825 +0xb7
k8s.io/test-infra/prow/interrupts.ListenAndServe.func1(0xc0003fa000)
  prow/interrupts/interrupts.go:156 +0x6e
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:154 +0x62

goroutine 84 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x0)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc0003ec180)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:159 +0x92

goroutine 56 [chan receive]:
k8s.io/test-infra/prow/config.(*Agent).Start.func1(0x7ffd505f479e, 0x17, 0x7ffd505f47fe, 0xf, 0xc0003e6700)
  prow/config/agent.go:58 +0xb1
created by k8s.io/test-infra/prow/config.(*Agent).Start
  prow/config/agent.go:52 +0xd2

goroutine 57 [chan receive]:
k8s.io/test-infra/prow/config/secret.(*Agent).reloadSecret(0xc002255c00, 0x7ffd505f47da, 0x11)
  prow/config/secret/agent.go:68 +0x122
created by k8s.io/test-infra/prow/config/secret.(*Agent).Start
  prow/config/secret/agent.go:52 +0x103

goroutine 58 [chan receive]:
k8s.io/test-infra/prow/github.(*client).Throttle.func1(0xc00440a820, 0xc0044708a0)
  prow/github/client.go:405 +0x59
created by k8s.io/test-infra/prow/github.(*client).Throttle
  prow/github/client.go:403 +0x26b

goroutine 59 [chan receive]:
k8s.io/test-infra/prow/github.(*client).Throttle.func1(0xc00440a8c0, 0xc0044709c0)
  prow/github/client.go:405 +0x59
created by k8s.io/test-infra/prow/github.(*client).Throttle
  prow/github/client.go:403 +0x26b

goroutine 233 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1ed08, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc008dc8798, 0x72, 0xff00, 0xffd8, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc008dc8780, 0xc001dd4000, 0xffd8, 0xffd8, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc008dc8780, 0xc001dd4000, 0xffd8, 0xffd8, 0x203002, 0x0, 0xffd3)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc008dd2000, 0xc001dd4000, 0xffd8, 0xffd8, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc009d05ac0, 0xc001dd4000, 0xffd8, 0xffd8, 0xd22, 0x1a0c4e0, 0xc00016e970)
  GOROOT/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc000130958, 0x1a0c2e0, 0xc009d05ac0, 0x40a1b5, 0x15f4ca0, 0x177b480)
  GOROOT/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc000130700, 0x1a0e6c0, 0xc008dd2000, 0x5, 0xc008dd2000, 0x203002)
  GOROOT/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc000130700, 0x0, 0x0, 0xc009d05aa0)
  GOROOT/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc000130700, 0xc008e11000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/crypto/tls/conn.go:1255 +0x161
bufio.(*Reader).Read(0xc008dd6c60, 0xc008df21f8, 0x9, 0x9, 0x3d67, 0xd04, 0x0)
  GOROOT/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x1a0c100, 0xc008dd6c60, 0xc008df21f8, 0x9, 0x9, 0x9, 0x0, 0x0, 0xc00016ef78)
  GOROOT/src/io/io.go:310 +0x87
io.ReadFull(...)
  GOROOT/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc008df21f8, 0x9, 0x9, 0x1a0c100, 0xc008dd6c60, 0x0, 0x0, 0xc006ec1c80, 0x0)
  external/org_golang_x_net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc008df21c0, 0xc006ec1c80, 0x0, 0x0, 0x0)
  external/org_golang_x_net/http2/frame.go:492 +0xa1
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00016efb8, 0x0, 0x0)
  external/org_golang_x_net/http2/transport.go:1710 +0x8c
golang.org/x/net/http2.(*ClientConn).readLoop(0xc008dd4180)
  external/org_golang_x_net/http2/transport.go:1638 +0xa3
created by golang.org/x/net/http2.(*Transport).newClientConn
  external/org_golang_x_net/http2/transport.go:673 +0x62f

goroutine 219 [chan receive, 259 minutes]:
k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0xc008da6fc0)
  external/io_k8s_apimachinery/pkg/watch/mux.go:207 +0x66
created by k8s.io/apimachinery/pkg/watch.NewBroadcaster
  external/io_k8s_apimachinery/pkg/watch/mux.go:75 +0xcc

goroutine 398 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x2)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc008f72720)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.OnInterrupt
  prow/interrupts/interrupts.go:235 +0x8e

goroutine 90 [chan receive, 19 minutes]:
k8s.io/test-infra/prow/tide.(*statusController).save(0xc00021a6c0, 0xc00440a0a0)
  prow/tide/status.go:385 +0x7c
created by k8s.io/test-infra/prow/tide.(*statusController).run
  prow/tide/status.go:417 +0xa7

goroutine 465 [chan receive, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start(0xc0092c60e0, 0xc008f8ede0)
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/informers_map.go:132 +0x4c
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/deleg_map.go:61 +0x80

goroutine 470 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0087840a0, 0x1a1d800, 0xc002363e80, 0xc00908fc00, 0xc001d0e540, 0xc008f8ede0, 0x0, 0x0)
  external/io_k8s_client_go/tools/cache/reflector.go:329 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0087840a0, 0xc008f8ede0, 0x0, 0x0)
  external/io_k8s_client_go/tools/cache/reflector.go:300 +0x833
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
  external/io_k8s_client_go/tools/cache/reflector.go:124 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc008e666f0)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00908fef0, 0x3b9aca00, 0x0, 0x1, 0xc008f8ede0)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0087840a0, 0xc008f8ede0)
  external/io_k8s_client_go/tools/cache/reflector.go:123 +0x169
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00168c020, 0xc004bb2120)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 477 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x4)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc00505e0a0)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:159 +0x92

goroutine 156 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x6)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc0084b3240)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.Tick
  prow/interrupts/interrupts.go:218 +0x10b

goroutine 394 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x1)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc00454de60)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.Run
  prow/interrupts/interrupts.go:145 +0xd2

goroutine 476 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e9c8, 0x72, 0x0)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc008f0a498, 0x72, 0x0, 0x0, 0x17e5a26)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc008f0a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc008f0a480, 0xc0000c1d50, 0x6e27e4, 0xc008412180)
  GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00505e0c0, 0x5dc43c1a, 0xc0000c1d50, 0x486416)
  GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00505e0c0, 0xc0000c1da0, 0x18, 0xc008e79200, 0x6e1cfe)
  GOROOT/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0084120e0, 0x1a46180, 0xc00505e0c0, 0x0, 0x0)
  GOROOT/src/net/http/server.go:2896 +0x286
net/http.(*Server).ListenAndServe(0xc0084120e0, 0x0, 0x0)
  GOROOT/src/net/http/server.go:2825 +0xb7
k8s.io/test-infra/prow/interrupts.ListenAndServe.func1(0xc0084120e0)
  prow/interrupts/interrupts.go:156 +0x6e
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:154 +0x62

goroutine 482 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x3)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc0050ca500)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:159 +0x92

goroutine 473 [select, 259 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0087840a0, 0xc008f8ede0, 0xc00f1bc480, 0xc001d0e540)
  external/io_k8s_client_go/tools/cache/reflector.go:235 +0x178
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
  external/io_k8s_client_go/tools/cache/reflector.go:229 +0x260

goroutine 368 [chan receive, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start(0xc0092c6000, 0xc008f8ede0)
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/informers_map.go:132 +0x4c
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/deleg_map.go:60 +0x50

goroutine 481 [IO wait, 259 minutes]:
internal/poll.runtime_pollWait(0x7f5da1a1edd8, 0x72, 0x0)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00842af18, 0x72, 0x0, 0x0, 0x17e5a26)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc00842af00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc00842af00, 0xc008e75d50, 0x6e27e4, 0xc008412340)
  GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0053dbb80, 0x5dc3ff7e, 0xc008e75d50, 0x486416)
  GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc0053dbb80, 0xc008e75da0, 0x18, 0xc008e79380, 0x6e1cfe)
  GOROOT/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0084122a0, 0x1a46180, 0xc0053dbb80, 0x0, 0x0)
  GOROOT/src/net/http/server.go:2896 +0x286
net/http.(*Server).ListenAndServe(0xc0084122a0, 0xc0090a1798, 0x40576f)
  GOROOT/src/net/http/server.go:2825 +0xb7
k8s.io/test-infra/prow/interrupts.ListenAndServe.func1(0xc0084122a0)
  prow/interrupts/interrupts.go:156 +0x6e
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:154 +0x62

goroutine 155 [semacquire]:
sync.runtime_Semacquire(0xc01373a208)
  GOROOT/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc01373a200)
  GOROOT/src/sync/waitgroup.go:130 +0x64
k8s.io/test-infra/prow/tide.subpoolsInParallel(0xf, 0xc00701a090, 0xc007ae13e0)
  prow/tide/tide.go:434 +0x1be
k8s.io/test-infra/prow/tide.(*Controller).Sync(0xc008509a80, 0x0, 0x0)
  prow/tide/tide.go:373 +0xbcb
main.sync(0xc008509a80)
  prow/cmd/tide/main.go:220 +0x2f
main.main.func3()
  prow/cmd/tide/main.go:213 +0x2a
k8s.io/test-infra/prow/interrupts.Tick.func1(0xc0084b3230, 0xc0020b7b60, 0xc0084b3220, 0xc00faa41c0)
  prow/interrupts/interrupts.go:210 +0x80
created by k8s.io/test-infra/prow/interrupts.Tick
  prow/interrupts/interrupts.go:196 +0xbd

goroutine 421 [chan receive, 259 minutes]:
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x1a1d840, 0xc0053bc420, 0xc0053bc3f0)
  external/io_k8s_client_go/tools/record/event.go:268 +0xc5
created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher
  external/io_k8s_client_go/tools/record/event.go:266 +0x6e

goroutine 468 [chan receive, 259 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc003d327e0, 0xc004df4b40)
  external/io_k8s_client_go/tools/cache/shared_informer.go:478 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00168c000, 0xc004bb20e0)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 422 [chan receive, 259 minutes]:
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x1a1d840, 0xc0053bc450, 0xc00924f280)
  external/io_k8s_client_go/tools/record/event.go:268 +0xc5
created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher
  external/io_k8s_client_go/tools/record/event.go:266 +0x6e

goroutine 469 [chan receive, 259 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc008f8ede0, 0xc008356000)
  external/io_k8s_client_go/tools/cache/controller.go:103 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
  external/io_k8s_client_go/tools/cache/controller.go:102 +0xae

goroutine 392 [select]:
net/http.(*persistConn).roundTrip(0xc00e57e6c0, 0xc013d1e660, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/transport.go:2417 +0x770
net/http.(*Transport).roundTrip(0x256bba0, 0xc0055ce600, 0xc0021d6258, 0xc00446d8b0, 0xc013d1e630)
  GOROOT/src/net/http/transport.go:535 +0x8f8
net/http.(*Transport).RoundTrip(0x256bba0, 0xc0055ce600, 0xc0055ce600, 0x0, 0x69961b)
  GOROOT/src/net/http/roundtrip.go:17 +0x35
golang.org/x/oauth2.(*Transport).RoundTrip(0xc00446d890, 0xc0055ce500, 0x0, 0x0, 0x0)
  external/org_golang_x_oauth2/transport.go:56 +0x124
net/http.send(0xc0055ce400, 0x1a0d460, 0xc00446d890, 0xbf692d31b22381b2, 0xe6d7101ce2a, 0x257be20, 0xc000314090, 0xbf692d31b22381b2, 0x1, 0x0)
  GOROOT/src/net/http/client.go:250 +0x43a
net/http.(*Client).send(0xc00446d8c0, 0xc0055ce400, 0xbf692d31b22381b2, 0xe6d7101ce2a, 0x257be20, 0xc000314090, 0x0, 0x1, 0xc0112d8080)
  GOROOT/src/net/http/client.go:174 +0xfa
net/http.(*Client).do(0xc00446d8c0, 0xc0055ce400, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/client.go:641 +0x3ce
net/http.(*Client).Do(...)
  GOROOT/src/net/http/client.go:509
github.com/shurcooL/go/ctxhttp.Do(0x1a4b9c0, 0xc000046020, 0xc00446d8c0, 0xc0055ce200, 0xc0046bc608, 0x16, 0x1a0c140)
  external/com_github_shurcool_go/ctxhttp/ctxhttp.go:34 +0xef
github.com/shurcooL/go/ctxhttp.Post(0x1a4b9c0, 0xc000046020, 0xc00446d8c0, 0x7ffd505f4872, 0x16, 0x17f111c, 0x10, 0x1a0c140, 0xc013d1e4b0, 0x140, ...)
  external/com_github_shurcool_go/ctxhttp/ctxhttp.go:72 +0x1af
github.com/shurcooL/graphql.(*Client).do(0xc002255cc0, 0x1a4b9c0, 0xc000046020, 0x0, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0x0, 0x0)
  external/com_github_shurcool_graphql/graphql.go:68 +0x1d3
github.com/shurcooL/graphql.(*Client).Query(...)
  external/com_github_shurcool_graphql/graphql.go:37
github.com/shurcooL/githubv4.(*Client).Query(0xc0001a1c00, 0x1a4b9c0, 0xc000046020, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0xc013d1e0f0, 0x0)
  external/com_github_shurcool_githubv4/githubv4.go:42 +0x68
k8s.io/test-infra/prow/github.(*throttler).Query(0xc0003fa230, 0x1a4b9c0, 0xc000046020, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0x0, 0x0)
  prow/github/client.go:378 +0xf6
k8s.io/test-infra/prow/github.(*client).Query(0xc003f6b600, 0x1a4b9c0, 0xc000046020, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0x0, 0x0)
  prow/github/client.go:2459 +0x70
k8s.io/test-infra/prow/tide.search(0xc0021d7528, 0xc012e7c1c0, 0xc001d54500, 0x22b5, 0x0, 0xed556329a, 0x0, 0xbf692ce6b21e153b, 0xe279797a9b6, 0x257be20, ...)
  prow/tide/search.go:64 +0x8c6
k8s.io/test-infra/prow/tide.(*statusController).search(0xc00021a6c0, 0xe2797920c70, 0x257be20, 0xc0114ae180)
  prow/tide/status.go:482 +0x544
k8s.io/test-infra/prow/tide.(*statusController).sync(0xc00021a6c0, 0xc007ae1380, 0xc01325ac90, 0xc01325acc0)
  prow/tide/status.go:461 +0x91
k8s.io/test-infra/prow/tide.(*statusController).waitSync(0xc00021a6c0)
  prow/tide/status.go:443 +0x1c1
k8s.io/test-infra/prow/tide.(*statusController).run(0xc00021a6c0)
  prow/tide/status.go:424 +0xb7
created by k8s.io/test-infra/prow/tide.NewController
  prow/tide/tide.go:232 +0x3e4

goroutine 466 [sync.Cond.Wait]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0004ab0a8, 0x2694)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0004ab098)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0004ab080, 0xc004504060, 0x0, 0x0, 0x0, 0x0)
  external/io_k8s_client_go/tools/cache/delta_fifo.go:423 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc008356000)
  external/io_k8s_client_go/tools/cache/controller.go:150 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000173e00)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0090dbe00, 0x3b9aca00, 0x0, 0xc001c26901, 0xc008f8ede0)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc008356000, 0xc008f8ede0)
  external/io_k8s_client_go/tools/cache/controller.go:124 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00875e090, 0xc008f8ede0)
  external/io_k8s_client_go/tools/cache/shared_informer.go:280 +0x47f
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start.func1
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/informers_map.go:126 +0x122

goroutine 393 [select, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start(0xc0092eec80, 0xc004df4240, 0x0, 0x0)
  external/io_k8s_sigs_controller_runtime/pkg/manager/internal.go:287 +0x170
main.main.func1(0x1a4b980, 0xc007e76ac0)
  prow/cmd/tide/main.go:178 +0x5b
k8s.io/test-infra/prow/interrupts.Run.func1(0xc004b78260, 0x1a4b980, 0xc007e76ac0)
  prow/interrupts/interrupts.go:142 +0x76
created by k8s.io/test-infra/prow/interrupts.Run
  prow/interrupts/interrupts.go:140 +0xb0

goroutine 365 [chan receive, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start(0xc0091fcde0, 0xc008f8ede0, 0x0, 0x0)
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/deleg_map.go:62 +0x97
sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).waitForCache.func1(0xc0092eec80)
  external/io_k8s_sigs_controller_runtime/pkg/manager/internal.go:343 +0x39
created by sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).waitForCache
  external/io_k8s_sigs_controller_runtime/pkg/manager/internal.go:342 +0x5a

goroutine 91 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1ea98, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc006372018, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc006372000, 0xc00045d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc006372000, 0xc00045d000, 0x1000, 0x1000, 0xc0008c7800, 0x7ea0fb, 0xc0083b82a0)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000314008, 0xc00045d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
net/http.(*connReader).Read(0xc0045ba120, 0xc00045d000, 0x1000, 0x1000, 0xc0008c78b8, 0xc0008c78a8, 0x48e00203004)
  GOROOT/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc001e12000)
  GOROOT/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).ReadSlice(0xc001e12000, 0xa, 0x7f5da0bb5878, 0xc0008c79a8, 0x40c1d6, 0xc00a88e500, 0x100)
  GOROOT/src/bufio/bufio.go:359 +0x3d
bufio.(*Reader).ReadLine(0xc001e12000, 0xc0008c79b0, 0xc0096ae000, 0x7f5da3cccb28, 0x0, 0x40ca48, 0x30)
  GOROOT/src/bufio/bufio.go:388 +0x34
net/textproto.(*Reader).readLineSlice(0xc013616240, 0xc00a88e500, 0xc006372000, 0x0, 0x0, 0x43112c)
  GOROOT/src/net/textproto/reader.go:57 +0x6c
net/textproto.(*Reader).ReadLine(...)
  GOROOT/src/net/textproto/reader.go:38
net/http.readRequest(0xc001e12000, 0x0, 0xc00a88e500, 0x0, 0x0)
  GOROOT/src/net/http/request.go:1012 +0x92
net/http.(*conn).readRequest(0xc00883a000, 0x1a4b980, 0xc009172040, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/server.go:965 +0x15f
net/http.(*conn).serve(0xc00883a000, 0x1a4b980, 0xc009172040)
  GOROOT/src/net/http/server.go:1817 +0x6d4
created by net/http.(*Server).Serve
  GOROOT/src/net/http/server.go:2927 +0x38e

goroutine 149 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e8f8, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0093a2518, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0093a2500, 0xc009702000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0093a2500, 0xc009702000, 0x1000, 0x1000, 0xc0090ef800, 0x7ea0fb, 0xc0084a5180)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00418c1a0, 0xc009702000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
net/http.(*connReader).Read(0xc0045bb5c0, 0xc009702000, 0x1000, 0x1000, 0xc0090ef8b8, 0xc0090ef8a8, 0x48e00203004)
  GOROOT/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc001e121e0)
  GOROOT/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).ReadSlice(0xc001e121e0, 0xa, 0x7f5da123ce30, 0xc0090ef9a8, 0x40c1d6, 0xc006895a00, 0x100)
  GOROOT/src/bufio/bufio.go:359 +0x3d
bufio.(*Reader).ReadLine(0xc001e121e0, 0xc0090ef9b0, 0xc0004b1880, 0x7f5da3ccb008, 0x0, 0x40ca48, 0x30)
  GOROOT/src/bufio/bufio.go:388 +0x34
net/textproto.(*Reader).readLineSlice(0xc012ccd1d0, 0xc006895a00, 0xc0093a2500, 0x0, 0x0, 0x43112c)
  GOROOT/src/net/textproto/reader.go:57 +0x6c
net/textproto.(*Reader).ReadLine(...)
  GOROOT/src/net/textproto/reader.go:38
net/http.readRequest(0xc001e121e0, 0x0, 0xc006895a00, 0x0, 0x0)
  GOROOT/src/net/http/request.go:1012 +0x92
net/http.(*conn).readRequest(0xc00883a280, 0x1a4b980, 0xc007bee200, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/server.go:965 +0x15f
net/http.(*conn).serve(0xc00883a280, 0x1a4b980, 0xc007bee200)
  GOROOT/src/net/http/server.go:1817 +0x6d4
created by net/http.(*Server).Serve
  GOROOT/src/net/http/server.go:2927 +0x38e

goroutine 77963 [select]:
net/http.(*persistConn).writeLoop(0xc00e57e6c0)
  GOROOT/src/net/http/transport.go:2196 +0x123
created by net/http.(*Transport).dialConn
  GOROOT/src/net/http/transport.go:1567 +0xb23

goroutine 77474 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f5da1a1e278, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc014e46518, 0x72, 0x1300, 0x1350, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc014e46500, 0xc0005e6a00, 0x1350, 0x1350, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc014e46500, 0xc0005e6a00, 0x1350, 0x1350, 0x203000, 0x437ef8, 0xc008362a80)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc008dd22f8, 0xc0005e6a00, 0x1350, 0x1350, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc002d9a400, 0xc0005e6a00, 0x1350, 0x1350, 0x27, 0x0, 0xc008e76970)
  GOROOT/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc000151058, 0x1a0c2e0, 0xc002d9a400, 0x40a1b5, 0x15f4ca0, 0x177b480)
  GOROOT/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc000150e00, 0x1a0e6c0, 0xc008dd22f8, 0x5, 0xc008dd22f8, 0x5)
  GOROOT/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc000150e00, 0x0, 0x0, 0x0)
  GOROOT/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc000150e00, 0xc0001b4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/crypto/tls/conn.go:1255 +0x161
bufio.(*Reader).Read(0xc005bda480, 0xc0003fa578, 0x9, 0x9, 0x11, 0x0, 0x0)
  GOROOT/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x1a0c100, 0xc005bda480, 0xc0003fa578, 0x9, 0x9, 0x9, 0xc008e76d20, 0xc008e76df0, 0x6c752a)
  GOROOT/src/io/io.go:310 +0x87
io.ReadFull(...)
  GOROOT/src/io/io.go:329
net/http.http2readFrameHeader(0xc0003fa578, 0x9, 0x9, 0x1a0c100, 0xc005bda480, 0x0, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/h2_bundle.go:1477 +0x87
net/http.(*http2Framer).ReadFrame(0xc0003fa540, 0xc005e50bc0, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/h2_bundle.go:1735 +0xa1
net/http.(*http2clientConnReadLoop).run(0xc008e76fb8, 0x1000000000001, 0x0)
  GOROOT/src/net/http/h2_bundle.go:8175 +0x8e
net/http.(*http2ClientConn).readLoop(0xc00fe9c180)
  GOROOT/src/net/http/h2_bundle.go:8103 +0xa3
created by net/http.(*http2Transport).newClientConn
  GOROOT/src/net/http/h2_bundle.go:7162 +0x62f

goroutine 78350 [syscall]:
syscall.Syscall6(0xf7, 0x1, 0x8bb, 0xc0021da338, 0x1000004, 0x0, 0x0, 0x4d7b01, 0xc00bd2df80, 0xc0021da378)
  src/syscall/asm_linux_amd64.s:44 +0x5
os.(*Process).blockUntilWaitable(0xc006cbfdd0, 0x203004, 0x160, 0x1)
  GOROOT/src/os/wait_waitid.go:31 +0x98
os.(*Process).wait(0xc006cbfdd0, 0x1890c10, 0x1890c18, 0x1890c08)
  GOROOT/src/os/exec_unix.go:22 +0x39
os.(*Process).Wait(...)
  GOROOT/src/os/exec.go:125
os/exec.(*Cmd).Wait(0xc0046bcb00, 0x0, 0x0)
  GOROOT/src/os/exec/exec.go:501 +0x60
os/exec.(*Cmd).Run(0xc0046bcb00, 0xc013aa8a20, 0xc0046bcb00)
  GOROOT/src/os/exec/exec.go:341 +0x5c
os/exec.(*Cmd).CombinedOutput(0xc0046bcb00, 0xc, 0xc008703e10, 0x1, 0x1, 0xc0046bcb00)
  GOROOT/src/os/exec/exec.go:561 +0x91
k8s.io/test-infra/prow/git.retryCmd(0xc003d33b90, 0xc006cbfcb0, 0x26, 0xc002684bc0, 0xc, 0xc008703e10, 0x1, 0x1, 0xc0021da6c0, 0x40c1d6, ...)
  prow/git/git.go:387 +0x22c
k8s.io/test-infra/prow/git.(*Client).Clone(0xc0035e6000, 0xc008b5da80, 0x10, 0x0, 0x0, 0x0)
  prow/git/git.go:163 +0x3c2
k8s.io/test-infra/prow/tide.(*Controller).pickBatch(0xc008509a80, 0xc016018540, 0xc006654820, 0x9, 0xc0066546f8, 0x6, 0xc0066542d0, 0x6, 0xc0061e5440, 0x28, ...)
  prow/tide/tide.go:814 +0x4f3
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc016018540, 0xc006654820, 0x9, 0xc0066546f8, 0x6, 0xc0066542d0, 0x6, 0xc0061e5440, 0x28, ...)
  prow/tide/tide.go:1097 +0x403
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc016018540, 0xc006654820, 0x9, 0xc0066546f8, 0x6, 0xc0066542d0, 0x6, 0xc0061e5440, 0x28, ...)
  prow/tide/tide.go:1301 +0xfad
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc0059c8900)
  prow/tide/tide.go:377 +0x157
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
  prow/tide/tide.go:430 +0x78
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
  prow/tide/tide.go:427 +0x19b

goroutine 78361 [runnable]:
github.com/json-iterator/go.(*sliceDecoder).Decode(0xc004bdede0, 0xc0108c69f0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_slice.go:59 +0x17f
github.com/json-iterator/go.(*placeholderDecoder).Decode(0xc008e35a80, 0xc0108c69f0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect.go:319 +0x47
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0029e1a00, 0xc0108c69c0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0029aa1c0, 0xc0108c69c0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0029aa1c0, 0xc0108c69c0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc0029aa1e0, 0xc0034b6dc8, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_optional.go:43 +0x93
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0021a0940, 0xc0034b6d20, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0021a0b00, 0xc0034b6d20, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0021a0b00, 0xc0034b6d20, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0020e81e0, 0xc0034b6c00, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*fiveFieldsStructDecoder).Decode(0xc00130d560, 0xc0034b6c00, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:720 +0xb1
github.com/json-iterator/go.(*Iterator).ReadVal(0xc0134e1200, 0x17b8ac0, 0xc0034b6c00)
  external/com_github_json_iterator_go/reflect.go:78 +0xae
github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc000218000, 0xc00002c000, 0xafd, 0xe00, 0x17b8ac0, 0xc0034b6c00, 0x0, 0x0)
  external/com_github_json_iterator_go/config.go:348 +0xd2
k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode(0xc00926cf80, 0xc00002c000, 0xafd, 0xe00, 0x0, 0x1a1e500, 0xc0034b6c00, 0x1, 0x0, 0x1a0e760, ...)
  external/io_k8s_apimachinery/pkg/runtime/serializer/json/json.go:255 +0x7d5
k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode(0x1a0df60, 0xc00926cf80, 0xc00002c000, 0xafd, 0xe00, 0x0, 0x1a1e500, 0xc0034b6c00, 0xc00a99a600, 0xc00a99a500, ...)
  external/io_k8s_apimachinery/pkg/runtime/helper.go:252 +0x97
k8s.io/client-go/rest.Result.Into(0xc00002c000, 0xafd, 0xe00, 0xc014c43a90, 0x10, 0x0, 0x0, 0xc9, 0x1a0fd20, 0xc007c5a300, ...)
  external/io_k8s_client_go/rest/request.go:1112 +0xba
sigs.k8s.io/controller-runtime/pkg/client.(*typedClient).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6c00, 0x0, 0x0, 0x0, 0x10, 0x154f600)
  external/io_k8s_sigs_controller_runtime/pkg/client/typed_client.go:48 +0x33e
sigs.k8s.io/controller-runtime/pkg/client.(*client).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6c00, 0x0, 0x0, 0x0, 0x32c7deb4, 0x8ae600ebca7e)
  external/io_k8s_sigs_controller_runtime/pkg/client/client.go:123 +0x11b
k8s.io/test-infra/prow/tide.(*Controller).trigger(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
  prow/tide/tide.go:1070 +0x6a0
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
  prow/tide/tide.go:1108 +0x2a7
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
  prow/tide/tide.go:1301 +0xfad
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc0134725a0)
  prow/tide/tide.go:377 +0x157
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
  prow/tide/tide.go:430 +0x78
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
  prow/tide/tide.go:427 +0x19b

goroutine 77964 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e348, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00bd2ded8, 0x72, 0x201, 0x200, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00bd2dec0, 0xc009636200, 0x200, 0x200, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
os.(*File).read(...)
  GOROOT/src/os/file_unix.go:259
os.(*File).Read(0xc000314148, 0xc009636200, 0x200, 0x200, 0x19f0350, 0xc0091e4eb0, 0xc0091e4ea0)
  GOROOT/src/os/file.go:116 +0x71
bytes.(*Buffer).ReadFrom(0xc013aa8a20, 0x1a0eb00, 0xc000314148, 0x7f5da0e6c550, 0xc013aa8a20, 0x19f0301)
  GOROOT/src/bytes/buffer.go:204 +0xb4
io.copyBuffer(0x1a0c160, 0xc013aa8a20, 0x1a0eb00, 0xc000314148, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0)
  GOROOT/src/io/io.go:388 +0x2ed
io.Copy(...)
  GOROOT/src/io/io.go:364
os/exec.(*Cmd).writerDescriptor.func1(0x0, 0x0)
  GOROOT/src/os/exec/exec.go:311 +0x63
os/exec.(*Cmd).Start.func1(0xc0046bcb00, 0xc00a80f5e0)
  GOROOT/src/os/exec/exec.go:435 +0x27
created by os/exec.(*Cmd).Start
  GOROOT/src/os/exec/exec.go:434 +0x608

goroutine 77957 [select]:
net/http.setRequestCancel.func3(0x0, 0xc013d1e600, 0xc003896190, 0xc013ba821c, 0xc014568240)
  GOROOT/src/net/http/client.go:321 +0xcf
created by net/http.setRequestCancel
  GOROOT/src/net/http/client.go:320 +0x24b

goroutine 75422 [runnable]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00a9b2680, 0xc00000012d)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00a9b2670)
  GOROOT/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc00a9b2668, 0xc00106c001, 0x3dff, 0x3dff, 0x0, 0x0, 0x0)
  external/org_golang_x_net/http2/pipe.go:64 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc00a9b2640, 0xc00106c001, 0x3dff, 0x3dff, 0x0, 0x0, 0x0)
  external/org_golang_x_net/http2/transport.go:1983 +0xac
encoding/json.(*Decoder).refill(0xc00bd2a2c0, 0xc004a5db0a, 0x9)
  GOROOT/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc00bd2a2c0, 0x0, 0x0, 0x15c0b00)
  GOROOT/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc00bd2a2c0, 0x15dd6c0, 0xc004a5dbe0, 0x0, 0x0)
  GOROOT/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0137e8c90, 0xc002306000, 0x4000, 0x4a80, 0x1a1e500, 0x0, 0x38)
  external/io_k8s_apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc008324e60, 0x0, 0x1a1d340, 0xc00232f580, 0x0, 0x0, 0x0, 0xc00291f1f8, 0x458e30)
  external/io_k8s_apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00d270100, 0xc004497fa8, 0x8, 0x1a1e500, 0xc00cef9200, 0x0, 0x0)
  external/io_k8s_client_go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc002363e80)
  external/io_k8s_apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
  external/io_k8s_apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 77962 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e758, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00137c618, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00137c600, 0xc000350000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00137c600, 0xc000350000, 0x1000, 0x1000, 0x1a0c4e0, 0xc00008e050, 0x259ae01)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0003140c8, 0xc000350000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00e57e6c0, 0xc000350000, 0x1000, 0x1000, 0x404c4d, 0x60, 0x0)
  GOROOT/src/net/http/transport.go:1744 +0x75
bufio.(*Reader).fill(0xc00bd2c840)
  GOROOT/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00bd2c840, 0x1, 0xc014568900, 0xc0091e4c98, 0x5bbe8d, 0x1652aa0, 0x7)
  GOROOT/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00e57e6c0)
  GOROOT/src/net/http/transport.go:1897 +0x1d6
created by net/http.(*Transport).dialConn
  GOROOT/src/net/http/transport.go:1566 +0xafe

/area prow/tide
/cc @cjwagner @alvaroaleman @fejta
/kind oncall-hostlist

Instructions for interacting with me using PR comments are available here. If you have questions or suggestions related to my behavior, please file an issue against the kubernetes/test-infra repository.

k8s-ci-robot on 7 Nov 2019

/retitle tide: segfault: concurrent map writes

stevekuznetsov on 7 Nov 2019

In response to this:

fatal error: concurrent map writes

goroutine 78348 [running]:
runtime.throw(0x17f7e36, 0x15)
  GOROOT/src/runtime/panic.go:774 +0x72 fp=0xc002e02e48 sp=0xc002e02e18 pc=0x42f302
runtime.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc012682f00)
  GOROOT/src/runtime/map.go:680 +0x5aa fp=0xc002e02ed0 sp=0xc002e02e48 pc=0x40e83a
reflect.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc008ed0990)
  GOROOT/src/runtime/map.go:1329 +0x3f fp=0xc002e02f00 sp=0xc002e02ed0 pc=0x4103bf
github.com/modern-go/reflect2.(*UnsafeMapType).UnsafeSetIndex(...)
  external/com_github_modern_go_reflect2/unsafe_map.go:76
github.com/json-iterator/go.(*mapDecoder).Decode(0xc0000da410, 0xc007957550, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_map.go:180 +0x1a5 fp=0xc002e02f80 sp=0xc002e02f00 pc=0xa22ae5
github.com/json-iterator/go.(*placeholderDecoder).Decode(0xc008ff00e0, 0xc007957550, 0xc012682f00)
  external/com_github_json_iterator_go/reflect.go:319 +0x47 fp=0xc002e02fa8 sp=0xc002e02f80 pc=0xa1bd37
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc00240aca0, 0xc007957500, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03038 sp=0xc002e02fa8 pc=0xa31b86
github.com/json-iterator/go.(*sevenFieldsStructDecoder).Decode(0xc002c7c900, 0xc007957500, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:806 +0x325 fp=0xc002e030b0 sp=0xc002e03038 pc=0xa30ef5
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc00240ad20, 0xc0136d8518, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_optional.go:43 +0x93 fp=0xc002e030e0 sp=0xc002e030b0 pc=0xa29753
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc00240b3a0, 0xc0136d8500, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03170 sp=0xc002e030e0 pc=0xa31b86
github.com/json-iterator/go.(*nineFieldsStructDecoder).Decode(0xc008540280, 0xc0136d8500, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:926 +0x295 fp=0xc002e031e8 sp=0xc002e03170 pc=0xa31585
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc00240b560, 0xc0034b6ae0, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_optional.go:43 +0x93 fp=0xc002e03218 sp=0xc002e031e8 pc=0xa29753
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0021a0a00, 0xc0034b6a20, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e032a8 sp=0xc002e03218 pc=0xa31b86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0021a0b00, 0xc0034b6a20, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212 fp=0xc002e03320 sp=0xc002e032a8 pc=0xa2f942
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0021a0b00, 0xc0034b6a20, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f fp=0xc002e033a0 sp=0xc002e03320 pc=0xa2f50f
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0020e81e0, 0xc0034b6900, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03430 sp=0xc002e033a0 pc=0xa31b86
github.com/json-iterator/go.(*fiveFieldsStructDecoder).Decode(0xc00130d560, 0xc0034b6900, 0xc012682f00)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:720 +0xb1 fp=0xc002e034a8 sp=0xc002e03430 pc=0xa30641
github.com/json-iterator/go.(*Iterator).ReadVal(0xc012682f00, 0x17b8ac0, 0xc0034b6900)
  external/com_github_json_iterator_go/reflect.go:78 +0xae fp=0xc002e03510 sp=0xc002e034a8 pc=0xa19cae
github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc000218000, 0xc00108e000, 0xcea, 0x1e00, 0x17b8ac0, 0xc0034b6900, 0x0, 0x0)
  external/com_github_json_iterator_go/config.go:348 +0xd2 fp=0xc002e03590 sp=0xc002e03510 pc=0xa10612
k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode(0xc00926cf80, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0x1, 0x0, 0x1a0e760, ...)
  external/io_k8s_apimachinery/pkg/runtime/serializer/json/json.go:255 +0x7d5 fp=0xc002e038d8 sp=0xc002e03590 pc=0xa7a665
k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode(0x1a0df60, 0xc00926cf80, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0xc00a99a200, 0xc00a99a100, ...)
  external/io_k8s_apimachinery/pkg/runtime/helper.go:252 +0x97 fp=0xc002e03970 sp=0xc002e038d8 pc=0x8eeb57
k8s.io/apimachinery/pkg/runtime.(*WithoutVersionDecoder).Decode(0xc007c5a300, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0xc002e03a38, 0x0, 0x0, ...)
  <autogenerated>:1 +0x9f fp=0xc002e039e8 sp=0xc002e03970 pc=0x8f9c1f
k8s.io/client-go/rest.Result.Into(0xc00108e000, 0xcea, 0x1e00, 0xc014c43a90, 0x10, 0x0, 0x0, 0xc9, 0x1a0fd20, 0xc007c5a300, ...)
  external/io_k8s_client_go/rest/request.go:1112 +0xba fp=0xc002e03ab0 sp=0xc002e039e8 pc=0xaa409a
sigs.k8s.io/controller-runtime/pkg/client.(*typedClient).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0x10, 0x154f600)
  external/io_k8s_sigs_controller_runtime/pkg/client/typed_client.go:48 +0x33e fp=0xc002e03bb0 sp=0xc002e03ab0 pc=0x109cd8e
sigs.k8s.io/controller-runtime/pkg/client.(*client).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0x32576709, 0x8ae6007b52fd)
  external/io_k8s_sigs_controller_runtime/pkg/client/client.go:123 +0x11b fp=0xc002e03c10 sp=0xc002e03bb0 pc=0x10999eb
sigs.k8s.io/controller-runtime/pkg/client.(*DelegatingClient).Create(0xc0053aa0c0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0xc007bdfa40, 0x38)
  <autogenerated>:1 +0x97 fp=0xc002e03c70 sp=0xc002e03c10 pc=0x10a2647
k8s.io/test-infra/prow/tide.(*Controller).trigger(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
  prow/tide/tide.go:1070 +0x6a0 fp=0xc002e04ef0 sp=0xc002e03c70 pc=0x1433540
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
  prow/tide/tide.go:1108 +0x2a7 fp=0xc002e05398 sp=0xc002e04ef0 pc=0x1433e97
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
  prow/tide/tide.go:1301 +0xfad fp=0xc002e059f0 sp=0xc002e05398 pc=0x143716d
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc01303b5f0)
  prow/tide/tide.go:377 +0x157 fp=0xc002e05f60 sp=0xc002e059f0 pc=0x143b597
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
  prow/tide/tide.go:430 +0x78 fp=0xc002e05fc8 sp=0xc002e05f60 pc=0x143b948
runtime.goexit()
  src/runtime/asm_amd64.s:1357 +0x1 fp=0xc002e05fd0 sp=0xc002e05fc8 pc=0x45eec1
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
  prow/tide/tide.go:427 +0x19b

goroutine 1 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x5)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0x1890718)
  prow/interrupts/interrupts.go:88 +0x80
k8s.io/test-infra/prow/interrupts.WaitForGracefulShutdown()
  prow/interrupts/interrupts.go:100 +0x3f
main.main()
  prow/cmd/tide/main.go:217 +0x138f

goroutine 6 [syscall, 259 minutes]:
os/signal.signal_recv(0x0)
  GOROOT/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
  GOROOT/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
  GOROOT/src/os/signal/signal_unix.go:29 +0x41

goroutine 7 [chan receive, 259 minutes]:
k8s.io/test-infra/prow/interrupts.handleInterrupt()
  prow/interrupts/interrupts.go:62 +0x84
created by k8s.io/test-infra/prow/interrupts.init.0
  prow/interrupts/interrupts.go:41 +0xb6

goroutine 18 [chan receive]:
k8s.io/klog.(*loggingT).flushDaemon(0x257c2c0)
  external/io_k8s_klog/klog.go:1010 +0x8b
created by k8s.io/klog.init.0
  external/io_k8s_klog/klog.go:411 +0xd6

goroutine 38 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc000192870)
  external/io_opencensus_go/stats/view/worker.go:154 +0x100
created by go.opencensus.io/stats/view.init.0
  external/io_opencensus_go/stats/view/worker.go:32 +0x57

goroutine 83 [IO wait, 259 minutes]:
internal/poll.runtime_pollWait(0x7f5da1a1eea8, 0x72, 0x0)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0001be198, 0x72, 0x0, 0x0, 0x17e5a26)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0001be180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc0001be180, 0xc0000c7d20, 0xc00020ee00, 0x7f5da3ccb008)
  GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00019e640, 0xc0000c7d50, 0x40ca48, 0x30)
  GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00019e640, 0x16d8ba0, 0xc0003dc360, 0x15bd140, 0x25601d0)
  GOROOT/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0003fa000, 0x1a46180, 0xc00019e640, 0x0, 0x0)
  GOROOT/src/net/http/server.go:2896 +0x286
net/http.(*Server).ListenAndServe(0xc0003fa000, 0xc000511798, 0x40576f)
  GOROOT/src/net/http/server.go:2825 +0xb7
k8s.io/test-infra/prow/interrupts.ListenAndServe.func1(0xc0003fa000)
  prow/interrupts/interrupts.go:156 +0x6e
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:154 +0x62

goroutine 84 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x0)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc0003ec180)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:159 +0x92

goroutine 56 [chan receive]:
k8s.io/test-infra/prow/config.(*Agent).Start.func1(0x7ffd505f479e, 0x17, 0x7ffd505f47fe, 0xf, 0xc0003e6700)
  prow/config/agent.go:58 +0xb1
created by k8s.io/test-infra/prow/config.(*Agent).Start
  prow/config/agent.go:52 +0xd2

goroutine 57 [chan receive]:
k8s.io/test-infra/prow/config/secret.(*Agent).reloadSecret(0xc002255c00, 0x7ffd505f47da, 0x11)
  prow/config/secret/agent.go:68 +0x122
created by k8s.io/test-infra/prow/config/secret.(*Agent).Start
  prow/config/secret/agent.go:52 +0x103

goroutine 58 [chan receive]:
k8s.io/test-infra/prow/github.(*client).Throttle.func1(0xc00440a820, 0xc0044708a0)
  prow/github/client.go:405 +0x59
created by k8s.io/test-infra/prow/github.(*client).Throttle
  prow/github/client.go:403 +0x26b

goroutine 59 [chan receive]:
k8s.io/test-infra/prow/github.(*client).Throttle.func1(0xc00440a8c0, 0xc0044709c0)
  prow/github/client.go:405 +0x59
created by k8s.io/test-infra/prow/github.(*client).Throttle
  prow/github/client.go:403 +0x26b

goroutine 233 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1ed08, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc008dc8798, 0x72, 0xff00, 0xffd8, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc008dc8780, 0xc001dd4000, 0xffd8, 0xffd8, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc008dc8780, 0xc001dd4000, 0xffd8, 0xffd8, 0x203002, 0x0, 0xffd3)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc008dd2000, 0xc001dd4000, 0xffd8, 0xffd8, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc009d05ac0, 0xc001dd4000, 0xffd8, 0xffd8, 0xd22, 0x1a0c4e0, 0xc00016e970)
  GOROOT/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc000130958, 0x1a0c2e0, 0xc009d05ac0, 0x40a1b5, 0x15f4ca0, 0x177b480)
  GOROOT/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc000130700, 0x1a0e6c0, 0xc008dd2000, 0x5, 0xc008dd2000, 0x203002)
  GOROOT/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc000130700, 0x0, 0x0, 0xc009d05aa0)
  GOROOT/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc000130700, 0xc008e11000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/crypto/tls/conn.go:1255 +0x161
bufio.(*Reader).Read(0xc008dd6c60, 0xc008df21f8, 0x9, 0x9, 0x3d67, 0xd04, 0x0)
  GOROOT/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x1a0c100, 0xc008dd6c60, 0xc008df21f8, 0x9, 0x9, 0x9, 0x0, 0x0, 0xc00016ef78)
  GOROOT/src/io/io.go:310 +0x87
io.ReadFull(...)
  GOROOT/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc008df21f8, 0x9, 0x9, 0x1a0c100, 0xc008dd6c60, 0x0, 0x0, 0xc006ec1c80, 0x0)
  external/org_golang_x_net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc008df21c0, 0xc006ec1c80, 0x0, 0x0, 0x0)
  external/org_golang_x_net/http2/frame.go:492 +0xa1
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00016efb8, 0x0, 0x0)
  external/org_golang_x_net/http2/transport.go:1710 +0x8c
golang.org/x/net/http2.(*ClientConn).readLoop(0xc008dd4180)
  external/org_golang_x_net/http2/transport.go:1638 +0xa3
created by golang.org/x/net/http2.(*Transport).newClientConn
  external/org_golang_x_net/http2/transport.go:673 +0x62f

goroutine 219 [chan receive, 259 minutes]:
k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0xc008da6fc0)
  external/io_k8s_apimachinery/pkg/watch/mux.go:207 +0x66
created by k8s.io/apimachinery/pkg/watch.NewBroadcaster
  external/io_k8s_apimachinery/pkg/watch/mux.go:75 +0xcc

goroutine 398 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x2)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc008f72720)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.OnInterrupt
  prow/interrupts/interrupts.go:235 +0x8e

goroutine 90 [chan receive, 19 minutes]:
k8s.io/test-infra/prow/tide.(*statusController).save(0xc00021a6c0, 0xc00440a0a0)
  prow/tide/status.go:385 +0x7c
created by k8s.io/test-infra/prow/tide.(*statusController).run
  prow/tide/status.go:417 +0xa7

goroutine 465 [chan receive, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start(0xc0092c60e0, 0xc008f8ede0)
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/informers_map.go:132 +0x4c
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/deleg_map.go:61 +0x80

goroutine 470 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0087840a0, 0x1a1d800, 0xc002363e80, 0xc00908fc00, 0xc001d0e540, 0xc008f8ede0, 0x0, 0x0)
  external/io_k8s_client_go/tools/cache/reflector.go:329 +0x1ec
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0087840a0, 0xc008f8ede0, 0x0, 0x0)
  external/io_k8s_client_go/tools/cache/reflector.go:300 +0x833
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
  external/io_k8s_client_go/tools/cache/reflector.go:124 +0x33
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc008e666f0)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00908fef0, 0x3b9aca00, 0x0, 0x1, 0xc008f8ede0)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0087840a0, 0xc008f8ede0)
  external/io_k8s_client_go/tools/cache/reflector.go:123 +0x169
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00168c020, 0xc004bb2120)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 477 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x4)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc00505e0a0)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:159 +0x92

goroutine 156 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x6)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc0084b3240)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.Tick
  prow/interrupts/interrupts.go:218 +0x10b

goroutine 394 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x1)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc00454de60)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.Run
  prow/interrupts/interrupts.go:145 +0xd2

goroutine 476 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e9c8, 0x72, 0x0)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc008f0a498, 0x72, 0x0, 0x0, 0x17e5a26)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc008f0a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc008f0a480, 0xc0000c1d50, 0x6e27e4, 0xc008412180)
  GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc00505e0c0, 0x5dc43c1a, 0xc0000c1d50, 0x486416)
  GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00505e0c0, 0xc0000c1da0, 0x18, 0xc008e79200, 0x6e1cfe)
  GOROOT/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0084120e0, 0x1a46180, 0xc00505e0c0, 0x0, 0x0)
  GOROOT/src/net/http/server.go:2896 +0x286
net/http.(*Server).ListenAndServe(0xc0084120e0, 0x0, 0x0)
  GOROOT/src/net/http/server.go:2825 +0xb7
k8s.io/test-infra/prow/interrupts.ListenAndServe.func1(0xc0084120e0)
  prow/interrupts/interrupts.go:156 +0x6e
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:154 +0x62

goroutine 482 [sync.Cond.Wait, 259 minutes]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0000b8310, 0x3)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0000b8300)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/test-infra/prow/interrupts.wait(0xc0050ca500)
  prow/interrupts/interrupts.go:88 +0x80
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:159 +0x92

goroutine 473 [select, 259 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0087840a0, 0xc008f8ede0, 0xc00f1bc480, 0xc001d0e540)
  external/io_k8s_client_go/tools/cache/reflector.go:235 +0x178
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
  external/io_k8s_client_go/tools/cache/reflector.go:229 +0x260

goroutine 368 [chan receive, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start(0xc0092c6000, 0xc008f8ede0)
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/informers_map.go:132 +0x4c
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/deleg_map.go:60 +0x50

goroutine 481 [IO wait, 259 minutes]:
internal/poll.runtime_pollWait(0x7f5da1a1edd8, 0x72, 0x0)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00842af18, 0x72, 0x0, 0x0, 0x17e5a26)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc00842af00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc00842af00, 0xc008e75d50, 0x6e27e4, 0xc008412340)
  GOROOT/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0053dbb80, 0x5dc3ff7e, 0xc008e75d50, 0x486416)
  GOROOT/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc0053dbb80, 0xc008e75da0, 0x18, 0xc008e79380, 0x6e1cfe)
  GOROOT/src/net/tcpsock.go:261 +0x47
net/http.(*Server).Serve(0xc0084122a0, 0x1a46180, 0xc0053dbb80, 0x0, 0x0)
  GOROOT/src/net/http/server.go:2896 +0x286
net/http.(*Server).ListenAndServe(0xc0084122a0, 0xc0090a1798, 0x40576f)
  GOROOT/src/net/http/server.go:2825 +0xb7
k8s.io/test-infra/prow/interrupts.ListenAndServe.func1(0xc0084122a0)
  prow/interrupts/interrupts.go:156 +0x6e
created by k8s.io/test-infra/prow/interrupts.ListenAndServe
  prow/interrupts/interrupts.go:154 +0x62

goroutine 155 [semacquire]:
sync.runtime_Semacquire(0xc01373a208)
  GOROOT/src/runtime/sema.go:56 +0x42
sync.(*WaitGroup).Wait(0xc01373a200)
  GOROOT/src/sync/waitgroup.go:130 +0x64
k8s.io/test-infra/prow/tide.subpoolsInParallel(0xf, 0xc00701a090, 0xc007ae13e0)
  prow/tide/tide.go:434 +0x1be
k8s.io/test-infra/prow/tide.(*Controller).Sync(0xc008509a80, 0x0, 0x0)
  prow/tide/tide.go:373 +0xbcb
main.sync(0xc008509a80)
  prow/cmd/tide/main.go:220 +0x2f
main.main.func3()
  prow/cmd/tide/main.go:213 +0x2a
k8s.io/test-infra/prow/interrupts.Tick.func1(0xc0084b3230, 0xc0020b7b60, 0xc0084b3220, 0xc00faa41c0)
  prow/interrupts/interrupts.go:210 +0x80
created by k8s.io/test-infra/prow/interrupts.Tick
  prow/interrupts/interrupts.go:196 +0xbd

goroutine 421 [chan receive, 259 minutes]:
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x1a1d840, 0xc0053bc420, 0xc0053bc3f0)
  external/io_k8s_client_go/tools/record/event.go:268 +0xc5
created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher
  external/io_k8s_client_go/tools/record/event.go:266 +0x6e

goroutine 468 [chan receive, 259 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc003d327e0, 0xc004df4b40)
  external/io_k8s_client_go/tools/cache/shared_informer.go:478 +0x49
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:54 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc00168c000, 0xc004bb20e0)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:71 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:69 +0x62

goroutine 422 [chan receive, 259 minutes]:
k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x1a1d840, 0xc0053bc450, 0xc00924f280)
  external/io_k8s_client_go/tools/record/event.go:268 +0xc5
created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher
  external/io_k8s_client_go/tools/record/event.go:266 +0x6e

goroutine 469 [chan receive, 259 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc008f8ede0, 0xc008356000)
  external/io_k8s_client_go/tools/cache/controller.go:103 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
  external/io_k8s_client_go/tools/cache/controller.go:102 +0xae

goroutine 392 [select]:
net/http.(*persistConn).roundTrip(0xc00e57e6c0, 0xc013d1e660, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/transport.go:2417 +0x770
net/http.(*Transport).roundTrip(0x256bba0, 0xc0055ce600, 0xc0021d6258, 0xc00446d8b0, 0xc013d1e630)
  GOROOT/src/net/http/transport.go:535 +0x8f8
net/http.(*Transport).RoundTrip(0x256bba0, 0xc0055ce600, 0xc0055ce600, 0x0, 0x69961b)
  GOROOT/src/net/http/roundtrip.go:17 +0x35
golang.org/x/oauth2.(*Transport).RoundTrip(0xc00446d890, 0xc0055ce500, 0x0, 0x0, 0x0)
  external/org_golang_x_oauth2/transport.go:56 +0x124
net/http.send(0xc0055ce400, 0x1a0d460, 0xc00446d890, 0xbf692d31b22381b2, 0xe6d7101ce2a, 0x257be20, 0xc000314090, 0xbf692d31b22381b2, 0x1, 0x0)
  GOROOT/src/net/http/client.go:250 +0x43a
net/http.(*Client).send(0xc00446d8c0, 0xc0055ce400, 0xbf692d31b22381b2, 0xe6d7101ce2a, 0x257be20, 0xc000314090, 0x0, 0x1, 0xc0112d8080)
  GOROOT/src/net/http/client.go:174 +0xfa
net/http.(*Client).do(0xc00446d8c0, 0xc0055ce400, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/client.go:641 +0x3ce
net/http.(*Client).Do(...)
  GOROOT/src/net/http/client.go:509
github.com/shurcooL/go/ctxhttp.Do(0x1a4b9c0, 0xc000046020, 0xc00446d8c0, 0xc0055ce200, 0xc0046bc608, 0x16, 0x1a0c140)
  external/com_github_shurcool_go/ctxhttp/ctxhttp.go:34 +0xef
github.com/shurcooL/go/ctxhttp.Post(0x1a4b9c0, 0xc000046020, 0xc00446d8c0, 0x7ffd505f4872, 0x16, 0x17f111c, 0x10, 0x1a0c140, 0xc013d1e4b0, 0x140, ...)
  external/com_github_shurcool_go/ctxhttp/ctxhttp.go:72 +0x1af
github.com/shurcooL/graphql.(*Client).do(0xc002255cc0, 0x1a4b9c0, 0xc000046020, 0x0, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0x0, 0x0)
  external/com_github_shurcool_graphql/graphql.go:68 +0x1d3
github.com/shurcooL/graphql.(*Client).Query(...)
  external/com_github_shurcool_graphql/graphql.go:37
github.com/shurcooL/githubv4.(*Client).Query(0xc0001a1c00, 0x1a4b9c0, 0xc000046020, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0xc013d1e0f0, 0x0)
  external/com_github_shurcool_githubv4/githubv4.go:42 +0x68
k8s.io/test-infra/prow/github.(*throttler).Query(0xc0003fa230, 0x1a4b9c0, 0xc000046020, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0x0, 0x0)
  prow/github/client.go:378 +0xf6
k8s.io/test-infra/prow/github.(*client).Query(0xc003f6b600, 0x1a4b9c0, 0xc000046020, 0x14f76a0, 0xc001eb6040, 0xc013d1e120, 0x0, 0x0)
  prow/github/client.go:2459 +0x70
k8s.io/test-infra/prow/tide.search(0xc0021d7528, 0xc012e7c1c0, 0xc001d54500, 0x22b5, 0x0, 0xed556329a, 0x0, 0xbf692ce6b21e153b, 0xe279797a9b6, 0x257be20, ...)
  prow/tide/search.go:64 +0x8c6
k8s.io/test-infra/prow/tide.(*statusController).search(0xc00021a6c0, 0xe2797920c70, 0x257be20, 0xc0114ae180)
  prow/tide/status.go:482 +0x544
k8s.io/test-infra/prow/tide.(*statusController).sync(0xc00021a6c0, 0xc007ae1380, 0xc01325ac90, 0xc01325acc0)
  prow/tide/status.go:461 +0x91
k8s.io/test-infra/prow/tide.(*statusController).waitSync(0xc00021a6c0)
  prow/tide/status.go:443 +0x1c1
k8s.io/test-infra/prow/tide.(*statusController).run(0xc00021a6c0)
  prow/tide/status.go:424 +0xb7
created by k8s.io/test-infra/prow/tide.NewController
  prow/tide/tide.go:232 +0x3e4

goroutine 466 [sync.Cond.Wait]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0004ab0a8, 0x2694)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0004ab098)
  GOROOT/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0004ab080, 0xc004504060, 0x0, 0x0, 0x0, 0x0)
  external/io_k8s_client_go/tools/cache/delta_fifo.go:423 +0xa6
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc008356000)
  external/io_k8s_client_go/tools/cache/controller.go:150 +0x40
k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000173e00)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:152 +0x5e
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0090dbe00, 0x3b9aca00, 0x0, 0xc001c26901, 0xc008f8ede0)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:153 +0xf8
k8s.io/apimachinery/pkg/util/wait.Until(...)
  external/io_k8s_apimachinery/pkg/util/wait/wait.go:88
k8s.io/client-go/tools/cache.(*controller).Run(0xc008356000, 0xc008f8ede0)
  external/io_k8s_client_go/tools/cache/controller.go:124 +0x2fd
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00875e090, 0xc008f8ede0)
  external/io_k8s_client_go/tools/cache/shared_informer.go:280 +0x47f
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start.func1
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/informers_map.go:126 +0x122

goroutine 393 [select, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start(0xc0092eec80, 0xc004df4240, 0x0, 0x0)
  external/io_k8s_sigs_controller_runtime/pkg/manager/internal.go:287 +0x170
main.main.func1(0x1a4b980, 0xc007e76ac0)
  prow/cmd/tide/main.go:178 +0x5b
k8s.io/test-infra/prow/interrupts.Run.func1(0xc004b78260, 0x1a4b980, 0xc007e76ac0)
  prow/interrupts/interrupts.go:142 +0x76
created by k8s.io/test-infra/prow/interrupts.Run
  prow/interrupts/interrupts.go:140 +0xb0

goroutine 365 [chan receive, 259 minutes]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start(0xc0091fcde0, 0xc008f8ede0, 0x0, 0x0)
  external/io_k8s_sigs_controller_runtime/pkg/cache/internal/deleg_map.go:62 +0x97
sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).waitForCache.func1(0xc0092eec80)
  external/io_k8s_sigs_controller_runtime/pkg/manager/internal.go:343 +0x39
created by sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).waitForCache
  external/io_k8s_sigs_controller_runtime/pkg/manager/internal.go:342 +0x5a

goroutine 91 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1ea98, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc006372018, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc006372000, 0xc00045d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc006372000, 0xc00045d000, 0x1000, 0x1000, 0xc0008c7800, 0x7ea0fb, 0xc0083b82a0)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000314008, 0xc00045d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
net/http.(*connReader).Read(0xc0045ba120, 0xc00045d000, 0x1000, 0x1000, 0xc0008c78b8, 0xc0008c78a8, 0x48e00203004)
  GOROOT/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc001e12000)
  GOROOT/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).ReadSlice(0xc001e12000, 0xa, 0x7f5da0bb5878, 0xc0008c79a8, 0x40c1d6, 0xc00a88e500, 0x100)
  GOROOT/src/bufio/bufio.go:359 +0x3d
bufio.(*Reader).ReadLine(0xc001e12000, 0xc0008c79b0, 0xc0096ae000, 0x7f5da3cccb28, 0x0, 0x40ca48, 0x30)
  GOROOT/src/bufio/bufio.go:388 +0x34
net/textproto.(*Reader).readLineSlice(0xc013616240, 0xc00a88e500, 0xc006372000, 0x0, 0x0, 0x43112c)
  GOROOT/src/net/textproto/reader.go:57 +0x6c
net/textproto.(*Reader).ReadLine(...)
  GOROOT/src/net/textproto/reader.go:38
net/http.readRequest(0xc001e12000, 0x0, 0xc00a88e500, 0x0, 0x0)
  GOROOT/src/net/http/request.go:1012 +0x92
net/http.(*conn).readRequest(0xc00883a000, 0x1a4b980, 0xc009172040, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/server.go:965 +0x15f
net/http.(*conn).serve(0xc00883a000, 0x1a4b980, 0xc009172040)
  GOROOT/src/net/http/server.go:1817 +0x6d4
created by net/http.(*Server).Serve
  GOROOT/src/net/http/server.go:2927 +0x38e

goroutine 149 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e8f8, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc0093a2518, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0093a2500, 0xc009702000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc0093a2500, 0xc009702000, 0x1000, 0x1000, 0xc0090ef800, 0x7ea0fb, 0xc0084a5180)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00418c1a0, 0xc009702000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
net/http.(*connReader).Read(0xc0045bb5c0, 0xc009702000, 0x1000, 0x1000, 0xc0090ef8b8, 0xc0090ef8a8, 0x48e00203004)
  GOROOT/src/net/http/server.go:785 +0xf4
bufio.(*Reader).fill(0xc001e121e0)
  GOROOT/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).ReadSlice(0xc001e121e0, 0xa, 0x7f5da123ce30, 0xc0090ef9a8, 0x40c1d6, 0xc006895a00, 0x100)
  GOROOT/src/bufio/bufio.go:359 +0x3d
bufio.(*Reader).ReadLine(0xc001e121e0, 0xc0090ef9b0, 0xc0004b1880, 0x7f5da3ccb008, 0x0, 0x40ca48, 0x30)
  GOROOT/src/bufio/bufio.go:388 +0x34
net/textproto.(*Reader).readLineSlice(0xc012ccd1d0, 0xc006895a00, 0xc0093a2500, 0x0, 0x0, 0x43112c)
  GOROOT/src/net/textproto/reader.go:57 +0x6c
net/textproto.(*Reader).ReadLine(...)
  GOROOT/src/net/textproto/reader.go:38
net/http.readRequest(0xc001e121e0, 0x0, 0xc006895a00, 0x0, 0x0)
  GOROOT/src/net/http/request.go:1012 +0x92
net/http.(*conn).readRequest(0xc00883a280, 0x1a4b980, 0xc007bee200, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/server.go:965 +0x15f
net/http.(*conn).serve(0xc00883a280, 0x1a4b980, 0xc007bee200)
  GOROOT/src/net/http/server.go:1817 +0x6d4
created by net/http.(*Server).Serve
  GOROOT/src/net/http/server.go:2927 +0x38e

goroutine 77963 [select]:
net/http.(*persistConn).writeLoop(0xc00e57e6c0)
  GOROOT/src/net/http/transport.go:2196 +0x123
created by net/http.(*Transport).dialConn
  GOROOT/src/net/http/transport.go:1567 +0xb23

goroutine 77474 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x7f5da1a1e278, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc014e46518, 0x72, 0x1300, 0x1350, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc014e46500, 0xc0005e6a00, 0x1350, 0x1350, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc014e46500, 0xc0005e6a00, 0x1350, 0x1350, 0x203000, 0x437ef8, 0xc008362a80)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc008dd22f8, 0xc0005e6a00, 0x1350, 0x1350, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc002d9a400, 0xc0005e6a00, 0x1350, 0x1350, 0x27, 0x0, 0xc008e76970)
  GOROOT/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc000151058, 0x1a0c2e0, 0xc002d9a400, 0x40a1b5, 0x15f4ca0, 0x177b480)
  GOROOT/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc000150e00, 0x1a0e6c0, 0xc008dd22f8, 0x5, 0xc008dd22f8, 0x5)
  GOROOT/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc000150e00, 0x0, 0x0, 0x0)
  GOROOT/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
  GOROOT/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc000150e00, 0xc0001b4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/crypto/tls/conn.go:1255 +0x161
bufio.(*Reader).Read(0xc005bda480, 0xc0003fa578, 0x9, 0x9, 0x11, 0x0, 0x0)
  GOROOT/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x1a0c100, 0xc005bda480, 0xc0003fa578, 0x9, 0x9, 0x9, 0xc008e76d20, 0xc008e76df0, 0x6c752a)
  GOROOT/src/io/io.go:310 +0x87
io.ReadFull(...)
  GOROOT/src/io/io.go:329
net/http.http2readFrameHeader(0xc0003fa578, 0x9, 0x9, 0x1a0c100, 0xc005bda480, 0x0, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/h2_bundle.go:1477 +0x87
net/http.(*http2Framer).ReadFrame(0xc0003fa540, 0xc005e50bc0, 0x0, 0x0, 0x0)
  GOROOT/src/net/http/h2_bundle.go:1735 +0xa1
net/http.(*http2clientConnReadLoop).run(0xc008e76fb8, 0x1000000000001, 0x0)
  GOROOT/src/net/http/h2_bundle.go:8175 +0x8e
net/http.(*http2ClientConn).readLoop(0xc00fe9c180)
  GOROOT/src/net/http/h2_bundle.go:8103 +0xa3
created by net/http.(*http2Transport).newClientConn
  GOROOT/src/net/http/h2_bundle.go:7162 +0x62f

goroutine 78350 [syscall]:
syscall.Syscall6(0xf7, 0x1, 0x8bb, 0xc0021da338, 0x1000004, 0x0, 0x0, 0x4d7b01, 0xc00bd2df80, 0xc0021da378)
  src/syscall/asm_linux_amd64.s:44 +0x5
os.(*Process).blockUntilWaitable(0xc006cbfdd0, 0x203004, 0x160, 0x1)
  GOROOT/src/os/wait_waitid.go:31 +0x98
os.(*Process).wait(0xc006cbfdd0, 0x1890c10, 0x1890c18, 0x1890c08)
  GOROOT/src/os/exec_unix.go:22 +0x39
os.(*Process).Wait(...)
  GOROOT/src/os/exec.go:125
os/exec.(*Cmd).Wait(0xc0046bcb00, 0x0, 0x0)
  GOROOT/src/os/exec/exec.go:501 +0x60
os/exec.(*Cmd).Run(0xc0046bcb00, 0xc013aa8a20, 0xc0046bcb00)
  GOROOT/src/os/exec/exec.go:341 +0x5c
os/exec.(*Cmd).CombinedOutput(0xc0046bcb00, 0xc, 0xc008703e10, 0x1, 0x1, 0xc0046bcb00)
  GOROOT/src/os/exec/exec.go:561 +0x91
k8s.io/test-infra/prow/git.retryCmd(0xc003d33b90, 0xc006cbfcb0, 0x26, 0xc002684bc0, 0xc, 0xc008703e10, 0x1, 0x1, 0xc0021da6c0, 0x40c1d6, ...)
  prow/git/git.go:387 +0x22c
k8s.io/test-infra/prow/git.(*Client).Clone(0xc0035e6000, 0xc008b5da80, 0x10, 0x0, 0x0, 0x0)
  prow/git/git.go:163 +0x3c2
k8s.io/test-infra/prow/tide.(*Controller).pickBatch(0xc008509a80, 0xc016018540, 0xc006654820, 0x9, 0xc0066546f8, 0x6, 0xc0066542d0, 0x6, 0xc0061e5440, 0x28, ...)
  prow/tide/tide.go:814 +0x4f3
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc016018540, 0xc006654820, 0x9, 0xc0066546f8, 0x6, 0xc0066542d0, 0x6, 0xc0061e5440, 0x28, ...)
  prow/tide/tide.go:1097 +0x403
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc016018540, 0xc006654820, 0x9, 0xc0066546f8, 0x6, 0xc0066542d0, 0x6, 0xc0061e5440, 0x28, ...)
  prow/tide/tide.go:1301 +0xfad
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc0059c8900)
  prow/tide/tide.go:377 +0x157
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
  prow/tide/tide.go:430 +0x78
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
  prow/tide/tide.go:427 +0x19b

goroutine 78361 [runnable]:
github.com/json-iterator/go.(*sliceDecoder).Decode(0xc004bdede0, 0xc0108c69f0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_slice.go:59 +0x17f
github.com/json-iterator/go.(*placeholderDecoder).Decode(0xc008e35a80, 0xc0108c69f0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect.go:319 +0x47
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0029e1a00, 0xc0108c69c0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0029aa1c0, 0xc0108c69c0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0029aa1c0, 0xc0108c69c0, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc0029aa1e0, 0xc0034b6dc8, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_optional.go:43 +0x93
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0021a0940, 0xc0034b6d20, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0021a0b00, 0xc0034b6d20, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0021a0b00, 0xc0034b6d20, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0020e81e0, 0xc0034b6c00, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*fiveFieldsStructDecoder).Decode(0xc00130d560, 0xc0034b6c00, 0xc0134e1200)
  external/com_github_json_iterator_go/reflect_struct_decoder.go:720 +0xb1
github.com/json-iterator/go.(*Iterator).ReadVal(0xc0134e1200, 0x17b8ac0, 0xc0034b6c00)
  external/com_github_json_iterator_go/reflect.go:78 +0xae
github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc000218000, 0xc00002c000, 0xafd, 0xe00, 0x17b8ac0, 0xc0034b6c00, 0x0, 0x0)
  external/com_github_json_iterator_go/config.go:348 +0xd2
k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode(0xc00926cf80, 0xc00002c000, 0xafd, 0xe00, 0x0, 0x1a1e500, 0xc0034b6c00, 0x1, 0x0, 0x1a0e760, ...)
  external/io_k8s_apimachinery/pkg/runtime/serializer/json/json.go:255 +0x7d5
k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode(0x1a0df60, 0xc00926cf80, 0xc00002c000, 0xafd, 0xe00, 0x0, 0x1a1e500, 0xc0034b6c00, 0xc00a99a600, 0xc00a99a500, ...)
  external/io_k8s_apimachinery/pkg/runtime/helper.go:252 +0x97
k8s.io/client-go/rest.Result.Into(0xc00002c000, 0xafd, 0xe00, 0xc014c43a90, 0x10, 0x0, 0x0, 0xc9, 0x1a0fd20, 0xc007c5a300, ...)
  external/io_k8s_client_go/rest/request.go:1112 +0xba
sigs.k8s.io/controller-runtime/pkg/client.(*typedClient).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6c00, 0x0, 0x0, 0x0, 0x10, 0x154f600)
  external/io_k8s_sigs_controller_runtime/pkg/client/typed_client.go:48 +0x33e
sigs.k8s.io/controller-runtime/pkg/client.(*client).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6c00, 0x0, 0x0, 0x0, 0x32c7deb4, 0x8ae600ebca7e)
  external/io_k8s_sigs_controller_runtime/pkg/client/client.go:123 +0x11b
k8s.io/test-infra/prow/tide.(*Controller).trigger(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
  prow/tide/tide.go:1070 +0x6a0
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
  prow/tide/tide.go:1108 +0x2a7
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
  prow/tide/tide.go:1301 +0xfad
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc0134725a0)
  prow/tide/tide.go:377 +0x157
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
  prow/tide/tide.go:430 +0x78
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
  prow/tide/tide.go:427 +0x19b

goroutine 77964 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e348, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00bd2ded8, 0x72, 0x201, 0x200, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00bd2dec0, 0xc009636200, 0x200, 0x200, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
os.(*File).read(...)
  GOROOT/src/os/file_unix.go:259
os.(*File).Read(0xc000314148, 0xc009636200, 0x200, 0x200, 0x19f0350, 0xc0091e4eb0, 0xc0091e4ea0)
  GOROOT/src/os/file.go:116 +0x71
bytes.(*Buffer).ReadFrom(0xc013aa8a20, 0x1a0eb00, 0xc000314148, 0x7f5da0e6c550, 0xc013aa8a20, 0x19f0301)
  GOROOT/src/bytes/buffer.go:204 +0xb4
io.copyBuffer(0x1a0c160, 0xc013aa8a20, 0x1a0eb00, 0xc000314148, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0)
  GOROOT/src/io/io.go:388 +0x2ed
io.Copy(...)
  GOROOT/src/io/io.go:364
os/exec.(*Cmd).writerDescriptor.func1(0x0, 0x0)
  GOROOT/src/os/exec/exec.go:311 +0x63
os/exec.(*Cmd).Start.func1(0xc0046bcb00, 0xc00a80f5e0)
  GOROOT/src/os/exec/exec.go:435 +0x27
created by os/exec.(*Cmd).Start
  GOROOT/src/os/exec/exec.go:434 +0x608

goroutine 77957 [select]:
net/http.setRequestCancel.func3(0x0, 0xc013d1e600, 0xc003896190, 0xc013ba821c, 0xc014568240)
  GOROOT/src/net/http/client.go:321 +0xcf
created by net/http.setRequestCancel
  GOROOT/src/net/http/client.go:320 +0x24b

goroutine 75422 [runnable]:
runtime.goparkunlock(...)
  GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00a9b2680, 0xc00000012d)
  GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00a9b2670)
  GOROOT/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc00a9b2668, 0xc00106c001, 0x3dff, 0x3dff, 0x0, 0x0, 0x0)
  external/org_golang_x_net/http2/pipe.go:64 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc00a9b2640, 0xc00106c001, 0x3dff, 0x3dff, 0x0, 0x0, 0x0)
  external/org_golang_x_net/http2/transport.go:1983 +0xac
encoding/json.(*Decoder).refill(0xc00bd2a2c0, 0xc004a5db0a, 0x9)
  GOROOT/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc00bd2a2c0, 0x0, 0x0, 0x15c0b00)
  GOROOT/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc00bd2a2c0, 0x15dd6c0, 0xc004a5dbe0, 0x0, 0x0)
  GOROOT/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0137e8c90, 0xc002306000, 0x4000, 0x4a80, 0x1a1e500, 0x0, 0x38)
  external/io_k8s_apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc008324e60, 0x0, 0x1a1d340, 0xc00232f580, 0x0, 0x0, 0x0, 0xc00291f1f8, 0x458e30)
  external/io_k8s_apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00d270100, 0xc004497fa8, 0x8, 0x1a1e500, 0xc00cef9200, 0x0, 0x0)
  external/io_k8s_client_go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc002363e80)
  external/io_k8s_apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
  external/io_k8s_apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 77962 [IO wait]:
internal/poll.runtime_pollWait(0x7f5da1a1e758, 0x72, 0xffffffffffffffff)
  GOROOT/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc00137c618, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
  GOROOT/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
  GOROOT/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00137c600, 0xc000350000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc00137c600, 0xc000350000, 0x1000, 0x1000, 0x1a0c4e0, 0xc00008e050, 0x259ae01)
  GOROOT/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0003140c8, 0xc000350000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
  GOROOT/src/net/net.go:184 +0x68
net/http.(*persistConn).Read(0xc00e57e6c0, 0xc000350000, 0x1000, 0x1000, 0x404c4d, 0x60, 0x0)
  GOROOT/src/net/http/transport.go:1744 +0x75
bufio.(*Reader).fill(0xc00bd2c840)
  GOROOT/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00bd2c840, 0x1, 0xc014568900, 0xc0091e4c98, 0x5bbe8d, 0x1652aa0, 0x7)
  GOROOT/src/bufio/bufio.go:138 +0x4f
net/http.(*persistConn).readLoop(0xc00e57e6c0)
  GOROOT/src/net/http/transport.go:1897 +0x1d6
created by net/http.(*Transport).dialConn
  GOROOT/src/net/http/transport.go:1566 +0xafe

/area prow/tide
/cc @cjwagner @alvaroaleman @fejta
/kind oncall-hostlist

k8s-ci-robot on 7 Nov 2019

Seems this is because two parallel create calls were made... I didn't realize we do that and I didn't know this is a problem

alvaroaleman on 7 Nov 2019

Creating what? I think all pools are handled in parallel.

stevekuznetsov on 7 Nov 2019

Creating what? I think all pools are handled in parallel.

Prowjobs, as the panic is triggered during a Create request and another one is going on as well. But maybe a red herring, I built a simple testcase where I have two goroutines create something in parallel but that doesn't trigger the race deterctor.

alvaroaleman on 7 Nov 2019

/assign @alvaroaleman @cjwagner @fejta

per https://github.com/kubernetes/test-infra/commits/master/prow/tide

fejta on 8 Nov 2019

Okay, so unless I am reading the dump wrong, the concurrent map access happens during the deserizaliation done within the Create call for a prowjob. Its not entirely clear what the other goroutine is.

I built a simple testcase that concurrently creates Prowjobs and calls the trigger func in which the Create call happens and this happily passes with -race:

$ go test  -count=1 -race -v ./ -run TestParallelCreate
=== RUN   TestParallelCreate
--- PASS: TestParallelCreate (0.80s)
PASS
ok      k8s.io/test-infra/prow/tide 1.843s

The only way I managed to get the race detector to complain is by passing in the same prowjob into two concurrent create request, which is expected and not something that can happen I think, as we generate the ProwJob in the same func that also Creates it

And with that I am out of ideas on how to reproduce this.

alvaroaleman on 8 Nov 2019

😕1

goroutine 78348 [running]:
runtime.throw(0x17f7e36, 0x15)
    GOROOT/src/runtime/panic.go:774 +0x72 fp=0xc002e02e48 sp=0xc002e02e18 pc=0x42f302
runtime.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc012682f00)
    GOROOT/src/runtime/map.go:680 +0x5aa fp=0xc002e02ed0 sp=0xc002e02e48 pc=0x40e83a
reflect.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc008ed0990)
    GOROOT/src/runtime/map.go:1329 +0x3f fp=0xc002e02f00 sp=0xc002e02ed0 pc=0x4103bf
github.com/modern-go/reflect2.(*UnsafeMapType).UnsafeSetIndex(...)
    external/com_github_modern_go_reflect2/unsafe_map.go:76
github.com/json-iterator/go.(*mapDecoder).Decode(0xc0000da410, 0xc007957550, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_map.go:180 +0x1a5 fp=0xc002e02f80 sp=0xc002e02f00 pc=0xa22ae5
github.com/json-iterator/go.(*placeholderDecoder).Decode(0xc008ff00e0, 0xc007957550, 0xc012682f00)
    external/com_github_json_iterator_go/reflect.go:319 +0x47 fp=0xc002e02fa8 sp=0xc002e02f80 pc=0xa1bd37
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc00240aca0, 0xc007957500, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03038 sp=0xc002e02fa8 pc=0xa31b86
github.com/json-iterator/go.(*sevenFieldsStructDecoder).Decode(0xc002c7c900, 0xc007957500, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:806 +0x325 fp=0xc002e030b0 sp=0xc002e03038 pc=0xa30ef5
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc00240ad20, 0xc0136d8518, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_optional.go:43 +0x93 fp=0xc002e030e0 sp=0xc002e030b0 pc=0xa29753
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc00240b3a0, 0xc0136d8500, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03170 sp=0xc002e030e0 pc=0xa31b86
github.com/json-iterator/go.(*nineFieldsStructDecoder).Decode(0xc008540280, 0xc0136d8500, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:926 +0x295 fp=0xc002e031e8 sp=0xc002e03170 pc=0xa31585
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc00240b560, 0xc0034b6ae0, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_optional.go:43 +0x93 fp=0xc002e03218 sp=0xc002e031e8 pc=0xa29753
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0021a0a00, 0xc0034b6a20, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e032a8 sp=0xc002e03218 pc=0xa31b86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0021a0b00, 0xc0034b6a20, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212 fp=0xc002e03320 sp=0xc002e032a8 pc=0xa2f942
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0021a0b00, 0xc0034b6a20, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f fp=0xc002e033a0 sp=0xc002e03320 pc=0xa2f50f
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0020e81e0, 0xc0034b6900, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86 fp=0xc002e03430 sp=0xc002e033a0 pc=0xa31b86
github.com/json-iterator/go.(*fiveFieldsStructDecoder).Decode(0xc00130d560, 0xc0034b6900, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:720 +0xb1 fp=0xc002e034a8 sp=0xc002e03430 pc=0xa30641
github.com/json-iterator/go.(*Iterator).ReadVal(0xc012682f00, 0x17b8ac0, 0xc0034b6900)
    external/com_github_json_iterator_go/reflect.go:78 +0xae fp=0xc002e03510 sp=0xc002e034a8 pc=0xa19cae
github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc000218000, 0xc00108e000, 0xcea, 0x1e00, 0x17b8ac0, 0xc0034b6900, 0x0, 0x0)
    external/com_github_json_iterator_go/config.go:348 +0xd2 fp=0xc002e03590 sp=0xc002e03510 pc=0xa10612
k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode(0xc00926cf80, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0x1, 0x0, 0x1a0e760, ...)
    external/io_k8s_apimachinery/pkg/runtime/serializer/json/json.go:255 +0x7d5 fp=0xc002e038d8 sp=0xc002e03590 pc=0xa7a665
k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode(0x1a0df60, 0xc00926cf80, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0xc00a99a200, 0xc00a99a100, ...)
    external/io_k8s_apimachinery/pkg/runtime/helper.go:252 +0x97 fp=0xc002e03970 sp=0xc002e038d8 pc=0x8eeb57
k8s.io/apimachinery/pkg/runtime.(*WithoutVersionDecoder).Decode(0xc007c5a300, 0xc00108e000, 0xcea, 0x1e00, 0x0, 0x1a1e500, 0xc0034b6900, 0xc002e03a38, 0x0, 0x0, ...)
    <autogenerated>:1 +0x9f fp=0xc002e039e8 sp=0xc002e03970 pc=0x8f9c1f
k8s.io/client-go/rest.Result.Into(0xc00108e000, 0xcea, 0x1e00, 0xc014c43a90, 0x10, 0x0, 0x0, 0xc9, 0x1a0fd20, 0xc007c5a300, ...)
    external/io_k8s_client_go/rest/request.go:1112 +0xba fp=0xc002e03ab0 sp=0xc002e039e8 pc=0xaa409a
sigs.k8s.io/controller-runtime/pkg/client.(*typedClient).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0x10, 0x154f600)
    external/io_k8s_sigs_controller_runtime/pkg/client/typed_client.go:48 +0x33e fp=0xc002e03bb0 sp=0xc002e03ab0 pc=0x109cd8e
sigs.k8s.io/controller-runtime/pkg/client.(*client).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0x32576709, 0x8ae6007b52fd)
    external/io_k8s_sigs_controller_runtime/pkg/client/client.go:123 +0x11b fp=0xc002e03c10 sp=0xc002e03bb0 pc=0x10999eb
sigs.k8s.io/controller-runtime/pkg/client.(*DelegatingClient).Create(0xc0053aa0c0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6900, 0x0, 0x0, 0x0, 0xc007bdfa40, 0x38)
    <autogenerated>:1 +0x97 fp=0xc002e03c70 sp=0xc002e03c10 pc=0x10a2647
k8s.io/test-infra/prow/tide.(*Controller).trigger(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
    prow/tide/tide.go:1070 +0x6a0 fp=0xc002e04ef0 sp=0xc002e03c70 pc=0x1433540
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
    prow/tide/tide.go:1108 +0x2a7 fp=0xc002e05398 sp=0xc002e04ef0 pc=0x1433e97
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc00b820fc0, 0xc01272d9e0, 0x9, 0xc013aed760, 0x17, 0xc01272d150, 0x6, 0xc003bac9c0, 0x28, ...)
    prow/tide/tide.go:1301 +0xfad fp=0xc002e059f0 sp=0xc002e05398 pc=0x143716d
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc01303b5f0)
    prow/tide/tide.go:377 +0x157 fp=0xc002e05f60 sp=0xc002e059f0 pc=0x143b597
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
    prow/tide/tide.go:430 +0x78 fp=0xc002e05fc8 sp=0xc002e05f60 pc=0x143b948
runtime.goexit()
    src/runtime/asm_amd64.s:1357 +0x1 fp=0xc002e05fd0 sp=0xc002e05fc8 pc=0x45eec1
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
    prow/tide/tide.go:427 +0x19b

goroutine 78361 [runnable]:
github.com/json-iterator/go.(*sliceDecoder).Decode(0xc004bdede0, 0xc0108c69f0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_slice.go:59 +0x17f
github.com/json-iterator/go.(*placeholderDecoder).Decode(0xc008e35a80, 0xc0108c69f0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect.go:319 +0x47
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0029e1a00, 0xc0108c69c0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0029aa1c0, 0xc0108c69c0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0029aa1c0, 0xc0108c69c0, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f
github.com/json-iterator/go.(*OptionalDecoder).Decode(0xc0029aa1e0, 0xc0034b6dc8, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_optional.go:43 +0x93
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0021a0940, 0xc0034b6d20, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*generalStructDecoder).decodeOneField(0xc0021a0b00, 0xc0034b6d20, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:548 +0x212
github.com/json-iterator/go.(*generalStructDecoder).Decode(0xc0021a0b00, 0xc0034b6d20, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:505 +0x6f
github.com/json-iterator/go.(*structFieldDecoder).Decode(0xc0020e81e0, 0xc0034b6c00, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:1010 +0x86
github.com/json-iterator/go.(*fiveFieldsStructDecoder).Decode(0xc00130d560, 0xc0034b6c00, 0xc0134e1200)
    external/com_github_json_iterator_go/reflect_struct_decoder.go:720 +0xb1
github.com/json-iterator/go.(*Iterator).ReadVal(0xc0134e1200, 0x17b8ac0, 0xc0034b6c00)
    external/com_github_json_iterator_go/reflect.go:78 +0xae
github.com/json-iterator/go.(*frozenConfig).Unmarshal(0xc000218000, 0xc00002c000, 0xafd, 0xe00, 0x17b8ac0, 0xc0034b6c00, 0x0, 0x0)
    external/com_github_json_iterator_go/config.go:348 +0xd2
k8s.io/apimachinery/pkg/runtime/serializer/json.(*Serializer).Decode(0xc00926cf80, 0xc00002c000, 0xafd, 0xe00, 0x0, 0x1a1e500, 0xc0034b6c00, 0x1, 0x0, 0x1a0e760, ...)
    external/io_k8s_apimachinery/pkg/runtime/serializer/json/json.go:255 +0x7d5
k8s.io/apimachinery/pkg/runtime.WithoutVersionDecoder.Decode(0x1a0df60, 0xc00926cf80, 0xc00002c000, 0xafd, 0xe00, 0x0, 0x1a1e500, 0xc0034b6c00, 0xc00a99a600, 0xc00a99a500, ...)
    external/io_k8s_apimachinery/pkg/runtime/helper.go:252 +0x97
k8s.io/client-go/rest.Result.Into(0xc00002c000, 0xafd, 0xe00, 0xc014c43a90, 0x10, 0x0, 0x0, 0xc9, 0x1a0fd20, 0xc007c5a300, ...)
    external/io_k8s_client_go/rest/request.go:1112 +0xba
sigs.k8s.io/controller-runtime/pkg/client.(*typedClient).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6c00, 0x0, 0x0, 0x0, 0x10, 0x154f600)
    external/io_k8s_sigs_controller_runtime/pkg/client/typed_client.go:48 +0x33e
sigs.k8s.io/controller-runtime/pkg/client.(*client).Create(0xc00906a4e0, 0x1a4b9c0, 0xc000046020, 0x1a1e500, 0xc0034b6c00, 0x0, 0x0, 0x0, 0x32c7deb4, 0x8ae600ebca7e)
    external/io_k8s_sigs_controller_runtime/pkg/client/client.go:123 +0x11b
k8s.io/test-infra/prow/tide.(*Controller).trigger(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
    prow/tide/tide.go:1070 +0x6a0
k8s.io/test-infra/prow/tide.(*Controller).takeAction(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
    prow/tide/tide.go:1108 +0x2a7
k8s.io/test-infra/prow/tide.(*Controller).syncSubpool(0xc008509a80, 0xc016019ea0, 0xc011de3eb0, 0x9, 0xc011de3de0, 0x7, 0xc011de3a88, 0x6, 0xc003ddea80, 0x28, ...)
    prow/tide/tide.go:1301 +0xfad
k8s.io/test-infra/prow/tide.(*Controller).Sync.func2(0xc0134725a0)
    prow/tide/tide.go:377 +0x157
k8s.io/test-infra/prow/tide.subpoolsInParallel.func1(0xc01373a200, 0xc00f4b63c0, 0xc007ae13e0)
    prow/tide/tide.go:430 +0x78
created by k8s.io/test-infra/prow/tide.subpoolsInParallel
    prow/tide/tide.go:427 +0x19b


goroutine 75422 [runnable]:
runtime.goparkunlock(...)
    GOROOT/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00a9b2680, 0xc00000012d)
    GOROOT/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc00a9b2670)
    GOROOT/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc00a9b2668, 0xc00106c001, 0x3dff, 0x3dff, 0x0, 0x0, 0x0)
    external/org_golang_x_net/http2/pipe.go:64 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc00a9b2640, 0xc00106c001, 0x3dff, 0x3dff, 0x0, 0x0, 0x0)
    external/org_golang_x_net/http2/transport.go:1983 +0xac
encoding/json.(*Decoder).refill(0xc00bd2a2c0, 0xc004a5db0a, 0x9)
    GOROOT/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc00bd2a2c0, 0x0, 0x0, 0x15c0b00)
    GOROOT/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc00bd2a2c0, 0x15dd6c0, 0xc004a5dbe0, 0x0, 0x0)
    GOROOT/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0137e8c90, 0xc002306000, 0x4000, 0x4a80, 0x1a1e500, 0x0, 0x38)
    external/io_k8s_apimachinery/pkg/util/framer/framer.go:150 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc008324e60, 0x0, 0x1a1d340, 0xc00232f580, 0x0, 0x0, 0x0, 0xc00291f1f8, 0x458e30)
    external/io_k8s_apimachinery/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00d270100, 0xc004497fa8, 0x8, 0x1a1e500, 0xc00cef9200, 0x0, 0x0)
    external/io_k8s_client_go/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc002363e80)
    external/io_k8s_apimachinery/pkg/watch/streamwatcher.go:104 +0x175
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
    external/io_k8s_apimachinery/pkg/watch/streamwatcher.go:71 +0xbe

fejta on 8 Nov 2019

https://github.com/kubernetes/test-infra/blob/7bf2a7ae54cc133daa75a5269fcb6510f8b9a694/prow/apis/prowjobs/v1/types.go#L141-L156

specFromJobBase() is not thread safe:
https://github.com/kubernetes/test-infra/blob/7bf2a7ae54cc133daa75a5269fcb6510f8b9a694/prow/pjutil/pjutil.go#L154-L181

So we're likely decoding into similar pointers:
https://github.com/kubernetes/test-infra/blob/1c44920cd06dc70f96c6a51f7d46151bffe17f65/prow/tide/tide.go#L1055-L1070

https://github.com/kubernetes-sigs/controller-runtime/blob/47744b5a819ddb704a85e6b11408cdf45443861b/pkg/client/typed_client.go#L48

func (c *typedClient) Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error {
    o, err := c.cache.getObjMeta(obj)
    if err != nil {
        return err
    }

    createOpts := &CreateOptions{}
    createOpts.ApplyOptions(opts)
    return o.Post().
        NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()).
        Resource(o.resource()).
        Body(obj).
        VersionedParams(createOpts.AsCreateOptions(), c.paramCodec).
        Context(ctx).
        Do().
        Into(obj)
}

https://github.com/kubernetes/client-go/blob/f8f007fd456c8d3a9ea48a1f69b246acf86c0d15/rest/request.go#L1154

https://github.com/kubernetes/apimachinery/blob/c18f71bf29474a53323469937882e0ebb873cd8c/pkg/runtime/serializer/json/json.go#L259

fejta on 8 Nov 2019

🚀2

Another interesting theory: https://github.com/kubernetes/test-infra/pull/15087

bumped json-iterator/go from 1.1.7 to 1.1.8

goroutine 78348 [running]:
runtime.throw(0x17f7e36, 0x15)
    GOROOT/src/runtime/panic.go:774 +0x72 fp=0xc002e02e48 sp=0xc002e02e18 pc=0x42f302
runtime.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc012682f00)
    GOROOT/src/runtime/map.go:680 +0x5aa fp=0xc002e02ed0 sp=0xc002e02e48 pc=0x40e83a
reflect.mapassign(0x15aabc0, 0xc0140e16e0, 0xc008ed0980, 0xc008ed0990)
    GOROOT/src/runtime/map.go:1329 +0x3f fp=0xc002e02f00 sp=0xc002e02ed0 pc=0x4103bf
github.com/modern-go/reflect2.(*UnsafeMapType).UnsafeSetIndex(...)
    external/com_github_modern_go_reflect2/unsafe_map.go:76
github.com/json-iterator/go.(*mapDecoder).Decode(0xc0000da410, 0xc007957550, 0xc012682f00)
    external/com_github_json_iterator_go/reflect_map.go:180 +0x1a5 fp=0xc002e02f80 sp=0xc002e02f00 pc=0xa22ae5

fejta on 8 Nov 2019

PodSpec maps:

NodeSelector
https://github.com/kubernetes/api/blob/59e77acf588f3c483e60c597b2cdbcdf0f37c240/core/v1/types.go#L2902

ResourceRequirements
https://github.com/kubernetes/api/blob/59e77acf588f3c483e60c597b2cdbcdf0f37c240/core/v1/types.go#L2173
https://github.com/kubernetes/api/blob/59e77acf588f3c483e60c597b2cdbcdf0f37c240/core/v1/types.go#L4579

fejta on 8 Nov 2019

Was this page helpful?

0 / 5 - 0 ratings