panic: deadlock detected: you have called conf.Get or conf.Watch before the frontend has been initialized (you may need to use a goroutine)
Created by: nicksnyder
I saw the following panic in sourcegraph-frontend when upgrading a Sourcegraph kubernetes cluster from 3.1.1 to 3.2. The pod eventually restarted due to health checks and did not panic the second time.
panic: deadlock detected: you have called conf.Get or conf.Watch before the frontend has been initialized (you may need to use a goroutine)
goroutine 10 [running]:
github.com/sourcegraph/sourcegraph/pkg/conf.(*Store).WaitUntilInitialized(0xc00018aea0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/store.go:134 +0x138
github.com/sourcegraph/sourcegraph/pkg/conf.(*client).Watch(0xc00015ab00, 0x1efdda8)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:111 +0xb8
github.com/sourcegraph/sourcegraph/pkg/conf.Watch(...)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:94
github.com/sourcegraph/sourcegraph/pkg/trace.init.0.func1()
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/trace/httptrace.go:59 +0x3a
created by github.com/sourcegraph/sourcegraph/pkg/trace.init.0
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/trace/httptrace.go:58 +0x117
goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7f3f30385e38, 0x77, 0x0)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00092a618, 0x77, 0xc000042000, 0x1, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitWrite(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:96
internal/poll.(*FD).WaitWrite(...)
/usr/local/go/src/internal/poll/fd_unix.go:498
net.(*netFD).connect(0xc00092a600, 0x20ceac0, 0xc000042058, 0x0, 0x0, 0x20a7cc0, 0xc000457d60, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/net/fd_unix.go:152 +0x284
net.(*netFD).dial(0xc00092a600, 0x20ceac0, 0xc000042058, 0x20d8540, 0x0, 0x20d8540, 0xc00094b740, 0x0, 0x0, 0xc0009015b0)
/usr/local/go/src/net/sock_posix.go:149 +0xff
net.socket(0x20ceac0, 0xc000042058, 0x15e6a54, 0x3, 0x2, 0x1, 0x0, 0x0, 0x20d8540, 0x0, ...)
/usr/local/go/src/net/sock_posix.go:70 +0x1ac
net.internetSocket(0x20ceac0, 0xc000042058, 0x15e6a54, 0x3, 0x20d8540, 0x0, 0x20d8540, 0xc00094b740, 0x1, 0x0, ...)
/usr/local/go/src/net/ipsock_posix.go:141 +0x141
net.(*sysDialer).doDialTCP(0xc00092a580, 0x20ceac0, 0xc000042058, 0x0, 0xc00094b740, 0xc0009015c0, 0x10, 0x10)
/usr/local/go/src/net/tcpsock_posix.go:65 +0xc2
net.(*sysDialer).dialTCP(0xc00092a580, 0x20ceac0, 0xc000042058, 0x0, 0xc00094b740, 0xae2c85a9e7, 0x7f3f0c706f3c, 0xc706f3c00000000)
/usr/local/go/src/net/tcpsock_posix.go:61 +0xd7
net.(*sysDialer).dialSingle(0xc00092a580, 0x20ceac0, 0xc000042058, 0x20b7400, 0xc00094b740, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/dial.go:571 +0x34b
net.(*sysDialer).dialSerial(0xc00092a580, 0x20ceac0, 0xc000042058, 0xc0009015c0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/dial.go:539 +0x221
net.(*Dialer).DialContext(0xc00125b538, 0x20ceac0, 0xc000042058, 0x15e6a54, 0x3, 0xc00033ee00, 0xa, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/net/dial.go:417 +0x65e
net.(*Dialer).Dial(...)
/usr/local/go/src/net/dial.go:340
net.Dial(...)
/usr/local/go/src/net/dial.go:311
github.com/lib/pq.defaultDialer.Dial(0x15e6a54, 0x3, 0xc00033ee00, 0xa, 0x10, 0x10, 0x1444e40, 0xc0002127d0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/pkg/mod/github.com/lib/[email protected]/conn.go:95 +0xab
github.com/lib/pq.dial(0x20b8500, 0x2d85088, 0xc00094b440, 0x0, 0xc000212778, 0x501, 0xc0012baa00)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/pkg/mod/github.com/lib/[email protected]/conn.go:393 +0x3a8
github.com/lib/pq.DialOpen(0x20b8500, 0x2d85088, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/pkg/mod/github.com/lib/[email protected]/conn.go:338 +0x53b
github.com/lib/pq.Open(...)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/pkg/mod/github.com/lib/[email protected]/conn.go:251
github.com/lib/pq.(*Driver).Open(0x2d85088, 0x0, 0x0, 0x39, 0xc00094b410, 0x203000, 0x8)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/pkg/mod/github.com/lib/[email protected]/conn.go:45 +0x4e
github.com/gchaincl/sqlhooks.(*Driver).Open(0xc000948740, 0x0, 0x0, 0x0, 0x0, 0x40bb00, 0xc000940000)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/pkg/mod/github.com/gchaincl/[email protected]/sqlhooks.go:47 +0x4b
database/sql.dsnConnector.Connect(...)
/usr/local/go/src/database/sql/sql.go:636
database/sql.(*DB).conn(0xc0004d6240, 0x20ceac0, 0xc000042058, 0x1, 0x0, 0x20a5480, 0xc000948740)
/usr/local/go/src/database/sql/sql.go:1176 +0x12b
database/sql.(*DB).PingContext(0xc0004d6240, 0x20ceac0, 0xc000042058, 0x0, 0x0)
/usr/local/go/src/database/sql/sql.go:730 +0x90
database/sql.(*DB).Ping(...)
/usr/local/go/src/database/sql/sql.go:748
github.com/sourcegraph/sourcegraph/pkg/db/dbconn.openDBWithStartupWait(0x0, 0x0, 0x15e6640, 0x3, 0x0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/db/dbconn/dbconn.go:94 +0x194
github.com/sourcegraph/sourcegraph/pkg/db/dbconn.ConnectToDB(0x0, 0x0, 0x0, 0x7f3f325f2d98)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/db/dbconn/dbconn.go:60 +0x16b
github.com/sourcegraph/sourcegraph/cmd/frontend/internal/cli.Main(0x0, 0x0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/cmd/frontend/internal/cli/serve_cmd.go:91 +0x7c
github.com/sourcegraph/sourcegraph/cmd/frontend/shared.Main()
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/cmd/frontend/shared/frontend.go:19 +0x2e
main.main()
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/main.go:56 +0x96
goroutine 32 [select]:
github.com/sourcegraph/sourcegraph/pkg/conf.(*Store).WaitUntilInitialized(0xc00018aea0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/store.go:125 +0xed
github.com/sourcegraph/sourcegraph/pkg/conf.(*client).Watch(0xc00015ab00, 0x1efd3f8)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:111 +0xb8
github.com/sourcegraph/sourcegraph/pkg/conf.Watch(...)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:94
github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/openidconnect.init.1.func1()
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/openidconnect/config_watch.go:30 +0x3a
created by github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/openidconnect.init.1
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/openidconnect/config_watch.go:29 +0x35
goroutine 31 [select]:
github.com/sourcegraph/sourcegraph/pkg/conf.(*Store).WaitUntilInitialized(0xc00018aea0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/store.go:125 +0xed
github.com/sourcegraph/sourcegraph/pkg/conf.(*client).Watch(0xc00015ab00, 0x1efd3b8)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:111 +0xb8
github.com/sourcegraph/sourcegraph/pkg/conf.Watch(...)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:94
github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/httpheader.init.1.func1()
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/httpheader/config_watch.go:11 +0x3a
created by github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/httpheader.init.1
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/httpheader/config_watch.go:10 +0x35
goroutine 68 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc00058f100)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/pkg/mod/[email protected]/stats/view/worker.go:147 +0xdd
created by go.opencensus.io/stats/view.init.0
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/pkg/mod/[email protected]/stats/view/worker.go:29 +0x57
goroutine 82 [select]:
github.com/sourcegraph/sourcegraph/pkg/conf.(*Store).WaitUntilInitialized(0xc00018aea0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/store.go:125 +0xed
github.com/sourcegraph/sourcegraph/pkg/conf.(*client).Watch(0xc00015ab00, 0x1efd0d8)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:111 +0xb8
github.com/sourcegraph/sourcegraph/pkg/conf.Watch(...)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:94
github.com/sourcegraph/sourcegraph/cmd/frontend/internal/auth/userpasswd.init.1.func1()
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/cmd/frontend/internal/auth/userpasswd/config_watch.go:11 +0x3a
created by github.com/sourcegraph/sourcegraph/cmd/frontend/internal/auth/userpasswd.init.1
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/cmd/frontend/internal/auth/userpasswd/config_watch.go:10 +0x35
goroutine 29 [select]:
github.com/sourcegraph/sourcegraph/pkg/conf.(*Store).WaitUntilInitialized(0xc00018aea0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/store.go:125 +0xed
github.com/sourcegraph/sourcegraph/pkg/conf.(*client).Watch(0xc00015ab00, 0x1efd368)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:111 +0xb8
github.com/sourcegraph/sourcegraph/pkg/conf.Watch(...)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:94
github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/githuboauth.init.0.func1()
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/githuboauth/config.go:13 +0x3e
created by github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/githuboauth.init.0
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/githuboauth/config.go:12 +0x35
goroutine 30 [select]:
github.com/sourcegraph/sourcegraph/pkg/conf.(*Store).WaitUntilInitialized(0xc00018aea0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/store.go:125 +0xed
github.com/sourcegraph/sourcegraph/pkg/conf.(*client).Watch(0xc00015ab00, 0x1efd398)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:111 +0xb8
github.com/sourcegraph/sourcegraph/pkg/conf.Watch(...)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:94
github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/gitlaboauth.init.0.func1()
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/gitlaboauth/config.go:16 +0x3e
created by github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/gitlaboauth.init.0
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/gitlaboauth/config.go:15 +0x35
goroutine 113 [select]:
github.com/sourcegraph/sourcegraph/pkg/conf.(*Store).WaitUntilInitialized(0xc00018aea0)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/store.go:125 +0xed
github.com/sourcegraph/sourcegraph/pkg/conf.(*client).Watch(0xc00015ab00, 0x1efd428)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:111 +0xb8
github.com/sourcegraph/sourcegraph/pkg/conf.Watch(...)
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/pkg/conf/client.go:94
github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/saml.init.1.func1()
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/saml/config_watch.go:31 +0x3a
created by github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/saml.init.1
/buildkite/builds/buildkite-agent-bf9c5945-v2n8n-1/sourcegraph/.golang/sourcegraph/src/github.com/sourcegraph/sourcegraph/enterprise/cmd/frontend/auth/saml/config_watch.go:30 +0x35
goroutine 78 [select]:
database/sql.(*DB).connectionResetter(0xc0004d6240, 0x20cea80, 0xc00015be80)
/usr/local/go/src/database/sql/sql.go:1013 +0xfb
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:671 +0x194
goroutine 77 [select]:
database/sql.(*DB).connectionOpener(0xc0004d6240, 0x20cea80, 0xc00015be80)
/usr/local/go/src/database/sql/sql.go:1000 +0xe8
created by database/sql.OpenDB
/usr/local/go/src/database/sql/sql.go:670 +0x15e