Panic: Concurrent map writes

[ukane-philemon]

I was testing and fixing bugs on dcrdata with the --reload-html flag, and out of the blue, I got a concurrent map writes panic when I attempted to refresh multiple times.

Panic log:

2024-06-05 22:36:51.762 [INF] PSQL: Project fund data is up-to-date.
fatal error: concurrent map writes

goroutine 1532335 [running]:
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*templates).addTemplate(0xc03890ba40, {0xf8476b1, 0xa})
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/templates.go:62 +0x1bf
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*templates).execWithReload(0xc03890ba40, {0xf8476b1, 0xa}, {0xfd943c0, 0xc0365e5b60})
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/templates.go:102 +0x30
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*explorerUI).SideChains(0xc04410c000, {0xfe69c98, 0xc073482620}, 0xc045867560)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/explorerroutes.go:263 +0x216
net/http.HandlerFunc.ServeHTTP(0x0?, {0xfe69c98?, 0xc073482620?}, 0x1?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*explorerUI).SyncStatusPageIntercept-fm.(*explorerUI).SyncStatusPageIntercept.func1({0xfe69c98, 0xc073482620}, 0xc045867560)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/explorermiddleware.go:160 +0xcc
net/http.HandlerFunc.ServeHTTP(0xc04585ebe0?, {0xfe69c98?, 0xc073482620?}, 0x0?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
github.com/go-chi/chi/v5.(*ChainHandler).ServeHTTP(0xfcf8ae0?, {0xfe69c98?, 0xc073482620?}, 0xc00003a16c?)
        /Users/[redacted]/go/pkg/mod/github.com/go-chi/chi/[email protected]/chain.go:31 +0x26
github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc04581cb40, {0xfe69c98, 0xc073482620}, 0xc045867560)
        /Users/[redacted]/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:444 +0x207
net/http.HandlerFunc.ServeHTTP(0xfe69c98?, {0xfe69c98?, 0xc073482620?}, 0x105d58c0?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
main._main.RequestBodyLimiter.func14.1({0xfe69c98, 0xc073482620}, 0xc045867560)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/middleware/apimiddleware.go:123 +0x87
net/http.HandlerFunc.ServeHTTP(0xc05a1eeeb0?, {0xfe69c98?, 0xc073482620?}, 0x10?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
github.com/go-chi/chi/v5/middleware.Recoverer.func1({0xfe69c98?, 0xc073482620?}, 0xf8422ad?)
        /Users/[redacted]/go/pkg/mod/github.com/go-chi/chi/[email protected]/middleware/recoverer.go:43 +0x78
net/http.HandlerFunc.ServeHTTP(0xc091e68ff0?, {0xfe69c98?, 0xc073482620?}, 0xf8437a1?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
main._main.Server.func13.1({0xfe69c98, 0xc073482620}, 0xc045867560)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/middleware/apimiddleware.go:594 +0x8c
net/http.HandlerFunc.ServeHTTP(0xfe6b3e0?, {0xfe69c98?, 0xc073482620?}, 0x10548350?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc04581cb40, {0xfe69c98, 0xc073482620}, 0xc05528eea0)
        /Users/[redacted]/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:90 +0x2ee
net/http.serverHandler.ServeHTTP({0xc09542bb00?}, {0xfe69c98?, 0xc073482620?}, 0x6?)
        /usr/local/go/src/net/http/server.go:3137 +0x8e
net/http.(*conn).serve(0xc012436990, {0xfe6b3a8, 0xc04586e6c0})
        /usr/local/go/src/net/http/server.go:2039 +0x5e8
created by net/http.(*Server).Serve in goroutine 1401573
        /usr/local/go/src/net/http/server.go:3285 +0x4b4

goroutine 1 [semacquire, 561 minutes]:
sync.runtime_Semacquire(0x0?)
        /usr/local/go/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xc04485be60?)
        /usr/local/go/src/sync/waitgroup.go:116 +0x48
main._main({0xfe6b3e0, 0xc0000ba1e0})
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/main.go:1113 +0x69b5
main.main()
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/main.go:60 +0x4a

goroutine 4 [chan receive, 579 minutes]:
main.withShutdownCancel.func1()
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/signal.go:46 +0x25
created by main.withShutdownCancel in goroutine 1
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/signal.go:45 +0x68

goroutine 5 [select, 579 minutes]:
main.shutdownListener()
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/signal.go:66 +0x9a
created by main.main in goroutine 1
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/main.go:58 +0x3b

goroutine 51 [syscall, 579 minutes]:
os/signal.signal_recv()
        /usr/local/go/src/runtime/sigqueue.go:149 +0x25
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:23 +0x13
created by os/signal.Notify.func1.1 in goroutine 5
        /usr/local/go/src/os/signal/signal.go:151 +0x1f

goroutine 16 [select]:
github.com/dgraph-io/badger.(*levelsController).runWorker(0xc0000ea770, 0xc000122228)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/levels.go:376 +0x1d8
created by github.com/dgraph-io/badger.(*levelsController).startCompact in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/levels.go:357 +0x59

goroutine 11 [IO wait]:
internal/poll.runtime_pollWait(0x57659f88, 0x72)
        /usr/local/go/src/runtime/netpoll.go:345 +0x85
internal/poll.(*pollDesc).wait(0xc0003ed480?, 0xc044894000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0003ed480, {0xc044894000, 0xa000, 0xa000})
        /usr/local/go/src/internal/poll/fd_unix.go:164 +0x27a
net.(*netFD).Read(0xc0003ed480, {0xc044894000?, 0xfcf4ef38?, 0xc0387847f8?})
        /usr/local/go/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc0000d61e8, {0xc044894000?, 0xc000169a10?, 0xee744fb?})
        /usr/local/go/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc0387847f8, {0xc044894000?, 0x0?, 0xc0387847f8?})
        /usr/local/go/src/crypto/tls/conn.go:806 +0x3b
bytes.(*Buffer).ReadFrom(0xc000005eb0, {0xfe64d20, 0xc0387847f8})
        /usr/local/go/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000005c08, {0x5765a080, 0xc0000d61e8}, 0xc000169a58?)
        /usr/local/go/src/crypto/tls/conn.go:828 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc000005c08, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:626 +0x3cf
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:588
crypto/tls.(*Conn).Read(0xc000005c08, {0xc0000ec000, 0x1000, 0xc000169dc0?})
        /usr/local/go/src/crypto/tls/conn.go:1370 +0x156
bufio.(*Reader).fill(0xc0000229c0)
        /usr/local/go/src/bufio/bufio.go:110 +0x103
bufio.(*Reader).Peek(0xc0000229c0, 0x2)
        /usr/local/go/src/bufio/bufio.go:148 +0x53
github.com/gorilla/websocket.(*Conn).read(0xc0004fdb80, 0xc0001eea80?)
        /Users/[redacted]/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:371 +0x26
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc0004fdb80)
        /Users/[redacted]/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:809 +0x6d
github.com/gorilla/websocket.(*Conn).NextReader(0xc0004fdb80)
        /Users/[redacted]/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1009 +0xb0
github.com/gorilla/websocket.(*Conn).ReadMessage(0xc0002c06e0?)
        /Users/[redacted]/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:1093 +0x13
github.com/decred/dcrd/rpcclient/v8.(*Client).wsInHandler(0xc0002c06e0)
        /Users/[redacted]/go/pkg/mod/github.com/decred/dcrd/rpcclient/[email protected]/infrastructure.go:455 +0xaf
created by github.com/decred/dcrd/rpcclient/v8.(*Client).start in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/decred/dcrd/rpcclient/[email protected]/infrastructure.go:1098 +0x198

goroutine 12 [select]:
github.com/decred/dcrd/rpcclient/v8.(*Client).wsOutHandler(0xc0002c06e0)
        /Users/[redacted]/go/pkg/mod/github.com/decred/dcrd/rpcclient/[email protected]/infrastructure.go:491 +0xa5
created by github.com/decred/dcrd/rpcclient/v8.(*Client).start in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/decred/dcrd/rpcclient/[email protected]/infrastructure.go:1099 +0x1d6

goroutine 13 [select]:
github.com/dgraph-io/badger/y.(*WaterMark).process(0xc0000b77c0, 0xc0000d4b10)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/y/watermark.go:229 +0x285
created by github.com/dgraph-io/badger/y.(*WaterMark).Init in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/y/watermark.go:78 +0x127

goroutine 14 [select]:
github.com/dgraph-io/badger/y.(*WaterMark).process(0xc0000b7800, 0xc0000d4b10)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/y/watermark.go:229 +0x285
created by github.com/dgraph-io/badger/y.(*WaterMark).Init in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/y/watermark.go:78 +0x127

goroutine 15 [select]:
github.com/dgraph-io/badger.(*DB).updateSize(0xc000116e08, 0xc0000d4b58)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/db.go:1029 +0x108
created by github.com/dgraph-io/badger.Open in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/db.go:293 +0xedd

goroutine 66 [select]:
github.com/dgraph-io/badger.(*levelsController).runWorker(0xc0000ea770, 0xc000122228)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/levels.go:376 +0x1d8
created by github.com/dgraph-io/badger.(*levelsController).startCompact in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/levels.go:357 +0x59

goroutine 67 [chan receive, 579 minutes]:
github.com/dgraph-io/badger.(*DB).flushMemtable(0xc000116e08, 0x0?)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/db.go:943 +0xb0
github.com/dgraph-io/badger.Open.func4()
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/db.go:310 +0x1e
created by github.com/dgraph-io/badger.Open in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/db.go:309 +0x11be

goroutine 70 [select]:
github.com/dgraph-io/badger.(*DB).doWrites(0xc000116e08, 0xc000122360)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/db.go:722 +0x211
created by github.com/dgraph-io/badger.Open in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/db.go:344 +0x176d

goroutine 69 [select, 579 minutes]:
github.com/dgraph-io/badger.(*valueLog).flushDiscardStats(0xc000116f78)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/value.go:1594 +0x14a
created by github.com/dgraph-io/badger.(*valueLog).open in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/value.go:849 +0x85

goroutine 71 [chan receive, 579 minutes]:
github.com/dgraph-io/badger.(*valueLog).waitOnGC(0xc000116f78, 0xc0002c06e0?)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/value.go:1501 +0x65
created by github.com/dgraph-io/badger.Open in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/db.go:347 +0x1885

goroutine 72 [select, 579 minutes]:
github.com/dgraph-io/badger.(*publisher).listenForUpdates(0xc000021230, 0xc0001223c0)
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/publisher.go:67 +0x125
created by github.com/dgraph-io/badger.Open in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/dgraph-io/[email protected]/db.go:350 +0x199a

goroutine 1401571 [select]:
github.com/decred/dcrdata/cmd/dcrdata/internal/api.(*appContext).StatusNtfnHandler(0xc0451e6a50, {0xfe6b3e0, 0xc0000ba1e0}, 0xc0000f6360?, 0xc04564ca20)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata:222 +0x12b
created by main._main in goroutine 1
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/main.go:647 +0x39e5

goroutine 1400140 [select, 256 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0xc00b02c8c0)
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/db_compaction.go:91 +0x149
created by github.com/syndtr/goleveldb/leveldb.openDB in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/db.go:148 +0x447

goroutine 1401240 [select, 578 minutes]:
database/sql.(*DB).connectionOpener(0xc000111c70, {0xfe6b3e0, 0xc03a8c6d70})
        /usr/local/go/src/database/sql/sql.go:1246 +0x87
created by database/sql.OpenDB in goroutine 1
        /usr/local/go/src/database/sql/sql.go:824 +0x14c

goroutine 1534661 [select]:
github.com/decred/dcrd/rpcclient/v8.receiveFuture({0xfe6b258?, 0x105d58c0?}, 0xc04ac89ce0)
        /Users/[redacted]/go/pkg/mod/github.com/decred/dcrd/rpcclient/[email protected]/infrastructure.go:855 +0x7c
github.com/decred/dcrd/rpcclient/v8.(*FutureGetBlockVerboseResult).Receive(0xc0002c06e0?)
        /Users/[redacted]/go/pkg/mod/github.com/decred/dcrd/rpcclient/[email protected]/chain.go:125 +0x25
github.com/decred/dcrd/rpcclient/v8.(*Client).GetBlockVerbose(0xc00a8bf838?, {0xfe6b258?, 0x105d58c0?}, 0x85a308d334e90c6c?, 0x89?)
        /Users/[redacted]/go/pkg/mod/github.com/decred/dcrd/rpcclient/[email protected]/chain.go:159 +0x25
github.com/decred/dcrdata/db/dcrpg/v8.(*ChainDB).GetBlockVerboseByHash(0xc03aa70000, {0xc05a40d600, 0x40}, 0x1)
        /Users/[redacted]/decred/dcrdata/db/dcrpg/insightapi.go:159 +0x118
github.com/decred/dcrdata/db/dcrpg/v8.(*ChainDB).GetExplorerBlock(0xc03aa70000, {0xc05a40d600, 0x40})
        /Users/[redacted]/decred/dcrdata/db/dcrpg/pgblockchain.go:5802 +0x13c
github.com/decred/dcrdata/v8/pubsub.(*PubSubHub).Store(0xc04485be60, 0xc0822f6380, 0xc02e876500)
        /Users/[redacted]/decred/dcrdata/pubsub/pubsubhub.go:614 +0x8e
github.com/decred/dcrdata/v8/blockdata.(*chainMonitor).ConnectBlock(0xc04ba35130, 0xc05519d110)
        /Users/[redacted]/decred/dcrdata/blockdata/chainmonitor.go:101 +0x1e4
github.com/decred/dcrdata/cmd/dcrdata/internal/notification.(*Notifier).processBlock.func1(0xc03a58d930)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/notification/notifier.go:308 +0xda
created by github.com/decred/dcrdata/cmd/dcrdata/internal/notification.(*Notifier).processBlock in goroutine 1464972
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/notification/notifier.go:301 +0x430

goroutine 1401573 [IO wait]:
internal/poll.runtime_pollWait(0x57659d98, 0x72)
        /usr/local/go/src/runtime/netpoll.go:345 +0x85
internal/poll.(*pollDesc).wait(0xc0451ded80?, 0x1?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0451ded80)
        /usr/local/go/src/internal/poll/fd_unix.go:611 +0x2ac
net.(*netFD).accept(0xc0451ded80)
        /usr/local/go/src/net/fd_unix.go:172 +0x29
net.(*TCPListener).accept(0xc04585c8e0)
        /usr/local/go/src/net/tcpsock_posix.go:159 +0x1e
net.(*TCPListener).Accept(0xc04585c8e0)
        /usr/local/go/src/net/tcpsock.go:327 +0x30
net/http.(*Server).Serve(0xc036df6a50, {0xfe69728, 0xc04585c8e0})
        /usr/local/go/src/net/http/server.go:3255 +0x33e
net/http.(*Server).ListenAndServe(0xc036df6a50)
        /usr/local/go/src/net/http/server.go:3184 +0x71
main.listenAndServeProto.func2()
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/main.go:1158 +0x53
created by main.listenAndServeProto in goroutine 1
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/main.go:1153 +0x2f1

goroutine 1400143 [select, 578 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0xc00b02c8c0)
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/db_compaction.go:773 +0x105
created by github.com/syndtr/goleveldb/leveldb.openDB in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/db.go:156 +0x536

goroutine 1401572 [chan receive, 578 minutes]:
main.listenAndServeProto.func1()
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/main.go:1136 +0x45
created by main.listenAndServeProto in goroutine 1
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/main.go:1134 +0x1c6

goroutine 1401556 [chan receive, 578 minutes]:
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*explorerUI).reloadTemplatesSig.func1()
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/explorer.go:256 +0x52
created by github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*explorerUI).reloadTemplatesSig in goroutine 1
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/explorer.go:254 +0xce

goroutine 1532597 [select]:
net/http.(*persistConn).writeLoop(0xc04b8907e0)
        /usr/local/go/src/net/http/transport.go:2444 +0xf0
created by net/http.(*Transport).dialConn in goroutine 1532403
        /usr/local/go/src/net/http/transport.go:1800 +0x1585

goroutine 1464972 [select]:
github.com/decred/dcrdata/cmd/dcrdata/internal/notification.(*Notifier).processBlock(0xc00001a090, 0xc05519d110)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/notification/notifier.go:319 +0x35a
github.com/decred/dcrdata/cmd/dcrdata/internal/notification.(*Notifier).superQueue(0xc00001a090, {0xfe6b3e0, 0xc0000ba1e0})
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/notification/notifier.go:149 +0x213
created by github.com/decred/dcrdata/cmd/dcrdata/internal/notification.(*Notifier).Listen in goroutine 1
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/notification/notifier.go:117 +0x1fa

goroutine 1532596 [select]:
net/http.(*persistConn).readLoop(0xc04b8907e0)
        /usr/local/go/src/net/http/transport.go:2261 +0xd3a
created by net/http.(*Transport).dialConn in goroutine 1532403
        /usr/local/go/src/net/http/transport.go:1799 +0x152f

goroutine 1401546 [select]:
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*WebsocketHub).periodicBufferSend(0xc0447fbb00)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/websocket.go:356 +0x75
created by github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*WebsocketHub).run in goroutine 1401554
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/websocket.go:205 +0xbb

goroutine 1401539 [chan receive]:
github.com/decred/dcrdata/v8/api/types.WatchPriorityQueue(0xc036ab3180)
        /Users/[redacted]/decred/dcrdata/api/types/apicache.go:43 +0x51
created by github.com/decred/dcrdata/v8/api/types.NewAPICache in goroutine 1
        /Users/[redacted]/decred/dcrdata/api/types/apicache.go:74 +0xe5

goroutine 1401557 [select]:
github.com/decred/dcrdata/v8/pubsub.(*WebsocketHub).Run(0xc04463c3f0)
        /Users/[redacted]/decred/dcrdata/pubsub/websocket.go:414 +0x2b0
created by github.com/decred/dcrdata/v8/pubsub.NewPubSubHub in goroutine 1
        /Users/[redacted]/decred/dcrdata/pubsub/pubsubhub.go:132 +0x3f6

goroutine 1534662 [semacquire]:
sync.runtime_Semacquire(0xc02cd545a0?)
        /usr/local/go/src/runtime/sema.go:62 +0x25
sync.(*WaitGroup).Wait(0xfe6b3e0?)
        /usr/local/go/src/sync/waitgroup.go:116 +0x48
github.com/decred/dcrdata/cmd/dcrdata/internal/notification.(*Notifier).processBlock.func2()
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/notification/notifier.go:316 +0x25
created by github.com/decred/dcrdata/cmd/dcrdata/internal/notification.(*Notifier).processBlock in goroutine 1464972
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/notification/notifier.go:315 +0x2fd

goroutine 1401570 [chan receive, 578 minutes]:
github.com/googollee/go-socket.io/engineio.(*Server).Accept(...)
        /Users/[redacted]/go/pkg/mod/github.com/googollee/[email protected]/engineio/server.go:114
github.com/googollee/go-socket%2eio.(*Server).Serve(0xc044afd480)
        /Users/[redacted]/go/pkg/mod/github.com/googollee/[email protected]/server.go:65 +0x4f
created by github.com/decred/dcrdata/cmd/dcrdata/internal/api/insight.NewSocketServer in goroutine 1
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/api/insight/socket.io.go:210 +0x839

goroutine 1401538 [chan receive]:
github.com/decred/dcrdata/v8/db/cache.(*AddressCache).Reporter(0xc00f024790)
        /Users/[redacted]/decred/dcrdata/db/cache/addresscache.go:757 +0xb9
created by github.com/decred/dcrdata/v8/db/cache.NewAddressCache.func1 in goroutine 1
        /Users/[redacted]/decred/dcrdata/db/cache/addresscache.go:728 +0x56

goroutine 1400142 [select, 4 minutes]:
github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0xc00b02c8c0)
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/db_compaction.go:836 +0x6ca
created by github.com/syndtr/goleveldb/leveldb.openDB in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/db.go:155 +0x4f6

goroutine 1401244 [select]:
github.com/decred/dcrdata/v8/pubsub.(*WebsocketHub).periodicTxBufferSend(0xc04463c3f0)
        /Users/[redacted]/decred/dcrdata/pubsub/websocket.go:552 +0x74
created by github.com/decred/dcrdata/v8/pubsub.(*WebsocketHub).Run in goroutine 1401557
        /Users/[redacted]/decred/dcrdata/pubsub/websocket.go:367 +0x105

goroutine 1401547 [select]:
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*WebsocketHub).pingClients.func1()
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/websocket.go:181 +0xf8
created by github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*WebsocketHub).pingClients in goroutine 1401554
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/websocket.go:175 +0x78

goroutine 1532330 [runnable]:
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*templates).execWithReload(0xc03890ba40, {0xf8476b1, 0xa}, {0xfd943c0, 0xc033ca1f00})
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/templates.go:106 +0x85
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*explorerUI).SideChains(0xc04410c000, {0xfe69c98, 0xc073482540}, 0xc045866480)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/explorerroutes.go:263 +0x216
net/http.HandlerFunc.ServeHTTP(0x0?, {0xfe69c98?, 0xc073482540?}, 0x1?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*explorerUI).SyncStatusPageIntercept-fm.(*explorerUI).SyncStatusPageIntercept.func1({0xfe69c98, 0xc073482540}, 0xc045866480)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/explorermiddleware.go:160 +0xcc
net/http.HandlerFunc.ServeHTTP(0xc04585ebe0?, {0xfe69c98?, 0xc073482540?}, 0x0?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
github.com/go-chi/chi/v5.(*ChainHandler).ServeHTTP(0xfcf8ae0?, {0xfe69c98?, 0xc073482540?}, 0xc00003a0f4?)
        /Users/[redacted]/go/pkg/mod/github.com/go-chi/chi/[email protected]/chain.go:31 +0x26
github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc04581cb40, {0xfe69c98, 0xc073482540}, 0xc045866480)
        /Users/[redacted]/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:444 +0x207
net/http.HandlerFunc.ServeHTTP(0xfe69c98?, {0xfe69c98?, 0xc073482540?}, 0x105d58c0?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
main._main.RequestBodyLimiter.func14.1({0xfe69c98, 0xc073482540}, 0xc045866480)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/middleware/apimiddleware.go:123 +0x87
net/http.HandlerFunc.ServeHTTP(0xc05a1eee20?, {0xfe69c98?, 0xc073482540?}, 0x10?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
github.com/go-chi/chi/v5/middleware.Recoverer.func1({0xfe69c98?, 0xc073482540?}, 0xf8422ad?)
        /Users/[redacted]/go/pkg/mod/github.com/go-chi/chi/[email protected]/middleware/recoverer.go:43 +0x78
net/http.HandlerFunc.ServeHTTP(0xc091e68d80?, {0xfe69c98?, 0xc073482540?}, 0xf8437a1?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
main._main.Server.func13.1({0xfe69c98, 0xc073482540}, 0xc045866480)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/middleware/apimiddleware.go:594 +0x8c
net/http.HandlerFunc.ServeHTTP(0xfe6b3e0?, {0xfe69c98?, 0xc073482540?}, 0x10548350?)
        /usr/local/go/src/net/http/server.go:2166 +0x29
github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc04581cb40, {0xfe69c98, 0xc073482540}, 0xc05528ed80)
        /Users/[redacted]/go/pkg/mod/github.com/go-chi/chi/[email protected]/mux.go:90 +0x2ee
net/http.serverHandler.ServeHTTP({0xc09542b7d0?}, {0xfe69c98?, 0xc073482540?}, 0x6?)
        /usr/local/go/src/net/http/server.go:3137 +0x8e
net/http.(*conn).serve(0xc0124366c0, {0xfe6b3a8, 0xc04586e6c0})
        /usr/local/go/src/net/http/server.go:2039 +0x5e8
created by net/http.(*Server).Serve in goroutine 1401573
        /usr/local/go/src/net/http/server.go:3285 +0x4b4

goroutine 1401245 [select]:
github.com/decred/dcrdata/v8/pubsub.(*WebsocketHub).pingClients.func1()
        /Users/[redacted]/decred/dcrdata/pubsub/websocket.go:326 +0xf8
created by github.com/decred/dcrdata/v8/pubsub.(*WebsocketHub).pingClients in goroutine 1401557
        /Users/[redacted]/decred/dcrdata/pubsub/websocket.go:320 +0x78

goroutine 1400141 [select]:
github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0xc00b02c8c0)
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/db_state.go:101 +0x9c
created by github.com/syndtr/goleveldb/leveldb.openDB in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/db.go:149 +0x485

goroutine 1401554 [select]:
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*WebsocketHub).run(0xc0447fbb00)
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/websocket.go:215 +0x245
created by github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.New in goroutine 1
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/explorer.go:384 +0xf8f

goroutine 1401239 [select, 4 minutes]:
github.com/syndtr/goleveldb/leveldb.(*session).refLoop(0xc0004d4000)
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/session_util.go:189 +0x59e
created by github.com/syndtr/goleveldb/leveldb.newSession in goroutine 1
        /Users/[redacted]/go/pkg/mod/github.com/syndtr/[email protected]/leveldb/session.go:93 +0x296

goroutine 1532098 [runnable]:
syscall.syscall(0xee6ae86?, 0xc00a8c47a8?, 0xef47b27?, 0x576597c8?)
        /usr/local/go/src/runtime/sys_darwin.go:23 +0x70
syscall.read(0xc0003ed780?, {0xc06f63065e?, 0x60ec6c60ddaaa000?, 0x4f476382b60007a3?})
        /usr/local/go/src/syscall/zsyscall_darwin_amd64.go:1245 +0x45
syscall.Read(...)
        /usr/local/go/src/syscall/syscall_unix.go:181
internal/poll.ignoringEINTRIO(...)
        /usr/local/go/src/internal/poll/fd_unix.go:736
internal/poll.(*FD).Read(0xc0003ed780, {0xc06f63065e, 0x39a2, 0x39a2})
        /usr/local/go/src/internal/poll/fd_unix.go:160 +0x2ae
net.(*netFD).Read(0xc0003ed780, {0xc06f63065e?, 0xfcac9f08?, 0xc03880ec90?})
        /usr/local/go/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc04a74e870, {0xc06f63065e?, 0xc00a8c4978?, 0xee744fb?})
        /usr/local/go/src/net/net.go:179 +0x45
crypto/tls.(*atLeastReader).Read(0xc03880ec90, {0xc06f63065e?, 0x0?, 0xc03880ec90?})
        /usr/local/go/src/crypto/tls/conn.go:806 +0x3b
bytes.(*Buffer).ReadFrom(0xc039b069b0, {0xfe64d20, 0xc03880ec90})
        /usr/local/go/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc039b06708, {0x5765a080, 0xc04a74e870}, 0xc00a8c49c0?)
        /usr/local/go/src/crypto/tls/conn.go:828 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc039b06708, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:677 +0xd3e
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:588
crypto/tls.(*Conn).Read(0xc039b06708, {0xc014697000, 0x1000, 0xc014697c9a?})
        /usr/local/go/src/crypto/tls/conn.go:1370 +0x156
net/http.(*persistConn).Read(0xc04b8907e0, {0xc014697000?, 0x104e0266?, 0x1?})
        /usr/local/go/src/net/http/transport.go:1977 +0x4a
bufio.(*Reader).Read(0xc04486aa20, {0xc02c734d43, 0x2bd, 0x2?})
        /usr/local/go/src/bufio/bufio.go:241 +0x197
net/http/internal.(*chunkedReader).Read(0xc09241afc0, {0xc02c734000?, 0x10?, 0x108d35b8?})
        /usr/local/go/src/net/http/internal/chunked.go:136 +0x16e
net/http.(*body).readLocked(0xc00f6ff5c0, {0xc02c734000?, 0xeeb8e09?, 0x8?})
        /usr/local/go/src/net/http/transfer.go:840 +0x3b
net/http.(*body).Read(0x0?, {0xc02c734000?, 0xd?, 0x1e?})
        /usr/local/go/src/net/http/transfer.go:832 +0x118
net/http.(*bodyEOFSignal).Read(0xc00f6ff600, {0xc02c734000, 0x1000, 0x1000})
        /usr/local/go/src/net/http/transport.go:2833 +0x13f
bufio.(*Reader).fill(0xc03a672120)
        /usr/local/go/src/bufio/bufio.go:110 +0x103
bufio.(*Reader).ReadByte(0xc03a672120)
        /usr/local/go/src/bufio/bufio.go:269 +0x27
compress/flate.(*decompressor).huffSym(0xc044af7308, 0xc044af7338)
        /usr/local/go/src/compress/flate/inflate.go:720 +0x102
compress/flate.(*decompressor).huffmanBlock(0xc044af7308?)
        /usr/local/go/src/compress/flate/inflate.go:495 +0x45
compress/flate.(*decompressor).nextBlock(0xc044af7308)
        /usr/local/go/src/compress/flate/inflate.go:328 +0x178
compress/flate.(*decompressor).Read(0xc044af7308, {0xc046ff8000, 0x8000, 0xc03a672120?})
        /usr/local/go/src/compress/flate/inflate.go:348 +0x5b
compress/gzip.(*Reader).Read(0xc0228622c8, {0xc046ff8000, 0x8000, 0x8000})
        /usr/local/go/src/compress/gzip/gunzip.go:252 +0xa2
net/http.(*gzipReader).Read(0xc04436bee0, {0xc046ff8000, 0x8000, 0x8000})
        /usr/local/go/src/net/http/transport.go:2897 +0x195
net/http.(*cancelTimerBody).Read(0xc04436bf00, {0xc046ff8000?, 0x1000?, 0x8000?})
        /usr/local/go/src/net/http/client.go:963 +0x2d
io.copyBuffer({0xfe647c0, 0xc04b8f7ae8}, {0xfe3bd440, 0xc04436bf00}, {0x0, 0x0, 0x0})
        /usr/local/go/src/io/io.go:429 +0x191
io.Copy(...)
        /usr/local/go/src/io/io.go:388
github.com/decred/politeia/util.RespBody(0xc021aebef0)
        /Users/[redacted]/go/pkg/mod/github.com/decred/[email protected]/util/net.go:95 +0x2a5
github.com/decred/politeia/politeiawww/client.(*Client).makeReq(0xc03a656c60, {0xf840245, 0x4}, {0xf84c8aa, 0xe}, {0xf845e2a?, 0x8?}, {0xfd318c0?, 0xc05a2b2b20?})
        /Users/[redacted]/go/pkg/mod/github.com/decred/[email protected]/politeiawww/client/client.go:137 +0xdde
github.com/decred/politeia/politeiawww/client.(*Client).TicketVoteResults(0xc03a656c60, {{0xc00a7beea0?, 0x2?}})
        /Users/[redacted]/go/pkg/mod/github.com/decred/[email protected]/politeiawww/client/ticketvote.go:109 +0x78
github.com/decred/dcrdata/gov/v6/politeia.(*ProposalsDB).fetchTicketVoteResults(0xc03a656c90, {0xc00a7beea0, 0x10})
        /Users/[redacted]/decred/dcrdata/gov/politeia/proposals.go:432 +0x217
github.com/decred/dcrdata/gov/v6/politeia.(*ProposalsDB).proposalsVoteResultsUpdate(0xc03a656c90)
        /Users/[redacted]/decred/dcrdata/gov/politeia/proposals.go:684 +0x846
github.com/decred/dcrdata/gov/v6/politeia.(*ProposalsDB).ProposalsSync(0xf1c5ef8?)
        /Users/[redacted]/decred/dcrdata/gov/politeia/proposals.go:158 +0x186
github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*explorerUI).Store.func3()
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/explorer.go:599 +0x24
created by github.com/decred/dcrdata/cmd/dcrdata/internal/explorer.(*explorerUI).Store in goroutine 1531562
        /Users/[redacted]/decred/dcrdata/cmd/dcrdata/internal/explorer/explorer.go:598 +0xd25