Deadlock when synchronizing FS mainnet #1472

Open
opened 2025-12-28 17:16:34 +00:00 by sami · 0 comments
Owner

Originally created by @roman-khimov on GitHub (Jan 14, 2025).

Current Behavior

Works good till stop at

2025-01-14T20:49:18.620+0300    INFO    runtime log     {"tx": "80fca8cf57303b9f548c9dbe847d45576cd9bf1ce9dd3a85dd45c1ab3cb5b6bc", "script": "dc1ec98d9d0c5f9dfade16144defe08cffc5ca55", "msg": "successfully transferred assets"}
2025-01-14T20:49:18.620+0300    INFO    runtime log     {"tx": "3021a1ae84546297304cff3b95117603eed4216f31d53af88c14fdb44f01f01a", "script": "dc1ec98d9d0c5f9dfade16144defe08cffc5ca55", "msg": "successfully transferred assets"}
2025-01-14T20:49:18.650+0300    INFO    runtime log     {"tx": "76699bc61d5eca08245529265d5612a432aa87d780024d691f8da9037f0a7e8b", "script": "7c5bdb23e36cc7cce95bf42f3ab9e452c2501df1", "msg": "process new epoch"}
2025-01-14T20:49:18.651+0300    INFO    persisted to disk       {"blocks": 1775, "keys": 38219, "headerHeight": 6574503, "blockHeight": 6574502, "took": "2.298806072s"}

which in fact is

goroutine 0 gp=0x1f36320 m=0 mp=0x1f37b60 [idle]:
runtime.futex(0x1f37ca0, 0x80, 0x0, 0x0, 0x0, 0x0)
        runtime/sys_linux_amd64.s:557 +0x21 fp=0x7ffddbbec180 sp=0x7ffddbbec178 pc=0x47f521
runtime.futexsleep(0x7ffddbbec1f8?, 0x40f2f0?, 0x18865?)
        runtime/os_linux.go:69 +0x30 fp=0x7ffddbbec1d0 sp=0x7ffddbbec180 pc=0x438510
runtime.notesleep(0x1f37ca0)
        runtime/lock_futex.go:170 +0x87 fp=0x7ffddbbec208 sp=0x7ffddbbec1d0 pc=0x40f487
runtime.mPark(...)
        runtime/proc.go:1866
runtime.stopm()
        runtime/proc.go:2885 +0x8c fp=0x7ffddbbec238 sp=0x7ffddbbec208 pc=0x443a6c
runtime.findRunnable()
        runtime/proc.go:3622 +0xd5c fp=0x7ffddbbec3b0 sp=0x7ffddbbec238 pc=0x4454dc
runtime.schedule()
        runtime/proc.go:3995 +0xb1 fp=0x7ffddbbec3e8 sp=0x7ffddbbec3b0 pc=0x4465b1
runtime.park_m(0xc0001bb6c0)
        runtime/proc.go:4102 +0x1eb fp=0x7ffddbbec440 sp=0x7ffddbbec3e8 pc=0x4469cb
runtime.mcall()
        runtime/asm_amd64.s:459 +0x4e fp=0x7ffddbbec458 sp=0x7ffddbbec440 pc=0x47b86e

goroutine 1 gp=0xc0000061c0 m=nil [sync.RWMutex.RLock, 46 minutes]:
runtime.gopark(0xc00b23f6b0?, 0x3?, 0xa0?, 0x2b?, 0xc00d240d20?)
        runtime/proc.go:424 +0xce fp=0xc0003692f0 sp=0xc0003692d0 pc=0x4755ce
runtime.goparkunlock(...)
        runtime/proc.go:430
runtime.semacquire1(0xc00026d04c, 0x0, 0x3, 0x0, 0x16)
        runtime/sema.go:178 +0x225 fp=0xc000369358 sp=0xc0003692f0 pc=0x4520c5
sync.runtime_SemacquireRWMutexR(0xc01032c270?, 0xc0?, 0xdd36d3?)
        runtime/sema.go:100 +0x25 fp=0xc000369390 sp=0xc000369358 pc=0x476ae5
sync.(*RWMutex).RLock(...)
        sync/rwmutex.go:72
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).rlock(0xc00fe6e9c0?)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:80 +0x32 fp=0xc0003693b8 sp=0xc000369390 pc=0xacbdd2
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Get(0xc00026d040, {0xc041d8a5a0, 0x1a, 0x45})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:93 +0x55 fp=0xc000369430 sp=0xc0003693b8 pc=0xacbeb5
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Get(0xc00fe5c960, {0xc041d8a5a0, 0x1a, 0x45})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:102 +0x1a2 fp=0xc0003694a8 sp=0xc000369430 pc=0xacc002
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Get(0xc00fe5cd20, {0xc041d8a5a0, 0x1a, 0x45})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:102 +0x1a2 fp=0xc000369520 sp=0xc0003694a8 pc=0xacc002
github.com/nspcc-dev/neo-go/pkg/core/dao.(*Simple).GetStorageItem(0xc00fe6ea80, 0x5, {0xc010316d08, 0x15, 0x0?})
        github.com/nspcc-dev/neo-go/pkg/core/dao/dao.go:357 +0x17b fp=0xc000369568 sp=0xc000369520 pc=0xcb257b
github.com/nspcc-dev/neo-go/pkg/core/interop/storage.Get(0xc00fe66ea0)
        github.com/nspcc-dev/neo-go/pkg/core/interop/storage/basic.go:49 +0x9c fp=0xc0003695d8 sp=0xc000369568 pc=0xda3f9c
github.com/nspcc-dev/neo-go/pkg/core/interop.(*Context).SyscallHandler(0xc00fe66ea0, 0x10d1cc0?, 0x369a38?)
        github.com/nspcc-dev/neo-go/pkg/core/interop/context.go:470 +0x99 fp=0xc000369630 sp=0xc0003695d8 pc=0xcbaf79
github.com/nspcc-dev/neo-go/pkg/core/interop.(*Context).SyscallHandler-fm(0xc04f434ebb?, 0x15d0240?)
        <autogenerated>:1 +0x28 fp=0xc000369658 sp=0xc000369630 pc=0xcc17a8
github.com/nspcc-dev/neo-go/pkg/vm.(*VM).execute(0xc00fe6e980, 0xc010328a80, 0x41, {0xc04f434ebb, 0x4, 0x22})
        github.com/nspcc-dev/neo-go/pkg/vm/vm.go:1523 +0x95d fp=0xc000369c70 sp=0xc000369658 pc=0xc741dd
github.com/nspcc-dev/neo-go/pkg/vm.(*VM).step(0xc00fe6e980, 0xc010328a80)
        github.com/nspcc-dev/neo-go/pkg/vm/vm.go:517 +0x1a5 fp=0xc000369d20 sp=0xc000369c70 pc=0xc734e5
github.com/nspcc-dev/neo-go/pkg/vm.(*VM).Run(0xc00fe6e980)
        github.com/nspcc-dev/neo-go/pkg/vm/vm.go:484 +0xd1 fp=0xc000369d40 sp=0xc000369d20 pc=0xc731f1
github.com/nspcc-dev/neo-go/pkg/core/interop.(*Context).Exec(0xc00fe6e980?)
        github.com/nspcc-dev/neo-go/pkg/core/interop/context.go:513 +0x50 fp=0xc000369d88 sp=0xc000369d40 pc=0xcbb630
github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).storeBlock(0xc000130908, 0xc010216900, 0xc00d2328f0)
        github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1718 +0x9aa fp=0xc00036a2c8 sp=0xc000369d88 pc=0xdf502a
github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).AddBlock(0xc000130908, 0xc010216900)
        github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1543 +0x68f fp=0xc00036a488 sp=0xc00036a2c8 pc=0xdf3b0f
github.com/nspcc-dev/neo-go/pkg/core/chaindump.Restore({0x15c6490, 0xc000130908}, 0xc0002b6420, 0x0, 0x6a5882, 0xc00036b650)
        github.com/nspcc-dev/neo-go/pkg/core/chaindump/dump.go:81 +0x2c8 fp=0xc00036a6d0 sp=0xc00036a488 pc=0xe0bf08
github.com/nspcc-dev/neo-go/cli/server.restoreDB(0xc00025d8c0)
        github.com/nspcc-dev/neo-go/cli/server/server.go:330 +0xac9 fp=0xc00036b6f8 sp=0xc00036a6d0 pc=0xeb8a49
github.com/urfave/cli/v2.(*Command).Run(0xc0001ad340, 0xc00025d8c0, {0xc00026cff0, 0x5, 0x5})
        github.com/urfave/cli/v2@v2.27.4/command.go:276 +0x7e2 fp=0xc00036b970 sp=0xc00036b6f8 pc=0x5fe9a2
github.com/urfave/cli/v2.(*Command).Run(0xc0001acf20, 0xc00025d780, {0xc000289ec0, 0x6, 0x6})
        github.com/urfave/cli/v2@v2.27.4/command.go:269 +0xa65 fp=0xc00036bbe8 sp=0xc00036b970 pc=0x5fec25
github.com/urfave/cli/v2.(*Command).Run(0xc0002a34a0, 0xc00025d640, {0xc00003e070, 0x7, 0x7})
        github.com/urfave/cli/v2@v2.27.4/command.go:269 +0xa65 fp=0xc00036be60 sp=0xc00036bbe8 pc=0x5fec25
github.com/urfave/cli/v2.(*App).RunContext(0xc0001b4a00, {0x15c6500, 0x1f58bc0}, {0xc00003e070, 0x7, 0x7})
        github.com/urfave/cli/v2@v2.27.4/app.go:333 +0x5a5 fp=0xc00036bec0 sp=0xc00036be60 pc=0x5fb405
github.com/urfave/cli/v2.(*App).Run(0xc000092740?, {0xc00003e070?, 0x68?, 0x0?})
        github.com/urfave/cli/v2@v2.27.4/app.go:307 +0x2f fp=0xc00036bf00 sp=0xc00036bec0 pc=0x5fae0f
main.main()
        ./main.go:13 +0x32 fp=0xc00036bf50 sp=0xc00036bf00 pc=0x1042df2
runtime.main()
        runtime/proc.go:272 +0x28b fp=0xc00036bfe0 sp=0xc00036bf50 pc=0x43edeb
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc00036bfe8 sp=0xc00036bfe0 pc=0x47d721

goroutine 2 gp=0xc000006700 m=nil [force gc (idle), 2 minutes]:
runtime.gopark(0x5b5c8dea658c?, 0x0?, 0x0?, 0x0?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000092fa8 sp=0xc000092f88 pc=0x4755ce
runtime.goparkunlock(...)
        runtime/proc.go:430
runtime.forcegchelper()
        runtime/proc.go:337 +0xb3 fp=0xc000092fe0 sp=0xc000092fa8 pc=0x43f133
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc000092fe8 sp=0xc000092fe0 pc=0x47d721
created by runtime.init.7 in goroutine 1
        runtime/proc.go:325 +0x1a

goroutine 3 gp=0xc000006c40 m=nil [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000093780 sp=0xc000093760 pc=0x4755ce
runtime.goparkunlock(...)
        runtime/proc.go:430
runtime.bgsweep(0xc000050080)
        runtime/mgcsweep.go:317 +0xdf fp=0xc0000937c8 sp=0xc000093780 pc=0x426a5f
runtime.gcenable.gowrap1()
        runtime/mgc.go:204 +0x25 fp=0xc0000937e0 sp=0xc0000937c8 pc=0x41b105
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0000937e8 sp=0xc0000937e0 pc=0x47d721
created by runtime.gcenable in goroutine 1
        runtime/mgc.go:204 +0x66

goroutine 4 gp=0xc000006e00 m=nil [GC scavenge wait, 2 minutes]:
runtime.gopark(0x440697?, 0x4354b5?, 0x0?, 0x0?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000093f78 sp=0xc000093f58 pc=0x4755ce
runtime.goparkunlock(...)
        runtime/proc.go:430
runtime.(*scavengerState).park(0x1f35960)
        runtime/mgcscavenge.go:425 +0x49 fp=0xc000093fa8 sp=0xc000093f78 pc=0x424429
runtime.bgscavenge(0xc000050080)
        runtime/mgcscavenge.go:658 +0x59 fp=0xc000093fc8 sp=0xc000093fa8 pc=0x4249b9
runtime.gcenable.gowrap2()
        runtime/mgc.go:205 +0x25 fp=0xc000093fe0 sp=0xc000093fc8 pc=0x41b0a5
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc000093fe8 sp=0xc000093fe0 pc=0x47d721
created by runtime.gcenable in goroutine 1
        runtime/mgc.go:205 +0xa5

goroutine 5 gp=0xc000007340 m=nil [finalizer wait, 123 minutes]:
runtime.gopark(0x490013?, 0xc000092660?, 0x9e?, 0x5?, 0x7fad2a0be128?)
        runtime/proc.go:424 +0xce fp=0xc000092620 sp=0xc000092600 pc=0x4755ce
runtime.runfinq()
        runtime/mfinal.go:193 +0x107 fp=0xc0000927e0 sp=0xc000092620 pc=0x41a187
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0000927e8 sp=0xc0000927e0 pc=0x47d721
created by runtime.createfing in goroutine 1
        runtime/mfinal.go:163 +0x3d

goroutine 6 gp=0xc0001bb6c0 m=nil [chan receive, 2 minutes]:
runtime.gopark(0xc000094760?, 0x688465?, 0x30?, 0x8c?, 0x15e2540?)
        runtime/proc.go:424 +0xce fp=0xc000094718 sp=0xc0000946f8 pc=0x4755ce
runtime.chanrecv(0xc0000be620, 0x0, 0x1)
        runtime/chan.go:639 +0x41c fp=0xc000094790 sp=0xc000094718 pc=0x409d7c
runtime.chanrecv1(0x0?, 0x0?)
        runtime/chan.go:489 +0x12 fp=0xc0000947b8 sp=0xc000094790 pc=0x409932
runtime.unique_runtime_registerUniqueMapCleanup.func1(...)
        runtime/mgc.go:1781
runtime.unique_runtime_registerUniqueMapCleanup.gowrap1()
        runtime/mgc.go:1784 +0x2f fp=0xc0000947e0 sp=0xc0000947b8 pc=0x41e12f
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0000947e8 sp=0xc0000947e0 pc=0x47d721
created by unique.runtime_registerUniqueMapCleanup in goroutine 1
        runtime/mgc.go:1779 +0x96

goroutine 104 gp=0xc0001bba40 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84fb4744d?, 0x1?, 0x6f?, 0xe9?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000094f38 sp=0xc000094f18 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc000094fc8 sp=0xc000094f38 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc000094fe0 sp=0xc000094fc8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc000094fe8 sp=0xc000094fe0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 105 gp=0xc0001bbc00 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84fcf64e0?, 0x1?, 0x70?, 0x81?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000095738 sp=0xc000095718 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc0000957c8 sp=0xc000095738 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc0000957e0 sp=0xc0000957c8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0000957e8 sp=0xc0000957e0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 106 gp=0xc0001bbdc0 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84fa81713?, 0x3?, 0xde?, 0x88?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000095f38 sp=0xc000095f18 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc000095fc8 sp=0xc000095f38 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc000095fe0 sp=0xc000095fc8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 107 gp=0xc0002d4000 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84facc1e2?, 0x3?, 0x12?, 0xb?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc00008e738 sp=0xc00008e718 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc00008e7c8 sp=0xc00008e738 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc00008e7e0 sp=0xc00008e7c8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc00008e7e8 sp=0xc00008e7e0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 108 gp=0xc0002d41c0 m=nil [GC worker (idle)]:
runtime.gopark(0x1f5b200?, 0x1?, 0x0?, 0xac?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc00008ef38 sp=0xc00008ef18 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc00008efc8 sp=0xc00008ef38 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc00008efe0 sp=0xc00008efc8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc00008efe8 sp=0xc00008efe0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 109 gp=0xc0002d4380 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84fa816af?, 0x3?, 0xb8?, 0x3d?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc00008f738 sp=0xc00008f718 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc00008f7c8 sp=0xc00008f738 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc00008f7e0 sp=0xc00008f7c8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc00008f7e8 sp=0xc00008f7e0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 110 gp=0xc0002d4540 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84fa8136f?, 0x3?, 0x48?, 0x7c?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc00008ff38 sp=0xc00008ff18 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc00008ffc8 sp=0xc00008ff38 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc00008ffe0 sp=0xc00008ffc8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc00008ffe8 sp=0xc00008ffe0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 111 gp=0xc0002d4700 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84fb366ce?, 0x1?, 0xc4?, 0x1?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000090738 sp=0xc000090718 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc0000907c8 sp=0xc000090738 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc0000907e0 sp=0xc0000907c8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0000907e8 sp=0xc0000907e0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 112 gp=0xc0002d48c0 m=nil [GC worker (idle), 2 minutes]:
runtime.gopark(0x5b5c8e3b6488?, 0x1?, 0xb?, 0x13?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000090f38 sp=0xc000090f18 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc000090fc8 sp=0xc000090f38 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc000090fe0 sp=0xc000090fc8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc000090fe8 sp=0xc000090fe0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 129 gp=0xc0002d4a80 m=nil [GC worker (idle), 47 minutes]:
runtime.gopark(0x58ccee83b7c9?, 0xc0002b0fa0?, 0x1a?, 0xa?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000091738 sp=0xc000091718 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc0000917c8 sp=0xc000091738 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc0000917e0 sp=0xc0000917c8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0000917e8 sp=0xc0000917e0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 130 gp=0xc0002d4c40 m=nil [GC worker (idle), 2 minutes]:
runtime.gopark(0x5b5c8e2fcce1?, 0x1?, 0x5b?, 0xd7?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc000091f38 sp=0xc000091f18 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc000091fc8 sp=0xc000091f38 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc000091fe0 sp=0xc000091fc8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc000091fe8 sp=0xc000091fe0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 131 gp=0xc0002d4e00 m=nil [GC worker (idle)]:
runtime.gopark(0x5b5c8e2fd70d?, 0x1?, 0x96?, 0xa3?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc0002da738 sp=0xc0002da718 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc0002da7c8 sp=0xc0002da738 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc0002da7e0 sp=0xc0002da7c8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0002da7e8 sp=0xc0002da7e0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 132 gp=0xc0002d4fc0 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84fa59fea?, 0x3?, 0x26?, 0xb2?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc0002daf38 sp=0xc0002daf18 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc0002dafc8 sp=0xc0002daf38 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc0002dafe0 sp=0xc0002dafc8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0002dafe8 sp=0xc0002dafe0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 133 gp=0xc0002d5180 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84fa8060d?, 0x3?, 0xa8?, 0x66?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc0002db738 sp=0xc0002db718 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc0002db7c8 sp=0xc0002db738 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc0002db7e0 sp=0xc0002db7c8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0002db7e8 sp=0xc0002db7e0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 134 gp=0xc0002d5340 m=nil [GC worker (idle), 47 minutes]:
runtime.gopark(0x58d62ae10690?, 0x1?, 0x23?, 0xbd?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc0002dbf38 sp=0xc0002dbf18 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc0002dbfc8 sp=0xc0002dbf38 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc0002dbfe0 sp=0xc0002dbfc8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0002dbfe8 sp=0xc0002dbfe0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 135 gp=0xc0002d5500 m=nil [GC worker (idle), 46 minutes]:
runtime.gopark(0x58d84facc68a?, 0x3?, 0xf5?, 0xa?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc0002dc738 sp=0xc0002dc718 pc=0x4755ce
runtime.gcBgMarkWorker(0xc0000be8c0)
        runtime/mgc.go:1412 +0xe9 fp=0xc0002dc7c8 sp=0xc0002dc738 pc=0x41d449
runtime.gcBgMarkStartWorkers.gowrap1()
        runtime/mgc.go:1328 +0x25 fp=0xc0002dc7e0 sp=0xc0002dc7c8 pc=0x41d325
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0002dc7e8 sp=0xc0002dc7e0 pc=0x47d721
created by runtime.gcBgMarkStartWorkers in goroutine 1
        runtime/mgc.go:1328 +0x105

goroutine 86 gp=0xc000105c00 m=nil [select, 46 minutes]:
runtime.gopark(0xc0003daef0?, 0x4?, 0x9?, 0x3?, 0xc0003dab48?)
        runtime/proc.go:424 +0xce fp=0xc0003da888 sp=0xc0003da868 pc=0x4755ce
runtime.selectgo(0xc0003daef0, 0xc0003dab40, 0xc0003dae10?, 0x0, 0x4?, 0x1)
        runtime/select.go:335 +0x7a5 fp=0xc0003da9b0 sp=0xc0003da888 pc=0x451025
github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).notificationDispatcher(0xc000130908)
        github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1378 +0x2ef fp=0xc0003dafc8 sp=0xc0003da9b0 pc=0xdf24af
github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).Run.gowrap1()
        github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1114 +0x25 fp=0xc0003dafe0 sp=0xc0003dafc8 pc=0xdf0465
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0003dafe8 sp=0xc0003dafe0 pc=0x47d721
created by github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).Run in goroutine 140
        github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1114 +0xe5

goroutine 140 gp=0xc000105dc0 m=nil [sync.RWMutex.Lock, 46 minutes]:
runtime.gopark(0xc045ac5f80?, 0x10?, 0xa0?, 0x65?, 0xc0041d0e38?)
        runtime/proc.go:424 +0xce fp=0xc0041d0de8 sp=0xc0041d0dc8 pc=0x4755ce
runtime.goparkunlock(...)
        runtime/proc.go:430
runtime.semacquire1(0xc0001ab3d8, 0x0, 0x3, 0x0, 0x17)
        runtime/sema.go:178 +0x225 fp=0xc0041d0e50 sp=0xc0041d0de8 pc=0x4520c5
sync.runtime_SemacquireRWMutex(0x47005d?, 0x30?, 0x47005d?)
        runtime/sema.go:105 +0x25 fp=0xc0041d0e88 sp=0xc0041d0e50 pc=0x476b45
sync.(*RWMutex).Lock(0x10000002000?)
        sync/rwmutex.go:153 +0x6a fp=0xc0041d0eb8 sp=0xc0041d0e88 pc=0x49778a
go.etcd.io/bbolt.(*DB).mmap(0xc0001ab208, 0x900000000)
        go.etcd.io/bbolt@v1.3.11/db.go:433 +0x4f fp=0xc0041d0fd0 sp=0xc0041d0eb8 pc=0xa68bcf
go.etcd.io/bbolt.(*DB).allocate(0xc0001ab208, 0xd0e, 0x1)
        go.etcd.io/bbolt@v1.3.11/db.go:1132 +0x10e fp=0xc0041d1028 sp=0xc0041d0fd0 pc=0xa6bbae
go.etcd.io/bbolt.(*Tx).allocate(0xc0011ed880, 0x1)
        go.etcd.io/bbolt@v1.3.11/tx.go:417 +0x36 fp=0xc0041d1058 sp=0xc0041d1028 pc=0xa75036
go.etcd.io/bbolt.(*node).spill(0xc03a532310)
        go.etcd.io/bbolt@v1.3.11/node.go:368 +0x212 fp=0xc0041d1138 sp=0xc0041d1058 pc=0xa71352
go.etcd.io/bbolt.(*node).spill(0xc03a5322a0)
        go.etcd.io/bbolt@v1.3.11/node.go:350 +0x9e fp=0xc0041d1218 sp=0xc0041d1138 pc=0xa711de
go.etcd.io/bbolt.(*node).spill(0xc03d25b880)
        go.etcd.io/bbolt@v1.3.11/node.go:350 +0x9e fp=0xc0041d12f8 sp=0xc0041d1218 pc=0xa711de
go.etcd.io/bbolt.(*node).spill(0xc0095eaee0)
        go.etcd.io/bbolt@v1.3.11/node.go:350 +0x9e fp=0xc0041d13d8 sp=0xc0041d12f8 pc=0xa711de
go.etcd.io/bbolt.(*node).spill(0xc02e5cc070)
        go.etcd.io/bbolt@v1.3.11/node.go:350 +0x9e fp=0xc0041d14b8 sp=0xc0041d13d8 pc=0xa711de
go.etcd.io/bbolt.(*Bucket).spill(0xc00d22e740)
        go.etcd.io/bbolt@v1.3.11/bucket.go:592 +0x365 fp=0xc0041d16c8 sp=0xc0041d14b8 pc=0xa65905
go.etcd.io/bbolt.(*Bucket).spill(0xc0011ed898)
        go.etcd.io/bbolt@v1.3.11/bucket.go:559 +0x105 fp=0xc0041d18d8 sp=0xc0041d16c8 pc=0xa656a5
go.etcd.io/bbolt.(*Tx).Commit(0xc0011ed880)
        go.etcd.io/bbolt@v1.3.11/tx.go:164 +0xf9 fp=0xc0041d1a00 sp=0xc0041d18d8 pc=0xa73df9
go.etcd.io/bbolt.(*DB).Update(0x470639?, 0xc0041d1a88)
        go.etcd.io/bbolt@v1.3.11/db.go:893 +0xca fp=0xc0041d1a68 sp=0xc0041d1a00 pc=0xa6ab6a
github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).PutChangeSet(0x0?, 0x0?, 0x0?)
        github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:94 +0x45 fp=0xc0041d1ab8 sp=0xc0041d1a68 pc=0xaca7c5
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).persist(0xc00026d040, 0x1)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:381 +0x26b fp=0xc0041d1c08 sp=0xc0041d1ab8 pc=0xacebeb
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).PersistSync(...)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:339
github.com/nspcc-dev/neo-go/pkg/core/dao.(*Simple).PersistSync(0x8ed06ef19cd93b14?)
        github.com/nspcc-dev/neo-go/pkg/core/dao/dao.go:971 +0xfb fp=0xc0041d1ce0 sp=0xc0041d1c08 pc=0xcb7ebb
github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).persist(0xc000130908, 0x1)
        github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:2161 +0x65 fp=0xc0041d1ea0 sp=0xc0041d1ce0 pc=0xdf8e85
github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).Run(0xc000130908)
        github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1127 +0x1bd fp=0xc0041d1fc8 sp=0xc0041d1ea0 pc=0xdf029d
github.com/nspcc-dev/neo-go/cli/server.initBCWithMetrics.gowrap1()
        github.com/nspcc-dev/neo-go/cli/server/server.go:151 +0x25 fp=0xc0041d1fe0 sp=0xc0041d1fc8 pc=0xeb7565
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0041d1fe8 sp=0xc0041d1fe0 pc=0x47d721
created by github.com/nspcc-dev/neo-go/cli/server.initBCWithMetrics in goroutine 1
        github.com/nspcc-dev/neo-go/cli/server/server.go:151 +0x13c

goroutine 141 gp=0xc0006ce380 m=nil [select, 123 minutes, locked to thread]:
runtime.gopark(0xc0002d8fa8?, 0x2?, 0xa8?, 0x8e?, 0xc0002d8f94?)
        runtime/proc.go:424 +0xce fp=0xc0002d8e30 sp=0xc0002d8e10 pc=0x4755ce
runtime.selectgo(0xc0002d8fa8, 0xc0002d8f90, 0x0?, 0x0, 0x0?, 0x1)
        runtime/select.go:335 +0x7a5 fp=0xc0002d8f58 sp=0xc0002d8e30 pc=0x451025
runtime.ensureSigM.func1()
        runtime/signal_unix.go:1077 +0x19f fp=0xc0002d8fe0 sp=0xc0002d8f58 pc=0x46cb3f
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc0002d8fe8 sp=0xc0002d8fe0 pc=0x47d721
created by runtime.ensureSigM in goroutine 1
        runtime/signal_unix.go:1060 +0xc8

goroutine 120 gp=0xc000704380 m=6 mp=0xc000700008 [syscall, 123 minutes]:
runtime.notetsleepg(0x1f59d20, 0xffffffffffffffff)
        runtime/lock_futex.go:246 +0x29 fp=0xc00033dfa0 sp=0xc00033df78 pc=0x40f769
os/signal.signal_recv()
        runtime/sigqueue.go:152 +0x29 fp=0xc00033dfc0 sp=0xc00033dfa0 pc=0x477509
os/signal.loop()
        os/signal/signal_unix.go:23 +0x13 fp=0xc00033dfe0 sp=0xc00033dfc0 pc=0xcf5c13
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc00033dfe8 sp=0xc00033dfe0 pc=0x47d721
created by os/signal.Notify.func1.1 in goroutine 1
        os/signal/signal.go:151 +0x1f

goroutine 121 gp=0xc000704540 m=nil [chan receive, 123 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc00033e718 sp=0xc00033e6f8 pc=0x4755ce
runtime.chanrecv(0xc00013d960, 0x0, 0x1)
        runtime/chan.go:639 +0x41c fp=0xc00033e790 sp=0xc00033e718 pc=0x409d7c
runtime.chanrecv1(0x0?, 0x0?)
        runtime/chan.go:489 +0x12 fp=0xc00033e7b8 sp=0xc00033e790 pc=0x409932
github.com/nspcc-dev/neo-go/cli/server.newGraceContext.func1()
        github.com/nspcc-dev/neo-go/cli/server/server.go:137 +0x25 fp=0xc00033e7e0 sp=0xc00033e7b8 pc=0xeb7185
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc00033e7e8 sp=0xc00033e7e0 pc=0x47d721
created by github.com/nspcc-dev/neo-go/cli/server.newGraceContext in goroutine 1
        github.com/nspcc-dev/neo-go/cli/server/server.go:136 +0xf7

goroutine 27196640 gp=0xc000704c40 m=nil [select, 46 minutes]:
runtime.gopark(0xc010081350?, 0x2?, 0x0?, 0x0?, 0xc010081344?)
        runtime/proc.go:424 +0xce fp=0xc0100811e8 sp=0xc0100811c8 pc=0x4755ce
runtime.selectgo(0xc010081350, 0xc010081340, 0x0?, 0x1, 0xc0100813a0?, 0x1)
        runtime/select.go:335 +0x7a5 fp=0xc010081310 sp=0xc0100811e8 pc=0x451025
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync.func1.1({0xc00fe7e526, 0x15, 0x1a}, {0x1f58bc0, 0x0, 0x0})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:175 +0xfd fp=0xc0100813b0 sp=0xc010081310 pc=0xacd31d
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc00fe7e500?, 0x1b?, 0xc010300b40?}, {0x1f58bc0, 0x0, 0x8?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100814c8 sp=0xc0100813b0 pc=0xace785
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc00fe7e4e0?, 0x1b?, 0xc010081688?}, {0x1f58bc0, 0x0, 0xc028b622b0?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100815e0 sp=0xc0100814c8 pc=0xace785
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc00fe7e4c0?, 0x1b?, 0x8?}, {0x1f58bc0, 0x0, 0xa67325?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100816f8 sp=0xc0100815e0 pc=0xace785
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0x7fab9f1479ba?, 0x1b?, 0x410845?}, {0x7fab9f147130, 0x0, 0x1?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc010081810 sp=0xc0100816f8 pc=0xace785
github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).Seek.func1(0xc010300b40?, {0x7fab9f1479ba?, 0x2?, 0x0?}, {0x7fab9f147130?, 0x3fe0000000000000?, 0xc00b5308f0?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:127 +0x34 fp=0xc010081850 sp=0xc010081810 pc=0xacadd4
github.com/nspcc-dev/neo-go/pkg/core/storage.boltSeek.func1(0x8?)
        github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:159 +0x2e4 fp=0xc010081918 sp=0xc010081850 pc=0xacb2e4
go.etcd.io/bbolt.(*DB).View(0x1216e40?, 0xc03bfc8360)
        go.etcd.io/bbolt@v1.3.11/db.go:917 +0x6c fp=0xc010081980 sp=0xc010081918 pc=0xa6ac8c
go.etcd.io/bbolt.(*DB).View-fm(0xc028b62258?)
        <autogenerated>:1 +0x26 fp=0xc0100819a0 sp=0xc010081980 pc=0xad9766
github.com/nspcc-dev/neo-go/pkg/core/storage.boltSeek(0xc00b530a50, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc00fb0dcb0)
        github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:136 +0x13a fp=0xc010081a00 sp=0xc0100819a0 pc=0xacaf7a
github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).Seek(0x0?, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e7018f0)
        github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:126 +0xf5 fp=0xc010081a70 sp=0xc010081a00 pc=0xacad15
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8d60, 0xc000096c48}, {0x0, 0x0, 0x0}, {{0xc028b62258, 0x6, 0x8}, ...}, ...)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010081b20 sp=0xc010081a70 pc=0xacdea2
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701880)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010081bb8 sp=0xc010081b20 pc=0xaccda8
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8da0, 0xc0261627d0}, {0x0, 0x0, 0x0}, {{0xc028b62258, 0x6, 0x8}, ...}, ...)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010081c68 sp=0xc010081bb8 pc=0xacdea2
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701810)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010081d00 sp=0xc010081c68 pc=0xaccda8
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8da0, 0xc00026d040}, {0x0, 0x0, 0x0}, {{0xc028b62258, 0x6, 0x8}, ...}, ...)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010081db0 sp=0xc010081d00 pc=0xacdea2
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e7017a0)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010081e48 sp=0xc010081db0 pc=0xaccda8
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6998, 0xc00fe5cdc0}, {0x15c8da0, 0xc00fe5c960}, {0x0, 0x0, 0x0}, {{0xc028b62258, 0x6, 0x8}, ...}, ...)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010081ef8 sp=0xc010081e48 pc=0xacdea2
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync.func1()
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:174 +0x169 fp=0xc010081fe0 sp=0xc010081ef8 pc=0xacd1e9
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc010081fe8 sp=0xc010081fe0 pc=0x47d721
created by github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync in goroutine 1
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:173 +0x1bc

goroutine 27196705 gp=0xc000d05dc0 m=nil [select, 46 minutes]:
runtime.gopark(0xc010089350?, 0x2?, 0x0?, 0x0?, 0xc010089344?)
        runtime/proc.go:424 +0xce fp=0xc0100891e8 sp=0xc0100891c8 pc=0x4755ce
runtime.selectgo(0xc010089350, 0xc010089340, 0x0?, 0x1, 0xc0100893a0?, 0x1)
        runtime/select.go:335 +0x7a5 fp=0xc010089310 sp=0xc0100891e8 pc=0x451025
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync.func1.1({0xc010324a66, 0x14, 0x1a}, {0xc027c83390, 0xd, 0x10})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:175 +0xfd fp=0xc0100893b0 sp=0xc010089310 pc=0xacd31d
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc010324a40?, 0x1a?, 0xc010300e09?}, {0xc027c83380, 0xd, 0x8?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100894c8 sp=0xc0100893b0 pc=0xace785
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc00fe7e8c0?, 0x1a?, 0xc010089609?}, {0xc025c388d0, 0xd, 0xc028b624c8?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100895e0 sp=0xc0100894c8 pc=0xace785
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc0103249e0?, 0x1a?, 0x8?}, {0xc027c83350, 0xd, 0xa67325?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:273 +0x443 fp=0xc0100896f8 sp=0xc0100895e0 pc=0xace5c3
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0x7fab9f12be85?, 0x1a?, 0x410845?}, {0x7fab9f12be9f, 0xd, 0x1?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc010089810 sp=0xc0100896f8 pc=0xace785
github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).Seek.func1(0xc010300e40?, {0x7fab9f12be85?, 0x2?, 0x0?}, {0x7fab9f12be9f?, 0x3fe0000000000000?, 0xc00b5308f0?})
        github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:127 +0x34 fp=0xc010089850 sp=0xc010089810 pc=0xacadd4
github.com/nspcc-dev/neo-go/pkg/core/storage.boltSeek.func1(0x8?)
        github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:159 +0x2e4 fp=0xc010089918 sp=0xc010089850 pc=0xacb2e4
go.etcd.io/bbolt.(*DB).View(0x1216e40?, 0xc03bfc8540)
        go.etcd.io/bbolt@v1.3.11/db.go:917 +0x6c fp=0xc010089980 sp=0xc010089918 pc=0xa6ac8c
go.etcd.io/bbolt.(*DB).View-fm(0xc028b62470?)
        <autogenerated>:1 +0x26 fp=0xc0100899a0 sp=0xc010089980 pc=0xad9766
github.com/nspcc-dev/neo-go/pkg/core/storage.boltSeek(0xc00b530a50, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc00fb0dd90)
        github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:136 +0x13a fp=0xc010089a00 sp=0xc0100899a0 pc=0xacaf7a
github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).Seek(0xc010313008?, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701c00)
        github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:126 +0xf5 fp=0xc010089a70 sp=0xc010089a00 pc=0xacad15
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8d60, 0xc000096c48}, {0xc010313008, 0x29, 0x49}, {{0xc028b62470, 0x6, 0x8}, ...}, ...)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010089b20 sp=0xc010089a70 pc=0xacdea2
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0xc010312008?, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701b20)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010089bb8 sp=0xc010089b20 pc=0xaccda8
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8da0, 0xc0261627d0}, {0xc010312008, 0x2b, 0x49}, {{0xc028b62470, 0x6, 0x8}, ...}, ...)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010089c68 sp=0xc010089bb8 pc=0xacdea2
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701a40)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010089d00 sp=0xc010089c68 pc=0xaccda8
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8da0, 0xc00026d040}, {0x0, 0x0, 0x0}, {{0xc028b62470, 0x6, 0x8}, ...}, ...)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010089db0 sp=0xc010089d00 pc=0xacdea2
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e7019d0)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010089e48 sp=0xc010089db0 pc=0xaccda8
github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6998, 0xc00fe5ce10}, {0x15c8da0, 0xc00fe5c960}, {0x0, 0x0, 0x0}, {{0xc028b62470, 0x6, 0x8}, ...}, ...)
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010089ef8 sp=0xc010089e48 pc=0xacdea2
github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync.func1()
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:174 +0x169 fp=0xc010089fe0 sp=0xc010089ef8 pc=0xacd1e9
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc010089fe8 sp=0xc010089fe0 pc=0x47d721
created by github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync in goroutine 1
        github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:173 +0x1bc

goroutine 27196638 gp=0xc001100a80 m=nil [chan receive, 46 minutes]:
runtime.gopark(0xc010216900?, 0xc00fe45e00?, 0x0?, 0x0?, 0x0?)
        runtime/proc.go:424 +0xce fp=0xc0100837a8 sp=0xc010083788 pc=0x4755ce
runtime.chanrecv(0xc03f47aaf0, 0xc010083a28, 0x1)
        runtime/chan.go:639 +0x41c fp=0xc010083820 sp=0xc0100837a8 pc=0x409d7c
runtime.chanrecv2(0x65f1a1808b62b04b?, 0x442cb62d94b4b97e?)
        runtime/chan.go:494 +0x12 fp=0xc010083848 sp=0xc010083820 pc=0x409952
github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).storeBlock.func1()
        github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1657 +0x1fb fp=0xc010083fe0 sp=0xc010083848 pc=0xdf651b
runtime.goexit({})
        runtime/asm_amd64.s:1700 +0x1 fp=0xc010083fe8 sp=0xc010083fe0 pc=0x47d721
created by github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).storeBlock in goroutine 1
        github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1622 +0x5be

Notice 46m stalls, these are locked ~since the last log message.

Expected Behavior

Working till the end.

Possible Solution

Fix it.

Steps to Reproduce

./bin/neo-go db restore --config-file config/protocol.mainnet.neofs.yml -i ../chains/mainnet-neofs-6969473.acc

config is changed for BoltDB:

--- a/config/protocol.mainnet.neofs.yml
+++ b/config/protocol.mainnet.neofs.yml
@@ -34,12 +34,12 @@ ApplicationConfiguration:
   # LogPath could be set up in case you need stdout logs to some proper file.
   # LogPath: "./log/neogo.log"
   DBConfiguration:
-    Type: "leveldb" #other options: 'inmemory','boltdb'
+    Type: "boltdb" #other options: 'inmemory','boltdb'
     # DB type options. Uncomment those you need in case you want to switch DB type.
     LevelDBOptions:
       DataDirectoryPath: "./chains/mainnet.neofs"
-  #    BoltDBOptions:
-  #      FilePath: "./chains/mainnet.bolt"
+    BoltDBOptions:
+      FilePath: "./chains/mainnet.neofs.bolt"
   P2P:
     Addresses:
       - ":40333" # in form of "[host]:[port][:announcedPort]"

Regression

Not likely.

Your Environment

  • Version of the product used: 0.107.2
  • Operating System and version (uname -a): openSUSE 15.6
Originally created by @roman-khimov on GitHub (Jan 14, 2025). ## Current Behavior Works good till stop at ``` 2025-01-14T20:49:18.620+0300 INFO runtime log {"tx": "80fca8cf57303b9f548c9dbe847d45576cd9bf1ce9dd3a85dd45c1ab3cb5b6bc", "script": "dc1ec98d9d0c5f9dfade16144defe08cffc5ca55", "msg": "successfully transferred assets"} 2025-01-14T20:49:18.620+0300 INFO runtime log {"tx": "3021a1ae84546297304cff3b95117603eed4216f31d53af88c14fdb44f01f01a", "script": "dc1ec98d9d0c5f9dfade16144defe08cffc5ca55", "msg": "successfully transferred assets"} 2025-01-14T20:49:18.650+0300 INFO runtime log {"tx": "76699bc61d5eca08245529265d5612a432aa87d780024d691f8da9037f0a7e8b", "script": "7c5bdb23e36cc7cce95bf42f3ab9e452c2501df1", "msg": "process new epoch"} 2025-01-14T20:49:18.651+0300 INFO persisted to disk {"blocks": 1775, "keys": 38219, "headerHeight": 6574503, "blockHeight": 6574502, "took": "2.298806072s"} ``` which in fact is ``` goroutine 0 gp=0x1f36320 m=0 mp=0x1f37b60 [idle]: runtime.futex(0x1f37ca0, 0x80, 0x0, 0x0, 0x0, 0x0) runtime/sys_linux_amd64.s:557 +0x21 fp=0x7ffddbbec180 sp=0x7ffddbbec178 pc=0x47f521 runtime.futexsleep(0x7ffddbbec1f8?, 0x40f2f0?, 0x18865?) runtime/os_linux.go:69 +0x30 fp=0x7ffddbbec1d0 sp=0x7ffddbbec180 pc=0x438510 runtime.notesleep(0x1f37ca0) runtime/lock_futex.go:170 +0x87 fp=0x7ffddbbec208 sp=0x7ffddbbec1d0 pc=0x40f487 runtime.mPark(...) runtime/proc.go:1866 runtime.stopm() runtime/proc.go:2885 +0x8c fp=0x7ffddbbec238 sp=0x7ffddbbec208 pc=0x443a6c runtime.findRunnable() runtime/proc.go:3622 +0xd5c fp=0x7ffddbbec3b0 sp=0x7ffddbbec238 pc=0x4454dc runtime.schedule() runtime/proc.go:3995 +0xb1 fp=0x7ffddbbec3e8 sp=0x7ffddbbec3b0 pc=0x4465b1 runtime.park_m(0xc0001bb6c0) runtime/proc.go:4102 +0x1eb fp=0x7ffddbbec440 sp=0x7ffddbbec3e8 pc=0x4469cb runtime.mcall() runtime/asm_amd64.s:459 +0x4e fp=0x7ffddbbec458 sp=0x7ffddbbec440 pc=0x47b86e goroutine 1 gp=0xc0000061c0 m=nil [sync.RWMutex.RLock, 46 minutes]: runtime.gopark(0xc00b23f6b0?, 0x3?, 0xa0?, 0x2b?, 0xc00d240d20?) runtime/proc.go:424 +0xce fp=0xc0003692f0 sp=0xc0003692d0 pc=0x4755ce runtime.goparkunlock(...) runtime/proc.go:430 runtime.semacquire1(0xc00026d04c, 0x0, 0x3, 0x0, 0x16) runtime/sema.go:178 +0x225 fp=0xc000369358 sp=0xc0003692f0 pc=0x4520c5 sync.runtime_SemacquireRWMutexR(0xc01032c270?, 0xc0?, 0xdd36d3?) runtime/sema.go:100 +0x25 fp=0xc000369390 sp=0xc000369358 pc=0x476ae5 sync.(*RWMutex).RLock(...) sync/rwmutex.go:72 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).rlock(0xc00fe6e9c0?) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:80 +0x32 fp=0xc0003693b8 sp=0xc000369390 pc=0xacbdd2 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Get(0xc00026d040, {0xc041d8a5a0, 0x1a, 0x45}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:93 +0x55 fp=0xc000369430 sp=0xc0003693b8 pc=0xacbeb5 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Get(0xc00fe5c960, {0xc041d8a5a0, 0x1a, 0x45}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:102 +0x1a2 fp=0xc0003694a8 sp=0xc000369430 pc=0xacc002 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Get(0xc00fe5cd20, {0xc041d8a5a0, 0x1a, 0x45}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:102 +0x1a2 fp=0xc000369520 sp=0xc0003694a8 pc=0xacc002 github.com/nspcc-dev/neo-go/pkg/core/dao.(*Simple).GetStorageItem(0xc00fe6ea80, 0x5, {0xc010316d08, 0x15, 0x0?}) github.com/nspcc-dev/neo-go/pkg/core/dao/dao.go:357 +0x17b fp=0xc000369568 sp=0xc000369520 pc=0xcb257b github.com/nspcc-dev/neo-go/pkg/core/interop/storage.Get(0xc00fe66ea0) github.com/nspcc-dev/neo-go/pkg/core/interop/storage/basic.go:49 +0x9c fp=0xc0003695d8 sp=0xc000369568 pc=0xda3f9c github.com/nspcc-dev/neo-go/pkg/core/interop.(*Context).SyscallHandler(0xc00fe66ea0, 0x10d1cc0?, 0x369a38?) github.com/nspcc-dev/neo-go/pkg/core/interop/context.go:470 +0x99 fp=0xc000369630 sp=0xc0003695d8 pc=0xcbaf79 github.com/nspcc-dev/neo-go/pkg/core/interop.(*Context).SyscallHandler-fm(0xc04f434ebb?, 0x15d0240?) <autogenerated>:1 +0x28 fp=0xc000369658 sp=0xc000369630 pc=0xcc17a8 github.com/nspcc-dev/neo-go/pkg/vm.(*VM).execute(0xc00fe6e980, 0xc010328a80, 0x41, {0xc04f434ebb, 0x4, 0x22}) github.com/nspcc-dev/neo-go/pkg/vm/vm.go:1523 +0x95d fp=0xc000369c70 sp=0xc000369658 pc=0xc741dd github.com/nspcc-dev/neo-go/pkg/vm.(*VM).step(0xc00fe6e980, 0xc010328a80) github.com/nspcc-dev/neo-go/pkg/vm/vm.go:517 +0x1a5 fp=0xc000369d20 sp=0xc000369c70 pc=0xc734e5 github.com/nspcc-dev/neo-go/pkg/vm.(*VM).Run(0xc00fe6e980) github.com/nspcc-dev/neo-go/pkg/vm/vm.go:484 +0xd1 fp=0xc000369d40 sp=0xc000369d20 pc=0xc731f1 github.com/nspcc-dev/neo-go/pkg/core/interop.(*Context).Exec(0xc00fe6e980?) github.com/nspcc-dev/neo-go/pkg/core/interop/context.go:513 +0x50 fp=0xc000369d88 sp=0xc000369d40 pc=0xcbb630 github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).storeBlock(0xc000130908, 0xc010216900, 0xc00d2328f0) github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1718 +0x9aa fp=0xc00036a2c8 sp=0xc000369d88 pc=0xdf502a github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).AddBlock(0xc000130908, 0xc010216900) github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1543 +0x68f fp=0xc00036a488 sp=0xc00036a2c8 pc=0xdf3b0f github.com/nspcc-dev/neo-go/pkg/core/chaindump.Restore({0x15c6490, 0xc000130908}, 0xc0002b6420, 0x0, 0x6a5882, 0xc00036b650) github.com/nspcc-dev/neo-go/pkg/core/chaindump/dump.go:81 +0x2c8 fp=0xc00036a6d0 sp=0xc00036a488 pc=0xe0bf08 github.com/nspcc-dev/neo-go/cli/server.restoreDB(0xc00025d8c0) github.com/nspcc-dev/neo-go/cli/server/server.go:330 +0xac9 fp=0xc00036b6f8 sp=0xc00036a6d0 pc=0xeb8a49 github.com/urfave/cli/v2.(*Command).Run(0xc0001ad340, 0xc00025d8c0, {0xc00026cff0, 0x5, 0x5}) github.com/urfave/cli/v2@v2.27.4/command.go:276 +0x7e2 fp=0xc00036b970 sp=0xc00036b6f8 pc=0x5fe9a2 github.com/urfave/cli/v2.(*Command).Run(0xc0001acf20, 0xc00025d780, {0xc000289ec0, 0x6, 0x6}) github.com/urfave/cli/v2@v2.27.4/command.go:269 +0xa65 fp=0xc00036bbe8 sp=0xc00036b970 pc=0x5fec25 github.com/urfave/cli/v2.(*Command).Run(0xc0002a34a0, 0xc00025d640, {0xc00003e070, 0x7, 0x7}) github.com/urfave/cli/v2@v2.27.4/command.go:269 +0xa65 fp=0xc00036be60 sp=0xc00036bbe8 pc=0x5fec25 github.com/urfave/cli/v2.(*App).RunContext(0xc0001b4a00, {0x15c6500, 0x1f58bc0}, {0xc00003e070, 0x7, 0x7}) github.com/urfave/cli/v2@v2.27.4/app.go:333 +0x5a5 fp=0xc00036bec0 sp=0xc00036be60 pc=0x5fb405 github.com/urfave/cli/v2.(*App).Run(0xc000092740?, {0xc00003e070?, 0x68?, 0x0?}) github.com/urfave/cli/v2@v2.27.4/app.go:307 +0x2f fp=0xc00036bf00 sp=0xc00036bec0 pc=0x5fae0f main.main() ./main.go:13 +0x32 fp=0xc00036bf50 sp=0xc00036bf00 pc=0x1042df2 runtime.main() runtime/proc.go:272 +0x28b fp=0xc00036bfe0 sp=0xc00036bf50 pc=0x43edeb runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc00036bfe8 sp=0xc00036bfe0 pc=0x47d721 goroutine 2 gp=0xc000006700 m=nil [force gc (idle), 2 minutes]: runtime.gopark(0x5b5c8dea658c?, 0x0?, 0x0?, 0x0?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000092fa8 sp=0xc000092f88 pc=0x4755ce runtime.goparkunlock(...) runtime/proc.go:430 runtime.forcegchelper() runtime/proc.go:337 +0xb3 fp=0xc000092fe0 sp=0xc000092fa8 pc=0x43f133 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc000092fe8 sp=0xc000092fe0 pc=0x47d721 created by runtime.init.7 in goroutine 1 runtime/proc.go:325 +0x1a goroutine 3 gp=0xc000006c40 m=nil [GC sweep wait]: runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000093780 sp=0xc000093760 pc=0x4755ce runtime.goparkunlock(...) runtime/proc.go:430 runtime.bgsweep(0xc000050080) runtime/mgcsweep.go:317 +0xdf fp=0xc0000937c8 sp=0xc000093780 pc=0x426a5f runtime.gcenable.gowrap1() runtime/mgc.go:204 +0x25 fp=0xc0000937e0 sp=0xc0000937c8 pc=0x41b105 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0000937e8 sp=0xc0000937e0 pc=0x47d721 created by runtime.gcenable in goroutine 1 runtime/mgc.go:204 +0x66 goroutine 4 gp=0xc000006e00 m=nil [GC scavenge wait, 2 minutes]: runtime.gopark(0x440697?, 0x4354b5?, 0x0?, 0x0?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000093f78 sp=0xc000093f58 pc=0x4755ce runtime.goparkunlock(...) runtime/proc.go:430 runtime.(*scavengerState).park(0x1f35960) runtime/mgcscavenge.go:425 +0x49 fp=0xc000093fa8 sp=0xc000093f78 pc=0x424429 runtime.bgscavenge(0xc000050080) runtime/mgcscavenge.go:658 +0x59 fp=0xc000093fc8 sp=0xc000093fa8 pc=0x4249b9 runtime.gcenable.gowrap2() runtime/mgc.go:205 +0x25 fp=0xc000093fe0 sp=0xc000093fc8 pc=0x41b0a5 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc000093fe8 sp=0xc000093fe0 pc=0x47d721 created by runtime.gcenable in goroutine 1 runtime/mgc.go:205 +0xa5 goroutine 5 gp=0xc000007340 m=nil [finalizer wait, 123 minutes]: runtime.gopark(0x490013?, 0xc000092660?, 0x9e?, 0x5?, 0x7fad2a0be128?) runtime/proc.go:424 +0xce fp=0xc000092620 sp=0xc000092600 pc=0x4755ce runtime.runfinq() runtime/mfinal.go:193 +0x107 fp=0xc0000927e0 sp=0xc000092620 pc=0x41a187 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0000927e8 sp=0xc0000927e0 pc=0x47d721 created by runtime.createfing in goroutine 1 runtime/mfinal.go:163 +0x3d goroutine 6 gp=0xc0001bb6c0 m=nil [chan receive, 2 minutes]: runtime.gopark(0xc000094760?, 0x688465?, 0x30?, 0x8c?, 0x15e2540?) runtime/proc.go:424 +0xce fp=0xc000094718 sp=0xc0000946f8 pc=0x4755ce runtime.chanrecv(0xc0000be620, 0x0, 0x1) runtime/chan.go:639 +0x41c fp=0xc000094790 sp=0xc000094718 pc=0x409d7c runtime.chanrecv1(0x0?, 0x0?) runtime/chan.go:489 +0x12 fp=0xc0000947b8 sp=0xc000094790 pc=0x409932 runtime.unique_runtime_registerUniqueMapCleanup.func1(...) runtime/mgc.go:1781 runtime.unique_runtime_registerUniqueMapCleanup.gowrap1() runtime/mgc.go:1784 +0x2f fp=0xc0000947e0 sp=0xc0000947b8 pc=0x41e12f runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0000947e8 sp=0xc0000947e0 pc=0x47d721 created by unique.runtime_registerUniqueMapCleanup in goroutine 1 runtime/mgc.go:1779 +0x96 goroutine 104 gp=0xc0001bba40 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84fb4744d?, 0x1?, 0x6f?, 0xe9?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000094f38 sp=0xc000094f18 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc000094fc8 sp=0xc000094f38 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc000094fe0 sp=0xc000094fc8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc000094fe8 sp=0xc000094fe0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 105 gp=0xc0001bbc00 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84fcf64e0?, 0x1?, 0x70?, 0x81?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000095738 sp=0xc000095718 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc0000957c8 sp=0xc000095738 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc0000957e0 sp=0xc0000957c8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0000957e8 sp=0xc0000957e0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 106 gp=0xc0001bbdc0 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84fa81713?, 0x3?, 0xde?, 0x88?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000095f38 sp=0xc000095f18 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc000095fc8 sp=0xc000095f38 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc000095fe0 sp=0xc000095fc8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc000095fe8 sp=0xc000095fe0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 107 gp=0xc0002d4000 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84facc1e2?, 0x3?, 0x12?, 0xb?, 0x0?) runtime/proc.go:424 +0xce fp=0xc00008e738 sp=0xc00008e718 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc00008e7c8 sp=0xc00008e738 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc00008e7e0 sp=0xc00008e7c8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc00008e7e8 sp=0xc00008e7e0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 108 gp=0xc0002d41c0 m=nil [GC worker (idle)]: runtime.gopark(0x1f5b200?, 0x1?, 0x0?, 0xac?, 0x0?) runtime/proc.go:424 +0xce fp=0xc00008ef38 sp=0xc00008ef18 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc00008efc8 sp=0xc00008ef38 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc00008efe0 sp=0xc00008efc8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc00008efe8 sp=0xc00008efe0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 109 gp=0xc0002d4380 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84fa816af?, 0x3?, 0xb8?, 0x3d?, 0x0?) runtime/proc.go:424 +0xce fp=0xc00008f738 sp=0xc00008f718 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc00008f7c8 sp=0xc00008f738 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc00008f7e0 sp=0xc00008f7c8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc00008f7e8 sp=0xc00008f7e0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 110 gp=0xc0002d4540 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84fa8136f?, 0x3?, 0x48?, 0x7c?, 0x0?) runtime/proc.go:424 +0xce fp=0xc00008ff38 sp=0xc00008ff18 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc00008ffc8 sp=0xc00008ff38 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc00008ffe0 sp=0xc00008ffc8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc00008ffe8 sp=0xc00008ffe0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 111 gp=0xc0002d4700 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84fb366ce?, 0x1?, 0xc4?, 0x1?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000090738 sp=0xc000090718 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc0000907c8 sp=0xc000090738 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc0000907e0 sp=0xc0000907c8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0000907e8 sp=0xc0000907e0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 112 gp=0xc0002d48c0 m=nil [GC worker (idle), 2 minutes]: runtime.gopark(0x5b5c8e3b6488?, 0x1?, 0xb?, 0x13?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000090f38 sp=0xc000090f18 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc000090fc8 sp=0xc000090f38 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc000090fe0 sp=0xc000090fc8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc000090fe8 sp=0xc000090fe0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 129 gp=0xc0002d4a80 m=nil [GC worker (idle), 47 minutes]: runtime.gopark(0x58ccee83b7c9?, 0xc0002b0fa0?, 0x1a?, 0xa?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000091738 sp=0xc000091718 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc0000917c8 sp=0xc000091738 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc0000917e0 sp=0xc0000917c8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0000917e8 sp=0xc0000917e0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 130 gp=0xc0002d4c40 m=nil [GC worker (idle), 2 minutes]: runtime.gopark(0x5b5c8e2fcce1?, 0x1?, 0x5b?, 0xd7?, 0x0?) runtime/proc.go:424 +0xce fp=0xc000091f38 sp=0xc000091f18 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc000091fc8 sp=0xc000091f38 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc000091fe0 sp=0xc000091fc8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc000091fe8 sp=0xc000091fe0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 131 gp=0xc0002d4e00 m=nil [GC worker (idle)]: runtime.gopark(0x5b5c8e2fd70d?, 0x1?, 0x96?, 0xa3?, 0x0?) runtime/proc.go:424 +0xce fp=0xc0002da738 sp=0xc0002da718 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc0002da7c8 sp=0xc0002da738 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc0002da7e0 sp=0xc0002da7c8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0002da7e8 sp=0xc0002da7e0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 132 gp=0xc0002d4fc0 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84fa59fea?, 0x3?, 0x26?, 0xb2?, 0x0?) runtime/proc.go:424 +0xce fp=0xc0002daf38 sp=0xc0002daf18 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc0002dafc8 sp=0xc0002daf38 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc0002dafe0 sp=0xc0002dafc8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0002dafe8 sp=0xc0002dafe0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 133 gp=0xc0002d5180 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84fa8060d?, 0x3?, 0xa8?, 0x66?, 0x0?) runtime/proc.go:424 +0xce fp=0xc0002db738 sp=0xc0002db718 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc0002db7c8 sp=0xc0002db738 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc0002db7e0 sp=0xc0002db7c8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0002db7e8 sp=0xc0002db7e0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 134 gp=0xc0002d5340 m=nil [GC worker (idle), 47 minutes]: runtime.gopark(0x58d62ae10690?, 0x1?, 0x23?, 0xbd?, 0x0?) runtime/proc.go:424 +0xce fp=0xc0002dbf38 sp=0xc0002dbf18 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc0002dbfc8 sp=0xc0002dbf38 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc0002dbfe0 sp=0xc0002dbfc8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0002dbfe8 sp=0xc0002dbfe0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 135 gp=0xc0002d5500 m=nil [GC worker (idle), 46 minutes]: runtime.gopark(0x58d84facc68a?, 0x3?, 0xf5?, 0xa?, 0x0?) runtime/proc.go:424 +0xce fp=0xc0002dc738 sp=0xc0002dc718 pc=0x4755ce runtime.gcBgMarkWorker(0xc0000be8c0) runtime/mgc.go:1412 +0xe9 fp=0xc0002dc7c8 sp=0xc0002dc738 pc=0x41d449 runtime.gcBgMarkStartWorkers.gowrap1() runtime/mgc.go:1328 +0x25 fp=0xc0002dc7e0 sp=0xc0002dc7c8 pc=0x41d325 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0002dc7e8 sp=0xc0002dc7e0 pc=0x47d721 created by runtime.gcBgMarkStartWorkers in goroutine 1 runtime/mgc.go:1328 +0x105 goroutine 86 gp=0xc000105c00 m=nil [select, 46 minutes]: runtime.gopark(0xc0003daef0?, 0x4?, 0x9?, 0x3?, 0xc0003dab48?) runtime/proc.go:424 +0xce fp=0xc0003da888 sp=0xc0003da868 pc=0x4755ce runtime.selectgo(0xc0003daef0, 0xc0003dab40, 0xc0003dae10?, 0x0, 0x4?, 0x1) runtime/select.go:335 +0x7a5 fp=0xc0003da9b0 sp=0xc0003da888 pc=0x451025 github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).notificationDispatcher(0xc000130908) github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1378 +0x2ef fp=0xc0003dafc8 sp=0xc0003da9b0 pc=0xdf24af github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).Run.gowrap1() github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1114 +0x25 fp=0xc0003dafe0 sp=0xc0003dafc8 pc=0xdf0465 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0003dafe8 sp=0xc0003dafe0 pc=0x47d721 created by github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).Run in goroutine 140 github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1114 +0xe5 goroutine 140 gp=0xc000105dc0 m=nil [sync.RWMutex.Lock, 46 minutes]: runtime.gopark(0xc045ac5f80?, 0x10?, 0xa0?, 0x65?, 0xc0041d0e38?) runtime/proc.go:424 +0xce fp=0xc0041d0de8 sp=0xc0041d0dc8 pc=0x4755ce runtime.goparkunlock(...) runtime/proc.go:430 runtime.semacquire1(0xc0001ab3d8, 0x0, 0x3, 0x0, 0x17) runtime/sema.go:178 +0x225 fp=0xc0041d0e50 sp=0xc0041d0de8 pc=0x4520c5 sync.runtime_SemacquireRWMutex(0x47005d?, 0x30?, 0x47005d?) runtime/sema.go:105 +0x25 fp=0xc0041d0e88 sp=0xc0041d0e50 pc=0x476b45 sync.(*RWMutex).Lock(0x10000002000?) sync/rwmutex.go:153 +0x6a fp=0xc0041d0eb8 sp=0xc0041d0e88 pc=0x49778a go.etcd.io/bbolt.(*DB).mmap(0xc0001ab208, 0x900000000) go.etcd.io/bbolt@v1.3.11/db.go:433 +0x4f fp=0xc0041d0fd0 sp=0xc0041d0eb8 pc=0xa68bcf go.etcd.io/bbolt.(*DB).allocate(0xc0001ab208, 0xd0e, 0x1) go.etcd.io/bbolt@v1.3.11/db.go:1132 +0x10e fp=0xc0041d1028 sp=0xc0041d0fd0 pc=0xa6bbae go.etcd.io/bbolt.(*Tx).allocate(0xc0011ed880, 0x1) go.etcd.io/bbolt@v1.3.11/tx.go:417 +0x36 fp=0xc0041d1058 sp=0xc0041d1028 pc=0xa75036 go.etcd.io/bbolt.(*node).spill(0xc03a532310) go.etcd.io/bbolt@v1.3.11/node.go:368 +0x212 fp=0xc0041d1138 sp=0xc0041d1058 pc=0xa71352 go.etcd.io/bbolt.(*node).spill(0xc03a5322a0) go.etcd.io/bbolt@v1.3.11/node.go:350 +0x9e fp=0xc0041d1218 sp=0xc0041d1138 pc=0xa711de go.etcd.io/bbolt.(*node).spill(0xc03d25b880) go.etcd.io/bbolt@v1.3.11/node.go:350 +0x9e fp=0xc0041d12f8 sp=0xc0041d1218 pc=0xa711de go.etcd.io/bbolt.(*node).spill(0xc0095eaee0) go.etcd.io/bbolt@v1.3.11/node.go:350 +0x9e fp=0xc0041d13d8 sp=0xc0041d12f8 pc=0xa711de go.etcd.io/bbolt.(*node).spill(0xc02e5cc070) go.etcd.io/bbolt@v1.3.11/node.go:350 +0x9e fp=0xc0041d14b8 sp=0xc0041d13d8 pc=0xa711de go.etcd.io/bbolt.(*Bucket).spill(0xc00d22e740) go.etcd.io/bbolt@v1.3.11/bucket.go:592 +0x365 fp=0xc0041d16c8 sp=0xc0041d14b8 pc=0xa65905 go.etcd.io/bbolt.(*Bucket).spill(0xc0011ed898) go.etcd.io/bbolt@v1.3.11/bucket.go:559 +0x105 fp=0xc0041d18d8 sp=0xc0041d16c8 pc=0xa656a5 go.etcd.io/bbolt.(*Tx).Commit(0xc0011ed880) go.etcd.io/bbolt@v1.3.11/tx.go:164 +0xf9 fp=0xc0041d1a00 sp=0xc0041d18d8 pc=0xa73df9 go.etcd.io/bbolt.(*DB).Update(0x470639?, 0xc0041d1a88) go.etcd.io/bbolt@v1.3.11/db.go:893 +0xca fp=0xc0041d1a68 sp=0xc0041d1a00 pc=0xa6ab6a github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).PutChangeSet(0x0?, 0x0?, 0x0?) github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:94 +0x45 fp=0xc0041d1ab8 sp=0xc0041d1a68 pc=0xaca7c5 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).persist(0xc00026d040, 0x1) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:381 +0x26b fp=0xc0041d1c08 sp=0xc0041d1ab8 pc=0xacebeb github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).PersistSync(...) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:339 github.com/nspcc-dev/neo-go/pkg/core/dao.(*Simple).PersistSync(0x8ed06ef19cd93b14?) github.com/nspcc-dev/neo-go/pkg/core/dao/dao.go:971 +0xfb fp=0xc0041d1ce0 sp=0xc0041d1c08 pc=0xcb7ebb github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).persist(0xc000130908, 0x1) github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:2161 +0x65 fp=0xc0041d1ea0 sp=0xc0041d1ce0 pc=0xdf8e85 github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).Run(0xc000130908) github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1127 +0x1bd fp=0xc0041d1fc8 sp=0xc0041d1ea0 pc=0xdf029d github.com/nspcc-dev/neo-go/cli/server.initBCWithMetrics.gowrap1() github.com/nspcc-dev/neo-go/cli/server/server.go:151 +0x25 fp=0xc0041d1fe0 sp=0xc0041d1fc8 pc=0xeb7565 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0041d1fe8 sp=0xc0041d1fe0 pc=0x47d721 created by github.com/nspcc-dev/neo-go/cli/server.initBCWithMetrics in goroutine 1 github.com/nspcc-dev/neo-go/cli/server/server.go:151 +0x13c goroutine 141 gp=0xc0006ce380 m=nil [select, 123 minutes, locked to thread]: runtime.gopark(0xc0002d8fa8?, 0x2?, 0xa8?, 0x8e?, 0xc0002d8f94?) runtime/proc.go:424 +0xce fp=0xc0002d8e30 sp=0xc0002d8e10 pc=0x4755ce runtime.selectgo(0xc0002d8fa8, 0xc0002d8f90, 0x0?, 0x0, 0x0?, 0x1) runtime/select.go:335 +0x7a5 fp=0xc0002d8f58 sp=0xc0002d8e30 pc=0x451025 runtime.ensureSigM.func1() runtime/signal_unix.go:1077 +0x19f fp=0xc0002d8fe0 sp=0xc0002d8f58 pc=0x46cb3f runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc0002d8fe8 sp=0xc0002d8fe0 pc=0x47d721 created by runtime.ensureSigM in goroutine 1 runtime/signal_unix.go:1060 +0xc8 goroutine 120 gp=0xc000704380 m=6 mp=0xc000700008 [syscall, 123 minutes]: runtime.notetsleepg(0x1f59d20, 0xffffffffffffffff) runtime/lock_futex.go:246 +0x29 fp=0xc00033dfa0 sp=0xc00033df78 pc=0x40f769 os/signal.signal_recv() runtime/sigqueue.go:152 +0x29 fp=0xc00033dfc0 sp=0xc00033dfa0 pc=0x477509 os/signal.loop() os/signal/signal_unix.go:23 +0x13 fp=0xc00033dfe0 sp=0xc00033dfc0 pc=0xcf5c13 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc00033dfe8 sp=0xc00033dfe0 pc=0x47d721 created by os/signal.Notify.func1.1 in goroutine 1 os/signal/signal.go:151 +0x1f goroutine 121 gp=0xc000704540 m=nil [chan receive, 123 minutes]: runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?) runtime/proc.go:424 +0xce fp=0xc00033e718 sp=0xc00033e6f8 pc=0x4755ce runtime.chanrecv(0xc00013d960, 0x0, 0x1) runtime/chan.go:639 +0x41c fp=0xc00033e790 sp=0xc00033e718 pc=0x409d7c runtime.chanrecv1(0x0?, 0x0?) runtime/chan.go:489 +0x12 fp=0xc00033e7b8 sp=0xc00033e790 pc=0x409932 github.com/nspcc-dev/neo-go/cli/server.newGraceContext.func1() github.com/nspcc-dev/neo-go/cli/server/server.go:137 +0x25 fp=0xc00033e7e0 sp=0xc00033e7b8 pc=0xeb7185 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc00033e7e8 sp=0xc00033e7e0 pc=0x47d721 created by github.com/nspcc-dev/neo-go/cli/server.newGraceContext in goroutine 1 github.com/nspcc-dev/neo-go/cli/server/server.go:136 +0xf7 goroutine 27196640 gp=0xc000704c40 m=nil [select, 46 minutes]: runtime.gopark(0xc010081350?, 0x2?, 0x0?, 0x0?, 0xc010081344?) runtime/proc.go:424 +0xce fp=0xc0100811e8 sp=0xc0100811c8 pc=0x4755ce runtime.selectgo(0xc010081350, 0xc010081340, 0x0?, 0x1, 0xc0100813a0?, 0x1) runtime/select.go:335 +0x7a5 fp=0xc010081310 sp=0xc0100811e8 pc=0x451025 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync.func1.1({0xc00fe7e526, 0x15, 0x1a}, {0x1f58bc0, 0x0, 0x0}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:175 +0xfd fp=0xc0100813b0 sp=0xc010081310 pc=0xacd31d github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc00fe7e500?, 0x1b?, 0xc010300b40?}, {0x1f58bc0, 0x0, 0x8?}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100814c8 sp=0xc0100813b0 pc=0xace785 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc00fe7e4e0?, 0x1b?, 0xc010081688?}, {0x1f58bc0, 0x0, 0xc028b622b0?}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100815e0 sp=0xc0100814c8 pc=0xace785 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc00fe7e4c0?, 0x1b?, 0x8?}, {0x1f58bc0, 0x0, 0xa67325?}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100816f8 sp=0xc0100815e0 pc=0xace785 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0x7fab9f1479ba?, 0x1b?, 0x410845?}, {0x7fab9f147130, 0x0, 0x1?}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc010081810 sp=0xc0100816f8 pc=0xace785 github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).Seek.func1(0xc010300b40?, {0x7fab9f1479ba?, 0x2?, 0x0?}, {0x7fab9f147130?, 0x3fe0000000000000?, 0xc00b5308f0?}) github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:127 +0x34 fp=0xc010081850 sp=0xc010081810 pc=0xacadd4 github.com/nspcc-dev/neo-go/pkg/core/storage.boltSeek.func1(0x8?) github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:159 +0x2e4 fp=0xc010081918 sp=0xc010081850 pc=0xacb2e4 go.etcd.io/bbolt.(*DB).View(0x1216e40?, 0xc03bfc8360) go.etcd.io/bbolt@v1.3.11/db.go:917 +0x6c fp=0xc010081980 sp=0xc010081918 pc=0xa6ac8c go.etcd.io/bbolt.(*DB).View-fm(0xc028b62258?) <autogenerated>:1 +0x26 fp=0xc0100819a0 sp=0xc010081980 pc=0xad9766 github.com/nspcc-dev/neo-go/pkg/core/storage.boltSeek(0xc00b530a50, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc00fb0dcb0) github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:136 +0x13a fp=0xc010081a00 sp=0xc0100819a0 pc=0xacaf7a github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).Seek(0x0?, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e7018f0) github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:126 +0xf5 fp=0xc010081a70 sp=0xc010081a00 pc=0xacad15 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8d60, 0xc000096c48}, {0x0, 0x0, 0x0}, {{0xc028b62258, 0x6, 0x8}, ...}, ...) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010081b20 sp=0xc010081a70 pc=0xacdea2 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701880) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010081bb8 sp=0xc010081b20 pc=0xaccda8 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8da0, 0xc0261627d0}, {0x0, 0x0, 0x0}, {{0xc028b62258, 0x6, 0x8}, ...}, ...) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010081c68 sp=0xc010081bb8 pc=0xacdea2 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701810) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010081d00 sp=0xc010081c68 pc=0xaccda8 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8da0, 0xc00026d040}, {0x0, 0x0, 0x0}, {{0xc028b62258, 0x6, 0x8}, ...}, ...) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010081db0 sp=0xc010081d00 pc=0xacdea2 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62258, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e7017a0) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010081e48 sp=0xc010081db0 pc=0xaccda8 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6998, 0xc00fe5cdc0}, {0x15c8da0, 0xc00fe5c960}, {0x0, 0x0, 0x0}, {{0xc028b62258, 0x6, 0x8}, ...}, ...) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010081ef8 sp=0xc010081e48 pc=0xacdea2 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync.func1() github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:174 +0x169 fp=0xc010081fe0 sp=0xc010081ef8 pc=0xacd1e9 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc010081fe8 sp=0xc010081fe0 pc=0x47d721 created by github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync in goroutine 1 github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:173 +0x1bc goroutine 27196705 gp=0xc000d05dc0 m=nil [select, 46 minutes]: runtime.gopark(0xc010089350?, 0x2?, 0x0?, 0x0?, 0xc010089344?) runtime/proc.go:424 +0xce fp=0xc0100891e8 sp=0xc0100891c8 pc=0x4755ce runtime.selectgo(0xc010089350, 0xc010089340, 0x0?, 0x1, 0xc0100893a0?, 0x1) runtime/select.go:335 +0x7a5 fp=0xc010089310 sp=0xc0100891e8 pc=0x451025 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync.func1.1({0xc010324a66, 0x14, 0x1a}, {0xc027c83390, 0xd, 0x10}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:175 +0xfd fp=0xc0100893b0 sp=0xc010089310 pc=0xacd31d github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc010324a40?, 0x1a?, 0xc010300e09?}, {0xc027c83380, 0xd, 0x8?}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100894c8 sp=0xc0100893b0 pc=0xace785 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc00fe7e8c0?, 0x1a?, 0xc010089609?}, {0xc025c388d0, 0xd, 0xc028b624c8?}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc0100895e0 sp=0xc0100894c8 pc=0xace785 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0xc0103249e0?, 0x1a?, 0x8?}, {0xc027c83350, 0xd, 0xa67325?}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:273 +0x443 fp=0xc0100896f8 sp=0xc0100895e0 pc=0xace5c3 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek.func2({0x7fab9f12be85?, 0x1a?, 0x410845?}, {0x7fab9f12be9f, 0xd, 0x1?}) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:290 +0x605 fp=0xc010089810 sp=0xc0100896f8 pc=0xace785 github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).Seek.func1(0xc010300e40?, {0x7fab9f12be85?, 0x2?, 0x0?}, {0x7fab9f12be9f?, 0x3fe0000000000000?, 0xc00b5308f0?}) github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:127 +0x34 fp=0xc010089850 sp=0xc010089810 pc=0xacadd4 github.com/nspcc-dev/neo-go/pkg/core/storage.boltSeek.func1(0x8?) github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:159 +0x2e4 fp=0xc010089918 sp=0xc010089850 pc=0xacb2e4 go.etcd.io/bbolt.(*DB).View(0x1216e40?, 0xc03bfc8540) go.etcd.io/bbolt@v1.3.11/db.go:917 +0x6c fp=0xc010089980 sp=0xc010089918 pc=0xa6ac8c go.etcd.io/bbolt.(*DB).View-fm(0xc028b62470?) <autogenerated>:1 +0x26 fp=0xc0100899a0 sp=0xc010089980 pc=0xad9766 github.com/nspcc-dev/neo-go/pkg/core/storage.boltSeek(0xc00b530a50, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc00fb0dd90) github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:136 +0x13a fp=0xc010089a00 sp=0xc0100899a0 pc=0xacaf7a github.com/nspcc-dev/neo-go/pkg/core/storage.(*BoltDBStore).Seek(0xc010313008?, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701c00) github.com/nspcc-dev/neo-go/pkg/core/storage/boltdb_store.go:126 +0xf5 fp=0xc010089a70 sp=0xc010089a00 pc=0xacad15 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8d60, 0xc000096c48}, {0xc010313008, 0x29, 0x49}, {{0xc028b62470, 0x6, 0x8}, ...}, ...) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010089b20 sp=0xc010089a70 pc=0xacdea2 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0xc010312008?, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701b20) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010089bb8 sp=0xc010089b20 pc=0xaccda8 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8da0, 0xc0261627d0}, {0xc010312008, 0x2b, 0x49}, {{0xc028b62470, 0x6, 0x8}, ...}, ...) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010089c68 sp=0xc010089bb8 pc=0xacdea2 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e701a40) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010089d00 sp=0xc010089c68 pc=0xaccda8 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6500, 0x1f58bc0}, {0x15c8da0, 0xc00026d040}, {0x0, 0x0, 0x0}, {{0xc028b62470, 0x6, 0x8}, ...}, ...) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010089db0 sp=0xc010089d00 pc=0xacdea2 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).Seek(0x0?, {{0xc028b62470, 0x6, 0x8}, {0x0, 0x0, 0x0}, 0x0, 0x0}, 0xc03e7019d0) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:155 +0xe8 fp=0xc010089e48 sp=0xc010089db0 pc=0xaccda8 github.com/nspcc-dev/neo-go/pkg/core/storage.performSeek({0x15c6998, 0xc00fe5ce10}, {0x15c8da0, 0xc00fe5c960}, {0x0, 0x0, 0x0}, {{0xc028b62470, 0x6, 0x8}, ...}, ...) github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:304 +0x3a2 fp=0xc010089ef8 sp=0xc010089e48 pc=0xacdea2 github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync.func1() github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:174 +0x169 fp=0xc010089fe0 sp=0xc010089ef8 pc=0xacd1e9 runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc010089fe8 sp=0xc010089fe0 pc=0x47d721 created by github.com/nspcc-dev/neo-go/pkg/core/storage.(*MemCachedStore).SeekAsync in goroutine 1 github.com/nspcc-dev/neo-go/pkg/core/storage/memcached_store.go:173 +0x1bc goroutine 27196638 gp=0xc001100a80 m=nil [chan receive, 46 minutes]: runtime.gopark(0xc010216900?, 0xc00fe45e00?, 0x0?, 0x0?, 0x0?) runtime/proc.go:424 +0xce fp=0xc0100837a8 sp=0xc010083788 pc=0x4755ce runtime.chanrecv(0xc03f47aaf0, 0xc010083a28, 0x1) runtime/chan.go:639 +0x41c fp=0xc010083820 sp=0xc0100837a8 pc=0x409d7c runtime.chanrecv2(0x65f1a1808b62b04b?, 0x442cb62d94b4b97e?) runtime/chan.go:494 +0x12 fp=0xc010083848 sp=0xc010083820 pc=0x409952 github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).storeBlock.func1() github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1657 +0x1fb fp=0xc010083fe0 sp=0xc010083848 pc=0xdf651b runtime.goexit({}) runtime/asm_amd64.s:1700 +0x1 fp=0xc010083fe8 sp=0xc010083fe0 pc=0x47d721 created by github.com/nspcc-dev/neo-go/pkg/core.(*Blockchain).storeBlock in goroutine 1 github.com/nspcc-dev/neo-go/pkg/core/blockchain.go:1622 +0x5be ```` Notice 46m stalls, these are locked ~since the last log message. ## Expected Behavior Working till the end. ## Possible Solution Fix it. ## Steps to Reproduce ``` ./bin/neo-go db restore --config-file config/protocol.mainnet.neofs.yml -i ../chains/mainnet-neofs-6969473.acc ``` config is changed for BoltDB: ``` --- a/config/protocol.mainnet.neofs.yml +++ b/config/protocol.mainnet.neofs.yml @@ -34,12 +34,12 @@ ApplicationConfiguration: # LogPath could be set up in case you need stdout logs to some proper file. # LogPath: "./log/neogo.log" DBConfiguration: - Type: "leveldb" #other options: 'inmemory','boltdb' + Type: "boltdb" #other options: 'inmemory','boltdb' # DB type options. Uncomment those you need in case you want to switch DB type. LevelDBOptions: DataDirectoryPath: "./chains/mainnet.neofs" - # BoltDBOptions: - # FilePath: "./chains/mainnet.bolt" + BoltDBOptions: + FilePath: "./chains/mainnet.neofs.bolt" P2P: Addresses: - ":40333" # in form of "[host]:[port][:announcedPort]" ``` ## Regression Not likely. ## Your Environment * Version of the product used: 0.107.2 * Operating System and version (`uname -a`): openSUSE 15.6
Sign in to join this conversation.
No milestone
No project
No assignees
1 participant
Notifications
Due date
The due date is invalid or out of range. Please use the format "yyyy-mm-dd".

No due date set.

Dependencies

No dependencies set.

Reference
nspcc-dev/neo-go#1472
No description provided.