I want to extract data from an API and save it to a boltDB database. The problem is that the process is frozen on the commit() call and sends no panic or error...
Anyone seeing something with the code below ? I put a maximum of 1500 candles at once maybe it's too many ? If so what would be the best practice for such amount ?
Thanks a lot !
type Candle struct {
Close float32 `json:"close"`
High float32 `json:"high"`
Low float32 `json:"low"`
Open float32 `json:"open"`
ClockTime float32 `json:"time"`
StartTime time.Time `json:"startTime"`
Volume float32 `json:"volume"`
Resolution string
}
func (s historyDataStore) saveCandles(mkt string, candles []Candle) error {
tx, err := s.db.Begin(true)
b := tx.Bucket([]byte(mainBucket)).Bucket([]byte(mkt))
i := 0
for _, candle := range candles {
i++
js, err := json.Marshal(candle)
if err != nil {
return err
}
key, err := b.NextSequence()
err = b.Put(itob(key), js)
if err != nil {
return err
}
}
err = tx.Commit()
if err != nil {
log.Fatal(err)
}
return err
}
This is the SIGQUIT info : It seems that this is the Lock() function in rwmutex.go that is waiting for its availability...
PC=0x7fff6b69b882 m=0 sigcode=0
goroutine 0 [idle]:
runtime.pthread_cond_wait(0x1572820, 0x15727e0, 0x0)
/usr/local/go/src/runtime/sys_darwin.go:384 +0x39
runtime.semasleep(0xffffffffffffffff, 0x7ffeefbff4e0)
/usr/local/go/src/runtime/os_darwin.go:63 +0x8d
runtime.notesleep(0x15725d0)
/usr/local/go/src/runtime/lock_sema.go:181 +0xdb
runtime.mPark()
/usr/local/go/src/runtime/proc.go:1340 +0x39
runtime.stopm()
/usr/local/go/src/runtime/proc.go:2301 +0x92
runtime.findrunnable(0xc00003b000, 0x0)
/usr/local/go/src/runtime/proc.go:2960 +0x72e
runtime.schedule()
/usr/local/go/src/runtime/proc.go:3169 +0x2d7
runtime.park_m(0xc000082900)
/usr/local/go/src/runtime/proc.go:3318 +0x9d
runtime.mcall(0x106fcf6)
/usr/local/go/src/runtime/asm_amd64.s:327 +0x5b
goroutine 1 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0xc0001201a0, 0xc0003ece00, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).Lock(0xc000120198)
/usr/local/go/src/sync/rwmutex.go:116 +0x85
github.com/boltdb/bolt.(*DB).mmap(0xc000120000, 0x9000, 0x0, 0x0)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:246 +0x69
github.com/boltdb/bolt.(*DB).allocate(0xc000120000, 0x1, 0xc0003ecfe0, 0x2, 0x2)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:849 +0x12a
github.com/boltdb/bolt.(*Tx).allocate(0xc000122000, 0x1, 0x8, 0x8, 0xc0002840a8)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:465 +0xef
github.com/boltdb/bolt.(*node).spill(0xc0002bb110, 0xc0000ba008, 0xc0003ed240)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/node.go:368 +0x1bd
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2080, 0xc00034e100, 0xc0003ed4f0)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:571 +0x5e5
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2000, 0xc00034e000, 0xc0003ed720)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Bucket).spill(0xc000122018, 0xc0000ba008, 0xc0003ed8a8)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Tx).Commit(0xc000122000, 0x5, 0xc0003edad0)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:165 +0x285
github.com/elRomano/gotrader/store/boltdb.historyDataStore.saveCandles(0xc000120000, 0x13690ca, 0xb, 0x1, 0x1367a43, 0x7, 0xc00075a000, 0x5dd, 0x638, 0x60d34ff4, ...)
/Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:123 +0x539
github.com/elRomano/gotrader/store/boltdb.historyDataStore.updateMarketHistory(0xc000120000, 0x13690ca, 0xb, 0x5, 0x1367a43, 0x7, 0x12fc9c0, 0x13c7f00)
/Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:71 +0x3ee
github.com/elRomano/gotrader/store/boltdb.(*historyDataStore).UpdateDb(0xc0000be4c0, 0xc0000be4a0, 0x2, 0x2, 0x1, 0x0)
/Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:43 +0xa5
main.main()
/Users/user/Documents/_DEV/Go/test/main.go:59 +0xa0d
rax 0x104
rbx 0x2
rcx 0x7ffeefbff2e8
rdx 0x6c00
rdi 0x1572820
rsi 0x6c0100006d00
rbp 0x7ffeefbff380
rsp 0x7ffeefbff2e8
r8 0x0
r9 0xa0
r10 0x0
r11 0x202
r12 0x1572820
r13 0x16
r14 0x6c0100006d00
r15 0xef99dc0
rip 0x7fff6b69b882
rflags 0x203
cs 0x7
fs 0x0
gs 0x0
exit status 2
Thanks to SIGQUIT tracing I've found out that there was a "Begin" transaction opened that was locking the other transactions I wanted to perform. I added a Rollback at the end and then it worked.
Problem solved ! Thanks @rustyx @mh-cbon @Adrian.