CGO decommission (#281)

* fix: Go implementation of rewriteBuffer

* feat: remove all C codes and replace by Go implementation

* fix: change the sort to a stable one

* fix: nanoseconds calculated by interval ticks can overflow when timeframe is not 1Sec

* chore: add unit tests for datatypes.go

* fix: delete old C files

* add a line to debug integration test on circleci

* fix: remove a test cause it's impossible to keep nanoseconds value correctly to 24H-timeframe bucket when import csv

* fix: round fractional second

README.ja.md

@@ -39,7 +39,7 @@ marketstore connect --url localhost:5993
 ```
 
 ### Source
-MarketStoreはGo (一部CGO)を用いて実装されているので、ソースコードからビルドすることも簡単です。Go `1.11` 以上のバージョンを使用し、また依存管理には`go mod` を使用しています。
+MarketStoreはGoを用いて実装されているので、ソースコードからビルドすることも簡単です。Go `1.11` 以上のバージョンを使用し、また依存管理には`go mod` を使用しています。
 ``` sh
 go get -u github.com/alpacahq/marketstore
 ```

README.md

@@ -39,7 +39,7 @@ marketstore connect --url localhost:5993
 ```
 
 ### Source
-MarketStore is implemented in Go (with some CGO), so you can build it from
+MarketStore is implemented in Go, so you can build it from
 source pretty easily. You need Go 1.11+ as it uses `go mod` to manage dependencies.
 ``` sh
 go get -u github.com/alpacahq/marketstore

executor/all_test.go

@@ -28,7 +28,6 @@ var (
 	_ = Suite(&TestSuite{nil, "", nil, nil})
 	_ = Suite(&DestructiveWALTests{nil, "", nil, nil})
 	_ = Suite(&DestructiveWALTest2{nil, "", nil, nil})
-	_ = Suite(&CGOTests{})
 )
 
 type TestSuite struct {
@@ -55,8 +54,6 @@ type DestructiveWALTest2 struct {
 	WALFile      *WALFileType
 }
 
-type CGOTests struct{}
-
 func (s *TestSuite) SetUpSuite(c *C) {
 	s.Rootdir = c.MkDir()
 	s.ItemsWritten = MakeDummyCurrencyDir(s.Rootdir, true, false)

executor/readvariable.go

@@ -1,21 +1,13 @@
 package executor
 
 import (
-	"os"
-	"unsafe"
-
 	"github.com/alpacahq/marketstore/utils"
 	"github.com/klauspost/compress/snappy"
+	"os"
 
 	. "github.com/alpacahq/marketstore/utils/io"
 )
 
-/*
-#include "rewriteBuffer.h"
-#cgo CFLAGS: -O3 -Wno-ignored-optimization-argument -std=c99
-*/
-import "C"
-
 func (r *reader) readSecondStage(bufMeta []bufferMeta, limitCount int32, direction DirectionEnum) (rb []byte, err error) {
 	/*
 		Here we use the bufFileMap which has index data for each file, then we read
@@ -92,12 +84,8 @@ func (r *reader) readSecondStage(bufMeta []bufferMeta, limitCount int32, directi
 					numVarRecords = numberLeftToRead
 				}
 			}
-			rbTemp := make([]byte, numVarRecords*(varRecLen+8)) // Add the extra space for epoch
-
-			arg1 := (*C.char)(unsafe.Pointer(&buffer[0]))
-			arg4 := (*C.char)(unsafe.Pointer(&rbTemp[0]))
-			C.rewriteBuffer(arg1, C.int(varRecLen), C.int(numVarRecords), arg4,
-				C.int64_t(md.Intervals), C.int64_t(intervalStartEpoch))
+			rbTemp := RewriteBuffer(buffer,
+				uint32(varRecLen), uint32(numVarRecords), uint32(md.Intervals), uint64(intervalStartEpoch))
 
 			//rb = append(rb, rbTemp...)
 			if (rbCursor + len(rbTemp)) > totalDatalen {

executor/rewritebuffer.go

@@ -0,0 +1,86 @@
+package executor
+
+import "C"
+import (
+	"encoding/binary"
+	"github.com/alpacahq/marketstore/utils/io"
+	_ "github.com/alpacahq/marketstore/utils/log"
+	_ "go.uber.org/zap"
+	"math"
+)
+
+// RewriteBuffer converts variable_length records to the result buffer.
+//
+// variable records in a file: [Actual Data (VarRecLen-4 byte) , Interval Ticks(4 byte) ]
+// rewriteBuffer converts the binary data to [EpochSecond(8 byte), Actual Data(VarRecLen-4 byte), Nanoseconds(4 byte) ] format.
+//
+// buffer
+// +-----------------------VarRecLen [byte]---+-----------------------+
+// +   Actual Data(Ask,Bid, etc.)             | IntevalTicks(4byte)    |
+// +------------------------------------------+------------------------+
+//
+// ↓ rewriteBuffer
+//
+// rbTemp (= temporary result buffer)
+// +--------------------+--VarRecLen + 8 [byte]-----+-------------------+
+// + EpochSecond(8byte) | Actual Data(Ask,Bid, etc) | Nanosecond(4byte) |
+// +--------------------+----------------------------+------------------+
+func RewriteBuffer(buffer []byte, varRecLen, numVarRecords uint32, intervalsPerDay uint32, intervalStartEpoch uint64) []byte {
+	// temporary result buffer
+	rbTemp := make([]byte, numVarRecords*(varRecLen+8)) // Add the extra space for epoch
+
+	var j, ii, cursor uint32
+	b := make([]byte, 8)
+	n := make([]byte, 4)
+	for j = 0; j < numVarRecords; j++ {
+
+		intervalTicks := buffer[(j+1)*varRecLen-4 : (j+1)*varRecLen]
+		it := io.ToUInt32(intervalTicks)
+
+		// Expand ticks (32-bit) into epoch and nanos
+		second, nanosecond := GetTimeFromTicks(intervalStartEpoch, intervalsPerDay, it)
+		binary.LittleEndian.PutUint64(b, second)
+
+		// copy Epoch second to the result buffer
+		cursor = j * (varRecLen + 8)
+		for ii = 0; ii < 8; ii++ {
+			rbTemp[cursor+ii] = b[ii]
+		}
+
+		// copy actual data (e.g. Ask, Bid) to the result buffer after the Epoch Second
+		for ii = 0; ii < varRecLen-4; ii++ {
+			rbTemp[cursor+8+ii] = buffer[(j*varRecLen)+ii]
+		}
+
+		// copy nanosecond to the result buffer after the Epoch Second
+		binary.LittleEndian.PutUint32(n, nanosecond)
+		for ii = 0; ii < 4; ii++ {
+			rbTemp[cursor+varRecLen+4+ii] = n[ii]
+		}
+	}
+
+	return rbTemp
+}
+
+// GetTimeFromTicks Takes two time components, the start of the interval and the number of
+// interval ticks to the timestamp and returns an epoch time (seconds) and
+// the number of nanoseconds of fractional time within the last second as a remainder
+func GetTimeFromTicks(intervalStart uint64, intervalsPerDay, intervalTicks uint32) (sec uint64, nanosec uint32) {
+	const ticksPerIntervalDivSecsPerDay float64 = 49710.269629629629629629629629629
+
+	var fractionalSeconds = float64(intervalTicks) / (float64(intervalsPerDay) * ticksPerIntervalDivSecsPerDay)
+	var subseconds = 1000000000 * (fractionalSeconds - math.Floor(fractionalSeconds))
+	if subseconds >= 1000000000 {
+		subseconds -= 1000000000
+		fractionalSeconds += 1
+	}
+
+	// in order to keep compatibility with the old rewriteBuffer implemented in C with some round error,
+	// fractionalSeconds should be rounded here.
+	sec = intervalStart + uint64(math.Round(fractionalSeconds*100000000)/100000000)
+	// round the subseconds after the decimal point to minimize the cancellation error of subseconds
+	// round( subseconds ) = (int32_t)(subseconds + 0.5)
+	nanosec = uint32(subseconds + 0.5)
+
+	return sec, nanosec
+}