Skip to content

Commit dfba8d3

Browse files
jancajthamljancajthaml
authored
Merge pull request #2 from jancajthaml-openbank/feature/faster-read
updated tests, added makefile, spedup read, created storage fascade
2 parents 1c048fb + 753c6f7 commit dfba8d3

File tree

3 files changed

+78
-39
lines changed

3 files changed

+78
-39
lines changed

‎Makefile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.ONESHELL:
2+
3+
.PHONY: all
4+
all: test
5+
6+
.PHONY: test
7+
test:
8+
GOMAXPROCS=1 \
9+
go test -v ./... -benchmem -bench=. -timeout=20s

‎storage.go‎

Lines changed: 32 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15-
package utils
15+
package storage
1616

1717
import (
1818
"bytes"
@@ -49,9 +49,22 @@ func nameFromDirent(dirent *syscall.Dirent) []byte {
4949
return name
5050
}
5151

52+
type Storage struct {
53+
Root string
54+
}
55+
56+
func NewStorage(root string) Storage {
57+
if root == "" || os.MkdirAll(filepath.Clean(root), os.ModePerm) != nil {
58+
panic("unable to assert root storage directory")
59+
}
60+
return Storage{
61+
Root: root,
62+
}
63+
}
64+
5265
// ListDirectory returns sorted slice of item names in given absolute path
5366
// default sorting is ascending
54-
func ListDirectory(absPath string, ascending bool) (result []string, err error) {
67+
func (storage Storage) ListDirectory(absPath string, ascending bool) (result []string, err error) {
5568
defer func() {
5669
if r := recover(); r != nil {
5770
if err == nil {
@@ -69,7 +82,7 @@ func ListDirectory(absPath string, ascending bool) (result []string, err error)
6982
de *syscall.Dirent
7083
)
7184

72-
dh, err = os.Open(filepath.Clean(absPath))
85+
dh, err = os.Open(filepath.Clean(storage.Root + "/" + absPath))
7386
if err != nil {
7487
return
7588
}
@@ -135,7 +148,7 @@ func ListDirectory(absPath string, ascending bool) (result []string, err error)
135148
}
136149

137150
// CountFiles returns number of items in directory
138-
func CountFiles(absPath string) (result int, err error) {
151+
func (storage Storage) CountFiles(absPath string) (result int, err error) {
139152
defer func() {
140153
if r := recover(); r != nil {
141154
if err == nil {
@@ -153,7 +166,7 @@ func CountFiles(absPath string) (result int, err error) {
153166
de *syscall.Dirent
154167
)
155168

156-
dh, err = os.Open(filepath.Clean(absPath))
169+
dh, err = os.Open(filepath.Clean(storage.Root + "/" + absPath))
157170
if err != nil {
158171
return
159172
}
@@ -194,10 +207,10 @@ func CountFiles(absPath string) (result int, err error) {
194207
}
195208

196209
// Exists returns true if absolute path exists
197-
func Exists(absPath string) (bool, error) {
210+
func (storage Storage) Exists(absPath string) (bool, error) {
198211
var (
199212
trusted = new(syscall.Stat_t)
200-
cleaned = filepath.Clean(absPath)
213+
cleaned = filepath.Clean(storage.Root + "/" + absPath)
201214
err error
202215
)
203216
err = syscall.Stat(cleaned, trusted)
@@ -211,8 +224,8 @@ func Exists(absPath string) (bool, error) {
211224
}
212225

213226
// TouchFile creates files given absolute path if file does not already exist
214-
func TouchFile(absPath string) error {
215-
cleanedPath := filepath.Clean(absPath)
227+
func (storage Storage) TouchFile(absPath string) error {
228+
cleanedPath := filepath.Clean(storage.Root + "/" + absPath)
216229
if err := os.MkdirAll(filepath.Dir(cleanedPath), os.ModePerm); err != nil {
217230
return err
218231
}
@@ -225,8 +238,8 @@ func TouchFile(absPath string) error {
225238
}
226239

227240
// ReadFileFully reads whole file given absolute path
228-
func ReadFileFully(absPath string) ([]byte, error) {
229-
f, err := os.OpenFile(filepath.Clean(absPath), os.O_RDONLY, os.ModePerm)
241+
func (storage Storage) ReadFileFully(absPath string) ([]byte, error) {
242+
f, err := os.OpenFile(filepath.Clean(storage.Root+"/"+absPath), os.O_RDONLY, os.ModePerm)
230243
if err != nil {
231244
return nil, err
232245
}
@@ -245,8 +258,8 @@ func ReadFileFully(absPath string) ([]byte, error) {
245258

246259
// WriteFile writes data given absolute path to a file if that file does not
247260
// already exists
248-
func WriteFile(absPath string, data []byte) error {
249-
cleanedPath := filepath.Clean(absPath)
261+
func (storage Storage) WriteFile(absPath string, data []byte) error {
262+
cleanedPath := filepath.Clean(storage.Root + "/" + absPath)
250263
if err := os.MkdirAll(filepath.Dir(cleanedPath), os.ModePerm); err != nil {
251264
return err
252265
}
@@ -262,14 +275,14 @@ func WriteFile(absPath string, data []byte) error {
262275
}
263276

264277
// DeleteFile removes file given absolute path if that file does exists
265-
func DeleteFile(absPath string) error {
266-
return os.Remove(filepath.Clean(absPath))
278+
func (storage Storage) DeleteFile(absPath string) error {
279+
return os.Remove(filepath.Clean(storage.Root + "/" + absPath))
267280
}
268281

269282
// UpdateFile rewrite file with data given absolute path to a file if that file
270283
// exist
271-
func UpdateFile(absPath string, data []byte) (err error) {
272-
cleanedPath := filepath.Clean(absPath)
284+
func (storage Storage) UpdateFile(absPath string, data []byte) (err error) {
285+
cleanedPath := filepath.Clean(storage.Root + "/" + absPath)
273286
var f *os.File
274287
f, err = os.OpenFile(cleanedPath, os.O_WRONLY|os.O_TRUNC, os.ModePerm)
275288
if err != nil {
@@ -282,8 +295,8 @@ func UpdateFile(absPath string, data []byte) (err error) {
282295

283296
// AppendFile appens data given absolute path to a file, creates it if it does
284297
// not exist
285-
func AppendFile(absPath string, data []byte) (err error) {
286-
cleanedPath := filepath.Clean(absPath)
298+
func (storage Storage) AppendFile(absPath string, data []byte) (err error) {
299+
cleanedPath := filepath.Clean(storage.Root + "/" + absPath)
287300
err = os.MkdirAll(filepath.Dir(cleanedPath), os.ModePerm)
288301
if err != nil {
289302
return err

‎storage_test.go‎

Lines changed: 37 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"io/ioutil"
66
"os"
7+
"path/filepath"
78
"testing"
89

910
"github.com/stretchr/testify/assert"
@@ -22,16 +23,16 @@ func TestExists(t *testing.T) {
2223
filename := file.Name()
2324
defer os.Remove(filename)
2425

25-
var (
26-
ok bool
27-
fail error
28-
)
26+
storage := NewStorage(tmpDir)
2927

30-
ok, fail = Exists(filename)
28+
var ok bool
29+
var fail error
30+
31+
ok, fail = storage.Exists(filepath.Base(filename))
3132
assert.Nil(t, fail)
3233
assert.True(t, ok)
3334

34-
ok, fail = Exists(filename + "xxx")
35+
ok, fail = storage.Exists(filepath.Base(filename + "xxx"))
3536
assert.Nil(t, fail)
3637
assert.False(t, ok)
3738
}
@@ -41,6 +42,8 @@ func TestListDirectory(t *testing.T) {
4142
require.Nil(t, err)
4243
defer os.RemoveAll(tmpdir)
4344

45+
storage := NewStorage(tmpDir)
46+
4447
NewSlice := func(start, end, step int) []int {
4548
if step <= 0 || end < start {
4649
return []int{}
@@ -63,7 +66,7 @@ func TestListDirectory(t *testing.T) {
6366
file.Close()
6467
}
6568

66-
list, err := ListDirectory(tmpdir, true)
69+
list, err := storage.ListDirectory(filepath.Base(tmpdir), true)
6770
require.Nil(t, err)
6871

6972
assert.NotNil(t, list)
@@ -77,8 +80,7 @@ func TestCountFiles(t *testing.T) {
7780
require.Nil(t, err)
7881
defer os.RemoveAll(tmpdir)
7982

80-
require.Nil(t, os.MkdirAll(tmpdir, os.ModePerm))
81-
defer os.RemoveAll(tmpdir)
83+
storage := NewStorage(tmpDir)
8284

8385
for i := 0; i < 60; i++ {
8486
file, err := os.Create(tmpdir + "/" + testPad(i) + "F")
@@ -91,7 +93,7 @@ func TestCountFiles(t *testing.T) {
9193
require.Nil(t, err)
9294
}
9395

94-
numberOfFiles, err := CountFiles(tmpdir)
96+
numberOfFiles, err := storage.CountFiles(filepath.Base(tmpdir))
9597
require.Nil(t, err)
9698
assert.Equal(t, 60, numberOfFiles)
9799
}
@@ -101,19 +103,20 @@ func BenchmarkCountFiles(b *testing.B) {
101103
require.Nil(b, err)
102104
defer os.RemoveAll(tmpdir)
103105

104-
os.MkdirAll(tmpdir, os.ModePerm)
105-
defer os.RemoveAll(tmpdir)
106+
storage := NewStorage(tmpDir)
106107

107108
for i := 0; i < 1000; i++ {
108109
file, err := os.Create(fmt.Sprintf("%s%010d", tmpdir, i))
109110
require.Nil(b, err)
110111
file.Close()
111112
}
112113

114+
basePath := filepath.Base(tmpdir)
115+
113116
b.ResetTimer()
114117
b.ReportAllocs()
115118
for n := 0; n < b.N; n++ {
116-
CountFiles(tmpdir)
119+
storage.CountFiles(basePath)
117120
}
118121
}
119122

@@ -122,16 +125,20 @@ func BenchmarkListDirectory(b *testing.B) {
122125
require.Nil(b, err)
123126
defer os.RemoveAll(tmpdir)
124127

128+
storage := NewStorage(tmpDir)
129+
125130
for i := 0; i < 1000; i++ {
126131
file, err := os.Create(fmt.Sprintf("%s%010d", tmpdir, i))
127132
require.Nil(b, err)
128133
file.Close()
129134
}
130135

136+
basePath := filepath.Base(tmpdir)
137+
131138
b.ResetTimer()
132139
b.ReportAllocs()
133140
for n := 0; n < b.N; n++ {
134-
ListDirectory(tmpdir, true)
141+
storage.ListDirectory(basePath, true)
135142
}
136143
}
137144

@@ -141,10 +148,13 @@ func BenchmarkExists(b *testing.B) {
141148
filename := file.Name()
142149
defer os.Remove(filename)
143150

151+
storage := NewStorage(tmpDir)
152+
basePath := filepath.Base(filename)
153+
144154
b.ResetTimer()
145155
b.ReportAllocs()
146156
for n := 0; n < b.N; n++ {
147-
Exists(filename)
157+
storage.Exists(basePath)
148158
}
149159
}
150160

@@ -154,12 +164,14 @@ func BenchmarkUpdateFile(b *testing.B) {
154164
filename := file.Name()
155165
defer os.Remove(filename)
156166

167+
storage := NewStorage(tmpDir)
168+
basePath := filepath.Base(filename)
157169
data := []byte("abcd")
158170

159171
b.ResetTimer()
160172
b.ReportAllocs()
161173
for n := 0; n < b.N; n++ {
162-
UpdateFile(filename, data)
174+
storage.UpdateFile(basePath, data)
163175
}
164176
}
165177

@@ -169,26 +181,31 @@ func BenchmarkAppendFile(b *testing.B) {
169181
filename := file.Name()
170182
defer os.Remove(filename)
171183

184+
storage := NewStorage(tmpDir)
185+
basePath := filepath.Base(filename)
172186
data := []byte("abcd")
173187

174188
b.ResetTimer()
175189
b.ReportAllocs()
176190
for n := 0; n < b.N; n++ {
177-
AppendFile(filename, data)
191+
storage.AppendFile(basePath, data)
178192
}
179193
}
180194

181195
func BenchmarkReadFileFully(b *testing.B) {
182-
file, err := ioutil.TempFile(tmpDir, "appended.*")
196+
file, err := ioutil.TempFile(tmpDir, "updated.*")
183197
require.Nil(b, err)
184198
filename := file.Name()
185199
defer os.Remove(filename)
186200

187-
require.Nil(b, UpdateFile(filename, []byte("abcd")))
201+
storage := NewStorage(tmpDir)
202+
basePath := filepath.Base(filename)
203+
204+
require.Nil(b, storage.UpdateFile(basePath, []byte("abcd")))
188205

189206
b.ResetTimer()
190207
b.ReportAllocs()
191208
for n := 0; n < b.N; n++ {
192-
ReadFileFully(filename)
209+
storage.ReadFileFully(basePath)
193210
}
194211
}

0 commit comments

Comments
 (0)