-
Notifications
You must be signed in to change notification settings - Fork 1
/
insert_1024d_data.py
53 lines (39 loc) · 1.47 KB
/
insert_1024d_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
# CREATE DATABASE a;
# USE a;
# CREATE TABLE speedtest (id int, one_k_vector vecf32(1024));
import binascii
import time
import numpy as np
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
table_name = "speedtest"
vec_len = 1024
num_inserts = 1024 * 8
num_vector_per_insert = 5
def to_db_binary(value, dim=None):
if value is None:
return value
value = np.asarray(value, dtype='<f')
if value.ndim != 1:
raise ValueError('expected ndim to be 1')
# return value.tobytes()
return binascii.b2a_hex(value)
def run():
engine = create_engine("mysql+pymysql://root:[email protected]:6001/a")
Session = sessionmaker(bind=engine)
session = Session()
with open('/Users/arjunsunilkumar/PycharmProjects/mo-benchmark-test/a.log', 'w') as oFile:
sql_insert = text("insert into speedtest (id, one_k_vector) values(:id, decode(:data,'hex') );")
for i in range(num_inserts * num_vector_per_insert):
arr = np.random.rand(vec_len)
arr = np.asarray(arr, dtype='<f')
arr_str = '[' + ','.join(map(str, arr)) + ']\n'
oFile.write(arr_str)
session.execute(sql_insert, {"id": i, "data": to_db_binary(arr)})
session.commit()
start = time.time()
run()
duration = time.time() - start
print(f"Result: vector dim={vec_len} vectors "
f"inserted={num_inserts * num_vector_per_insert} "
f"insert/second={num_inserts * num_vector_per_insert / duration}")