From 0b16ce55bb5b816a5021725ac4cacdd3a2ae1ca7 Mon Sep 17 00:00:00 2001 From: bennyelg Date: Mon, 10 Sep 2018 22:41:50 +0300 Subject: [PATCH 1/6] add option to export result --- src/csvql.nim | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/csvql.nim b/src/csvql.nim index a2769c5..4fe72e2 100644 --- a/src/csvql.nim +++ b/src/csvql.nim @@ -248,7 +248,17 @@ proc getLongestWordsByPosition(rs: seq[tuple[r: Row, length: int]]): seq[int] = return lengths -proc displayResults(db: Database, csvs: seq[Csv], query: string) = +proc exportResults(columns: seq[string], resultSet: seq[seq[string]]): string = + let dt = format(now(), "yyyy-mm-ddHH:mm:ss").replace("-","_").replace(":", "_") + var fs = newFileStream("/tmp/" & dt & ".csv", fmWrite) + for idx, row in resultSet: + if idx == 0: + fs.writeLine(columns.join(",")) + fs.writeLine(row.join(",")) + + return "/tmp/" & dt & ".csv" + +proc displayResults(db: Database, csvs: seq[Csv], query: string, exportResult: bool = false) = var queryColumns = getQueryColumns(csvs, query) var rows: seq[tuple[r: Row, length: int]] = @[] for row in db.connection.fastRows(SqlQuery(query)): @@ -277,7 +287,16 @@ proc displayResults(db: Database, csvs: seq[Csv], query: string) = for row in fin: echo("|" & row.join("|") & " |") echo(row.join("|").len * "-") - + + if exportResult: + let exportResultHeader = """ +---------- +::Export:: +---------- +""" + styledWriteLine(stdout, fgRed, exportResultHeader, resetStyle) + let generatedCsvPath = exportResults(queryColumns, rows.mapIt(it.r)) + styledWriteLine(stdout, fgGreen, "File is ready & can be located in: " & generatedCsvPath, resetStyle) proc parseQuery(query: string): (seq[Csv], string) = @@ -297,7 +316,7 @@ proc parseQuery(query: string): (seq[Csv], string) = newQuery = newQuery.replace(csvPath.replace("'", ""), fmt"t{idx + 1}").replace("'", "") return (csvs, newQuery) -proc csvQL(query: string) = +proc csvQL(query: string, exportResult: bool = false) = let startTime = cpuTime() let db = openConnection() let (csvs, adjustedQuery) = parseQuery(query) @@ -317,7 +336,7 @@ proc csvQL(query: string) = ::Result:: ----------""" styledWriteLine(stdout, fgRed, queryResultHeader, resetStyle) - displayResults(db, csvs, adjustedQuery) + displayResults(db, csvs, adjustedQuery, exportResult) styledWriteLine(stdout, fgYellow, fmt"* Total Duration: {cpuTime() - startTime} seconds.", resetStyle) when isMainModule: @@ -326,6 +345,6 @@ when isMainModule: i.e: 1) SELECT name, day, hour FROM '/path/to/csv/test.csv' as t1 LIMIT 10 2) SELECT name, lastname, birthday FROM '/path/to/csv/test.csv' as t1 LEFT JOIN '/path/to/csv/test2.csv' as t2 ON t1.name = t2.name -""" }) +""", "exportResult": "set to true if you want to export the result set." }) From 01dad3fc9b2309e3b31eb0a72f886ff9a50dcbdb Mon Sep 17 00:00:00 2001 From: bennyelg Date: Tue, 11 Sep 2018 09:45:31 +0300 Subject: [PATCH 2/6] generic temp dir. --- src/csvql.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/csvql.nim b/src/csvql.nim index 4fe72e2..c6e5db4 100644 --- a/src/csvql.nim +++ b/src/csvql.nim @@ -250,7 +250,7 @@ proc getLongestWordsByPosition(rs: seq[tuple[r: Row, length: int]]): seq[int] = proc exportResults(columns: seq[string], resultSet: seq[seq[string]]): string = let dt = format(now(), "yyyy-mm-ddHH:mm:ss").replace("-","_").replace(":", "_") - var fs = newFileStream("/tmp/" & dt & ".csv", fmWrite) + var fs = newFileStream(getTempDir() & dt & ".csv", fmWrite) for idx, row in resultSet: if idx == 0: fs.writeLine(columns.join(",")) From d48da796d107faffb46d292dd56b78c6ab523394 Mon Sep 17 00:00:00 2001 From: bennyelg Date: Tue, 11 Sep 2018 10:05:31 +0300 Subject: [PATCH 3/6] cleaning start --- src/csvql.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/csvql.nim b/src/csvql.nim index c6e5db4..89701b6 100644 --- a/src/csvql.nim +++ b/src/csvql.nim @@ -250,13 +250,14 @@ proc getLongestWordsByPosition(rs: seq[tuple[r: Row, length: int]]): seq[int] = proc exportResults(columns: seq[string], resultSet: seq[seq[string]]): string = let dt = format(now(), "yyyy-mm-ddHH:mm:ss").replace("-","_").replace(":", "_") - var fs = newFileStream(getTempDir() & dt & ".csv", fmWrite) + let generatedFilePath = getTempDir() & dt & ".csv" + var fs = newFileStream(generatedFilePath, fmWrite) for idx, row in resultSet: if idx == 0: fs.writeLine(columns.join(",")) fs.writeLine(row.join(",")) - return "/tmp/" & dt & ".csv" + return generatedFilePath proc displayResults(db: Database, csvs: seq[Csv], query: string, exportResult: bool = false) = var queryColumns = getQueryColumns(csvs, query) From 57329c10cbb87e5281d5f8c858a5b44a4bbc77f7 Mon Sep 17 00:00:00 2001 From: bennyelg Date: Tue, 11 Sep 2018 10:11:32 +0300 Subject: [PATCH 4/6] gitignotr --- .gitignore | 3 +++ src/csvql.nim | 8 -------- 2 files changed, 3 insertions(+), 8 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3a71767 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/nimcache/* +csvql +src/nimcache/* diff --git a/src/csvql.nim b/src/csvql.nim index 89701b6..d6e4911 100644 --- a/src/csvql.nim +++ b/src/csvql.nim @@ -76,7 +76,6 @@ proc guessCsvDelimiter(filePath: string): char = else: return ',' - proc appendCsv* (path, alias: string, hasHeader: bool = false): Csv = ## return a new Csv object. @@ -114,7 +113,6 @@ proc figureColumnsTypes(rowsSamples: seq[seq[string]]): seq[string] = let csvTypes = getTypesWithMostProbability(types) return csvTypes - proc parseCsv* (csv: Csv) = const numOfSamplingRows = 50 var csvStream = newFileStream(csv.path, fmRead) @@ -150,7 +148,6 @@ proc parseCsv* (csv: Csv) = csv.types = figureColumnsTypes(rowsSampleCount) - proc openConnection* (): Database = Database( connection: open(":memory:", nil, nil, nil) @@ -168,7 +165,6 @@ proc createTableUsingCsvProperties(db: Database, csv: Csv) = """ db.connection.exec(SqlQuery(statement)) - proc executeChunk(args: tuple[db: Database, tableName: string, columns: seq[string], rows: seq[seq[string]]]) = var statement = fmt""" INSERT INTO {args.tableName}({args.columns.join(",")}) @@ -181,7 +177,6 @@ proc executeChunk(args: tuple[db: Database, tableName: string, columns: seq[stri let executableStatement = statement & insertableRows.join(",") & ";" discard args.db.connection.tryExec(SqlQuery(executableStatement)) - proc insertCsvRowsIntoTable(db: Database, csv: Csv) = db.connection.exec(sql"PRAGMA synchronous=OFF") db.connection.exec(sql"BEGIN TRANSACTION;") @@ -203,7 +198,6 @@ proc insertCsvRowsIntoTable(db: Database, csv: Csv) = db.connection.exec(sql"COMMIT;") - proc `*`(size: int, del: string): string = result = "+" for i in countup(0, size): @@ -247,7 +241,6 @@ proc getLongestWordsByPosition(rs: seq[tuple[r: Row, length: int]]): seq[int] = lengths[idx] = word.len return lengths - proc exportResults(columns: seq[string], resultSet: seq[seq[string]]): string = let dt = format(now(), "yyyy-mm-ddHH:mm:ss").replace("-","_").replace(":", "_") let generatedFilePath = getTempDir() & dt & ".csv" @@ -299,7 +292,6 @@ proc displayResults(db: Database, csvs: seq[Csv], query: string, exportResult: b let generatedCsvPath = exportResults(queryColumns, rows.mapIt(it.r)) styledWriteLine(stdout, fgGreen, "File is ready & can be located in: " & generatedCsvPath, resetStyle) - proc parseQuery(query: string): (seq[Csv], string) = let csvsPaths = re.findAll(query, re"'(.*?)'") var csvs = newSeqOfCap[Csv](csvsPaths.len) From 4dfd4d88a90638c690b3e5babbe65f6399e2ed3c Mon Sep 17 00:00:00 2001 From: bennyelg Date: Wed, 12 Sep 2018 15:19:52 +0300 Subject: [PATCH 5/6] Fix bug - anything qouted was turn into table t identifier. --- src/csvql.nim | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/csvql.nim b/src/csvql.nim index d6e4911..b0d4211 100644 --- a/src/csvql.nim +++ b/src/csvql.nim @@ -293,7 +293,8 @@ proc displayResults(db: Database, csvs: seq[Csv], query: string, exportResult: b styledWriteLine(stdout, fgGreen, "File is ready & can be located in: " & generatedCsvPath, resetStyle) proc parseQuery(query: string): (seq[Csv], string) = - let csvsPaths = re.findAll(query, re"'(.*?)'") + let csvsPaths = re.findAll(query, re"'(.*?).csv'") + echo(csvsPaths) var csvs = newSeqOfCap[Csv](csvsPaths.len) var newQuery = query let propertiesHeader =""" @@ -306,9 +307,11 @@ proc parseQuery(query: string): (seq[Csv], string) = styledWriteLine(stdout, fgGreen, fmt"t{idx + 1} = {csvPath}", resetStyle) let csv = appendCsv(csvPath.replace("'", ""), fmt"t{idx + 1}", true) csvs.add(csv) - newQuery = newQuery.replace(csvPath.replace("'", ""), fmt"t{idx + 1}").replace("'", "") - return (csvs, newQuery) + newQuery = newQuery.replace(csvPath.replace("'", ""), fmt"t{idx + 1}") + + return (csvs, newQuery) + proc csvQL(query: string, exportResult: bool = false) = let startTime = cpuTime() let db = openConnection() From 45edbedb0fa872497925dd79e07f1f9d4763bc87 Mon Sep 17 00:00:00 2001 From: bennyelg Date: Wed, 12 Sep 2018 15:20:09 +0300 Subject: [PATCH 6/6] remove print --- src/csvql.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/src/csvql.nim b/src/csvql.nim index b0d4211..adce673 100644 --- a/src/csvql.nim +++ b/src/csvql.nim @@ -294,7 +294,6 @@ proc displayResults(db: Database, csvs: seq[Csv], query: string, exportResult: b proc parseQuery(query: string): (seq[Csv], string) = let csvsPaths = re.findAll(query, re"'(.*?).csv'") - echo(csvsPaths) var csvs = newSeqOfCap[Csv](csvsPaths.len) var newQuery = query let propertiesHeader ="""