Merge branch 'develop' into stable

CHANGELOG.md

@@ -1,6 +1,6 @@
 # VisiData version history
 
-# v1.4.1 (2018-10-29)
+# v1.5 (2018-11-02)
 
 ## Bugfixes
 - [clipboard] fix broken `gzY` (syscopy-cells)
@@ -24,9 +24,10 @@
 - [tsv] add unnamed columns if extra cells in rows
 - [diff] now based on display value (more intuitive)
 - [mouse] move to column also
+- [mouse] right-click to rename-col, rename-sheet, or edit-cell
 - [cosmetic] addcol-new (`za`) input new column name on top of new column
 - [cosmetic] include file iteration in progress meter
-
+- [xls xlsx] use options.header to determine column names
 
 # v1.4 (2018-09-23)
 

README.md

@@ -1,4 +1,4 @@
-# VisiData v1.4 [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/stable.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/stable)
+# VisiData v1.5 [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/stable.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/stable)
 
 A terminal interface for exploring and arranging tabular data.
 

bin/vd

@@ -3,7 +3,7 @@
 # Usage: $0 [<options>] [<input> ...]
 #        $0 [<options>] --play <cmdlog> [--batch] [-w <waitsecs>] [-o <output>] [field=value ...]
 
-__version__ = '1.4'
+__version__ = '1.5'
 __version_info__ = 'saul.pw/VisiData v' + __version__
 
 import os

docs/edit.md

@@ -40,10 +40,18 @@ In most cases, `^` is the preferred command. Examples which demo `^` can be seen
 
 ###### How to set the header in an Excel sheet?
 
-For the standard filetypes (e.g. csv, tsv) the loaders assume that the dataset's first `options.header` rows contain the column names. The Excel loader does not because an Excel file can have multiple sheets with varying number of header rows.
+For most filetypes (e.g. csv, tsv, xls(x)) the loaders assume that the dataset's first `options.header` rows contain the column names.
 
-1. Press `s` or `t` to select the rows which represent the header rows.
-2. Press `g^` to set the names of the headers to the contents of selected rows.
+If the Excel file has multiple sheets with varying number of header rows:
+
+1. Pass `--header==0` while loading the file.
+
+~~~
+vd file.xlsx --header=0
+~~~
+
+2. For each sheet, press `s` or `t` to select the rows which represent the header rows.
+3. Press `g^` to set the names of the headers to the contents of selected rows.
 
 ###### How to rename headers using the Columns sheet
 

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup
 # tox can't actually run python3 setup.py: https://github.com/tox-dev/tox/issues/96
 #from visidata import __version__
-__version__ = '1.4'
+__version__ = '1.5'
 
 setup(name='visidata',
       version=__version__,

tests/load-xlsx.vd

@@ -1,5 +1,3 @@
-sheet	col	row	keystrokes	input	comment
-			o	sample_data/sample-sales-reps.xlsx	open file
-sample-sales-reps		キsample-salesv4	^J		load the entire table into memory
-sample-sales-reps_sample-salesv4			g^		set names of all visible columns to contents of selected rows (or current row)
-sample-sales-reps_sample-salesv4		0	d		delete current row and move it to clipboard
+sheet	col	row	longname	input	keystrokes	comment
+			open-file	sample_data/sample-sales-reps.xlsx	o	
+sample-sales-reps		キsample-salesv4	dive-row		^J	

tests/xlsx-header.vd

@@ -0,0 +1,6 @@
+sheet	col	row	longname	input	keystrokes	comment
+	override	header	set-option	0		
+			open-file	sample_data/sample-sales-reps.xlsx	o	
+sample-sales-reps		キsample-salesv4	dive-row		^J	
+sample-sales-reps_sample-salesv4		0	rename-cols-row		g^	
+sample-sales-reps_sample-salesv4		0	delete-row		d	

visidata/data.py

@@ -5,7 +5,7 @@
 
 option('confirm_overwrite', True, 'whether to prompt for overwrite confirmation on save')
 replayableOption('safe_error', '#ERR', 'error string to use while saving')
-replayableOption('header', 1, 'parse first N rows of .csv/.tsv as column names')
+replayableOption('header', 1, 'parse first N rows of certain formats as column names')
 replayableOption('delimiter', '\t', 'delimiter to use for tsv filetype')
 replayableOption('filetype', '', 'specify file type')
 replayableOption('save_filetype', 'tsv', 'specify default file type to save as')

visidata/loaders/xlsx.py

@@ -40,8 +40,18 @@ def reload(self):
 
         self.columns = []
         self.rows = []
-        for row in Progress(worksheet.iter_rows(), worksheet.max_row or 0):
-            row = list(wrapply(getattr, cell, 'value') for cell in row)
+
+        rows = worksheet.iter_rows()
+        hdrs = [list(wrapply(getattr, cell, 'value')
+                   for cell in next(rows))
+                       for i in range(options.header)
+               ]
+        colnames = ['\n'.join(str(hdr[i]) for i in range(len(hdr))) for hdr in zip(*hdrs)]
+        for i, colname in enumerate(colnames):
+            self.addColumn(ColumnItem(colname, i))
+
+        for r in Progress(rows, worksheet.max_row or 0):
+            row = list(wrapply(getattr, cell, 'value') for cell in r)
             for i in range(len(self.columns), len(row)):  # no-op if already done
                 self.addColumn(ColumnItem(None, i, width=8))
             self.addRow(row)
@@ -78,9 +88,16 @@ class xlsSheet(Sheet):
     def reload(self):
         worksheet = self.source
         self.columns = []
-        for i in range(worksheet.ncols):
-            self.addColumn(ColumnItem(None, i, width=8))
+        if options.header:
+            hdrs = [list(worksheet.cell(rownum, colnum).value for colnum in range(worksheet.ncols))
+                        for rownum in range(options.header)]
+            colnames = ['\\n'.join(str(hdr[i]) for i in range(len(hdr))) for hdr in zip(*hdrs)]
+        else:
+            colnames = ['']*worksheet.ncols
+
+        for i, colname in enumerate(colnames):
+            self.addColumn(ColumnItem(colname, i))
 
         self.rows = []
-        for rownum in Progress(range(worksheet.nrows)):
+        for rownum in Progress(range(options.header, worksheet.nrows)):
             self.addRow(list(worksheet.cell(rownum, colnum).value for colnum in range(worksheet.ncols)))

visidata/man/vd.1

@@ -1,4 +1,4 @@
-.Dd September 21, 2018
+.Dd November 1, 2018
 .Dt vd \&1 "Quick Reference Guide" 
 .Os Linux/MacOS
 .
@@ -678,7 +678,7 @@ overwrite existing files without confirmation
 .It Sy --encoding Ns = Ns Ar "str              " No "utf-8"
 encoding passed to codecs.open
 .It Sy --encoding-errors Ns = Ns Ar "str       " No "surrogateescape"
-encoding errors passed to codecs.open
+encoding_errors passed to codecs.open
 .It Sy --regex-flags Ns = Ns Ar "str           " No "I"
 flags to pass to re.compile() [AILMSUX]
 .It Sy --default-width Ns = Ns Ar "int         " No "20"
@@ -719,10 +719,10 @@ amount to scroll with scrollwheel
 skip first N lines of text input
 .It Sy --confirm-overwrite Ns = Ns Ar "bool    " No "True"
 whether to prompt for overwrite confirmation on save
-.It Sy --save-errors Ns = Ns Ar "bool          " No "True"
-whether to save or discard errors while saving
+.It Sy --safe-error Ns = Ns Ar "str            " No "#ERR"
+error string to use while saving
 .It Sy --header Ns = Ns Ar "int                " No "1"
-parse first N rows of .csv/.tsv as column names
+parse first N rows of certain formats as column names
 .It Sy --delimiter Ns = Ns Ar "str             " No "	"
 delimiter to use for tsv filetype
 .It Sy --filetype Ns = Ns Ar "str              " No ""

visidata/man/vd.inc

@@ -1,4 +1,4 @@
-.Dd September 21, 2018
+.Dd November 1, 2018
 .Dt vd \&1 "Quick Reference Guide" 
 .Os Linux/MacOS
 .

visidata/vdtui.py

@@ -26,7 +26,7 @@
 # Just include this whole file in your project as-is.  If you do make
 # modifications, please keep the base vdtui version and append your own id and
 # version.
-__version__ = '1.4'
+__version__ = '1.5'
 __version_info__ = 'saul.pw/vdtui v' + __version__
 __author__ = 'Saul Pwanson <[email protected]>'
 __license__ = 'MIT'
@@ -37,6 +37,7 @@
 import sys
 import os
 import collections
+from collections import defaultdict
 from copy import copy, deepcopy
 from contextlib import suppress
 import curses
@@ -255,7 +256,7 @@ def replayableOption(optname, default, helpstr):
 
 
 replayableOption('encoding', 'utf-8', 'encoding passed to codecs.open')
-replayableOption('encoding_errors', 'surrogateescape', 'encoding errors passed to codecs.open')
+replayableOption('encoding_errors', 'surrogateescape', 'encoding_errors passed to codecs.open')
 
 replayableOption('regex_flags', 'I', 'flags to pass to re.compile() [AILMSUX]')
 replayableOption('default_width', 20, 'default column width')
@@ -561,6 +562,11 @@ def __init__(self):
         self.prefixWaiting = False
         self.scr = None  # curses scr
         self.hooks = collections.defaultdict(list)  # [hookname] -> list(hooks)
+        self.mousereg = defaultdict(  # [scr] ->
+                  lambda: defaultdict(  # [y] ->
+                    lambda: defaultdict(  # [x] ->
+                      lambda: defaultdict(  # [button] (like "BUTTON1_CLICKED") ->
+                        lambda: None))))      # func(y, x, button)
         self.threads = [] # all long-running threads, including main and finished
         self.addThread(threading.current_thread(), endTime=0)
         self.addHook('rstatus', lambda sheet,self=self: (self.keystrokes, 'color_keystrokes'))
@@ -746,6 +752,12 @@ def exceptionCaught(self, exc=None, **kwargs):
         if options.debug:
             raise
 
+    def onMouse(self, scr, y, x, h, w, **kwargs):
+        for i in range(y, y+h):
+            for j in range(x, x+w):
+                for button, func in kwargs.items():
+                    self.mousereg[scr][i][j][button] = func
+
     def drawLeftStatus(self, scr, vs):
         'Draw left side of status bar.'
         cattr = CursesAttr(colors.color_status)
@@ -762,6 +774,10 @@ def drawLeftStatus(self, scr, vs):
 
             y = self.windowHeight-1
             x = clipdraw(scr, y, 0, lstatus, attr)
+            self.onMouse(scr, y, 0, 1, x,
+                            BUTTON1_PRESSED='sheets',
+                            BUTTON3_PRESSED='rename-sheet')
+
             one = False
             for (pri, msgparts), n in sorted(self.statuses.items(), key=lambda k: -k[0][0]):
                 if x > self.windowWidth:
@@ -851,22 +867,37 @@ def run(self, scr):
                 self.statuses.clear()
 
                 if keystroke == 'KEY_MOUSE':
+                    self.keystrokes = ''
+                    clicktype = ''
                     try:
                         devid, x, y, z, bstate = curses.getmouse()
+                        sheet.mouseX, sheet.mouseY = x, y
                         if bstate & curses.BUTTON_CTRL:
-                            self.keystrokes += "CTRL-"
+                            clicktype += "CTRL-"
                             bstate &= ~curses.BUTTON_CTRL
                         if bstate & curses.BUTTON_ALT:
-                            self.keystrokes += "ALT-"
+                            clicktype += "ALT-"
                             bstate &= ~curses.BUTTON_ALT
                         if bstate & curses.BUTTON_SHIFT:
-                            self.keystrokes += "SHIFT-"
+                            clicktype += "SHIFT-"
                             bstate &= ~curses.BUTTON_SHIFT
 
-                        keystroke = curses.mouseEvents.get(bstate, str(bstate))
-                        sheet.mouseX, sheet.mouseY = x, y
+                        keystroke = clicktype + curses.mouseEvents.get(bstate, str(bstate))
+
+                        f = self.mousereg[scr][y][x][keystroke]
+                        if f:
+                            if isinstance(f, str):
+                                for cmd in f.split():
+                                    sheet.exec_keystrokes(cmd)
+                            else:
+                                f(y, x, keystroke)
+
+                            self.keystrokes = keystroke
+                            keystroke = ''
                     except curses.error:
-                        keystroke = ''
+                        pass
+                    except Exception as e:
+                        exceptionCaught(e)
 
                 self.keystrokes += keystroke
 
@@ -1695,6 +1726,7 @@ def drawColHeader(self, scr, y, vcolidx):
             N = N[:colwidth-len(options.disp_truncator)] + options.disp_truncator
         clipdraw(scr, y, x, N, hdrattr.attr, colwidth)
         clipdraw(scr, y, x+colwidth-len(T), T, hdrattr.attr, len(T))
+        vd.onMouse(scr, y, x, 1, colwidth, BUTTON3_RELEASED='rename-col')
 
         if vcolidx == self.leftVisibleColIndex and col not in self.keyCols and self.nonKeyVisibleCols.index(col) > 0:
             A = options.disp_more_left
@@ -1764,6 +1796,7 @@ def draw(self, scr):
                         clipdraw(scr, y, x+colwidth-len(note), note, noteattr.attr, len(note))
 
                     clipdraw(scr, y, x, disp_column_fill+cellval.display, attr.attr, colwidth-(1 if note else 0))
+                    vd.onMouse(scr, y, x, 1, colwidth, BUTTON3_RELEASED='edit-cell')
 
                     sepchars = disp_column_sep
                     if (self.keyCols and col is self.keyCols[-1]) or vcolidx == self.rightVisibleColIndex:
@@ -1820,9 +1853,13 @@ def editCell(self, vcolidx=None, rowidx=None, **kwargs):
 
 Sheet.addCommand('BUTTON1_PRESSED', 'go-mouse', 'sheet.cursorRowIndex=topRowIndex+mouseY-1; sheet.cursorVisibleColIndex=visibleColAtX(mouseX)'),
 Sheet.addCommand('BUTTON1_RELEASED', 'scroll-mouse', 'sheet.topRowIndex=cursorRowIndex-mouseY+1'),
+
 Sheet.addCommand('BUTTON4_PRESSED', 'scroll-up', 'cursorDown(options.scroll_incr); sheet.topRowIndex += options.scroll_incr'),
 Sheet.addCommand('REPORT_MOUSE_POSITION', 'scroll-down', 'cursorDown(-options.scroll_incr); sheet.topRowIndex -= options.scroll_incr'),
 
+Sheet.bindkey('BUTTON1_CLICKED', 'go-mouse')
+Sheet.bindkey('BUTTON3_PRESSED', 'go-mouse')
+
 Sheet.addCommand('^G', 'show-cursor', 'status(statusLine)'),
 
 Sheet.addCommand('_', 'resize-col-max', 'cursorCol.toggleWidth(cursorCol.getMaxWidth(visibleRows))'),