Merge branch 'develop' into feat/ipaddress-types

saulpw · Jul 2, 2023 · 4dc7547 · 4dc7547
2 parents 7d79167 + 694680f
commit 4dc7547
Show file tree

Hide file tree

Showing 56 changed files with 611 additions and 189 deletions.
diff --git a/docs/assets/gmail_oauth/1.png b/docs/assets/gmail_oauth/1.png
diff --git a/docs/assets/gmail_oauth/10.png b/docs/assets/gmail_oauth/10.png
diff --git a/docs/assets/gmail_oauth/11.png b/docs/assets/gmail_oauth/11.png
diff --git a/docs/assets/gmail_oauth/12.png b/docs/assets/gmail_oauth/12.png
diff --git a/docs/assets/gmail_oauth/13.png b/docs/assets/gmail_oauth/13.png
diff --git a/docs/assets/gmail_oauth/14.png b/docs/assets/gmail_oauth/14.png
diff --git a/docs/assets/gmail_oauth/15.png b/docs/assets/gmail_oauth/15.png
diff --git a/docs/assets/gmail_oauth/16.png b/docs/assets/gmail_oauth/16.png
diff --git a/docs/assets/gmail_oauth/17.png b/docs/assets/gmail_oauth/17.png
diff --git a/docs/assets/gmail_oauth/18.png b/docs/assets/gmail_oauth/18.png
diff --git a/docs/assets/gmail_oauth/2.png b/docs/assets/gmail_oauth/2.png
diff --git a/docs/assets/gmail_oauth/3.png b/docs/assets/gmail_oauth/3.png
diff --git a/docs/assets/gmail_oauth/4.png b/docs/assets/gmail_oauth/4.png
diff --git a/docs/assets/gmail_oauth/5.png b/docs/assets/gmail_oauth/5.png
diff --git a/docs/assets/gmail_oauth/6.png b/docs/assets/gmail_oauth/6.png
diff --git a/docs/assets/gmail_oauth/7.png b/docs/assets/gmail_oauth/7.png
diff --git a/docs/assets/gmail_oauth/8.png b/docs/assets/gmail_oauth/8.png
diff --git a/docs/assets/gmail_oauth/9.png b/docs/assets/gmail_oauth/9.png
diff --git a/docs/assets/gmail_oauth/enable/1.png b/docs/assets/gmail_oauth/enable/1.png
diff --git a/docs/assets/gmail_oauth/enable/2.png b/docs/assets/gmail_oauth/enable/2.png
diff --git a/docs/assets/gmail_oauth/enable/3.png b/docs/assets/gmail_oauth/enable/3.png
diff --git a/docs/formats.md b/docs/formats.md
@@ -16,6 +16,7 @@ eleventyNavigation:
 |xls                 |Excel spreadsheets  |0\.42            |                |0\.42          |1987     |Microsoft           |xlrd                |
 |[fixed](#fixed)|fixed width text    |0\.97            |                |0\.97          |         |                    |                    |
 |[postgres](#postgres)|PostgreSQL database |0\.97            |                |0\.97          |1996     |                    |                    |
+|[imap](#imap)|Internet Message Access Protocol |2\.12            |                |2\.12          |1988     |                    |                    |
 |[vd](#vd)      |VisiData command log|0\.97            |                |0\.97          |2017     |VisiData            |                    |
 |vds           |              |VisiData Sheet      |2\.2             |yes                 |2021     |VisiData            |                    |                    |
 |[mbtiles](#mbtiles)|MapBox Tileset      |0\.98            |                |0\.98          |2011     |MapBox              |mapbox\-vector\-tile|
@@ -102,6 +103,12 @@ eleventyNavigation:
     - `postgres_schema` (default: 'public') the desired schema for the Postgres database
 - `vd postgres://`*username*`:`*password*`@`*hostname*`:`*port*`/`*database* opens a connection to the given postgres database.
 
+## imap {#imap}
+- `vd "imap://[email protected]:[email protected]"` opens a connection to the IMAP server
+    - e.g.  `vd "imap://[email protected]:[email protected]:993"`
+    - e.g.  `vd "imap://[email protected]@imap.gmail.com"`
+        - note that you don't specifiy a password for gmail here -- instead, you will be prompted to follow some instructions
+
 ### using VisiData as a pager within psql
 
 In psql:

diff --git a/docs/gmail.md b/docs/gmail.md
@@ -0,0 +1,165 @@
+# Using Gmail with OAuth 2.0
+
+## Why
+
+As of May 30 2022, Google [doesn't allow](https://support.google.com/accounts/answer/6010255?hl=en) you to log into your IMAP gmail account using only a username/password combination.
+So to open your Gmail in Visidata you need to create a Google API App, attach some scopes to it, generate a client ID and secret, then let Visidata use that client ID and secret (in a json file).
+
+## How
+
+First, go to the [Google Console](https://console.cloud.google.com/apis/dashboard)
+
+<br/>
+
+![](assets/gmail_oauth/1.png)
+
+<br/>
+
+And click `CREATE PROJECT`
+
+---
+
+
+![](assets/gmail_oauth/2.png)
+
+Give the project a name then click `CREATE`
+
+
+---
+
+Open a new tab and go to the [API Library](https://console.cloud.google.com/apis/library)
+
+<br/>
+
+![](assets/gmail_oauth/enable/1.png)
+
+Search for `gmail`
+
+---
+
+![](assets/gmail_oauth/enable/2.png)
+
+Click the search result `Gmail API`
+
+---
+
+![](assets/gmail_oauth/enable/3.png)
+
+Click `ENABLE`
+
+---
+
+Go back to your first tab
+
+On the left, select `OAuth consent screen` 
+
+![](assets/gmail_oauth/3.png)
+
+Then select `External` and click `CREATE` to create an App 
+
+---
+
+![](assets/gmail_oauth/4.png)
+
+Give the App a name and input your gmail address.
+
+---
+
+![](assets/gmail_oauth/5.png)
+
+Click `ADD OR REMOVE SCOPES`
+
+---
+
+![](assets/gmail_oauth/6.png)
+
+Search for `gmail`
+
+---
+
+![](assets/gmail_oauth/7.png)
+
+Click the checkbox by the row with the scope value `https://mail.google.com/`
+
+Then scroll to the bottom
+
+---
+
+![](assets/gmail_oauth/8.png)
+
+And click `UPDATE`
+
+---
+
+![](assets/gmail_oauth/9.png)
+
+You should see your selected scopes.
+
+Click `SAVE AND CONTINUE`
+
+---
+
+![](assets/gmail_oauth/10.png)
+
+Click `ADD USERS`
+
+---
+
+![](assets/gmail_oauth/11.png)
+
+Type in your gmail email address then click `ADD`
+
+---
+
+![](assets/gmail_oauth/12.png)
+
+On the left, click `Credentials`
+
+---
+
+![](assets/gmail_oauth/13.png)
+
+Near the top click `CREATE CREDENTIALS`
+
+Then click `OAuth client ID`
+
+---
+
+![](assets/gmail_oauth/14.png)
+
+Select the application type `Desktop App` and give your OAuth 2.0 client a name then click `CREATE`
+
+---
+
+![](assets/gmail_oauth/15.png)
+
+Click `DOWNLOAD JSON` and move the downloaded file into the visidata project directory at the path `vdplus/api/google/` and call the file `google-creds.json`
+
+---
+
+Now, on the command line run the equivalent for you:
+
+`vd "imap://[email protected]@imap.gmail.com"`
+
+Then you should get a web browser popup:
+
+![](assets/gmail_oauth/16.png)
+
+Select the account whose email address you have been using in these instructions.
+
+---
+
+![](assets/gmail_oauth/17.png)
+
+Click `Select all`
+
+---
+
+![](assets/gmail_oauth/18.png)
+
+See your gmail in Visidata.
+
+<Chef's Kiss>
+
+---
+
diff --git a/docs/rows.md b/docs/rows.md
@@ -29,7 +29,7 @@ An example usage follows.
 
 ## How to filter rows
 
-1. Press `s` or `t` on the rows to filter.
+1. Press `s` or `t` on the rows to be filtered.
 
 2. Press
 

diff --git a/tests/join-merge.vd b/tests/join-merge.vd
@@ -1,8 +1,8 @@
 sheet	col	row	longname	input	keystrokes	comment
-			open-file	tests/join-merge-1.jsonl	o	
 			open-file	tests/join-merge-2.jsonl	o	
-join-merge-2	colA		key-col		!	
+			open-file	tests/join-merge-1.jsonl	o	
 join-merge-1	colA		key-col		!	
-join-merge-2			join-sheets-top2	merge	&	
-join-merge-2+join-merge-1	colA		type-int		#	
-join-merge-2+join-merge-1	colA		sort-asc		[	
+join-merge-2	colA		key-col		!	
+join-merge-1			join-sheets-top2	merge	&	
+join-merge-1+join-merge-2	colA		type-int		#	
+join-merge-1+join-merge-2	colA		sort-asc		[	
diff --git a/vdplus/api/google/__init__.py b/vdplus/api/google/__init__.py
@@ -1,35 +1,2 @@
-
-from visidata import vd, VisiData, Sheet, IndexSheet, SequenceSheet, ColumnItem, Path, AttrDict, ColumnAttr, asyncthread
 from .gsheets import *
 from .gdrive import *
-
-
-def _google_creds_fn():
-    from pkg_resources import resource_filename
-    return resource_filename('vdplus.api.google', 'google-creds.json')
-
-
-@VisiData.api
-def google_auth(vd, scopes='spreadsheets.readonly'):
-    import pickle
-    import os.path
-    GSHEETS_TOKEN_FILE = Path(vd.options.visidata_dir)/f'google-{scopes}.pickle'
-    creds = None
-    if os.path.exists(GSHEETS_TOKEN_FILE):
-        with open(GSHEETS_TOKEN_FILE, 'rb') as fp:
-            creds = pickle.load(fp)
-
-    if not creds or not creds.valid:
-        if creds and creds.expired and creds.refresh_token:
-            from google.auth.transport.requests import Request
-            creds.refresh(Request())
-        else:
-            from google_auth_oauthlib.flow import InstalledAppFlow
-            SCOPES = [f'https://www.googleapis.com/auth/{x}' for x in scopes.split()]
-            flow = InstalledAppFlow.from_client_secrets_file(_google_creds_fn(), SCOPES)
-            creds = flow.run_local_server(port=0)
-
-        with open(GSHEETS_TOKEN_FILE, 'wb') as fp:
-            pickle.dump(creds, fp)
-
-    return creds
diff --git a/vdplus/api/google/google-creds.json b/vdplus/api/google/google-creds.json