From e400b70ac6066cecd248c80172a99a5dad848e4a Mon Sep 17 00:00:00 2001 From: artste Date: Sun, 14 May 2023 18:16:08 +0200 Subject: [PATCH 1/2] FEAT: properly deal with special statements like del,assrt, with, for. --- nbs/01_ast.ipynb | 95 ++++++++++++++++++++++++++++++++++++++++++++- testcell/_modidx.py | 1 + testcell/core.py | 33 ++++++++++------ 3 files changed, 117 insertions(+), 12 deletions(-) diff --git a/nbs/01_ast.ipynb b/nbs/01_ast.ipynb index b1dd798..d2f75b2 100644 --- a/nbs/01_ast.ipynb +++ b/nbs/01_ast.ipynb @@ -60,9 +60,12 @@ " if len(tree.body)==0: return None\n", " src = tree.body[-1]\n", " last_node = None\n", + " parent_node = None\n", " for node in ast.walk(src):\n", " if isinstance(node, ast.stmt):\n", + " parent_node = last_node\n", " last_node = node\n", + " if parent_node is not None: return None # deal with nested statements like \"for loop\".\n", " return last_node" ] }, @@ -119,6 +122,45 @@ "test_eq(node_source(last_node(sample_code),sample_code), 'my_function(123)')" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| test\n", + "sample_code = ''\n", + "test_eq(node_source(last_node(sample_code),sample_code), None) # No code should display nothing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| test\n", + "sample_code = '''\n", + "for i in [1,2,3]:i\n", + "'''\n", + "test_eq(node_source(last_node(sample_code),sample_code), None) # should not display anyhting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| test\n", + "sample_code = '''\n", + "t=0 # sample assignment in the same cell\n", + "with open('test.txt') as f: \n", + " f.readlines()\n", + "'''\n", + "test_eq(node_source(last_node(sample_code),sample_code), None) # with block should catch implicit output" + ] + }, { "cell_type": "code", "execution_count": null, @@ -242,6 +284,38 @@ "test_eq( is_import_statement(last_node('from PIL import Image')) , True )" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def is_ast_node(x,ref):\n", + " for t in ref:\n", + " if isinstance(x,t): return True\n", + " return False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| test\n", + "test_eq(is_ast_node(last_node('del a'),[ast.Delete]), True)\n", + "test_eq(is_ast_node(last_node('del a'),[ast.Assert]), False)\n", + "test_eq(is_ast_node(last_node('a==1'),[ast.Assert]), False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "NOTE: I can't came around with any common propety to mark statements like `del a` and `assert b==1`. The only way I've found is to hardcode a comparison against these language statements." + ] + }, { "cell_type": "code", "execution_count": null, @@ -251,7 +325,9 @@ "#| export\n", "def need_display(node):\n", " if node is None: return False\n", + " if is_assignment(node): return False\n", " if is_function_call(node,names=['print','display']): return False\n", + " if is_ast_node(node,ref=[ast.Delete, ast.Assert, ast.Global, ast.Nonlocal]): return False\n", " if is_import_statement(node): return False\n", " return True" ] @@ -263,17 +339,34 @@ "outputs": [], "source": [ "#| test\n", + "# this is a bunch of real use cases\n", + "# NOTE: not considering \";\"\n", "def test_need_display(code): return need_display(last_node(code))\n", "\n", + "# SHOULD BE TRUE\n", "test_eq( test_need_display('a') , True )\n", "#test_eq( test_need_display('a;') , False ) # This is not supported with ast: we should do it differently\n", "test_eq( test_need_display('func(a)') , True )\n", "test_eq( test_need_display('{1:1,2:2}') , True )\n", + "test_eq( test_need_display('a in b') , True )\n", + "test_eq( test_need_display('a in b') , True )\n", + "test_eq( test_need_display('1 if True else None'), True)\n", + "\n", + "# SHOULD BE FALSE\n", "test_eq( test_need_display('display(a)') , False )\n", "test_eq( test_need_display('# xxx') , False )\n", "test_eq( test_need_display('print(a)') , False )\n", "test_eq( test_need_display('import xxx') , False )\n", - "test_eq( test_need_display('from xxx import yyy') , False )" + "test_eq( test_need_display('from xxx import yyy') , False )\n", + "test_eq( test_need_display('a=1') , False )\n", + "test_eq( test_need_display('for a in [1,2,3]: a') , False )\n", + "test_eq( test_need_display('del a') , False )\n", + "test_eq( test_need_display('a=1; del a') , False )\n", + "test_eq( test_need_display('assert a(b)==1') , False )\n", + "test_eq( test_need_display('try: a=0\\nexcept: a=1') , False )\n", + "test_eq( test_need_display('from numpy import array') , False )\n", + "test_eq( test_need_display('global a') , False )\n", + "test_eq( test_need_display('nonlocal a') , False )" ] }, { diff --git a/testcell/_modidx.py b/testcell/_modidx.py index 6fea211..fca6c0c 100644 --- a/testcell/_modidx.py +++ b/testcell/_modidx.py @@ -9,6 +9,7 @@ 'testcell.core.code_till_node': ('ast.html#code_till_node', 'testcell/core.py'), 'testcell.core.extract_call': ('ast.html#extract_call', 'testcell/core.py'), 'testcell.core.is_assignment': ('ast.html#is_assignment', 'testcell/core.py'), + 'testcell.core.is_ast_node': ('ast.html#is_ast_node', 'testcell/core.py'), 'testcell.core.is_function_call': ('ast.html#is_function_call', 'testcell/core.py'), 'testcell.core.is_import_statement': ('ast.html#is_import_statement', 'testcell/core.py'), 'testcell.core.last_node': ('ast.html#last_node', 'testcell/core.py'), diff --git a/testcell/core.py b/testcell/core.py index 2880440..238f6ab 100644 --- a/testcell/core.py +++ b/testcell/core.py @@ -1,8 +1,8 @@ # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_ast.ipynb. # %% auto 0 -__all__ = ['last_node', 'node_source', 'is_assignment', 'extract_call', 'is_function_call', 'is_import_statement', 'need_display', - 'wrap_node', 'last_statement_has_semicolon', 'code_till_node', 'auto_display'] +__all__ = ['last_node', 'node_source', 'is_assignment', 'extract_call', 'is_function_call', 'is_import_statement', 'is_ast_node', + 'need_display', 'wrap_node', 'last_statement_has_semicolon', 'code_till_node', 'auto_display'] # %% ../nbs/01_ast.ipynb 3 import ast @@ -13,20 +13,23 @@ def last_node(code): if len(tree.body)==0: return None src = tree.body[-1] last_node = None + parent_node = None for node in ast.walk(src): if isinstance(node, ast.stmt): + parent_node = last_node last_node = node + if parent_node is not None: return None # deal with nested statements like "for loop". return last_node # %% ../nbs/01_ast.ipynb 7 def node_source(node,code): return ast.get_source_segment(code,node) -# %% ../nbs/01_ast.ipynb 10 +# %% ../nbs/01_ast.ipynb 13 def is_assignment(node): return isinstance(node, ast.Assign) -# %% ../nbs/01_ast.ipynb 12 +# %% ../nbs/01_ast.ipynb 15 def extract_call(node): if not isinstance(node, ast.Expr): return None node = node.value # step in @@ -37,24 +40,32 @@ def extract_call(node): if isinstance(n, ast.Attribute): return n.attr return None # all the rest is not supported -# %% ../nbs/01_ast.ipynb 14 +# %% ../nbs/01_ast.ipynb 17 def is_function_call(node,names): function_name = extract_call(node) if function_name is None: return False # this is not a function call return function_name in names -# %% ../nbs/01_ast.ipynb 16 +# %% ../nbs/01_ast.ipynb 19 def is_import_statement(node): return isinstance(node, (ast.Import, ast.ImportFrom)) -# %% ../nbs/01_ast.ipynb 18 +# %% ../nbs/01_ast.ipynb 21 +def is_ast_node(x,ref): + for t in ref: + if isinstance(x,t): return True + return False + +# %% ../nbs/01_ast.ipynb 24 def need_display(node): if node is None: return False + if is_assignment(node): return False if is_function_call(node,names=['print','display']): return False + if is_ast_node(node,ref=[ast.Delete, ast.Assert, ast.Global, ast.Nonlocal]): return False if is_import_statement(node): return False return True -# %% ../nbs/01_ast.ipynb 20 +# %% ../nbs/01_ast.ipynb 26 def wrap_node(node,function_name): return ast.Expr( value=ast.Call( @@ -63,13 +74,13 @@ def wrap_node(node,function_name): keywords=[]) ) -# %% ../nbs/01_ast.ipynb 23 +# %% ../nbs/01_ast.ipynb 29 def last_statement_has_semicolon(code): t = [x.strip() for x in code.splitlines()] t = [x for x in t if not x.startswith('#')] return t[-1].endswith(';') -# %% ../nbs/01_ast.ipynb 25 +# %% ../nbs/01_ast.ipynb 31 def code_till_node(code:str,node): t = code.splitlines() t = t[:node.lineno] @@ -77,7 +88,7 @@ def code_till_node(code:str,node): if len(t[-1])==0: t = t[:-1] return '\n'.join(t) -# %% ../nbs/01_ast.ipynb 28 +# %% ../nbs/01_ast.ipynb 34 def auto_display(code): if last_statement_has_semicolon(code): return code From 255af8cc8795ff923aa1a0941c07d8264862f03e Mon Sep 17 00:00:00 2001 From: artste Date: Sun, 14 May 2023 23:56:31 +0200 Subject: [PATCH 2/2] Bump version --- settings.ini | 2 +- testcell/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/settings.ini b/settings.ini index 37d3da6..27139e4 100644 --- a/settings.ini +++ b/settings.ini @@ -5,7 +5,7 @@ ### Python library ### repo = testcell lib_name = %(repo)s -version = 0.0.4 +version = 0.0.5 min_python = 3.7 license = apache2 black_formatting = False diff --git a/testcell/__init__.py b/testcell/__init__.py index f768fc7..8a55310 100644 --- a/testcell/__init__.py +++ b/testcell/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.0.4" +__version__ = "0.0.5" # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_testcell.ipynb. # %% auto 0