Support **$... for keyword arguments.

This pretty much just works out of the box, albeit it's ordering-dependent.

For instance, to match any call to `gfile.Stat` which does _not_ include `stat_proto=True`, you can use:

```python
AllOf(
  ExprPattern("gfile.Stat(*$..., **$...)"),
  Unless(ExprPattern("$_(*$..., **$..., stat_proto=True, **$...)"))
)
```

Or similar.

A future change to refex (which I do want to make eventually...) could eventually mean keyword parameter and dict ordering is ignored by default, so that we can remove one of the `**$...`.

PiperOrigin-RevId: 568317030

Author: ssbr

refex/python/matcher.py

@@ -203,74 +203,6 @@ def register_constant(name: str, constant: Any):
   registered_constants[name] = constant
 
 
-def coerce(value):  # Nobody uses coerce. pylint: disable=redefined-builtin
-  """Returns the 'intended' matcher given by `value`.
-
-  If `value` is already a matcher, then this is what is returned.
-
-  If `value` is anything else, then coerce returns `ImplicitEquals(value)`.
-
-  Args:
-    value: Either a Matcher, or a value to compare for equality.
-  """
-  if isinstance(value, Matcher):
-    return value
-  else:
-    return ImplicitEquals(value)
-
-
-def _coerce_list(values):
-  return [coerce(v) for v in values]
-
-
-# TODO(b/199577701): drop the **kwargs: Any in the *_attrib functions.
-
-_IS_SUBMATCHER_ATTRIB = __name__ + '._IS_SUBMATCHER_ATTRIB'
-_IS_SUBMATCHER_LIST_ATTRIB = __name__ + '._IS_SUBMATCHER_LIST_ATTRIB'
-
-
-def submatcher_attrib(*args, walk: bool = True, **kwargs: Any):
-  """Creates an attr.ib that is marked as a submatcher.
-
-  This will cause the matcher to be automatically walked as part of the
-  computation of .bind_variables. Any submatcher that can introduce a binding
-  must be listed as a submatcher_attrib or submatcher_list_attrib.
-
-  Args:
-    *args: Forwarded to attr.ib.
-    walk: Whether or not to walk to accumulate .bind_variables.
-    **kwargs: Forwarded to attr.ib.
-
-  Returns:
-    An attr.ib()
-  """
-  if walk:
-    kwargs.setdefault('metadata', {})[_IS_SUBMATCHER_ATTRIB] = True
-  kwargs.setdefault('converter', coerce)
-  return attr.ib(*args, **kwargs)
-
-
-def submatcher_list_attrib(*args, walk: bool = True, **kwargs: Any):
-  """Creates an attr.ib that is marked as an iterable of submatchers.
-
-  This will cause the matcher to be automatically walked as part of the
-  computation of .bind_variables. Any submatcher that can introduce a binding
-  must be listed as a submatcher_attrib or submatcher_list_attrib.
-
-  Args:
-    *args: Forwarded to attr.ib.
-    walk: Whether or not to walk to accumulate .bind_variables.
-    **kwargs: Forwarded to attr.ib.
-
-  Returns:
-    An attr.ib()
-  """
-  if walk:
-    kwargs.setdefault('metadata', {})[_IS_SUBMATCHER_LIST_ATTRIB] = True
-  kwargs.setdefault('converter', _coerce_list)
-  return attr.ib(*args, **kwargs)
-
-
 # TODO: make MatchObject, MatchInfo, and Matcher generic, parameterized
 # by match type. Since pytype doesn't support generics yet, that's not an
 # option, but it'd greatly clarify the API by allowing us to classify matchers
@@ -921,6 +853,16 @@ def bind_variables(self):
   type_filter = None
 
 
+class ContextualMatcher(Matcher):
+  """A matcher which requires special understanding in context.
+
+  By default, contextual matchers are not allowed inside of a submatcher
+  attribute.
+  To allow one, specify, for instance,
+  ``submatcher_attrib(contextual=MyContextualMatcher)``.
+  """
+
+  pass
 
 
 def accumulating_matcher(f):
@@ -1026,6 +968,106 @@ class ParseError(Exception):
   """
 
 
+def coerce(value):  # Nobody uses coerce. pylint: disable=redefined-builtin
+  """Returns the 'intended' matcher given by `value`.
+
+  If `value` is already a matcher, then this is what is returned.
+
+  If `value` is anything else, then coerce returns `ImplicitEquals(value)`.
+
+  Args:
+    value: Either a Matcher, or a value to compare for equality.
+  """
+  if isinstance(value, Matcher):
+    return value
+  else:
+    return ImplicitEquals(value)
+
+
+def _coerce_list(values):
+  return [coerce(v) for v in values]
+
+
+# TODO(b/199577701): drop the **kwargs: Any in the *_attrib functions.
+
+_IS_SUBMATCHER_ATTRIB = __name__ + '._IS_SUBMATCHER_ATTRIB'
+_IS_SUBMATCHER_LIST_ATTRIB = __name__ + '._IS_SUBMATCHER_LIST_ATTRIB'
+
+
+def _submatcher_validator(old_validator, contextual):
+  def validator(o: object, attribute: attr.Attribute, m: Matcher):
+    if isinstance(m, ContextualMatcher) and not isinstance(m, contextual):
+      raise TypeError(
+          f'Cannot use a `{m}` in `{type(o).__name__}.{attribute.name}`.'
+      )
+    if old_validator is not None:
+      old_validator(o, attribute, m)
+
+  return validator
+
+
+def submatcher_attrib(
+    *args,
+    walk: bool = True,
+    contextual: type[ContextualMatcher]
+    | tuple[type[ContextualMatcher], ...] = (),
+    **kwargs: Any,
+):
+  """Creates an attr.ib that is marked as a submatcher.
+
+  This will cause the matcher to be automatically walked as part of the
+  computation of .bind_variables. Any submatcher that can introduce a binding
+  must be listed as a submatcher_attrib or submatcher_list_attrib.
+
+  Args:
+    *args: Forwarded to attr.ib.
+    walk: Whether or not to walk to accumulate .bind_variables.
+    contextual: The contextual matcher classes to allow, if any.
+    **kwargs: Forwarded to attr.ib.
+
+  Returns:
+    An attr.ib()
+  """
+  if walk:
+    kwargs.setdefault('metadata', {})[_IS_SUBMATCHER_ATTRIB] = True
+  kwargs.setdefault('converter', coerce)
+  kwargs['validator'] = _submatcher_validator(
+      kwargs.get('validator'), contextual
+  )
+  return attr.ib(*args, **kwargs)
+
+
+def submatcher_list_attrib(
+    *args,
+    walk: bool = True,
+    contextual: type[ContextualMatcher]
+    | tuple[type[ContextualMatcher], ...] = (),
+    **kwargs: Any,
+):
+  """Creates an attr.ib that is marked as an iterable of submatchers.
+
+  This will cause the matcher to be automatically walked as part of the
+  computation of .bind_variables. Any submatcher that can introduce a binding
+  must be listed as a submatcher_attrib or submatcher_list_attrib.
+
+  Args:
+    *args: Forwarded to attr.ib.
+    walk: Whether or not to walk to accumulate .bind_variables.
+    contextual: The contextual matcher classes to allow, if any.
+    **kwargs: Forwarded to attr.ib.
+
+  Returns:
+    An attr.ib()
+  """
+  if walk:
+    kwargs.setdefault('metadata', {})[_IS_SUBMATCHER_LIST_ATTRIB] = True
+  kwargs.setdefault('converter', _coerce_list)
+  kwargs['validator'] = attr.validators.deep_iterable(
+      _submatcher_validator(kwargs.get('validator'), contextual)
+  )
+  return attr.ib(*args, **kwargs)
+
+
 @attr.s(frozen=True)
 class _CompareById:
   """Wrapper object to compare things by identity."""

refex/python/matchers/ast_matchers.py

@@ -206,3 +206,33 @@ def _match(self, context, candidate):
                              (type(...),))
 
     type_filter = frozenset({ast.Constant})
+
+
+#####################################
+# High level AST matching overrides #
+#####################################
+
+# Firstly, a `*$...` is always equivalent to a `$...`.
+
+_old_starred = Starred  # pylint: disable=undefined-variable
+
+
+def Starred(**kw):  # pylint: disable=invalid-name
+  value = kw.get('value')
+  if isinstance(value, base_matchers.GlobStar):
+    return value
+  return _old_starred(**kw)
+
+
+# Similarly, a `**$...` is always morally-equivalent to a `$...=$...` in a call.
+# (But the latter isn't valid syntax atm, so this is the only way to spell it.)
+
+_old_keyword = keyword  # pylint: disable=undefined-variable
+
+
+def keyword(**kw):
+  value = kw.get('value')
+  arg = kw.get('arg')
+  if arg == base_matchers.Equals(None) and isinstance(value, base_matchers.GlobStar):
+    return value
+  return _old_keyword(**kw)

refex/python/matchers/base_matchers.py

@@ -125,15 +125,17 @@ def _match(self, context, candidate):
     raise TestOnlyRaisedError(self.message)
 
 
-@attr.s(init=False, frozen=True)
+@attr.s(frozen=True)
 class _NAryMatcher(matcher.Matcher):
   """Base class for matchers which take arbitrarily many submatchers in init."""
 
   _matchers = matcher.submatcher_list_attrib()
 
+
+# override __init__ to take *args
+class _NAryMatcher(_NAryMatcher):
   def __init__(self, *matchers):
-    super(_NAryMatcher, self).__init__()
-    self.__dict__['_matchers'] = matchers
+    super().__init__(matchers)
 
 
 @matcher.safe_to_eval
@@ -275,7 +277,6 @@ class Bind(matcher.Matcher):
       <refex.python.matcher.BindMerge>`, or None for the default strategy
       (``KEEP_LAST``).
   """
-  _NAME_REGEX = re.compile(r'\A(?!__)[a-zA-Z_]\w*\Z')
 
   name = attr.ib()
   _submatcher = matcher.submatcher_attrib(default=Anything())
@@ -284,7 +285,12 @@ class Bind(matcher.Matcher):
       validator=attr.validators.in_(frozenset(matcher.BindConflict) | {None}))
   _on_merge = attr.ib(
       default=None,
-      validator=attr.validators.in_(frozenset(matcher.BindMerge) | {None}))
+      validator=attr.validators.in_(frozenset(matcher.BindMerge) | {None}),
+  )
+
+  # Constants go after attrs-fields to work around static analysis tooling:
+  # b/301979723
+  _NAME_REGEX = re.compile(r'\A(?!__)[a-zA-Z_]\w*\Z')
 
   @name.validator
   def _name_validator(self, attribute, value):
@@ -781,12 +787,15 @@ def _match(self, context, candidate):
 # bindings -- you can't add a bound GlobStar() :(
 # @matcher.safe_to_eval
 @attr.s(frozen=True)
-class GlobStar(matcher.Matcher):
+class GlobStar(matcher.ContextualMatcher):
   """Matches any sequence of items in a sequence.
 
-  Only valid within :class:`Glob`.
+  Only valid within special matchers like :class:`Glob`.
   """
 
+  def __str__(self):
+    return '$...'
+
   def _match(self, context, candidate):
     del context, candidate  # unused
     # _match isn't called by GlobMatcher; it instead specially recognizes it
@@ -827,7 +836,7 @@ class Glob(matcher.Matcher):
   class:`GlobStar()` is only valid directly within the body of a `Glob`.
   """
 
-  _matchers = matcher.submatcher_list_attrib()
+  _matchers = matcher.submatcher_list_attrib(contextual=GlobStar)
 
   @cached_property.cached_property
   def _blocked_matchers(self):