|
| 1 | +# Copyright 2024 The kauldron Authors. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +"""Masks utils.""" |
| 16 | + |
| 17 | +from collections.abc import Callable, Sequence |
| 18 | +import re |
| 19 | +from typing import Any |
| 20 | + |
| 21 | +import jax |
| 22 | + |
| 23 | +_PyTree = Any |
| 24 | + |
| 25 | + |
| 26 | +# Improvements: |
| 27 | +# * Could add `exclude=` kwargs, similar to `glob()`. |
| 28 | + |
| 29 | + |
| 30 | +def select(pattern: str | Sequence[str]) -> Callable[[_PyTree], _PyTree]: |
| 31 | + r"""Create a mask which selects only the sub-pytree matching the pattern. |
| 32 | +
|
| 33 | + * `xx` will match all `{'xx': ...}` dict anywhere inside the tree. Note that |
| 34 | + the match is strict, so `xx` will NOT match `{'xxyy': }` |
| 35 | + * `xx.yy` will match `{'xx': {'yy': ...}}` dict |
| 36 | + * Regex are supported, when using regex, make sure to escape `.` (e.g. |
| 37 | + `xx\.yy[0-9]+`) |
| 38 | +
|
| 39 | + Example: |
| 40 | +
|
| 41 | + ```python |
| 42 | + mask_fn = kg.optim.select("lora") |
| 43 | +
|
| 44 | + mask_fn({ |
| 45 | + 'layer0': { |
| 46 | + 'lora': { |
| 47 | + 'a': jnp.zeros(), |
| 48 | + 'b': jnp.zeros(), |
| 49 | + }, |
| 50 | + 'weights': jnp.zeros(), |
| 51 | + 'bias': jnp.zeros(), |
| 52 | + } |
| 53 | + }) == { |
| 54 | + 'layer0': { |
| 55 | + 'lora': { |
| 56 | + 'a': True, |
| 57 | + 'b': True, |
| 58 | + }, |
| 59 | + 'weights': False, |
| 60 | + 'bias': False, |
| 61 | + } |
| 62 | + } |
| 63 | + ``` |
| 64 | +
|
| 65 | + Args: |
| 66 | + pattern: The pattern to include. Everything else will be `False`. |
| 67 | +
|
| 68 | + Returns: |
| 69 | + The optax mask factory. |
| 70 | + """ |
| 71 | + |
| 72 | + # Convert the pattern to a regex. |
| 73 | + if isinstance(pattern, str): |
| 74 | + pattern = [pattern] |
| 75 | + |
| 76 | + pattern_regexes = [_make_regex(p) for p in pattern] |
| 77 | + |
| 78 | + def _path_match_pattern(path: jax.tree_util.KeyPath) -> bool: |
| 79 | + path_str = ".".join(_jax_key_entry_to_str(p) for p in path) |
| 80 | + return any(bool(p.search(path_str)) for p in pattern_regexes) |
| 81 | + |
| 82 | + def _make_mask(tree: _PyTree) -> _PyTree: |
| 83 | + # TODO(epot): Replace by `jax.tree.flatten_with_path` once Colab is updated |
| 84 | + leaves_with_path, treedef = jax.tree_util.tree_flatten_with_path(tree) |
| 85 | + |
| 86 | + # Parse each leaves |
| 87 | + leaves = [] |
| 88 | + for path, _ in leaves_with_path: |
| 89 | + leaves.append(_path_match_pattern(path)) |
| 90 | + |
| 91 | + # Restore the tree structure. |
| 92 | + return jax.tree.unflatten(treedef, leaves) |
| 93 | + |
| 94 | + return _make_mask |
| 95 | + |
| 96 | + |
| 97 | +def exclude(pattern: str | Sequence[str]) -> Callable[[_PyTree], _PyTree]: |
| 98 | + """Create a mask which selects all nodes except the ones matching the pattern. |
| 99 | +
|
| 100 | + This is the inverse of `select()`. |
| 101 | +
|
| 102 | + Example: |
| 103 | +
|
| 104 | + ```python |
| 105 | + optax.masked( |
| 106 | + optax.set_to_zero(), |
| 107 | + kd.optim.exclude("lora"), # Only `lora` weights are trained. |
| 108 | + ) |
| 109 | + ``` |
| 110 | +
|
| 111 | + Args: |
| 112 | + pattern: The pattern to exclude. See `select()` for more details. |
| 113 | +
|
| 114 | + Returns: |
| 115 | + The optax mask factory. |
| 116 | + """ |
| 117 | + make_select_mask = select(pattern) |
| 118 | + |
| 119 | + def _make_mask(tree: _PyTree) -> _PyTree: |
| 120 | + # Invert the select mask. |
| 121 | + tree = make_select_mask(tree) |
| 122 | + return jax.tree.map(lambda x: not x, tree) |
| 123 | + |
| 124 | + return _make_mask |
| 125 | + |
| 126 | + |
| 127 | +_REGEX_SPECIAL_CHARS = set("()[]?+*^$|\\") |
| 128 | + |
| 129 | + |
| 130 | +def _make_regex(pattern: str) -> re.Pattern[str]: |
| 131 | + # Auto-detect regex and forward them as-is. |
| 132 | + if any(c in _REGEX_SPECIAL_CHARS for c in pattern): |
| 133 | + pass |
| 134 | + else: # Otherwise, escape special characters (`.`). |
| 135 | + pattern = re.escape(pattern) |
| 136 | + |
| 137 | + pattern = rf"(?:^|\.){pattern}(?:$|\.)" |
| 138 | + return re.compile(pattern) |
| 139 | + |
| 140 | + |
| 141 | +def _jax_key_entry_to_str( |
| 142 | + jax_key_entry: jax.tree_util.KeyEntry, |
| 143 | +) -> str: |
| 144 | + """Convert a JaxKeyEntry into a valid `kontext.Path` element.""" |
| 145 | + match jax_key_entry: |
| 146 | + case jax.tree_util.DictKey(key): |
| 147 | + return key |
| 148 | + case _: |
| 149 | + raise TypeError(f"Unknown key entry type {type(jax_key_entry)}") |
0 commit comments