Merge pull request #191 from FragileTech/utf8

Make enforcing utf8 output optional

.github/workflows/push.yml

@@ -30,7 +30,7 @@ jobs:
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
-        python-version: "3.8"
+        python-version: "3.10"
     - name: Install lint dependencies
       run: |
         python -m pip install --upgrade pip
@@ -44,7 +44,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.6', '3.7', '3.8', '3.9']
+        python-version: ['3.10']
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}

Makefile

@@ -8,12 +8,12 @@ VERSION ?= latest
 style:
 	black .
 	isort .
+	ruff check --fix-only .
 
 .POSIX:
 check:
-	!(grep -R /tmp ${PROJECT}/tests)
-	flakehell lint ${PROJECT}
-	pylint ${PROJECT}
+	!(grep -R /tmp tests)
+	ruff check ${PROJECT}
 	black --check ${PROJECT}
 
 .PHONY: test

flogging/flogging.py

@@ -1,5 +1,4 @@
-"""
-Structured logging.
+"""Structured logging.
 
 This module defines functions to setup the standard Python logging subsystem to do one of the
 two things:
@@ -18,6 +17,7 @@
 import argparse
 import base64
 import codecs
+from collections.abc import Callable
 import datetime
 import functools
 import io
@@ -30,7 +30,6 @@
 import sys
 import threading
 import traceback
-from typing import Callable, Optional, Tuple, Union
 import uuid
 
 import xxhash
@@ -62,7 +61,7 @@ def get_datetime_now() -> datetime:
     return datetime.datetime.now(datetime.timezone.utc)
 
 
-def get_timezone() -> Tuple[datetime.tzinfo, str]:
+def get_timezone() -> tuple[datetime.tzinfo, str]:
     """Discover the current time zone and it's standard string representation."""
     dt = get_datetime_now().astimezone()
     tzstr = dt.strftime("%z")
@@ -103,7 +102,7 @@ def reduce_thread_id(thread_id: int) -> str:
 
 def repr_array(arr: ndarray) -> str:
     """repr() with shape."""
-    return "array(shape=%r, %s" % (arr.shape, _default_array_repr(arr).split("(", 1)[1])
+    return f"array(shape={arr.shape.__repr__()}, {_default_array_repr(arr).split('(', 1)[1]}"  # noqa: E501
 
 
 def with_logger(cls):
@@ -116,8 +115,7 @@ def with_logger(cls):
 
 
 def check_trailing_dot(func: Callable) -> Callable:
-    """
-    Decorate a function to check if the log message ends with a dot.
+    """Decorate a function to check if the log message ends with a dot.
 
     AssertionError is raised if so.
     """
@@ -129,10 +127,11 @@ def decorated_with_check_trailing_dot(*args):
         if record.name not in trailing_dot_exceptions:
             msg = record.msg
             if isinstance(msg, str) and msg.endswith(".") and not msg.endswith(".."):
-                raise AssertionError(
-                    'Log message is not allowed to have a trailing dot: %s: "%s"'
-                    % (record.name, msg),
+                err_msg = (
+                    f"Log message is not allowed to have a trailing dot:"
+                    f' {record.name}: "{msg}"'
                 )
+                raise AssertionError(err_msg)
         args = list(args)
         args[-1] = record
         return func(*args)
@@ -141,6 +140,7 @@ def decorated_with_check_trailing_dot(*args):
 
 
 class AwesomeFormatter(logging.Formatter):
+
     """logging.Formatter which adds colors to messages and shortens thread ids."""
 
     GREEN_MARKERS = [
@@ -156,7 +156,7 @@ class AwesomeFormatter(logging.Formatter):
     ]
     GREEN_RE = re.compile("(?<![_a-zA-Z0-9])(%s)(?![_a-zA-Z0-9])" % "|".join(GREEN_MARKERS))
 
-    def formatMessage(self, record: logging.LogRecord) -> str:
+    def formatMessage(self, record: logging.LogRecord) -> str:  # noqa: N802
         """Convert the already filled log record to a string."""
         level_color = "0"
         text_color = "0"
@@ -186,10 +186,11 @@ def formatMessage(self, record: logging.LogRecord) -> str:
 
 
 class StructuredHandler(logging.Handler):
+
     """logging handler for structured logging."""
 
     def __init__(
-        self, level=logging.NOTSET, level_from_msg: Optional[Callable[[str], Optional[str]]] = None
+        self, level=logging.NOTSET, level_from_msg: Callable[[str], str | None] | None = None
     ):
         """Initialize a new StructuredHandler."""
         super().__init__(level)
@@ -238,13 +239,13 @@ def flush(self):
 
 
 def setup(
-    level: Optional[Union[str, int]] = os.environ.get("LOG_LEVEL", "INFO"),  # noqa: B008
-    structured: bool = os.getenv("LOG_STRUCTURED", False),  # noqa: B008
+    level: str | int | None = os.environ.get("LOG_LEVEL", "INFO"),
+    structured: bool = os.getenv("LOG_STRUCTURED", False),  # noqa: PLW1508, FBT003
     allow_trailing_dot: bool = False,
-    level_from_msg: Optional[Callable[[str], Optional[str]]] = None,
+    level_from_msg: Callable[[str], str | None] | None = None,
+    ensure_utf8_streams: bool = True,
 ) -> None:
-    """
-    Make stdout and stderr unicode friendly in case of configured \
+    """Make stdout and stderr unicode friendly in case of configured \
     environments, initializes the logging, structured logging and \
     enables colored logs if it is appropriate.
 
@@ -254,6 +255,7 @@ def setup(
                                when a logging message ends with a dot.
     :param level_from_msg: Customize the logging level depending on the formatted message. \
                            Returning None means no change of the level.
+    :param ensure_utf8_streams: Ensure that stdout and stderr are utf-8 streams.
     :return: Nothing.
     """
     global logs_are_structured
@@ -268,13 +270,14 @@ def ensure_utf8_stream(stream):
             stream.encoding = "utf-8"
         return stream
 
-    sys.stdout, sys.stderr = (ensure_utf8_stream(s) for s in (sys.stdout, sys.stderr))
+    if ensure_utf8_streams:
+        sys.stdout, sys.stderr = (ensure_utf8_stream(s) for s in (sys.stdout, sys.stderr))
     np_set_string_function(repr_array)
 
     # basicConfig is only called to make sure there is at least one handler for the root logger.
     # All the output level setting is down right afterwards.
     logging.basicConfig()
-    logging.captureWarnings(True)
+    logging.captureWarnings(capture=True)
     for key, val in os.environ.items():
         if key.startswith("flog_"):
             domain = key[len("flog_") :]
@@ -314,10 +317,9 @@ def add_logging_args(
     patch: bool = True,
     erase_args: bool = True,
     allow_trailing_dot: bool = False,
-    level_from_msg: Optional[Callable[[str], Optional[str]]] = None,
+    level_from_msg: Callable[[str], str | None] | None = None,
 ) -> None:
-    """
-    Add command line flags specific to logging.
+    """Add command line flags specific to logging.
 
     :param parser: `argparse` parser where to add new flags.
     :param erase_args: Automatically remove logging-related flags from parsed args.
@@ -361,8 +363,7 @@ def _patched_parse_args(args=None, namespace=None) -> argparse.Namespace:
 
 
 def log_multipart(log: logging.Logger, data: bytes) -> str:
-    """
-    Log something big enough to be compressed and split in pieces.
+    """Log something big enough to be compressed and split in pieces.
 
     :return: Log record ID that we can later search for.
     """

pyproject.toml

@@ -1,12 +1,71 @@
-# NOTE: you have to use single-quoted strings in TOML for regular expressions.
-# It's the equivalent of r-strings in Python.  Multiline strings are treated as
-# verbose regular expressions by Black.  Use [ ] to denote a significant space
-# character.
+[tool.ruff]
+include = ["*.py", "*.pyi", "**/pyproject.toml"]#, "*.ipynb"]
+select = [
+    "ARG", "C4", "D", "E", "EM", "F", "FBT",
+    "FLY", "FIX", "FURB", "N", "NPY",
+    "INP", "ISC", "PERF", "PIE", "PL",
+    "PTH", "RET", "RUF", "S", "T10",
+    "TD", "T20", "UP", "YTT", "W",
+]
+ignore = ["D100", "D211", "D213", "D104", "D301", "D407", "FBT001", "FBT002", "PLR0913", "N801",
+    # From here on are disabled rules to match the current project style.
+    "D101", "D103", "ARG001", "PLR2004", "RUF001", "PLW0603", "D205", "RUF012"]
 
-# Example configuration for Black.
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = ["I"]
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".idea",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "output",
+    "venv",
+    ".pytest_cache",
+    " **/.ipynb_checkpoints/**",
+    "__main__.py",
+]
+
+# Same as Black.
+line-length = 99
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+# Assume Python 3.10
+target-version = "py310"
+
+[tool.ruff.flake8-quotes]
+docstring-quotes = "double"
+
+[tool.ruff.per-file-ignores]
+"__init__.py" = ["E402", "F401"]
+"**/docs/**" = ["INP001", "PTH100"]
+"**/{tests,docs,tools}/*" = ["E402", "F401", "F811", "D", "S101"]
+
+# black is the tool to format the source code
 [tool.black]
 line-length = 99
-target-version = ['py36', 'py37', 'py38']
+target-version = ['py310']
 include = '\.pyi?$'
 exclude = '''
 /(
@@ -20,9 +79,10 @@ exclude = '''
   | buck-out
   | build
   | dist
+  | venv
 )/
 '''
-# Isort configuration to manage imports
+# isort orders and lints imports
 [tool.isort]
 profile = "black"
 line_length = 99
@@ -36,50 +96,6 @@ color_output = true
 lines_after_imports = 2
 honor_noqa = true
 
-# Code coverage config
-[tool.coverage.run]
-branch = true
-source = ["flogging"]
-
-[tool.coverage.report]
-exclude_lines =["no cover",
-    'raise NotImplementedError',
-    'if __name__ == "__main__":']
-ignore_errors = true
-omit = ["flogging/tests/*"]
-
-# Flakehell config
-[tool.flakehell]
-# optionally inherit from remote config (or local if you want)
-base = "https://raw.githubusercontent.com/life4/flakehell/master/pyproject.toml"
-# specify any flake8 options. For example, exclude "example.py":
-exclude = [".git", "docs", ".ipynb*", "*.ipynb", ".pytest_cache"]
-format = "grouped" # make output nice
-max_line_length = 99 # show line of source code in output
-show_source = true
-inline_quotes='"'
-import_order_style = "appnexus"
-application_package_names = ["flogging"]
-application_import_names = ["flogging"]
-
-[tool.flakehell.plugins]
-'flake8*' = ["+*", "-D*"] # disable docs by default
-pylint = ["+*", "-D*"]
-pyflakes = ["+*"]
-pycodestyle = ["+*" ,"-B008","-B301","-C815","-C816","-C812","-D100",
-    "-D105","-D200","-D202","-D301","-D402","-E121","-E123","-E126","-E203","-E226",
-    "-E24","-E704","-F821","-W503","-W504"]
-
-[tool.flakehell.exceptions."**/__init__.py"]
-"flake8*" = ["-D*"]
-pylint = ["-D*"]
-pyflakes = ["-F401"]
-
-# match by prefix
-[tool.flakehell.exceptions."**/tests/*"]
-pycodestyle = ["-F401", "-F811"] # disable a check
-pyflakes = ["-*"]           # disable a plugin
-
 [tool.pylint.master]
 ignore = 'tests'
 load-plugins =' pylint.extensions.docparams'

requirements-lint.txt

@@ -1,10 +1,35 @@
-black ==21.4b0
-flake8==3.8.4
-flake8-bugbear==20.11.1
-flake8-docstrings==1.6.0
-flake8-import-order==0.18.1
-flake8-quotes==3.2.0
-isort==5.8.0
-pylint==2.7.0
-pydocstyle==6.0.0
-flakehell==0.9.0
+black ==23.9.1
+flake8==6.1.0
+flake8-2020==1.8.1
+flake8-bandit==4.1.1
+flake8-boolean-trap==1.0.1
+flake8-bugbear==23.7.10
+flake8-commas==2.1.0
+flake8-comprehensions==3.14.0
+flake8-debugger==4.1.2
+flake8-docstrings==1.7.0
+flake8-errmsg==0.4.0
+flake8-fixme==1.1.1
+flake8-import-order==0.18.2
+flake8-implicit-str-concat==0.4.0
+flake8-no-pep420==2.7.0
+flake8-pie==0.16.0
+flake8-print==5.0.0
+flake8-pytest-style==1.7.2
+flake8-quotes==3.3.2
+flake8-return==1.2.0
+flake8-todos==0.3.0
+flake8-unused-arguments==0.0.13
+flake8-use-pathlib==0.3.0
+flynt==1.0.1
+isort==5.12.0
+pep8-naming==0.13.3
+perflint==0.7.3
+pre-commit==3.4.0
+pylint==2.17.5
+pydocstyle==6.3.0
+pycodestyle==2.11.0
+pyupgrade==3.10.1
+refurb==1.21.0
+ruff==0.0.289
+tryceratops==2.3.2

requirements-test.txt

@@ -1,3 +1,3 @@
-pytest==6.2.3
-pytest-cov==2.11.1
-tomli==1.2.3
+pytest==7.4.2
+pytest-cov==4.1.0
+tomli==2.0.1

requirements.txt

@@ -1,2 +1,2 @@
-numpy==1.19.5
-xxhash==2.0.2
+numpy==1.26.0
+xxhash==3.4.1