Fix ANSI measurement / truncation / wrapping edge cases

lib/cli/ui.rb

@@ -346,7 +346,7 @@ def link(url, text, format: true, blue_underline: format)
 
         text = "{{blue:{{underline:#{text}}}}}" if blue_underline
         text = CLI::UI.fmt(text) if format
-        "\x1b]8;;#{url}\x1b\\#{text}\x1b]8;;\x1b\\"
+        ANSI.hyperlink(url, text)
       end
     end
 

lib/cli/ui/ansi.rb

@@ -1,6 +1,9 @@
 # typed: true
 # frozen_string_literal: true
 
+require 'strscan'
+require_relative 'ansi/terminal_width'
+
 module CLI
   module UI
     module ANSI
@@ -9,35 +12,82 @@ module ANSI
 
       ESC = "\x1b"
       # https://ghostty.org/docs/vt/concepts/sequences#csi-sequences
-      CSI_SEQUENCE = /\x1b\[[\d;:]+[\x20-\x2f]*?[\x40-\x7e]/
+      CSI_SEQUENCE = /\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]/
       # https://ghostty.org/docs/vt/concepts/sequences#osc-sequences
       # OSC sequences can be terminated with either ST (\x1b\x5c) or BEL (\x07)
       OSC_SEQUENCE = /\x1b\][^\x07\x1b]*?(?:\x07|\x1b\x5c)/
-
+      # An OSC 8 hyperlink: \x1b]8;params;URI, terminated like any OSC
+      # sequence. One with a URI opens a link, one without closes it.
+      # Anchored, to classify a whole sequence as yielded by each_token.
+      HYPERLINK = /\A\x1b\]8;[^;]*;(?<uri>.*)(?:\x07|\x1b\x5c)\z/m
+      HYPERLINK_END = "\x1b]8;;\x1b\x5c"
+      # Any whole control sequence, for walking a string as alternating
+      # sequence and text runs.
+      SEQUENCE = Regexp.union(CSI_SEQUENCE, OSC_SEQUENCE)
+      # A CSI or OSC introducer whose sequence runs to the end of the
+      # string without a terminator — usually one sliced open by an
+      # upstream cut. Treating it as a sequence keeps its bytes out of
+      # width measurements and truncation windows.
+      UNTERMINATED_SEQUENCE = /\x1b[\[\]][^\x1b]*\z/
+      TEXT_RUN = /[^\x1b]+/
       class << self
-        # ANSI escape sequences (like \x1b[31m) have zero width.
-        # when calculating the padding width, we must exclude them.
-        # This also implements a basic version of utf8 character width calculation like
-        # we could get for real from something like utf8proc.
+        # Yields str as alternating runs of :sequence (one whole CSI or OSC
+        # sequence) and :text (everything between them). Sequences never
+        # straddle tokens, so a consumer that measures or cuts only at
+        # token boundaries cannot slice one open. A CSI or OSC sequence
+        # left unterminated at the end of the string is yielded as one
+        # :sequence token; any other stray ESC is yielded as text.
+        #
+        #: (String str) ?{ (Symbol kind, String token) -> void } -> Enumerator[[Symbol, String]]?
+        def each_token(str, &block)
+          return to_enum(:each_token, str) unless block_given?
+
+          scanner = StringScanner.new(str)
+          until scanner.eos?
+            if (sequence = scanner.scan(SEQUENCE) || scanner.scan(UNTERMINATED_SEQUENCE))
+              yield(:sequence, sequence)
+            elsif (text = scanner.scan(TEXT_RUN))
+              yield(:text, text)
+            else
+              yield(:text, scanner.getch.to_s)
+            end
+          end
+        end
+
+        # The number of terminal columns str occupies when printed: control
+        # sequences take none, and each grapheme cluster (not codepoint:
+        # 👩‍💻 is one cluster) is measured by grapheme_width.
         #
         #: (String str) -> Integer
         def printing_width(str)
-          zwj = false #: bool
-          strip_codes(str).codepoints.reduce(0) do |acc, cp|
-            if zwj
-              zwj = false
-              next acc
-            end
-            case cp
-            when 0x200d # zero-width joiner
-              zwj = true
-              acc
-            when "\n"
-              acc
+          # ASCII fast paths. Every ASCII grapheme cluster is one character
+          # wide except \n and \r, which are zero, so counting stands in for
+          # the cluster walk; with no ESC there are no sequences to skip and
+          # the whole string can be counted without tokenizing.
+          if str.ascii_only? && !str.include?(ESC)
+            return str.length - str.count("\n\r")
+          end
+
+          width = 0 #: Integer
+          each_token(str) do |kind, token|
+            next unless kind == :text
+
+            if token.ascii_only?
+              width += token.length - token.count("\n\r")
             else
-              acc + 1
+              token.grapheme_clusters.each do |cluster|
+                width += grapheme_width(cluster)
+              end
             end
           end
+          width
+        end
+
+        # The number of terminal columns one grapheme cluster occupies.
+        #
+        #: (String cluster) -> Integer
+        def grapheme_width(cluster)
+          TerminalWidth.grapheme_width(cluster)
         end
 
         # Strips ANSI codes from a str
@@ -96,6 +146,13 @@ def sgr(params)
           control(params, 'm')
         end
 
+        # Renders text as an OSC 8 hyperlink to url
+        #
+        #: (String url, String text) -> String
+        def hyperlink(url, text)
+          "\x1b]8;;#{url}\x1b\x5c#{text}#{HYPERLINK_END}"
+        end
+
         # Cursor Movement
 
         # Move the cursor up n lines

lib/cli/ui/ansi/replay.rb

@@ -474,9 +474,26 @@ def escape(screen, scanner)
               when "\e" then next
               when "\x18", "\x1a" then return :ground
               when /[\x20-\x2f]/
-                # nF sequences: intermediate bytes, then one final byte.
-                scanner.skip(/[\x20-\x2f]*[\x30-\x7e]?/)
-                return :ground
+                return escape_intermediate(screen, scanner)
+              when SIMPLE_CONTROL then simple_control(screen, char)
+              else return :ground
+              end
+            end
+            :ground
+          end
+
+          # ESC intermediate state: collect through the final byte while
+          # executing embedded C0 controls and ignoring DEL. Bulk-skipping this
+          # tail would end the sequence at an embedded control, exposing its
+          # final byte as printable text.
+          #: (Screen screen, StringScanner scanner) -> Symbol
+          def escape_intermediate(screen, scanner)
+            until scanner.eos?
+              case (char = scanner.getch.to_s)
+              when /[\x30-\x7e]/ then return :ground
+              when /[\x20-\x2f]/ then next
+              when "\e" then return :escape
+              when "\x18", "\x1a" then return :ground
               when SIMPLE_CONTROL then simple_control(screen, char)
               else return :ground
               end
@@ -548,7 +565,10 @@ def apply(screen, params, intermediates, final)
             # tracks, except the alternate screen: a full-screen UI draws
             # there and a terminal discards it on exit, so it must not reach
             # the replayed scrollback either.
-            if params == '?1049'
+            if params.match?(/\A\?[\d;]*\z/)
+              modes = params.delete_prefix('?').split(';')
+              return unless modes.include?('1049')
+
               case final
               when 'h' then screen.enter_alternate
               when 'l' then screen.exit_alternate

lib/cli/ui/truncater.rb

@@ -5,74 +5,57 @@ module CLI
   module UI
     # Truncater truncates a string to a provided printable width.
     module Truncater
-      PARSE_ROOT = :root
-      PARSE_ANSI = :ansi
-      PARSE_ESC  = :esc
-      PARSE_ZWJ  = :zwj
-
-      ESC                 = 0x1b
-      LEFT_SQUARE_BRACKET = 0x5b
-      ZWJ                 = 0x200d # emojipedia.org/emoji-zwj-sequences
-      SEMICOLON           = 0x3b
-
-      # EMOJI_RANGE in particular is super inaccurate. This is best-effort.
-      # If you need this to be more accurate, we'll almost certainly accept a
-      # PR improving it.
-      EMOJI_RANGE    = 0x1f300..0x1f5ff
-      NUMERIC_RANGE  = 0x30..0x39
-      LC_ALPHA_RANGE = 0x40..0x5a
-      UC_ALPHA_RANGE = 0x60..0x71
-
       TRUNCATED = "\x1b[0m…"
 
       class << self
         #: (String text, Integer printing_width) -> String
         def call(text, printing_width)
-          return text if text.size <= printing_width
+          # Fast path. Only sound for ASCII, where no character is wider
+          # than a column: an emoji string can occupy up to twice as many
+          # columns as it has characters.
+          return text if text.ascii_only? && text.size <= printing_width
 
-          width            = 0
-          mode             = PARSE_ROOT
-          truncation_index = nil #: Integer?
+          width = 0 #: Integer
+          truncated = false #: bool
+          open_hyperlink = false #: bool
+          # Preserve the caller's encoding. Printer can deliberately pass
+          # ASCII-compatible strings in encodings other than UTF-8, and an
+          # empty UTF-8 buffer becomes incompatible after binary text has
+          # been appended to it.
+          prefix = String.new(encoding: text.encoding)
 
-          codepoints = text.codepoints
-          codepoints.each.with_index do |cp, index|
-            case mode
-            when PARSE_ROOT
-              case cp
-              when ESC # non-printable, followed by some more non-printables.
-                mode = PARSE_ESC
-              when ZWJ # non-printable, followed by another non-printable.
-                mode = PARSE_ZWJ
-              else
-                width += width(cp)
-                if width >= printing_width
-                  truncation_index ||= index
-                  # it looks like we could break here but we still want the
-                  # width calculation for the rest of the characters.
-                end
-              end
-            when PARSE_ESC
-              mode = case cp
-              when LEFT_SQUARE_BRACKET
-                PARSE_ANSI
-              else
-                PARSE_ROOT
+          ANSI.each_token(text) do |kind, token|
+            case kind
+            when :sequence
+              # Sequences occupy no columns. Any that fall past the cut are
+              # dropped: TRUNCATED resets SGR state itself, and an open
+              # hyperlink gets closed below.
+              next if truncated
+
+              prefix << token
+              if (match = ANSI::HYPERLINK.match(token))
+                open_hyperlink = !match[:uri].to_s.empty?
               end
-            when PARSE_ANSI
-              # ANSI escape codes preeeetty much have the format of:
-              # \x1b[0-9;]+[A-Za-z]
-              case cp
-              when NUMERIC_RANGE, SEMICOLON
-              when LC_ALPHA_RANGE, UC_ALPHA_RANGE
-                mode = PARSE_ROOT
-              else
-                # unexpected. let's just go back to the root state I guess?
-                mode = PARSE_ROOT
+            when :text
+              token.grapheme_clusters.each do |cluster|
+                # A line break is zero columns to printing_width, but a
+                # truncated string must stay one line: count it as a column
+                # so the cut lands before it, never absorbing it silently.
+                # Other zero-width clusters remain zero-width.
+                cluster_width = case cluster
+                when "\n", "\r", "\r\n"
+                  1
+                else
+                  ANSI.grapheme_width(cluster)
+                end
+                width += cluster_width
+                # We cut before the cluster that reaches printing_width,
+                # leaving one column for TRUNCATED's ellipsis, but keep
+                # measuring: if the rest of the string turns out not to
+                # exceed printing_width after all, no cut is needed.
+                truncated ||= width >= printing_width
+                prefix << cluster unless truncated
               end
-            when PARSE_ZWJ
-              # consume any character and consider it as having no width
-              # width(x+ZWJ+y) = width(x).
-              mode = PARSE_ROOT
             end
           end
 
@@ -81,22 +64,21 @@ def call(text, printing_width)
           # It's specifically for the case where we decided "Yes, this is the
           # point at which we'd have to add a truncation!" but it's actually
           # the end of the string.
-          return text if !truncation_index || width <= printing_width
+          return text if !truncated || width <= printing_width
 
-          slice = codepoints[0...truncation_index] #: as !nil
-          slice.pack('U*') + TRUNCATED
+          prefix << ANSI::HYPERLINK_END.encode(text.encoding) if open_hyperlink
+          prefix << truncation_marker(text.encoding)
         end
 
         private
 
-        #: (Integer printable_codepoint) -> Integer
-        def width(printable_codepoint)
-          case printable_codepoint
-          when EMOJI_RANGE
-            2
-          else
-            1
-          end
+        # Keep the reset and marker in the input encoding. Some
+        # ASCII-compatible encodings cannot represent U+2026; a one-column
+        # question mark preserves the width contract in that case.
+        #
+        #: (Encoding encoding) -> String
+        def truncation_marker(encoding)
+          TRUNCATED.encode(encoding, invalid: :replace, undef: :replace, replace: '?')
         end
       end
     end