From b5c70cded5ad1db53c7b33a32683145e3935c537 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Tue, 11 Aug 2026 14:41:13 -0400 Subject: [PATCH 01/13] Strip parameter-less and private CSI sequences CSI_SEQUENCE required at least one parameter character, so strip_codes passed \e[K, \e[m, and private-mode sequences like \e[?25l through as text, and printing_width counted their bytes as printed columns: printing_width("\e[?25lx\e[K") returned 10 for one visible character. Match the full CSI grammar instead: any parameter bytes (including the private-mode markers), then intermediate bytes, then one final byte. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/ansi.rb | 2 +- test/cli/ui/ansi_test.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/cli/ui/ansi.rb b/lib/cli/ui/ansi.rb index fbdcda13..07fec554 100644 --- a/lib/cli/ui/ansi.rb +++ b/lib/cli/ui/ansi.rb @@ -6,7 +6,7 @@ module UI module ANSI ESC = "\x1b" # https://ghostty.org/docs/vt/concepts/sequences#csi-sequences - CSI_SEQUENCE = /\x1b\[[\d;:]+[\x20-\x2f]*?[\x40-\x7e]/ + CSI_SEQUENCE = /\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]/ # https://ghostty.org/docs/vt/concepts/sequences#osc-sequences # OSC sequences can be terminated with either ST (\x1b\x5c) or BEL (\x07) OSC_SEQUENCE = /\x1b\][^\x07\x1b]*?(?:\x07|\x1b\x5c)/ diff --git a/test/cli/ui/ansi_test.rb b/test/cli/ui/ansi_test.rb index 86c82f19..0b2b5cf6 100644 --- a/test/cli/ui/ansi_test.rb +++ b/test/cli/ui/ansi_test.rb @@ -19,6 +19,20 @@ def test_printing_width assert_equal(4, ANSI.printing_width(UI.link('url', 'text'))) end + # CSI sequences aren't required to carry parameters (\e[K, \e[m), and + # private-mode sequences mark theirs with ? (\e[?25l). None of them + # print anything. + def test_printing_width_of_parameterless_and_private_sequences_is_zero + assert_equal(1, ANSI.printing_width("\e[?25lx\e[K")) + assert_equal(4, ANSI.printing_width("\e[mtest\e[0m")) + end + + def test_strip_codes_removes_parameterless_and_private_sequences + assert_equal('x', ANSI.strip_codes("\e[?25lx\e[K")) + assert_equal('shown', ANSI.strip_codes("#{ANSI.hide_cursor}shown#{ANSI.show_cursor}")) + assert_equal('saved', ANSI.strip_codes("#{ANSI.cursor_save}saved#{ANSI.cursor_restore}")) + end + def test_strip_codes_preserves_text_between_osc8_hyperlinks hyperlink = CLI::UI.link('https://example.com', 'text', format: false) input = "Before #{hyperlink} after" From 717b8ee9d0b2d3cde9357c61dce5f47c99954910 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Tue, 11 Aug 2026 16:48:06 -0400 Subject: [PATCH 02/13] Add ANSI.each_token and measure width by grapheme cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ANSI.each_token walks a string as alternating runs of whole control sequences (CSI or OSC) and the text between them, so consumers that measure or cut at token boundaries can no longer slice a sequence open or count its bytes as printable. Without a block it returns an Enumerator, per the each_* convention. (The unused-looking &block parameter stays: Sorbet's rbs-inline support rejects a declared block type without a matching parameter.) printing_width is rebuilt on top of it, fixing three bugs: - Newlines counted as one column (the old `when "\n"` branch compared a String against Integer codepoints, so it never matched); they now count as zero. - Emoji counted as one column while Truncater counted them as two; both now share ANSI.grapheme_width, which says two. - ZWJ sequences and combining marks were hand-rolled or miscounted; String#grapheme_clusters now groups them, so ๐Ÿ‘ฉโ€๐Ÿ’ป and e+U+0301 are each one cluster. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/ansi.rb | 75 ++++++++++++++++++++++++++++++---------- test/cli/ui/ansi_test.rb | 38 ++++++++++++++++++-- 2 files changed, 93 insertions(+), 20 deletions(-) diff --git a/lib/cli/ui/ansi.rb b/lib/cli/ui/ansi.rb index 07fec554..9759ff1d 100644 --- a/lib/cli/ui/ansi.rb +++ b/lib/cli/ui/ansi.rb @@ -1,6 +1,8 @@ # typed: true # frozen_string_literal: true +require 'strscan' + module CLI module UI module ANSI @@ -10,31 +12,68 @@ module ANSI # https://ghostty.org/docs/vt/concepts/sequences#osc-sequences # OSC sequences can be terminated with either ST (\x1b\x5c) or BEL (\x07) OSC_SEQUENCE = /\x1b\][^\x07\x1b]*?(?:\x07|\x1b\x5c)/ + # Any whole control sequence, for walking a string as alternating + # sequence and text runs. + SEQUENCE = Regexp.union(CSI_SEQUENCE, OSC_SEQUENCE) + TEXT_RUN = /[^\x1b]+/ + # EMOJI_RANGE is super inaccurate. This is best-effort. If you need + # this to be more accurate, we'll almost certainly accept a PR + # improving it. + EMOJI_RANGE = 0x1f300..0x1f5ff class << self - # ANSI escape sequences (like \x1b[31m) have zero width. - # when calculating the padding width, we must exclude them. - # This also implements a basic version of utf8 character width calculation like - # we could get for real from something like utf8proc. + # Yields str as alternating runs of :sequence (one whole CSI or OSC + # sequence) and :text (everything between them). Sequences never + # straddle tokens, so a consumer that measures or cuts only at + # token boundaries cannot slice one open. A stray ESC heading no + # well-formed sequence is yielded as text. + # + #: (String str) ?{ (Symbol kind, String token) -> void } -> Enumerator[[Symbol, String]]? + def each_token(str, &block) + return to_enum(:each_token, str) unless block_given? + + scanner = StringScanner.new(str) + until scanner.eos? + if (sequence = scanner.scan(SEQUENCE)) + yield(:sequence, sequence) + elsif (text = scanner.scan(TEXT_RUN)) + yield(:text, text) + else + yield(:text, scanner.getch.to_s) + end + end + end + + # The number of terminal columns str occupies when printed: control + # sequences take none, and each grapheme cluster (not codepoint: + # ๐Ÿ‘ฉโ€๐Ÿ’ป is one cluster) is measured by grapheme_width. # #: (String str) -> Integer def printing_width(str) - zwj = false #: bool - strip_codes(str).codepoints.reduce(0) do |acc, cp| - if zwj - zwj = false - next acc - end - case cp - when 0x200d # zero-width joiner - zwj = true - acc - when "\n" - acc - else - acc + 1 + width = 0 #: Integer + each_token(str) do |kind, token| + next unless kind == :text + + token.grapheme_clusters.each do |cluster| + width += grapheme_width(cluster) end end + width + end + + # The number of terminal columns one grapheme cluster occupies: + # none for a newline, two for emoji, one for everything else. Still + # a basic version of the width tables something like utf8proc would + # give us for real (wide CJK characters, for one, are counted 1). + # + #: (String cluster) -> Integer + def grapheme_width(cluster) + case cluster + when "\n", "\r", "\r\n" + 0 + else + EMOJI_RANGE.cover?(cluster.ord) ? 2 : 1 + end end # Strips ANSI codes from a str diff --git a/test/cli/ui/ansi_test.rb b/test/cli/ui/ansi_test.rb index 0b2b5cf6..a5fd1fa9 100644 --- a/test/cli/ui/ansi_test.rb +++ b/test/cli/ui/ansi_test.rb @@ -13,12 +13,46 @@ def test_printing_width assert_equal(4, ANSI.printing_width("\x1b[38;2;100;100;100mtest\x1b[0m")) assert_equal(0, ANSI.printing_width('')) - assert_equal(3, ANSI.printing_width('>๐Ÿ”ง<')) - assert_equal(1, ANSI.printing_width('๐Ÿ‘ฉโ€๐Ÿ’ป')) + # Emoji occupy two columns, matching what Truncater has always + # assumed. A ZWJ sequence is one grapheme cluster, so one emoji. + assert_equal(4, ANSI.printing_width('>๐Ÿ”ง<')) + assert_equal(2, ANSI.printing_width('๐Ÿ‘ฉโ€๐Ÿ’ป')) + + # Newlines and combining marks occupy no columns. + assert_equal(2, ANSI.printing_width("a\nb")) + assert_equal(1, ANSI.printing_width("e\u0301")) assert_equal(4, ANSI.printing_width(UI.link('url', 'text'))) end + def test_each_token_yields_whole_sequences_and_text + tokens = [] + ANSI.each_token("a\e[?25l\e]8;;https://x\e\\b") { |kind, token| tokens << [kind, token] } + assert_equal( + [ + [:text, 'a'], + [:sequence, "\e[?25l"], + [:sequence, "\e]8;;https://x\e\\"], + [:text, 'b'], + ], + tokens, + ) + end + + def test_each_token_without_a_block_returns_an_enumerator + enum = ANSI.each_token("a\e[31mb") + + assert_kind_of(Enumerator, enum) + assert_equal([[:text, 'a'], [:sequence, "\e[31m"], [:text, 'b']], enum.to_a) + assert_equal([], ANSI.each_token('').to_a) + end + + def test_each_token_yields_stray_escape_as_text + tokens = [] + ANSI.each_token("a\eb") { |kind, token| tokens << [kind, token] } + assert_equal([[:text, 'a'], [:text, "\e"], [:text, 'b']], tokens) + end + # CSI sequences aren't required to carry parameters (\e[K, \e[m), and # private-mode sequences mark theirs with ? (\e[?25l). None of them # print anything. From 44aefc19cefa9a29a5c65ad4e8afa89b4bfa7c47 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Tue, 11 Aug 2026 16:48:53 -0400 Subject: [PATCH 03/13] Rebuild Truncater on the shared ANSI grammar Truncater's hand-rolled codepoint state machine predated the fixes to CSI_SEQUENCE and knew even less of the grammar: its parameter-byte set had no `?`, so a private-mode sequence like \x1b[?25l was sliced after the `?` and its body counted as printable text, and OSC sequences weren't recognized at all, so truncating inside an OSC 8 hyperlink emitted a dangling, unterminated link. It now walks ANSI.each_token: sequences pass through whole and spend no width, text is measured and cut at grapheme-cluster boundaries (the ZWJ special-casing goes away with it), and if the cut lands inside an OSC 8 hyperlink the truncation suffix closes it before resetting SGR. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/truncater.rb | 107 +++++++++++----------------------- test/cli/ui/truncater_test.rb | 20 +++++++ 2 files changed, 53 insertions(+), 74 deletions(-) diff --git a/lib/cli/ui/truncater.rb b/lib/cli/ui/truncater.rb index f5c2332e..05634c9a 100644 --- a/lib/cli/ui/truncater.rb +++ b/lib/cli/ui/truncater.rb @@ -5,74 +5,45 @@ module CLI module UI # Truncater truncates a string to a provided printable width. module Truncater - PARSE_ROOT = :root - PARSE_ANSI = :ansi - PARSE_ESC = :esc - PARSE_ZWJ = :zwj - - ESC = 0x1b - LEFT_SQUARE_BRACKET = 0x5b - ZWJ = 0x200d # emojipedia.org/emoji-zwj-sequences - SEMICOLON = 0x3b - - # EMOJI_RANGE in particular is super inaccurate. This is best-effort. - # If you need this to be more accurate, we'll almost certainly accept a - # PR improving it. - EMOJI_RANGE = 0x1f300..0x1f5ff - NUMERIC_RANGE = 0x30..0x39 - LC_ALPHA_RANGE = 0x40..0x5a - UC_ALPHA_RANGE = 0x60..0x71 - TRUNCATED = "\x1b[0mโ€ฆ" + # An OSC 8 hyperlink sequence: \x1b]8;params;URI terminated by BEL or + # ST. One with a URI starts a link; one without ends it. + HYPERLINK = /\A\x1b\]8;[^;]*;(?.*)(?:\x07|\x1b\x5c)\z/m + HYPERLINK_END = "\x1b]8;;\x1b\x5c" + class << self #: (String text, Integer printing_width) -> String def call(text, printing_width) return text if text.size <= printing_width - width = 0 - mode = PARSE_ROOT - truncation_index = nil #: Integer? - - codepoints = text.codepoints - codepoints.each.with_index do |cp, index| - case mode - when PARSE_ROOT - case cp - when ESC # non-printable, followed by some more non-printables. - mode = PARSE_ESC - when ZWJ # non-printable, followed by another non-printable. - mode = PARSE_ZWJ - else - width += width(cp) - if width >= printing_width - truncation_index ||= index - # it looks like we could break here but we still want the - # width calculation for the rest of the characters. - end + width = 0 #: Integer + truncated = false #: bool + open_hyperlink = false #: bool + prefix = +'' + + ANSI.each_token(text) do |kind, token| + case kind + when :sequence + # Sequences occupy no columns. Any that fall past the cut are + # dropped: TRUNCATED resets SGR state itself, and an open + # hyperlink gets closed below. + next if truncated + + prefix << token + if (match = HYPERLINK.match(token)) + open_hyperlink = !match[:uri].to_s.empty? end - when PARSE_ESC - mode = case cp - when LEFT_SQUARE_BRACKET - PARSE_ANSI - else - PARSE_ROOT + when :text + token.grapheme_clusters.each do |cluster| + width += ANSI.grapheme_width(cluster) + # We cut before the cluster that reaches printing_width, + # leaving one column for TRUNCATED's ellipsis, but keep + # measuring: if the rest of the string turns out not to + # exceed printing_width after all, no cut is needed. + truncated ||= width >= printing_width + prefix << cluster unless truncated end - when PARSE_ANSI - # ANSI escape codes preeeetty much have the format of: - # \x1b[0-9;]+[A-Za-z] - case cp - when NUMERIC_RANGE, SEMICOLON - when LC_ALPHA_RANGE, UC_ALPHA_RANGE - mode = PARSE_ROOT - else - # unexpected. let's just go back to the root state I guess? - mode = PARSE_ROOT - end - when PARSE_ZWJ - # consume any character and consider it as having no width - # width(x+ZWJ+y) = width(x). - mode = PARSE_ROOT end end @@ -81,22 +52,10 @@ def call(text, printing_width) # It's specifically for the case where we decided "Yes, this is the # point at which we'd have to add a truncation!" but it's actually # the end of the string. - return text if !truncation_index || width <= printing_width - - slice = codepoints[0...truncation_index] #: as !nil - slice.pack('U*') + TRUNCATED - end + return text if !truncated || width <= printing_width - private - - #: (Integer printable_codepoint) -> Integer - def width(printable_codepoint) - case printable_codepoint - when EMOJI_RANGE - 2 - else - 1 - end + prefix << HYPERLINK_END if open_hyperlink + prefix << TRUNCATED end end end diff --git a/test/cli/ui/truncater_test.rb b/test/cli/ui/truncater_test.rb index b399face..00bc29f9 100644 --- a/test/cli/ui/truncater_test.rb +++ b/test/cli/ui/truncater_test.rb @@ -22,6 +22,26 @@ def test_truncate assert_example(3, 'AB' + MAN_COOKING, 'AB' + Truncater::TRUNCATED) end + def test_truncate_never_slices_a_sequence + # Private-mode (\x1b[?25l) and parameterless (\x1b[K) sequences pass + # through whole and spend no width; those past the cut are dropped. + assert_example(3, "\x1b[?25lfoobar\x1b[K", "\x1b[?25lfo" + Truncater::TRUNCATED) + end + + def test_truncate_closes_an_open_hyperlink + link = "\x1b]8;;https://example.com\x1b\\foobar\x1b]8;;\x1b\\" + assert_example( + 3, + link, + "\x1b]8;;https://example.com\x1b\\fo" + Truncater::HYPERLINK_END + Truncater::TRUNCATED, + ) + end + + def test_truncate_does_not_close_an_already_closed_hyperlink + input = "\x1b]8;;u\x1b\\a\x1b]8;;\x1b\\bcdef" + assert_example(3, input, "\x1b]8;;u\x1b\\a\x1b]8;;\x1b\\b" + Truncater::TRUNCATED) + end + private def assert_example(width, from, to) From 9a103ed7b3e7213d8a3ef71d23c0d2cf1b8b6230 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Tue, 11 Aug 2026 16:49:15 -0400 Subject: [PATCH 04/13] Rebuild Wrap on ANSI.each_token MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrap built its own lexer: an alternation of twenty lookbehind patterns (one per SGR parameter length, since lookbehinds can't quantify) spliced into a split regex. Walking ANSI.each_token replaces all of it, and any non-SGR sequence (cursor movement, OSC) now passes through as a unit instead of being split as text. It also fixes a dead branch: the reset case was written as the single-quoted literal '\x1B[0?m', which no token ever equals, so the SGR codes tracked for re-sending after each wrap were never cleared โ€” a reset color could come back on the next wrapped line. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/wrap.rb | 67 ++++++++++++++++++++++------------------ test/cli/ui/wrap_test.rb | 12 +++++++ 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/lib/cli/ui/wrap.rb b/lib/cli/ui/wrap.rb index 05ce0afd..3098db17 100644 --- a/lib/cli/ui/wrap.rb +++ b/lib/cli/ui/wrap.rb @@ -5,6 +5,9 @@ module CLI module UI class Wrap + SGR_RESET = /\A\x1b\[0?m\z/ + SGR = /\A\x1b\[[\d;]*m\z/ + #: (String input) -> void def initialize(input) @input = input @@ -14,43 +17,47 @@ def initialize(input) def wrap(total_width = Terminal.width) max_width = total_width - Frame.prefix_width width = 0 #: Integer - final = [] - # Create an alternation of format codes of parameter lengths 1-20, since + and {1,n} not allowed in lookbehind - format_codes = (1..20).map { |n| /\x1b\[[\d;]{#{n}}m/ }.join('|') - codes = '' - @input.split(/(?=\s|\x1b\[[\d;]+m|\r)|(?<=\s|#{format_codes})/).each do |token| - case token - when '\x1B[0?m' - codes = '' - final << token - when /\x1b\[[\d;]+m/ - codes += token # Track in use format codes so that they are resent after frame coloring + final = +'' + # SGR codes in effect, resent after each wrap so that frame coloring + # doesn't clobber them mid-paragraph. + codes = +'' + + ANSI.each_token(@input) do |kind, token| + if kind == :sequence + case token + when SGR_RESET + codes = +'' + when SGR + codes << token + end final << token - when "\n" - final << "\n#{codes}" - width = 0 - when /\s/ - token_width = ANSI.printing_width(token) - if width + token_width <= max_width - final << token - width += token_width - else - final << "\n#{codes}" + next + end + + # Split the text run so each whitespace character is its own + # token: lines break at whitespace, and a space that would sit in + # the last column becomes the break itself. + token.split(/(?=\s)|(?<=\s)/).each do |chunk| + if chunk == "\n" + final << "\n" << codes width = 0 + next end - else - token_width = ANSI.printing_width(token) - if width + token_width <= max_width - final << token - width += token_width + + chunk_width = ANSI.printing_width(chunk) + if width + chunk_width <= max_width + final << chunk + width += chunk_width + elsif chunk.match?(/\A\s\z/) + final << "\n" << codes + width = 0 else - final << "\n#{codes}" - final << token - width = token_width + final << "\n" << codes << chunk + width = chunk_width end end end - final.join + final end end end diff --git a/test/cli/ui/wrap_test.rb b/test/cli/ui/wrap_test.rb index 41050913..bfca71eb 100644 --- a/test/cli/ui/wrap_test.rb +++ b/test/cli/ui/wrap_test.rb @@ -14,6 +14,18 @@ def test_wrap Terminal.stubs(:width).returns(20) assert_equal(ex, w.wrap) end + + def test_wrap_resends_active_codes_after_a_break + wrapped = Wrap.new("\x1b[31maaaa bbbb cccc").wrap(9) + + assert_equal("\x1b[31maaaa bbbb\n\x1b[31mcccc", wrapped) + end + + def test_wrap_stops_resending_codes_after_a_reset + wrapped = Wrap.new("\x1b[31maaaa\x1b[0m bbbb cccc").wrap(9) + + assert_equal("\x1b[31maaaa\x1b[0m bbbb\ncccc", wrapped) + end end end end From 497ee88e1571208a580cf0ac075daa851a1464c4 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Tue, 11 Aug 2026 17:02:40 -0400 Subject: [PATCH 05/13] Move the OSC 8 hyperlink grammar into ANSI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLI::UI.link built hyperlinks while Truncater parsed and closed them, each with its own copy of the escape-sequence grammar โ€” the same split that motivated ANSI.each_token. Both halves now live in ANSI next to the other sequence definitions: HYPERLINK classifies a sequence and captures its URI, HYPERLINK_END closes a link, and ANSI.hyperlink builds one. Co-Authored-By: Claude Fable 5 --- lib/cli/ui.rb | 2 +- lib/cli/ui/ansi.rb | 12 ++++++++++++ test/cli/ui/ansi_test.rb | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/cli/ui.rb b/lib/cli/ui.rb index 3360d3b5..52e717e4 100644 --- a/lib/cli/ui.rb +++ b/lib/cli/ui.rb @@ -346,7 +346,7 @@ def link(url, text, format: true, blue_underline: format) text = "{{blue:{{underline:#{text}}}}}" if blue_underline text = CLI::UI.fmt(text) if format - "\x1b]8;;#{url}\x1b\\#{text}\x1b]8;;\x1b\\" + ANSI.hyperlink(url, text) end end diff --git a/lib/cli/ui/ansi.rb b/lib/cli/ui/ansi.rb index 9759ff1d..e238fc7e 100644 --- a/lib/cli/ui/ansi.rb +++ b/lib/cli/ui/ansi.rb @@ -12,6 +12,11 @@ module ANSI # https://ghostty.org/docs/vt/concepts/sequences#osc-sequences # OSC sequences can be terminated with either ST (\x1b\x5c) or BEL (\x07) OSC_SEQUENCE = /\x1b\][^\x07\x1b]*?(?:\x07|\x1b\x5c)/ + # An OSC 8 hyperlink: \x1b]8;params;URI, terminated like any OSC + # sequence. One with a URI opens a link, one without closes it. + # Anchored, to classify a whole sequence as yielded by each_token. + HYPERLINK = /\A\x1b\]8;[^;]*;(?.*)(?:\x07|\x1b\x5c)\z/m + HYPERLINK_END = "\x1b]8;;\x1b\x5c" # Any whole control sequence, for walking a string as alternating # sequence and text runs. SEQUENCE = Regexp.union(CSI_SEQUENCE, OSC_SEQUENCE) @@ -105,6 +110,13 @@ def sgr(params) control(params, 'm') end + # Renders text as an OSC 8 hyperlink to url + # + #: (String url, String text) -> String + def hyperlink(url, text) + "\x1b]8;;#{url}\x1b\x5c#{text}#{HYPERLINK_END}" + end + # Cursor Movement # Move the cursor up n lines diff --git a/test/cli/ui/ansi_test.rb b/test/cli/ui/ansi_test.rb index a5fd1fa9..c76d02f2 100644 --- a/test/cli/ui/ansi_test.rb +++ b/test/cli/ui/ansi_test.rb @@ -9,6 +9,10 @@ def test_sgr assert_equal("\x1b[1;34m", ANSI.sgr('1;34')) end + def test_hyperlink + assert_equal("\e]8;;https://example.com\e\\text\e]8;;\e\\", ANSI.hyperlink('https://example.com', 'text')) + end + def test_printing_width assert_equal(4, ANSI.printing_width("\x1b[38;2;100;100;100mtest\x1b[0m")) assert_equal(0, ANSI.printing_width('')) From dc5e1e6e16c05e024fe4b00c9b66893a86744b34 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Tue, 11 Aug 2026 17:03:05 -0400 Subject: [PATCH 06/13] Measure Truncater's fast path by columns, not characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The early return compared codepoint count against printing width, the one measurement in the file grapheme_width doesn't make. Character count only bounds column count for ASCII โ€” an emoji string occupies up to twice as many columns as it has characters, so Truncater.call("๐ŸŒˆ๐ŸŒˆ๐ŸŒˆ", 3) returned all six columns untouched. The fast path now applies only to ASCII strings. Also adopts ANSI's OSC 8 grammar in place of the local copy. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/truncater.rb | 14 ++++++-------- test/cli/ui/truncater_test.rb | 9 ++++++++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/lib/cli/ui/truncater.rb b/lib/cli/ui/truncater.rb index 05634c9a..76d9e568 100644 --- a/lib/cli/ui/truncater.rb +++ b/lib/cli/ui/truncater.rb @@ -7,15 +7,13 @@ module UI module Truncater TRUNCATED = "\x1b[0mโ€ฆ" - # An OSC 8 hyperlink sequence: \x1b]8;params;URI terminated by BEL or - # ST. One with a URI starts a link; one without ends it. - HYPERLINK = /\A\x1b\]8;[^;]*;(?.*)(?:\x07|\x1b\x5c)\z/m - HYPERLINK_END = "\x1b]8;;\x1b\x5c" - class << self #: (String text, Integer printing_width) -> String def call(text, printing_width) - return text if text.size <= printing_width + # Fast path. Only sound for ASCII, where no character is wider + # than a column: an emoji string can occupy up to twice as many + # columns as it has characters. + return text if text.ascii_only? && text.size <= printing_width width = 0 #: Integer truncated = false #: bool @@ -31,7 +29,7 @@ def call(text, printing_width) next if truncated prefix << token - if (match = HYPERLINK.match(token)) + if (match = ANSI::HYPERLINK.match(token)) open_hyperlink = !match[:uri].to_s.empty? end when :text @@ -54,7 +52,7 @@ def call(text, printing_width) # the end of the string. return text if !truncated || width <= printing_width - prefix << HYPERLINK_END if open_hyperlink + prefix << ANSI::HYPERLINK_END if open_hyperlink prefix << TRUNCATED end end diff --git a/test/cli/ui/truncater_test.rb b/test/cli/ui/truncater_test.rb index 00bc29f9..116ee110 100644 --- a/test/cli/ui/truncater_test.rb +++ b/test/cli/ui/truncater_test.rb @@ -28,12 +28,19 @@ def test_truncate_never_slices_a_sequence assert_example(3, "\x1b[?25lfoobar\x1b[K", "\x1b[?25lfo" + Truncater::TRUNCATED) end + def test_truncate_measures_by_column_not_character + # Each ๐ŸŒˆ is one character but two columns; a character-count + # shortcut would pass these through six columns wide. + assert_example(1, '๐Ÿ”ง', Truncater::TRUNCATED) + assert_example(3, '๐ŸŒˆ๐ŸŒˆ๐ŸŒˆ', '๐ŸŒˆ' + Truncater::TRUNCATED) + end + def test_truncate_closes_an_open_hyperlink link = "\x1b]8;;https://example.com\x1b\\foobar\x1b]8;;\x1b\\" assert_example( 3, link, - "\x1b]8;;https://example.com\x1b\\fo" + Truncater::HYPERLINK_END + Truncater::TRUNCATED, + "\x1b]8;;https://example.com\x1b\\fo" + ANSI::HYPERLINK_END + Truncater::TRUNCATED, ) end From be73d036185803270b3e75128cb409eede3e2346 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Tue, 11 Aug 2026 17:03:28 -0400 Subject: [PATCH 07/13] Carry hyperlinks and colon-form SGR across wraps Wrap resends the SGR codes in effect after each line break so frame coloring doesn't clobber them, but an OSC 8 hyperlink spanning a break was left open across the newline, putting the next line's frame gutter inside the link. Breaks now close an open hyperlink and reopen it after the resent codes, exactly as Truncater closes one at a cut. The SGR tracking also accepts the colon form of extended colors (\x1b[38:2::255:0:0m), which the previous [\d;] parameter set silently dropped from the resend list. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/wrap.rb | 28 ++++++++++++++++++++-------- test/cli/ui/wrap_test.rb | 14 ++++++++++++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/lib/cli/ui/wrap.rb b/lib/cli/ui/wrap.rb index 3098db17..ff8cd3a5 100644 --- a/lib/cli/ui/wrap.rb +++ b/lib/cli/ui/wrap.rb @@ -6,7 +6,9 @@ module CLI module UI class Wrap SGR_RESET = /\A\x1b\[0?m\z/ - SGR = /\A\x1b\[[\d;]*m\z/ + # SGR parameters are separated by ; or, in the underspecified-but-real + # colon form of extended colors (\x1b[38:2::255:0:0m), by :. + SGR = /\A\x1b\[[\d;:]*m\z/ #: (String input) -> void def initialize(input) @@ -18,9 +20,17 @@ def wrap(total_width = Terminal.width) max_width = total_width - Frame.prefix_width width = 0 #: Integer final = +'' - # SGR codes in effect, resent after each wrap so that frame coloring - # doesn't clobber them mid-paragraph. + # SGR codes in effect, resent after each line break so that frame + # coloring doesn't clobber them mid-paragraph. An open hyperlink + # likewise gets closed at the break and reopened after it, keeping + # the frame gutter outside the link. codes = +'' + open_hyperlink = nil #: String? + break_line = -> do + final << ANSI::HYPERLINK_END if open_hyperlink + final << "\n" << codes << open_hyperlink.to_s + width = 0 + end ANSI.each_token(@input) do |kind, token| if kind == :sequence @@ -29,6 +39,9 @@ def wrap(total_width = Terminal.width) codes = +'' when SGR codes << token + when ANSI::HYPERLINK + match = ANSI::HYPERLINK.match(token) #: as !nil + open_hyperlink = match[:uri].to_s.empty? ? nil : token end final << token next @@ -39,8 +52,7 @@ def wrap(total_width = Terminal.width) # the last column becomes the break itself. token.split(/(?=\s)|(?<=\s)/).each do |chunk| if chunk == "\n" - final << "\n" << codes - width = 0 + break_line.call next end @@ -49,10 +61,10 @@ def wrap(total_width = Terminal.width) final << chunk width += chunk_width elsif chunk.match?(/\A\s\z/) - final << "\n" << codes - width = 0 + break_line.call else - final << "\n" << codes << chunk + break_line.call + final << chunk width = chunk_width end end diff --git a/test/cli/ui/wrap_test.rb b/test/cli/ui/wrap_test.rb index bfca71eb..f8307c59 100644 --- a/test/cli/ui/wrap_test.rb +++ b/test/cli/ui/wrap_test.rb @@ -26,6 +26,20 @@ def test_wrap_stops_resending_codes_after_a_reset assert_equal("\x1b[31maaaa\x1b[0m bbbb\ncccc", wrapped) end + + def test_wrap_tracks_colon_form_sgr_codes + wrapped = Wrap.new("\e[38:2::255:0:0maaaa bbbb cccc").wrap(9) + + assert_equal("\e[38:2::255:0:0maaaa bbbb\n\e[38:2::255:0:0mcccc", wrapped) + end + + def test_wrap_reopens_a_hyperlink_after_a_break + open_link = "\e]8;;https://example.com\e\\" + close_link = ANSI::HYPERLINK_END + wrapped = Wrap.new("#{open_link}aaaa bbbb cccc#{close_link}").wrap(9) + + assert_equal("#{open_link}aaaa bbbb#{close_link}\n#{open_link}cccc#{close_link}", wrapped) + end end end end From 0f9ee3afe9c858dc0ca75f67053f45a0b61b7e58 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Wed, 12 Aug 2026 10:01:06 -0400 Subject: [PATCH 08/13] Fast-path ASCII text in printing_width MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every ASCII grapheme cluster is one character wide except \n and \r, which are zero, so ASCII text can be measured by character count instead of walking clusters โ€” and a string with no ESC at all needs no tokenizing either. Both paths fall through to the cluster walk the moment anything non-ASCII appears. Benchmarks (ยตs/op, Ruby 3.4, arm64 macOS) against main and the tokenizer as first written: main tokenizer fast path ascii, 1000 chars 46 206 0.4 sgr-heavy 32 147 28 emoji 18 35 35 Wrap paragraph @ 20 718 233 60 Co-Authored-By: Claude Fable 5 --- lib/cli/ui/ansi.rb | 16 ++++++++++++++-- test/cli/ui/ansi_test.rb | 3 +++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/cli/ui/ansi.rb b/lib/cli/ui/ansi.rb index e238fc7e..15b2a9a0 100644 --- a/lib/cli/ui/ansi.rb +++ b/lib/cli/ui/ansi.rb @@ -55,12 +55,24 @@ def each_token(str, &block) # #: (String str) -> Integer def printing_width(str) + # ASCII fast paths. Every ASCII grapheme cluster is one character + # wide except \n and \r, which are zero, so counting stands in for + # the cluster walk; with no ESC there are no sequences to skip and + # the whole string can be counted without tokenizing. + if str.ascii_only? && !str.include?(ESC) + return str.length - str.count("\n\r") + end + width = 0 #: Integer each_token(str) do |kind, token| next unless kind == :text - token.grapheme_clusters.each do |cluster| - width += grapheme_width(cluster) + if token.ascii_only? + width += token.length - token.count("\n\r") + else + token.grapheme_clusters.each do |cluster| + width += grapheme_width(cluster) + end end end width diff --git a/test/cli/ui/ansi_test.rb b/test/cli/ui/ansi_test.rb index c76d02f2..4ab27d93 100644 --- a/test/cli/ui/ansi_test.rb +++ b/test/cli/ui/ansi_test.rb @@ -26,6 +26,9 @@ def test_printing_width assert_equal(2, ANSI.printing_width("a\nb")) assert_equal(1, ANSI.printing_width("e\u0301")) + # Mixed sequences, emoji, and ASCII in one string. + assert_equal(5, ANSI.printing_width("\e[31m\u{1f527} ok\e[0m")) + assert_equal(4, ANSI.printing_width(UI.link('url', 'text'))) end From 6214df50bd843a108bf420645620986b61cf08de Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Wed, 12 Aug 2026 12:58:33 -0400 Subject: [PATCH 09/13] Count line breaks as one column when truncating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit printing_width counts a newline as zero columns, which is right for measurement but wrong for truncation: call("ab\ncd", 3) returned "ab\nโ€ฆ", embedding a line break in output whose contract is one line, where main cut to "abโ€ฆ". A spin-group title holding a newline would break its frame instead of being cut. Truncater now counts line-breaking clusters as one column, so the cut lands before the break, exactly where main put it. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/truncater.rb | 5 ++++- test/cli/ui/truncater_test.rb | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/cli/ui/truncater.rb b/lib/cli/ui/truncater.rb index 76d9e568..b6a45164 100644 --- a/lib/cli/ui/truncater.rb +++ b/lib/cli/ui/truncater.rb @@ -34,7 +34,10 @@ def call(text, printing_width) end when :text token.grapheme_clusters.each do |cluster| - width += ANSI.grapheme_width(cluster) + # A line break is zero columns to printing_width, but a + # truncated string must stay one line: count it as a column + # so the cut lands before it, never absorbing it silently. + width += [ANSI.grapheme_width(cluster), 1].max # We cut before the cluster that reaches printing_width, # leaving one column for TRUNCATED's ellipsis, but keep # measuring: if the rest of the string turns out not to diff --git a/test/cli/ui/truncater_test.rb b/test/cli/ui/truncater_test.rb index 116ee110..208220d0 100644 --- a/test/cli/ui/truncater_test.rb +++ b/test/cli/ui/truncater_test.rb @@ -35,6 +35,13 @@ def test_truncate_measures_by_column_not_character assert_example(3, '๐ŸŒˆ๐ŸŒˆ๐ŸŒˆ', '๐ŸŒˆ' + Truncater::TRUNCATED) end + def test_truncate_cuts_before_a_line_break + # printing_width counts a newline as zero columns, but a truncated + # string must stay one line: the cut lands before the break. + assert_example(3, "ab\ncd", 'ab' + Truncater::TRUNCATED) + assert_example(3, "๐ŸŒˆ\ncd", '๐ŸŒˆ' + Truncater::TRUNCATED) + end + def test_truncate_closes_an_open_hyperlink link = "\x1b]8;;https://example.com\x1b\\foobar\x1b]8;;\x1b\\" assert_example( From 11b2052ba329be9265ea3ca383a35d7b75e4d5ce Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Wed, 12 Aug 2026 12:58:34 -0400 Subject: [PATCH 10/13] Detect SGR resets hidden in parameter lists Wrap resends active SGR codes after each break, and stops when it sees a reset -- but only recognized \e[0m and \e[m. A reset riding a parameter list (\e[0;33m, \e[;1m) was treated as one more code to accumulate, so codes grew without bound and stale attributes were resent at every subsequent break. Parameters now reset at any 0 or empty entry: everything before the last reset dies there, and only the survivors are resent. Colon-form parameters (38:2::255:0:0) never reset; a 0 there is a subparameter. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/wrap.rb | 20 +++++++++++++++++++- test/cli/ui/wrap_test.rb | 15 +++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/cli/ui/wrap.rb b/lib/cli/ui/wrap.rb index ff8cd3a5..a2dcde1e 100644 --- a/lib/cli/ui/wrap.rb +++ b/lib/cli/ui/wrap.rb @@ -38,7 +38,7 @@ def wrap(total_width = Terminal.width) when SGR_RESET codes = +'' when SGR - codes << token + codes = track_sgr(token, codes) when ANSI::HYPERLINK match = ANSI::HYPERLINK.match(token) #: as !nil open_hyperlink = match[:uri].to_s.empty? ? nil : token @@ -71,6 +71,24 @@ def wrap(total_width = Terminal.width) end final end + + private + + # The SGR codes in effect after token applies to codes. A reset can + # hide mid-list: parameters reset at a 0 or an empty entry (\e[0;33m, + # \e[;1m), killing every code before it, so only the parameters after + # the last reset survive. Colon-form parameters (38:2::255:0:0) never + # reset: a 0 there is a subparameter, not a command. + # + #: (String token, String codes) -> String + def track_sgr(token, codes) + params = token[2...-1].to_s.split(';', -1) + last_reset = params.rindex { |param| param.match?(/\A0*\z/) } + return codes << token unless last_reset + + survivors = params[(last_reset + 1)..].to_a + survivors.empty? ? +'' : +"\e[#{survivors.join(";")}m" + end end end end diff --git a/test/cli/ui/wrap_test.rb b/test/cli/ui/wrap_test.rb index f8307c59..a31a74f4 100644 --- a/test/cli/ui/wrap_test.rb +++ b/test/cli/ui/wrap_test.rb @@ -27,6 +27,21 @@ def test_wrap_stops_resending_codes_after_a_reset assert_equal("\x1b[31maaaa\x1b[0m bbbb\ncccc", wrapped) end + def test_wrap_detects_a_reset_hidden_in_a_parameter_list + # \e[0;33m resets, then applies 33: earlier codes die at the reset + # and only the survivors are resent after a break. + wrapped = Wrap.new("\x1b[1m\x1b[0;33maaaa bbbb cccc").wrap(9) + + assert_equal("\x1b[1m\x1b[0;33maaaa bbbb\n\x1b[33mcccc", wrapped) + end + + def test_wrap_detects_an_empty_parameter_as_a_reset + # An empty SGR parameter (\e[;m) is a 0 to a terminal. + wrapped = Wrap.new("\x1b[31maaaa\x1b[;m bbbb cccc").wrap(9) + + assert_equal("\x1b[31maaaa\x1b[;m bbbb\ncccc", wrapped) + end + def test_wrap_tracks_colon_form_sgr_codes wrapped = Wrap.new("\e[38:2::255:0:0maaaa bbbb cccc").wrap(9) From 6306c3a25ca745f0a3b337f691031c998186e376 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Wed, 12 Aug 2026 13:05:38 -0400 Subject: [PATCH 11/13] Widen the wide-glyph table beyond the core emoji block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EMOJI_RANGE covered only U+1F300..U+1F5FF, so โœ…, โญ, ๐Ÿš€, ๐Ÿ˜€, ๐Ÿ›’, CJK text, and cli-ui's own Glyph::WARNING (โš ๏ธ, U+26A0 + VS16) all measured one column while rendering two, shifting frames and truncation by a column per glyph. Replace it with WIDE_RANGES โ€” the East Asian Wide and Fullwidth blocks, the emoji planes, and the scattered BMP emoji with default emoji presentation โ€” searched with bsearch, plus an explicit rule that a VS16 in the cluster means emoji presentation and two columns. wcwidth(3) counts VS16 forms as one column, but the terminals cli-ui targets render them two, so we side with the terminals. The VS16 check compares codepoints rather than substrings because Printer measures strings before re-encoding them, and include? against a UTF-8 needle raises on ISO-8859-1 input. Still an approximation of a real wcwidth table (ambiguous-width East Asian characters are counted 1); vendoring one remains the exact fix. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/ansi.rb | 103 +++++++++++++++++++++++++++++++++++---- test/cli/ui/ansi_test.rb | 24 +++++++++ 2 files changed, 118 insertions(+), 9 deletions(-) diff --git a/lib/cli/ui/ansi.rb b/lib/cli/ui/ansi.rb index 15b2a9a0..2bddf3aa 100644 --- a/lib/cli/ui/ansi.rb +++ b/lib/cli/ui/ansi.rb @@ -21,10 +21,78 @@ module ANSI # sequence and text runs. SEQUENCE = Regexp.union(CSI_SEQUENCE, OSC_SEQUENCE) TEXT_RUN = /[^\x1b]+/ - # EMOJI_RANGE is super inaccurate. This is best-effort. If you need - # this to be more accurate, we'll almost certainly accept a PR - # improving it. - EMOJI_RANGE = 0x1f300..0x1f5ff + # Codepoints whose glyphs occupy two terminal columns: the East Asian + # Wide and Fullwidth blocks, the emoji planes, and the scattered BMP + # emoji with default emoji presentation (โœ…, โญ, โŒ). Consolidated + # across unassigned gaps inside CJK blocks; sorted and non-overlapping + # for bsearch. Still an approximation of a real wcwidth table + # (ambiguous-width East Asian characters, for one, are counted 1); + # vendoring one would be the exact fix, and we'd almost certainly + # accept a PR doing it. + WIDE_RANGES = [ + 0x1100..0x115F, # Hangul jamo, leading consonants + 0x231A..0x231B, # watch, hourglass + 0x2329..0x232A, # angle brackets + 0x23E9..0x23EC, # media-control arrows + 0x23F0..0x23F0, # alarm clock + 0x23F3..0x23F3, # hourglass with sand + 0x25FD..0x25FE, # small squares + 0x2614..0x2615, # umbrella, hot beverage + 0x2648..0x2653, # zodiac + 0x267F..0x267F, # wheelchair + 0x2693..0x2693, # anchor + 0x26A1..0x26A1, # high voltage + 0x26AA..0x26AB, # circles + 0x26BD..0x26BE, # soccer, baseball + 0x26C4..0x26C5, # snowman, sun behind cloud + 0x26CE..0x26CE, # ophiuchus + 0x26D4..0x26D4, # no entry + 0x26EA..0x26EA, # church + 0x26F2..0x26F3, # fountain, golf flag + 0x26F5..0x26F5, # sailboat + 0x26FA..0x26FA, # tent + 0x26FD..0x26FD, # fuel pump + 0x2705..0x2705, # check mark button + 0x270A..0x270B, # raised fist, raised hand + 0x2728..0x2728, # sparkles + 0x274C..0x274C, # cross mark + 0x274E..0x274E, # cross mark button + 0x2753..0x2755, # question and exclamation ornaments + 0x2757..0x2757, # exclamation mark + 0x2795..0x2797, # plus, minus, divide + 0x27B0..0x27B0, # curly loop + 0x27BF..0x27BF, # double curly loop + 0x2B1B..0x2B1C, # large squares + 0x2B50..0x2B50, # star + 0x2B55..0x2B55, # hollow circle + 0x2E80..0x303E, # CJK radicals through CJK punctuation + 0x3041..0x33FF, # kana, Hangul compatibility, CJK compatibility + 0x3400..0x4DBF, # CJK extension A + 0x4E00..0x9FFF, # CJK unified ideographs + 0xA000..0xA4CF, # Yi + 0xA960..0xA97F, # Hangul jamo extended-A + 0xAC00..0xD7A3, # Hangul syllables + 0xF900..0xFAFF, # CJK compatibility ideographs + 0xFE10..0xFE19, # vertical forms + 0xFE30..0xFE6B, # CJK compatibility and small forms + 0xFF00..0xFF60, # fullwidth forms + 0xFFE0..0xFFE6, # fullwidth signs + 0x1F004..0x1F004, # mahjong red dragon + 0x1F0CF..0x1F0CF, # joker + 0x1F18E..0x1F18E, # AB button + 0x1F191..0x1F19A, # squared CL through VS + 0x1F1E6..0x1F1FF, # regional indicators: a flag pairs two into one wide cluster + 0x1F200..0x1F265, # enclosed ideographic supplement + 0x1F300..0x1F64F, # pictographs and emoticons + 0x1F680..0x1F6FF, # transport + 0x1F7E0..0x1F7FF, # colored shapes + 0x1F900..0x1F9FF, # supplemental pictographs + 0x1FA70..0x1FAFF, # extended pictographs + 0x20000..0x3FFFD, # CJK extensions B and beyond + ].freeze + # VARIATION SELECTOR 16 (U+FE0F) requests emoji presentation for a + # character that defaults to text (โš ๏ธ is U+26A0 + VS16). + VS16 = 0xFE0F class << self # Yields str as alternating runs of :sequence (one whole CSI or OSC @@ -78,10 +146,12 @@ def printing_width(str) width end - # The number of terminal columns one grapheme cluster occupies: - # none for a newline, two for emoji, one for everything else. Still - # a basic version of the width tables something like utf8proc would - # give us for real (wide CJK characters, for one, are counted 1). + # The number of terminal columns one grapheme cluster occupies: none + # for a line break, two when the base character is in WIDE_RANGES or + # the cluster requests emoji presentation with VS16, one for + # everything else. wcwidth(3) counts VS16 forms as 1 โ€” the base + # character's width โ€” but the terminals cli-ui targets render them + # two columns wide, so we side with the terminals. # #: (String cluster) -> Integer def grapheme_width(cluster) @@ -89,7 +159,22 @@ def grapheme_width(cluster) when "\n", "\r", "\r\n" 0 else - EMOJI_RANGE.cover?(cluster.ord) ? 2 : 1 + codepoint = cluster.ord + wide = WIDE_RANGES.bsearch do |range| + if codepoint < range.begin + -1 + elsif codepoint > range.end + 1 + else + 0 + end + end + return 2 if wide + + # Compare codepoints, not substrings: the cluster can arrive in + # a non-UTF-8 encoding (Printer measures before re-encoding), + # where include? against a UTF-8 needle raises. + cluster.length > 1 && cluster.each_codepoint.include?(VS16) ? 2 : 1 end end diff --git a/test/cli/ui/ansi_test.rb b/test/cli/ui/ansi_test.rb index 4ab27d93..c282067e 100644 --- a/test/cli/ui/ansi_test.rb +++ b/test/cli/ui/ansi_test.rb @@ -32,6 +32,30 @@ def test_printing_width assert_equal(4, ANSI.printing_width(UI.link('url', 'text'))) end + def test_printing_width_covers_wide_glyphs_beyond_the_core_emoji_block + # BMP emoji outside the U+1F300 block, SMP emoji beyond U+1F5FF, + # and CJK are all two columns wide. + assert_equal(2, ANSI.printing_width('โœ…')) + assert_equal(2, ANSI.printing_width('โญ')) + assert_equal(2, ANSI.printing_width('๐Ÿš€')) + assert_equal(2, ANSI.printing_width('๐Ÿ›’')) + assert_equal(2, ANSI.printing_width('๐Ÿ˜€')) + assert_equal(4, ANSI.printing_width('ๆผขๅญ—')) + + # VS16 asks for emoji presentation: U+26A0 alone is a narrow text + # glyph, but โš ๏ธ (U+26A0 + VS16) renders two columns wide. This is + # Glyph::WARNING's form. + assert_equal(1, ANSI.printing_width("\u{26a0}")) + assert_equal(2, ANSI.printing_width("\u{26a0}\u{fe0f}")) + + # A flag is two regional indicators forming one wide cluster. + assert_equal(2, ANSI.printing_width('๐Ÿ‡จ๐Ÿ‡ฆ')) + + # Narrow neighbours of wide ranges stay narrow. + assert_equal(1, ANSI.printing_width('โœ“')) + assert_equal(1, ANSI.printing_width('โญ‘')) + end + def test_each_token_yields_whole_sequences_and_text tokens = [] ANSI.each_token("a\e[?25l\e]8;;https://x\e\\b") { |kind, token| tokens << [kind, token] } From 7167537fdad54babfeca47937d2a2724af674326 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Wed, 12 Aug 2026 13:08:39 -0400 Subject: [PATCH 12/13] Tokenize a trailing unterminated sequence as a sequence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit each_token yielded a CSI or OSC sequence missing its terminator at the end of the string as text, so Truncater counted its bytes as columns and could slice it a second time: an unterminated hyperlink open cut at width 5 came back as \e]8;\e[0mโ€ฆ, mangled further rather than dropped. Strings like this reach cli-ui when something upstream โ€” a log pipeline, a byte-limited buffer โ€” has already cut a sequence open. Only a missing terminator at the end of the string is unambiguous, so the new UNTERMINATED_SEQUENCE alternative anchors there; an unterminated sequence mid-string still tokenizes as text, ESC first. Co-Authored-By: Claude Fable 5 --- lib/cli/ui/ansi.rb | 12 +++++++++--- test/cli/ui/ansi_test.rb | 15 +++++++++++++++ test/cli/ui/truncater_test.rb | 8 ++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/lib/cli/ui/ansi.rb b/lib/cli/ui/ansi.rb index 2bddf3aa..90228f37 100644 --- a/lib/cli/ui/ansi.rb +++ b/lib/cli/ui/ansi.rb @@ -20,6 +20,11 @@ module ANSI # Any whole control sequence, for walking a string as alternating # sequence and text runs. SEQUENCE = Regexp.union(CSI_SEQUENCE, OSC_SEQUENCE) + # A CSI or OSC introducer whose sequence runs to the end of the + # string without a terminator โ€” usually one sliced open by an + # upstream cut. Treating it as a sequence keeps its bytes out of + # width measurements and truncation windows. + UNTERMINATED_SEQUENCE = /\x1b[\[\]][^\x1b]*\z/ TEXT_RUN = /[^\x1b]+/ # Codepoints whose glyphs occupy two terminal columns: the East Asian # Wide and Fullwidth blocks, the emoji planes, and the scattered BMP @@ -98,8 +103,9 @@ class << self # Yields str as alternating runs of :sequence (one whole CSI or OSC # sequence) and :text (everything between them). Sequences never # straddle tokens, so a consumer that measures or cuts only at - # token boundaries cannot slice one open. A stray ESC heading no - # well-formed sequence is yielded as text. + # token boundaries cannot slice one open. A CSI or OSC sequence + # left unterminated at the end of the string is yielded as one + # :sequence token; any other stray ESC is yielded as text. # #: (String str) ?{ (Symbol kind, String token) -> void } -> Enumerator[[Symbol, String]]? def each_token(str, &block) @@ -107,7 +113,7 @@ def each_token(str, &block) scanner = StringScanner.new(str) until scanner.eos? - if (sequence = scanner.scan(SEQUENCE)) + if (sequence = scanner.scan(SEQUENCE) || scanner.scan(UNTERMINATED_SEQUENCE)) yield(:sequence, sequence) elsif (text = scanner.scan(TEXT_RUN)) yield(:text, text) diff --git a/test/cli/ui/ansi_test.rb b/test/cli/ui/ansi_test.rb index c282067e..039ce01c 100644 --- a/test/cli/ui/ansi_test.rb +++ b/test/cli/ui/ansi_test.rb @@ -84,6 +84,21 @@ def test_each_token_yields_stray_escape_as_text assert_equal([[:text, 'a'], [:text, "\e"], [:text, 'b']], tokens) end + def test_each_token_yields_a_trailing_unterminated_sequence_whole + # A sequence sliced open by an upstream cut runs to the end of the + # string with no terminator. It stays one zero-width token instead + # of being counted (and sliced again) as text. + assert_equal([[:text, 'a'], [:sequence, "\e[31"]], ANSI.each_token("a\e[31").to_a) + assert_equal([[:sequence, "\e]8;;http://x"]], ANSI.each_token("\e]8;;http://x").to_a) + + # Mid-string, an unterminated sequence is still text: only at the + # end of the string is a missing terminator unambiguous. + assert_equal( + [[:text, "\e"], [:text, '[31'], [:sequence, "\e[0m"]], + ANSI.each_token("\e[31\e[0m").to_a, + ) + end + # CSI sequences aren't required to carry parameters (\e[K, \e[m), and # private-mode sequences mark theirs with ? (\e[?25l). None of them # print anything. diff --git a/test/cli/ui/truncater_test.rb b/test/cli/ui/truncater_test.rb index 208220d0..b3c8d039 100644 --- a/test/cli/ui/truncater_test.rb +++ b/test/cli/ui/truncater_test.rb @@ -28,6 +28,14 @@ def test_truncate_never_slices_a_sequence assert_example(3, "\x1b[?25lfoobar\x1b[K", "\x1b[?25lfo" + Truncater::TRUNCATED) end + def test_truncate_treats_a_trailing_unterminated_sequence_as_a_sequence + # A sequence already sliced open (by an upstream cut, say) spends + # no width: past the cut it drops, and alone it passes unchanged. + assert_example(3, "foobar\x1b]8;;http://x", 'fo' + Truncater::TRUNCATED) + input = "\x1b]8;;http://x no-terminator" + assert_example(5, input, input) + end + def test_truncate_measures_by_column_not_character # Each ๐ŸŒˆ is one character but two columns; a character-count # shortcut would pass these through six columns wide. From b4ff7ba410c2865682a14927312e1f33b40e9d80 Mon Sep 17 00:00:00 2001 From: Gord Pearson Date: Wed, 12 Aug 2026 13:13:07 -0400 Subject: [PATCH 13/13] Lock wide-glyph widths into layout with an integration test The width table's unit tests measure printing_width against itself; if a glyph's width regressed, the measurement asserting the layout would regress with it. These tests pin frame padding, table column alignment, and spin-group truncation as exact rendered strings, so a wide glyph measured one column short shifts the expected output. Co-Authored-By: Claude Fable 5 --- test/cli/ui/wide_glyph_layout_test.rb | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 test/cli/ui/wide_glyph_layout_test.rb diff --git a/test/cli/ui/wide_glyph_layout_test.rb b/test/cli/ui/wide_glyph_layout_test.rb new file mode 100644 index 00000000..5b6f9f8b --- /dev/null +++ b/test/cli/ui/wide_glyph_layout_test.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'test_helper' + +module CLI + module UI + # Locks ANSI's width table into real layout: a wide glyph measured one + # column short would shift every character after it in these strings. + # Expectations are spelled out as exact output rather than measured + # with printing_width, which is the very thing under test. Color and + # cursor movement are disabled so layout arrives as plain text instead + # of repaints. + class WideGlyphLayoutTest < Minitest::Test + def setup + CLI::UI.enable_color = false + CLI::UI.enable_cursor = false + super + end + + def teardown + CLI::UI.enable_color = true + CLI::UI.enable_cursor = true + super + end + + def test_frame_pads_an_emoji_title_like_a_plain_one + Terminal.stubs(:width).returns(20) + + with_emoji = capture_io { Frame.open('๐Ÿš€ go', timing: false) {} }.first.lines.first.chomp + plain = capture_io { Frame.open('ab go', timing: false) {} }.first.lines.first.chomp + + assert_equal('โ”โ”โ” ๐Ÿš€ go โ”โ”โ”โ”โ”โ”โ”โ”โ”', with_emoji) + # ๐Ÿš€ spans two columns, like 'ab': the rules must line up. + assert_equal('โ”โ”โ” ab go โ”โ”โ”โ”โ”โ”โ”โ”โ”', plain) + end + + def test_table_pads_emoji_cells_by_column + rows = Table.capture_table([['โœ… pass', 'ok'], ['status', 'ok']]) + + assert_equal(['โœ… pass ok', 'status ok'], rows) + end + + def test_spin_group_truncates_a_vs16_glyph_title_by_column + Terminal.stubs(:width).returns(12) + + out, _ = capture_io do + StdoutRouter.ensure_activated + sg = Spinner::SpinGroup.new + sg.add('โš ๏ธ wide glyph title') { true } + sg.wait + end + + # โš ๏ธ (U+26A0 + VS16) takes two columns, so twelve fill at the g. + assert_equal("โœ“ โš ๏ธ wide g\e[0mโ€ฆ", out.lines.last.chomp) + end + end + end +end