WIP: improve colr emoji detection and testing - #620
Conversation
288fa99 to
d7b53ec
Compare
|
I tried using Ragel/Colm to generate Rust code, but the generated code cannot be used directly. The quality is much worse than C. It has not been updated for a long time. |
|
The hand-written C port (https://github.com/chansen/c-emoji) referenced in google/emoji-segmenter#17 may also be worth looking at. |
fundon
changed the title
Can you elaborate please? We use the ragel Rust output in HarfRust and seems to work fine, except for a minor issue: https://github.com/harfbuzz/harfrust/blob/main/docs/ragel.md#the-universal-state-machine |
|
I'm using this version: emoji-segmenter on main
λ ~/.colm-suite/bin/ragel-rust -v
Ragel State Machine Compiler version 7.1.0-pre.1 April 2026
Copyright (c) 2001-2026 by Dr. Adrian D. Thurston et al.Try to generate rust code: λ ~/.colm-suite/bin/ragel-rust -F1 -e emoji_presentation_scanner.rlDetails#include <stdbool.h>
#ifndef EMOJI_LINKAGE
#define EMOJI_LINKAGE static
#endif
static _emoji_presentation_trans_keys: [unsigned char ; 28] = [ 0, 13, 14, 15, 0, 13, 9, 12, 10, 12, 10, 10, 4, 12, 4, 12, 6, 6, 8, 12, 8, 8, 8, 10, 9, 14, 0, 0 ];
static _emoji_presentation_char_class: [i8 ; 19] = [ 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0 , 0 ];
static _emoji_presentation_index_offsets: [i8 ; 15] = [ 0, 14, 16, 30, 34, 37, 38, 47, 56, 57, 62, 63, 66, 0 , 0 ];
static _emoji_presentation_indices: [i8 ; 74] = [ 1, 1, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 4, 5, 7, 8, 9, 10, 5, 11, 12, 13, 11, 11, 11, 11, 11, 14, 5, 15, 16, 17, 15, 18, 19, 15, 19, 18, 18, 18, 18, 18, 15, 18, 19, 19, 0, 0, 0, 0, 5, 15, 16, 17, 5, 21, 5, 15, 22, 23, 16, 26, 25, 15, 5, 15, 16, 17, 18, 4, 0 , 0 ];
static _emoji_presentation_index_defaults: [i8 ; 15] = [ 0, 3, 11, 0, 18, 0, 18, 0, 18, 20, 24, 25, 18, 0 , 0 ];
static _emoji_presentation_cond_targs: [i8 ; 29] = [ 2, 4, 6, 2, 1, 2, 2, 3, 3, 7, 7, 2, 8, 9, 12, 0, 2, 5, 2, 5, 2, 2, 10, 11, 2, 2, 2, 0 , 0 ];
static _emoji_presentation_cond_actions: [i8 ; 29] = [ 1, 2, 2, 3, 0, 4, 0, 7, 2, 7, 2, 8, 0, 7, 2, 0, 9, 10, 11, 2, 12, 13, 0, 10, 14, 15, 16, 0 , 0 ];
static _emoji_presentation_to_state_actions: [i8 ; 15] = [ 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 ];
static _emoji_presentation_from_state_actions: [i8 ; 15] = [ 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 , 0 ];
static _emoji_presentation_eof_trans: [i8 ; 15] = [ 1, 4, 7, 1, 19, 1, 19, 1, 19, 21, 25, 26, 19, 0 , 0 ];
static emoji_presentation_start : i32 = 2;
static emoji_presentation_en_text_and_emoji_run : i32 = 2;
EMOJI_LINKAGE emoji_text_iter_t
scan_emoji_presentation (emoji_text_iter_t p,
const emoji_text_iter_t pe,
bool* is_emoji,
bool* has_vs)
{
emoji_text_iter_t ts;
emoji_text_iter_t te;
const emoji_text_iter_t eof = pe;
(void)ts;
unsigned act;
int cs;
{
cs = ( emoji_presentation_start ) as i32;
ts = 0;
te = 0;
act = 0;
}
{
let mut _trans = 0;
let mut _keys :i32= 0;
let mut _inds :i32= 0;
let mut _ic = 0;
'_resume: while ( p != pe || p == eof ) {
'_again: while ( true ) {
match ( _emoji_presentation_from_state_actions[(cs) as usize] ) {
6 => {
{{ts = p;
}}
}
_ => {}
}
if ( p == eof ) {
{
if ( _emoji_presentation_eof_trans[(cs) as usize]> 0 ) {
{
_trans = ( _emoji_presentation_eof_trans ) as u32[(cs) as usize]- 1;
}
}
}
}
else {
{
_keys = ( (cs<<1) ) as i32;
_inds = ( _emoji_presentation_index_offsets[(cs) as usize] ) as i32;
if ( ( data[(p ) as usize]
) <= 16 ) {
{
_ic = ( _emoji_presentation_char_class ) as i32[(( ( data[(p ) as usize]
) ) as i32- 0) as usize];
if ( _ic <= ( _emoji_presentation_trans_keys[(_keys+1 ) as usize]
) as i32&& _ic >= ( _emoji_presentation_trans_keys[(_keys ) as usize]
) as i32 ) {
_trans = ( _emoji_presentation_indices[(_inds + ( ( _ic - ( _emoji_presentation_trans_keys[(_keys ) as usize]
) as i32) ) as i32) as usize]
) as u32;
}
else {
_trans = ( _emoji_presentation_index_defaults ) as u32[(cs) as usize];
}
}
}
else {
{
_trans = ( _emoji_presentation_index_defaults ) as u32[(cs) as usize];
}
}
}
}
cs = ( _emoji_presentation_cond_targs ) as i32[(_trans) as usize];
if ( _emoji_presentation_cond_actions[(_trans) as usize]!= 0 ) {
{
match ( _emoji_presentation_cond_actions[(_trans) as usize] ) {
9 => {
{{te = p+1;
{*is_emoji = false; *has_vs = true; return te; }
}}
}
16 => {
{{te = p+1;
{*is_emoji = true; *has_vs = true; return te; }
}}
}
4 => {
{{te = p+1;
{*is_emoji = true; *has_vs = false; return te; }
}}
}
13 => {
{{te = p+1;
{*is_emoji = true; *has_vs = false; return te; }
}}
}
8 => {
{{te = p+1;
{*is_emoji = false; *has_vs = false; return te; }
}}
}
14 => {
{{te = p;
p = p - 1;
{*is_emoji = false; *has_vs = true; return te; }
}}
}
15 => {
{{te = p;
p = p - 1;
{*is_emoji = true; *has_vs = true; return te; }
}}
}
11 => {
{{te = p;
p = p - 1;
{*is_emoji = true; *has_vs = false; return te; }
}}
}
12 => {
{{te = p;
p = p - 1;
{*is_emoji = false; *has_vs = false; return te; }
}}
}
3 => {
{{p = ((te))-1;
{*is_emoji = true; *has_vs = false; return te; }
}}
}
1 => {
{{match ( act ) {
2 => {
p = ((te))-1;
{*is_emoji = true; *has_vs = true; return te; }
}
3 => {
p = ((te))-1;
{*is_emoji = true; *has_vs = false; return te; }
}
5 => {
p = ((te))-1;
{*is_emoji = false; *has_vs = false; return te; }
}
_ => {}
}
}
}
}
10 => {
{{te = p+1;
}}
{{act = 2;
}}
}
2 => {
{{te = p+1;
}}
{{act = 3;
}}
}
7 => {
{{te = p+1;
}}
{{act = 5;
}}
}
_ => {}
}
}
}
break '_again;
}
if ( p == eof ) {
{
if ( cs >= 2 ) {
break '_resume;
}
}
}
else {
{
match ( _emoji_presentation_to_state_actions[(cs) as usize] ) {
5 => {
{{ts = 0;
}}
}
_ => {}
}
p += 1;
continue '_resume;
}
}
break '_resume;
}
}
*is_emoji = false;
*has_vs = false;
return p;
}It looks like the generated code is not quite correct. BTW: I pulled the latest colm-suite source code and added a patch: Details |
That looks bad indeed. Thanks for the steps. In HarfBuzz / HarfRust we heavily rely on ragel, and historically it's latest versions have indeed be flaky. That's unfortunate. I'll take a look and see if I can make this work (changing emoji-segmenter to remove the C/C++ out of the .rl at least). |
There was a problem hiding this comment.
Thank you so much, and apologies that this has taken much too long to review. Overall, I'm pretty positive on these changes.
The only blocking concern is making sure that we're properly following Emoji Segmenter’s license.
If you want us to take this over, please let me know!
| @@ -0,0 +1,615 @@ | |||
| // Copyright 2026 the Parley Authors | |||
| // SPDX-License-Identifier: Apache-2.0 OR MIT | |||
There was a problem hiding this comment.
Again, is this license header right, if it's heavily adapted from Apache 2.0 only code?
| style: EmojiPresentationStyle, | ||
| } | ||
|
|
||
| fn assert_emoji_segmenters_produce_same_result(entity: TestEntity<'_>) { |
There was a problem hiding this comment.
Which are the segmenters producing the same result in this name?
| // FIXME: segmented clusters are incorrect | ||
| // ✖️, [[0x1F1FA, 0x1F1FA], [0x1F1F8]] | ||
| // ✔️, [[0x1F1FA], [0x1F1FA, 0x1F1F8]] |
| write!(&mut file, "{code_extra}").unwrap(); | ||
| } | ||
|
|
||
| fn generate_emojis(emojis: &[(u32, u32)]) -> String { |
There was a problem hiding this comment.
Is there any sensible alternative compression we could do? I haven't looked into how much binary size this adds yet. (cc @taj-p)
There was a problem hiding this comment.
It's very small: static EMOJI_COMPOSITE_U8: [u8; 10] = [2, 3, 4, 5, 11, 15, 29, 35, 43, 77];
But the match is very large.
There was a problem hiding this comment.
Details
static EMOJI_COMPOSITE_U8: [u8; 10] = [2, 3, 4, 5, 11, 15, 29, 35, 43, 77];
#[allow(missing_docs, reason = "generated code")]
#[inline]
pub const fn emoji_composite_get(cp: u32) -> u32 {
let idx = match cp {
0x1F02C..=0x1F02F
| 0x1F094..=0x1F09F
| 0x1F0AF..=0x1F0B0
| 0x1F0C0
| 0x1F0D0
| 0x1F0F6..=0x1F0FF
| 0x1F1AE..=0x1F1E5
| 0x1F203..=0x1F20F
| 0x1F23C..=0x1F23F
| 0x1F249..=0x1F24F
| 0x1F252..=0x1F25F
| 0x1F266..=0x1F2FF
| 0x1F6D9..=0x1F6DB
| 0x1F6ED..=0x1F6EF
| 0x1F6FD..=0x1F6FF
| 0x1F7DA..=0x1F7DF
| 0x1F7EC..=0x1F7EF
| 0x1F7F1..=0x1F7FF
| 0x1F80C..=0x1F80F
| 0x1F848..=0x1F84F
| 0x1F85A..=0x1F85F
| 0x1F888..=0x1F88F
| 0x1F8AE..=0x1F8AF
| 0x1F8BC..=0x1F8BF
| 0x1F8C2..=0x1F8CF
| 0x1F8D9..=0x1F8FF
| 0x1FA58..=0x1FA5F
| 0x1FA6E..=0x1FA6F
| 0x1FA7D..=0x1FA7F
| 0x1FA8B..=0x1FA8D
| 0x1FAC7
| 0x1FAC9..=0x1FACC
| 0x1FADD..=0x1FADE
| 0x1FAEB..=0x1FAEE
| 0x1FAF9..=0x1FAFF
| 0x1FC00..=0x1FFFD => 0,
0xA9
| 0xAE
| 0x203C
| 0x2049
| 0x2122
| 0x2139
| 0x2194..=0x2199
| 0x21A9..=0x21AA
| 0x2328
| 0x23CF
| 0x23ED..=0x23EF
| 0x23F1..=0x23F2
| 0x23F8..=0x23FA
| 0x24C2
| 0x25AA..=0x25AB
| 0x25B6
| 0x25C0
| 0x25FB..=0x25FC
| 0x2600..=0x2604
| 0x260E
| 0x2611
| 0x2618
| 0x2620
| 0x2622..=0x2623
| 0x2626
| 0x262A
| 0x262E..=0x262F
| 0x2638..=0x263A
| 0x2640
| 0x2642
| 0x265F..=0x2660
| 0x2663
| 0x2665..=0x2666
| 0x2668
| 0x267B
| 0x267E
| 0x2692
| 0x2694..=0x2697
| 0x2699
| 0x269B..=0x269C
| 0x26A0
| 0x26A7
| 0x26B0..=0x26B1
| 0x26C8
| 0x26CF
| 0x26D1
| 0x26D3
| 0x26E9
| 0x26F0..=0x26F1
| 0x26F4
| 0x26F7..=0x26F8
| 0x2702
| 0x2708..=0x2709
| 0x270F
| 0x2712
| 0x2714
| 0x2716
| 0x271D
| 0x2721
| 0x2733..=0x2734
| 0x2744
| 0x2747
| 0x2763..=0x2764
| 0x27A1
| 0x2934..=0x2935
| 0x2B05..=0x2B07
| 0x3030
| 0x303D
| 0x3297
| 0x3299
| 0x1F170..=0x1F171
| 0x1F17E..=0x1F17F
| 0x1F202
| 0x1F237
| 0x1F321
| 0x1F324..=0x1F32C
| 0x1F336
| 0x1F37D
| 0x1F396..=0x1F397
| 0x1F399..=0x1F39B
| 0x1F39E..=0x1F39F
| 0x1F3CD..=0x1F3CE
| 0x1F3D4..=0x1F3DF
| 0x1F3F3
| 0x1F3F5
| 0x1F3F7
| 0x1F43F
| 0x1F441
| 0x1F4FD
| 0x1F549..=0x1F54A
| 0x1F56F..=0x1F570
| 0x1F573
| 0x1F576..=0x1F579
| 0x1F587
| 0x1F58A..=0x1F58D
| 0x1F5A5
| 0x1F5A8
| 0x1F5B1..=0x1F5B2
| 0x1F5BC
| 0x1F5C2..=0x1F5C4
| 0x1F5D1..=0x1F5D3
| 0x1F5DC..=0x1F5DE
| 0x1F5E1
| 0x1F5E3
| 0x1F5E8
| 0x1F5EF
| 0x1F5F3
| 0x1F5FA
| 0x1F6CB
| 0x1F6CD..=0x1F6CF
| 0x1F6E0..=0x1F6E5
| 0x1F6E9
| 0x1F6F0
| 0x1F6F3 => 1,
0x200D | 0x20E3 | 0xFE0F | 0xE0020..=0xE007F => 2,
0x23 | 0x2A | 0x30..=0x39 => 3,
0x231A..=0x231B
| 0x23E9..=0x23EC
| 0x23F0
| 0x23F3
| 0x25FD..=0x25FE
| 0x2614..=0x2615
| 0x2648..=0x2653
| 0x267F
| 0x2693
| 0x26A1
| 0x26AA..=0x26AB
| 0x26BD..=0x26BE
| 0x26C4..=0x26C5
| 0x26CE
| 0x26D4
| 0x26EA
| 0x26F2..=0x26F3
| 0x26F5
| 0x26FA
| 0x26FD
| 0x2705
| 0x2728
| 0x274C
| 0x274E
| 0x2753..=0x2755
| 0x2757
| 0x2795..=0x2797
| 0x27B0
| 0x27BF
| 0x2B1B..=0x2B1C
| 0x2B50
| 0x2B55
| 0x1F004
| 0x1F0CF
| 0x1F18E
| 0x1F191..=0x1F19A
| 0x1F201
| 0x1F21A
| 0x1F22F
| 0x1F232..=0x1F236
| 0x1F238..=0x1F23A
| 0x1F250..=0x1F251
| 0x1F300..=0x1F320
| 0x1F32D..=0x1F335
| 0x1F337..=0x1F37C
| 0x1F37E..=0x1F384
| 0x1F386..=0x1F393
| 0x1F3A0..=0x1F3C1
| 0x1F3C5..=0x1F3C6
| 0x1F3C8..=0x1F3C9
| 0x1F3CF..=0x1F3D3
| 0x1F3E0..=0x1F3F0
| 0x1F3F4
| 0x1F3F8..=0x1F3FA
| 0x1F400..=0x1F43E
| 0x1F440
| 0x1F444..=0x1F445
| 0x1F451..=0x1F465
| 0x1F479..=0x1F47B
| 0x1F47D..=0x1F480
| 0x1F484
| 0x1F488..=0x1F48E
| 0x1F490
| 0x1F492..=0x1F4A9
| 0x1F4AB..=0x1F4FC
| 0x1F4FF..=0x1F53D
| 0x1F54B..=0x1F54E
| 0x1F550..=0x1F567
| 0x1F5A4
| 0x1F5FB..=0x1F644
| 0x1F648..=0x1F64A
| 0x1F680..=0x1F6A2
| 0x1F6A4..=0x1F6B3
| 0x1F6B7..=0x1F6BF
| 0x1F6C1..=0x1F6C5
| 0x1F6D0..=0x1F6D2
| 0x1F6D5..=0x1F6D8
| 0x1F6DC..=0x1F6DF
| 0x1F6EB..=0x1F6EC
| 0x1F6F4..=0x1F6FC
| 0x1F7E0..=0x1F7EB
| 0x1F7F0
| 0x1F90D..=0x1F90E
| 0x1F910..=0x1F917
| 0x1F920..=0x1F925
| 0x1F927..=0x1F92F
| 0x1F93A
| 0x1F93F..=0x1F945
| 0x1F947..=0x1F976
| 0x1F978..=0x1F9AF
| 0x1F9B4
| 0x1F9B7
| 0x1F9BA
| 0x1F9BC..=0x1F9CC
| 0x1F9D0
| 0x1F9DE..=0x1F9FF
| 0x1FA70..=0x1FA7C
| 0x1FA80..=0x1FA8A
| 0x1FA8E..=0x1FAC2
| 0x1FAC6
| 0x1FAC8
| 0x1FACD..=0x1FADC
| 0x1FADF..=0x1FAEA
| 0x1FAEF => 4,
0x1F9B0..=0x1F9B3 => 5,
0x1F3FB..=0x1F3FF => 6,
0x261D | 0x26F9 | 0x270C..=0x270D | 0x1F3CB..=0x1F3CC | 0x1F574..=0x1F575 | 0x1F590 => 7,
0x270A..=0x270B
| 0x1F385
| 0x1F3C2..=0x1F3C4
| 0x1F3C7
| 0x1F3CA
| 0x1F442..=0x1F443
| 0x1F446..=0x1F450
| 0x1F466..=0x1F478
| 0x1F47C
| 0x1F481..=0x1F483
| 0x1F485..=0x1F487
| 0x1F48F
| 0x1F491
| 0x1F4AA
| 0x1F57A
| 0x1F595..=0x1F596
| 0x1F645..=0x1F647
| 0x1F64B..=0x1F64F
| 0x1F6A3
| 0x1F6B4..=0x1F6B6
| 0x1F6C0
| 0x1F6CC
| 0x1F90C
| 0x1F90F
| 0x1F918..=0x1F91F
| 0x1F926
| 0x1F930..=0x1F939
| 0x1F93C..=0x1F93E
| 0x1F977
| 0x1F9B5..=0x1F9B6
| 0x1F9B8..=0x1F9B9
| 0x1F9BB
| 0x1F9CD..=0x1F9CF
| 0x1F9D1..=0x1F9DD
| 0x1FAC3..=0x1FAC5
| 0x1FAF0..=0x1FAF8 => 8,
0x1F1E6..=0x1F1FF => 9,
_ => return 0,
};
EMOJI_COMPOSITE_U8[idx as usize] as u32
}There was a problem hiding this comment.
These tests need rethinking; I think we need both a better end-to-end tests, and better cross-platform tests.
As an example, rendering the same thing in multiple sizes doesn't make any sense here; this is more testing that Vello does stuff properly. I'd lean towards just maintaining the current tests at the moment, and deferring testing to a follow-up.
Let's continue. I'll fix the conflict first. |
|
Hi! Nice work on the Rust implementation. I noticed that this appears to be a port of my C library, https://github.com/chansen/c-emoji I'd appreciate it if the PR (or the resulting source/README) acknowledged that the implementation was originally derived from Also, if it's useful, you're welcome to reuse the test cases from https://github.com/chansen/c-emoji/blob/main/test/emoji_scan_test.c They cover a number of edge cases and should help validate that the Rust implementation behaves the same as the original. Thanks! |
Yes, Thanks for the reminder and for your work. I will source the info. |
|
Oh yikes, the licensing gets even more complicated then... For future reference, you really need to be explicit about where code you use as a base for porting comes from, so that we can follow the license properly. @chansen from my perspective as a maintainer here, I'm definitely more than happy to provide attribution. I also note that it's required anyway through your MIT license's copyright notice. I think the least bad option is probably to move the emoji segmentstion to it's own crate, which would then have to have the license be |
It looks like the only part derived from the Google code is the test suite. I wrote the Perl script that generates those test cases. If it's useful, I'm happy to share the script, and you can license it however you like. emoji_test_data.pl.gz |
f9e57ad to
8099d59
Compare
923047e to
14e258b
Compare
6 participants
emojimodule and implementedscan_emoji_presentation, its based onemoji-segmenter's Ragel grammar.By the way, this PR should help #492.
Related to #617.