exclude ALMOST ALL symbols and punctuation (more aggressive, using Unicode classes) from filenames

Author: emilykl

src/lib/slugify.js

@@ -1,17 +1,25 @@
 'use strict';
 
 // precompile for speed
-var HTML_TAGS_REGEX = /<[^>]*>/g;                 // anything contained in < > tags
-var FORBIDDEN_CHARS_REGEX = /[\\/:*?"<>|$%&!@#~.^`'(){}[\],=+;]/g; // Characters in the set: \/:*?"<>|$%&!@#~.^`'(){}[],=+;
-var CONTROL_CHARS_REGEX = /\p{Cc}/gu; // Unicode control characters
+var HTML_TAGS_REGEX = /<[^>]*>/g;  // Anything contained in < > tags
+// Forbid ANY character in the Unicode Symbol or Punctuation categories,
+// which includes pretty much every common non-word character,
+// including all of the following: \/:*?"<>|$%&!@#~.^`'(){}[]-_,=+;
+// We actually want to allow hyphens (WORD_SEP_CHAR), but it's not easy
+// to exclude them from the regex, so we'll handle them in the match function.
+var FORBIDDEN_CHARS_REGEX = /[\p{S}\p{P}]/gu;
+// Control chars, format chars (e.g. ZWJ), variation selectors, and the
+// combining enclosing keycap. Some of these may be left behind after emoji
+// symbols are removed, so we explicitly remove them here.
+var INVISIBLE_CHARS_REGEX = /[\p{Cc}\p{Cf}\uFE00-\uFE0F\u20E3]/gu;
 
-var UNICODE_REPLACEMENT_CHAR_REGEX = /�/g;        // U+FFFD, the Unicode replacement character
-var WHITESPACE_REGEX = /\s+/g;
+var UNICODE_REPLACEMENT_CHAR_REGEX = /�/g;  // U+FFFD, the Unicode replacement character
+var WHITESPACE_REGEX = /\s+/g;  // All whitespace characters
 
-var WORD_SEP_CHAR = '-';                                                       // character used to separate words (replaces whitespace)
+var WORD_SEP_CHAR = '-';                                                        // Character used to separate words (replaces whitespace)
 var _WORD_SEP_ESCAPED = WORD_SEP_CHAR.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-var WORD_SEP_CHARS_REGEX = new RegExp(_WORD_SEP_ESCAPED + '{3,}', 'g');         // three or more consecutive word separator chars
-var TRAILING_WORD_SEP_CHAR_REGEX = new RegExp(_WORD_SEP_ESCAPED + '$', 'g');    // trailing word separator char
+var WORD_SEP_CHARS_REGEX = new RegExp(_WORD_SEP_ESCAPED + '{3,}', 'g');         // Three or more consecutive word separator chars
+var TRAILING_WORD_SEP_CHAR_REGEX = new RegExp(_WORD_SEP_ESCAPED + '$', 'g');    // Trailing word separator char
 
 // Safely under the limit for most filesystems
 var DEFAULT_MAX_LEN = 60;
@@ -39,15 +47,17 @@ function toWellFormed(str) {
  * @return {string}
  */
 module.exports = function slugify(str, maxLen = DEFAULT_MAX_LEN) {
-    var slug = toWellFormed(str ?? '')                 // Guarantee well-formed Unicode text
-        .replace(UNICODE_REPLACEMENT_CHAR_REGEX, '')   // Drop Unicode replacement chars left by previous step
-        .replace(HTML_TAGS_REGEX, ' ')                 // Remove < > tags, such as <br> (replace with space)
-        .replace(FORBIDDEN_CHARS_REGEX, '')            // Remove forbidden filename characters
-        .toLowerCase()                                 // Lowercase everything
-        .trim()                                        // Strip leading/trailing whitespace
-        .replace(WHITESPACE_REGEX, WORD_SEP_CHAR)      // Replace any remaining whitespace with the word sep char
-        .replace(CONTROL_CHARS_REGEX, '')              // Remove control characters (after whitespace)
-        .replace(WORD_SEP_CHARS_REGEX, WORD_SEP_CHAR); // Replace multiple word sep chars with a single one
+    var slug = toWellFormed(str ?? '')                      // Guarantee well-formed Unicode text
+        .replace(UNICODE_REPLACEMENT_CHAR_REGEX, '')        // Drop Unicode replacement chars left by previous step
+        .replace(HTML_TAGS_REGEX, ' ')                      // Remove < > tags, such as <br> (replace with space)
+        .replace(FORBIDDEN_CHARS_REGEX, (c) => {            // Remove forbidden filename characters (but allow the word sep char)
+            return c === WORD_SEP_CHAR ? c : '';
+        })
+        .toLowerCase()                                      // Lowercase everything
+        .trim()                                             // Strip leading/trailing whitespace
+        .replace(WHITESPACE_REGEX, WORD_SEP_CHAR)           // Replace any remaining whitespace with the word sep char
+        .replace(INVISIBLE_CHARS_REGEX, '')                 // Remove control/format chars and emoji glue (after whitespace)
+        .replace(WORD_SEP_CHARS_REGEX, WORD_SEP_CHAR);      // Replace multiple word sep chars with a single one
 
     if (slug.length <= maxLen) return slug;
     // Apply maxLen to the resulting string. Use Array.from().slice() instead of String.prototype.split()

test/jasmine/tests/lib_test.js

@@ -1956,15 +1956,16 @@ describe('Test lib.js:', function () {
                 .toBe('abcdefghijklmnopqrst');
             expect(Lib.slugify('a>b<c)d(e]f[g}h{i')).toBe('abcdefghi');
             expect(Lib.slugify('a=b+c;d')).toBe('abcd');
+            expect(Lib.slugify('a_b😀c★d©e')).toBe('abcde');
         });
 
-        it('removes control characters', function () {
+        it('removes control characters and emoji glue', function () {
             expect(Lib.slugify('a\x00b\x07c\x1Fd')).toBe('abcd');
+            expect(Lib.slugify('a👨‍👩‍👧b❤️c')).toBe('abc');
         });
 
-        it('preserves unicode letters (accents, CJK, emoji)', function () {
+        it('preserves unicode letters (accents, CJK)', function () {
             expect(Lib.slugify('Café 北京')).toBe('café-北京');
-            expect(Lib.slugify('pair😀')).toBe('pair😀');
         });
 
         it('drops unpaired surrogates so the result is valid UTF-8', function () {
@@ -1990,8 +1991,8 @@ describe('Test lib.js:', function () {
         });
 
         it('never splits a surrogate pair when capping length', function () {
-            // each emoji is one code point (a surrogate pair); the cap counts code points
-            expect(Lib.slugify('😀😀😀😀😀', 3)).toBe('😀😀😀');
+            // each CJK char is one code point (a surrogate pair); the cap counts code points
+            expect(Lib.slugify('𠀀𠀀𠀀𠀀𠀀', 3)).toBe('𠀀𠀀𠀀');
         });
     });