Merge pull request #358 from eccenca/feature/provideHtmlEntitiesDecoding-CMEM-7032

Provide functionality to decode HTML entities (CMEM-7032)

Author: andreas-schultz

CHANGELOG.md

@@ -54,6 +54,7 @@ This is a major release, and it might be not compatible with your current usage
     -   `colorCalculateDistance()`: calculates the difference between 2 colors using the simple CIE76 formula
     -   `textToColorHash()`: calculates a color from a text string
     -   `reduceToText`: shrinks HTML content and React elements to plain text, used for `<TextReducer />`
+    -   `decodeHtmlEntities`: decode a string of HTML text, map HTML entities back to UTF-8 chars
 -   SCSS color functions
     -   `eccgui-color-var`: returns a var of a custom property used for palette color
     -   `eccgui-color-mix`: mix 2 colors in `srgb`, works with all types of color values and CSS custom properties

package.json

@@ -87,6 +87,7 @@
         "codemirror": "^6.0.1",
         "color": "^4.2.3",
         "compute-scroll-into-view": "^3.1.1",
+        "he": "^1.2.0",
         "jshint": "^2.13.6",
         "lodash": "^4.17.21",
         "n3": "^1.25.1",
@@ -134,6 +135,7 @@
         "@testing-library/react": "^12.1.5",
         "@types/codemirror": "^5.60.15",
         "@types/color": "^3.0.6",
+        "@types/he": "^1.2.3",
         "@types/jest": "^29.5.14",
         "@types/jshint": "^2.12.4",
         "@types/lodash": "^4.17.16",

src/common/index.ts

@@ -1,3 +1,5 @@
+import { decode } from "he";
+
 import { invisibleZeroWidthCharacters } from "./utils/characters";
 import { colorCalculateDistance } from "./utils/colorCalculateDistance";
 import decideContrastColorValue from "./utils/colorDecideContrastvalue";
@@ -6,7 +8,8 @@ import getColorConfiguration from "./utils/getColorConfiguration";
 import { getScrollParent } from "./utils/getScrollParent";
 import { getGlobalVar, setGlobalVar } from "./utils/globalVars";
 import { openInNewTab } from "./utils/openInNewTab";
-import { reduceToText } from "./utils/reduceToText"
+import { reduceToText } from "./utils/reduceToText";
+export type { DecodeOptions as DecodeHtmlEntitiesOptions } from "he";
 export type { IntentTypes as IntentBaseTypes } from "./Intent";
 
 export const utils = {
@@ -20,5 +23,6 @@ export const utils = {
     getScrollParent,
     getEnabledColorsFromPalette,
     textToColorHash,
-    reduceToText
+    reduceToText,
+    decodeHtmlEntities: decode,
 };

src/common/utils/reduceToText.tsx

@@ -3,19 +3,20 @@ import { renderToString } from "react-dom/server";
 import * as ReactIs from "react-is";
 
 import { TextReducerProps } from "./../../components/TextReducer/TextReducer";
+import { DecodeHtmlEntitiesOptions, utils } from "./../";
 
 export interface ReduceToTextFuncType {
     (
         /**
          *  Component or text to reduce HTML markup content to plain text.
          */
         input: React.ReactNode | React.ReactNode[] | string,
-        options?: Pick<TextReducerProps, "maxNodes" | "maxLength">
+        options?: Pick<TextReducerProps, "maxNodes" | "maxLength" | "decodeHtmlEntities" | "decodeHtmlEntitiesOptions">
     ): string;
 }
 
 export const reduceToText: ReduceToTextFuncType = (input, options) => {
-    const { maxNodes, maxLength } = options || {};
+    const { maxNodes, maxLength, decodeHtmlEntities } = options || {};
     const content: React.ReactNode | React.ReactNode[] = input;
     let nodeCount = 0;
 
@@ -46,6 +47,33 @@ export const reduceToText: ReduceToTextFuncType = (input, options) => {
     // Basic HTML cleanup
     text = text.replace(/<[^\s][^>]*>/g, "").replace(/\n/g, " ");
 
+    if (decodeHtmlEntities) {
+        const decodeDefaultOptions = {
+            isAttributeValue: true,
+            strict: true,
+        } as DecodeHtmlEntitiesOptions;
+        let decodeErrors = 0;
+        // we decode in pieces to apply some error tolerance even in strict mode
+        text = text
+            .split(" ")
+            .map((value) => {
+                try {
+                    return utils.decodeHtmlEntities(value, {
+                        ...decodeDefaultOptions,
+                        ...options?.decodeHtmlEntitiesOptions,
+                    });
+                } catch {
+                    decodeErrors++;
+                    return value;
+                }
+            })
+            .join(" ");
+        if (decodeErrors > 0) {
+            // eslint-disable-next-line no-console
+            console.warn(`${decodeErrors} parse error(s) for decodeHtmlEntities, return un-decoded text`, text);
+        }
+    }
+
     if (typeof maxLength === "number") {
         text = text.slice(0, maxLength);
     }

src/components/TextReducer/TextReducer.stories.tsx

@@ -18,8 +18,9 @@ Default.args = {
         <LoremIpsum p={1} avgSentencesPerParagraph={1} random={false} />,
         "Simple text with URL http://example.com/ that should not get parsed.",
         "a < b to test equations in text like b > a.",
+        `Something with a "quote" in it.`,
         <>
-            <Markdown>{`* This\n* is\n* a\n* list\n\nwritten in Markdown.`}</Markdown>
+            <Markdown>{`* This\n* is\n* a\n* list\n\nwritten in Markdown\n* containing a few HTML 'entities' & "quotes".`}</Markdown>
             <HtmlContentBlock>
                 <h1>Block with sub elements</h1>
                 <LoremIpsum p={3} avgSentencesPerParagraph={3} random={false} />

src/components/TextReducer/TextReducer.test.tsx

@@ -0,0 +1,44 @@
+import React from "react";
+import {render, RenderResult} from "@testing-library/react";
+
+import "@testing-library/jest-dom";
+
+import { Markdown, TextReducer } from "./../../";
+import { Default as TextReducerStory } from "./TextReducer.stories";
+
+describe("TextReducer", () => {
+    const textMustExist = (queryByText: RenderResult["queryByText"], text: string) => {
+        expect(queryByText(text, { exact: false })).not.toBeNull();
+    }
+    const textMustNotExist = (queryByText: RenderResult["queryByText"], text: string) => {
+        expect(queryByText(text, { exact: false })).toBeNull();
+    }
+    it("should display encoded HTML entities by default if they are used in the transformed markup", () => {
+        const { queryByText } = render(<TextReducer {...TextReducerStory.args} />);
+        textMustExist(queryByText, "&#x27;entities&#x27; &amp; &quot;quotes&quot;");
+        textMustNotExist(queryByText, `'entities' & "quotes"`);
+    });
+    it("should not display encoded HTML entities if `decodeHtmlEntities` is enabled", () => {
+        const { queryByText } = render(<TextReducer {...TextReducerStory.args} decodeHtmlEntities />);
+        textMustNotExist(queryByText, "&#x27;entities&#x27; &amp; &quot;quotes&quot;");
+        textMustExist(queryByText, `'entities' & "quotes"`);
+    });
+    it("should only decode if correct encoded HTML entities are found (strict mode)", () => {
+        const { queryByText } = render(
+            <TextReducer decodeHtmlEntities>
+                <Markdown>&</Markdown>&amp foo&ampbar
+            </TextReducer>
+        );
+        textMustExist(queryByText, "& &amp foo&ampbar");
+        textMustNotExist(queryByText, "& & foo&ampbar");
+    });
+    it("should allow decoding non-strict encoded HTML entities", () => {
+        const { queryByText } = render(
+            <TextReducer decodeHtmlEntities decodeHtmlEntitiesOptions={{ strict: false }}>
+                <Markdown>&</Markdown>&amp foo&ampbar
+            </TextReducer>
+        );
+        textMustNotExist(queryByText, "& &amp foo&ampbar");
+        textMustExist(queryByText, "& & foo&ampbar");
+    });
+});

src/components/TextReducer/TextReducer.tsx

@@ -1,6 +1,6 @@
 import React from "react";
 
-import { reduceToText } from "../../common/utils/reduceToText";
+import { DecodeHtmlEntitiesOptions, utils } from "../../common";
 import { CLASSPREFIX as eccgui } from "../../configuration/constants";
 
 import { OverflowText, OverflowTextProps } from "./../Typography";
@@ -24,6 +24,17 @@ export interface TextReducerProps extends Pick<React.HTMLAttributes<HTMLElement>
      * Specify more `OverflowText` properties used when `useOverflowTextWrapper` is set to `true`.
      */
     overflowTextProps?: Omit<OverflowTextProps, "passDown">;
+    /**
+     * If you transform HTML markup to text then the result could contain HTML entity encoded strings.
+     * By enabling this option they are decoded back to it's original char.
+     */
+    decodeHtmlEntities?: boolean;
+    /**
+     * Set the options used to decode the HTML entities, if `decodeHtmlEntities` is enabled.
+     * Internally we use `he` library, see their [documentation on decode options](https://www.npmjs.com/package/he#hedecodehtml-options).
+     * If not set we use `{ isAttributeValue: true, strict: true }` as default value.
+     */
+    decodeHtmlEntitiesOptions?: DecodeHtmlEntitiesOptions;
 }
 
 /**
@@ -32,16 +43,15 @@ export interface TextReducerProps extends Pick<React.HTMLAttributes<HTMLElement>
  */
 export const TextReducer = ({
     children,
-    maxNodes,
-    maxLength,
     useOverflowTextWrapper,
     overflowTextProps,
+    ...reduceToTextOptions
 }: TextReducerProps) => {
     if (typeof children === "undefined") {
         return <></>;
     }
 
-    const shrinkedContent = reduceToText(children, { maxLength, maxNodes });
+    const shrinkedContent = utils.reduceToText(children, reduceToTextOptions);
 
     return useOverflowTextWrapper ? (
         <OverflowText

yarn.lock

@@ -3079,6 +3079,11 @@
   dependencies:
     "@types/unist" "*"
 
+"@types/he@^1.2.3":
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/@types/he/-/he-1.2.3.tgz#c33ca3096f30cbd5d68d78211572de3f9adff75a"
+  integrity sha512-q67/qwlxblDzEDvzHhVkwc1gzVWxaNxeyHUBF4xElrvjL11O+Ytze+1fGpBHlr/H9myiBUaUXNnNPmBHxxfAcA==
+
 "@types/hoist-non-react-statics@^3.3.0":
   version "3.3.6"
   resolved "https://registry.yarnpkg.com/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.6.tgz#6bba74383cdab98e8db4e20ce5b4a6b98caed010"