diff --git a/en/use-dify/workspace/api-extension/moderation-api-extension.mdx b/en/use-dify/workspace/api-extension/moderation-api-extension.mdx index 1a290d79f..d6230765e 100644 --- a/en/use-dify/workspace/api-extension/moderation-api-extension.mdx +++ b/en/use-dify/workspace/api-extension/moderation-api-extension.mdx @@ -41,10 +41,10 @@ When **Content Moderation > Review Input Content** is enabled in applications su "params": { "app_id": "61248ab4-1125-45be-ae32-0ce91334d021", "inputs": { - "var_1": "I will kill you.", - "var_2": "I will fuck you." + "var_1": "My SSN is 123-45-6789.", + "var_2": "My phone number is 123-456-7890." }, - "query": "Happy everydays." + "query": "Please help me update my account." } } ``` @@ -81,10 +81,10 @@ When **Content Moderation > Review Input Content** is enabled in applications su "flagged": true, "action": "overridden", "inputs": { - "var_1": "I will *** you.", - "var_2": "I will *** you." + "var_1": "My SSN is ***-**-****.", + "var_2": "My phone number is ***-***-****." }, - "query": "Happy everydays." + "query": "Please help me update my account." } ``` @@ -110,7 +110,7 @@ When **Content Moderation > Review Output Content** is enabled in applications s "point": "app.moderation.output", "params": { "app_id": "61248ab4-1125-45be-ae32-0ce91334d021", - "text": "I will kill you." + "text": "My SSN is 123-45-6789." } } ``` @@ -141,7 +141,7 @@ When **Content Moderation > Review Output Content** is enabled in applications s { "flagged": true, "action": "overridden", - "text": "I will *** you." + "text": "My SSN is ***-**-****." } ``` diff --git a/ja/use-dify/workspace/api-extension/moderation-api-extension.mdx b/ja/use-dify/workspace/api-extension/moderation-api-extension.mdx index b3d94c656..be48f07c8 100644 --- a/ja/use-dify/workspace/api-extension/moderation-api-extension.mdx +++ b/ja/use-dify/workspace/api-extension/moderation-api-extension.mdx @@ -41,10 +41,10 @@ Chatflow、Agent、チャットアシスタントなどのアプリケーショ "params": { "app_id": "61248ab4-1125-45be-ae32-0ce91334d021", "inputs": { - "var_1": "I will kill you.", - "var_2": "I will fuck you." + "var_1": "私のマイナンバーは 1234-5678-9012 です。", + "var_2": "私の電話番号は 090-1234-5678 です。" }, - "query": "Happy everydays." + "query": "アカウント情報を更新してください。" } } ``` @@ -81,10 +81,10 @@ Chatflow、Agent、チャットアシスタントなどのアプリケーショ "flagged": true, "action": "overridden", "inputs": { - "var_1": "I will *** you.", - "var_2": "I will *** you." + "var_1": "私のマイナンバーは ****-****-**** です。", + "var_2": "私の電話番号は ***-****-**** です。" }, - "query": "Happy everydays." + "query": "アカウント情報を更新してください。" } ``` @@ -110,7 +110,7 @@ Chatflow、Agent、チャットアシスタントなどのアプリケーショ "point": "app.moderation.output", "params": { "app_id": "61248ab4-1125-45be-ae32-0ce91334d021", - "text": "I will kill you." + "text": "私のマイナンバーは 1234-5678-9012 です。" } } ``` @@ -141,7 +141,7 @@ Chatflow、Agent、チャットアシスタントなどのアプリケーショ { "flagged": true, "action": "overridden", - "text": "I will *** you." + "text": "私のマイナンバーは ****-****-**** です。" } ``` diff --git a/versions/3-0-x/en/user-guide/application-orchestrate/app-toolkits/moderation-tool.mdx b/versions/3-0-x/en/user-guide/application-orchestrate/app-toolkits/moderation-tool.mdx index dd252bbc8..7fb9ae134 100644 --- a/versions/3-0-x/en/user-guide/application-orchestrate/app-toolkits/moderation-tool.mdx +++ b/versions/3-0-x/en/user-guide/application-orchestrate/app-toolkits/moderation-tool.mdx @@ -16,9 +16,7 @@ Now you can also directly call the OpenAI Moderation API on Dify; you can review ## Keywords -Developers can customize the sensitive words they need to review, such as using "kill" as a keyword to perform an audit action when users input. The preset reply content should be "The content is violating usage policies." It can be anticipated that when a user inputs a text chuck containing "kill" at the terminal, it will trigger the sensitive word review tool and return the preset reply content. - -![Configuring Load Balancing from Add Model](https://assets-docs.dify.ai/dify-enterprise-mintlify/en/guides/application-orchestrate/app-toolkits/15652f65c3af15d931598a866a7baa17.png) +Developers can customize the sensitive words they need to review, such as using "confidential" as a keyword to perform an audit action when users input. The preset reply content should be "The content is violating usage policies." It can be anticipated that when a user inputs a text chunk containing "confidential" at the terminal, it will trigger the sensitive word review tool and return the preset reply content. ## Moderation Extension diff --git a/versions/3-0-x/en/user-guide/extension/api-based-extension/moderation.mdx b/versions/3-0-x/en/user-guide/extension/api-based-extension/moderation.mdx index 3e3ab91cb..17762b3d6 100644 --- a/versions/3-0-x/en/user-guide/extension/api-based-extension/moderation.mdx +++ b/versions/3-0-x/en/user-guide/extension/api-based-extension/moderation.mdx @@ -38,10 +38,10 @@ This module is used to review the content input by end-users and the output cont "params": { "app_id": "61248ab4-1125-45be-ae32-0ce91334d021", "inputs": { - "var_1": "I will kill you.", - "var_2": "I will fuck you." + "var_1": "My SSN is 123-45-6789.", + "var_2": "My phone number is 123-456-7890." }, - "query": "Happy everydays." + "query": "Please help me update my account." } } ``` @@ -77,10 +77,10 @@ This module is used to review the content input by end-users and the output cont "flagged": true, "action": "overridden", "inputs": { - "var_1": "I will *** you.", - "var_2": "I will *** you." + "var_1": "My SSN is ***-**-****.", + "var_2": "My phone number is ***-***-****." }, - "query": "Happy everydays." + "query": "Please help me update my account." } ``` @@ -104,7 +104,7 @@ This module is used to review the content input by end-users and the output cont "point": "app.moderation.output", "params": { "app_id": "61248ab4-1125-45be-ae32-0ce91334d021", - "text": "I will kill you." + "text": "My SSN is 123-45-6789." } } ``` @@ -134,6 +134,6 @@ This module is used to review the content input by end-users and the output cont { "flagged": true, "action": "overridden", - "text": "I will *** you." + "text": "My SSN is ***-**-****." } ``` diff --git a/versions/3-0-x/zh/user-guide/application-orchestrate/app-toolkits/moderation-tool.mdx b/versions/3-0-x/zh/user-guide/application-orchestrate/app-toolkits/moderation-tool.mdx index 9f34d2c23..4393d7867 100644 --- a/versions/3-0-x/zh/user-guide/application-orchestrate/app-toolkits/moderation-tool.mdx +++ b/versions/3-0-x/zh/user-guide/application-orchestrate/app-toolkits/moderation-tool.mdx @@ -15,9 +15,7 @@ OpenAI 和大多数 LLM 公司提供的模型，都带有内容审查功能， ### 功能二：自定义关键词 -开发者可以自定义需要审查的敏感词，比如把“kill”作为关键词，在用户输入的时候作审核动作，要求预设回复内容为“The content is violating usage policies.”可以预见的结果是当用户在终端输入包含“kill”的语料片段，就会触发敏感词审查工具，返回预设回复内容。 - -![Keywords](https://assets-docs.dify.ai/dify-enterprise-mintlify/zh_CN/guides/application-orchestrate/app-toolkits/644c2b024f59497aed3cd8ac984c96e3.png) +开发者可以自定义需要审查的敏感词，比如把“密钥”作为关键词，在用户输入的时候作审核动作，要求预设回复内容为“The content is violating usage policies.”。可以预见的结果是当用户在终端输入包含“密钥”的语料片段，就会触发敏感词审查工具，返回预设回复内容。 ### 功能三： 敏感词审查 Moderation 扩展 @@ -25,6 +23,4 @@ OpenAI 和大多数 LLM 公司提供的模型，都带有内容审查功能， ![Moderation Settings](https://assets-docs.dify.ai/dify-enterprise-mintlify/zh_CN/guides/application-orchestrate/app-toolkits/d8b6dff6fce6d70795b87aefc56eb02b.png) -比如我们在自己的本地服务中自定义敏感词审查规则：不能查询有关美国总统的名字的问题。当用户在`query`变量输入"Trump"，则在对话时会返回 "Your content violates our usage policy." 测试效果如下： - -![Moderation Test](https://assets-docs.dify.ai/dify-enterprise-mintlify/zh_CN/guides/application-orchestrate/app-toolkits/970c894a68f017def62c0f7253b0f44e.png) +比如我们在自己的本地服务中自定义敏感词审查规则：不能查询有关公司内部薪资的问题。当用户在`query`变量输入"薪资"，则在对话时会返回 "Your content violates our usage policy."。 diff --git a/zh/use-dify/workspace/api-extension/moderation-api-extension.mdx b/zh/use-dify/workspace/api-extension/moderation-api-extension.mdx index f6f3278af..919166778 100644 --- a/zh/use-dify/workspace/api-extension/moderation-api-extension.mdx +++ b/zh/use-dify/workspace/api-extension/moderation-api-extension.mdx @@ -41,10 +41,10 @@ title: 敏感内容审查 "params": { "app_id": "61248ab4-1125-45be-ae32-0ce91334d021", "inputs": { - "var_1": "I will kill you.", - "var_2": "I will fuck you." + "var_1": "我的身份证号是 123456789012345678。", + "var_2": "我的手机号是 123-4567-8901。" }, - "query": "Happy everydays." + "query": "请帮我更新账户信息。" } } ``` @@ -81,10 +81,10 @@ title: 敏感内容审查 "flagged": true, "action": "overridden", "inputs": { - "var_1": "I will *** you.", - "var_2": "I will *** you." + "var_1": "我的身份证号是 ******************。", + "var_2": "我的手机号是 ***-****-****。" }, - "query": "Happy everydays." + "query": "请帮我更新账户信息。" } ``` @@ -110,7 +110,7 @@ title: 敏感内容审查 "point": "app.moderation.output", "params": { "app_id": "61248ab4-1125-45be-ae32-0ce91334d021", - "text": "I will kill you." + "text": "我的身份证号是 123456789012345678。" } } ``` @@ -141,7 +141,7 @@ title: 敏感内容审查 { "flagged": true, "action": "overridden", - "text": "I will *** you." + "text": "我的身份证号是 ******************。" } ```