diff --git a/src/lib/helpers/http.js b/src/lib/helpers/http.js index ab66e444..91eef803 100644 --- a/src/lib/helpers/http.js +++ b/src/lib/helpers/http.js @@ -185,7 +185,7 @@ function skipLoader(config) { new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/create', 'g'), new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/page', 'g'), new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/upload', 'g'), - new RegExp('http(s*)://(.*?)/knowledge/tokenize', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/entity/analyze', 'g'), new RegExp('http(s*)://(.*?)/users', 'g'), new RegExp('http(s*)://(.*?)/instruct/(.*?)', 'g'), new RegExp('http(s*)://(.*?)/agent/(.*?)/code-scripts', 'g'), @@ -229,9 +229,9 @@ function skipLoader(config) { new RegExp('http(s*)://(.*?)/llm-provider/(.*?)/models', 'g'), new RegExp('http(s*)://(.*?)/knowledge/vector/collections', 'g'), new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/exist', 'g'), - new RegExp('http(s*)://(.*?)/knowledge/document/processors', 'g'), - new RegExp('http(s*)://(.*?)/knowledge/tokenizer/providers', 'g'), - new RegExp('http(s*)://(.*?)/knowledge/tokenizer/data-providers', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/processors', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/entity/analyzers', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/entity/data-providers', 'g'), new RegExp('http(s*)://(.*?)/logger/instruction/log', 'g'), new RegExp('http(s*)://(.*?)/logger/instruction/log/keys', 'g'), new RegExp('http(s*)://(.*?)/logger/conversation/(.*?)/content-log', 'g'), diff --git a/src/lib/helpers/types/knowledgeTypes.js b/src/lib/helpers/types/knowledgeTypes.js index 635ca3c3..23cad223 100644 --- a/src/lib/helpers/types/knowledgeTypes.js +++ b/src/lib/helpers/types/knowledgeTypes.js @@ -162,14 +162,14 @@ */ /** - * @typedef {Object} TokenizeRequest + * @typedef {Object} EntityAnalysisRequest * @property {string} text * @property {string?} [provider] - * @property {TokenizeOptions?} [options] + * @property {EntityAnalysisOptions?} [options] */ /** - * @typedef {Object} TokenizeOptions + * @typedef {Object} EntityAnalysisOptions * @property {string[]?} [data_providers] * @property {number?} [max_ngram] * @property {number?} [cutoff] @@ -177,14 +177,14 @@ */ /** - * @typedef {Object} TokenizeResponse - * @property {TokenizeResult[]} [results] + * @typedef {Object} EntityAnalysisResponse + * @property {EntityAnalysisResult[]} [results] * @property {boolean?} [success] * @property {string?} [error_message] */ /** - * @typedef {Object} TokenizeResult + * @typedef {Object} EntityAnalysisResult * @property {string} token * @property {string?} [canonical_text] * @property {any} data diff --git a/src/lib/services/api-endpoints.js b/src/lib/services/api-endpoints.js index 59331f46..1dd73bd3 100644 --- a/src/lib/services/api-endpoints.js +++ b/src/lib/services/api-endpoints.js @@ -113,11 +113,11 @@ export const endpoints = { knowledgeDocumentDeleteUrl: `${host}/knowledge/document/{collection}/delete/{fileId}`, knowledgeDocumentDeleteAllUrl: `${host}/knowledge/document/{collection}/delete`, knowledgeDocumentPageListUrl: `${host}/knowledge/document/{collection}/page`, - knowledgeDocumentProcessorsUrl: `${host}/knowledge/document/processors`, + knowledgeProcessorsUrl: `${host}/knowledge/processors`, - tokenizersUrl: `${host}/knowledge/tokenizer/providers`, - tokenizerDataLoadersUrl: `${host}/knowledge/tokenizer/data-providers`, - tokenizeUrl: `${host}/knowledge/tokenize`, + entityAnalyzersUrl: `${host}/knowledge/entity/analyzers`, + entityDataLoadersUrl: `${host}/knowledge/entity/data-providers`, + entityAnalyzeUrl: `${host}/knowledge/entity/analyze`, // chathub chatHubUrl: `${host}/chatHub`, diff --git a/src/lib/services/knowledge-base-service.js b/src/lib/services/knowledge-base-service.js index 91f5455c..84827911 100644 --- a/src/lib/services/knowledge-base-service.js +++ b/src/lib/services/knowledge-base-service.js @@ -196,8 +196,8 @@ export async function getKnowledgeDocumentPageList(collection, request) { /** * @returns {Promise} */ -export async function getKnowledgeDocumentProcessors() { - const url = endpoints.knowledgeDocumentProcessorsUrl; +export async function getKnowledgeProcessors() { + const url = endpoints.knowledgeProcessorsUrl; const response = await axios.get(url); return response.data; } @@ -291,8 +291,8 @@ export async function deleteVectorIndexes(collection, options) { /** * @returns {Promise} */ -export async function getTokenizers() { - const url = endpoints.tokenizersUrl; +export async function getEntityAnalyzers() { + const url = endpoints.entityAnalyzersUrl; const response = await axios.get(url); return response.data; } @@ -300,18 +300,18 @@ export async function getTokenizers() { /** * @returns {Promise} */ -export async function getTokenizerDataLoaders() { - const url = endpoints.tokenizerDataLoadersUrl; +export async function getEntityDataLoaders() { + const url = endpoints.entityDataLoadersUrl; const response = await axios.get(url); return response.data; } /** - * @param {import('$knowledgeTypes').TokenizeRequest} request - * @returns {Promise} + * @param {import('$knowledgeTypes').EntityAnalysisRequest} request + * @returns {Promise} */ -export async function tokenize(request) { - const url = endpoints.tokenizeUrl; +export async function analyzeEntity(request) { + const url = endpoints.entityAnalyzeUrl; const response = await axios.post(url, { ...request }); diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-instruction.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-instruction.svelte index 49d4aa0f..f745a866 100644 --- a/src/routes/page/agent/[agentId]/agent-components/agent-instruction.svelte +++ b/src/routes/page/agent/[agentId]/agent-components/agent-instruction.svelte @@ -87,6 +87,22 @@ handleAgentChange(); } + /** @param {any} e */ + function onKeyDown(e) { + if (e.key === 'Tab') { + e.preventDefault(); + + if (e.target) { + const start = e.target.selectionStart || 0; + const end = e.target.selectionEnd || 0; + const value = e.target.value || ''; + e.target.value = value.substring(0, start) + "\t" + value.substring(end); + e.target.selectionStart = start + 1; + e.target.selectionEnd = start + 1; + } + } + } + function addChannel() { inner_instructions = [ ...inner_instructions, @@ -193,6 +209,7 @@ value={selected_instruction.instruction} rows={20} on:input={(e) => changePrompt(e)} + on:keydown={(e) => onKeyDown(e)} placeholder="Enter your instruction" /> diff --git a/src/routes/page/agent/[agentId]/agent-components/agent-template.svelte b/src/routes/page/agent/[agentId]/agent-components/agent-template.svelte index 3b1531fd..ab45fe53 100644 --- a/src/routes/page/agent/[agentId]/agent-components/agent-template.svelte +++ b/src/routes/page/agent/[agentId]/agent-components/agent-template.svelte @@ -80,6 +80,22 @@ handleAgentChange(); } + /** @param {any} e */ + function onKeyDown(e) { + if (e.key === 'Tab') { + e.preventDefault(); + + if (e.target) { + const start = e.target.selectionStart || 0; + const end = e.target.selectionEnd || 0; + const value = e.target.value || ''; + e.target.value = value.substring(0, start) + "\t" + value.substring(end); + e.target.selectionStart = start + 1; + e.target.selectionEnd = start + 1; + } + } + } + function addTemplate() { inner_templates = [ ...inner_templates, @@ -164,10 +180,11 @@ type="textarea" class="form-control" style="scrollbar-width: thin; resize: none;" + placeholder="Enter your content" value={selected_template.content} rows={15} on:input={(e) => changePrompt(e)} - placeholder="Enter your content" + on:keydown={(e) => onKeyDown(e)} /> {/if} diff --git a/src/routes/page/knowledge-base/dictionary/+page.svelte b/src/routes/page/knowledge-base/dictionary/+page.svelte index 94065f53..89fda89e 100644 --- a/src/routes/page/knowledge-base/dictionary/+page.svelte +++ b/src/routes/page/knowledge-base/dictionary/+page.svelte @@ -10,9 +10,9 @@ Table } from '@sveltestrap/sveltestrap'; import { - getTokenizers, - getTokenizerDataLoaders, - tokenize + getEntityAnalyzers, + getEntityDataLoaders, + analyzeEntity } from '$lib/services/knowledge-base-service'; import Breadcrumb from '$lib/common/Breadcrumb.svelte'; import HeadTitle from '$lib/common/HeadTitle.svelte'; @@ -40,19 +40,19 @@ let elapsedTime = ''; /** @type {string | null} */ - let selectedTokenizer = null; + let selectedAnalyzer = null; /** @type {string[]} */ let selectedDataLoaders = []; - /** @type {import('$knowledgeTypes').TokenizeResult[]} */ + /** @type {import('$knowledgeTypes').EntityAnalysisResult[]} */ let items = []; /** @type {import('$commonTypes').LabelValuePair[]} */ - let tokenizers = []; + let analyzers = []; /** @type {import('$commonTypes').LabelValuePair[]} */ - let tokenizerDataLoaders = []; + let dataLoaders = []; /** @type {number | null | undefined} */ let totalDataCount; @@ -68,7 +68,7 @@ $: { disableSearchBtn = false; - if (!selectedTokenizer || isSearching) { + if (!selectedAnalyzer || isSearching) { disableSearchBtn = true; } else if (!text || util.trim(text).length === 0) { disableSearchBtn = true; @@ -82,8 +82,8 @@ function initData() { isLoading = true; Promise.all([ - getTokenizerProviders(), - getTokenizerDataLoaderProviders() + getAnalyzerProviders(), + getDataLoaderProviders() ]).finally(() => { isLoading = false; }); @@ -102,7 +102,7 @@ elapsedTime = ''; const start = new Date(); - getTokenizeResult().finally(() => { + getAnalysisResult().finally(() => { isSearching = false; searchDone = true; const gap = new Date().getTime() - start.getTime(); @@ -137,45 +137,45 @@ selectedDataLoaders = []; } - function getTokenizerProviders() { + function getAnalyzerProviders() { return new Promise((resolve, reject) => { - getTokenizers().then(res => { + getEntityAnalyzers().then(res => { const retProviders = res?.map(x => ({ label: x, value: x })) || []; - tokenizers = [ ...retProviders ]; - selectedTokenizer = tokenizers[0]?.value; + analyzers = [ ...retProviders ]; + selectedAnalyzer = analyzers[0]?.value; resolve(res); }).catch(err => { - tokenizers = []; - selectedTokenizer = tokenizers[0]?.value; + analyzers = []; + selectedAnalyzer = null; reject(err); }); }); } - function getTokenizerDataLoaderProviders() { + function getDataLoaderProviders() { return new Promise((resolve, reject) => { - getTokenizerDataLoaders().then(res => { + getEntityDataLoaders().then(res => { const retProviders = res?.map(x => ({ label: x, value: x })) || []; - tokenizerDataLoaders = [ ...retProviders ]; + dataLoaders = [ ...retProviders ]; resolve(res); }).catch(err => { - tokenizerDataLoaders = []; + dataLoaders = []; reject(err); }); }); } - function getTokenizeResult() { + function getAnalysisResult() { return new Promise((resolve, reject) => { const request = { text: util.trim(text), - provider: selectedTokenizer, + provider: selectedAnalyzer, options: { data_providers: selectedDataLoaders?.length > 0 ? selectedDataLoaders : null } }; - tokenize(request).then(res => { + analyzeEntity(request).then(res => { items = res?.results || []; totalDataCount = items.length; resolve(res); @@ -188,13 +188,13 @@ } /** @param {any} e */ - function changeTokenizer(e) { + function changeAnalyzer(e) { const selectedValues = e?.detail?.selecteds || []; - selectedTokenizer = selectedValues[0]?.value; + selectedAnalyzer = selectedValues[0]?.value; } /** @param {any} e */ - function changeTokenizerDataLoaders(e) { + function changeDataLoaders(e) { const selectedValues = e?.detail?.selecteds || []; // @ts-ignore selectedDataLoaders = selectedValues.map(x => x.value); @@ -314,27 +314,27 @@
-
Tokenizer
+
Analyzer
changeTokenizerDataLoaders(e)} + options={dataLoaders} + on:select={e => changeDataLoaders(e)} />
diff --git a/src/routes/page/knowledge-base/documents/+page.svelte b/src/routes/page/knowledge-base/documents/+page.svelte index 9c164153..99881cc3 100644 --- a/src/routes/page/knowledge-base/documents/+page.svelte +++ b/src/routes/page/knowledge-base/documents/+page.svelte @@ -26,7 +26,7 @@ createVectorIndexes, deleteVectorIndexes, getVectorCollectionDetails, - getKnowledgeDocumentProcessors + getKnowledgeProcessors } from '$lib/services/knowledge-base-service'; import Breadcrumb from '$lib/common/Breadcrumb.svelte'; import HeadTitle from '$lib/common/HeadTitle.svelte'; @@ -464,7 +464,7 @@ function getDocProcessors() { return new Promise((resolve, reject) => { - getKnowledgeDocumentProcessors().then(res => { + getKnowledgeProcessors().then(res => { const retProcessors = res?.map(x => ({ label: x, value: x })) || []; docProcessors = [ ...retProcessors ]; resolve(res);