From c5935cf5ce50062339ecdad5ed4d96a254d53aba Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 12 Dec 2025 16:53:26 -0600 Subject: [PATCH 1/3] add knowledge base dictionary --- src/lib/common/Breadcrumb.svelte | 2 +- src/lib/helpers/http.js | 3 + src/lib/helpers/types/commonTypes.js | 6 + src/lib/helpers/types/knowledgeTypes.js | 29 ++ src/lib/helpers/utils/common.js | 5 +- src/lib/scss/custom/pages/_knowledgebase.scss | 24 +- src/lib/services/api-endpoints.js | 4 + src/lib/services/knowledge-base-service.js | 33 ++ .../common/table/table-item.svelte | 119 ++++++ .../knowledge-base/dictionary/+page.svelte | 374 ++++++++++++++++++ .../knowledge-base/documents/+page.svelte | 4 +- .../question-answer/+page.svelte | 4 +- svelte.config.js | 3 +- 13 files changed, 598 insertions(+), 12 deletions(-) create mode 100644 src/routes/page/knowledge-base/common/table/table-item.svelte create mode 100644 src/routes/page/knowledge-base/dictionary/+page.svelte diff --git a/src/lib/common/Breadcrumb.svelte b/src/lib/common/Breadcrumb.svelte index 438bcc74..02cf6bf0 100644 --- a/src/lib/common/Breadcrumb.svelte +++ b/src/lib/common/Breadcrumb.svelte @@ -2,7 +2,7 @@ import { Row, Col, Breadcrumb, BreadcrumbItem } from '@sveltestrap/sveltestrap'; import Link from 'svelte-link'; - export let title = ''; + export let title = ''; export let pagetitle = ''; diff --git a/src/lib/helpers/http.js b/src/lib/helpers/http.js index 8e49d548..56d7e19f 100644 --- a/src/lib/helpers/http.js +++ b/src/lib/helpers/http.js @@ -184,6 +184,7 @@ function skipLoader(config) { new RegExp('http(s*)://(.*?)/knowledge/(.*?)/search', 'g'), new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/create', 'g'), new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/page', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/tokenize', 'g'), new RegExp('http(s*)://(.*?)/users', 'g'), new RegExp('http(s*)://(.*?)/instruct/(.*?)', 'g'), new RegExp('http(s*)://(.*?)/agent/(.*?)/code-scripts', 'g'), @@ -227,6 +228,8 @@ function skipLoader(config) { new RegExp('http(s*)://(.*?)/llm-provider/(.*?)/models', 'g'), new RegExp('http(s*)://(.*?)/knowledge/vector/collections', 'g'), new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/exist', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/tokenizer/providers', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/tokenizer/data-providers', 'g'), new RegExp('http(s*)://(.*?)/logger/instruction/log', 'g'), new RegExp('http(s*)://(.*?)/logger/instruction/log/keys', 'g'), new RegExp('http(s*)://(.*?)/logger/conversation/(.*?)/content-log', 'g'), diff --git a/src/lib/helpers/types/commonTypes.js b/src/lib/helpers/types/commonTypes.js index a97df90b..4158299c 100644 --- a/src/lib/helpers/types/commonTypes.js +++ b/src/lib/helpers/types/commonTypes.js @@ -82,4 +82,10 @@ * @property {any[]} fail */ +/** + * @typedef {Object} TableItemConfig + * @property {string} dataName + * @property {string} displayName + */ + export default {}; \ No newline at end of file diff --git a/src/lib/helpers/types/knowledgeTypes.js b/src/lib/helpers/types/knowledgeTypes.js index 5d105db1..ead3e86e 100644 --- a/src/lib/helpers/types/knowledgeTypes.js +++ b/src/lib/helpers/types/knowledgeTypes.js @@ -160,4 +160,33 @@ * @property {string} field_schema_type */ +/** + * @typedef {Object} TokenizeRequest + * @property {string} text + * @property {string?} [provider] + * @property {TokenizeOptions?} [options] + */ + +/** + * @typedef {Object} TokenizeOptions + * @property {string[]?} [data_providers] + * @property {number?} [max_ngram] + * @property {number?} [cutoff] + * @property {number?} [top_k] + */ + +/** + * @typedef {Object} TokenizeResponse + * @property {TokenizeResult[]} [results] + * @property {boolean?} [success] + * @property {string?} [error_message] + */ + +/** + * @typedef {Object} TokenizeResult + * @property {string} token + * @property {string?} [canonical_text] + * @property {any} data + */ + export default {}; \ No newline at end of file diff --git a/src/lib/helpers/utils/common.js b/src/lib/helpers/utils/common.js index 5a348daf..590f2ae5 100644 --- a/src/lib/helpers/utils/common.js +++ b/src/lib/helpers/utils/common.js @@ -161,11 +161,12 @@ export function scrollToBottom(container, behavior = 'smooth') { /** * @param {string} str + * @param {string} separator */ -export function splitTextByCase(str) { +export function splitTextByCase(str, separator = '_') { if (!str) return str; - let words = str.split("_"); + let words = str.split(separator); if (words.length === 1) { // split by camel case words = str.split(/(?=[A-Z])/); diff --git a/src/lib/scss/custom/pages/_knowledgebase.scss b/src/lib/scss/custom/pages/_knowledgebase.scss index 1c683c41..4cc25e6d 100644 --- a/src/lib/scss/custom/pages/_knowledgebase.scss +++ b/src/lib/scss/custom/pages/_knowledgebase.scss @@ -288,10 +288,11 @@ display: flex; gap: 5px; justify-content: flex-end; - min-width: 300px; + flex-wrap: wrap; .collection-dropdown { - width: 100%; + flex: 1 1 auto; + min-width: 150px; } .collection-action-btn { @@ -304,6 +305,17 @@ outline: none !important; box-shadow: none !important; } + + @media (max-width: 350px) { + flex-direction: column; + justify-content: center; + gap: 5px; + + .collection-dropdown { + width: 100%; + min-width: 100%; + } + } } .collection-add-container { @@ -385,15 +397,19 @@ .more-detail-list { margin: 1px 0px 15px 0px; - font-size: 10px; + font-size: 12px; padding-left: 4rem; overflow-y: auto; scrollbar-width: none; max-height: 200px; - .more-dtail-item { + .more-detail-item { list-style-type: square; } + + ul { + font-size: 15px; + } } } } diff --git a/src/lib/services/api-endpoints.js b/src/lib/services/api-endpoints.js index a958f2d7..ad5c635a 100644 --- a/src/lib/services/api-endpoints.js +++ b/src/lib/services/api-endpoints.js @@ -114,6 +114,10 @@ export const endpoints = { knowledgeDocumentDeleteAllUrl: `${host}/knowledge/document/{collection}/delete`, knowledgeDocumentPageListUrl: `${host}/knowledge/document/{collection}/page`, + tokenizersUrl: `${host}/knowledge/tokenizer/providers`, + tokenizerDataLoadersUrl: `${host}/knowledge/tokenizer/data-providers`, + tokenizeUrl: `${host}/knowledge/tokenize`, + // chathub chatHubUrl: `${host}/chatHub`, diff --git a/src/lib/services/knowledge-base-service.js b/src/lib/services/knowledge-base-service.js index 919d9214..2c2b721d 100644 --- a/src/lib/services/knowledge-base-service.js +++ b/src/lib/services/knowledge-base-service.js @@ -274,4 +274,37 @@ export async function deleteVectorIndexes(collection, options) { } }); return response.data; +} + + + + +/** + * @returns {Promise} + */ +export async function getTokenizers() { + const url = endpoints.tokenizersUrl; + const response = await axios.get(url); + return response.data; +} + +/** + * @returns {Promise} + */ +export async function getTokenizerDataLoaders() { + const url = endpoints.tokenizerDataLoadersUrl; + const response = await axios.get(url); + return response.data; +} + +/** + * @param {import('$knowledgeTypes').TokenizeRequest} request + * @returns {Promise} + */ +export async function tokenize(request) { + const url = endpoints.tokenizeUrl; + const response = await axios.post(url, { + ...request + }); + return response.data; } \ No newline at end of file diff --git a/src/routes/page/knowledge-base/common/table/table-item.svelte b/src/routes/page/knowledge-base/common/table/table-item.svelte new file mode 100644 index 00000000..223f52a2 --- /dev/null +++ b/src/routes/page/knowledge-base/common/table/table-item.svelte @@ -0,0 +1,119 @@ + + +{#if isLoading} + +{/if} + + + {#if columns?.length > 0} + {#each columns as column, idx (idx)} + +
{item[column.dataName] || ''}
+ + {/each} + {/if} + +
    + {#if detailKey} +
  • + +
  • + {/if} +
+ + + +{#if open} + + +
+
    + {#each columns as column, idx (idx)} +
  • +
    + {column.displayName || column.dataName} +
    +
    + {item[column.dataName] || ''} +
    +
  • + {/each} +
+
+ +
+ {#if loadMore} +
    + {#if detailKey && item[detailKey]} + {#if useJsonDisplay} + + {:else} + {#each Object.keys(item[detailKey]) as key, idx (idx)} +
  • + {splitTextByCase(key)}: + {item[detailKey][key]} +
  • + {/each} + {/if} + {/if} +
+ {/if} +
+ + +{/if} \ No newline at end of file diff --git a/src/routes/page/knowledge-base/dictionary/+page.svelte b/src/routes/page/knowledge-base/dictionary/+page.svelte new file mode 100644 index 00000000..94065f53 --- /dev/null +++ b/src/routes/page/knowledge-base/dictionary/+page.svelte @@ -0,0 +1,374 @@ + + + + + + + +
+
+ +
+ +
+ +
+
+ +
+
+ {#if showDemo} +
+
+