diff --git a/src/lib/common/Breadcrumb.svelte b/src/lib/common/Breadcrumb.svelte index 438bcc74..02cf6bf0 100644 --- a/src/lib/common/Breadcrumb.svelte +++ b/src/lib/common/Breadcrumb.svelte @@ -2,7 +2,7 @@ import { Row, Col, Breadcrumb, BreadcrumbItem } from '@sveltestrap/sveltestrap'; import Link from 'svelte-link'; - export let title = ''; + export let title = ''; export let pagetitle = ''; diff --git a/src/lib/helpers/http.js b/src/lib/helpers/http.js index 8e49d548..ab66e444 100644 --- a/src/lib/helpers/http.js +++ b/src/lib/helpers/http.js @@ -184,6 +184,8 @@ function skipLoader(config) { new RegExp('http(s*)://(.*?)/knowledge/(.*?)/search', 'g'), new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/create', 'g'), new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/page', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/document/(.*?)/upload', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/tokenize', 'g'), new RegExp('http(s*)://(.*?)/users', 'g'), new RegExp('http(s*)://(.*?)/instruct/(.*?)', 'g'), new RegExp('http(s*)://(.*?)/agent/(.*?)/code-scripts', 'g'), @@ -227,6 +229,9 @@ function skipLoader(config) { new RegExp('http(s*)://(.*?)/llm-provider/(.*?)/models', 'g'), new RegExp('http(s*)://(.*?)/knowledge/vector/collections', 'g'), new RegExp('http(s*)://(.*?)/knowledge/vector/(.*?)/exist', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/document/processors', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/tokenizer/providers', 'g'), + new RegExp('http(s*)://(.*?)/knowledge/tokenizer/data-providers', 'g'), new RegExp('http(s*)://(.*?)/logger/instruction/log', 'g'), new RegExp('http(s*)://(.*?)/logger/instruction/log/keys', 'g'), new RegExp('http(s*)://(.*?)/logger/conversation/(.*?)/content-log', 'g'), diff --git a/src/lib/helpers/types/commonTypes.js b/src/lib/helpers/types/commonTypes.js index a97df90b..4158299c 100644 --- a/src/lib/helpers/types/commonTypes.js +++ b/src/lib/helpers/types/commonTypes.js @@ -82,4 +82,10 @@ * @property {any[]} fail */ +/** + * @typedef {Object} TableItemConfig + * @property {string} dataName + * @property {string} displayName + */ + export default {}; \ No newline at end of file diff --git a/src/lib/helpers/types/knowledgeTypes.js b/src/lib/helpers/types/knowledgeTypes.js index 5d105db1..635ca3c3 100644 --- a/src/lib/helpers/types/knowledgeTypes.js +++ b/src/lib/helpers/types/knowledgeTypes.js @@ -98,6 +98,7 @@ /** * @typedef {Object} VectorKnowledgeUploadRequest * @property {import('$fileTypes').FileModel[]} files - The files. + * @property {any} [options] */ /** @@ -160,4 +161,33 @@ * @property {string} field_schema_type */ +/** + * @typedef {Object} TokenizeRequest + * @property {string} text + * @property {string?} [provider] + * @property {TokenizeOptions?} [options] + */ + +/** + * @typedef {Object} TokenizeOptions + * @property {string[]?} [data_providers] + * @property {number?} [max_ngram] + * @property {number?} [cutoff] + * @property {number?} [top_k] + */ + +/** + * @typedef {Object} TokenizeResponse + * @property {TokenizeResult[]} [results] + * @property {boolean?} [success] + * @property {string?} [error_message] + */ + +/** + * @typedef {Object} TokenizeResult + * @property {string} token + * @property {string?} [canonical_text] + * @property {any} data + */ + export default {}; \ No newline at end of file diff --git a/src/lib/helpers/utils/common.js b/src/lib/helpers/utils/common.js index 5a348daf..590f2ae5 100644 --- a/src/lib/helpers/utils/common.js +++ b/src/lib/helpers/utils/common.js @@ -161,11 +161,12 @@ export function scrollToBottom(container, behavior = 'smooth') { /** * @param {string} str + * @param {string} separator */ -export function splitTextByCase(str) { +export function splitTextByCase(str, separator = '_') { if (!str) return str; - let words = str.split("_"); + let words = str.split(separator); if (words.length === 1) { // split by camel case words = str.split(/(?=[A-Z])/); diff --git a/src/lib/scss/custom/pages/_knowledgebase.scss b/src/lib/scss/custom/pages/_knowledgebase.scss index 1c683c41..1184ef12 100644 --- a/src/lib/scss/custom/pages/_knowledgebase.scss +++ b/src/lib/scss/custom/pages/_knowledgebase.scss @@ -288,10 +288,11 @@ display: flex; gap: 5px; justify-content: flex-end; - min-width: 300px; + flex-wrap: wrap; .collection-dropdown { - width: 100%; + flex: 1 1 auto; + min-width: 150px; } .collection-action-btn { @@ -304,6 +305,17 @@ outline: none !important; box-shadow: none !important; } + + @media (max-width: 350px) { + flex-direction: column; + justify-content: center; + gap: 5px; + + .collection-dropdown { + width: 100%; + min-width: 100%; + } + } } .collection-add-container { @@ -351,6 +363,9 @@ border-color: var(--#{$prefix}light) !important; background-color: var(--#{$prefix}light) !important; position: relative; + overflow-y: auto; + scrollbar-width: none; + max-height: 300px; ul { margin: 15px 0px; @@ -385,15 +400,16 @@ .more-detail-list { margin: 1px 0px 15px 0px; - font-size: 10px; + font-size: 12px; padding-left: 4rem; - overflow-y: auto; - scrollbar-width: none; - max-height: 200px; - .more-dtail-item { + .more-detail-item { list-style-type: square; } + + ul { + font-size: 15px; + } } } } @@ -466,7 +482,7 @@ } .doc-upload-footer { - margin-top: 30px; + margin-top: 0px; .load-doc-btn { font-size: 12px; @@ -804,4 +820,25 @@ gap: 0; } } +} + + +.knowledge-upload-modal-container { + .doc-upload-body { + width: 100%; + height: 100%; + + .gallery-item, .doc-drop-zone { + width: 100%; + min-height: 200px; + } + + i { + font-size: 5em; + } + + .doc-drop-icon { + text-align: center; + } + } } \ No newline at end of file diff --git a/src/lib/services/api-endpoints.js b/src/lib/services/api-endpoints.js index a958f2d7..59331f46 100644 --- a/src/lib/services/api-endpoints.js +++ b/src/lib/services/api-endpoints.js @@ -113,6 +113,11 @@ export const endpoints = { knowledgeDocumentDeleteUrl: `${host}/knowledge/document/{collection}/delete/{fileId}`, knowledgeDocumentDeleteAllUrl: `${host}/knowledge/document/{collection}/delete`, knowledgeDocumentPageListUrl: `${host}/knowledge/document/{collection}/page`, + knowledgeDocumentProcessorsUrl: `${host}/knowledge/document/processors`, + + tokenizersUrl: `${host}/knowledge/tokenizer/providers`, + tokenizerDataLoadersUrl: `${host}/knowledge/tokenizer/data-providers`, + tokenizeUrl: `${host}/knowledge/tokenize`, // chathub chatHubUrl: `${host}/chatHub`, diff --git a/src/lib/services/knowledge-base-service.js b/src/lib/services/knowledge-base-service.js index 919d9214..91f5455c 100644 --- a/src/lib/services/knowledge-base-service.js +++ b/src/lib/services/knowledge-base-service.js @@ -193,6 +193,15 @@ export async function getKnowledgeDocumentPageList(collection, request) { return response.data; } +/** + * @returns {Promise} + */ +export async function getKnowledgeDocumentProcessors() { + const url = endpoints.knowledgeDocumentProcessorsUrl; + const response = await axios.get(url); + return response.data; +} + /** * @param {import('$knowledgeTypes').CreateVectorCollectionRequest} request @@ -274,4 +283,37 @@ export async function deleteVectorIndexes(collection, options) { } }); return response.data; +} + + + + +/** + * @returns {Promise} + */ +export async function getTokenizers() { + const url = endpoints.tokenizersUrl; + const response = await axios.get(url); + return response.data; +} + +/** + * @returns {Promise} + */ +export async function getTokenizerDataLoaders() { + const url = endpoints.tokenizerDataLoadersUrl; + const response = await axios.get(url); + return response.data; +} + +/** + * @param {import('$knowledgeTypes').TokenizeRequest} request + * @returns {Promise} + */ +export async function tokenize(request) { + const url = endpoints.tokenizeUrl; + const response = await axios.post(url, { + ...request + }); + return response.data; } \ No newline at end of file diff --git a/src/routes/page/knowledge-base/common/table/table-item.svelte b/src/routes/page/knowledge-base/common/table/table-item.svelte new file mode 100644 index 00000000..223f52a2 --- /dev/null +++ b/src/routes/page/knowledge-base/common/table/table-item.svelte @@ -0,0 +1,119 @@ + + +{#if isLoading} + +{/if} + + + {#if columns?.length > 0} + {#each columns as column, idx (idx)} + +
{item[column.dataName] || ''}
+ + {/each} + {/if} + +
    + {#if detailKey} +
  • + +
  • + {/if} +
+ + + +{#if open} + + +
+
    + {#each columns as column, idx (idx)} +
  • +
    + {column.displayName || column.dataName} +
    +
    + {item[column.dataName] || ''} +
    +
  • + {/each} +
+
+ +
+ {#if loadMore} +
    + {#if detailKey && item[detailKey]} + {#if useJsonDisplay} + + {:else} + {#each Object.keys(item[detailKey]) as key, idx (idx)} +
  • + {splitTextByCase(key)}: + {item[detailKey][key]} +
  • + {/each} + {/if} + {/if} +
+ {/if} +
+ + +{/if} \ No newline at end of file diff --git a/src/routes/page/knowledge-base/dictionary/+page.svelte b/src/routes/page/knowledge-base/dictionary/+page.svelte new file mode 100644 index 00000000..94065f53 --- /dev/null +++ b/src/routes/page/knowledge-base/dictionary/+page.svelte @@ -0,0 +1,374 @@ + + + + + + + +
+
+ +
+ +
+ +
+
+ +
+
+ {#if showDemo} +
+
+