- {Object.keys(segmentationRuleMap).map((field) => {
- return (
- = ({ datasetId, batchId, documents = [], indexingType, retrievalMethod }) => {
+// Action buttons component
+const ActionButtons: FC<{
+ apiReferenceUrl: string
+ onNavToDocuments: () => void
+}> = ({ apiReferenceUrl, onNavToDocuments }) => {
const { t } = useTranslation()
+
+ return (
+ = ({
+ datasetId,
+ batchId,
+ documents = [],
+ indexingType,
+ retrievalMethod,
+}) => {
const { enableBilling, plan } = useProviderContext()
-
- const getFirstDocument = documents[0]
-
- const [indexingStatusBatchDetail, setIndexingStatusDetail] = useState([])
- const fetchIndexingStatus = async () => {
- const status = await doFetchIndexingStatus({ datasetId, batchId })
- setIndexingStatusDetail(status.data)
- return status.data
- }
-
- const [isStopQuery, setIsStopQuery] = useState(false)
- const isStopQueryRef = useRef(isStopQuery)
- useEffect(() => {
- isStopQueryRef.current = isStopQuery
- }, [isStopQuery])
- const stopQueryStatus = () => {
- setIsStopQuery(true)
- }
-
- const startQueryStatus = async () => {
- if (isStopQueryRef.current)
- return
-
- try {
- const indexingStatusBatchDetail = await fetchIndexingStatus()
- const isCompleted = indexingStatusBatchDetail.every(indexingStatusDetail => ['completed', 'error', 'paused'].includes(indexingStatusDetail.indexing_status))
- if (isCompleted) {
- stopQueryStatus()
- return
- }
- await sleep(2500)
- await startQueryStatus()
- }
- catch {
- await sleep(2500)
- await startQueryStatus()
- }
- }
-
- useEffect(() => {
- setIsStopQuery(false)
- startQueryStatus()
- return () => {
- stopQueryStatus()
- }
- }, [])
-
- // get rule
- const { data: ruleDetail } = useProcessRule(getFirstDocument?.id)
-
const router = useRouter()
const invalidDocumentList = useInvalidDocumentList()
- const navToDocumentList = () => {
+ const apiReferenceUrl = useDatasetApiAccessUrl()
+
+ // Polling hook for indexing status
+ const { statusList, isEmbedding, isEmbeddingCompleted } = useIndexingStatusPolling({
+ datasetId,
+ batchId,
+ })
+
+ // Get process rule for the first document
+ const firstDocumentId = documents[0]?.id
+ const { data: ruleDetail } = useProcessRule(firstDocumentId)
+
+ // Document lookup utilities - memoized for performance
+ const documentLookup = useMemo(
+ () => createDocumentLookup(documents),
+ [documents],
+ )
+
+ const handleNavToDocuments = () => {
invalidDocumentList()
router.push(`/datasets/${datasetId}/documents`)
}
- const apiReferenceUrl = useDatasetApiAccessUrl()
- const isEmbedding = useMemo(() => {
- return indexingStatusBatchDetail.some(indexingStatusDetail => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || ''))
- }, [indexingStatusBatchDetail])
- const isEmbeddingCompleted = useMemo(() => {
- return indexingStatusBatchDetail.every(indexingStatusDetail => ['completed', 'error', 'paused'].includes(indexingStatusDetail?.indexing_status || ''))
- }, [indexingStatusBatchDetail])
-
- const getSourceName = (id: string) => {
- const doc = documents.find(document => document.id === id)
- return doc?.name
- }
- const getFileType = (name?: string) => name?.split('.').pop() || 'txt'
- const getSourcePercent = (detail: IndexingStatusResponse) => {
- const completedCount = detail.completed_segments || 0
- const totalCount = detail.total_segments || 0
- if (totalCount === 0)
- return 0
- const percent = Math.round(completedCount * 100 / totalCount)
- return percent > 100 ? 100 : percent
- }
- const getSourceType = (id: string) => {
- const doc = documents.find(document => document.id === id)
- return doc?.data_source_type as DataSourceType
- }
-
- const isLegacyDataSourceInfo = (info: DataSourceInfo): info is LegacyDataSourceInfo => {
- return info != null && typeof (info as LegacyDataSourceInfo).upload_file === 'object'
- }
-
- const getIcon = (id: string) => {
- const doc = documents.find(document => document.id === id)
- const info = doc?.data_source_info
- if (info && isLegacyDataSourceInfo(info))
- return info.notion_page_icon
- return undefined
- }
- const isSourceEmbedding = (detail: IndexingStatusResponse) =>
- ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'].includes(detail.indexing_status || '')
+ const showUpgradeBanner = enableBilling && plan.type !== Plan.team
return (
<>
+
+
+ )
+ }
+
+ return null
+}
+
+// Source type icon component
+const SourceTypeIcon: FC<{
+ sourceType?: DataSourceType
+ name?: string
+ notionIcon?: string
+}> = ({ sourceType, name, notionIcon }) => {
+ if (sourceType === DataSourceType.FILE) {
+ return (
+ = ({
+ detail,
+ name,
+ sourceType,
+ notionIcon,
+ enableBilling,
+}) => {
+ const isEmbedding = isSourceEmbedding(detail)
+ const percent = getSourcePercent(detail)
+ const isError = detail.indexing_status === 'error'
+
+ return (
+ > = {
+ [RETRIEVE_METHOD.fullText]: retrievalIcon.fullText,
+ [RETRIEVE_METHOD.hybrid]: retrievalIcon.hybrid,
+ [RETRIEVE_METHOD.semantic]: retrievalIcon.vector,
+ [RETRIEVE_METHOD.invertedIndex]: retrievalIcon.fullText,
+ [RETRIEVE_METHOD.keywordSearch]: retrievalIcon.fullText,
+}
+
+const isNumber = (value: unknown): value is number => typeof value === 'number'
+
+const RuleDetail: FC = ({ sourceData, indexingType, retrievalMethod }) => {
+ const { t } = useTranslation()
+
+ const segmentationRuleLabels = {
+ mode: t('embedding.mode', { ns: 'datasetDocuments' }),
+ segmentLength: t('embedding.segmentLength', { ns: 'datasetDocuments' }),
+ textCleaning: t('embedding.textCleaning', { ns: 'datasetDocuments' }),
+ }
+
+ const getRuleName = useCallback((key: string): string | undefined => {
+ const translationKey = PRE_PROCESSING_RULE_KEYS[key as keyof typeof PRE_PROCESSING_RULE_KEYS]
+ return translationKey ? t(translationKey, { ns: 'datasetCreation' }) : undefined
+ }, [t])
+
+ const getModeValue = useCallback((): string => {
+ if (!sourceData?.mode)
+ return '-'
+
+ if (sourceData.mode === ProcessMode.general)
+ return t('embedding.custom', { ns: 'datasetDocuments' })
+
+ const parentModeLabel = sourceData.rules?.parent_mode === 'paragraph'
+ ? t('parentMode.paragraph', { ns: 'dataset' })
+ : t('parentMode.fullDoc', { ns: 'dataset' })
+
+ return `${t('embedding.hierarchical', { ns: 'datasetDocuments' })} · ${parentModeLabel}`
+ }, [sourceData, t])
+
+ const getSegmentLengthValue = useCallback((): string | number => {
+ if (!sourceData?.mode)
+ return '-'
+
+ const maxTokens = isNumber(sourceData.rules?.segmentation?.max_tokens)
+ ? sourceData.rules.segmentation.max_tokens
+ : '-'
+
+ if (sourceData.mode === ProcessMode.general)
+ return maxTokens
+
+ const childMaxTokens = isNumber(sourceData.rules?.subchunk_segmentation?.max_tokens)
+ ? sourceData.rules.subchunk_segmentation.max_tokens
+ : '-'
+
+ return `${t('embedding.parentMaxTokens', { ns: 'datasetDocuments' })} ${maxTokens}; ${t('embedding.childMaxTokens', { ns: 'datasetDocuments' })} ${childMaxTokens}`
+ }, [sourceData, t])
+
+ const getTextCleaningValue = useCallback((): string => {
+ if (!sourceData?.mode)
+ return '-'
+
+ const enabledRules = sourceData.rules?.pre_processing_rules?.filter(rule => rule.enabled) || []
+ const ruleNames = enabledRules
+ .map((rule) => {
+ const name = getRuleName(rule.id)
+ return typeof name === 'string' ? name : ''
+ })
+ .filter(name => name)
+ return ruleNames.length > 0 ? ruleNames.join(',') : '-'
+ }, [sourceData, getRuleName])
+
+ const fieldValueGetters: Record string | number> = {
+ mode: getModeValue,
+ segmentLength: getSegmentLengthValue,
+ textCleaning: getTextCleaningValue,
+ }
+
+ const isEconomical = indexingType === IndexingType.ECONOMICAL
+ const indexMethodIconSrc = isEconomical ? indexMethodIcon.economical : indexMethodIcon.high_quality
+ const indexModeLabel = t(`stepTwo.${isEconomical ? 'economical' : 'qualified'}`, { ns: 'datasetCreation' })
+
+ const effectiveRetrievalMethod = isEconomical ? 'keyword_search' : (retrievalMethod ?? 'semantic_search')
+ const retrievalLabel = t(`retrieval.${effectiveRetrievalMethod}.title`, { ns: 'dataset' })
+ const retrievalIconSrc = RETRIEVAL_ICON_MAP[retrievalMethod as keyof typeof RETRIEVAL_ICON_MAP] ?? retrievalIcon.vector
+
+ return (
+ ([])
+ const isStopPollingRef = useRef(false)
+
+ useEffect(() => {
+ // Reset polling state on mount
+ isStopPollingRef.current = false
+ let timeoutId: ReturnType | null = null
+
+ const fetchStatus = async (): Promise => {
+ const response = await fetchIndexingStatusBatch({ datasetId, batchId })
+ setStatusList(response.data)
+ return response.data
+ }
+
+ const poll = async (): Promise => {
+ if (isStopPollingRef.current)
+ return
+
+ try {
+ const data = await fetchStatus()
+ if (isAllCompleted(data)) {
+ isStopPollingRef.current = true
+ return
+ }
+ }
+ catch {
+ // Continue polling on error
+ }
+
+ if (!isStopPollingRef.current) {
+ timeoutId = setTimeout(() => {
+ poll()
+ }, POLLING_INTERVAL)
+ }
+ }
+
+ poll()
+
+ return () => {
+ isStopPollingRef.current = true
+ if (timeoutId)
+ clearTimeout(timeoutId)
+ }
+ }, [datasetId, batchId])
+
+ const isEmbedding = statusList.some(item =>
+ EMBEDDING_STATUSES.includes(item?.indexing_status as typeof EMBEDDING_STATUSES[number]),
+ )
+
+ const isEmbeddingCompleted = statusList.length > 0 && isAllCompleted(statusList)
+
+ return {
+ statusList,
+ isEmbedding,
+ isEmbeddingCompleted,
+ }
+}
diff --git a/web/app/components/datasets/create/embedding-process/utils.ts b/web/app/components/datasets/create/embedding-process/utils.ts
new file mode 100644
index 0000000000..6fbefb0230
--- /dev/null
+++ b/web/app/components/datasets/create/embedding-process/utils.ts
@@ -0,0 +1,64 @@
+import type {
+ DataSourceInfo,
+ DataSourceType,
+ FullDocumentDetail,
+ IndexingStatusResponse,
+ LegacyDataSourceInfo,
+} from '@/models/datasets'
+
+const EMBEDDING_STATUSES = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const
+
+/**
+ * Type guard for legacy data source info with upload_file property
+ */
+export const isLegacyDataSourceInfo = (info: DataSourceInfo): info is LegacyDataSourceInfo => {
+ return info != null && typeof (info as LegacyDataSourceInfo).upload_file === 'object'
+}
+
+/**
+ * Check if a status indicates the source is being embedded
+ */
+export const isSourceEmbedding = (detail: IndexingStatusResponse): boolean =>
+ EMBEDDING_STATUSES.includes(detail.indexing_status as typeof EMBEDDING_STATUSES[number])
+
+/**
+ * Calculate the progress percentage for a document
+ */
+export const getSourcePercent = (detail: IndexingStatusResponse): number => {
+ const completedCount = detail.completed_segments || 0
+ const totalCount = detail.total_segments || 0
+
+ if (totalCount === 0)
+ return 0
+
+ const percent = Math.round(completedCount * 100 / totalCount)
+ return Math.min(percent, 100)
+}
+
+/**
+ * Get file extension from filename, defaults to 'txt'
+ */
+export const getFileType = (name?: string): string =>
+ name?.split('.').pop() || 'txt'
+
+/**
+ * Document lookup utilities - provides document info by ID from a list
+ */
+export const createDocumentLookup = (documents: FullDocumentDetail[]) => {
+ const documentMap = new Map(documents.map(doc => [doc.id, doc]))
+
+ return {
+ getDocument: (id: string) => documentMap.get(id),
+
+ getName: (id: string) => documentMap.get(id)?.name,
+
+ getSourceType: (id: string) => documentMap.get(id)?.data_source_type as DataSourceType | undefined,
+
+ getNotionIcon: (id: string) => {
+ const info = documentMap.get(id)?.data_source_info
+ if (info && isLegacyDataSourceInfo(info))
+ return info.notion_page_icon
+ return undefined
+ },
+ }
+}
diff --git a/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx b/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx
new file mode 100644
index 0000000000..5140c902f5
--- /dev/null
+++ b/web/app/components/datasets/create/step-two/components/general-chunking-options.tsx
@@ -0,0 +1,199 @@
+'use client'
+
+import type { FC } from 'react'
+import type { PreProcessingRule } from '@/models/datasets'
+import {
+ RiAlertFill,
+ RiSearchEyeLine,
+} from '@remixicon/react'
+import Image from 'next/image'
+import { useTranslation } from 'react-i18next'
+import Button from '@/app/components/base/button'
+import Checkbox from '@/app/components/base/checkbox'
+import Divider from '@/app/components/base/divider'
+import Tooltip from '@/app/components/base/tooltip'
+import { IS_CE_EDITION } from '@/config'
+import { ChunkingMode } from '@/models/datasets'
+import SettingCog from '../../assets/setting-gear-mod.svg'
+import s from '../index.module.css'
+import LanguageSelect from '../language-select'
+import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs'
+import { OptionCard } from './option-card'
+
+type TextLabelProps = {
+ children: React.ReactNode
+}
+
+const TextLabel: FC = ({ children }) => {
+ return
+}
+
+type GeneralChunkingOptionsProps = {
+ // State
+ segmentIdentifier: string
+ maxChunkLength: number
+ overlap: number
+ rules: PreProcessingRule[]
+ currentDocForm: ChunkingMode
+ docLanguage: string
+ // Flags
+ isActive: boolean
+ isInUpload: boolean
+ isNotUploadInEmptyDataset: boolean
+ hasCurrentDatasetDocForm: boolean
+ // Actions
+ onSegmentIdentifierChange: (value: string) => void
+ onMaxChunkLengthChange: (value: number) => void
+ onOverlapChange: (value: number) => void
+ onRuleToggle: (id: string) => void
+ onDocFormChange: (form: ChunkingMode) => void
+ onDocLanguageChange: (lang: string) => void
+ onPreview: () => void
+ onReset: () => void
+ // Locale
+ locale: string
+}
+
+export const GeneralChunkingOptions: FC = ({
+ segmentIdentifier,
+ maxChunkLength,
+ overlap,
+ rules,
+ currentDocForm,
+ docLanguage,
+ isActive,
+ isInUpload,
+ isNotUploadInEmptyDataset,
+ hasCurrentDatasetDocForm,
+ onSegmentIdentifierChange,
+ onMaxChunkLengthChange,
+ onOverlapChange,
+ onRuleToggle,
+ onDocFormChange,
+ onDocLanguageChange,
+ onPreview,
+ onReset,
+ locale,
+}) => {
+ const { t } = useTranslation()
+
+ const getRuleName = (key: string): string => {
+ const ruleNameMap: Record = {
+ remove_extra_spaces: t('stepTwo.removeExtraSpaces', { ns: 'datasetCreation' }),
+ remove_urls_emails: t('stepTwo.removeUrlEmails', { ns: 'datasetCreation' }),
+ remove_stopwords: t('stepTwo.removeStopwords', { ns: 'datasetCreation' }),
+ }
+ return ruleNameMap[key] ?? key
+ }
+
+ return (
+ = ({
+ indexType,
+ hasSetIndexType,
+ docForm,
+ embeddingModel,
+ embeddingModelList,
+ retrievalConfig,
+ showMultiModalTip,
+ isModelAndRetrievalConfigDisabled,
+ datasetId,
+ isQAConfirmDialogOpen,
+ onIndexTypeChange,
+ onEmbeddingModelChange,
+ onRetrievalConfigChange,
+ onQAConfirmDialogClose,
+ onQAConfirmDialogConfirm,
+}) => {
+ const { t } = useTranslation()
+ const docLink = useDocLink()
+
+ const getIndexingTechnique = () => indexType
+
+ return (
+ <>
+ {/* Index Mode */}
+
+
+
+
+
+ {docForm === ChunkingMode.qa
+ ? t('stepTwo.notAvailableForQA', { ns: 'datasetCreation' })
+ : t('stepTwo.notAvailableForParentChild', { ns: 'datasetCreation' })}
+
+ )}
+ noDecoration
+ position="top"
+ asChild={false}
+ triggerClassName="flex-1 self-stretch"
+ >
+
+ {isEmbedding && (
+ <>
+
)
}
-const EmbeddingProcess: FC
+
+
+
+
+
+ )
+}
+
+const EmbeddingProcess: FC
-
-
- {isEmbedding && (
- <>
-
- {
- enableBilling && plan.type !== Plan.team && (
-
-
- )
- }
+
-
-
- {t('plansCommon.documentProcessingPriorityUpgrade', { ns: 'billing' })}
-
-
- {indexingStatusBatchDetail.map(indexingStatusDetail => (
-
+
- {isSourceEmbedding(indexingStatusDetail) && (
-
- )}
-
+ {statusList.map(detail => (
+
- {getSourceType(indexingStatusDetail.id) === DataSourceType.FILE && (
-
-
-
- )}
- {indexingStatusDetail.indexing_status === 'completed' && (
-
-
-
- {isSourceEmbedding(indexingStatusDetail) && (
-
- {getSourceName(indexingStatusDetail.id)}
-
- {
- enableBilling && (
- {`${getSourcePercent(indexingStatusDetail)}%`}
- )}
- {indexingStatusDetail.indexing_status === 'error' && (
-
-
-
-
-
-
+
+
+ {isEmbedding && (
+
+ )}
+
+ )
+}
+
+export default IndexingProgressItem
diff --git a/web/app/components/datasets/create/embedding-process/rule-detail.tsx b/web/app/components/datasets/create/embedding-process/rule-detail.tsx
new file mode 100644
index 0000000000..dff35100cb
--- /dev/null
+++ b/web/app/components/datasets/create/embedding-process/rule-detail.tsx
@@ -0,0 +1,133 @@
+import type { FC } from 'react'
+import type { ProcessRuleResponse } from '@/models/datasets'
+import Image from 'next/image'
+import { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata'
+import { ProcessMode } from '@/models/datasets'
+import { RETRIEVE_METHOD } from '@/types/app'
+import { indexMethodIcon, retrievalIcon } from '../icons'
+import { IndexingType } from '../step-two'
+
+type RuleDetailProps = {
+ sourceData?: ProcessRuleResponse
+ indexingType?: string
+ retrievalMethod?: RETRIEVE_METHOD
+}
+
+// Lookup table for pre-processing rule names
+const PRE_PROCESSING_RULE_KEYS = {
+ remove_extra_spaces: 'stepTwo.removeExtraSpaces',
+ remove_urls_emails: 'stepTwo.removeUrlEmails',
+ remove_stopwords: 'stepTwo.removeStopwords',
+} as const
+
+// Lookup table for retrieval method icons
+const RETRIEVAL_ICON_MAP: Partial
+
+
+
+ {isEmbedding && (
+
+ {name}
+
+ {enableBilling && {`${percent}%`}
+ )}
+
+ {Object.keys(segmentationRuleLabels).map(field => (
+
+ )
+}
+
+export default RuleDetail
diff --git a/web/app/components/datasets/create/embedding-process/upgrade-banner.tsx b/web/app/components/datasets/create/embedding-process/upgrade-banner.tsx
new file mode 100644
index 0000000000..49e5fe99a1
--- /dev/null
+++ b/web/app/components/datasets/create/embedding-process/upgrade-banner.tsx
@@ -0,0 +1,22 @@
+import type { FC } from 'react'
+import { useTranslation } from 'react-i18next'
+import { ZapFast } from '@/app/components/base/icons/src/vender/solid/general'
+import UpgradeBtn from '@/app/components/billing/upgrade-btn'
+
+const UpgradeBanner: FC = () => {
+ const { t } = useTranslation()
+
+ return (
+
+
+ )
+}
+
+export default UpgradeBanner
diff --git a/web/app/components/datasets/create/embedding-process/use-indexing-status-polling.ts b/web/app/components/datasets/create/embedding-process/use-indexing-status-polling.ts
new file mode 100644
index 0000000000..f8e69e47af
--- /dev/null
+++ b/web/app/components/datasets/create/embedding-process/use-indexing-status-polling.ts
@@ -0,0 +1,90 @@
+import type { IndexingStatusResponse } from '@/models/datasets'
+import { useEffect, useRef, useState } from 'react'
+import { fetchIndexingStatusBatch } from '@/service/datasets'
+
+const POLLING_INTERVAL = 2500
+const COMPLETED_STATUSES = ['completed', 'error', 'paused'] as const
+const EMBEDDING_STATUSES = ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'] as const
+
+type IndexingStatusPollingParams = {
+ datasetId: string
+ batchId: string
+}
+
+type IndexingStatusPollingResult = {
+ statusList: IndexingStatusResponse[]
+ isEmbedding: boolean
+ isEmbeddingCompleted: boolean
+}
+
+const isStatusCompleted = (status: string): boolean =>
+ COMPLETED_STATUSES.includes(status as typeof COMPLETED_STATUSES[number])
+
+const isAllCompleted = (statusList: IndexingStatusResponse[]): boolean =>
+ statusList.every(item => isStatusCompleted(item.indexing_status))
+
+/**
+ * Custom hook for polling indexing status with automatic stop on completion.
+ * Handles the polling lifecycle and provides derived states for UI rendering.
+ */
+export const useIndexingStatusPolling = ({
+ datasetId,
+ batchId,
+}: IndexingStatusPollingParams): IndexingStatusPollingResult => {
+ const [statusList, setStatusList] = useState
+
+
+ {t('plansCommon.documentProcessingPriorityUpgrade', { ns: 'billing' })}
+
+
+
+
+ )
+}
diff --git a/web/app/components/datasets/create/step-two/components/index.ts b/web/app/components/datasets/create/step-two/components/index.ts
new file mode 100644
index 0000000000..d5382e0c4b
--- /dev/null
+++ b/web/app/components/datasets/create/step-two/components/index.ts
@@ -0,0 +1,5 @@
+export { GeneralChunkingOptions } from './general-chunking-options'
+export { IndexingModeSection } from './indexing-mode-section'
+export { ParentChildOptions } from './parent-child-options'
+export { PreviewPanel } from './preview-panel'
+export { StepTwoFooter } from './step-two-footer'
diff --git a/web/app/components/datasets/create/step-two/components/indexing-mode-section.tsx b/web/app/components/datasets/create/step-two/components/indexing-mode-section.tsx
new file mode 100644
index 0000000000..ee49f42903
--- /dev/null
+++ b/web/app/components/datasets/create/step-two/components/indexing-mode-section.tsx
@@ -0,0 +1,253 @@
+'use client'
+
+import type { FC } from 'react'
+import type { DefaultModel, Model } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import type { RetrievalConfig } from '@/types/app'
+import Image from 'next/image'
+import Link from 'next/link'
+import { useTranslation } from 'react-i18next'
+import Badge from '@/app/components/base/badge'
+import Button from '@/app/components/base/button'
+import CustomDialog from '@/app/components/base/dialog'
+import Divider from '@/app/components/base/divider'
+import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
+import Tooltip from '@/app/components/base/tooltip'
+import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config'
+import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config'
+import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
+import { useDocLink } from '@/context/i18n'
+import { ChunkingMode } from '@/models/datasets'
+import { cn } from '@/utils/classnames'
+import { indexMethodIcon } from '../../icons'
+import { IndexingType } from '../hooks'
+import s from '../index.module.css'
+import { OptionCard } from './option-card'
+
+type IndexingModeSectionProps = {
+ // State
+ indexType: IndexingType
+ hasSetIndexType: boolean
+ docForm: ChunkingMode
+ embeddingModel: DefaultModel
+ embeddingModelList?: Model[]
+ retrievalConfig: RetrievalConfig
+ showMultiModalTip: boolean
+ // Flags
+ isModelAndRetrievalConfigDisabled: boolean
+ datasetId?: string
+ // Modal state
+ isQAConfirmDialogOpen: boolean
+ // Actions
+ onIndexTypeChange: (type: IndexingType) => void
+ onEmbeddingModelChange: (model: DefaultModel) => void
+ onRetrievalConfigChange: (config: RetrievalConfig) => void
+ onQAConfirmDialogClose: () => void
+ onQAConfirmDialogConfirm: () => void
+}
+
+export const IndexingModeSection: FC
+ onSegmentIdentifierChange(e.target.value)}
+ />
+
+
+
+
+
+
+ {t('stepTwo.rules', { ns: 'datasetCreation' })}
+
+
+ {rules.map(rule => (
+
+ onRuleToggle(rule.id)}
+ >
+
+ ))}
+ {IS_CE_EDITION && (
+ <>
+
+
+ {currentDocForm === ChunkingMode.qa && (
+ {
+ if (hasCurrentDatasetDocForm)
+ return
+ if (currentDocForm === ChunkingMode.qa)
+ onDocFormChange(ChunkingMode.text)
+ else
+ onDocFormChange(ChunkingMode.qa)
+ }}
+ >
+
+
+
+ )}
+
+ )}
+
+ {t('stepTwo.indexMode', { ns: 'datasetCreation' })}
+
+
+ {/* Qualified option */}
+ {(!hasSetIndexType || (hasSetIndexType && indexType === IndexingType.QUALIFIED)) && (
+
+ {t('stepTwo.qualified', { ns: 'datasetCreation' })}
+
+ {t('stepTwo.recommend', { ns: 'datasetCreation' })}
+
+
+ {!hasSetIndexType && }
+
+
+ )}
+ description={t('stepTwo.qualifiedTip', { ns: 'datasetCreation' })}
+ icon={+ {t('stepTwo.qaSwitchHighQualityTipTitle', { ns: 'datasetCreation' })} +
++ {t('stepTwo.qaSwitchHighQualityTipContent', { ns: 'datasetCreation' })} +
+
+
+
+
+
+
+
+ )}
+
+ {/* Economical index setting tip */}
+ {hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
+
+
+
+ {t('stepTwo.highQualityTip', { ns: 'datasetCreation' })}
+
+
+ {t('stepTwo.indexSettingTip', { ns: 'datasetCreation' })}
+
+ {t('stepTwo.datasetSettingLink', { ns: 'datasetCreation' })}
+
+
+ )}
+
+ {/* Embedding model */}
+ {indexType === IndexingType.QUALIFIED && (
+
+
+ )}
+
+
+ {t('form.embeddingModel', { ns: 'datasetSettings' })}
+
+
+ {t('stepTwo.indexSettingTip', { ns: 'datasetCreation' })}
+
+ {t('stepTwo.datasetSettingLink', { ns: 'datasetCreation' })}
+
+
+ )}
+
+ {!isModelAndRetrievalConfigDisabled
+ ? (
+
+
+ )
+}
diff --git a/web/app/components/datasets/create/step-two/inputs.tsx b/web/app/components/datasets/create/step-two/components/inputs.tsx
similarity index 100%
rename from web/app/components/datasets/create/step-two/inputs.tsx
rename to web/app/components/datasets/create/step-two/components/inputs.tsx
diff --git a/web/app/components/datasets/create/step-two/option-card.tsx b/web/app/components/datasets/create/step-two/components/option-card.tsx
similarity index 100%
rename from web/app/components/datasets/create/step-two/option-card.tsx
rename to web/app/components/datasets/create/step-two/components/option-card.tsx
diff --git a/web/app/components/datasets/create/step-two/components/parent-child-options.tsx b/web/app/components/datasets/create/step-two/components/parent-child-options.tsx
new file mode 100644
index 0000000000..e46aa5817b
--- /dev/null
+++ b/web/app/components/datasets/create/step-two/components/parent-child-options.tsx
@@ -0,0 +1,191 @@
+'use client'
+
+import type { FC } from 'react'
+import type { ParentChildConfig } from '../hooks'
+import type { ParentMode, PreProcessingRule } from '@/models/datasets'
+import { RiSearchEyeLine } from '@remixicon/react'
+import Image from 'next/image'
+import { useTranslation } from 'react-i18next'
+import Button from '@/app/components/base/button'
+import Checkbox from '@/app/components/base/checkbox'
+import Divider from '@/app/components/base/divider'
+import { ParentChildChunk } from '@/app/components/base/icons/src/vender/knowledge'
+import RadioCard from '@/app/components/base/radio-card'
+import { ChunkingMode } from '@/models/datasets'
+import FileList from '../../assets/file-list-3-fill.svg'
+import Note from '../../assets/note-mod.svg'
+import BlueEffect from '../../assets/option-card-effect-blue.svg'
+import s from '../index.module.css'
+import { DelimiterInput, MaxLengthInput } from './inputs'
+import { OptionCard } from './option-card'
+
+type TextLabelProps = {
+ children: React.ReactNode
+}
+
+const TextLabel: FC
+
+ )
+ : (
+
+ {t('form.retrievalSetting.title', { ns: 'datasetSettings' })}
+
+
+
+ {t('form.retrievalSetting.learnMore', { ns: 'datasetSettings' })}
+
+ {t('form.retrievalSetting.longDescription', { ns: 'datasetSettings' })}
+
+
+
+ )}
+
+ {t('form.retrievalSetting.title', { ns: 'datasetSettings' })}
+
+ {getIndexingTechnique() === IndexingType.QUALIFIED
+ ? (
+
+
+ {/* Parent chunk for context */}
+
+
+ )
+}
diff --git a/web/app/components/datasets/create/step-two/components/preview-panel.tsx b/web/app/components/datasets/create/step-two/components/preview-panel.tsx
new file mode 100644
index 0000000000..4f25cee5bd
--- /dev/null
+++ b/web/app/components/datasets/create/step-two/components/preview-panel.tsx
@@ -0,0 +1,171 @@
+'use client'
+
+import type { FC } from 'react'
+import type { ParentChildConfig } from '../hooks'
+import type { DataSourceType, FileIndexingEstimateResponse } from '@/models/datasets'
+import { RiSearchEyeLine } from '@remixicon/react'
+import { noop } from 'es-toolkit/function'
+import { useTranslation } from 'react-i18next'
+import Badge from '@/app/components/base/badge'
+import FloatRightContainer from '@/app/components/base/float-right-container'
+import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton'
+import { FULL_DOC_PREVIEW_LENGTH } from '@/config'
+import { ChunkingMode } from '@/models/datasets'
+import { cn } from '@/utils/classnames'
+import { ChunkContainer, QAPreview } from '../../../chunk'
+import PreviewDocumentPicker from '../../../common/document-picker/preview-document-picker'
+import { PreviewSlice } from '../../../formatted-text/flavours/preview-slice'
+import { FormattedText } from '../../../formatted-text/formatted'
+import PreviewContainer from '../../../preview/container'
+import { PreviewHeader } from '../../../preview/header'
+
+type PreviewPanelProps = {
+ // State
+ isMobile: boolean
+ dataSourceType: DataSourceType
+ currentDocForm: ChunkingMode
+ estimate?: FileIndexingEstimateResponse
+ parentChildConfig: ParentChildConfig
+ isSetting?: boolean
+ // Picker
+ pickerFiles: Array<{ id: string, name: string, extension: string }>
+ pickerValue: { id: string, name: string, extension: string }
+ // Mutation state
+ isIdle: boolean
+ isPending: boolean
+ // Actions
+ onPickerChange: (selected: { id: string, name: string }) => void
+}
+
+export const PreviewPanel: FC
+
+
+ {/* Child chunk for retrieval */}
+
+
+
+ {t('stepTwo.parentChunkForContext', { ns: 'datasetCreation' })}
+
+
+ onParentDelimiterChange(e.target.value)}
+ />
+
+ )}
+ />
+
+
+
+ {/* Rules */}
+
+
+
+ {t('stepTwo.childChunkForRetrieval', { ns: 'datasetCreation' })}
+
+
+ onChildDelimiterChange(e.target.value)}
+ />
+
+
+
+
+
+
+ {t('stepTwo.rules', { ns: 'datasetCreation' })}
+
+
+ {rules.map(rule => (
+
+ onRuleToggle(rule.id)}
+ >
+
+ ))}
+
+ >}
+ onChange={onPickerChange}
+ value={isSetting ? pickerFiles[0] : pickerValue}
+ />
+ {currentDocForm !== ChunkingMode.qa && (
+
+
+ )}
+ className={cn('relative flex h-full w-1/2 shrink-0 p-4 pr-0', isMobile && 'w-full max-w-[524px]')}
+ mainClassName="space-y-6"
+ >
+ {/* QA Preview */}
+ {currentDocForm === ChunkingMode.qa && estimate?.qa_preview && (
+ estimate.qa_preview.map((item, index) => (
+
+
+ )}
+
+ {/* Loading State */}
+ {isPending && (
+
+
+ + {t('stepTwo.previewChunkTip', { ns: 'datasetCreation' })} +
+
+ {Array.from({ length: 10 }, (_, i) => (
+
+
+
+
+ ))}
+
+ )}
+
+
+
+
+ )
+ }
+
+ return (
+
+
+
+
+ )
+}
diff --git a/web/app/components/datasets/create/step-two/escape.ts b/web/app/components/datasets/create/step-two/hooks/escape.ts
similarity index 100%
rename from web/app/components/datasets/create/step-two/escape.ts
rename to web/app/components/datasets/create/step-two/hooks/escape.ts
diff --git a/web/app/components/datasets/create/step-two/hooks/index.ts b/web/app/components/datasets/create/step-two/hooks/index.ts
new file mode 100644
index 0000000000..f16daaaea5
--- /dev/null
+++ b/web/app/components/datasets/create/step-two/hooks/index.ts
@@ -0,0 +1,14 @@
+export { useDocumentCreation } from './use-document-creation'
+export type { DocumentCreation, ValidationParams } from './use-document-creation'
+
+export { IndexingType, useIndexingConfig } from './use-indexing-config'
+export type { IndexingConfig } from './use-indexing-config'
+
+export { useIndexingEstimate } from './use-indexing-estimate'
+export type { IndexingEstimate } from './use-indexing-estimate'
+
+export { usePreviewState } from './use-preview-state'
+export type { PreviewState } from './use-preview-state'
+
+export { DEFAULT_MAXIMUM_CHUNK_LENGTH, DEFAULT_OVERLAP, DEFAULT_SEGMENT_IDENTIFIER, defaultParentChildConfig, MAXIMUM_CHUNK_TOKEN_LENGTH, useSegmentationState } from './use-segmentation-state'
+export type { ParentChildConfig, SegmentationState } from './use-segmentation-state'
diff --git a/web/app/components/datasets/create/step-two/unescape.ts b/web/app/components/datasets/create/step-two/hooks/unescape.ts
similarity index 100%
rename from web/app/components/datasets/create/step-two/unescape.ts
rename to web/app/components/datasets/create/step-two/hooks/unescape.ts
diff --git a/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts b/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts
new file mode 100644
index 0000000000..fd132b38ef
--- /dev/null
+++ b/web/app/components/datasets/create/step-two/hooks/use-document-creation.ts
@@ -0,0 +1,279 @@
+import type { DefaultModel, Model } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import type { NotionPage } from '@/models/common'
+import type {
+ ChunkingMode,
+ CrawlOptions,
+ CrawlResultItem,
+ CreateDocumentReq,
+ createDocumentResponse,
+ CustomFile,
+ FullDocumentDetail,
+ ProcessRule,
+} from '@/models/datasets'
+import type { RetrievalConfig, RETRIEVE_METHOD } from '@/types/app'
+import { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import { trackEvent } from '@/app/components/base/amplitude'
+import Toast from '@/app/components/base/toast'
+import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
+import { DataSourceProvider } from '@/models/common'
+import {
+ DataSourceType,
+} from '@/models/datasets'
+import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument } from '@/service/knowledge/use-create-dataset'
+import { useInvalidDatasetList } from '@/service/knowledge/use-dataset'
+import { IndexingType } from './use-indexing-config'
+import { MAXIMUM_CHUNK_TOKEN_LENGTH } from './use-segmentation-state'
+
+export type UseDocumentCreationOptions = {
+ datasetId?: string
+ isSetting?: boolean
+ documentDetail?: FullDocumentDetail
+ dataSourceType: DataSourceType
+ files: CustomFile[]
+ notionPages: NotionPage[]
+ notionCredentialId: string
+ websitePages: CrawlResultItem[]
+ crawlOptions?: CrawlOptions
+ websiteCrawlProvider?: DataSourceProvider
+ websiteCrawlJobId?: string
+ // Callbacks
+ onStepChange?: (delta: number) => void
+ updateIndexingTypeCache?: (type: string) => void
+ updateResultCache?: (res: createDocumentResponse) => void
+ updateRetrievalMethodCache?: (method: RETRIEVE_METHOD | '') => void
+ onSave?: () => void
+ mutateDatasetRes?: () => void
+}
+
+export type ValidationParams = {
+ segmentationType: string
+ maxChunkLength: number
+ limitMaxChunkLength: number
+ overlap: number
+ indexType: IndexingType
+ embeddingModel: DefaultModel
+ rerankModelList: Model[]
+ retrievalConfig: RetrievalConfig
+}
+
+export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
+ const { t } = useTranslation()
+ const {
+ datasetId,
+ isSetting,
+ documentDetail,
+ dataSourceType,
+ files,
+ notionPages,
+ notionCredentialId,
+ websitePages,
+ crawlOptions,
+ websiteCrawlProvider = DataSourceProvider.jinaReader,
+ websiteCrawlJobId = '',
+ onStepChange,
+ updateIndexingTypeCache,
+ updateResultCache,
+ updateRetrievalMethodCache,
+ onSave,
+ mutateDatasetRes,
+ } = options
+
+ const createFirstDocumentMutation = useCreateFirstDocument()
+ const createDocumentMutation = useCreateDocument(datasetId!)
+ const invalidDatasetList = useInvalidDatasetList()
+
+ const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending
+
+ // Validate creation params
+ const validateParams = useCallback((params: ValidationParams): boolean => {
+ const {
+ segmentationType,
+ maxChunkLength,
+ limitMaxChunkLength,
+ overlap,
+ indexType,
+ embeddingModel,
+ rerankModelList,
+ retrievalConfig,
+ } = params
+
+ if (segmentationType === 'general' && overlap > maxChunkLength) {
+ Toast.notify({ type: 'error', message: t('stepTwo.overlapCheck', { ns: 'datasetCreation' }) })
+ return false
+ }
+
+ if (segmentationType === 'general' && maxChunkLength > limitMaxChunkLength) {
+ Toast.notify({
+ type: 'error',
+ message: t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: limitMaxChunkLength }),
+ })
+ return false
+ }
+
+ if (!isSetting) {
+ if (indexType === IndexingType.QUALIFIED && (!embeddingModel.model || !embeddingModel.provider)) {
+ Toast.notify({
+ type: 'error',
+ message: t('datasetConfig.embeddingModelRequired', { ns: 'appDebug' }),
+ })
+ return false
+ }
+
+ if (!isReRankModelSelected({
+ rerankModelList,
+ retrievalConfig,
+ indexMethod: indexType,
+ })) {
+ Toast.notify({ type: 'error', message: t('datasetConfig.rerankModelRequired', { ns: 'appDebug' }) })
+ return false
+ }
+ }
+
+ return true
+ }, [t, isSetting])
+
+ // Build creation params
+ const buildCreationParams = useCallback((
+ currentDocForm: ChunkingMode,
+ docLanguage: string,
+ processRule: ProcessRule,
+ retrievalConfig: RetrievalConfig,
+ embeddingModel: DefaultModel,
+ indexingTechnique: string,
+ ): CreateDocumentReq | null => {
+ if (isSetting) {
+ return {
+ original_document_id: documentDetail?.id,
+ doc_form: currentDocForm,
+ doc_language: docLanguage,
+ process_rule: processRule,
+ retrieval_model: retrievalConfig,
+ embedding_model: embeddingModel.model,
+ embedding_model_provider: embeddingModel.provider,
+ indexing_technique: indexingTechnique,
+ } as CreateDocumentReq
+ }
+
+ const params: CreateDocumentReq = {
+ data_source: {
+ type: dataSourceType,
+ info_list: {
+ data_source_type: dataSourceType,
+ },
+ },
+ indexing_technique: indexingTechnique,
+ process_rule: processRule,
+ doc_form: currentDocForm,
+ doc_language: docLanguage,
+ retrieval_model: retrievalConfig,
+ embedding_model: embeddingModel.model,
+ embedding_model_provider: embeddingModel.provider,
+ } as CreateDocumentReq
+
+ // Add data source specific info
+ if (dataSourceType === DataSourceType.FILE) {
+ params.data_source!.info_list.file_info_list = {
+ file_ids: files.map(file => file.id || '').filter(Boolean),
+ }
+ }
+ if (dataSourceType === DataSourceType.NOTION)
+ params.data_source!.info_list.notion_info_list = getNotionInfo(notionPages, notionCredentialId)
+
+ if (dataSourceType === DataSourceType.WEB) {
+ params.data_source!.info_list.website_info_list = getWebsiteInfo({
+ websiteCrawlProvider,
+ websiteCrawlJobId,
+ websitePages,
+ crawlOptions,
+ })
+ }
+
+ return params
+ }, [
+ isSetting,
+ documentDetail,
+ dataSourceType,
+ files,
+ notionPages,
+ notionCredentialId,
+ websitePages,
+ websiteCrawlProvider,
+ websiteCrawlJobId,
+ crawlOptions,
+ ])
+
+ // Execute creation
+ const executeCreation = useCallback(async (
+ params: CreateDocumentReq,
+ indexType: IndexingType,
+ retrievalConfig: RetrievalConfig,
+ ) => {
+ if (!datasetId) {
+ await createFirstDocumentMutation.mutateAsync(params, {
+ onSuccess(data) {
+ updateIndexingTypeCache?.(indexType)
+ updateResultCache?.(data)
+ updateRetrievalMethodCache?.(retrievalConfig.search_method as RETRIEVE_METHOD)
+ },
+ })
+ }
+ else {
+ await createDocumentMutation.mutateAsync(params, {
+ onSuccess(data) {
+ updateIndexingTypeCache?.(indexType)
+ updateResultCache?.(data)
+ updateRetrievalMethodCache?.(retrievalConfig.search_method as RETRIEVE_METHOD)
+ },
+ })
+ }
+
+ mutateDatasetRes?.()
+ invalidDatasetList()
+
+ trackEvent('create_datasets', {
+ data_source_type: dataSourceType,
+ indexing_technique: indexType,
+ })
+
+ onStepChange?.(+1)
+
+ if (isSetting)
+ onSave?.()
+ }, [
+ datasetId,
+ createFirstDocumentMutation,
+ createDocumentMutation,
+ updateIndexingTypeCache,
+ updateResultCache,
+ updateRetrievalMethodCache,
+ mutateDatasetRes,
+ invalidDatasetList,
+ dataSourceType,
+ onStepChange,
+ isSetting,
+ onSave,
+ ])
+
+ // Validate preview params
+ const validatePreviewParams = useCallback((maxChunkLength: number): boolean => {
+ if (maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) {
+ Toast.notify({
+ type: 'error',
+ message: t('stepTwo.maxLengthCheck', { ns: 'datasetCreation', limit: MAXIMUM_CHUNK_TOKEN_LENGTH }),
+ })
+ return false
+ }
+ return true
+ }, [t])
+
+ return {
+ isCreating,
+ validateParams,
+ buildCreationParams,
+ executeCreation,
+ validatePreviewParams,
+ }
+}
+
+export type DocumentCreation = ReturnType{t('stepTwo.segmentation', { ns: 'datasetCreation' })}
- {((isInUpload && [ChunkingMode.text, ChunkingMode.qa].includes(currentDataset!.doc_form))
- || isUploadInEmptyDataset
- || isInInit)
- && (
-
-
-
+ {showGeneralOption && (
+
- setSegmentIdentifier(e.target.value, true)}
- />
-
-
-
-
-
-
- {t('stepTwo.rules', { ns: 'datasetCreation' })}
-
-
- {rules.map(rule => (
-
- {
- ruleChangeHandle(rule.id)
- }}
- >
-
- ))}
- {IS_CE_EDITION && (
- <>
-
-
- {currentDocForm === ChunkingMode.qa && (
- {
- if (currentDataset?.doc_form)
- return
- if (docForm === ChunkingMode.qa)
- handleChangeDocform(ChunkingMode.text)
- else
- handleChangeDocform(ChunkingMode.qa)
- }}
- >
-
-
-
- )}
-
- )}
-
-
-
- )
- }
-
-
-
-
-
-
- {t('stepTwo.parentChunkForContext', { ns: 'datasetCreation' })}
-
-
- setParentChildConfig({
- ...parentChildConfig,
- parent: {
- ...parentChildConfig.parent,
- delimiter: e.target.value ? escape(e.target.value) : '',
- },
- })}
- />
- setParentChildConfig({
- ...parentChildConfig,
- parent: {
- ...parentChildConfig.parent,
- maxLength: value,
- },
- })}
- />
-
- )}
- />
-
-
-
-
-
- {t('stepTwo.childChunkForRetrieval', { ns: 'datasetCreation' })}
-
-
- setParentChildConfig({
- ...parentChildConfig,
- child: {
- ...parentChildConfig.child,
- delimiter: e.target.value ? escape(e.target.value) : '',
- },
- })}
- />
- setParentChildConfig({
- ...parentChildConfig,
- child: {
- ...parentChildConfig.child,
- maxLength: value,
- },
- })}
- />
-
-
-
-
-
-
- {t('stepTwo.rules', { ns: 'datasetCreation' })}
-
-
- {rules.map(rule => (
-
- {
- ruleChangeHandle(rule.id)
- }}
- >
-
- ))}
- {t('stepTwo.indexMode', { ns: 'datasetCreation' })}
-
- {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && (
-
- {t('stepTwo.qualified', { ns: 'datasetCreation' })}
-
- {t('stepTwo.recommend', { ns: 'datasetCreation' })}
-
-
- {!hasSetIndexType && }
-
-
- )}
- description={t('stepTwo.qualifiedTip', { ns: 'datasetCreation' })}
- icon={- {t('stepTwo.qaSwitchHighQualityTipTitle', { ns: 'datasetCreation' })} -
-- {t('stepTwo.qaSwitchHighQualityTipContent', { ns: 'datasetCreation' })} -
-
-
-
-
-
-
-
- )}
- {hasSetIndexType && indexType === IndexingType.ECONOMICAL && (
-
-
- {t('stepTwo.highQualityTip', { ns: 'datasetCreation' })}
-
- {t('stepTwo.indexSettingTip', { ns: 'datasetCreation' })}
- {t('stepTwo.datasetSettingLink', { ns: 'datasetCreation' })}
-
- )}
- {/* Embedding model */}
- {indexType === IndexingType.QUALIFIED && (
-
- {
- setEmbeddingModel(model)
- }}
- />
- {isModelAndRetrievalConfigDisabled && (
-
+ {showParentChildOption && (
+ {t('form.embeddingModel', { ns: 'datasetSettings' })}
-
- {t('stepTwo.indexSettingTip', { ns: 'datasetCreation' })}
- {t('stepTwo.datasetSettingLink', { ns: 'datasetCreation' })}
-
- )}
-
- {!isModelAndRetrievalConfigDisabled
- ? (
-
-
- {!isSetting
- ? (
-
-
- )
- : (
- {t('form.retrievalSetting.title', { ns: 'datasetSettings' })}
-
-
- {t('form.retrievalSetting.learnMore', { ns: 'datasetSettings' })}
-
- {t('form.retrievalSetting.longDescription', { ns: 'datasetSettings' })}
-
-
-
- )}
-
- {t('form.retrievalSetting.title', { ns: 'datasetSettings' })}
-
- {
- getIndexing_technique() === IndexingType.QUALIFIED
- ? (
-
-
-
-
-
- )
- : (
-
-
-
-
- )}
+
- {dataSourceType === DataSourceType.FILE
- && (
- >}
- onChange={(selected) => {
- currentEstimateMutation.reset()
- setPreviewFile(selected)
- currentEstimateMutation.mutate()
- }}
- // when it is from setting, it just has one file
- value={isSetting ? (files[0]! as Required) : previewFile}
- />
- )}
- {dataSourceType === DataSourceType.NOTION
- && (
- ({
- id: page.page_id,
- name: page.page_name,
- extension: 'md',
- }))
- }
- onChange={(selected) => {
- currentEstimateMutation.reset()
- const selectedPage = notionPages.find(page => page.page_id === selected.id)
- setPreviewNotionPage(selectedPage!)
- currentEstimateMutation.mutate()
- }}
- value={{
- id: previewNotionPage?.page_id || '',
- name: previewNotionPage?.page_name || '',
- extension: 'md',
- }}
- />
- )}
- {dataSourceType === DataSourceType.WEB
- && (
- ({
- id: page.source_url,
- name: page.title,
- extension: 'md',
- }))
- }
- onChange={(selected) => {
- currentEstimateMutation.reset()
- const selectedPage = websitePages.find(page => page.source_url === selected.id)
- setPreviewWebsitePage(selectedPage!)
- currentEstimateMutation.mutate()
- }}
- value={
- {
- id: previewWebsitePage?.source_url || '',
- name: previewWebsitePage?.title || '',
- extension: 'md',
- }
- }
- />
- )}
- {
- currentDocForm !== ChunkingMode.qa
- && (
-
-
- )}
- className={cn('relative flex h-full w-1/2 shrink-0 p-4 pr-0', isMobile && 'w-full max-w-[524px]')}
- mainClassName="space-y-6"
- >
- {currentDocForm === ChunkingMode.qa && estimate?.qa_preview && (
- estimate?.qa_preview.map((item, index) => (
-
-
- )}
- {currentEstimateMutation.isPending && (
-
-
- - {t('stepTwo.previewChunkTip', { ns: 'datasetCreation' })} -
-
- {Array.from({ length: 10 }, (_, i) => (
-
-
-
-
- ))}
-
- )}
-
- handleChange(e.target.value)}
- />
-
- )
-}
-
-// Plugin type change test component
-const PluginTypeChangeTestComponent = () => {
- const handleChange = useMarketplaceContext(v => v.handleActivePluginTypeChange)
- return (
-
- )
-}
-
-// Page change test component
-const PageChangeTestComponent = () => {
- const handlePageChange = useMarketplaceContext(v => v.handlePageChange)
- return (
-
- )
-}
-
// ================================
// Constants Tests
// ================================
@@ -490,7 +446,7 @@ describe('utils', () => {
org: 'test-org',
name: 'test-plugin',
tags: [{ name: 'search' }],
- }
+ } as unknown as Plugin
const formatted = getFormattedPlugin(rawPlugin)
@@ -504,7 +460,7 @@ describe('utils', () => {
name: 'test-bundle',
description: 'Bundle description',
labels: { 'en-US': 'Test Bundle' },
- }
+ } as unknown as Plugin
const formatted = getFormattedPlugin(rawBundle)
@@ -1514,955 +1470,6 @@ describe('flatMap Coverage', () => {
})
})
-// ================================
-// Context Tests
-// ================================
-describe('MarketplaceContext', () => {
- beforeEach(() => {
- vi.clearAllMocks()
- mockPortalOpenState = false
- })
-
- describe('MarketplaceContext default values', () => {
- it('should have correct default context values', () => {
- expect(MarketplaceContext).toBeDefined()
- })
- })
-
- describe('useMarketplaceContext', () => {
- it('should return selected value from context', () => {
- const TestComponent = () => {
- const searchText = useMarketplaceContext(v => v.searchPluginText)
- return {searchText}
- {searchText}
- }
-
- render(
- Test Child
-
-
- )
- }
-
- render(
- {activePluginType}
- {filterPluginTags.join(',')}
- {sort.sortBy}
- {page}
- {searchText}
- }
-
- render(
-
-
-
- )
- }
-
- render(
- {tags.join(',')}
-
-
-
- )
- }
-
- render(
- {activeType}
-
-
-
- )
- }
-
- render(
- {`${sort.sortBy}-${sort.sortOrder}`}
-
-
-
- )
- }
-
- render(
- {searchText}
- {`${sort.sortBy}-${sort.sortOrder}`}
-
-
-
- )
- }
-
- render(
- {plugins ? 'has plugins' : 'no plugins'}
- {isLoading.toString()}
- }
-
- render(
- Child
- Child
-
-
- )
- }
-
- render(
- handleChange('all')}
- data-testid="all-option"
- >
- All
-
- handleChange('tool')}
- data-testid="tool-option"
- >
- Tools
-
-
-
- )
- }
-
- render(
- handleChange('all')}
- data-testid="all-option"
- >
- All
-
-
-
- )
- }
-
- render(
- handleChange('tool')}
- data-testid="tool-option"
- >
- Tools
-
- {activeType}
-
-
- )
- }
-
- render(
- handleChange('model')}
- data-testid="model-option"
- >
- Models
-
- {activeType}
- {activeType}
- }
-
- render(
-
- Content
-
- {searchText || 'empty'}
- }
-
- const SearchInput = () => {
- const handleChange = useMarketplaceContext(v => v.handleSearchPluginTextChange)
- return (
- handleChange(e.target.value)}
- />
- )
- }
-
- render(
-
-
-
-
- )
- }
-
- render(
- {tags.join(',') || 'none'}
-
-
-
-
- )
- }
-
- render(
- {sort.sortBy}
-
- {Object.entries(PLUGIN_TYPE_SEARCH_MAP).map(([key, value]) => (
-
- ))}
-
- )
- }
-
- render(
- {activeType}
- {searchText || 'empty'}
- }
-
- render(
- {tags.length === 0 ? 'no tags' : tags.join(',')}
- }
-
- render(
- {plugins === undefined ? 'undefined' : 'defined'}
- }
-
- render(
-
-
-
-
-
- )
- }
-
- render(
- {activeType}
-
- handleChange(e.target.value)}
- />
-
- )
- }
-
- render(
- {searchText.length}
-
-
-
- )
- }
-
- render(
- {tags.length}
-
-
-
- )
- }
-
- render(
- {`${sort.sortBy}-${sort.sortOrder}`}
-
-
- )
- }
-
- render(
- {activeType}
-
-
-
- )
- }
-
- render(
-
-
- )
- }
-
- render(
- {activeType}
-
-
- )
- }
-
- render(
- {activeType}
-
-
-
- )
- }
-
- render(
-
-
-
- )
- }
-
- render(
- {isLoading.toString()}
- }
-
- render(
- {isSuccess.toString()}
- }
-
- render(
- {total || 0}
- }
-
- render(
- {collection.description[getLanguage(locale)]}
{
- collection.searchable && onMoreClick && (
+ collection.searchable && (
onMoreClick?.(collection.search_params)}
+ onClick={() => onMoreClick(collection.search_params)}
>
{t('marketplace.viewMore', { ns: 'plugin' })}
showInstallButton?: boolean
}
const ListWrapper = ({
- marketplaceCollections,
- marketplaceCollectionPluginsMap,
showInstallButton,
}: ListWrapperProps) => {
const { t } = useTranslation()
- const plugins = useMarketplaceContext(v => v.plugins)
- const pluginsTotal = useMarketplaceContext(v => v.pluginsTotal)
- const marketplaceCollectionsFromClient = useMarketplaceContext(v => v.marketplaceCollectionsFromClient)
- const marketplaceCollectionPluginsMapFromClient = useMarketplaceContext(v => v.marketplaceCollectionPluginsMapFromClient)
- const isLoading = useMarketplaceContext(v => v.isLoading)
- const isSuccessCollections = useMarketplaceContext(v => v.isSuccessCollections)
- const handleQueryPlugins = useMarketplaceContext(v => v.handleQueryPlugins)
- const searchPluginText = useMarketplaceContext(v => v.searchPluginText)
- const filterPluginTags = useMarketplaceContext(v => v.filterPluginTags)
- const page = useMarketplaceContext(v => v.page)
- const handleMoreClick = useMarketplaceContext(v => v.handleMoreClick)
- useEffect(() => {
- if (
- !marketplaceCollectionsFromClient?.length
- && isSuccessCollections
- && !searchPluginText
- && !filterPluginTags.length
- ) {
- handleQueryPlugins()
- }
- }, [handleQueryPlugins, marketplaceCollections, marketplaceCollectionsFromClient, isSuccessCollections, searchPluginText, filterPluginTags])
+ const {
+ plugins,
+ pluginsTotal,
+ marketplaceCollections,
+ marketplaceCollectionPluginsMap,
+ isLoading,
+ page,
+ } = useMarketplaceData()
return (
1) && (
= ({
},
]
- const [pluginType, setPluginType] = useState(PLUGIN_TYPE_SEARCH_MAP.all)
+ const [pluginType, setPluginType] = useState(PLUGIN_TYPE_SEARCH_MAP.all)
const [query, setQuery] = useState('')
const [tags, setTags] = useState([])
const { data, isLoading } = useInstalledPluginList()
diff --git a/web/app/components/share/text-generation/run-once/index.tsx b/web/app/components/share/text-generation/run-once/index.tsx
index 6094147bbd..b8193fd944 100644
--- a/web/app/components/share/text-generation/run-once/index.tsx
+++ b/web/app/components/share/text-generation/run-once/index.tsx
@@ -195,7 +195,7 @@ const RunOnce: FC = ({
noWrapper
className="bg h-[80px] overflow-y-auto rounded-[10px] bg-components-input-bg-normal p-1"
placeholder={
- = ({
const { t } = useTranslation()
const { type } = payload
const fileSettings = useHooksStore(s => s.configsMap?.fileSettings)
+ const jsonSchemaPlaceholder = React.useMemo(() => {
+ const schema = (payload as any)?.json_schema
+ if (!schema)
+ return ''
+ return typeof schema === 'string' ? schema : JSON.stringify(schema, null, 2)
+ }, [payload])
const handleArrayItemChange = useCallback((index: number) => {
return (newValue: any) => {
@@ -211,7 +217,7 @@ const FormItem: FC = ({
noWrapper
className="bg h-[80px] overflow-y-auto rounded-[10px] bg-components-input-bg-normal p-1"
placeholder={
- // for jsonObject type
} & Partial
export type ModelConfig = {
diff --git a/web/context/query-client-server.ts b/web/context/query-client-server.ts
new file mode 100644
index 0000000000..3650e30f52
--- /dev/null
+++ b/web/context/query-client-server.ts
@@ -0,0 +1,16 @@
+import { QueryClient } from '@tanstack/react-query'
+import { cache } from 'react'
+
+const STALE_TIME = 1000 * 60 * 30 // 30 minutes
+
+export function makeQueryClient() {
+ return new QueryClient({
+ defaultOptions: {
+ queries: {
+ staleTime: STALE_TIME,
+ },
+ },
+ })
+}
+
+export const getQueryClientServer = cache(makeQueryClient)
diff --git a/web/context/query-client.tsx b/web/context/query-client.tsx
index 9562686f6f..a72393490c 100644
--- a/web/context/query-client.tsx
+++ b/web/context/query-client.tsx
@@ -1,23 +1,27 @@
'use client'
+import type { QueryClient } from '@tanstack/react-query'
import type { FC, PropsWithChildren } from 'react'
-import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
+import { QueryClientProvider } from '@tanstack/react-query'
+import { useState } from 'react'
import { TanStackDevtoolsLoader } from '@/app/components/devtools/tanstack/loader'
+import { makeQueryClient } from './query-client-server'
-const STALE_TIME = 1000 * 60 * 30 // 30 minutes
+let browserQueryClient: QueryClient | undefined
-const client = new QueryClient({
- defaultOptions: {
- queries: {
- staleTime: STALE_TIME,
- },
- },
-})
+function getQueryClient() {
+ if (typeof window === 'undefined') {
+ return makeQueryClient()
+ }
+ if (!browserQueryClient)
+ browserQueryClient = makeQueryClient()
+ return browserQueryClient
+}
-export const TanstackQueryInitializer: FC = (props) => {
- const { children } = props
+export const TanstackQueryInitializer: FC = ({ children }) => {
+ const [queryClient] = useState(getQueryClient)
return (
-
+
{children}
diff --git a/web/hooks/use-query-params.spec.tsx b/web/hooks/use-query-params.spec.tsx
index 2aa6b7998f..b187471809 100644
--- a/web/hooks/use-query-params.spec.tsx
+++ b/web/hooks/use-query-params.spec.tsx
@@ -8,7 +8,6 @@ import {
PRICING_MODAL_QUERY_PARAM,
PRICING_MODAL_QUERY_VALUE,
useAccountSettingModal,
- useMarketplaceFilters,
usePluginInstallation,
usePricingModal,
} from './use-query-params'
@@ -302,174 +301,6 @@ describe('useQueryParams hooks', () => {
})
})
- // Marketplace filters query behavior.
- describe('useMarketplaceFilters', () => {
- it('should return default filters when query params are missing', () => {
- // Arrange
- const { result } = renderWithAdapter(() => useMarketplaceFilters())
-
- // Act
- const [filters] = result.current
-
- // Assert
- expect(filters.q).toBe('')
- expect(filters.category).toBe('all')
- expect(filters.tags).toEqual([])
- })
-
- it('should parse filters when query params are present', () => {
- // Arrange
- const { result } = renderWithAdapter(
- () => useMarketplaceFilters(),
- '?q=prompt&category=tool&tags=ai,ml',
- )
-
- // Act
- const [filters] = result.current
-
- // Assert
- expect(filters.q).toBe('prompt')
- expect(filters.category).toBe('tool')
- expect(filters.tags).toEqual(['ai', 'ml'])
- })
-
- it('should treat empty tags param as empty array', () => {
- // Arrange
- const { result } = renderWithAdapter(
- () => useMarketplaceFilters(),
- '?tags=',
- )
-
- // Act
- const [filters] = result.current
-
- // Assert
- expect(filters.tags).toEqual([])
- })
-
- it('should preserve other filters when updating a single field', async () => {
- // Arrange
- const { result } = renderWithAdapter(
- () => useMarketplaceFilters(),
- '?category=tool&tags=ai,ml',
- )
-
- // Act
- act(() => {
- result.current[1]({ q: 'search' })
- })
-
- // Assert
- await waitFor(() => expect(result.current[0].q).toBe('search'))
- expect(result.current[0].category).toBe('tool')
- expect(result.current[0].tags).toEqual(['ai', 'ml'])
- })
-
- it('should clear q param when q is empty', async () => {
- // Arrange
- const { result, onUrlUpdate } = renderWithAdapter(
- () => useMarketplaceFilters(),
- '?q=search',
- )
-
- // Act
- act(() => {
- result.current[1]({ q: '' })
- })
-
- // Assert
- await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled())
- const update = onUrlUpdate.mock.calls[onUrlUpdate.mock.calls.length - 1][0]
- expect(update.searchParams.has('q')).toBe(false)
- })
-
- it('should serialize tags as comma-separated values', async () => {
- // Arrange
- const { result, onUrlUpdate } = renderWithAdapter(() => useMarketplaceFilters())
-
- // Act
- act(() => {
- result.current[1]({ tags: ['ai', 'ml'] })
- })
-
- // Assert
- await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled())
- const update = onUrlUpdate.mock.calls[onUrlUpdate.mock.calls.length - 1][0]
- expect(update.searchParams.get('tags')).toBe('ai,ml')
- })
-
- it('should remove tags param when list is empty', async () => {
- // Arrange
- const { result, onUrlUpdate } = renderWithAdapter(
- () => useMarketplaceFilters(),
- '?tags=ai,ml',
- )
-
- // Act
- act(() => {
- result.current[1]({ tags: [] })
- })
-
- // Assert
- await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled())
- const update = onUrlUpdate.mock.calls[onUrlUpdate.mock.calls.length - 1][0]
- expect(update.searchParams.has('tags')).toBe(false)
- })
-
- it('should keep category in the URL when set to default', async () => {
- // Arrange
- const { result, onUrlUpdate } = renderWithAdapter(
- () => useMarketplaceFilters(),
- '?category=tool',
- )
-
- // Act
- act(() => {
- result.current[1]({ category: 'all' })
- })
-
- // Assert
- await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled())
- const update = onUrlUpdate.mock.calls[onUrlUpdate.mock.calls.length - 1][0]
- expect(update.searchParams.get('category')).toBe('all')
- })
-
- it('should clear all marketplace filters when set to null', async () => {
- // Arrange
- const { result, onUrlUpdate } = renderWithAdapter(
- () => useMarketplaceFilters(),
- '?q=search&category=tool&tags=ai,ml',
- )
-
- // Act
- act(() => {
- result.current[1](null)
- })
-
- // Assert
- await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled())
- const update = onUrlUpdate.mock.calls[onUrlUpdate.mock.calls.length - 1][0]
- expect(update.searchParams.has('q')).toBe(false)
- expect(update.searchParams.has('category')).toBe(false)
- expect(update.searchParams.has('tags')).toBe(false)
- })
-
- it('should use replace history when updating filters', async () => {
- // Arrange
- const { result, onUrlUpdate } = renderWithAdapter(() => useMarketplaceFilters())
-
- // Act
- act(() => {
- result.current[1]({ q: 'search' })
- })
-
- // Assert
- await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled())
- const update = onUrlUpdate.mock.calls[onUrlUpdate.mock.calls.length - 1][0]
- expect(update.options.history).toBe('replace')
- })
- })
-
// Plugin installation query behavior.
describe('usePluginInstallation', () => {
it('should parse package ids from JSON arrays', () => {
diff --git a/web/hooks/use-query-params.ts b/web/hooks/use-query-params.ts
index e0d7cc3c02..73798a4a4f 100644
--- a/web/hooks/use-query-params.ts
+++ b/web/hooks/use-query-params.ts
@@ -15,7 +15,6 @@
import {
createParser,
- parseAsArrayOf,
parseAsString,
useQueryState,
useQueryStates,
@@ -93,39 +92,6 @@ export function useAccountSettingModal() {
return [{ isOpen, payload: currentTab }, setState] as const
}
-/**
- * Marketplace Search Query Parameters
- */
-export type MarketplaceFilters = {
- q: string // search query
- category: string // plugin category
- tags: string[] // array of tags
-}
-
-/**
- * Hook to manage marketplace search/filter state via URL
- * Provides atomic updates - all params update together
- *
- * @example
- * const [filters, setFilters] = useMarketplaceFilters()
- * setFilters({ q: 'search', category: 'tool', tags: ['ai'] }) // Updates all at once
- * setFilters({ q: '' }) // Only updates q, keeps others
- * setFilters(null) // Clears all marketplace params
- */
-export function useMarketplaceFilters() {
- return useQueryStates(
- {
- q: parseAsString.withDefault(''),
- category: parseAsString.withDefault('all').withOptions({ clearOnDefault: false }),
- tags: parseAsArrayOf(parseAsString).withDefault([]),
- },
- {
- // Update URL without pushing to history (replaceState behavior)
- history: 'replace',
- },
- )
-}
-
/**
* Plugin Installation Query Parameters
*/
diff --git a/web/models/debug.ts b/web/models/debug.ts
index 5290268fe9..73d0910e82 100644
--- a/web/models/debug.ts
+++ b/web/models/debug.ts
@@ -62,7 +62,7 @@ export type PromptVariable = {
icon?: string
icon_background?: string
hide?: boolean // used in frontend to hide variable
- json_schema?: string
+ json_schema?: string | Record
}
export type CompletionParams = {
diff --git a/web/service/use-apps.ts b/web/service/use-apps.ts
index d16d44af20..74e7662492 100644
--- a/web/service/use-apps.ts
+++ b/web/service/use-apps.ts
@@ -10,13 +10,14 @@ import type {
AppVoicesListResponse,
WorkflowDailyConversationsResponse,
} from '@/models/app'
-import type { App, AppModeEnum } from '@/types/app'
+import type { App } from '@/types/app'
import {
keepPreviousData,
useInfiniteQuery,
useQuery,
useQueryClient,
} from '@tanstack/react-query'
+import { AppModeEnum } from '@/types/app'
import { get, post } from './base'
import { useInvalid } from './use-base'
@@ -36,6 +37,16 @@ type DateRangeParams = {
end?: string
}
+// Allowed app modes for filtering; defined at module scope to avoid re-creating on every call
+const allowedModes = new Set([
+ 'all',
+ AppModeEnum.WORKFLOW,
+ AppModeEnum.ADVANCED_CHAT,
+ AppModeEnum.CHAT,
+ AppModeEnum.AGENT_CHAT,
+ AppModeEnum.COMPLETION,
+])
+
const normalizeAppListParams = (params: AppListParams) => {
const {
page = 1,
@@ -46,11 +57,13 @@ const normalizeAppListParams = (params: AppListParams) => {
is_created_by_me,
} = params
+ const safeMode = allowedModes.has((mode as any)) ? mode : undefined
+
return {
page,
limit,
name,
- ...(mode && mode !== 'all' ? { mode } : {}),
+ ...(safeMode && safeMode !== 'all' ? { mode: safeMode } : {}),
...(tag_ids?.length ? { tag_ids } : {}),
...(is_created_by_me ? { is_created_by_me } : {}),
}
diff --git a/web/service/workflow-payload.ts b/web/service/workflow-payload.ts
index b294141cb7..5e2cdebdb3 100644
--- a/web/service/workflow-payload.ts
+++ b/web/service/workflow-payload.ts
@@ -66,7 +66,30 @@ export const sanitizeWorkflowDraftPayload = (params: WorkflowDraftSyncParams): W
if (!graph?.nodes?.length)
return params
- const sanitizedNodes = graph.nodes.map(node => sanitizeTriggerPluginNode(node as Node))
+ const sanitizedNodes = graph.nodes.map((node) => {
+ // First sanitize known node types (TriggerPlugin)
+ const n = sanitizeTriggerPluginNode(node as Node) as Node
+
+ // Normalize Start node variable json_schema: ensure dict, not string
+ if ((n.data as any)?.type === BlockEnum.Start && Array.isArray((n.data as any).variables)) {
+ const next = { ...n, data: { ...n.data } }
+ next.data.variables = (n.data as any).variables.map((v: any) => {
+ if (v && v.type === 'json_object' && typeof v.json_schema === 'string') {
+ try {
+ const obj = JSON.parse(v.json_schema)
+ return { ...v, json_schema: obj }
+ }
+ catch {
+ return v
+ }
+ }
+ return v
+ })
+ return next
+ }
+
+ return n
+ })
return {
...params,
@@ -126,7 +149,25 @@ export const hydrateWorkflowDraftResponse = (draft: FetchWorkflowDraftResponse):
if (node.data)
removeTempProperties(node.data as Record)
- return hydrateTriggerPluginNode(node)
+ let n = hydrateTriggerPluginNode(node)
+ // Normalize Start node variable json_schema to object when loading
+ if ((n.data as any)?.type === BlockEnum.Start && Array.isArray((n.data as any).variables)) {
+ const next = { ...n, data: { ...n.data } } as Node
+ next.data.variables = (n.data as any).variables.map((v: any) => {
+ if (v && v.type === 'json_object' && typeof v.json_schema === 'string') {
+ try {
+ const obj = JSON.parse(v.json_schema)
+ return { ...v, json_schema: obj }
+ }
+ catch {
+ return v
+ }
+ }
+ return v
+ })
+ n = next
+ }
+ return n
})
}
diff --git a/web/service/workflow.ts b/web/service/workflow.ts
index 7571e804a9..3a37db791b 100644
--- a/web/service/workflow.ts
+++ b/web/service/workflow.ts
@@ -9,6 +9,7 @@ import type {
} from '@/types/workflow'
import { get, post } from './base'
import { getFlowPrefix } from './utils'
+import { sanitizeWorkflowDraftPayload } from './workflow-payload'
export const fetchWorkflowDraft = (url: string) => {
return get(url, {}, { silent: true }) as Promise
@@ -18,7 +19,8 @@ export const syncWorkflowDraft = ({ url, params }: {
url: string
params: Pick
}) => {
- return post(url, { body: params }, { silent: true })
+ const sanitized = sanitizeWorkflowDraftPayload(params)
+ return post(url, { body: sanitized }, { silent: true })
}
export const fetchNodesDefaultConfigs = (url: string) => {
{
handleActivePluginTypeChange(option.value)
+ if (PLUGIN_CATEGORY_WITH_COLLECTIONS.has(option.value)) {
+ setSearchMode(null)
+ }
}}
>
{option.icon}
diff --git a/web/app/components/plugins/marketplace/query.ts b/web/app/components/plugins/marketplace/query.ts
new file mode 100644
index 0000000000..c5a1421146
--- /dev/null
+++ b/web/app/components/plugins/marketplace/query.ts
@@ -0,0 +1,38 @@
+import type { CollectionsAndPluginsSearchParams, PluginsSearchParams } from './types'
+import { useInfiniteQuery, useQuery } from '@tanstack/react-query'
+import { getMarketplaceCollectionsAndPlugins, getMarketplacePlugins } from './utils'
+
+// TODO: Avoid manual maintenance of query keys and better service management,
+// https://github.com/langgenius/dify/issues/30342
+
+export const marketplaceKeys = {
+ all: ['marketplace'] as const,
+ collections: (params?: CollectionsAndPluginsSearchParams) => [...marketplaceKeys.all, 'collections', params] as const,
+ collectionPlugins: (collectionId: string, params?: CollectionsAndPluginsSearchParams) => [...marketplaceKeys.all, 'collectionPlugins', collectionId, params] as const,
+ plugins: (params?: PluginsSearchParams) => [...marketplaceKeys.all, 'plugins', params] as const,
+}
+
+export function useMarketplaceCollectionsAndPlugins(
+ collectionsParams: CollectionsAndPluginsSearchParams,
+) {
+ return useQuery({
+ queryKey: marketplaceKeys.collections(collectionsParams),
+ queryFn: ({ signal }) => getMarketplaceCollectionsAndPlugins(collectionsParams, { signal }),
+ })
+}
+
+export function useMarketplacePlugins(
+ queryParams: PluginsSearchParams | undefined,
+) {
+ return useInfiniteQuery({
+ queryKey: marketplaceKeys.plugins(queryParams),
+ queryFn: ({ pageParam = 1, signal }) => getMarketplacePlugins(queryParams, pageParam, signal),
+ getNextPageParam: (lastPage) => {
+ const nextPage = lastPage.page + 1
+ const loaded = lastPage.page * lastPage.pageSize
+ return loaded < (lastPage.total || 0) ? nextPage : undefined
+ },
+ initialPageParam: 1,
+ enabled: queryParams !== undefined,
+ })
+}
diff --git a/web/app/components/plugins/marketplace/search-box/index.spec.tsx b/web/app/components/plugins/marketplace/search-box/index.spec.tsx
index 3e9cc40be0..85be82cb33 100644
--- a/web/app/components/plugins/marketplace/search-box/index.spec.tsx
+++ b/web/app/components/plugins/marketplace/search-box/index.spec.tsx
@@ -26,16 +26,19 @@ vi.mock('#i18n', () => ({
}),
}))
-// Mock useMarketplaceContext
-const mockContextValues = {
- searchPluginText: '',
- handleSearchPluginTextChange: vi.fn(),
- filterPluginTags: [] as string[],
- handleFilterPluginTagsChange: vi.fn(),
-}
+// Mock marketplace state hooks
+const { mockSearchPluginText, mockHandleSearchPluginTextChange, mockFilterPluginTags, mockHandleFilterPluginTagsChange } = vi.hoisted(() => {
+ return {
+ mockSearchPluginText: '',
+ mockHandleSearchPluginTextChange: vi.fn(),
+ mockFilterPluginTags: [] as string[],
+ mockHandleFilterPluginTagsChange: vi.fn(),
+ }
+})
-vi.mock('../context', () => ({
- useMarketplaceContext: (selector: (v: typeof mockContextValues) => unknown) => selector(mockContextValues),
+vi.mock('../atoms', () => ({
+ useSearchPluginText: () => [mockSearchPluginText, mockHandleSearchPluginTextChange],
+ useFilterPluginTags: () => [mockFilterPluginTags, mockHandleFilterPluginTagsChange],
}))
// Mock useTags hook
@@ -430,9 +433,6 @@ describe('SearchBoxWrapper', () => {
beforeEach(() => {
vi.clearAllMocks()
mockPortalOpenState = false
- // Reset context values
- mockContextValues.searchPluginText = ''
- mockContextValues.filterPluginTags = []
})
describe('Rendering', () => {
@@ -456,28 +456,14 @@ describe('SearchBoxWrapper', () => {
})
})
- describe('Context Integration', () => {
- it('should use searchPluginText from context', () => {
- mockContextValues.searchPluginText = 'context search'
- render((Object.values(PLUGIN_TYPE_SEARCH_MAP) as ActivePluginType[]).withDefault('all').withOptions({ history: 'replace', clearOnDefault: false }),
+ q: parseAsString.withDefault('').withOptions({ history: 'replace' }),
+ tags: parseAsArrayOf(parseAsString).withDefault([]).withOptions({ history: 'replace' }),
+}
diff --git a/web/app/components/plugins/marketplace/sort-dropdown/index.spec.tsx b/web/app/components/plugins/marketplace/sort-dropdown/index.spec.tsx
index 3ed7d78b07..f91c7ba4d3 100644
--- a/web/app/components/plugins/marketplace/sort-dropdown/index.spec.tsx
+++ b/web/app/components/plugins/marketplace/sort-dropdown/index.spec.tsx
@@ -1,4 +1,3 @@
-import type { MarketplaceContextValue } from '../context'
import { fireEvent, render, screen, within } from '@testing-library/react'
import userEvent from '@testing-library/user-event'
import { beforeEach, describe, expect, it, vi } from 'vitest'
@@ -28,18 +27,12 @@ vi.mock('#i18n', () => ({
}),
}))
-// Mock marketplace context with controllable values
-let mockSort = { sortBy: 'install_count', sortOrder: 'DESC' }
+// Mock marketplace atoms with controllable values
+let mockSort: { sortBy: string, sortOrder: string } = { sortBy: 'install_count', sortOrder: 'DESC' }
const mockHandleSortChange = vi.fn()
-vi.mock('../context', () => ({
- useMarketplaceContext: (selector: (value: MarketplaceContextValue) => unknown) => {
- const contextValue = {
- sort: mockSort,
- handleSortChange: mockHandleSortChange,
- } as unknown as MarketplaceContextValue
- return selector(contextValue)
- },
+vi.mock('../atoms', () => ({
+ useMarketplaceSort: () => [mockSort, mockHandleSortChange],
}))
// Mock portal component with controllable open state
diff --git a/web/app/components/plugins/marketplace/sort-dropdown/index.tsx b/web/app/components/plugins/marketplace/sort-dropdown/index.tsx
index 984b114d03..1f7bab1005 100644
--- a/web/app/components/plugins/marketplace/sort-dropdown/index.tsx
+++ b/web/app/components/plugins/marketplace/sort-dropdown/index.tsx
@@ -10,7 +10,7 @@ import {
PortalToFollowElemContent,
PortalToFollowElemTrigger,
} from '@/app/components/base/portal-to-follow-elem'
-import { useMarketplaceContext } from '../context'
+import { useMarketplaceSort } from '../atoms'
const SortDropdown = () => {
const { t } = useTranslation()
@@ -36,8 +36,7 @@ const SortDropdown = () => {
text: t('marketplace.sortOption.firstReleased', { ns: 'plugin' }),
},
]
- const sort = useMarketplaceContext(v => v.sort)
- const handleSortChange = useMarketplaceContext(v => v.handleSortChange)
+ const [sort, handleSortChange] = useMarketplaceSort()
const [open, setOpen] = useState(false)
const selectedOption = options.find(option => option.value === sort.sortBy && option.order === sort.sortOrder) ?? options[0]
diff --git a/web/app/components/plugins/marketplace/state.ts b/web/app/components/plugins/marketplace/state.ts
new file mode 100644
index 0000000000..1c1abfc0a1
--- /dev/null
+++ b/web/app/components/plugins/marketplace/state.ts
@@ -0,0 +1,54 @@
+import type { PluginsSearchParams } from './types'
+import { useDebounce } from 'ahooks'
+import { useCallback, useMemo } from 'react'
+import { useActivePluginType, useFilterPluginTags, useMarketplaceSearchMode, useMarketplaceSortValue, useSearchPluginText } from './atoms'
+import { PLUGIN_TYPE_SEARCH_MAP } from './constants'
+import { useMarketplaceContainerScroll } from './hooks'
+import { useMarketplaceCollectionsAndPlugins, useMarketplacePlugins } from './query'
+import { getCollectionsParams, getMarketplaceListFilterType } from './utils'
+
+export function useMarketplaceData() {
+ const [searchPluginTextOriginal] = useSearchPluginText()
+ const searchPluginText = useDebounce(searchPluginTextOriginal, { wait: 500 })
+ const [filterPluginTags] = useFilterPluginTags()
+ const [activePluginType] = useActivePluginType()
+
+ const collectionsQuery = useMarketplaceCollectionsAndPlugins(
+ getCollectionsParams(activePluginType),
+ )
+
+ const sort = useMarketplaceSortValue()
+ const isSearchMode = useMarketplaceSearchMode()
+ const queryParams = useMemo((): PluginsSearchParams | undefined => {
+ if (!isSearchMode)
+ return undefined
+ return {
+ query: searchPluginText,
+ category: activePluginType === PLUGIN_TYPE_SEARCH_MAP.all ? undefined : activePluginType,
+ tags: filterPluginTags,
+ sortBy: sort.sortBy,
+ sortOrder: sort.sortOrder,
+ type: getMarketplaceListFilterType(activePluginType),
+ }
+ }, [isSearchMode, searchPluginText, activePluginType, filterPluginTags, sort])
+
+ const pluginsQuery = useMarketplacePlugins(queryParams)
+ const { hasNextPage, fetchNextPage, isFetching } = pluginsQuery
+
+ const handlePageChange = useCallback(() => {
+ if (hasNextPage && !isFetching)
+ fetchNextPage()
+ }, [fetchNextPage, hasNextPage, isFetching])
+
+ // Scroll pagination
+ useMarketplaceContainerScroll(handlePageChange)
+
+ return {
+ marketplaceCollections: collectionsQuery.data?.marketplaceCollections,
+ marketplaceCollectionPluginsMap: collectionsQuery.data?.marketplaceCollectionPluginsMap,
+ plugins: pluginsQuery.data?.pages.flatMap(page => page.plugins),
+ pluginsTotal: pluginsQuery.data?.pages[0]?.total,
+ page: pluginsQuery.data?.pages.length || 1,
+ isLoading: collectionsQuery.isLoading || pluginsQuery.isLoading,
+ }
+}
diff --git a/web/app/components/plugins/marketplace/sticky-search-and-switch-wrapper.tsx b/web/app/components/plugins/marketplace/sticky-search-and-switch-wrapper.tsx
index 3d3530c83e..4da3844c0a 100644
--- a/web/app/components/plugins/marketplace/sticky-search-and-switch-wrapper.tsx
+++ b/web/app/components/plugins/marketplace/sticky-search-and-switch-wrapper.tsx
@@ -6,12 +6,10 @@ import SearchBoxWrapper from './search-box/search-box-wrapper'
type StickySearchAndSwitchWrapperProps = {
pluginTypeSwitchClassName?: string
- showSearchParams?: boolean
}
const StickySearchAndSwitchWrapper = ({
pluginTypeSwitchClassName,
- showSearchParams,
}: StickySearchAndSwitchWrapperProps) => {
const hasCustomTopClass = pluginTypeSwitchClassName?.includes('top-')
@@ -24,9 +22,7 @@ const StickySearchAndSwitchWrapper = ({
)}
>
)
}
diff --git a/web/app/components/plugins/marketplace/utils.ts b/web/app/components/plugins/marketplace/utils.ts
index e51c9b76a6..eaf299314c 100644
--- a/web/app/components/plugins/marketplace/utils.ts
+++ b/web/app/components/plugins/marketplace/utils.ts
@@ -1,16 +1,19 @@
+import type { ActivePluginType } from './constants'
import type {
CollectionsAndPluginsSearchParams,
MarketplaceCollection,
+ PluginsSearchParams,
} from '@/app/components/plugins/marketplace/types'
-import type { Plugin } from '@/app/components/plugins/types'
+import type { Plugin, PluginsFromMarketplaceResponse } from '@/app/components/plugins/types'
import { PluginCategoryEnum } from '@/app/components/plugins/types'
import {
APP_VERSION,
IS_MARKETPLACE,
MARKETPLACE_API_PREFIX,
} from '@/config'
+import { postMarketplace } from '@/service/base'
import { getMarketplaceUrl } from '@/utils/var'
-import { PLUGIN_TYPE_SEARCH_MAP } from './plugin-type-switch'
+import { PLUGIN_TYPE_SEARCH_MAP } from './constants'
type MarketplaceFetchOptions = {
signal?: AbortSignal
@@ -26,12 +29,13 @@ export const getPluginIconInMarketplace = (plugin: Plugin) => {
return `${MARKETPLACE_API_PREFIX}/plugins/${plugin.org}/${plugin.name}/icon`
}
-export const getFormattedPlugin = (bundle: any) => {
+export const getFormattedPlugin = (bundle: Plugin): Plugin => {
if (bundle.type === 'bundle') {
return {
...bundle,
icon: getPluginIconInMarketplace(bundle),
brief: bundle.description,
+ // @ts-expect-error I do not have enough information
label: bundle.labels,
}
}
@@ -129,6 +133,64 @@ export const getMarketplaceCollectionsAndPlugins = async (
}
}
+export const getMarketplacePlugins = async (
+ queryParams: PluginsSearchParams | undefined,
+ pageParam: number,
+ signal?: AbortSignal,
+) => {
+ if (!queryParams) {
+ return {
+ plugins: [] as Plugin[],
+ total: 0,
+ page: 1,
+ pageSize: 40,
+ }
+ }
+
+ const {
+ query,
+ sortBy,
+ sortOrder,
+ category,
+ tags,
+ type,
+ pageSize = 40,
+ } = queryParams
+ const pluginOrBundle = type === 'bundle' ? 'bundles' : 'plugins'
+
+ try {
+ const res = await postMarketplace<{ data: PluginsFromMarketplaceResponse }>(`/${pluginOrBundle}/search/advanced`, {
+ body: {
+ page: pageParam,
+ page_size: pageSize,
+ query,
+ sort_by: sortBy,
+ sort_order: sortOrder,
+ category: category !== 'all' ? category : '',
+ tags,
+ type,
+ },
+ signal,
+ })
+ const resPlugins = res.data.bundles || res.data.plugins || []
+
+ return {
+ plugins: resPlugins.map(plugin => getFormattedPlugin(plugin)),
+ total: res.data.total,
+ page: pageParam,
+ pageSize,
+ }
+ }
+ catch {
+ return {
+ plugins: [],
+ total: 0,
+ page: pageParam,
+ pageSize,
+ }
+ }
+}
+
export const getMarketplaceListCondition = (pluginType: string) => {
if ([PluginCategoryEnum.tool, PluginCategoryEnum.agent, PluginCategoryEnum.model, PluginCategoryEnum.datasource, PluginCategoryEnum.trigger].includes(pluginType as PluginCategoryEnum))
return `category=${pluginType}`
@@ -142,7 +204,7 @@ export const getMarketplaceListCondition = (pluginType: string) => {
return ''
}
-export const getMarketplaceListFilterType = (category: string) => {
+export const getMarketplaceListFilterType = (category: ActivePluginType) => {
if (category === PLUGIN_TYPE_SEARCH_MAP.all)
return undefined
@@ -151,3 +213,14 @@ export const getMarketplaceListFilterType = (category: string) => {
return 'plugin'
}
+
+export function getCollectionsParams(category: ActivePluginType): CollectionsAndPluginsSearchParams {
+ if (category === PLUGIN_TYPE_SEARCH_MAP.all) {
+ return {}
+ }
+ return {
+ category,
+ condition: getMarketplaceListCondition(category),
+ type: getMarketplaceListFilterType(category),
+ }
+}
diff --git a/web/app/components/plugins/plugin-page/index.tsx b/web/app/components/plugins/plugin-page/index.tsx
index b8fc891254..1f88f691ef 100644
--- a/web/app/components/plugins/plugin-page/index.tsx
+++ b/web/app/components/plugins/plugin-page/index.tsx
@@ -27,7 +27,7 @@ import { cn } from '@/utils/classnames'
import { PLUGIN_PAGE_TABS_MAP } from '../hooks'
import InstallFromLocalPackage from '../install-plugin/install-from-local-package'
import InstallFromMarketplace from '../install-plugin/install-from-marketplace'
-import { PLUGIN_TYPE_SEARCH_MAP } from '../marketplace/plugin-type-switch'
+import { PLUGIN_TYPE_SEARCH_MAP } from '../marketplace/constants'
import {
PluginPageContextProvider,
usePluginPageContext,
diff --git a/web/app/components/plugins/reference-setting-modal/auto-update-setting/index.spec.tsx b/web/app/components/plugins/reference-setting-modal/auto-update-setting/index.spec.tsx
index d65b0b7957..1008ef461d 100644
--- a/web/app/components/plugins/reference-setting-modal/auto-update-setting/index.spec.tsx
+++ b/web/app/components/plugins/reference-setting-modal/auto-update-setting/index.spec.tsx
@@ -262,7 +262,7 @@ vi.mock('@/app/components/base/icons/src/vender/other', () => ({
}))
// Mock PLUGIN_TYPE_SEARCH_MAP
-vi.mock('../../marketplace/plugin-type-switch', () => ({
+vi.mock('../../marketplace/constants', () => ({
PLUGIN_TYPE_SEARCH_MAP: {
all: 'all',
model: 'model',
diff --git a/web/app/components/plugins/reference-setting-modal/auto-update-setting/tool-picker.tsx b/web/app/components/plugins/reference-setting-modal/auto-update-setting/tool-picker.tsx
index a91df6c793..4e681a6b67 100644
--- a/web/app/components/plugins/reference-setting-modal/auto-update-setting/tool-picker.tsx
+++ b/web/app/components/plugins/reference-setting-modal/auto-update-setting/tool-picker.tsx
@@ -1,5 +1,6 @@
'use client'
import type { FC } from 'react'
+import type { ActivePluginType } from '../../marketplace/constants'
import * as React from 'react'
import { useCallback, useMemo, useState } from 'react'
import { useTranslation } from 'react-i18next'
@@ -12,7 +13,7 @@ import {
import SearchBox from '@/app/components/plugins/marketplace/search-box'
import { useInstalledPluginList } from '@/service/use-plugins'
import { cn } from '@/utils/classnames'
-import { PLUGIN_TYPE_SEARCH_MAP } from '../../marketplace/plugin-type-switch'
+import { PLUGIN_TYPE_SEARCH_MAP } from '../../marketplace/constants'
import { PluginSource } from '../../types'
import NoDataPlaceholder from './no-data-placeholder'
import ToolItem from './tool-item'
@@ -73,7 +74,7 @@ const ToolPicker: FC{item.json_schema}
+ {typeof item.json_schema === 'string' ? item.json_schema : JSON.stringify(item.json_schema || '', null, 2)}
}
/>
)}
diff --git a/web/app/components/workflow/nodes/_base/components/before-run-form/form-item.tsx b/web/app/components/workflow/nodes/_base/components/before-run-form/form-item.tsx
index 81a8453582..3eef34bd7b 100644
--- a/web/app/components/workflow/nodes/_base/components/before-run-form/form-item.tsx
+++ b/web/app/components/workflow/nodes/_base/components/before-run-form/form-item.tsx
@@ -48,6 +48,12 @@ const FormItem: FC{payload.json_schema}
+ {jsonSchemaPlaceholder}
}
/>
)}
diff --git a/web/app/components/workflow/nodes/_base/components/variable/utils.ts b/web/app/components/workflow/nodes/_base/components/variable/utils.ts
index 9f77be0ce2..e5e8174456 100644
--- a/web/app/components/workflow/nodes/_base/components/variable/utils.ts
+++ b/web/app/components/workflow/nodes/_base/components/variable/utils.ts
@@ -353,7 +353,7 @@ const formatItem = (
try {
if (type === VarType.object && v.json_schema) {
varRes.children = {
- schema: JSON.parse(v.json_schema),
+ schema: typeof v.json_schema === 'string' ? JSON.parse(v.json_schema) : v.json_schema,
}
}
}
diff --git a/web/app/components/workflow/types.ts b/web/app/components/workflow/types.ts
index 2accac4047..02ee45aa7a 100644
--- a/web/app/components/workflow/types.ts
+++ b/web/app/components/workflow/types.ts
@@ -226,7 +226,7 @@ export type InputVar = {
getVarValueFromDependent?: boolean
hide?: boolean
isFileItem?: boolean
- json_schema?: string // for jsonObject type
+ json_schema?: string | Record